t/t5310-pack-bitmaps.sh

   1 #!/bin/sh
   2
   3 test_description='exercise basic bitmap functionality'
   4
   5 . ./test-lib.sh
   6 . "$TEST_DIRECTORY"/lib-bitmap.sh
   7
   8 # t5310 deals only with single-pack bitmaps, so don't write MIDX bitmaps in
   9 # their place.
  10 GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=0
  11
  12 objpath () {
  13         echo ".git/objects/$(echo "$1" | sed -e 's|\(..\)|\1/|')"
  14 }
  15
  16 # show objects present in pack ($1 should be associated *.idx)
  17 list_packed_objects () {
  18         git show-index <"$1" >object-list &&
  19         cut -d' ' -f2 object-list
  20 }
  21
  22 # has_any pattern-file content-file
  23 # tests whether content-file has any entry from pattern-file with entries being
  24 # whole lines.
  25 has_any () {
  26         grep -Ff "$1" "$2"
  27 }
  28
  29 test_bitmap_cases () {
  30         writeLookupTable=false
  31         for i in "$@"
  32         do
  33                 case "$i" in
  34                 "pack.writeBitmapLookupTable") writeLookupTable=true;;
  35                 esac
  36         done
  37
  38         test_expect_success 'setup test repository' '
  39                 rm -fr * .git &&
  40                 git init &&
  41                 git config pack.writeBitmapLookupTable '"$writeLookupTable"'
  42         '
  43         setup_bitmap_history
  44
  45         test_expect_success 'setup writing bitmaps during repack' '
  46                 git config repack.writeBitmaps true
  47         '
  48
  49         test_expect_success 'full repack creates bitmaps' '
  50                 GIT_TRACE2_EVENT="$(pwd)/trace" \
  51                         git repack -ad &&
  52                 ls .git/objects/pack/ | grep bitmap >output &&
  53                 test_line_count = 1 output &&
  54                 grep "\"key\":\"num_selected_commits\",\"value\":\"106\"" trace &&
  55                 grep "\"key\":\"num_maximal_commits\",\"value\":\"107\"" trace
  56         '
  57
  58         basic_bitmap_tests
  59
  60         test_expect_success 'pack-objects respects --local (non-local loose)' '
  61                 git init --bare alt.git &&
  62                 echo $(pwd)/alt.git/objects >.git/objects/info/alternates &&
  63                 echo content1 >file1 &&
  64                 # non-local loose object which is not present in bitmapped pack
  65                 altblob=$(GIT_DIR=alt.git git hash-object -w file1) &&
  66                 # non-local loose object which is also present in bitmapped pack
  67                 git cat-file blob $blob | GIT_DIR=alt.git git hash-object -w --stdin &&
  68                 git add file1 &&
  69                 test_tick &&
  70                 git commit -m commit_file1 &&
  71                 echo HEAD | git pack-objects --local --stdout --revs >1.pack &&
  72                 git index-pack 1.pack &&
  73                 list_packed_objects 1.idx >1.objects &&
  74                 printf "%s\n" "$altblob" "$blob" >nonlocal-loose &&
  75                 ! has_any nonlocal-loose 1.objects
  76         '
  77
  78         test_expect_success 'pack-objects respects --honor-pack-keep (local non-bitmapped pack)' '
  79                 echo content2 >file2 &&
  80                 blob2=$(git hash-object -w file2) &&
  81                 git add file2 &&
  82                 test_tick &&
  83                 git commit -m commit_file2 &&
  84                 printf "%s\n" "$blob2" "$bitmaptip" >keepobjects &&
  85                 pack2=$(git pack-objects pack2 <keepobjects) &&
  86                 mv pack2-$pack2.* .git/objects/pack/ &&
  87                 >.git/objects/pack/pack2-$pack2.keep &&
  88                 rm $(objpath $blob2) &&
  89                 echo HEAD | git pack-objects --honor-pack-keep --stdout --revs >2a.pack &&
  90                 git index-pack 2a.pack &&
  91                 list_packed_objects 2a.idx >2a.objects &&
  92                 ! has_any keepobjects 2a.objects
  93         '
  94
  95         test_expect_success 'pack-objects respects --local (non-local pack)' '
  96                 mv .git/objects/pack/pack2-$pack2.* alt.git/objects/pack/ &&
  97                 echo HEAD | git pack-objects --local --stdout --revs >2b.pack &&
  98                 git index-pack 2b.pack &&
  99                 list_packed_objects 2b.idx >2b.objects &&
 100                 ! has_any keepobjects 2b.objects
 101         '
 102
 103         test_expect_success 'pack-objects respects --honor-pack-keep (local bitmapped pack)' '
 104                 ls .git/objects/pack/ | grep bitmap >output &&
 105                 test_line_count = 1 output &&
 106                 packbitmap=$(basename $(cat output) .bitmap) &&
 107                 list_packed_objects .git/objects/pack/$packbitmap.idx >packbitmap.objects &&
 108                 test_when_finished "rm -f .git/objects/pack/$packbitmap.keep" &&
 109                 >.git/objects/pack/$packbitmap.keep &&
 110                 echo HEAD | git pack-objects --honor-pack-keep --stdout --revs >3a.pack &&
 111                 git index-pack 3a.pack &&
 112                 list_packed_objects 3a.idx >3a.objects &&
 113                 ! has_any packbitmap.objects 3a.objects
 114         '
 115
 116         test_expect_success 'pack-objects respects --local (non-local bitmapped pack)' '
 117                 mv .git/objects/pack/$packbitmap.* alt.git/objects/pack/ &&
 118                 rm -f .git/objects/pack/multi-pack-index &&
 119                 test_when_finished "mv alt.git/objects/pack/$packbitmap.* .git/objects/pack/" &&
 120                 echo HEAD | git pack-objects --local --stdout --revs >3b.pack &&
 121                 git index-pack 3b.pack &&
 122                 list_packed_objects 3b.idx >3b.objects &&
 123                 ! has_any packbitmap.objects 3b.objects
 124         '
 125
 126         test_expect_success 'pack-objects to file can use bitmap' '
 127                 # make sure we still have 1 bitmap index from previous tests
 128                 ls .git/objects/pack/ | grep bitmap >output &&
 129                 test_line_count = 1 output &&
 130                 # verify equivalent packs are generated with/without using bitmap index
 131                 packasha1=$(git pack-objects --no-use-bitmap-index --all packa </dev/null) &&
 132                 packbsha1=$(git pack-objects --use-bitmap-index --all packb </dev/null) &&
 133                 list_packed_objects packa-$packasha1.idx >packa.objects &&
 134                 list_packed_objects packb-$packbsha1.idx >packb.objects &&
 135                 test_cmp packa.objects packb.objects
 136         '
 137
 138         test_expect_success 'full repack, reusing previous bitmaps' '
 139                 git repack -ad &&
 140                 ls .git/objects/pack/ | grep bitmap >output &&
 141                 test_line_count = 1 output
 142         '
 143
 144         test_expect_success 'fetch (full bitmap)' '
 145                 git --git-dir=clone.git fetch origin second:second &&
 146                 git rev-parse HEAD >expect &&
 147                 git --git-dir=clone.git rev-parse HEAD >actual &&
 148                 test_cmp expect actual
 149         '
 150
 151         test_expect_success 'create objects for missing-HAVE tests' '
 152                 blob=$(echo "missing have" | git hash-object -w --stdin) &&
 153                 tree=$(printf "100644 blob $blob\tfile\n" | git mktree) &&
 154                 parent=$(echo parent | git commit-tree $tree) &&
 155                 commit=$(echo commit | git commit-tree $tree -p $parent) &&
 156                 cat >revs <<-EOF
 157                 HEAD
 158                 ^HEAD^
 159                 ^$commit
 160                 EOF
 161         '
 162
 163         test_expect_success 'pack-objects respects --incremental' '
 164                 cat >revs2 <<-EOF &&
 165                 HEAD
 166                 $commit
 167                 EOF
 168                 git pack-objects --incremental --stdout --revs <revs2 >4.pack &&
 169                 git index-pack 4.pack &&
 170                 list_packed_objects 4.idx >4.objects &&
 171                 test_line_count = 4 4.objects &&
 172                 git rev-list --objects $commit >revlist &&
 173                 cut -d" " -f1 revlist |sort >objects &&
 174                 test_cmp 4.objects objects
 175         '
 176
 177         test_expect_success 'pack with missing blob' '
 178                 rm $(objpath $blob) &&
 179                 git pack-objects --stdout --revs <revs >/dev/null
 180         '
 181
 182         test_expect_success 'pack with missing tree' '
 183                 rm $(objpath $tree) &&
 184                 git pack-objects --stdout --revs <revs >/dev/null
 185         '
 186
 187         test_expect_success 'pack with missing parent' '
 188                 rm $(objpath $parent) &&
 189                 git pack-objects --stdout --revs <revs >/dev/null
 190         '
 191
 192         test_expect_success JGIT,SHA1 'we can read jgit bitmaps' '
 193                 git clone --bare . compat-jgit.git &&
 194                 (
 195                         cd compat-jgit.git &&
 196                         rm -f objects/pack/*.bitmap &&
 197                         jgit gc &&
 198                         git rev-list --test-bitmap HEAD
 199                 )
 200         '
 201
 202         test_expect_success JGIT,SHA1 'jgit can read our bitmaps' '
 203                 git clone --bare . compat-us.git &&
 204                 (
 205                         cd compat-us.git &&
 206                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
 207                         git repack -adb &&
 208                         # jgit gc will barf if it does not like our bitmaps
 209                         jgit gc
 210                 )
 211         '
 212
 213         test_expect_success 'splitting packs does not generate bogus bitmaps' '
 214                 test-tool genrandom foo $((1024 * 1024)) >rand &&
 215                 git add rand &&
 216                 git commit -m "commit with big file" &&
 217                 git -c pack.packSizeLimit=500k repack -adb &&
 218                 git init --bare no-bitmaps.git &&
 219                 git -C no-bitmaps.git fetch .. HEAD
 220         '
 221
 222         test_expect_success 'set up reusable pack' '
 223                 rm -f .git/objects/pack/*.keep &&
 224                 git repack -adb &&
 225                 reusable_pack () {
 226                         git for-each-ref --format="%(objectname)" |
 227                         git pack-objects --delta-base-offset --revs --stdout "$@"
 228                 }
 229         '
 230
 231         test_expect_success 'pack reuse respects --honor-pack-keep' '
 232                 test_when_finished "rm -f .git/objects/pack/*.keep" &&
 233                 for i in .git/objects/pack/*.pack
 234                 do
 235                         >${i%.pack}.keep || return 1
 236                 done &&
 237                 reusable_pack --honor-pack-keep >empty.pack &&
 238                 git index-pack empty.pack &&
 239                 git show-index <empty.idx >actual &&
 240                 test_must_be_empty actual
 241         '
 242
 243         test_expect_success 'pack reuse respects --local' '
 244                 mv .git/objects/pack/* alt.git/objects/pack/ &&
 245                 test_when_finished "mv alt.git/objects/pack/* .git/objects/pack/" &&
 246                 reusable_pack --local >empty.pack &&
 247                 git index-pack empty.pack &&
 248                 git show-index <empty.idx >actual &&
 249                 test_must_be_empty actual
 250         '
 251
 252         test_expect_success 'pack reuse respects --incremental' '
 253                 reusable_pack --incremental >empty.pack &&
 254                 git index-pack empty.pack &&
 255                 git show-index <empty.idx >actual &&
 256                 test_must_be_empty actual
 257         '
 258
 259         test_expect_success 'truncated bitmap fails gracefully (ewah)' '
 260                 test_config pack.writebitmaphashcache false &&
 261                 test_config pack.writebitmaplookuptable false &&
 262                 git repack -ad &&
 263                 git rev-list --use-bitmap-index --count --all >expect &&
 264                 bitmap=$(ls .git/objects/pack/*.bitmap) &&
 265                 test_when_finished "rm -f $bitmap" &&
 266                 test_copy_bytes 256 <$bitmap >$bitmap.tmp &&
 267                 mv -f $bitmap.tmp $bitmap &&
 268                 git rev-list --use-bitmap-index --count --all >actual 2>stderr &&
 269                 test_cmp expect actual &&
 270                 test_i18ngrep corrupt.ewah.bitmap stderr
 271         '
 272
 273         test_expect_success 'truncated bitmap fails gracefully (cache)' '
 274                 git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
 275                 git repack -ad &&
 276                 git rev-list --use-bitmap-index --count --all >expect &&
 277                 bitmap=$(ls .git/objects/pack/*.bitmap) &&
 278                 test_when_finished "rm -f $bitmap" &&
 279                 test_copy_bytes 512 <$bitmap >$bitmap.tmp &&
 280                 mv -f $bitmap.tmp $bitmap &&
 281                 git rev-list --use-bitmap-index --count --all >actual 2>stderr &&
 282                 test_cmp expect actual &&
 283                 test_i18ngrep corrupted.bitmap.index stderr
 284         '
 285
 286         # Create a state of history with these properties:
 287         #
 288         #  - refs that allow a client to fetch some new history, while sharing some old
 289         #    history with the server; we use branches delta-reuse-old and
 290         #    delta-reuse-new here
 291         #
 292         #  - the new history contains an object that is stored on the server as a delta
 293         #    against a base that is in the old history
 294         #
 295         #  - the base object is not immediately reachable from the tip of the old
 296         #    history; finding it would involve digging down through history we know the
 297         #    other side has
 298         #
 299         # This should result in a state where fetching from old->new would not
 300         # traditionally reuse the on-disk delta (because we'd have to dig to realize
 301         # that the client has it), but we will do so if bitmaps can tell us cheaply
 302         # that the other side has it.
 303         test_expect_success 'set up thin delta-reuse parent' '
 304                 # This first commit contains the buried base object.
 305                 test-tool genrandom delta 16384 >file &&
 306                 git add file &&
 307                 git commit -m "delta base" &&
 308                 base=$(git rev-parse --verify HEAD:file) &&
 309
 310                 # These intermediate commits bury the base back in history.
 311                 # This becomes the "old" state.
 312                 for i in 1 2 3 4 5
 313                 do
 314                         echo $i >file &&
 315                         git commit -am "intermediate $i" || return 1
 316                 done &&
 317                 git branch delta-reuse-old &&
 318
 319                 # And now our new history has a delta against the buried base. Note
 320                 # that this must be smaller than the original file, since pack-objects
 321                 # prefers to create deltas from smaller objects to larger.
 322                 test-tool genrandom delta 16300 >file &&
 323                 git commit -am "delta result" &&
 324                 delta=$(git rev-parse --verify HEAD:file) &&
 325                 git branch delta-reuse-new &&
 326
 327                 # Repack with bitmaps and double check that we have the expected delta
 328                 # relationship.
 329                 git repack -adb &&
 330                 have_delta $delta $base
 331         '
 332
 333         # Now we can sanity-check the non-bitmap behavior (that the server is not able
 334         # to reuse the delta). This isn't strictly something we care about, so this
 335         # test could be scrapped in the future. But it makes sure that the next test is
 336         # actually triggering the feature we want.
 337         #
 338         # Note that our tools for working with on-the-wire "thin" packs are limited. So
 339         # we actually perform the fetch, retain the resulting pack, and inspect the
 340         # result.
 341         test_expect_success 'fetch without bitmaps ignores delta against old base' '
 342                 test_config pack.usebitmaps false &&
 343                 test_when_finished "rm -rf client.git" &&
 344                 git init --bare client.git &&
 345                 (
 346                         cd client.git &&
 347                         git config transfer.unpackLimit 1 &&
 348                         git fetch .. delta-reuse-old:delta-reuse-old &&
 349                         git fetch .. delta-reuse-new:delta-reuse-new &&
 350                         have_delta $delta $ZERO_OID
 351                 )
 352         '
 353
 354         # And do the same for the bitmap case, where we do expect to find the delta.
 355         test_expect_success 'fetch with bitmaps can reuse old base' '
 356                 test_config pack.usebitmaps true &&
 357                 test_when_finished "rm -rf client.git" &&
 358                 git init --bare client.git &&
 359                 (
 360                         cd client.git &&
 361                         git config transfer.unpackLimit 1 &&
 362                         git fetch .. delta-reuse-old:delta-reuse-old &&
 363                         git fetch .. delta-reuse-new:delta-reuse-new &&
 364                         have_delta $delta $base
 365                 )
 366         '
 367
 368         test_expect_success 'pack.preferBitmapTips' '
 369                 git init repo &&
 370                 test_when_finished "rm -fr repo" &&
 371                 (
 372                         cd repo &&
 373                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
 374
 375                         # create enough commits that not all are receive bitmap
 376                         # coverage even if they are all at the tip of some reference.
 377                         test_commit_bulk --message="%s" 103 &&
 378
 379                         git rev-list HEAD >commits.raw &&
 380                         sort <commits.raw >commits &&
 381
 382                         git log --format="create refs/tags/%s %H" HEAD >refs &&
 383                         git update-ref --stdin <refs &&
 384
 385                         git repack -adb &&
 386                         test-tool bitmap list-commits | sort >bitmaps &&
 387
 388                         # remember which commits did not receive bitmaps
 389                         comm -13 bitmaps commits >before &&
 390                         test_file_not_empty before &&
 391
 392                         # mark the commits which did not receive bitmaps as preferred,
 393                         # and generate the bitmap again
 394                         perl -pe "s{^}{create refs/tags/include/$. }" <before |
 395                                 git update-ref --stdin &&
 396                         git -c pack.preferBitmapTips=refs/tags/include repack -adb &&
 397
 398                         # finally, check that the commit(s) without bitmap coverage
 399                         # are not the same ones as before
 400                         test-tool bitmap list-commits | sort >bitmaps &&
 401                         comm -13 bitmaps commits >after &&
 402
 403                         ! test_cmp before after
 404                 )
 405         '
 406
 407         test_expect_success 'complains about multiple pack bitmaps' '
 408                 rm -fr repo &&
 409                 git init repo &&
 410                 test_when_finished "rm -fr repo" &&
 411                 (
 412                         cd repo &&
 413                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
 414
 415                         test_commit base &&
 416
 417                         git repack -adb &&
 418                         bitmap="$(ls .git/objects/pack/pack-*.bitmap)" &&
 419                         mv "$bitmap" "$bitmap.bak" &&
 420
 421                         test_commit other &&
 422                         git repack -ab &&
 423
 424                         mv "$bitmap.bak" "$bitmap" &&
 425
 426                         find .git/objects/pack -type f -name "*.pack" >packs &&
 427                         find .git/objects/pack -type f -name "*.bitmap" >bitmaps &&
 428                         test_line_count = 2 packs &&
 429                         test_line_count = 2 bitmaps &&
 430
 431                         GIT_TRACE2_EVENT=$(pwd)/trace2.txt git rev-list --use-bitmap-index HEAD &&
 432                         grep "opened bitmap" trace2.txt &&
 433                         grep "ignoring extra bitmap" trace2.txt
 434                 )
 435         '
 436 }
 437
 438 test_bitmap_cases
 439
 440 test_expect_success 'incremental repack fails when bitmaps are requested' '
 441         test_commit more-1 &&
 442         test_must_fail git repack -d 2>err &&
 443         test_i18ngrep "Incremental repacks are incompatible with bitmap" err
 444 '
 445
 446 test_expect_success 'incremental repack can disable bitmaps' '
 447         test_commit more-2 &&
 448         git repack -d --no-write-bitmap-index
 449 '
 450
 451 test_bitmap_cases "pack.writeBitmapLookupTable"
 452
 453 test_expect_success 'verify writing bitmap lookup table when enabled' '
 454         GIT_TRACE2_EVENT="$(pwd)/trace2" \
 455                 git repack -ad &&
 456         grep "\"label\":\"writing_lookup_table\"" trace2
 457 '
 458
 459 test_expect_success 'truncated bitmap fails gracefully (lookup table)' '
 460         test_config pack.writebitmaphashcache false &&
 461         git repack -adb &&
 462         git rev-list --use-bitmap-index --count --all >expect &&
 463         bitmap=$(ls .git/objects/pack/*.bitmap) &&
 464         test_when_finished "rm -f $bitmap" &&
 465         test_copy_bytes 512 <$bitmap >$bitmap.tmp &&
 466         mv -f $bitmap.tmp $bitmap &&
 467         git rev-list --use-bitmap-index --count --all >actual 2>stderr &&
 468         test_cmp expect actual &&
 469         test_i18ngrep corrupted.bitmap.index stderr
 470 '
 471
 472 test_done