t/t5326-multi-pack-bitmaps.sh

   1 #!/bin/sh
   2
   3 test_description='exercise basic multi-pack bitmap functionality'
   4
   5 TEST_PASSES_SANITIZE_LEAK=true
   6 . ./test-lib.sh
   7 . "${TEST_DIRECTORY}/lib-bitmap.sh"
   8
   9 # We'll be writing our own MIDX, so avoid getting confused by the
  10 # automatic ones.
  11 GIT_TEST_MULTI_PACK_INDEX=0
  12 GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL=0
  13
  14 # This test exercise multi-pack bitmap functionality where the object order is
  15 # stored and read from a special chunk within the MIDX, so use the default
  16 # behavior here.
  17 sane_unset GIT_TEST_MIDX_WRITE_REV
  18 sane_unset GIT_TEST_MIDX_READ_RIDX
  19
  20 bitmap_reuse_tests() {
  21         from=$1
  22         to=$2
  23         writeLookupTable=false
  24
  25         for i in $3-${$#}
  26         do
  27                 case $i in
  28                 "pack.writeBitmapLookupTable") writeLookupTable=true;;
  29                 esac
  30         done
  31
  32         test_expect_success "setup pack reuse tests ($from -> $to)" '
  33                 rm -fr repo &&
  34                 git init repo &&
  35                 (
  36                         cd repo &&
  37                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
  38                         test_commit_bulk 16 &&
  39                         git tag old-tip &&
  40
  41                         git config core.multiPackIndex true &&
  42                         if test "MIDX" = "$from"
  43                         then
  44                                 git repack -Ad &&
  45                                 git multi-pack-index write --bitmap
  46                         else
  47                                 git repack -Adb
  48                         fi
  49                 )
  50         '
  51
  52         test_expect_success "build bitmap from existing ($from -> $to)" '
  53                 (
  54                         cd repo &&
  55                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
  56                         test_commit_bulk --id=further 16 &&
  57                         git tag new-tip &&
  58
  59                         if test "MIDX" = "$to"
  60                         then
  61                                 git repack -d &&
  62                                 git multi-pack-index write --bitmap
  63                         else
  64                                 git repack -Adb
  65                         fi
  66                 )
  67         '
  68
  69         test_expect_success "verify resulting bitmaps ($from -> $to)" '
  70                 (
  71                         cd repo &&
  72                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
  73                         git for-each-ref &&
  74                         git rev-list --test-bitmap refs/tags/old-tip &&
  75                         git rev-list --test-bitmap refs/tags/new-tip
  76                 )
  77         '
  78 }
  79
  80 test_midx_bitmap_cases () {
  81         writeLookupTable=false
  82         writeBitmapLookupTable=
  83
  84         for i in "$@"
  85         do
  86                 case $i in
  87                 "pack.writeBitmapLookupTable")
  88                         writeLookupTable=true
  89                         writeBitmapLookupTable="$i"
  90                         ;;
  91                 esac
  92         done
  93
  94         test_expect_success 'setup test_repository' '
  95                 rm -rf * .git &&
  96                 git init &&
  97                 git config pack.writeBitmapLookupTable '"$writeLookupTable"'
  98         '
  99
 100         midx_bitmap_core
 101
 102         bitmap_reuse_tests 'pack' 'MIDX' "$writeBitmapLookupTable"
 103         bitmap_reuse_tests 'MIDX' 'pack' "$writeBitmapLookupTable"
 104         bitmap_reuse_tests 'MIDX' 'MIDX' "$writeBitmapLookupTable"
 105
 106         test_expect_success 'missing object closure fails gracefully' '
 107                 rm -fr repo &&
 108                 git init repo &&
 109                 test_when_finished "rm -fr repo" &&
 110                 (
 111                         cd repo &&
 112                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
 113
 114                         test_commit loose &&
 115                         test_commit packed &&
 116
 117                         # Do not pass "--revs"; we want a pack without the "loose"
 118                         # commit.
 119                         git pack-objects $objdir/pack/pack <<-EOF &&
 120                         $(git rev-parse packed)
 121                         EOF
 122
 123                         test_must_fail git multi-pack-index write --bitmap 2>err &&
 124                         grep "doesn.t have full closure" err &&
 125                         test_path_is_missing $midx
 126                 )
 127         '
 128
 129         midx_bitmap_partial_tests
 130
 131         test_expect_success 'removing a MIDX clears stale bitmaps' '
 132                 rm -fr repo &&
 133                 git init repo &&
 134                 test_when_finished "rm -fr repo" &&
 135                 (
 136                         cd repo &&
 137                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
 138                         test_commit base &&
 139                         git repack &&
 140                         git multi-pack-index write --bitmap &&
 141
 142                         # Write a MIDX and bitmap; remove the MIDX but leave the bitmap.
 143                         stale_bitmap=$midx-$(midx_checksum $objdir).bitmap &&
 144                         rm $midx &&
 145
 146                         # Then write a new MIDX.
 147                         test_commit new &&
 148                         git repack &&
 149                         git multi-pack-index write --bitmap &&
 150
 151                         test_path_is_file $midx &&
 152                         test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
 153                         test_path_is_missing $stale_bitmap
 154                 )
 155         '
 156
 157         test_expect_success 'pack.preferBitmapTips' '
 158                 git init repo &&
 159                 test_when_finished "rm -fr repo" &&
 160                 (
 161                         cd repo &&
 162                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
 163
 164                         test_commit_bulk --message="%s" 103 &&
 165
 166                         git log --format="%H" >commits.raw &&
 167                         sort <commits.raw >commits &&
 168
 169                         git log --format="create refs/tags/%s %H" HEAD >refs &&
 170                         git update-ref --stdin <refs &&
 171
 172                         git multi-pack-index write --bitmap &&
 173                         test_path_is_file $midx &&
 174                         test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
 175
 176                         test-tool bitmap list-commits | sort >bitmaps &&
 177                         comm -13 bitmaps commits >before &&
 178                         test_line_count = 1 before &&
 179
 180                         perl -ne "printf(\"create refs/tags/include/%d \", $.); print" \
 181                                 <before | git update-ref --stdin &&
 182
 183                         rm -fr $midx-$(midx_checksum $objdir).bitmap &&
 184                         rm -fr $midx &&
 185
 186                         git -c pack.preferBitmapTips=refs/tags/include \
 187                                 multi-pack-index write --bitmap &&
 188                         test-tool bitmap list-commits | sort >bitmaps &&
 189                         comm -13 bitmaps commits >after &&
 190
 191                         ! test_cmp before after
 192                 )
 193         '
 194
 195         test_expect_success 'writing a bitmap with --refs-snapshot' '
 196                 git init repo &&
 197                 test_when_finished "rm -fr repo" &&
 198                 (
 199                         cd repo &&
 200                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
 201
 202                         test_commit one &&
 203                         test_commit two &&
 204
 205                         git rev-parse one >snapshot &&
 206
 207                         git repack -ad &&
 208
 209                         # First, write a MIDX which see both refs/tags/one and
 210                         # refs/tags/two (causing both of those commits to receive
 211                         # bitmaps).
 212                         git multi-pack-index write --bitmap &&
 213
 214                         test_path_is_file $midx &&
 215                         test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
 216
 217                         test-tool bitmap list-commits | sort >bitmaps &&
 218                         grep "$(git rev-parse one)" bitmaps &&
 219                         grep "$(git rev-parse two)" bitmaps &&
 220
 221                         rm -fr $midx-$(midx_checksum $objdir).bitmap &&
 222                         rm -fr $midx &&
 223
 224                         # Then again, but with a refs snapshot which only sees
 225                         # refs/tags/one.
 226                         git multi-pack-index write --bitmap --refs-snapshot=snapshot &&
 227
 228                         test_path_is_file $midx &&
 229                         test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
 230
 231                         test-tool bitmap list-commits | sort >bitmaps &&
 232                         grep "$(git rev-parse one)" bitmaps &&
 233                         ! grep "$(git rev-parse two)" bitmaps
 234                 )
 235         '
 236
 237         test_expect_success 'write a bitmap with --refs-snapshot (preferred tips)' '
 238                 git init repo &&
 239                 test_when_finished "rm -fr repo" &&
 240                 (
 241                         cd repo &&
 242                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
 243
 244                         test_commit_bulk --message="%s" 103 &&
 245
 246                         git log --format="%H" >commits.raw &&
 247                         sort <commits.raw >commits &&
 248
 249                         git log --format="create refs/tags/%s %H" HEAD >refs &&
 250                         git update-ref --stdin <refs &&
 251
 252                         git multi-pack-index write --bitmap &&
 253                         test_path_is_file $midx &&
 254                         test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
 255
 256                         test-tool bitmap list-commits | sort >bitmaps &&
 257                         comm -13 bitmaps commits >before &&
 258                         test_line_count = 1 before &&
 259
 260                         (
 261                                 grep -vf before commits.raw &&
 262                                 # mark missing commits as preferred
 263                                 sed "s/^/+/" before
 264                         ) >snapshot &&
 265
 266                         rm -fr $midx-$(midx_checksum $objdir).bitmap &&
 267                         rm -fr $midx &&
 268
 269                         git multi-pack-index write --bitmap --refs-snapshot=snapshot &&
 270                         test-tool bitmap list-commits | sort >bitmaps &&
 271                         comm -13 bitmaps commits >after &&
 272
 273                         ! test_cmp before after
 274                 )
 275         '
 276
 277         test_expect_success 'hash-cache values are propagated from pack bitmaps' '
 278                 rm -fr repo &&
 279                 git init repo &&
 280                 test_when_finished "rm -fr repo" &&
 281                 (
 282                         cd repo &&
 283                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
 284
 285                         test_commit base &&
 286                         test_commit base2 &&
 287                         git repack -adb &&
 288
 289                         test-tool bitmap dump-hashes >pack.raw &&
 290                         test_file_not_empty pack.raw &&
 291                         sort pack.raw >pack.hashes &&
 292
 293                         test_commit new &&
 294                         git repack &&
 295                         git multi-pack-index write --bitmap &&
 296
 297                         test-tool bitmap dump-hashes >midx.raw &&
 298                         sort midx.raw >midx.hashes &&
 299
 300                         # ensure that every namehash in the pack bitmap can be found in
 301                         # the midx bitmap (i.e., that there are no oid-namehash pairs
 302                         # unique to the pack bitmap).
 303                         comm -23 pack.hashes midx.hashes >dropped.hashes &&
 304                         test_must_be_empty dropped.hashes
 305                 )
 306         '
 307
 308         test_expect_success 'no .bitmap is written without any objects' '
 309                 rm -fr repo &&
 310                 git init repo &&
 311                 test_when_finished "rm -fr repo" &&
 312                 (
 313                         cd repo &&
 314                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
 315
 316                         empty="$(git pack-objects $objdir/pack/pack </dev/null)" &&
 317                         cat >packs <<-EOF &&
 318                         pack-$empty.idx
 319                         EOF
 320
 321                         git multi-pack-index write --bitmap --stdin-packs \
 322                                 <packs 2>err &&
 323
 324                         grep "bitmap without any objects" err &&
 325
 326                         test_path_is_file $midx &&
 327                         test_path_is_missing $midx-$(midx_checksum $objdir).bitmap
 328                 )
 329         '
 330
 331         test_expect_success 'graceful fallback when missing reverse index' '
 332                 rm -fr repo &&
 333                 git init repo &&
 334                 test_when_finished "rm -fr repo" &&
 335                 (
 336                         cd repo &&
 337                         git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
 338
 339                         test_commit base &&
 340
 341                         # write a pack and MIDX bitmap containing base
 342                         git repack -adb &&
 343                         git multi-pack-index write --bitmap &&
 344
 345                         GIT_TEST_MIDX_READ_RIDX=0 \
 346                                 git rev-list --use-bitmap-index HEAD 2>err &&
 347                         ! grep "ignoring extra bitmap file" err
 348                 )
 349         '
 350 }
 351
 352 test_midx_bitmap_cases
 353
 354 test_midx_bitmap_cases "pack.writeBitmapLookupTable"
 355
 356 test_expect_success 'multi-pack-index write writes lookup table if enabled' '
 357         rm -fr repo &&
 358         git init repo &&
 359         test_when_finished "rm -fr repo" &&
 360         (
 361                 cd repo &&
 362                 test_commit base &&
 363                 git config pack.writeBitmapLookupTable true &&
 364                 git repack -ad &&
 365                 GIT_TRACE2_EVENT="$(pwd)/trace" \
 366                         git multi-pack-index write --bitmap &&
 367                 grep "\"label\":\"writing_lookup_table\"" trace
 368         )
 369 '
 370
 371 test_expect_success 'preferred pack change with existing MIDX bitmap' '
 372         git init preferred-pack-with-existing &&
 373         (
 374                 cd preferred-pack-with-existing &&
 375
 376                 test_commit base &&
 377                 test_commit other &&
 378
 379                 git rev-list --objects --no-object-names base >p1.objects &&
 380                 git rev-list --objects --no-object-names other >p2.objects &&
 381
 382                 p1="$(git pack-objects "$objdir/pack/pack" \
 383                         --delta-base-offset <p1.objects)" &&
 384                 p2="$(git pack-objects "$objdir/pack/pack" \
 385                         --delta-base-offset <p2.objects)" &&
 386
 387                 # Generate a MIDX containing the first two packs,
 388                 # marking p1 as preferred, and ensure that it can be
 389                 # successfully cloned.
 390                 git multi-pack-index write --bitmap \
 391                         --preferred-pack="pack-$p1.pack" &&
 392                 test_path_is_file $midx &&
 393                 test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
 394                 git clone --no-local . clone1 &&
 395
 396                 # Then generate a new pack which sorts ahead of any
 397                 # existing pack (by tweaking the pack prefix).
 398                 test_commit foo &&
 399                 git pack-objects --all --unpacked $objdir/pack/pack0 &&
 400
 401                 # Generate a new MIDX which changes the preferred pack
 402                 # to a pack contained in the existing MIDX.
 403                 git multi-pack-index write --bitmap \
 404                         --preferred-pack="pack-$p2.pack" &&
 405                 test_path_is_file $midx &&
 406                 test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
 407
 408                 # When the above circumstances are met, the preferred
 409                 # pack should change appropriately and clones should
 410                 # (still) succeed.
 411                 git clone --no-local . clone2
 412         )
 413 '
 414
 415 test_expect_success 'tagged commits are selected for bitmapping' '
 416         rm -fr repo &&
 417         git init repo &&
 418         test_when_finished "rm -fr repo" &&
 419         (
 420                 cd repo &&
 421
 422                 test_commit --annotate base &&
 423                 git repack -d &&
 424
 425                 # Remove refs/heads/main which points at the commit directly,
 426                 # leaving only a reference to the annotated tag.
 427                 git branch -M main &&
 428                 git checkout base &&
 429                 git branch -d main &&
 430
 431                 git multi-pack-index write --bitmap &&
 432
 433                 git rev-parse HEAD >want &&
 434                 test-tool bitmap list-commits >actual &&
 435                 grep $(cat want) actual
 436         )
 437 '
 438
 439 test_expect_success 'do not follow replace objects for MIDX bitmap' '
 440         rm -fr repo &&
 441         git init repo &&
 442         test_when_finished "rm -fr repo" &&
 443         (
 444                 cd repo &&
 445
 446                 test_commit A &&
 447                 test_commit B &&
 448                 git checkout --orphan=orphan A &&
 449                 test_commit orphan &&
 450
 451                 git replace A HEAD &&
 452                 git repack -ad --write-midx --write-bitmap-index &&
 453
 454                 # generating reachability bitmaps with replace refs
 455                 # enabled will result in broken clones
 456                 git clone --no-local --bare . clone.git
 457         )
 458 '
 459
 460 corrupt_file () {
 461         chmod a+w "$1" &&
 462         printf "bogus" | dd of="$1" bs=1 seek="12" conv=notrunc
 463 }
 464
 465 test_expect_success 'git fsck correctly identifies good and bad bitmaps' '
 466         git init valid &&
 467         test_when_finished rm -rf valid &&
 468
 469         test_commit_bulk 20 &&
 470         git repack -adbf &&
 471
 472         # Move pack-bitmap aside so it is not deleted
 473         # in next repack.
 474         packbitmap=$(ls .git/objects/pack/pack-*.bitmap) &&
 475         mv "$packbitmap" "$packbitmap.bak" &&
 476
 477         test_commit_bulk 10 &&
 478         git repack -b --write-midx &&
 479         midxbitmap=$(ls .git/objects/pack/multi-pack-index-*.bitmap) &&
 480
 481         # Copy MIDX bitmap to backup. Copy pack bitmap from backup.
 482         cp "$midxbitmap" "$midxbitmap.bak" &&
 483         cp "$packbitmap.bak" "$packbitmap" &&
 484
 485         # fsck works at first
 486         git fsck 2>err &&
 487         test_must_be_empty err &&
 488
 489         corrupt_file "$packbitmap" &&
 490         test_must_fail git fsck 2>err &&
 491         grep "bitmap file '\''$packbitmap'\'' has invalid checksum" err &&
 492
 493         cp "$packbitmap.bak" "$packbitmap" &&
 494         corrupt_file "$midxbitmap" &&
 495         test_must_fail git fsck 2>err &&
 496         grep "bitmap file '\''$midxbitmap'\'' has invalid checksum" err &&
 497
 498         corrupt_file "$packbitmap" &&
 499         test_must_fail git fsck 2>err &&
 500         grep "bitmap file '\''$midxbitmap'\'' has invalid checksum" err &&
 501         grep "bitmap file '\''$packbitmap'\'' has invalid checksum" err
 502 '
 503
 504 test_expect_success 'corrupt MIDX with bitmap causes fallback' '
 505         git init corrupt-midx-bitmap &&
 506         (
 507                 cd corrupt-midx-bitmap &&
 508
 509                 test_commit first &&
 510                 git repack -d &&
 511                 test_commit second &&
 512                 git repack -d &&
 513
 514                 git multi-pack-index write --bitmap &&
 515                 checksum=$(midx_checksum $objdir) &&
 516                 for f in $midx $midx-$checksum.bitmap
 517                 do
 518                         mv $f $f.bak || return 1
 519                 done &&
 520
 521                 # pack everything together, invalidating the MIDX
 522                 git repack -ad &&
 523                 # then restore the now-stale MIDX
 524                 for f in $midx $midx-$checksum.bitmap
 525                 do
 526                         mv $f.bak $f || return 1
 527                 done &&
 528
 529                 git rev-list --count --objects --use-bitmap-index HEAD >out 2>err &&
 530                 # should attempt opening the broken pack twice (once
 531                 # from the attempt to load it via the stale bitmap, and
 532                 # again when attempting to load it from the stale MIDX)
 533                 # before falling back to the non-MIDX case
 534                 test 2 -eq $(grep -c "could not open pack" err) &&
 535                 test 6 -eq $(cat out)
 536         )
 537 '
 538
 539 for allow_pack_reuse in single multi
 540 do
 541         test_expect_success "reading MIDX without BTMP chunk does not complain with $allow_pack_reuse pack reuse" '
 542                 test_when_finished "rm -rf midx-without-btmp" &&
 543                 git init midx-without-btmp &&
 544                 (
 545                         cd midx-without-btmp &&
 546                         test_commit initial &&
 547
 548                         git repack -Adbl --write-bitmap-index --write-midx &&
 549                         GIT_TEST_MIDX_READ_BTMP=false git -c pack.allowPackReuse=$allow_pack_reuse \
 550                                 pack-objects --all --use-bitmap-index --stdout </dev/null >/dev/null 2>err &&
 551                         test_must_be_empty err
 552                 )
 553         '
 554 done
 555
 556 test_expect_success 'remove one packfile between MIDX bitmap writes' '
 557         git init remove-pack-between-writes &&
 558         (
 559                 cd remove-pack-between-writes &&
 560
 561                 test_commit A &&
 562                 test_commit B &&
 563                 test_commit C &&
 564
 565                 # Create packs with the prefix "pack-A", "pack-B",
 566                 # "pack-C" to impose a lexicographic order on these
 567                 # packs so the pack being removed is always from the
 568                 # middle.
 569                 packdir=.git/objects/pack &&
 570                 A="$(echo A | git pack-objects $packdir/pack-A --revs)" &&
 571                 B="$(echo B | git pack-objects $packdir/pack-B --revs)" &&
 572                 C="$(echo C | git pack-objects $packdir/pack-C --revs)" &&
 573
 574                 git multi-pack-index write --bitmap &&
 575
 576                 cat >in <<-EOF &&
 577                 pack-A-$A.idx
 578                 pack-C-$C.idx
 579                 EOF
 580                 git multi-pack-index write --bitmap --stdin-packs <in &&
 581
 582                 git rev-list --test-bitmap HEAD
 583         )
 584 '
 585
 586 test_done