t/t5310-pack-bitmaps.sh

   1 #!/bin/sh
   2
   3 test_description='exercise basic bitmap functionality'
   4 . ./test-lib.sh
   5
   6 objpath () {
   7         echo ".git/objects/$(echo "$1" | sed -e 's|\(..\)|\1/|')"
   8 }
   9
  10 # show objects present in pack ($1 should be associated *.idx)
  11 list_packed_objects () {
  12         git show-index <"$1" >object-list &&
  13         cut -d' ' -f2 object-list
  14 }
  15
  16 # has_any pattern-file content-file
  17 # tests whether content-file has any entry from pattern-file with entries being
  18 # whole lines.
  19 has_any () {
  20         grep -Ff "$1" "$2"
  21 }
  22
  23 test_expect_success 'setup repo with moderate-sized history' '
  24         test_commit_bulk --id=file 100 &&
  25         git branch -M second &&
  26         git checkout -b other HEAD~5 &&
  27         test_commit_bulk --id=side 10 &&
  28         git checkout second &&
  29         bitmaptip=$(git rev-parse second) &&
  30         blob=$(echo tagged-blob | git hash-object -w --stdin) &&
  31         git tag tagged-blob $blob &&
  32         git config repack.writebitmaps true
  33 '
  34
  35 test_expect_success 'full repack creates bitmaps' '
  36         git repack -ad &&
  37         ls .git/objects/pack/ | grep bitmap >output &&
  38         test_line_count = 1 output
  39 '
  40
  41 test_expect_success 'rev-list --test-bitmap verifies bitmaps' '
  42         git rev-list --test-bitmap HEAD
  43 '
  44
  45 rev_list_tests_head () {
  46         test_expect_success "counting commits via bitmap ($state, $branch)" '
  47                 git rev-list --count $branch >expect &&
  48                 git rev-list --use-bitmap-index --count $branch >actual &&
  49                 test_cmp expect actual
  50         '
  51
  52         test_expect_success "counting partial commits via bitmap ($state, $branch)" '
  53                 git rev-list --count $branch~5..$branch >expect &&
  54                 git rev-list --use-bitmap-index --count $branch~5..$branch >actual &&
  55                 test_cmp expect actual
  56         '
  57
  58         test_expect_success "counting commits with limit ($state, $branch)" '
  59                 git rev-list --count -n 1 $branch >expect &&
  60                 git rev-list --use-bitmap-index --count -n 1 $branch >actual &&
  61                 test_cmp expect actual
  62         '
  63
  64         test_expect_success "counting non-linear history ($state, $branch)" '
  65                 git rev-list --count other...second >expect &&
  66                 git rev-list --use-bitmap-index --count other...second >actual &&
  67                 test_cmp expect actual
  68         '
  69
  70         test_expect_success "counting commits with limiting ($state, $branch)" '
  71                 git rev-list --count $branch -- 1.t >expect &&
  72                 git rev-list --use-bitmap-index --count $branch -- 1.t >actual &&
  73                 test_cmp expect actual
  74         '
  75
  76         test_expect_success "counting objects via bitmap ($state, $branch)" '
  77                 git rev-list --count --objects $branch >expect &&
  78                 git rev-list --use-bitmap-index --count --objects $branch >actual &&
  79                 test_cmp expect actual
  80         '
  81
  82         test_expect_success "enumerate commits ($state, $branch)" '
  83                 git rev-list --use-bitmap-index $branch >actual &&
  84                 git rev-list $branch >expect &&
  85                 test_bitmap_traversal --no-confirm-bitmaps expect actual
  86         '
  87
  88         test_expect_success "enumerate --objects ($state, $branch)" '
  89                 git rev-list --objects --use-bitmap-index $branch >actual &&
  90                 git rev-list --objects $branch >expect &&
  91                 test_bitmap_traversal expect actual
  92         '
  93
  94         test_expect_success "bitmap --objects handles non-commit objects ($state, $branch)" '
  95                 git rev-list --objects --use-bitmap-index $branch tagged-blob >actual &&
  96                 grep $blob actual
  97         '
  98 }
  99
 100 rev_list_tests () {
 101         state=$1
 102
 103         for branch in "second" "other"
 104         do
 105                 rev_list_tests_head
 106         done
 107 }
 108
 109 rev_list_tests 'full bitmap'
 110
 111 test_expect_success 'clone from bitmapped repository' '
 112         git clone --no-local --bare . clone.git &&
 113         git rev-parse HEAD >expect &&
 114         git --git-dir=clone.git rev-parse HEAD >actual &&
 115         test_cmp expect actual
 116 '
 117
 118 test_expect_success 'partial clone from bitmapped repository' '
 119         test_config uploadpack.allowfilter true &&
 120         git clone --no-local --bare --filter=blob:none . partial-clone.git &&
 121         (
 122                 cd partial-clone.git &&
 123                 pack=$(echo objects/pack/*.pack) &&
 124                 git verify-pack -v "$pack" >have &&
 125                 awk "/blob/ { print \$1 }" <have >blobs &&
 126                 # we expect this single blob because of the direct ref
 127                 git rev-parse refs/tags/tagged-blob >expect &&
 128                 test_cmp expect blobs
 129         )
 130 '
 131
 132 test_expect_success 'setup further non-bitmapped commits' '
 133         test_commit_bulk --id=further 10
 134 '
 135
 136 rev_list_tests 'partial bitmap'
 137
 138 test_expect_success 'fetch (partial bitmap)' '
 139         git --git-dir=clone.git fetch origin second:second &&
 140         git rev-parse HEAD >expect &&
 141         git --git-dir=clone.git rev-parse HEAD >actual &&
 142         test_cmp expect actual
 143 '
 144
 145 test_expect_success 'incremental repack fails when bitmaps are requested' '
 146         test_commit more-1 &&
 147         test_must_fail git repack -d 2>err &&
 148         test_i18ngrep "Incremental repacks are incompatible with bitmap" err
 149 '
 150
 151 test_expect_success 'incremental repack can disable bitmaps' '
 152         test_commit more-2 &&
 153         git repack -d --no-write-bitmap-index
 154 '
 155
 156 test_expect_success 'pack-objects respects --local (non-local loose)' '
 157         git init --bare alt.git &&
 158         echo $(pwd)/alt.git/objects >.git/objects/info/alternates &&
 159         echo content1 >file1 &&
 160         # non-local loose object which is not present in bitmapped pack
 161         altblob=$(GIT_DIR=alt.git git hash-object -w file1) &&
 162         # non-local loose object which is also present in bitmapped pack
 163         git cat-file blob $blob | GIT_DIR=alt.git git hash-object -w --stdin &&
 164         git add file1 &&
 165         test_tick &&
 166         git commit -m commit_file1 &&
 167         echo HEAD | git pack-objects --local --stdout --revs >1.pack &&
 168         git index-pack 1.pack &&
 169         list_packed_objects 1.idx >1.objects &&
 170         printf "%s\n" "$altblob" "$blob" >nonlocal-loose &&
 171         ! has_any nonlocal-loose 1.objects
 172 '
 173
 174 test_expect_success 'pack-objects respects --honor-pack-keep (local non-bitmapped pack)' '
 175         echo content2 >file2 &&
 176         blob2=$(git hash-object -w file2) &&
 177         git add file2 &&
 178         test_tick &&
 179         git commit -m commit_file2 &&
 180         printf "%s\n" "$blob2" "$bitmaptip" >keepobjects &&
 181         pack2=$(git pack-objects pack2 <keepobjects) &&
 182         mv pack2-$pack2.* .git/objects/pack/ &&
 183         >.git/objects/pack/pack2-$pack2.keep &&
 184         rm $(objpath $blob2) &&
 185         echo HEAD | git pack-objects --honor-pack-keep --stdout --revs >2a.pack &&
 186         git index-pack 2a.pack &&
 187         list_packed_objects 2a.idx >2a.objects &&
 188         ! has_any keepobjects 2a.objects
 189 '
 190
 191 test_expect_success 'pack-objects respects --local (non-local pack)' '
 192         mv .git/objects/pack/pack2-$pack2.* alt.git/objects/pack/ &&
 193         echo HEAD | git pack-objects --local --stdout --revs >2b.pack &&
 194         git index-pack 2b.pack &&
 195         list_packed_objects 2b.idx >2b.objects &&
 196         ! has_any keepobjects 2b.objects
 197 '
 198
 199 test_expect_success 'pack-objects respects --honor-pack-keep (local bitmapped pack)' '
 200         ls .git/objects/pack/ | grep bitmap >output &&
 201         test_line_count = 1 output &&
 202         packbitmap=$(basename $(cat output) .bitmap) &&
 203         list_packed_objects .git/objects/pack/$packbitmap.idx >packbitmap.objects &&
 204         test_when_finished "rm -f .git/objects/pack/$packbitmap.keep" &&
 205         >.git/objects/pack/$packbitmap.keep &&
 206         echo HEAD | git pack-objects --honor-pack-keep --stdout --revs >3a.pack &&
 207         git index-pack 3a.pack &&
 208         list_packed_objects 3a.idx >3a.objects &&
 209         ! has_any packbitmap.objects 3a.objects
 210 '
 211
 212 test_expect_success 'pack-objects respects --local (non-local bitmapped pack)' '
 213         mv .git/objects/pack/$packbitmap.* alt.git/objects/pack/ &&
 214         rm -f .git/objects/pack/multi-pack-index &&
 215         test_when_finished "mv alt.git/objects/pack/$packbitmap.* .git/objects/pack/" &&
 216         echo HEAD | git pack-objects --local --stdout --revs >3b.pack &&
 217         git index-pack 3b.pack &&
 218         list_packed_objects 3b.idx >3b.objects &&
 219         ! has_any packbitmap.objects 3b.objects
 220 '
 221
 222 test_expect_success 'pack-objects to file can use bitmap' '
 223         # make sure we still have 1 bitmap index from previous tests
 224         ls .git/objects/pack/ | grep bitmap >output &&
 225         test_line_count = 1 output &&
 226         # verify equivalent packs are generated with/without using bitmap index
 227         packasha1=$(git pack-objects --no-use-bitmap-index --all packa </dev/null) &&
 228         packbsha1=$(git pack-objects --use-bitmap-index --all packb </dev/null) &&
 229         list_packed_objects packa-$packasha1.idx >packa.objects &&
 230         list_packed_objects packb-$packbsha1.idx >packb.objects &&
 231         test_cmp packa.objects packb.objects
 232 '
 233
 234 test_expect_success 'full repack, reusing previous bitmaps' '
 235         git repack -ad &&
 236         ls .git/objects/pack/ | grep bitmap >output &&
 237         test_line_count = 1 output
 238 '
 239
 240 test_expect_success 'fetch (full bitmap)' '
 241         git --git-dir=clone.git fetch origin second:second &&
 242         git rev-parse HEAD >expect &&
 243         git --git-dir=clone.git rev-parse HEAD >actual &&
 244         test_cmp expect actual
 245 '
 246
 247 test_expect_success 'create objects for missing-HAVE tests' '
 248         blob=$(echo "missing have" | git hash-object -w --stdin) &&
 249         tree=$(printf "100644 blob $blob\tfile\n" | git mktree) &&
 250         parent=$(echo parent | git commit-tree $tree) &&
 251         commit=$(echo commit | git commit-tree $tree -p $parent) &&
 252         cat >revs <<-EOF
 253         HEAD
 254         ^HEAD^
 255         ^$commit
 256         EOF
 257 '
 258
 259 test_expect_success 'pack-objects respects --incremental' '
 260         cat >revs2 <<-EOF &&
 261         HEAD
 262         $commit
 263         EOF
 264         git pack-objects --incremental --stdout --revs <revs2 >4.pack &&
 265         git index-pack 4.pack &&
 266         list_packed_objects 4.idx >4.objects &&
 267         test_line_count = 4 4.objects &&
 268         git rev-list --objects $commit >revlist &&
 269         cut -d" " -f1 revlist |sort >objects &&
 270         test_cmp 4.objects objects
 271 '
 272
 273 test_expect_success 'pack with missing blob' '
 274         rm $(objpath $blob) &&
 275         git pack-objects --stdout --revs <revs >/dev/null
 276 '
 277
 278 test_expect_success 'pack with missing tree' '
 279         rm $(objpath $tree) &&
 280         git pack-objects --stdout --revs <revs >/dev/null
 281 '
 282
 283 test_expect_success 'pack with missing parent' '
 284         rm $(objpath $parent) &&
 285         git pack-objects --stdout --revs <revs >/dev/null
 286 '
 287
 288 test_expect_success JGIT 'we can read jgit bitmaps' '
 289         git clone --bare . compat-jgit.git &&
 290         (
 291                 cd compat-jgit.git &&
 292                 rm -f objects/pack/*.bitmap &&
 293                 jgit gc &&
 294                 git rev-list --test-bitmap HEAD
 295         )
 296 '
 297
 298 test_expect_success JGIT 'jgit can read our bitmaps' '
 299         git clone --bare . compat-us.git &&
 300         (
 301                 cd compat-us.git &&
 302                 git repack -adb &&
 303                 # jgit gc will barf if it does not like our bitmaps
 304                 jgit gc
 305         )
 306 '
 307
 308 test_expect_success 'splitting packs does not generate bogus bitmaps' '
 309         test-tool genrandom foo $((1024 * 1024)) >rand &&
 310         git add rand &&
 311         git commit -m "commit with big file" &&
 312         git -c pack.packSizeLimit=500k repack -adb &&
 313         git init --bare no-bitmaps.git &&
 314         git -C no-bitmaps.git fetch .. HEAD
 315 '
 316
 317 test_expect_success 'set up reusable pack' '
 318         rm -f .git/objects/pack/*.keep &&
 319         git repack -adb &&
 320         reusable_pack () {
 321                 git for-each-ref --format="%(objectname)" |
 322                 git pack-objects --delta-base-offset --revs --stdout "$@"
 323         }
 324 '
 325
 326 test_expect_success 'pack reuse respects --honor-pack-keep' '
 327         test_when_finished "rm -f .git/objects/pack/*.keep" &&
 328         for i in .git/objects/pack/*.pack
 329         do
 330                 >${i%.pack}.keep
 331         done &&
 332         reusable_pack --honor-pack-keep >empty.pack &&
 333         git index-pack empty.pack &&
 334         git show-index <empty.idx >actual &&
 335         test_must_be_empty actual
 336 '
 337
 338 test_expect_success 'pack reuse respects --local' '
 339         mv .git/objects/pack/* alt.git/objects/pack/ &&
 340         test_when_finished "mv alt.git/objects/pack/* .git/objects/pack/" &&
 341         reusable_pack --local >empty.pack &&
 342         git index-pack empty.pack &&
 343         git show-index <empty.idx >actual &&
 344         test_must_be_empty actual
 345 '
 346
 347 test_expect_success 'pack reuse respects --incremental' '
 348         reusable_pack --incremental >empty.pack &&
 349         git index-pack empty.pack &&
 350         git show-index <empty.idx >actual &&
 351         test_must_be_empty actual
 352 '
 353
 354 test_expect_success 'truncated bitmap fails gracefully (ewah)' '
 355         test_config pack.writebitmaphashcache false &&
 356         git repack -ad &&
 357         git rev-list --use-bitmap-index --count --all >expect &&
 358         bitmap=$(ls .git/objects/pack/*.bitmap) &&
 359         test_when_finished "rm -f $bitmap" &&
 360         test_copy_bytes 256 <$bitmap >$bitmap.tmp &&
 361         mv -f $bitmap.tmp $bitmap &&
 362         git rev-list --use-bitmap-index --count --all >actual 2>stderr &&
 363         test_cmp expect actual &&
 364         test_i18ngrep corrupt.ewah.bitmap stderr
 365 '
 366
 367 test_expect_success 'truncated bitmap fails gracefully (cache)' '
 368         git repack -ad &&
 369         git rev-list --use-bitmap-index --count --all >expect &&
 370         bitmap=$(ls .git/objects/pack/*.bitmap) &&
 371         test_when_finished "rm -f $bitmap" &&
 372         test_copy_bytes 512 <$bitmap >$bitmap.tmp &&
 373         mv -f $bitmap.tmp $bitmap &&
 374         git rev-list --use-bitmap-index --count --all >actual 2>stderr &&
 375         test_cmp expect actual &&
 376         test_i18ngrep corrupted.bitmap.index stderr
 377 '
 378
 379 # have_delta <obj> <expected_base>
 380 #
 381 # Note that because this relies on cat-file, it might find _any_ copy of an
 382 # object in the repository. The caller is responsible for making sure
 383 # there's only one (e.g., via "repack -ad", or having just fetched a copy).
 384 have_delta () {
 385         echo $2 >expect &&
 386         echo $1 | git cat-file --batch-check="%(deltabase)" >actual &&
 387         test_cmp expect actual
 388 }
 389
 390 # Create a state of history with these properties:
 391 #
 392 #  - refs that allow a client to fetch some new history, while sharing some old
 393 #    history with the server; we use branches delta-reuse-old and
 394 #    delta-reuse-new here
 395 #
 396 #  - the new history contains an object that is stored on the server as a delta
 397 #    against a base that is in the old history
 398 #
 399 #  - the base object is not immediately reachable from the tip of the old
 400 #    history; finding it would involve digging down through history we know the
 401 #    other side has
 402 #
 403 # This should result in a state where fetching from old->new would not
 404 # traditionally reuse the on-disk delta (because we'd have to dig to realize
 405 # that the client has it), but we will do so if bitmaps can tell us cheaply
 406 # that the other side has it.
 407 test_expect_success 'set up thin delta-reuse parent' '
 408         # This first commit contains the buried base object.
 409         test-tool genrandom delta 16384 >file &&
 410         git add file &&
 411         git commit -m "delta base" &&
 412         base=$(git rev-parse --verify HEAD:file) &&
 413
 414         # These intermediate commits bury the base back in history.
 415         # This becomes the "old" state.
 416         for i in 1 2 3 4 5
 417         do
 418                 echo $i >file &&
 419                 git commit -am "intermediate $i" || return 1
 420         done &&
 421         git branch delta-reuse-old &&
 422
 423         # And now our new history has a delta against the buried base. Note
 424         # that this must be smaller than the original file, since pack-objects
 425         # prefers to create deltas from smaller objects to larger.
 426         test-tool genrandom delta 16300 >file &&
 427         git commit -am "delta result" &&
 428         delta=$(git rev-parse --verify HEAD:file) &&
 429         git branch delta-reuse-new &&
 430
 431         # Repack with bitmaps and double check that we have the expected delta
 432         # relationship.
 433         git repack -adb &&
 434         have_delta $delta $base
 435 '
 436
 437 # Now we can sanity-check the non-bitmap behavior (that the server is not able
 438 # to reuse the delta). This isn't strictly something we care about, so this
 439 # test could be scrapped in the future. But it makes sure that the next test is
 440 # actually triggering the feature we want.
 441 #
 442 # Note that our tools for working with on-the-wire "thin" packs are limited. So
 443 # we actually perform the fetch, retain the resulting pack, and inspect the
 444 # result.
 445 test_expect_success 'fetch without bitmaps ignores delta against old base' '
 446         test_config pack.usebitmaps false &&
 447         test_when_finished "rm -rf client.git" &&
 448         git init --bare client.git &&
 449         (
 450                 cd client.git &&
 451                 git config transfer.unpackLimit 1 &&
 452                 git fetch .. delta-reuse-old:delta-reuse-old &&
 453                 git fetch .. delta-reuse-new:delta-reuse-new &&
 454                 have_delta $delta $ZERO_OID
 455         )
 456 '
 457
 458 # And do the same for the bitmap case, where we do expect to find the delta.
 459 test_expect_success 'fetch with bitmaps can reuse old base' '
 460         test_config pack.usebitmaps true &&
 461         test_when_finished "rm -rf client.git" &&
 462         git init --bare client.git &&
 463         (
 464                 cd client.git &&
 465                 git config transfer.unpackLimit 1 &&
 466                 git fetch .. delta-reuse-old:delta-reuse-old &&
 467                 git fetch .. delta-reuse-new:delta-reuse-new &&
 468                 have_delta $delta $base
 469         )
 470 '
 471
 472 test_done