t/t5317-pack-objects-filter-objects.sh

   1 #!/bin/sh
   2
   3 test_description='git pack-objects using object filtering'
   4
   5 GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
   6 export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
   7
   8 . ./test-lib.sh
   9
  10 # Test blob:none filter.
  11
  12 test_expect_success 'setup r1' '
  13         git init r1 &&
  14         for n in 1 2 3 4 5
  15         do
  16                 echo "This is file: $n" > r1/file.$n &&
  17                 git -C r1 add file.$n &&
  18                 git -C r1 commit -m "$n" || return 1
  19         done
  20 '
  21
  22 parse_verify_pack_blob_oid () {
  23         awk '{print $1}' -
  24 }
  25
  26 test_expect_success 'verify blob count in normal packfile' '
  27         git -C r1 ls-files -s file.1 file.2 file.3 file.4 file.5 \
  28                 >ls_files_result &&
  29         test_parse_ls_files_stage_oids <ls_files_result |
  30         sort >expected &&
  31
  32         git -C r1 pack-objects --revs --stdout >all.pack <<-EOF &&
  33         HEAD
  34         EOF
  35         git -C r1 index-pack ../all.pack &&
  36
  37         git -C r1 verify-pack -v ../all.pack >verify_result &&
  38         grep blob verify_result |
  39         parse_verify_pack_blob_oid |
  40         sort >observed &&
  41
  42         test_cmp expected observed
  43 '
  44
  45 test_expect_success 'verify blob:none packfile has no blobs' '
  46         git -C r1 pack-objects --revs --stdout --filter=blob:none >filter.pack <<-EOF &&
  47         HEAD
  48         EOF
  49         git -C r1 index-pack ../filter.pack &&
  50
  51         git -C r1 verify-pack -v ../filter.pack >verify_result &&
  52         ! grep blob verify_result
  53 '
  54
  55 test_expect_success 'verify normal and blob:none packfiles have same commits/trees' '
  56         git -C r1 verify-pack -v ../all.pack >verify_result &&
  57         grep -E "commit|tree" verify_result |
  58         parse_verify_pack_blob_oid |
  59         sort >expected &&
  60
  61         git -C r1 verify-pack -v ../filter.pack >verify_result &&
  62         grep -E "commit|tree" verify_result |
  63         parse_verify_pack_blob_oid |
  64         sort >observed &&
  65
  66         test_cmp expected observed
  67 '
  68
  69 test_expect_success 'get an error for missing tree object' '
  70         git init r5 &&
  71         echo foo >r5/foo &&
  72         git -C r5 add foo &&
  73         git -C r5 commit -m "foo" &&
  74         git -C r5 rev-parse HEAD^{tree} >tree &&
  75         del=$(sed "s|..|&/|" tree) &&
  76         rm r5/.git/objects/$del &&
  77         test_must_fail git -C r5 pack-objects --revs --stdout 2>bad_tree <<-EOF &&
  78         HEAD
  79         EOF
  80         grep "bad tree object" bad_tree
  81 '
  82
  83 test_expect_success 'setup for tests of tree:0' '
  84         mkdir r1/subtree &&
  85         echo "This is a file in a subtree" >r1/subtree/file &&
  86         git -C r1 add subtree/file &&
  87         git -C r1 commit -m subtree
  88 '
  89
  90 test_expect_success 'verify tree:0 packfile has no blobs or trees' '
  91         git -C r1 pack-objects --revs --stdout --filter=tree:0 >commitsonly.pack <<-EOF &&
  92         HEAD
  93         EOF
  94         git -C r1 index-pack ../commitsonly.pack &&
  95         git -C r1 verify-pack -v ../commitsonly.pack >objs &&
  96         ! grep -E "tree|blob" objs
  97 '
  98
  99 test_expect_success 'grab tree directly when using tree:0' '
 100         # We should get the tree specified directly but not its blobs or subtrees.
 101         git -C r1 pack-objects --revs --stdout --filter=tree:0 >commitsonly.pack <<-EOF &&
 102         HEAD:
 103         EOF
 104         git -C r1 index-pack ../commitsonly.pack &&
 105         git -C r1 verify-pack -v ../commitsonly.pack >objs &&
 106         awk "/tree|blob/{print \$1}" objs >trees_and_blobs &&
 107         git -C r1 rev-parse HEAD: >expected &&
 108         test_cmp expected trees_and_blobs
 109 '
 110
 111 # Test blob:limit=<n>[kmg] filter.
 112 # We boundary test around the size parameter.  The filter is strictly less than
 113 # the value, so size 500 and 1000 should have the same results, but 1001 should
 114 # filter more.
 115
 116 test_expect_success 'setup r2' '
 117         git init r2 &&
 118         for n in 1000 10000
 119         do
 120                 printf "%"$n"s" X > r2/large.$n &&
 121                 git -C r2 add large.$n &&
 122                 git -C r2 commit -m "$n" || return 1
 123         done
 124 '
 125
 126 test_expect_success 'verify blob count in normal packfile' '
 127         git -C r2 ls-files -s large.1000 large.10000 >ls_files_result &&
 128         test_parse_ls_files_stage_oids <ls_files_result |
 129         sort >expected &&
 130
 131         git -C r2 pack-objects --revs --stdout >all.pack <<-EOF &&
 132         HEAD
 133         EOF
 134         git -C r2 index-pack ../all.pack &&
 135
 136         git -C r2 verify-pack -v ../all.pack >verify_result &&
 137         grep blob verify_result |
 138         parse_verify_pack_blob_oid |
 139         sort >observed &&
 140
 141         test_cmp expected observed
 142 '
 143
 144 test_expect_success 'verify blob:limit=500 omits all blobs' '
 145         git -C r2 pack-objects --revs --stdout --filter=blob:limit=500 >filter.pack <<-EOF &&
 146         HEAD
 147         EOF
 148         git -C r2 index-pack ../filter.pack &&
 149
 150         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 151         ! grep blob verify_result
 152 '
 153
 154 test_expect_success 'verify blob:limit=1000' '
 155         git -C r2 pack-objects --revs --stdout --filter=blob:limit=1000 >filter.pack <<-EOF &&
 156         HEAD
 157         EOF
 158         git -C r2 index-pack ../filter.pack &&
 159
 160         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 161         ! grep blob verify_result
 162 '
 163
 164 test_expect_success 'verify blob:limit=1001' '
 165         git -C r2 ls-files -s large.1000 >ls_files_result &&
 166         test_parse_ls_files_stage_oids <ls_files_result |
 167         sort >expected &&
 168
 169         git -C r2 pack-objects --revs --stdout --filter=blob:limit=1001 >filter.pack <<-EOF &&
 170         HEAD
 171         EOF
 172         git -C r2 index-pack ../filter.pack &&
 173
 174         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 175         grep blob verify_result |
 176         parse_verify_pack_blob_oid |
 177         sort >observed &&
 178
 179         test_cmp expected observed
 180 '
 181
 182 test_expect_success 'verify blob:limit=10001' '
 183         git -C r2 ls-files -s large.1000 large.10000 >ls_files_result &&
 184         test_parse_ls_files_stage_oids <ls_files_result |
 185         sort >expected &&
 186
 187         git -C r2 pack-objects --revs --stdout --filter=blob:limit=10001 >filter.pack <<-EOF &&
 188         HEAD
 189         EOF
 190         git -C r2 index-pack ../filter.pack &&
 191
 192         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 193         grep blob verify_result |
 194         parse_verify_pack_blob_oid |
 195         sort >observed &&
 196
 197         test_cmp expected observed
 198 '
 199
 200 test_expect_success 'verify blob:limit=1k' '
 201         git -C r2 ls-files -s large.1000 >ls_files_result &&
 202         test_parse_ls_files_stage_oids <ls_files_result |
 203         sort >expected &&
 204
 205         git -C r2 pack-objects --revs --stdout --filter=blob:limit=1k >filter.pack <<-EOF &&
 206         HEAD
 207         EOF
 208         git -C r2 index-pack ../filter.pack &&
 209
 210         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 211         grep blob verify_result |
 212         parse_verify_pack_blob_oid |
 213         sort >observed &&
 214
 215         test_cmp expected observed
 216 '
 217
 218 test_expect_success 'verify explicitly specifying oversized blob in input' '
 219         git -C r2 ls-files -s large.1000 large.10000 >ls_files_result &&
 220         test_parse_ls_files_stage_oids <ls_files_result |
 221         sort >expected &&
 222
 223         echo HEAD >objects &&
 224         git -C r2 rev-parse HEAD:large.10000 >>objects &&
 225         git -C r2 pack-objects --revs --stdout --filter=blob:limit=1k <objects >filter.pack &&
 226         git -C r2 index-pack ../filter.pack &&
 227
 228         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 229         grep blob verify_result |
 230         parse_verify_pack_blob_oid |
 231         sort >observed &&
 232
 233         test_cmp expected observed
 234 '
 235
 236 test_expect_success 'verify blob:limit=1m' '
 237         git -C r2 ls-files -s large.1000 large.10000 >ls_files_result &&
 238         test_parse_ls_files_stage_oids <ls_files_result |
 239         sort >expected &&
 240
 241         git -C r2 pack-objects --revs --stdout --filter=blob:limit=1m >filter.pack <<-EOF &&
 242         HEAD
 243         EOF
 244         git -C r2 index-pack ../filter.pack &&
 245
 246         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 247         grep blob verify_result |
 248         parse_verify_pack_blob_oid |
 249         sort >observed &&
 250
 251         test_cmp expected observed
 252 '
 253
 254 test_expect_success 'verify normal and blob:limit packfiles have same commits/trees' '
 255         git -C r2 verify-pack -v ../all.pack >verify_result &&
 256         grep -E "commit|tree" verify_result |
 257         parse_verify_pack_blob_oid |
 258         sort >expected &&
 259
 260         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 261         grep -E "commit|tree" verify_result |
 262         parse_verify_pack_blob_oid |
 263         sort >observed &&
 264
 265         test_cmp expected observed
 266 '
 267
 268 test_expect_failure 'verify small limit and big limit results in small limit' '
 269         git -C r2 ls-files -s large.1000 >ls_files_result &&
 270         test_parse_ls_files_stage_oids <ls_files_result |
 271         sort >expected &&
 272
 273         git -C r2 pack-objects --revs --stdout --filter=blob:limit=1001 \
 274                 --filter=blob:limit=10001 >filter.pack <<-EOF &&
 275         HEAD
 276         EOF
 277         git -C r2 index-pack ../filter.pack &&
 278
 279         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 280         grep blob verify_result |
 281         parse_verify_pack_blob_oid |
 282         sort >observed &&
 283
 284         test_cmp expected observed
 285 '
 286
 287 test_expect_success 'verify big limit and small limit results in small limit' '
 288         git -C r2 ls-files -s large.1000 >ls_files_result &&
 289         test_parse_ls_files_stage_oids <ls_files_result |
 290         sort >expected &&
 291
 292         git -C r2 pack-objects --revs --stdout --filter=blob:limit=10001 \
 293                 --filter=blob:limit=1001 >filter.pack <<-EOF &&
 294         HEAD
 295         EOF
 296         git -C r2 index-pack ../filter.pack &&
 297
 298         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 299         grep blob verify_result |
 300         parse_verify_pack_blob_oid |
 301         sort >observed &&
 302
 303         test_cmp expected observed
 304 '
 305
 306 # Test sparse:path=<path> filter.
 307 # !!!!
 308 # NOTE: sparse:path filter support has been dropped for security reasons,
 309 # so the tests have been changed to make sure that using it fails.
 310 # !!!!
 311 # Use a local file containing a sparse-checkout specification to filter
 312 # out blobs not required for the corresponding sparse-checkout.  We do not
 313 # require sparse-checkout to actually be enabled.
 314
 315 test_expect_success 'setup r3' '
 316         git init r3 &&
 317         mkdir r3/dir1 &&
 318         for n in sparse1 sparse2
 319         do
 320                 echo "This is file: $n" > r3/$n &&
 321                 git -C r3 add $n &&
 322                 echo "This is file: dir1/$n" > r3/dir1/$n &&
 323                 git -C r3 add dir1/$n || return 1
 324         done &&
 325         git -C r3 commit -m "sparse" &&
 326         echo dir1/ >pattern1 &&
 327         echo sparse1 >pattern2
 328 '
 329
 330 test_expect_success 'verify blob count in normal packfile' '
 331         git -C r3 ls-files -s sparse1 sparse2 dir1/sparse1 dir1/sparse2 \
 332                 >ls_files_result &&
 333         test_parse_ls_files_stage_oids <ls_files_result |
 334         sort >expected &&
 335
 336         git -C r3 pack-objects --revs --stdout >all.pack <<-EOF &&
 337         HEAD
 338         EOF
 339         git -C r3 index-pack ../all.pack &&
 340
 341         git -C r3 verify-pack -v ../all.pack >verify_result &&
 342         grep blob verify_result |
 343         parse_verify_pack_blob_oid |
 344         sort >observed &&
 345
 346         test_cmp expected observed
 347 '
 348
 349 test_expect_success 'verify sparse:path=pattern1 fails' '
 350         test_must_fail git -C r3 pack-objects --revs --stdout \
 351                 --filter=sparse:path=../pattern1 <<-EOF
 352         HEAD
 353         EOF
 354 '
 355
 356 test_expect_success 'verify sparse:path=pattern2 fails' '
 357         test_must_fail git -C r3 pack-objects --revs --stdout \
 358                 --filter=sparse:path=../pattern2 <<-EOF
 359         HEAD
 360         EOF
 361 '
 362
 363 # Test sparse:oid=<oid-ish> filter.
 364 # Use a blob containing a sparse-checkout specification to filter
 365 # out blobs not required for the corresponding sparse-checkout.  We do not
 366 # require sparse-checkout to actually be enabled.
 367
 368 test_expect_success 'setup r4' '
 369         git init r4 &&
 370         mkdir r4/dir1 &&
 371         for n in sparse1 sparse2
 372         do
 373                 echo "This is file: $n" > r4/$n &&
 374                 git -C r4 add $n &&
 375                 echo "This is file: dir1/$n" > r4/dir1/$n &&
 376                 git -C r4 add dir1/$n || return 1
 377         done &&
 378         echo dir1/ >r4/pattern &&
 379         git -C r4 add pattern &&
 380         git -C r4 commit -m "pattern"
 381 '
 382
 383 test_expect_success 'verify blob count in normal packfile' '
 384         git -C r4 ls-files -s pattern sparse1 sparse2 dir1/sparse1 dir1/sparse2 \
 385                 >ls_files_result &&
 386         test_parse_ls_files_stage_oids <ls_files_result |
 387         sort >expected &&
 388
 389         git -C r4 pack-objects --revs --stdout >all.pack <<-EOF &&
 390         HEAD
 391         EOF
 392         git -C r4 index-pack ../all.pack &&
 393
 394         git -C r4 verify-pack -v ../all.pack >verify_result &&
 395         grep blob verify_result |
 396         parse_verify_pack_blob_oid |
 397         sort >observed &&
 398
 399         test_cmp expected observed
 400 '
 401
 402 test_expect_success 'verify sparse:oid=OID' '
 403         git -C r4 ls-files -s dir1/sparse1 dir1/sparse2 >ls_files_result &&
 404         test_parse_ls_files_stage_oids <ls_files_result |
 405         sort >expected &&
 406
 407         git -C r4 ls-files -s pattern >staged &&
 408         oid=$(test_parse_ls_files_stage_oids <staged) &&
 409         git -C r4 pack-objects --revs --stdout --filter=sparse:oid=$oid >filter.pack <<-EOF &&
 410         HEAD
 411         EOF
 412         git -C r4 index-pack ../filter.pack &&
 413
 414         git -C r4 verify-pack -v ../filter.pack >verify_result &&
 415         grep blob verify_result |
 416         parse_verify_pack_blob_oid |
 417         sort >observed &&
 418
 419         test_cmp expected observed
 420 '
 421
 422 test_expect_success 'verify sparse:oid=oid-ish' '
 423         git -C r4 ls-files -s dir1/sparse1 dir1/sparse2 >ls_files_result &&
 424         test_parse_ls_files_stage_oids <ls_files_result |
 425         sort >expected &&
 426
 427         git -C r4 pack-objects --revs --stdout --filter=sparse:oid=main:pattern >filter.pack <<-EOF &&
 428         HEAD
 429         EOF
 430         git -C r4 index-pack ../filter.pack &&
 431
 432         git -C r4 verify-pack -v ../filter.pack >verify_result &&
 433         grep blob verify_result |
 434         parse_verify_pack_blob_oid |
 435         sort >observed &&
 436
 437         test_cmp expected observed
 438 '
 439
 440 # Delete some loose objects and use pack-objects, but WITHOUT any filtering.
 441 # This models previously omitted objects that we did not receive.
 442
 443 test_expect_success 'setup r1 - delete loose blobs' '
 444         git -C r1 ls-files -s file.1 file.2 file.3 file.4 file.5 \
 445                 >ls_files_result &&
 446         test_parse_ls_files_stage_oids <ls_files_result |
 447         sort >expected &&
 448
 449         for id in `cat expected | sed "s|..|&/|"`
 450         do
 451                 rm r1/.git/objects/$id || return 1
 452         done
 453 '
 454
 455 test_expect_success 'verify pack-objects fails w/ missing objects' '
 456         test_must_fail git -C r1 pack-objects --revs --stdout >miss.pack <<-EOF
 457         HEAD
 458         EOF
 459 '
 460
 461 test_expect_success 'verify pack-objects fails w/ --missing=error' '
 462         test_must_fail git -C r1 pack-objects --revs --stdout --missing=error >miss.pack <<-EOF
 463         HEAD
 464         EOF
 465 '
 466
 467 test_expect_success 'verify pack-objects w/ --missing=allow-any' '
 468         git -C r1 pack-objects --revs --stdout --missing=allow-any >miss.pack <<-EOF
 469         HEAD
 470         EOF
 471 '
 472
 473 test_done