t/t5317-pack-objects-filter-objects.sh

   1 #!/bin/sh
   2
   3 test_description='git pack-objects using object filtering'
   4
   5 GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
   6 export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
   7
   8 . ./test-lib.sh
   9
  10 # Test blob:none filter.
  11
  12 test_expect_success 'setup r1' '
  13         git init r1 &&
  14         for n in 1 2 3 4 5
  15         do
  16                 echo "This is file: $n" > r1/file.$n &&
  17                 git -C r1 add file.$n &&
  18                 git -C r1 commit -m "$n" || return 1
  19         done
  20 '
  21
  22 parse_verify_pack_blob_oid () {
  23         awk '{print $1}' -
  24 }
  25
  26 test_expect_success 'verify blob count in normal packfile' '
  27         git -C r1 ls-files -s file.1 file.2 file.3 file.4 file.5 |
  28         test_parse_ls_files_stage_oids |
  29         sort >expected &&
  30
  31         git -C r1 pack-objects --revs --stdout >all.pack <<-EOF &&
  32         HEAD
  33         EOF
  34         git -C r1 index-pack ../all.pack &&
  35
  36         git -C r1 verify-pack -v ../all.pack >verify_result &&
  37         grep blob verify_result |
  38         parse_verify_pack_blob_oid |
  39         sort >observed &&
  40
  41         test_cmp expected observed
  42 '
  43
  44 test_expect_success 'verify blob:none packfile has no blobs' '
  45         git -C r1 pack-objects --revs --stdout --filter=blob:none >filter.pack <<-EOF &&
  46         HEAD
  47         EOF
  48         git -C r1 index-pack ../filter.pack &&
  49
  50         git -C r1 verify-pack -v ../filter.pack >verify_result &&
  51         ! grep blob verify_result
  52 '
  53
  54 test_expect_success 'verify normal and blob:none packfiles have same commits/trees' '
  55         git -C r1 verify-pack -v ../all.pack >verify_result &&
  56         grep -E "commit|tree" verify_result |
  57         parse_verify_pack_blob_oid |
  58         sort >expected &&
  59
  60         git -C r1 verify-pack -v ../filter.pack >verify_result &&
  61         grep -E "commit|tree" verify_result |
  62         parse_verify_pack_blob_oid |
  63         sort >observed &&
  64
  65         test_cmp expected observed
  66 '
  67
  68 test_expect_success 'get an error for missing tree object' '
  69         git init r5 &&
  70         echo foo >r5/foo &&
  71         git -C r5 add foo &&
  72         git -C r5 commit -m "foo" &&
  73         git -C r5 rev-parse HEAD^{tree} >tree &&
  74         del=$(sed "s|..|&/|" tree) &&
  75         rm r5/.git/objects/$del &&
  76         test_must_fail git -C r5 pack-objects --revs --stdout 2>bad_tree <<-EOF &&
  77         HEAD
  78         EOF
  79         grep "bad tree object" bad_tree
  80 '
  81
  82 test_expect_success 'setup for tests of tree:0' '
  83         mkdir r1/subtree &&
  84         echo "This is a file in a subtree" >r1/subtree/file &&
  85         git -C r1 add subtree/file &&
  86         git -C r1 commit -m subtree
  87 '
  88
  89 test_expect_success 'verify tree:0 packfile has no blobs or trees' '
  90         git -C r1 pack-objects --revs --stdout --filter=tree:0 >commitsonly.pack <<-EOF &&
  91         HEAD
  92         EOF
  93         git -C r1 index-pack ../commitsonly.pack &&
  94         git -C r1 verify-pack -v ../commitsonly.pack >objs &&
  95         ! grep -E "tree|blob" objs
  96 '
  97
  98 test_expect_success 'grab tree directly when using tree:0' '
  99         # We should get the tree specified directly but not its blobs or subtrees.
 100         git -C r1 pack-objects --revs --stdout --filter=tree:0 >commitsonly.pack <<-EOF &&
 101         HEAD:
 102         EOF
 103         git -C r1 index-pack ../commitsonly.pack &&
 104         git -C r1 verify-pack -v ../commitsonly.pack >objs &&
 105         awk "/tree|blob/{print \$1}" objs >trees_and_blobs &&
 106         git -C r1 rev-parse HEAD: >expected &&
 107         test_cmp expected trees_and_blobs
 108 '
 109
 110 # Test blob:limit=<n>[kmg] filter.
 111 # We boundary test around the size parameter.  The filter is strictly less than
 112 # the value, so size 500 and 1000 should have the same results, but 1001 should
 113 # filter more.
 114
 115 test_expect_success 'setup r2' '
 116         git init r2 &&
 117         for n in 1000 10000
 118         do
 119                 printf "%"$n"s" X > r2/large.$n &&
 120                 git -C r2 add large.$n &&
 121                 git -C r2 commit -m "$n" || return 1
 122         done
 123 '
 124
 125 test_expect_success 'verify blob count in normal packfile' '
 126         git -C r2 ls-files -s large.1000 large.10000 |
 127         test_parse_ls_files_stage_oids |
 128         sort >expected &&
 129
 130         git -C r2 pack-objects --revs --stdout >all.pack <<-EOF &&
 131         HEAD
 132         EOF
 133         git -C r2 index-pack ../all.pack &&
 134
 135         git -C r2 verify-pack -v ../all.pack >verify_result &&
 136         grep blob verify_result |
 137         parse_verify_pack_blob_oid |
 138         sort >observed &&
 139
 140         test_cmp expected observed
 141 '
 142
 143 test_expect_success 'verify blob:limit=500 omits all blobs' '
 144         git -C r2 pack-objects --revs --stdout --filter=blob:limit=500 >filter.pack <<-EOF &&
 145         HEAD
 146         EOF
 147         git -C r2 index-pack ../filter.pack &&
 148
 149         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 150         ! grep blob verify_result
 151 '
 152
 153 test_expect_success 'verify blob:limit=1000' '
 154         git -C r2 pack-objects --revs --stdout --filter=blob:limit=1000 >filter.pack <<-EOF &&
 155         HEAD
 156         EOF
 157         git -C r2 index-pack ../filter.pack &&
 158
 159         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 160         ! grep blob verify_result
 161 '
 162
 163 test_expect_success 'verify blob:limit=1001' '
 164         git -C r2 ls-files -s large.1000 |
 165         test_parse_ls_files_stage_oids |
 166         sort >expected &&
 167
 168         git -C r2 pack-objects --revs --stdout --filter=blob:limit=1001 >filter.pack <<-EOF &&
 169         HEAD
 170         EOF
 171         git -C r2 index-pack ../filter.pack &&
 172
 173         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 174         grep blob verify_result |
 175         parse_verify_pack_blob_oid |
 176         sort >observed &&
 177
 178         test_cmp expected observed
 179 '
 180
 181 test_expect_success 'verify blob:limit=10001' '
 182         git -C r2 ls-files -s large.1000 large.10000 |
 183         test_parse_ls_files_stage_oids |
 184         sort >expected &&
 185
 186         git -C r2 pack-objects --revs --stdout --filter=blob:limit=10001 >filter.pack <<-EOF &&
 187         HEAD
 188         EOF
 189         git -C r2 index-pack ../filter.pack &&
 190
 191         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 192         grep blob verify_result |
 193         parse_verify_pack_blob_oid |
 194         sort >observed &&
 195
 196         test_cmp expected observed
 197 '
 198
 199 test_expect_success 'verify blob:limit=1k' '
 200         git -C r2 ls-files -s large.1000 |
 201         test_parse_ls_files_stage_oids |
 202         sort >expected &&
 203
 204         git -C r2 pack-objects --revs --stdout --filter=blob:limit=1k >filter.pack <<-EOF &&
 205         HEAD
 206         EOF
 207         git -C r2 index-pack ../filter.pack &&
 208
 209         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 210         grep blob verify_result |
 211         parse_verify_pack_blob_oid |
 212         sort >observed &&
 213
 214         test_cmp expected observed
 215 '
 216
 217 test_expect_success 'verify explicitly specifying oversized blob in input' '
 218         git -C r2 ls-files -s large.1000 large.10000 |
 219         test_parse_ls_files_stage_oids |
 220         sort >expected &&
 221
 222         echo HEAD >objects &&
 223         git -C r2 rev-parse HEAD:large.10000 >>objects &&
 224         git -C r2 pack-objects --revs --stdout --filter=blob:limit=1k <objects >filter.pack &&
 225         git -C r2 index-pack ../filter.pack &&
 226
 227         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 228         grep blob verify_result |
 229         parse_verify_pack_blob_oid |
 230         sort >observed &&
 231
 232         test_cmp expected observed
 233 '
 234
 235 test_expect_success 'verify blob:limit=1m' '
 236         git -C r2 ls-files -s large.1000 large.10000 |
 237         test_parse_ls_files_stage_oids |
 238         sort >expected &&
 239
 240         git -C r2 pack-objects --revs --stdout --filter=blob:limit=1m >filter.pack <<-EOF &&
 241         HEAD
 242         EOF
 243         git -C r2 index-pack ../filter.pack &&
 244
 245         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 246         grep blob verify_result |
 247         parse_verify_pack_blob_oid |
 248         sort >observed &&
 249
 250         test_cmp expected observed
 251 '
 252
 253 test_expect_success 'verify normal and blob:limit packfiles have same commits/trees' '
 254         git -C r2 verify-pack -v ../all.pack >verify_result &&
 255         grep -E "commit|tree" verify_result |
 256         parse_verify_pack_blob_oid |
 257         sort >expected &&
 258
 259         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 260         grep -E "commit|tree" verify_result |
 261         parse_verify_pack_blob_oid |
 262         sort >observed &&
 263
 264         test_cmp expected observed
 265 '
 266
 267 # Test sparse:path=<path> filter.
 268 # !!!!
 269 # NOTE: sparse:path filter support has been dropped for security reasons,
 270 # so the tests have been changed to make sure that using it fails.
 271 # !!!!
 272 # Use a local file containing a sparse-checkout specification to filter
 273 # out blobs not required for the corresponding sparse-checkout.  We do not
 274 # require sparse-checkout to actually be enabled.
 275
 276 test_expect_success 'setup r3' '
 277         git init r3 &&
 278         mkdir r3/dir1 &&
 279         for n in sparse1 sparse2
 280         do
 281                 echo "This is file: $n" > r3/$n &&
 282                 git -C r3 add $n &&
 283                 echo "This is file: dir1/$n" > r3/dir1/$n &&
 284                 git -C r3 add dir1/$n || return 1
 285         done &&
 286         git -C r3 commit -m "sparse" &&
 287         echo dir1/ >pattern1 &&
 288         echo sparse1 >pattern2
 289 '
 290
 291 test_expect_success 'verify blob count in normal packfile' '
 292         git -C r3 ls-files -s sparse1 sparse2 dir1/sparse1 dir1/sparse2 |
 293         test_parse_ls_files_stage_oids |
 294         sort >expected &&
 295
 296         git -C r3 pack-objects --revs --stdout >all.pack <<-EOF &&
 297         HEAD
 298         EOF
 299         git -C r3 index-pack ../all.pack &&
 300
 301         git -C r3 verify-pack -v ../all.pack >verify_result &&
 302         grep blob verify_result |
 303         parse_verify_pack_blob_oid |
 304         sort >observed &&
 305
 306         test_cmp expected observed
 307 '
 308
 309 test_expect_success 'verify sparse:path=pattern1 fails' '
 310         test_must_fail git -C r3 pack-objects --revs --stdout \
 311                 --filter=sparse:path=../pattern1 <<-EOF
 312         HEAD
 313         EOF
 314 '
 315
 316 test_expect_success 'verify sparse:path=pattern2 fails' '
 317         test_must_fail git -C r3 pack-objects --revs --stdout \
 318                 --filter=sparse:path=../pattern2 <<-EOF
 319         HEAD
 320         EOF
 321 '
 322
 323 # Test sparse:oid=<oid-ish> filter.
 324 # Use a blob containing a sparse-checkout specification to filter
 325 # out blobs not required for the corresponding sparse-checkout.  We do not
 326 # require sparse-checkout to actually be enabled.
 327
 328 test_expect_success 'setup r4' '
 329         git init r4 &&
 330         mkdir r4/dir1 &&
 331         for n in sparse1 sparse2
 332         do
 333                 echo "This is file: $n" > r4/$n &&
 334                 git -C r4 add $n &&
 335                 echo "This is file: dir1/$n" > r4/dir1/$n &&
 336                 git -C r4 add dir1/$n || return 1
 337         done &&
 338         echo dir1/ >r4/pattern &&
 339         git -C r4 add pattern &&
 340         git -C r4 commit -m "pattern"
 341 '
 342
 343 test_expect_success 'verify blob count in normal packfile' '
 344         git -C r4 ls-files -s pattern sparse1 sparse2 dir1/sparse1 dir1/sparse2 |
 345         test_parse_ls_files_stage_oids |
 346         sort >expected &&
 347
 348         git -C r4 pack-objects --revs --stdout >all.pack <<-EOF &&
 349         HEAD
 350         EOF
 351         git -C r4 index-pack ../all.pack &&
 352
 353         git -C r4 verify-pack -v ../all.pack >verify_result &&
 354         grep blob verify_result |
 355         parse_verify_pack_blob_oid |
 356         sort >observed &&
 357
 358         test_cmp expected observed
 359 '
 360
 361 test_expect_success 'verify sparse:oid=OID' '
 362         git -C r4 ls-files -s dir1/sparse1 dir1/sparse2 |
 363         test_parse_ls_files_stage_oids |
 364         sort >expected &&
 365
 366         git -C r4 ls-files -s pattern >staged &&
 367         oid=$(test_parse_ls_files_stage_oids <staged) &&
 368         git -C r4 pack-objects --revs --stdout --filter=sparse:oid=$oid >filter.pack <<-EOF &&
 369         HEAD
 370         EOF
 371         git -C r4 index-pack ../filter.pack &&
 372
 373         git -C r4 verify-pack -v ../filter.pack >verify_result &&
 374         grep blob verify_result |
 375         parse_verify_pack_blob_oid |
 376         sort >observed &&
 377
 378         test_cmp expected observed
 379 '
 380
 381 test_expect_success 'verify sparse:oid=oid-ish' '
 382         git -C r4 ls-files -s dir1/sparse1 dir1/sparse2 |
 383         test_parse_ls_files_stage_oids |
 384         sort >expected &&
 385
 386         git -C r4 pack-objects --revs --stdout --filter=sparse:oid=main:pattern >filter.pack <<-EOF &&
 387         HEAD
 388         EOF
 389         git -C r4 index-pack ../filter.pack &&
 390
 391         git -C r4 verify-pack -v ../filter.pack >verify_result &&
 392         grep blob verify_result |
 393         parse_verify_pack_blob_oid |
 394         sort >observed &&
 395
 396         test_cmp expected observed
 397 '
 398
 399 # Delete some loose objects and use pack-objects, but WITHOUT any filtering.
 400 # This models previously omitted objects that we did not receive.
 401
 402 test_expect_success 'setup r1 - delete loose blobs' '
 403         git -C r1 ls-files -s file.1 file.2 file.3 file.4 file.5 |
 404         test_parse_ls_files_stage_oids |
 405         sort >expected &&
 406
 407         for id in `cat expected | sed "s|..|&/|"`
 408         do
 409                 rm r1/.git/objects/$id || return 1
 410         done
 411 '
 412
 413 test_expect_success 'verify pack-objects fails w/ missing objects' '
 414         test_must_fail git -C r1 pack-objects --revs --stdout >miss.pack <<-EOF
 415         HEAD
 416         EOF
 417 '
 418
 419 test_expect_success 'verify pack-objects fails w/ --missing=error' '
 420         test_must_fail git -C r1 pack-objects --revs --stdout --missing=error >miss.pack <<-EOF
 421         HEAD
 422         EOF
 423 '
 424
 425 test_expect_success 'verify pack-objects w/ --missing=allow-any' '
 426         git -C r1 pack-objects --revs --stdout --missing=allow-any >miss.pack <<-EOF
 427         HEAD
 428         EOF
 429 '
 430
 431 test_done