t/t5317-pack-objects-filter-objects.sh

   1 #!/bin/sh
   2
   3 test_description='git pack-objects using object filtering'
   4
   5 GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
   6 export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
   7
   8 . ./test-lib.sh
   9
  10 # Test blob:none filter.
  11
  12 test_expect_success 'setup r1' '
  13         echo "{print \$1}" >print_1.awk &&
  14         echo "{print \$2}" >print_2.awk &&
  15
  16         git init r1 &&
  17         for n in 1 2 3 4 5
  18         do
  19                 echo "This is file: $n" > r1/file.$n
  20                 git -C r1 add file.$n
  21                 git -C r1 commit -m "$n"
  22         done
  23 '
  24
  25 test_expect_success 'verify blob count in normal packfile' '
  26         git -C r1 ls-files -s file.1 file.2 file.3 file.4 file.5 \
  27                 >ls_files_result &&
  28         awk -f print_2.awk ls_files_result |
  29         sort >expected &&
  30
  31         git -C r1 pack-objects --revs --stdout >all.pack <<-EOF &&
  32         HEAD
  33         EOF
  34         git -C r1 index-pack ../all.pack &&
  35
  36         git -C r1 verify-pack -v ../all.pack >verify_result &&
  37         grep blob verify_result |
  38         awk -f print_1.awk |
  39         sort >observed &&
  40
  41         test_cmp expected observed
  42 '
  43
  44 test_expect_success 'verify blob:none packfile has no blobs' '
  45         git -C r1 pack-objects --revs --stdout --filter=blob:none >filter.pack <<-EOF &&
  46         HEAD
  47         EOF
  48         git -C r1 index-pack ../filter.pack &&
  49
  50         git -C r1 verify-pack -v ../filter.pack >verify_result &&
  51         ! grep blob verify_result
  52 '
  53
  54 test_expect_success 'verify normal and blob:none packfiles have same commits/trees' '
  55         git -C r1 verify-pack -v ../all.pack >verify_result &&
  56         grep -E "commit|tree" verify_result |
  57         awk -f print_1.awk |
  58         sort >expected &&
  59
  60         git -C r1 verify-pack -v ../filter.pack >verify_result &&
  61         grep -E "commit|tree" verify_result |
  62         awk -f print_1.awk |
  63         sort >observed &&
  64
  65         test_cmp expected observed
  66 '
  67
  68 test_expect_success 'get an error for missing tree object' '
  69         git init r5 &&
  70         echo foo >r5/foo &&
  71         git -C r5 add foo &&
  72         git -C r5 commit -m "foo" &&
  73         git -C r5 rev-parse HEAD^{tree} >tree &&
  74         del=$(sed "s|..|&/|" tree) &&
  75         rm r5/.git/objects/$del &&
  76         test_must_fail git -C r5 pack-objects --revs --stdout 2>bad_tree <<-EOF &&
  77         HEAD
  78         EOF
  79         grep "bad tree object" bad_tree
  80 '
  81
  82 test_expect_success 'setup for tests of tree:0' '
  83         mkdir r1/subtree &&
  84         echo "This is a file in a subtree" >r1/subtree/file &&
  85         git -C r1 add subtree/file &&
  86         git -C r1 commit -m subtree
  87 '
  88
  89 test_expect_success 'verify tree:0 packfile has no blobs or trees' '
  90         git -C r1 pack-objects --revs --stdout --filter=tree:0 >commitsonly.pack <<-EOF &&
  91         HEAD
  92         EOF
  93         git -C r1 index-pack ../commitsonly.pack &&
  94         git -C r1 verify-pack -v ../commitsonly.pack >objs &&
  95         ! grep -E "tree|blob" objs
  96 '
  97
  98 test_expect_success 'grab tree directly when using tree:0' '
  99         # We should get the tree specified directly but not its blobs or subtrees.
 100         git -C r1 pack-objects --revs --stdout --filter=tree:0 >commitsonly.pack <<-EOF &&
 101         HEAD:
 102         EOF
 103         git -C r1 index-pack ../commitsonly.pack &&
 104         git -C r1 verify-pack -v ../commitsonly.pack >objs &&
 105         awk "/tree|blob/{print \$1}" objs >trees_and_blobs &&
 106         git -C r1 rev-parse HEAD: >expected &&
 107         test_cmp expected trees_and_blobs
 108 '
 109
 110 # Test blob:limit=<n>[kmg] filter.
 111 # We boundary test around the size parameter.  The filter is strictly less than
 112 # the value, so size 500 and 1000 should have the same results, but 1001 should
 113 # filter more.
 114
 115 test_expect_success 'setup r2' '
 116         git init r2 &&
 117         for n in 1000 10000
 118         do
 119                 printf "%"$n"s" X > r2/large.$n
 120                 git -C r2 add large.$n
 121                 git -C r2 commit -m "$n"
 122         done
 123 '
 124
 125 test_expect_success 'verify blob count in normal packfile' '
 126         git -C r2 ls-files -s large.1000 large.10000 >ls_files_result &&
 127         awk -f print_2.awk ls_files_result |
 128         sort >expected &&
 129
 130         git -C r2 pack-objects --revs --stdout >all.pack <<-EOF &&
 131         HEAD
 132         EOF
 133         git -C r2 index-pack ../all.pack &&
 134
 135         git -C r2 verify-pack -v ../all.pack >verify_result &&
 136         grep blob verify_result |
 137         awk -f print_1.awk |
 138         sort >observed &&
 139
 140         test_cmp expected observed
 141 '
 142
 143 test_expect_success 'verify blob:limit=500 omits all blobs' '
 144         git -C r2 pack-objects --revs --stdout --filter=blob:limit=500 >filter.pack <<-EOF &&
 145         HEAD
 146         EOF
 147         git -C r2 index-pack ../filter.pack &&
 148
 149         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 150         ! grep blob verify_result
 151 '
 152
 153 test_expect_success 'verify blob:limit=1000' '
 154         git -C r2 pack-objects --revs --stdout --filter=blob:limit=1000 >filter.pack <<-EOF &&
 155         HEAD
 156         EOF
 157         git -C r2 index-pack ../filter.pack &&
 158
 159         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 160         ! grep blob verify_result
 161 '
 162
 163 test_expect_success 'verify blob:limit=1001' '
 164         git -C r2 ls-files -s large.1000 >ls_files_result &&
 165         awk -f print_2.awk ls_files_result |
 166         sort >expected &&
 167
 168         git -C r2 pack-objects --revs --stdout --filter=blob:limit=1001 >filter.pack <<-EOF &&
 169         HEAD
 170         EOF
 171         git -C r2 index-pack ../filter.pack &&
 172
 173         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 174         grep blob verify_result |
 175         awk -f print_1.awk |
 176         sort >observed &&
 177
 178         test_cmp expected observed
 179 '
 180
 181 test_expect_success 'verify blob:limit=10001' '
 182         git -C r2 ls-files -s large.1000 large.10000 >ls_files_result &&
 183         awk -f print_2.awk ls_files_result |
 184         sort >expected &&
 185
 186         git -C r2 pack-objects --revs --stdout --filter=blob:limit=10001 >filter.pack <<-EOF &&
 187         HEAD
 188         EOF
 189         git -C r2 index-pack ../filter.pack &&
 190
 191         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 192         grep blob verify_result |
 193         awk -f print_1.awk |
 194         sort >observed &&
 195
 196         test_cmp expected observed
 197 '
 198
 199 test_expect_success 'verify blob:limit=1k' '
 200         git -C r2 ls-files -s large.1000 >ls_files_result &&
 201         awk -f print_2.awk ls_files_result |
 202         sort >expected &&
 203
 204         git -C r2 pack-objects --revs --stdout --filter=blob:limit=1k >filter.pack <<-EOF &&
 205         HEAD
 206         EOF
 207         git -C r2 index-pack ../filter.pack &&
 208
 209         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 210         grep blob verify_result |
 211         awk -f print_1.awk |
 212         sort >observed &&
 213
 214         test_cmp expected observed
 215 '
 216
 217 test_expect_success 'verify explicitly specifying oversized blob in input' '
 218         git -C r2 ls-files -s large.1000 large.10000 >ls_files_result &&
 219         awk -f print_2.awk ls_files_result |
 220         sort >expected &&
 221
 222         echo HEAD >objects &&
 223         git -C r2 rev-parse HEAD:large.10000 >>objects &&
 224         git -C r2 pack-objects --revs --stdout --filter=blob:limit=1k <objects >filter.pack &&
 225         git -C r2 index-pack ../filter.pack &&
 226
 227         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 228         grep blob verify_result |
 229         awk -f print_1.awk |
 230         sort >observed &&
 231
 232         test_cmp expected observed
 233 '
 234
 235 test_expect_success 'verify blob:limit=1m' '
 236         git -C r2 ls-files -s large.1000 large.10000 >ls_files_result &&
 237         awk -f print_2.awk ls_files_result |
 238         sort >expected &&
 239
 240         git -C r2 pack-objects --revs --stdout --filter=blob:limit=1m >filter.pack <<-EOF &&
 241         HEAD
 242         EOF
 243         git -C r2 index-pack ../filter.pack &&
 244
 245         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 246         grep blob verify_result |
 247         awk -f print_1.awk |
 248         sort >observed &&
 249
 250         test_cmp expected observed
 251 '
 252
 253 test_expect_success 'verify normal and blob:limit packfiles have same commits/trees' '
 254         git -C r2 verify-pack -v ../all.pack >verify_result &&
 255         grep -E "commit|tree" verify_result |
 256         awk -f print_1.awk |
 257         sort >expected &&
 258
 259         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 260         grep -E "commit|tree" verify_result |
 261         awk -f print_1.awk |
 262         sort >observed &&
 263
 264         test_cmp expected observed
 265 '
 266
 267 # Test sparse:path=<path> filter.
 268 # !!!!
 269 # NOTE: sparse:path filter support has been dropped for security reasons,
 270 # so the tests have been changed to make sure that using it fails.
 271 # !!!!
 272 # Use a local file containing a sparse-checkout specification to filter
 273 # out blobs not required for the corresponding sparse-checkout.  We do not
 274 # require sparse-checkout to actually be enabled.
 275
 276 test_expect_success 'setup r3' '
 277         git init r3 &&
 278         mkdir r3/dir1 &&
 279         for n in sparse1 sparse2
 280         do
 281                 echo "This is file: $n" > r3/$n
 282                 git -C r3 add $n
 283                 echo "This is file: dir1/$n" > r3/dir1/$n
 284                 git -C r3 add dir1/$n
 285         done &&
 286         git -C r3 commit -m "sparse" &&
 287         echo dir1/ >pattern1 &&
 288         echo sparse1 >pattern2
 289 '
 290
 291 test_expect_success 'verify blob count in normal packfile' '
 292         git -C r3 ls-files -s sparse1 sparse2 dir1/sparse1 dir1/sparse2 \
 293                 >ls_files_result &&
 294         awk -f print_2.awk ls_files_result |
 295         sort >expected &&
 296
 297         git -C r3 pack-objects --revs --stdout >all.pack <<-EOF &&
 298         HEAD
 299         EOF
 300         git -C r3 index-pack ../all.pack &&
 301
 302         git -C r3 verify-pack -v ../all.pack >verify_result &&
 303         grep blob verify_result |
 304         awk -f print_1.awk |
 305         sort >observed &&
 306
 307         test_cmp expected observed
 308 '
 309
 310 test_expect_success 'verify sparse:path=pattern1 fails' '
 311         test_must_fail git -C r3 pack-objects --revs --stdout \
 312                 --filter=sparse:path=../pattern1 <<-EOF
 313         HEAD
 314         EOF
 315 '
 316
 317 test_expect_success 'verify sparse:path=pattern2 fails' '
 318         test_must_fail git -C r3 pack-objects --revs --stdout \
 319                 --filter=sparse:path=../pattern2 <<-EOF
 320         HEAD
 321         EOF
 322 '
 323
 324 # Test sparse:oid=<oid-ish> filter.
 325 # Use a blob containing a sparse-checkout specification to filter
 326 # out blobs not required for the corresponding sparse-checkout.  We do not
 327 # require sparse-checkout to actually be enabled.
 328
 329 test_expect_success 'setup r4' '
 330         git init r4 &&
 331         mkdir r4/dir1 &&
 332         for n in sparse1 sparse2
 333         do
 334                 echo "This is file: $n" > r4/$n
 335                 git -C r4 add $n
 336                 echo "This is file: dir1/$n" > r4/dir1/$n
 337                 git -C r4 add dir1/$n
 338         done &&
 339         echo dir1/ >r4/pattern &&
 340         git -C r4 add pattern &&
 341         git -C r4 commit -m "pattern"
 342 '
 343
 344 test_expect_success 'verify blob count in normal packfile' '
 345         git -C r4 ls-files -s pattern sparse1 sparse2 dir1/sparse1 dir1/sparse2 \
 346                 >ls_files_result &&
 347         awk -f print_2.awk ls_files_result |
 348         sort >expected &&
 349
 350         git -C r4 pack-objects --revs --stdout >all.pack <<-EOF &&
 351         HEAD
 352         EOF
 353         git -C r4 index-pack ../all.pack &&
 354
 355         git -C r4 verify-pack -v ../all.pack >verify_result &&
 356         grep blob verify_result |
 357         awk -f print_1.awk |
 358         sort >observed &&
 359
 360         test_cmp expected observed
 361 '
 362
 363 test_expect_success 'verify sparse:oid=OID' '
 364         git -C r4 ls-files -s dir1/sparse1 dir1/sparse2 >ls_files_result &&
 365         awk -f print_2.awk ls_files_result |
 366         sort >expected &&
 367
 368         git -C r4 ls-files -s pattern >staged &&
 369         oid=$(awk -f print_2.awk staged) &&
 370         git -C r4 pack-objects --revs --stdout --filter=sparse:oid=$oid >filter.pack <<-EOF &&
 371         HEAD
 372         EOF
 373         git -C r4 index-pack ../filter.pack &&
 374
 375         git -C r4 verify-pack -v ../filter.pack >verify_result &&
 376         grep blob verify_result |
 377         awk -f print_1.awk |
 378         sort >observed &&
 379
 380         test_cmp expected observed
 381 '
 382
 383 test_expect_success 'verify sparse:oid=oid-ish' '
 384         git -C r4 ls-files -s dir1/sparse1 dir1/sparse2 >ls_files_result &&
 385         awk -f print_2.awk ls_files_result |
 386         sort >expected &&
 387
 388         git -C r4 pack-objects --revs --stdout --filter=sparse:oid=main:pattern >filter.pack <<-EOF &&
 389         HEAD
 390         EOF
 391         git -C r4 index-pack ../filter.pack &&
 392
 393         git -C r4 verify-pack -v ../filter.pack >verify_result &&
 394         grep blob verify_result |
 395         awk -f print_1.awk |
 396         sort >observed &&
 397
 398         test_cmp expected observed
 399 '
 400
 401 # Delete some loose objects and use pack-objects, but WITHOUT any filtering.
 402 # This models previously omitted objects that we did not receive.
 403
 404 test_expect_success 'setup r1 - delete loose blobs' '
 405         git -C r1 ls-files -s file.1 file.2 file.3 file.4 file.5 \
 406                 >ls_files_result &&
 407         awk -f print_2.awk ls_files_result |
 408         sort >expected &&
 409
 410         for id in `cat expected | sed "s|..|&/|"`
 411         do
 412                 rm r1/.git/objects/$id
 413         done
 414 '
 415
 416 test_expect_success 'verify pack-objects fails w/ missing objects' '
 417         test_must_fail git -C r1 pack-objects --revs --stdout >miss.pack <<-EOF
 418         HEAD
 419         EOF
 420 '
 421
 422 test_expect_success 'verify pack-objects fails w/ --missing=error' '
 423         test_must_fail git -C r1 pack-objects --revs --stdout --missing=error >miss.pack <<-EOF
 424         HEAD
 425         EOF
 426 '
 427
 428 test_expect_success 'verify pack-objects w/ --missing=allow-any' '
 429         git -C r1 pack-objects --revs --stdout --missing=allow-any >miss.pack <<-EOF
 430         HEAD
 431         EOF
 432 '
 433
 434 test_done