t/helper: teach test-regex to report pattern errors (like REG_ILLSEQ)
[git/debian.git] / t / t6112-rev-list-filters-objects.sh
blobde0e5a5d3646cdd7e60b98b566bb075200f311ad
1 #!/bin/sh
3 test_description='git rev-list using object filtering'
5 . ./test-lib.sh
7 # Test the blob:none filter.
9 test_expect_success 'setup r1' '
10 echo "{print \$1}" >print_1.awk &&
11 echo "{print \$2}" >print_2.awk &&
13 git init r1 &&
14 for n in 1 2 3 4 5
16 echo "This is file: $n" > r1/file.$n
17 git -C r1 add file.$n
18 git -C r1 commit -m "$n"
19 done
22 test_expect_success 'verify blob:none omits all 5 blobs' '
23 git -C r1 ls-files -s file.1 file.2 file.3 file.4 file.5 \
24 >ls_files_result &&
25 awk -f print_2.awk ls_files_result |
26 sort >expected &&
28 git -C r1 rev-list --quiet --objects --filter-print-omitted \
29 --filter=blob:none HEAD >revs &&
30 awk -f print_1.awk revs |
31 sed "s/~//" |
32 sort >observed &&
34 test_cmp expected observed
37 test_expect_success 'specify blob explicitly prevents filtering' '
38 file_3=$(git -C r1 ls-files -s file.3 |
39 awk -f print_2.awk) &&
41 file_4=$(git -C r1 ls-files -s file.4 |
42 awk -f print_2.awk) &&
44 git -C r1 rev-list --objects --filter=blob:none HEAD $file_3 >observed &&
45 grep "$file_3" observed &&
46 ! grep "$file_4" observed
49 test_expect_success 'verify emitted+omitted == all' '
50 git -C r1 rev-list --objects HEAD >revs &&
51 awk -f print_1.awk revs |
52 sort >expected &&
54 git -C r1 rev-list --objects --filter-print-omitted --filter=blob:none \
55 HEAD >revs &&
56 awk -f print_1.awk revs |
57 sed "s/~//" |
58 sort >observed &&
60 test_cmp expected observed
64 # Test blob:limit=<n>[kmg] filter.
65 # We boundary test around the size parameter. The filter is strictly less than
66 # the value, so size 500 and 1000 should have the same results, but 1001 should
67 # filter more.
69 test_expect_success 'setup r2' '
70 git init r2 &&
71 for n in 1000 10000
73 printf "%"$n"s" X > r2/large.$n
74 git -C r2 add large.$n
75 git -C r2 commit -m "$n"
76 done
79 test_expect_success 'verify blob:limit=500 omits all blobs' '
80 git -C r2 ls-files -s large.1000 large.10000 >ls_files_result &&
81 awk -f print_2.awk ls_files_result |
82 sort >expected &&
84 git -C r2 rev-list --quiet --objects --filter-print-omitted \
85 --filter=blob:limit=500 HEAD >revs &&
86 awk -f print_1.awk revs |
87 sed "s/~//" |
88 sort >observed &&
90 test_cmp expected observed
93 test_expect_success 'verify emitted+omitted == all' '
94 git -C r2 rev-list --objects HEAD >revs &&
95 awk -f print_1.awk revs |
96 sort >expected &&
98 git -C r2 rev-list --objects --filter-print-omitted \
99 --filter=blob:limit=500 HEAD >revs &&
100 awk -f print_1.awk revs |
101 sed "s/~//" |
102 sort >observed &&
104 test_cmp expected observed
107 test_expect_success 'verify blob:limit=1000' '
108 git -C r2 ls-files -s large.1000 large.10000 >ls_files_result &&
109 awk -f print_2.awk ls_files_result |
110 sort >expected &&
112 git -C r2 rev-list --quiet --objects --filter-print-omitted \
113 --filter=blob:limit=1000 HEAD >revs &&
114 awk -f print_1.awk revs |
115 sed "s/~//" |
116 sort >observed &&
118 test_cmp expected observed
121 test_expect_success 'verify blob:limit=1001' '
122 git -C r2 ls-files -s large.10000 >ls_files_result &&
123 awk -f print_2.awk ls_files_result |
124 sort >expected &&
126 git -C r2 rev-list --quiet --objects --filter-print-omitted \
127 --filter=blob:limit=1001 HEAD >revs &&
128 awk -f print_1.awk revs |
129 sed "s/~//" |
130 sort >observed &&
132 test_cmp expected observed
135 test_expect_success 'verify blob:limit=1k' '
136 git -C r2 ls-files -s large.10000 >ls_files_result &&
137 awk -f print_2.awk ls_files_result |
138 sort >expected &&
140 git -C r2 rev-list --quiet --objects --filter-print-omitted \
141 --filter=blob:limit=1k HEAD >revs &&
142 awk -f print_1.awk revs |
143 sed "s/~//" |
144 sort >observed &&
146 test_cmp expected observed
149 test_expect_success 'verify blob:limit=1m' '
150 git -C r2 rev-list --quiet --objects --filter-print-omitted \
151 --filter=blob:limit=1m HEAD >revs &&
152 awk -f print_1.awk revs |
153 sed "s/~//" |
154 sort >observed &&
156 test_must_be_empty observed
159 # Test sparse:path=<path> filter.
160 # !!!!
161 # NOTE: sparse:path filter support has been dropped for security reasons,
162 # so the tests have been changed to make sure that using it fails.
163 # !!!!
164 # Use a local file containing a sparse-checkout specification to filter
165 # out blobs not required for the corresponding sparse-checkout. We do not
166 # require sparse-checkout to actually be enabled.
168 test_expect_success 'setup r3' '
169 git init r3 &&
170 mkdir r3/dir1 &&
171 for n in sparse1 sparse2
173 echo "This is file: $n" > r3/$n
174 git -C r3 add $n
175 echo "This is file: dir1/$n" > r3/dir1/$n
176 git -C r3 add dir1/$n
177 done &&
178 git -C r3 commit -m "sparse" &&
179 echo dir1/ >pattern1 &&
180 echo sparse1 >pattern2
183 test_expect_success 'verify sparse:path=pattern1 fails' '
184 test_must_fail git -C r3 rev-list --quiet --objects \
185 --filter-print-omitted --filter=sparse:path=../pattern1 HEAD
188 test_expect_success 'verify sparse:path=pattern2 fails' '
189 test_must_fail git -C r3 rev-list --quiet --objects \
190 --filter-print-omitted --filter=sparse:path=../pattern2 HEAD
193 # Test sparse:oid=<oid-ish> filter.
194 # Use a blob containing a sparse-checkout specification to filter
195 # out blobs not required for the corresponding sparse-checkout. We do not
196 # require sparse-checkout to actually be enabled.
198 test_expect_success 'setup r3 part 2' '
199 echo dir1/ >r3/pattern &&
200 git -C r3 add pattern &&
201 git -C r3 commit -m "pattern"
204 test_expect_success 'verify sparse:oid=OID omits top-level files' '
205 git -C r3 ls-files -s pattern sparse1 sparse2 >ls_files_result &&
206 awk -f print_2.awk ls_files_result |
207 sort >expected &&
209 oid=$(git -C r3 ls-files -s pattern | awk -f print_2.awk) &&
211 git -C r3 rev-list --quiet --objects --filter-print-omitted \
212 --filter=sparse:oid=$oid HEAD >revs &&
213 awk -f print_1.awk revs |
214 sed "s/~//" |
215 sort >observed &&
217 test_cmp expected observed
220 test_expect_success 'verify sparse:oid=oid-ish omits top-level files' '
221 git -C r3 ls-files -s pattern sparse1 sparse2 >ls_files_result &&
222 awk -f print_2.awk ls_files_result |
223 sort >expected &&
225 git -C r3 rev-list --quiet --objects --filter-print-omitted \
226 --filter=sparse:oid=master:pattern HEAD >revs &&
227 awk -f print_1.awk revs |
228 sed "s/~//" |
229 sort >observed &&
231 test_cmp expected observed
234 test_expect_success 'rev-list W/ --missing=print and --missing=allow-any for trees' '
235 TREE=$(git -C r3 rev-parse HEAD:dir1) &&
237 # Create a spare repo because we will be deleting objects from this one.
238 git clone r3 r3.b &&
240 rm r3.b/.git/objects/$(echo $TREE | sed "s|^..|&/|") &&
242 git -C r3.b rev-list --quiet --missing=print --objects HEAD \
243 >missing_objs 2>rev_list_err &&
244 echo "?$TREE" >expected &&
245 test_cmp expected missing_objs &&
247 # do not complain when a missing tree cannot be parsed
248 test_must_be_empty rev_list_err &&
250 git -C r3.b rev-list --missing=allow-any --objects HEAD \
251 >objs 2>rev_list_err &&
252 ! grep $TREE objs &&
253 test_must_be_empty rev_list_err
256 # Test tree:0 filter.
258 test_expect_success 'verify tree:0 includes trees in "filtered" output' '
259 git -C r3 rev-list --quiet --objects --filter-print-omitted \
260 --filter=tree:0 HEAD >revs &&
262 awk -f print_1.awk revs |
263 sed s/~// |
264 xargs -n1 git -C r3 cat-file -t >unsorted_filtered_types &&
266 sort -u unsorted_filtered_types >filtered_types &&
267 test_write_lines blob tree >expected &&
268 test_cmp expected filtered_types
271 # Make sure tree:0 does not iterate through any trees.
273 test_expect_success 'verify skipping tree iteration when not collecting omits' '
274 GIT_TRACE=1 git -C r3 rev-list \
275 --objects --filter=tree:0 HEAD 2>filter_trace &&
276 grep "Skipping contents of tree [.][.][.]" filter_trace >actual &&
277 # One line for each commit traversed.
278 test_line_count = 2 actual &&
280 # Make sure no other trees were considered besides the root.
281 ! grep "Skipping contents of tree [^.]" filter_trace &&
283 # Try this again with "combine:". If both sub-filters are skipping
284 # trees, the composite filter should also skip trees. This is not
285 # important unless the user does combine:tree:X+tree:Y or another filter
286 # besides "tree:" is implemented in the future which can skip trees.
287 GIT_TRACE=1 git -C r3 rev-list \
288 --objects --filter=combine:tree:1+tree:3 HEAD 2>filter_trace &&
290 # Only skip the dir1/ tree, which is shared between the two commits.
291 grep "Skipping contents of tree " filter_trace >actual &&
292 test_write_lines "Skipping contents of tree dir1/..." >expected &&
293 test_cmp expected actual
296 # Test tree:# filters.
298 expect_has () {
299 commit=$1 &&
300 name=$2 &&
302 hash=$(git -C r3 rev-parse $commit:$name) &&
303 grep "^$hash $name$" actual
306 test_expect_success 'verify tree:1 includes root trees' '
307 git -C r3 rev-list --objects --filter=tree:1 HEAD >actual &&
309 # We should get two root directories and two commits.
310 expect_has HEAD "" &&
311 expect_has HEAD~1 "" &&
312 test_line_count = 4 actual
315 test_expect_success 'verify tree:2 includes root trees and immediate children' '
316 git -C r3 rev-list --objects --filter=tree:2 HEAD >actual &&
318 expect_has HEAD "" &&
319 expect_has HEAD~1 "" &&
320 expect_has HEAD dir1 &&
321 expect_has HEAD pattern &&
322 expect_has HEAD sparse1 &&
323 expect_has HEAD sparse2 &&
325 # There are also 2 commit objects
326 test_line_count = 8 actual
329 test_expect_success 'verify tree:3 includes everything expected' '
330 git -C r3 rev-list --objects --filter=tree:3 HEAD >actual &&
332 expect_has HEAD "" &&
333 expect_has HEAD~1 "" &&
334 expect_has HEAD dir1 &&
335 expect_has HEAD dir1/sparse1 &&
336 expect_has HEAD dir1/sparse2 &&
337 expect_has HEAD pattern &&
338 expect_has HEAD sparse1 &&
339 expect_has HEAD sparse2 &&
341 # There are also 2 commit objects
342 test_line_count = 10 actual
345 test_expect_success 'combine:... for a simple combination' '
346 git -C r3 rev-list --objects --filter=combine:tree:2+blob:none HEAD \
347 >actual &&
349 expect_has HEAD "" &&
350 expect_has HEAD~1 "" &&
351 expect_has HEAD dir1 &&
353 # There are also 2 commit objects
354 test_line_count = 5 actual &&
356 cp actual expected &&
358 # Try again using repeated --filter - this is equivalent to a manual
359 # combine with "combine:...+..."
360 git -C r3 rev-list --objects --filter=combine:tree:2 \
361 --filter=blob:none HEAD >actual &&
363 test_cmp expected actual
366 test_expect_success 'combine:... with URL encoding' '
367 git -C r3 rev-list --objects \
368 --filter=combine:tree%3a2+blob:%6Eon%65 HEAD >actual &&
370 expect_has HEAD "" &&
371 expect_has HEAD~1 "" &&
372 expect_has HEAD dir1 &&
374 # There are also 2 commit objects
375 test_line_count = 5 actual
378 expect_invalid_filter_spec () {
379 spec="$1" &&
380 err="$2" &&
382 test_must_fail git -C r3 rev-list --objects --filter="$spec" HEAD \
383 >actual 2>actual_stderr &&
384 test_must_be_empty actual &&
385 test_i18ngrep "$err" actual_stderr
388 test_expect_success 'combine:... while URL-encoding things that should not be' '
389 expect_invalid_filter_spec combine%3Atree:2+blob:none \
390 "invalid filter-spec"
393 test_expect_success 'combine: with nothing after the :' '
394 expect_invalid_filter_spec combine: "expected something after combine:"
397 test_expect_success 'parse error in first sub-filter in combine:' '
398 expect_invalid_filter_spec combine:tree:asdf+blob:none \
399 "expected .tree:<depth>."
402 test_expect_success 'combine:... with non-encoded reserved chars' '
403 expect_invalid_filter_spec combine:tree:2+sparse:@xyz \
404 "must escape char in sub-filter-spec: .@." &&
405 expect_invalid_filter_spec combine:tree:2+sparse:\` \
406 "must escape char in sub-filter-spec: .\`." &&
407 expect_invalid_filter_spec combine:tree:2+sparse:~abc \
408 "must escape char in sub-filter-spec: .\~."
411 test_expect_success 'validate err msg for "combine:<valid-filter>+"' '
412 expect_invalid_filter_spec combine:tree:2+ "expected .tree:<depth>."
415 test_expect_success 'combine:... with edge-case hex digits: Ff Aa 0 9' '
416 git -C r3 rev-list --objects --filter="combine:tree:2+bl%6Fb:n%6fne" \
417 HEAD >actual &&
418 test_line_count = 5 actual &&
419 git -C r3 rev-list --objects --filter="combine:tree%3A2+blob%3anone" \
420 HEAD >actual &&
421 test_line_count = 5 actual &&
422 git -C r3 rev-list --objects --filter="combine:tree:%30" HEAD >actual &&
423 test_line_count = 2 actual &&
424 git -C r3 rev-list --objects --filter="combine:tree:%39+blob:none" \
425 HEAD >actual &&
426 test_line_count = 5 actual
429 test_expect_success 'add sparse pattern blobs whose paths have reserved chars' '
430 cp r3/pattern r3/pattern1+renamed% &&
431 cp r3/pattern "r3/p;at%ter+n" &&
432 cp r3/pattern r3/^~pattern &&
433 git -C r3 add pattern1+renamed% "p;at%ter+n" ^~pattern &&
434 git -C r3 commit -m "add sparse pattern files with reserved chars"
437 test_expect_success 'combine:... with more than two sub-filters' '
438 git -C r3 rev-list --objects \
439 --filter=combine:tree:3+blob:limit=40+sparse:oid=master:pattern \
440 HEAD >actual &&
442 expect_has HEAD "" &&
443 expect_has HEAD~1 "" &&
444 expect_has HEAD~2 "" &&
445 expect_has HEAD dir1 &&
446 expect_has HEAD dir1/sparse1 &&
447 expect_has HEAD dir1/sparse2 &&
449 # Should also have 3 commits
450 test_line_count = 9 actual &&
452 # Try again, this time making sure the last sub-filter is only
453 # URL-decoded once.
454 cp actual expect &&
456 git -C r3 rev-list --objects \
457 --filter=combine:tree:3+blob:limit=40+sparse:oid=master:pattern1%2brenamed%25 \
458 HEAD >actual &&
459 test_cmp expect actual &&
461 # Use the same composite filter again, but with a pattern file name that
462 # requires encoding multiple characters, and use implicit filter
463 # combining.
464 test_when_finished "rm -f trace1" &&
465 GIT_TRACE=$(pwd)/trace1 git -C r3 rev-list --objects \
466 --filter=tree:3 --filter=blob:limit=40 \
467 --filter=sparse:oid="master:p;at%ter+n" \
468 HEAD >actual &&
470 test_cmp expect actual &&
471 grep "Add to combine filter-spec: sparse:oid=master:p%3bat%25ter%2bn" \
472 trace1 &&
474 # Repeat the above test, but this time, the characters to encode are in
475 # the LHS of the combined filter.
476 test_when_finished "rm -f trace2" &&
477 GIT_TRACE=$(pwd)/trace2 git -C r3 rev-list --objects \
478 --filter=sparse:oid=master:^~pattern \
479 --filter=tree:3 --filter=blob:limit=40 \
480 HEAD >actual &&
482 test_cmp expect actual &&
483 grep "Add to combine filter-spec: sparse:oid=master:%5e%7epattern" \
484 trace2
487 # Test provisional omit collection logic with a repo that has objects appearing
488 # at multiple depths - first deeper than the filter's threshold, then shallow.
490 test_expect_success 'setup r4' '
491 git init r4 &&
493 echo foo > r4/foo &&
494 mkdir r4/subdir &&
495 echo bar > r4/subdir/bar &&
497 mkdir r4/filt &&
498 cp -r r4/foo r4/subdir r4/filt &&
500 git -C r4 add foo subdir filt &&
501 git -C r4 commit -m "commit msg"
504 expect_has_with_different_name () {
505 repo=$1 &&
506 name=$2 &&
508 hash=$(git -C $repo rev-parse HEAD:$name) &&
509 ! grep "^$hash $name$" actual &&
510 grep "^$hash " actual &&
511 ! grep "~$hash" actual
514 test_expect_success 'test tree:# filter provisional omit for blob and tree' '
515 git -C r4 rev-list --objects --filter-print-omitted --filter=tree:2 \
516 HEAD >actual &&
517 expect_has_with_different_name r4 filt/foo &&
518 expect_has_with_different_name r4 filt/subdir
521 test_expect_success 'verify skipping tree iteration when collecting omits' '
522 GIT_TRACE=1 git -C r4 rev-list --filter-print-omitted \
523 --objects --filter=tree:0 HEAD 2>filter_trace &&
524 grep "^Skipping contents of tree " filter_trace >actual &&
526 echo "Skipping contents of tree subdir/..." >expect &&
527 test_cmp expect actual
530 test_expect_success 'setup r5' '
531 git init r5 &&
532 mkdir -p r5/subdir &&
534 echo 1 >r5/short-root &&
535 echo 12345 >r5/long-root &&
536 echo a >r5/subdir/short-subdir &&
537 echo abcde >r5/subdir/long-subdir &&
539 git -C r5 add short-root long-root subdir &&
540 git -C r5 commit -m "commit msg"
543 test_expect_success 'verify collecting omits in combined: filter' '
544 # Note that this test guards against the naive implementation of simply
545 # giving both filters the same "omits" set and expecting it to
546 # automatically merge them.
547 git -C r5 rev-list --objects --quiet --filter-print-omitted \
548 --filter=combine:tree:2+blob:limit=3 HEAD >actual &&
550 # Expect 0 trees/commits, 3 blobs omitted (all blobs except short-root)
551 omitted_1=$(echo 12345 | git hash-object --stdin) &&
552 omitted_2=$(echo a | git hash-object --stdin) &&
553 omitted_3=$(echo abcde | git hash-object --stdin) &&
555 grep ~$omitted_1 actual &&
556 grep ~$omitted_2 actual &&
557 grep ~$omitted_3 actual &&
558 test_line_count = 3 actual
561 # Test tree:<depth> where a tree is iterated to twice - once where a subentry is
562 # too deep to be included, and again where the blob inside it is shallow enough
563 # to be included. This makes sure we don't use LOFR_MARK_SEEN incorrectly (we
564 # can't use it because a tree can be iterated over again at a lower depth).
566 test_expect_success 'tree:<depth> where we iterate over tree at two levels' '
567 git init r5 &&
569 mkdir -p r5/a/subdir/b &&
570 echo foo > r5/a/subdir/b/foo &&
572 mkdir -p r5/subdir/b &&
573 echo foo > r5/subdir/b/foo &&
575 git -C r5 add a subdir &&
576 git -C r5 commit -m "commit msg" &&
578 git -C r5 rev-list --objects --filter=tree:4 HEAD >actual &&
579 expect_has_with_different_name r5 a/subdir/b/foo
582 test_expect_success 'tree:<depth> which filters out blob but given as arg' '
583 blob_hash=$(git -C r4 rev-parse HEAD:subdir/bar) &&
585 git -C r4 rev-list --objects --filter=tree:1 HEAD $blob_hash >actual &&
586 grep ^$blob_hash actual
589 # Delete some loose objects and use rev-list, but WITHOUT any filtering.
590 # This models previously omitted objects that we did not receive.
592 test_expect_success 'rev-list W/ --missing=print' '
593 git -C r1 ls-files -s file.1 file.2 file.3 file.4 file.5 \
594 >ls_files_result &&
595 awk -f print_2.awk ls_files_result |
596 sort >expected &&
598 for id in `cat expected | sed "s|..|&/|"`
600 rm r1/.git/objects/$id
601 done &&
603 git -C r1 rev-list --quiet --missing=print --objects HEAD >revs &&
604 awk -f print_1.awk revs |
605 sed "s/?//" |
606 sort >observed &&
608 test_cmp expected observed
611 test_expect_success 'rev-list W/O --missing fails' '
612 test_must_fail git -C r1 rev-list --quiet --objects HEAD
615 test_expect_success 'rev-list W/ missing=allow-any' '
616 git -C r1 rev-list --quiet --missing=allow-any --objects HEAD
619 # Test expansion of filter specs.
621 test_expect_success 'expand blob limit in protocol' '
622 git -C r2 config --local uploadpack.allowfilter 1 &&
623 GIT_TRACE_PACKET="$(pwd)/trace" git -c protocol.version=2 clone \
624 --filter=blob:limit=1k "file://$(pwd)/r2" limit &&
625 ! grep "blob:limit=1k" trace &&
626 grep "blob:limit=1024" trace
629 test_done