t/t0021-conversion.sh

   1 #!/bin/sh
   2
   3 test_description='blob conversion via gitattributes'
   4
   5 GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
   6 export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
   7
   8 . ./test-lib.sh
   9
  10 TEST_ROOT="$PWD"
  11 PATH=$TEST_ROOT:$PATH
  12
  13 write_script <<\EOF "$TEST_ROOT/rot13.sh"
  14 tr \
  15   'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' \
  16   'nopqrstuvwxyzabcdefghijklmNOPQRSTUVWXYZABCDEFGHIJKLM'
  17 EOF
  18
  19 write_script rot13-filter.pl "$PERL_PATH" \
  20         <"$TEST_DIRECTORY"/t0021/rot13-filter.pl
  21
  22 generate_random_characters () {
  23         LEN=$1
  24         NAME=$2
  25         test-tool genrandom some-seed $LEN |
  26                 perl -pe "s/./chr((ord($&) % 26) + ord('a'))/sge" >"$TEST_ROOT/$NAME"
  27 }
  28
  29 filter_git () {
  30         rm -f *.log &&
  31         git "$@"
  32 }
  33
  34 # Compare two files and ensure that `clean` and `smudge` respectively are
  35 # called at least once if specified in the `expect` file. The actual
  36 # invocation count is not relevant because their number can vary.
  37 # c.f. https://lore.kernel.org/git/xmqqshv18i8i.fsf@gitster.mtv.corp.google.com/
  38 test_cmp_count () {
  39         expect=$1
  40         actual=$2
  41         for FILE in "$expect" "$actual"
  42         do
  43                 sort "$FILE" | uniq -c |
  44                 sed -e "s/^ *[0-9][0-9]*[       ]*IN: /x IN: /" >"$FILE.tmp"
  45         done &&
  46         test_cmp "$expect.tmp" "$actual.tmp" &&
  47         rm "$expect.tmp" "$actual.tmp"
  48 }
  49
  50 # Compare two files but exclude all `clean` invocations because Git can
  51 # call `clean` zero or more times.
  52 # c.f. https://lore.kernel.org/git/xmqqshv18i8i.fsf@gitster.mtv.corp.google.com/
  53 test_cmp_exclude_clean () {
  54         expect=$1
  55         actual=$2
  56         for FILE in "$expect" "$actual"
  57         do
  58                 grep -v "IN: clean" "$FILE" >"$FILE.tmp"
  59         done &&
  60         test_cmp "$expect.tmp" "$actual.tmp" &&
  61         rm "$expect.tmp" "$actual.tmp"
  62 }
  63
  64 # Check that the contents of two files are equal and that their rot13 version
  65 # is equal to the committed content.
  66 test_cmp_committed_rot13 () {
  67         test_cmp "$1" "$2" &&
  68         rot13.sh <"$1" >expected &&
  69         git cat-file blob :"$2" >actual &&
  70         test_cmp expected actual
  71 }
  72
  73 test_expect_success setup '
  74         git config filter.rot13.smudge ./rot13.sh &&
  75         git config filter.rot13.clean ./rot13.sh &&
  76
  77         {
  78             echo "*.t filter=rot13"
  79             echo "*.i ident"
  80         } >.gitattributes &&
  81
  82         {
  83             echo a b c d e f g h i j k l m
  84             echo n o p q r s t u v w x y z
  85             echo '\''$Id$'\''
  86         } >test &&
  87         cat test >test.t &&
  88         cat test >test.o &&
  89         cat test >test.i &&
  90         git add test test.t test.i &&
  91         rm -f test test.t test.i &&
  92         git checkout -- test test.t test.i &&
  93
  94         echo "content-test2" >test2.o &&
  95         echo "content-test3 - filename with special characters" >"test3 '\''sq'\'',\$x=.o"
  96 '
  97
  98 script='s/^\$Id: \([0-9a-f]*\) \$/\1/p'
  99
 100 test_expect_success check '
 101
 102         test_cmp test.o test &&
 103         test_cmp test.o test.t &&
 104
 105         # ident should be stripped in the repository
 106         git diff --raw --exit-code :test :test.i &&
 107         id=$(git rev-parse --verify :test) &&
 108         embedded=$(sed -ne "$script" test.i) &&
 109         test "z$id" = "z$embedded" &&
 110
 111         git cat-file blob :test.t >test.r &&
 112
 113         ./rot13.sh <test.o >test.t &&
 114         test_cmp test.r test.t
 115 '
 116
 117 # If an expanded ident ever gets into the repository, we want to make sure that
 118 # it is collapsed before being expanded again on checkout
 119 test_expect_success expanded_in_repo '
 120         {
 121                 echo "File with expanded keywords"
 122                 echo "\$Id\$"
 123                 echo "\$Id:\$"
 124                 echo "\$Id: 0000000000000000000000000000000000000000 \$"
 125                 echo "\$Id: NoSpaceAtEnd\$"
 126                 echo "\$Id:NoSpaceAtFront \$"
 127                 echo "\$Id:NoSpaceAtEitherEnd\$"
 128                 echo "\$Id: NoTerminatingSymbol"
 129                 echo "\$Id: Foreign Commit With Spaces \$"
 130         } >expanded-keywords.0 &&
 131
 132         {
 133                 cat expanded-keywords.0 &&
 134                 printf "\$Id: NoTerminatingSymbolAtEOF"
 135         } >expanded-keywords &&
 136         cat expanded-keywords >expanded-keywords-crlf &&
 137         git add expanded-keywords expanded-keywords-crlf &&
 138         git commit -m "File with keywords expanded" &&
 139         id=$(git rev-parse --verify :expanded-keywords) &&
 140
 141         {
 142                 echo "File with expanded keywords"
 143                 echo "\$Id: $id \$"
 144                 echo "\$Id: $id \$"
 145                 echo "\$Id: $id \$"
 146                 echo "\$Id: $id \$"
 147                 echo "\$Id: $id \$"
 148                 echo "\$Id: $id \$"
 149                 echo "\$Id: NoTerminatingSymbol"
 150                 echo "\$Id: Foreign Commit With Spaces \$"
 151         } >expected-output.0 &&
 152         {
 153                 cat expected-output.0 &&
 154                 printf "\$Id: NoTerminatingSymbolAtEOF"
 155         } >expected-output &&
 156         {
 157                 append_cr <expected-output.0 &&
 158                 printf "\$Id: NoTerminatingSymbolAtEOF"
 159         } >expected-output-crlf &&
 160         {
 161                 echo "expanded-keywords ident"
 162                 echo "expanded-keywords-crlf ident text eol=crlf"
 163         } >>.gitattributes &&
 164
 165         rm -f expanded-keywords expanded-keywords-crlf &&
 166
 167         git checkout -- expanded-keywords &&
 168         test_cmp expected-output expanded-keywords &&
 169
 170         git checkout -- expanded-keywords-crlf &&
 171         test_cmp expected-output-crlf expanded-keywords-crlf
 172 '
 173
 174 # The use of %f in a filter definition is expanded to the path to
 175 # the filename being smudged or cleaned.  It must be shell escaped.
 176 # First, set up some interesting file names and pet them in
 177 # .gitattributes.
 178 test_expect_success 'filter shell-escaped filenames' '
 179         cat >argc.sh <<-EOF &&
 180         #!$SHELL_PATH
 181         cat >/dev/null
 182         echo argc: \$# "\$@"
 183         EOF
 184         normal=name-no-magic &&
 185         special="name  with '\''sq'\'' and \$x" &&
 186         echo some test text >"$normal" &&
 187         echo some test text >"$special" &&
 188         git add "$normal" "$special" &&
 189         git commit -q -m "add files" &&
 190         echo "name* filter=argc" >.gitattributes &&
 191
 192         # delete the files and check them out again, using a smudge filter
 193         # that will count the args and echo the command-line back to us
 194         test_config filter.argc.smudge "sh ./argc.sh %f" &&
 195         rm "$normal" "$special" &&
 196         git checkout -- "$normal" "$special" &&
 197
 198         # make sure argc.sh counted the right number of args
 199         echo "argc: 1 $normal" >expect &&
 200         test_cmp expect "$normal" &&
 201         echo "argc: 1 $special" >expect &&
 202         test_cmp expect "$special" &&
 203
 204         # do the same thing, but with more args in the filter expression
 205         test_config filter.argc.smudge "sh ./argc.sh %f --my-extra-arg" &&
 206         rm "$normal" "$special" &&
 207         git checkout -- "$normal" "$special" &&
 208
 209         # make sure argc.sh counted the right number of args
 210         echo "argc: 2 $normal --my-extra-arg" >expect &&
 211         test_cmp expect "$normal" &&
 212         echo "argc: 2 $special --my-extra-arg" >expect &&
 213         test_cmp expect "$special" &&
 214         :
 215 '
 216
 217 test_expect_success 'required filter should filter data' '
 218         test_config filter.required.smudge ./rot13.sh &&
 219         test_config filter.required.clean ./rot13.sh &&
 220         test_config filter.required.required true &&
 221
 222         echo "*.r filter=required" >.gitattributes &&
 223
 224         cat test.o >test.r &&
 225         git add test.r &&
 226
 227         rm -f test.r &&
 228         git checkout -- test.r &&
 229         test_cmp test.o test.r &&
 230
 231         ./rot13.sh <test.o >expected &&
 232         git cat-file blob :test.r >actual &&
 233         test_cmp expected actual
 234 '
 235
 236 test_expect_success 'required filter smudge failure' '
 237         test_config filter.failsmudge.smudge false &&
 238         test_config filter.failsmudge.clean cat &&
 239         test_config filter.failsmudge.required true &&
 240
 241         echo "*.fs filter=failsmudge" >.gitattributes &&
 242
 243         echo test >test.fs &&
 244         git add test.fs &&
 245         rm -f test.fs &&
 246         test_must_fail git checkout -- test.fs
 247 '
 248
 249 test_expect_success 'required filter clean failure' '
 250         test_config filter.failclean.smudge cat &&
 251         test_config filter.failclean.clean false &&
 252         test_config filter.failclean.required true &&
 253
 254         echo "*.fc filter=failclean" >.gitattributes &&
 255
 256         echo test >test.fc &&
 257         test_must_fail git add test.fc
 258 '
 259
 260 test_expect_success 'required filter with absent clean field' '
 261         test_config filter.absentclean.smudge cat &&
 262         test_config filter.absentclean.required true &&
 263
 264         echo "*.ac filter=absentclean" >.gitattributes &&
 265
 266         echo test >test.ac &&
 267         test_must_fail git add test.ac 2>stderr &&
 268         test_i18ngrep "fatal: test.ac: clean filter .absentclean. failed" stderr
 269 '
 270
 271 test_expect_success 'required filter with absent smudge field' '
 272         test_config filter.absentsmudge.clean cat &&
 273         test_config filter.absentsmudge.required true &&
 274
 275         echo "*.as filter=absentsmudge" >.gitattributes &&
 276
 277         echo test >test.as &&
 278         git add test.as &&
 279         rm -f test.as &&
 280         test_must_fail git checkout -- test.as 2>stderr &&
 281         test_i18ngrep "fatal: test.as: smudge filter absentsmudge failed" stderr
 282 '
 283
 284 test_expect_success 'filtering large input to small output should use little memory' '
 285         test_config filter.devnull.clean "cat >/dev/null" &&
 286         test_config filter.devnull.required true &&
 287         for i in $(test_seq 1 30); do printf "%1048576d" 1; done >30MB &&
 288         echo "30MB filter=devnull" >.gitattributes &&
 289         GIT_MMAP_LIMIT=1m GIT_ALLOC_LIMIT=1m git add 30MB
 290 '
 291
 292 test_expect_success 'filter that does not read is fine' '
 293         test-tool genrandom foo $((128 * 1024 + 1)) >big &&
 294         echo "big filter=epipe" >.gitattributes &&
 295         test_config filter.epipe.clean "echo xyzzy" &&
 296         git add big &&
 297         git cat-file blob :big >actual &&
 298         echo xyzzy >expect &&
 299         test_cmp expect actual
 300 '
 301
 302 test_expect_success EXPENSIVE 'filter large file' '
 303         test_config filter.largefile.smudge cat &&
 304         test_config filter.largefile.clean cat &&
 305         for i in $(test_seq 1 2048); do printf "%1048576d" 1; done >2GB &&
 306         echo "2GB filter=largefile" >.gitattributes &&
 307         git add 2GB 2>err &&
 308         test_must_be_empty err &&
 309         rm -f 2GB &&
 310         git checkout -- 2GB 2>err &&
 311         test_must_be_empty err
 312 '
 313
 314 test_expect_success "filter: clean empty file" '
 315         test_config filter.in-repo-header.clean  "echo cleaned && cat" &&
 316         test_config filter.in-repo-header.smudge "sed 1d" &&
 317
 318         echo "empty-in-worktree    filter=in-repo-header" >>.gitattributes &&
 319         >empty-in-worktree &&
 320
 321         echo cleaned >expected &&
 322         git add empty-in-worktree &&
 323         git show :empty-in-worktree >actual &&
 324         test_cmp expected actual
 325 '
 326
 327 test_expect_success "filter: smudge empty file" '
 328         test_config filter.empty-in-repo.clean "cat >/dev/null" &&
 329         test_config filter.empty-in-repo.smudge "echo smudged && cat" &&
 330
 331         echo "empty-in-repo filter=empty-in-repo" >>.gitattributes &&
 332         echo dead data walking >empty-in-repo &&
 333         git add empty-in-repo &&
 334
 335         echo smudged >expected &&
 336         git checkout-index --prefix=filtered- empty-in-repo &&
 337         test_cmp expected filtered-empty-in-repo
 338 '
 339
 340 test_expect_success 'disable filter with empty override' '
 341         test_config_global filter.disable.smudge false &&
 342         test_config_global filter.disable.clean false &&
 343         test_config filter.disable.smudge false &&
 344         test_config filter.disable.clean false &&
 345
 346         echo "*.disable filter=disable" >.gitattributes &&
 347
 348         echo test >test.disable &&
 349         git -c filter.disable.clean= add test.disable 2>err &&
 350         test_must_be_empty err &&
 351         rm -f test.disable &&
 352         git -c filter.disable.smudge= checkout -- test.disable 2>err &&
 353         test_must_be_empty err
 354 '
 355
 356 test_expect_success 'diff does not reuse worktree files that need cleaning' '
 357         test_config filter.counter.clean "echo . >>count; sed s/^/clean:/" &&
 358         echo "file filter=counter" >.gitattributes &&
 359         test_commit one file &&
 360         test_commit two file &&
 361
 362         >count &&
 363         git diff-tree -p HEAD &&
 364         test_line_count = 0 count
 365 '
 366
 367 test_expect_success PERL 'required process filter should filter data' '
 368         test_config_global filter.protocol.process "rot13-filter.pl debug.log clean smudge" &&
 369         test_config_global filter.protocol.required true &&
 370         rm -rf repo &&
 371         mkdir repo &&
 372         (
 373                 cd repo &&
 374                 git init &&
 375
 376                 echo "*.r filter=protocol" >.gitattributes &&
 377                 git add . &&
 378                 git commit -m "test commit 1" &&
 379                 git branch empty-branch &&
 380
 381                 cp "$TEST_ROOT/test.o" test.r &&
 382                 cp "$TEST_ROOT/test2.o" test2.r &&
 383                 mkdir testsubdir &&
 384                 cp "$TEST_ROOT/test3 '\''sq'\'',\$x=.o" "testsubdir/test3 '\''sq'\'',\$x=.r" &&
 385                 >test4-empty.r &&
 386
 387                 S=$(test_file_size test.r) &&
 388                 S2=$(test_file_size test2.r) &&
 389                 S3=$(test_file_size "testsubdir/test3 '\''sq'\'',\$x=.r") &&
 390                 M=$(git hash-object test.r) &&
 391                 M2=$(git hash-object test2.r) &&
 392                 M3=$(git hash-object "testsubdir/test3 '\''sq'\'',\$x=.r") &&
 393                 EMPTY=$(git hash-object /dev/null) &&
 394
 395                 filter_git add . &&
 396                 cat >expected.log <<-EOF &&
 397                         START
 398                         init handshake complete
 399                         IN: clean test.r $S [OK] -- OUT: $S . [OK]
 400                         IN: clean test2.r $S2 [OK] -- OUT: $S2 . [OK]
 401                         IN: clean test4-empty.r 0 [OK] -- OUT: 0  [OK]
 402                         IN: clean testsubdir/test3 '\''sq'\'',\$x=.r $S3 [OK] -- OUT: $S3 . [OK]
 403                         STOP
 404                 EOF
 405                 test_cmp_count expected.log debug.log &&
 406
 407                 git commit -m "test commit 2" &&
 408                 MAIN=$(git rev-parse --verify main) &&
 409                 META="ref=refs/heads/main treeish=$MAIN" &&
 410                 rm -f test2.r "testsubdir/test3 '\''sq'\'',\$x=.r" &&
 411
 412                 filter_git checkout --quiet --no-progress . &&
 413                 cat >expected.log <<-EOF &&
 414                         START
 415                         init handshake complete
 416                         IN: smudge test2.r blob=$M2 $S2 [OK] -- OUT: $S2 . [OK]
 417                         IN: smudge testsubdir/test3 '\''sq'\'',\$x=.r blob=$M3 $S3 [OK] -- OUT: $S3 . [OK]
 418                         STOP
 419                 EOF
 420                 test_cmp_exclude_clean expected.log debug.log &&
 421
 422                 # Make sure that the file appears dirty, so checkout below has to
 423                 # run the configured filter.
 424                 touch test.r &&
 425                 filter_git checkout --quiet --no-progress empty-branch &&
 426                 cat >expected.log <<-EOF &&
 427                         START
 428                         init handshake complete
 429                         IN: clean test.r $S [OK] -- OUT: $S . [OK]
 430                         STOP
 431                 EOF
 432                 test_cmp_exclude_clean expected.log debug.log &&
 433
 434                 filter_git checkout --quiet --no-progress main &&
 435                 cat >expected.log <<-EOF &&
 436                         START
 437                         init handshake complete
 438                         IN: smudge test.r $META blob=$M $S [OK] -- OUT: $S . [OK]
 439                         IN: smudge test2.r $META blob=$M2 $S2 [OK] -- OUT: $S2 . [OK]
 440                         IN: smudge test4-empty.r $META blob=$EMPTY 0 [OK] -- OUT: 0  [OK]
 441                         IN: smudge testsubdir/test3 '\''sq'\'',\$x=.r $META blob=$M3 $S3 [OK] -- OUT: $S3 . [OK]
 442                         STOP
 443                 EOF
 444                 test_cmp_exclude_clean expected.log debug.log &&
 445
 446                 test_cmp_committed_rot13 "$TEST_ROOT/test.o" test.r &&
 447                 test_cmp_committed_rot13 "$TEST_ROOT/test2.o" test2.r &&
 448                 test_cmp_committed_rot13 "$TEST_ROOT/test3 '\''sq'\'',\$x=.o" "testsubdir/test3 '\''sq'\'',\$x=.r"
 449         )
 450 '
 451
 452 test_expect_success PERL 'required process filter should filter data for various subcommands' '
 453         test_config_global filter.protocol.process "rot13-filter.pl debug.log clean smudge" &&
 454         test_config_global filter.protocol.required true &&
 455         (
 456                 cd repo &&
 457
 458                 S=$(test_file_size test.r) &&
 459                 S2=$(test_file_size test2.r) &&
 460                 S3=$(test_file_size "testsubdir/test3 '\''sq'\'',\$x=.r") &&
 461                 M=$(git hash-object test.r) &&
 462                 M2=$(git hash-object test2.r) &&
 463                 M3=$(git hash-object "testsubdir/test3 '\''sq'\'',\$x=.r") &&
 464                 EMPTY=$(git hash-object /dev/null) &&
 465
 466                 MAIN=$(git rev-parse --verify main) &&
 467
 468                 cp "$TEST_ROOT/test.o" test5.r &&
 469                 git add test5.r &&
 470                 git commit -m "test commit 3" &&
 471                 git checkout empty-branch &&
 472                 filter_git rebase --onto empty-branch main^^ main &&
 473                 MAIN2=$(git rev-parse --verify main) &&
 474                 META="ref=refs/heads/main treeish=$MAIN2" &&
 475                 cat >expected.log <<-EOF &&
 476                         START
 477                         init handshake complete
 478                         IN: smudge test.r $META blob=$M $S [OK] -- OUT: $S . [OK]
 479                         IN: smudge test2.r $META blob=$M2 $S2 [OK] -- OUT: $S2 . [OK]
 480                         IN: smudge test4-empty.r $META blob=$EMPTY 0 [OK] -- OUT: 0  [OK]
 481                         IN: smudge test5.r $META blob=$M $S [OK] -- OUT: $S . [OK]
 482                         IN: smudge testsubdir/test3 '\''sq'\'',\$x=.r $META blob=$M3 $S3 [OK] -- OUT: $S3 . [OK]
 483                         STOP
 484                 EOF
 485                 test_cmp_exclude_clean expected.log debug.log &&
 486
 487                 git reset --hard empty-branch &&
 488                 filter_git reset --hard $MAIN &&
 489                 META="treeish=$MAIN" &&
 490                 cat >expected.log <<-EOF &&
 491                         START
 492                         init handshake complete
 493                         IN: smudge test.r $META blob=$M $S [OK] -- OUT: $S . [OK]
 494                         IN: smudge test2.r $META blob=$M2 $S2 [OK] -- OUT: $S2 . [OK]
 495                         IN: smudge test4-empty.r $META blob=$EMPTY 0 [OK] -- OUT: 0  [OK]
 496                         IN: smudge testsubdir/test3 '\''sq'\'',\$x=.r $META blob=$M3 $S3 [OK] -- OUT: $S3 . [OK]
 497                         STOP
 498                 EOF
 499                 test_cmp_exclude_clean expected.log debug.log &&
 500
 501                 git branch old-main $MAIN &&
 502                 git reset --hard empty-branch &&
 503                 filter_git reset --hard old-main &&
 504                 META="ref=refs/heads/old-main treeish=$MAIN" &&
 505                 cat >expected.log <<-EOF &&
 506                         START
 507                         init handshake complete
 508                         IN: smudge test.r $META blob=$M $S [OK] -- OUT: $S . [OK]
 509                         IN: smudge test2.r $META blob=$M2 $S2 [OK] -- OUT: $S2 . [OK]
 510                         IN: smudge test4-empty.r $META blob=$EMPTY 0 [OK] -- OUT: 0  [OK]
 511                         IN: smudge testsubdir/test3 '\''sq'\'',\$x=.r $META blob=$M3 $S3 [OK] -- OUT: $S3 . [OK]
 512                         STOP
 513                 EOF
 514                 test_cmp_exclude_clean expected.log debug.log &&
 515
 516                 git checkout -b merge empty-branch &&
 517                 git branch -f main $MAIN2 &&
 518                 filter_git merge main &&
 519                 META="treeish=$MAIN2" &&
 520                 cat >expected.log <<-EOF &&
 521                         START
 522                         init handshake complete
 523                         IN: smudge test.r $META blob=$M $S [OK] -- OUT: $S . [OK]
 524                         IN: smudge test2.r $META blob=$M2 $S2 [OK] -- OUT: $S2 . [OK]
 525                         IN: smudge test4-empty.r $META blob=$EMPTY 0 [OK] -- OUT: 0  [OK]
 526                         IN: smudge test5.r $META blob=$M $S [OK] -- OUT: $S . [OK]
 527                         IN: smudge testsubdir/test3 '\''sq'\'',\$x=.r $META blob=$M3 $S3 [OK] -- OUT: $S3 . [OK]
 528                         STOP
 529                 EOF
 530                 test_cmp_exclude_clean expected.log debug.log &&
 531
 532                 filter_git archive main >/dev/null &&
 533                 META="ref=refs/heads/main treeish=$MAIN2" &&
 534                 cat >expected.log <<-EOF &&
 535                         START
 536                         init handshake complete
 537                         IN: smudge test.r $META blob=$M $S [OK] -- OUT: $S . [OK]
 538                         IN: smudge test2.r $META blob=$M2 $S2 [OK] -- OUT: $S2 . [OK]
 539                         IN: smudge test4-empty.r $META blob=$EMPTY 0 [OK] -- OUT: 0  [OK]
 540                         IN: smudge test5.r $META blob=$M $S [OK] -- OUT: $S . [OK]
 541                         IN: smudge testsubdir/test3 '\''sq'\'',\$x=.r $META blob=$M3 $S3 [OK] -- OUT: $S3 . [OK]
 542                         STOP
 543                 EOF
 544                 test_cmp_exclude_clean expected.log debug.log &&
 545
 546                 TREE="$(git rev-parse $MAIN2^{tree})" &&
 547                 filter_git archive $TREE >/dev/null &&
 548                 META="treeish=$TREE" &&
 549                 cat >expected.log <<-EOF &&
 550                         START
 551                         init handshake complete
 552                         IN: smudge test.r $META blob=$M $S [OK] -- OUT: $S . [OK]
 553                         IN: smudge test2.r $META blob=$M2 $S2 [OK] -- OUT: $S2 . [OK]
 554                         IN: smudge test4-empty.r $META blob=$EMPTY 0 [OK] -- OUT: 0  [OK]
 555                         IN: smudge test5.r $META blob=$M $S [OK] -- OUT: $S . [OK]
 556                         IN: smudge testsubdir/test3 '\''sq'\'',\$x=.r $META blob=$M3 $S3 [OK] -- OUT: $S3 . [OK]
 557                         STOP
 558                 EOF
 559                 test_cmp_exclude_clean expected.log debug.log
 560         )
 561 '
 562
 563 test_expect_success PERL 'required process filter takes precedence' '
 564         test_config_global filter.protocol.clean false &&
 565         test_config_global filter.protocol.process "rot13-filter.pl debug.log clean" &&
 566         test_config_global filter.protocol.required true &&
 567         rm -rf repo &&
 568         mkdir repo &&
 569         (
 570                 cd repo &&
 571                 git init &&
 572
 573                 echo "*.r filter=protocol" >.gitattributes &&
 574                 cp "$TEST_ROOT/test.o" test.r &&
 575                 S=$(test_file_size test.r) &&
 576
 577                 # Check that the process filter is invoked here
 578                 filter_git add . &&
 579                 cat >expected.log <<-EOF &&
 580                         START
 581                         init handshake complete
 582                         IN: clean test.r $S [OK] -- OUT: $S . [OK]
 583                         STOP
 584                 EOF
 585                 test_cmp_count expected.log debug.log
 586         )
 587 '
 588
 589 test_expect_success PERL 'required process filter should be used only for "clean" operation only' '
 590         test_config_global filter.protocol.process "rot13-filter.pl debug.log clean" &&
 591         rm -rf repo &&
 592         mkdir repo &&
 593         (
 594                 cd repo &&
 595                 git init &&
 596
 597                 echo "*.r filter=protocol" >.gitattributes &&
 598                 cp "$TEST_ROOT/test.o" test.r &&
 599                 S=$(test_file_size test.r) &&
 600
 601                 filter_git add . &&
 602                 cat >expected.log <<-EOF &&
 603                         START
 604                         init handshake complete
 605                         IN: clean test.r $S [OK] -- OUT: $S . [OK]
 606                         STOP
 607                 EOF
 608                 test_cmp_count expected.log debug.log &&
 609
 610                 rm test.r &&
 611
 612                 filter_git checkout --quiet --no-progress . &&
 613                 # If the filter would be used for "smudge", too, we would see
 614                 # "IN: smudge test.r 57 [OK] -- OUT: 57 . [OK]" here
 615                 cat >expected.log <<-EOF &&
 616                         START
 617                         init handshake complete
 618                         STOP
 619                 EOF
 620                 test_cmp_exclude_clean expected.log debug.log
 621         )
 622 '
 623
 624 test_expect_success PERL 'required process filter should process multiple packets' '
 625         test_config_global filter.protocol.process "rot13-filter.pl debug.log clean smudge" &&
 626         test_config_global filter.protocol.required true &&
 627
 628         rm -rf repo &&
 629         mkdir repo &&
 630         (
 631                 cd repo &&
 632                 git init &&
 633
 634                 # Generate data requiring 1, 2, 3 packets
 635                 S=65516 && # PKTLINE_DATA_MAXLEN -> Maximal size of a packet
 636                 generate_random_characters $(($S    )) 1pkt_1__.file &&
 637                 generate_random_characters $(($S  +1)) 2pkt_1+1.file &&
 638                 generate_random_characters $(($S*2-1)) 2pkt_2-1.file &&
 639                 generate_random_characters $(($S*2  )) 2pkt_2__.file &&
 640                 generate_random_characters $(($S*2+1)) 3pkt_2+1.file &&
 641
 642                 for FILE in "$TEST_ROOT"/*.file
 643                 do
 644                         cp "$FILE" . &&
 645                         rot13.sh <"$FILE" >"$FILE.rot13"
 646                 done &&
 647
 648                 echo "*.file filter=protocol" >.gitattributes &&
 649                 filter_git add *.file .gitattributes &&
 650                 cat >expected.log <<-EOF &&
 651                         START
 652                         init handshake complete
 653                         IN: clean 1pkt_1__.file $(($S    )) [OK] -- OUT: $(($S    )) . [OK]
 654                         IN: clean 2pkt_1+1.file $(($S  +1)) [OK] -- OUT: $(($S  +1)) .. [OK]
 655                         IN: clean 2pkt_2-1.file $(($S*2-1)) [OK] -- OUT: $(($S*2-1)) .. [OK]
 656                         IN: clean 2pkt_2__.file $(($S*2  )) [OK] -- OUT: $(($S*2  )) .. [OK]
 657                         IN: clean 3pkt_2+1.file $(($S*2+1)) [OK] -- OUT: $(($S*2+1)) ... [OK]
 658                         STOP
 659                 EOF
 660                 test_cmp_count expected.log debug.log &&
 661
 662                 M1="blob=$(git hash-object 1pkt_1__.file)" &&
 663                 M2="blob=$(git hash-object 2pkt_1+1.file)" &&
 664                 M3="blob=$(git hash-object 2pkt_2-1.file)" &&
 665                 M4="blob=$(git hash-object 2pkt_2__.file)" &&
 666                 M5="blob=$(git hash-object 3pkt_2+1.file)" &&
 667                 rm -f *.file debug.log &&
 668
 669                 filter_git checkout --quiet --no-progress -- *.file &&
 670                 cat >expected.log <<-EOF &&
 671                         START
 672                         init handshake complete
 673                         IN: smudge 1pkt_1__.file $M1 $(($S    )) [OK] -- OUT: $(($S    )) . [OK]
 674                         IN: smudge 2pkt_1+1.file $M2 $(($S  +1)) [OK] -- OUT: $(($S  +1)) .. [OK]
 675                         IN: smudge 2pkt_2-1.file $M3 $(($S*2-1)) [OK] -- OUT: $(($S*2-1)) .. [OK]
 676                         IN: smudge 2pkt_2__.file $M4 $(($S*2  )) [OK] -- OUT: $(($S*2  )) .. [OK]
 677                         IN: smudge 3pkt_2+1.file $M5 $(($S*2+1)) [OK] -- OUT: $(($S*2+1)) ... [OK]
 678                         STOP
 679                 EOF
 680                 test_cmp_exclude_clean expected.log debug.log &&
 681
 682                 for FILE in *.file
 683                 do
 684                         test_cmp_committed_rot13 "$TEST_ROOT/$FILE" $FILE
 685                 done
 686         )
 687 '
 688
 689 test_expect_success PERL 'required process filter with clean error should fail' '
 690         test_config_global filter.protocol.process "rot13-filter.pl debug.log clean smudge" &&
 691         test_config_global filter.protocol.required true &&
 692         rm -rf repo &&
 693         mkdir repo &&
 694         (
 695                 cd repo &&
 696                 git init &&
 697
 698                 echo "*.r filter=protocol" >.gitattributes &&
 699
 700                 cp "$TEST_ROOT/test.o" test.r &&
 701                 echo "this is going to fail" >clean-write-fail.r &&
 702                 echo "content-test3-subdir" >test3.r &&
 703
 704                 test_must_fail git add .
 705         )
 706 '
 707
 708 test_expect_success PERL 'process filter should restart after unexpected write failure' '
 709         test_config_global filter.protocol.process "rot13-filter.pl debug.log clean smudge" &&
 710         rm -rf repo &&
 711         mkdir repo &&
 712         (
 713                 cd repo &&
 714                 git init &&
 715
 716                 echo "*.r filter=protocol" >.gitattributes &&
 717
 718                 cp "$TEST_ROOT/test.o" test.r &&
 719                 cp "$TEST_ROOT/test2.o" test2.r &&
 720                 echo "this is going to fail" >smudge-write-fail.o &&
 721                 cp smudge-write-fail.o smudge-write-fail.r &&
 722
 723                 S=$(test_file_size test.r) &&
 724                 S2=$(test_file_size test2.r) &&
 725                 SF=$(test_file_size smudge-write-fail.r) &&
 726                 M=$(git hash-object test.r) &&
 727                 M2=$(git hash-object test2.r) &&
 728                 MF=$(git hash-object smudge-write-fail.r) &&
 729                 rm -f debug.log &&
 730
 731                 git add . &&
 732                 rm -f *.r &&
 733
 734                 rm -f debug.log &&
 735                 git checkout --quiet --no-progress . 2>git-stderr.log &&
 736
 737                 grep "smudge write error at" git-stderr.log &&
 738                 test_i18ngrep "error: external filter" git-stderr.log &&
 739
 740                 cat >expected.log <<-EOF &&
 741                         START
 742                         init handshake complete
 743                         IN: smudge smudge-write-fail.r blob=$MF $SF [OK] -- [WRITE FAIL]
 744                         START
 745                         init handshake complete
 746                         IN: smudge test.r blob=$M $S [OK] -- OUT: $S . [OK]
 747                         IN: smudge test2.r blob=$M2 $S2 [OK] -- OUT: $S2 . [OK]
 748                         STOP
 749                 EOF
 750                 test_cmp_exclude_clean expected.log debug.log &&
 751
 752                 test_cmp_committed_rot13 "$TEST_ROOT/test.o" test.r &&
 753                 test_cmp_committed_rot13 "$TEST_ROOT/test2.o" test2.r &&
 754
 755                 # Smudge failed
 756                 ! test_cmp smudge-write-fail.o smudge-write-fail.r &&
 757                 rot13.sh <smudge-write-fail.o >expected &&
 758                 git cat-file blob :smudge-write-fail.r >actual &&
 759                 test_cmp expected actual
 760         )
 761 '
 762
 763 test_expect_success PERL 'process filter should not be restarted if it signals an error' '
 764         test_config_global filter.protocol.process "rot13-filter.pl debug.log clean smudge" &&
 765         rm -rf repo &&
 766         mkdir repo &&
 767         (
 768                 cd repo &&
 769                 git init &&
 770
 771                 echo "*.r filter=protocol" >.gitattributes &&
 772
 773                 cp "$TEST_ROOT/test.o" test.r &&
 774                 cp "$TEST_ROOT/test2.o" test2.r &&
 775                 echo "this will cause an error" >error.o &&
 776                 cp error.o error.r &&
 777
 778                 S=$(test_file_size test.r) &&
 779                 S2=$(test_file_size test2.r) &&
 780                 SE=$(test_file_size error.r) &&
 781                 M=$(git hash-object test.r) &&
 782                 M2=$(git hash-object test2.r) &&
 783                 ME=$(git hash-object error.r) &&
 784                 rm -f debug.log &&
 785
 786                 git add . &&
 787                 rm -f *.r &&
 788
 789                 filter_git checkout --quiet --no-progress . &&
 790                 cat >expected.log <<-EOF &&
 791                         START
 792                         init handshake complete
 793                         IN: smudge error.r blob=$ME $SE [OK] -- [ERROR]
 794                         IN: smudge test.r blob=$M $S [OK] -- OUT: $S . [OK]
 795                         IN: smudge test2.r blob=$M2 $S2 [OK] -- OUT: $S2 . [OK]
 796                         STOP
 797                 EOF
 798                 test_cmp_exclude_clean expected.log debug.log &&
 799
 800                 test_cmp_committed_rot13 "$TEST_ROOT/test.o" test.r &&
 801                 test_cmp_committed_rot13 "$TEST_ROOT/test2.o" test2.r &&
 802                 test_cmp error.o error.r
 803         )
 804 '
 805
 806 test_expect_success PERL 'process filter abort stops processing of all further files' '
 807         test_config_global filter.protocol.process "rot13-filter.pl debug.log clean smudge" &&
 808         rm -rf repo &&
 809         mkdir repo &&
 810         (
 811                 cd repo &&
 812                 git init &&
 813
 814                 echo "*.r filter=protocol" >.gitattributes &&
 815
 816                 cp "$TEST_ROOT/test.o" test.r &&
 817                 cp "$TEST_ROOT/test2.o" test2.r &&
 818                 echo "error this blob and all future blobs" >abort.o &&
 819                 cp abort.o abort.r &&
 820
 821                 M="blob=$(git hash-object abort.r)" &&
 822                 rm -f debug.log &&
 823                 SA=$(test_file_size abort.r) &&
 824
 825                 git add . &&
 826                 rm -f *.r &&
 827
 828
 829                 # Note: This test assumes that Git filters files in alphabetical
 830                 # order ("abort.r" before "test.r").
 831                 filter_git checkout --quiet --no-progress . &&
 832                 cat >expected.log <<-EOF &&
 833                         START
 834                         init handshake complete
 835                         IN: smudge abort.r $M $SA [OK] -- [ABORT]
 836                         STOP
 837                 EOF
 838                 test_cmp_exclude_clean expected.log debug.log &&
 839
 840                 test_cmp "$TEST_ROOT/test.o" test.r &&
 841                 test_cmp "$TEST_ROOT/test2.o" test2.r &&
 842                 test_cmp abort.o abort.r
 843         )
 844 '
 845
 846 test_expect_success PERL 'invalid process filter must fail (and not hang!)' '
 847         test_config_global filter.protocol.process cat &&
 848         test_config_global filter.protocol.required true &&
 849         rm -rf repo &&
 850         mkdir repo &&
 851         (
 852                 cd repo &&
 853                 git init &&
 854
 855                 echo "*.r filter=protocol" >.gitattributes &&
 856
 857                 cp "$TEST_ROOT/test.o" test.r &&
 858                 test_must_fail git add . 2>git-stderr.log &&
 859                 grep "expected git-filter-server" git-stderr.log
 860         )
 861 '
 862
 863 test_expect_success PERL 'delayed checkout in process filter' '
 864         test_config_global filter.a.process "rot13-filter.pl a.log clean smudge delay" &&
 865         test_config_global filter.a.required true &&
 866         test_config_global filter.b.process "rot13-filter.pl b.log clean smudge delay" &&
 867         test_config_global filter.b.required true &&
 868
 869         rm -rf repo &&
 870         mkdir repo &&
 871         (
 872                 cd repo &&
 873                 git init &&
 874                 echo "*.a filter=a" >.gitattributes &&
 875                 echo "*.b filter=b" >>.gitattributes &&
 876                 cp "$TEST_ROOT/test.o" test.a &&
 877                 cp "$TEST_ROOT/test.o" test-delay10.a &&
 878                 cp "$TEST_ROOT/test.o" test-delay11.a &&
 879                 cp "$TEST_ROOT/test.o" test-delay20.a &&
 880                 cp "$TEST_ROOT/test.o" test-delay10.b &&
 881                 git add . &&
 882                 git commit -m "test commit"
 883         ) &&
 884
 885         S=$(test_file_size "$TEST_ROOT/test.o") &&
 886         PM="ref=refs/heads/main treeish=$(git -C repo rev-parse --verify main) " &&
 887         M="${PM}blob=$(git -C repo rev-parse --verify main:test.a)" &&
 888         cat >a.exp <<-EOF &&
 889                 START
 890                 init handshake complete
 891                 IN: smudge test.a $M $S [OK] -- OUT: $S . [OK]
 892                 IN: smudge test-delay10.a $M $S [OK] -- [DELAYED]
 893                 IN: smudge test-delay11.a $M $S [OK] -- [DELAYED]
 894                 IN: smudge test-delay20.a $M $S [OK] -- [DELAYED]
 895                 IN: list_available_blobs test-delay10.a test-delay11.a [OK]
 896                 IN: smudge test-delay10.a $M 0 [OK] -- OUT: $S . [OK]
 897                 IN: smudge test-delay11.a $M 0 [OK] -- OUT: $S . [OK]
 898                 IN: list_available_blobs test-delay20.a [OK]
 899                 IN: smudge test-delay20.a $M 0 [OK] -- OUT: $S . [OK]
 900                 IN: list_available_blobs [OK]
 901                 STOP
 902         EOF
 903         cat >b.exp <<-EOF &&
 904                 START
 905                 init handshake complete
 906                 IN: smudge test-delay10.b $M $S [OK] -- [DELAYED]
 907                 IN: list_available_blobs test-delay10.b [OK]
 908                 IN: smudge test-delay10.b $M 0 [OK] -- OUT: $S . [OK]
 909                 IN: list_available_blobs [OK]
 910                 STOP
 911         EOF
 912
 913         rm -rf repo-cloned &&
 914         filter_git clone repo repo-cloned &&
 915         test_cmp_count a.exp repo-cloned/a.log &&
 916         test_cmp_count b.exp repo-cloned/b.log &&
 917
 918         (
 919                 cd repo-cloned &&
 920                 test_cmp_committed_rot13 "$TEST_ROOT/test.o" test.a &&
 921                 test_cmp_committed_rot13 "$TEST_ROOT/test.o" test-delay10.a &&
 922                 test_cmp_committed_rot13 "$TEST_ROOT/test.o" test-delay11.a &&
 923                 test_cmp_committed_rot13 "$TEST_ROOT/test.o" test-delay20.a &&
 924                 test_cmp_committed_rot13 "$TEST_ROOT/test.o" test-delay10.b &&
 925
 926                 rm *.a *.b &&
 927                 filter_git checkout . &&
 928                 # We are not checking out a ref here, so filter out ref metadata.
 929                 sed -e "s!$PM!!" ../a.exp >a.exp.filtered &&
 930                 sed -e "s!$PM!!" ../b.exp >b.exp.filtered &&
 931                 test_cmp_count a.exp.filtered a.log &&
 932                 test_cmp_count b.exp.filtered b.log &&
 933
 934                 test_cmp_committed_rot13 "$TEST_ROOT/test.o" test.a &&
 935                 test_cmp_committed_rot13 "$TEST_ROOT/test.o" test-delay10.a &&
 936                 test_cmp_committed_rot13 "$TEST_ROOT/test.o" test-delay11.a &&
 937                 test_cmp_committed_rot13 "$TEST_ROOT/test.o" test-delay20.a &&
 938                 test_cmp_committed_rot13 "$TEST_ROOT/test.o" test-delay10.b
 939         )
 940 '
 941
 942 test_expect_success PERL 'missing file in delayed checkout' '
 943         test_config_global filter.bug.process "rot13-filter.pl bug.log clean smudge delay" &&
 944         test_config_global filter.bug.required true &&
 945
 946         rm -rf repo &&
 947         mkdir repo &&
 948         (
 949                 cd repo &&
 950                 git init &&
 951                 echo "*.a filter=bug" >.gitattributes &&
 952                 cp "$TEST_ROOT/test.o" missing-delay.a &&
 953                 git add . &&
 954                 git commit -m "test commit"
 955         ) &&
 956
 957         rm -rf repo-cloned &&
 958         test_must_fail git clone repo repo-cloned 2>git-stderr.log &&
 959         grep "error: .missing-delay\.a. was not filtered properly" git-stderr.log
 960 '
 961
 962 test_expect_success PERL 'invalid file in delayed checkout' '
 963         test_config_global filter.bug.process "rot13-filter.pl bug.log clean smudge delay" &&
 964         test_config_global filter.bug.required true &&
 965
 966         rm -rf repo &&
 967         mkdir repo &&
 968         (
 969                 cd repo &&
 970                 git init &&
 971                 echo "*.a filter=bug" >.gitattributes &&
 972                 cp "$TEST_ROOT/test.o" invalid-delay.a &&
 973                 cp "$TEST_ROOT/test.o" unfiltered &&
 974                 git add . &&
 975                 git commit -m "test commit"
 976         ) &&
 977
 978         rm -rf repo-cloned &&
 979         test_must_fail git clone repo repo-cloned 2>git-stderr.log &&
 980         grep "error: external filter .* signaled that .unfiltered. is now available although it has not been delayed earlier" git-stderr.log
 981 '
 982
 983 for mode in 'case' 'utf-8'
 984 do
 985         case "$mode" in
 986         case)   dir='A' symlink='a' mode_prereq='CASE_INSENSITIVE_FS' ;;
 987         utf-8)
 988                 dir=$(printf "\141\314\210") symlink=$(printf "\303\244")
 989                 mode_prereq='UTF8_NFD_TO_NFC' ;;
 990         esac
 991
 992         test_expect_success PERL,SYMLINKS,$mode_prereq \
 993         "delayed checkout with $mode-collision don't write to the wrong place" '
 994                 test_config_global filter.delay.process \
 995                         "\"$TEST_ROOT/rot13-filter.pl\" --always-delay delayed.log clean smudge delay" &&
 996                 test_config_global filter.delay.required true &&
 997
 998                 git init $mode-collision &&
 999                 (
1000                         cd $mode-collision &&
1001                         mkdir target-dir &&
1002
1003                         empty_oid=$(printf "" | git hash-object -w --stdin) &&
1004                         symlink_oid=$(printf "%s" "$PWD/target-dir" | git hash-object -w --stdin) &&
1005                         attr_oid=$(echo "$dir/z filter=delay" | git hash-object -w --stdin) &&
1006
1007                         cat >objs <<-EOF &&
1008                         100644 blob $empty_oid  $dir/x
1009                         100644 blob $empty_oid  $dir/y
1010                         100644 blob $empty_oid  $dir/z
1011                         120000 blob $symlink_oid        $symlink
1012                         100644 blob $attr_oid   .gitattributes
1013                         EOF
1014
1015                         git update-index --index-info <objs &&
1016                         git commit -m "test commit"
1017                 ) &&
1018
1019                 git clone $mode-collision $mode-collision-cloned &&
1020                 # Make sure z was really delayed
1021                 grep "IN: smudge $dir/z .* \\[DELAYED\\]" $mode-collision-cloned/delayed.log &&
1022
1023                 # Should not create $dir/z at $symlink/z
1024                 test_path_is_missing $mode-collision/target-dir/z
1025         '
1026 done
1027
1028 test_expect_success PERL,SYMLINKS,CASE_INSENSITIVE_FS \
1029 "delayed checkout with submodule collision don't write to the wrong place" '
1030         git init collision-with-submodule &&
1031         (
1032                 cd collision-with-submodule &&
1033                 git config filter.delay.process "\"$TEST_ROOT/rot13-filter.pl\" --always-delay delayed.log clean smudge delay" &&
1034                 git config filter.delay.required true &&
1035
1036                 # We need Git to treat the submodule "a" and the
1037                 # leading dir "A" as different paths in the index.
1038                 git config --local core.ignoreCase false &&
1039
1040                 empty_oid=$(printf "" | git hash-object -w --stdin) &&
1041                 attr_oid=$(echo "A/B/y filter=delay" | git hash-object -w --stdin) &&
1042                 cat >objs <<-EOF &&
1043                 100644 blob $empty_oid  A/B/x
1044                 100644 blob $empty_oid  A/B/y
1045                 100644 blob $attr_oid   .gitattributes
1046                 EOF
1047                 git update-index --index-info <objs &&
1048
1049                 git init a &&
1050                 mkdir target-dir &&
1051                 symlink_oid=$(printf "%s" "$PWD/target-dir" | git -C a hash-object -w --stdin) &&
1052                 echo "120000 blob $symlink_oid  b" >objs &&
1053                 git -C a update-index --index-info <objs &&
1054                 git -C a commit -m sub &&
1055                 git submodule add ./a &&
1056                 git commit -m super &&
1057
1058                 git checkout --recurse-submodules . &&
1059                 grep "IN: smudge A/B/y .* \\[DELAYED\\]" delayed.log &&
1060                 test_path_is_missing target-dir/y
1061         )
1062 '
1063
1064 test_done