testsuite/tests-to-run/parallel-local-0.3s.sh

   1 #!/bin/bash
   2
   3 # SPDX-FileCopyrightText: 2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc.
   4 #
   5 # SPDX-License-Identifier: GPL-3.0-or-later
   6
   7 # Simple jobs that never fails
   8 # Each should be taking 0.3-1s and be possible to run in parallel
   9 # I.e.: No race conditions, no logins
  10
  11 stdsort() {
  12     "$@" 2>&1 | LC_ALL=C sort;
  13 }
  14 export -f stdsort
  15
  16 # Test amount of parallelization
  17 # parallel --shuf --jl /tmp/myjl -j1 'export JOBS={1};'bash tests-to-run/parallel-local-0.3s.sh ::: {1..16} ::: {1..5}
  18
  19 par_crnl() {
  20     echo '### Give a warning if input is DOS-ascii'
  21     printf "b\r\nc\r\nd\r\ne\r\nf\r\n" | stdout parallel -k echo {}a
  22     echo This should give no warning because -d is set
  23     printf "b\r\nc\r\nd\r\ne\r\nf\r\n" | parallel -k -d '\r\n' echo {}a
  24     echo This should give no warning because line2 has newline only
  25     printf "b\r\nc\nd\r\ne\r\nf\r\n" | parallel -k echo {}a
  26 }
  27
  28 par_tmpl1() {
  29     tmp1=$(mktemp)
  30     tmp2=$(mktemp)
  31     cat <<'EOF' > "$tmp1"
  32     Template1
  33     Xval: {x}
  34     Yval: {y}
  35     FixedValue: 9
  36     Seq: {#}
  37     Slot: {%}
  38     # x with 2 decimals
  39     DecimalX: {=x $_=sprintf("%.2f",$_) =}
  40     TenX: {=x $_=$_*10 =}
  41     RandomVal: {=1 $_=rand() =}
  42
  43 EOF
  44
  45     cat <<'EOF' > "$tmp2"
  46     Template2
  47     X,Y: {x},{y}
  48     val1,val2: {1},{2}
  49
  50 EOF
  51     myprog() {
  52         echo "$@"
  53         cat "$@"
  54     }
  55     export -f myprog
  56     parallel -k --header : --tmpl "$tmp1"={#}.t1 \
  57              --tmpl "$tmp2"=/tmp/tmpl-{x}-{y}.t2 \
  58              myprog {#}.t1 /tmp/tmpl-{x}-{y}.t2 \
  59              ::: x 1.1 2.22 3.333 ::: y 111.111 222.222 333.333 |
  60         perl -pe 's/0.\d{13,}/0.RANDOM_NUMBER/' |
  61         perl -pe 's/Slot: \d/Slot: X/'
  62     rm "$tmp1" "$tmp2"
  63 }
  64
  65 par_tmpl2() {
  66     tmp1=$(mktemp)
  67     tmp2=$(mktemp)
  68     cat <<'EOF' > "$tmp1"
  69     === Start tmpl1 ===
  70     1: Job:{#} Slot:{%} All:{} Arg[1]:{1} Arg[-1]:{-1} Perl({}+4):{=$_+=4=}
  71 EOF
  72
  73     cat <<'EOF' > "$tmp2"
  74     2: Job:{#} Slot:{%} All:{} Arg[1]:{1} Arg[-1]:{-1} Perl({}+4):{=$_+=4=}
  75 EOF
  76     parallel --colsep , -j2 --cleanup --tmpl $tmp1=t1.{#} --tmpl $tmp2=t2.{%} \
  77              'sleep 0.{#}; cat t1.{#} t2.{%}' ::: 1,a 1,b 2,a 2,b
  78     echo should give no files
  79     ls t[12].*
  80     parallel            -j2 --cleanup --tmpl $tmp1=t1.{#} --tmpl $tmp2=t2.{%} \
  81              'sleep 0.{#}; cat t1.{#} t2.{%}' ::: 1 2 ::: a b
  82     echo should give no files
  83     ls t[12].*
  84 }
  85
  86 par_resume_k() {
  87     echo '### --resume -k'
  88     tmp=$(tempfile)
  89     parallel -k --resume --joblog $tmp echo job{}id\;exit {} ::: 0 1 2 3 0 5
  90     echo try 2 = nothing
  91     parallel -k --resume --joblog $tmp echo job{}id\;exit {} ::: 0 1 2 3 0 5
  92     echo two extra
  93     parallel -k --resume --joblog $tmp echo job{}id\;exit 0 ::: 0 1 2 3 0 5 6 7
  94     rm -f $tmp
  95 }
  96
  97 par_empty_string_quote() {
  98     echo "bug #37694: Empty string argument skipped when using --quote"
  99     parallel -q --nonall perl -le 'print scalar @ARGV' 'a' 'b' ''
 100 }
 101
 102 par_trim_illegal_value() {
 103     echo '### Test of --trim illegal'
 104     stdout parallel --trim fj ::: echo
 105 }
 106
 107 par_dirnamereplace() {
 108     echo '### Test --dnr'
 109     parallel --dnr II -k echo II {} ::: a a/b a/b/c
 110
 111     echo '### Test --dirnamereplace'
 112     parallel --dirnamereplace II -k echo II {} ::: a a/b a/b/c
 113 }
 114
 115 par_negative_replacement() {
 116     echo '### Negative replacement strings'
 117     parallel -X -j1 -N 6 echo {-1}orrec{1} ::: t B X D E c
 118     parallel -N 6 echo {-1}orrect ::: A B X D E c
 119     parallel --colsep ' ' echo '{2} + {4} = {2} + {-1}=' '$(( {2} + {-1} ))' ::: "1 2 3 4"
 120     parallel --colsep ' ' echo '{-3}orrect' ::: "1 c 3 4"
 121 }
 122
 123 par_replacement_string_on_utf8() {
 124     echo '### test {} {.} on UTF8 input'
 125     inputlist() {
 126         echo "中国 (Zhōngguó)/China's (中国) road.jpg"
 127         echo "中国.(中国)"
 128         echo /tmp/test-of-{.}-parallel/subdir/file
 129         echo '/tmp/test-of-{.}-parallel/subdir/file{.}.funkyextension}}'
 130     }
 131     inputlist | parallel -k echo {} {.}
 132 }
 133
 134 par_rpl_repeats() {
 135     echo '### Test {.} does not repeat more than {}'
 136     seq 15 | perl -pe 's/$/.gif/' | parallel -j1 -s 80 -kX echo a{}b{.}c{.}
 137     seq 15 | perl -pe 's/$/.gif/' | parallel -j1 -s 80 -km echo a{}b{.}c{.}
 138 }
 139
 140 par_do_not_export_PARALLEL_ENV() {
 141     echo '### Do not export $PARALLEL_ENV to children'
 142     doit() {
 143         echo Should be 0
 144         echo "$PARALLEL_ENV" | wc
 145         echo Should give 60k and not overflow
 146         PARALLEL_ENV="$PARALLEL_ENV" parallel echo '{=$_="\""x$_=}' ::: 60000 | wc
 147     }
 148     . `which env_parallel.bash`
 149     # Make PARALLEL_ENV as big as possible
 150     PARALLEL_ENV="a='$(seq 100000 | head -c $((139000-$(set|wc -c) )) )'"
 151     env_parallel doit ::: 1
 152 }
 153
 154 par_compress_stdout_stderr() {
 155     echo '### Test compress - stdout'
 156     parallel --compress echo ::: OK
 157     echo '### Test compress - stderr'
 158     parallel --compress ls /{} ::: OK-if-missing-file 2>&1 >/dev/null
 159 }
 160
 161 par_regexp_chars_in_template() {
 162     echo '### Test regexp chars in template'
 163     seq 1 6 | parallel -j1 -I :: -X echo 'a::b::^c::[.}c'
 164 }
 165
 166 par_test_m_X() {
 167     echo '### Test -m vs -X'
 168     (echo foo;echo bar;echo joe.gif) | parallel -j1 -km echo 1{}2{.}3 A{.}B{.}C
 169     (echo foo;echo bar;echo joe.gif) | parallel -j1 -kX echo 1{}2{.}3 A{.}B{.}C
 170     seq 1 6 | parallel -k printf '{}.gif\\n' | parallel -j1 -km echo a{}b{.}c{.}
 171     seq 1 6 | parallel -k printf '{}.gif\\n' | parallel -j1 -kX echo a{}b{.}c{.}
 172
 173     echo '### Test -q {.}'
 174     echo a | parallel -qX echo  "'"{.}"' "
 175     echo a | parallel -qX echo  "'{.}'"
 176 }
 177
 178 par_i_t() {
 179     echo '### Test -i'
 180     (echo a; echo END; echo b) | parallel -k -i -eEND echo repl{.}ce
 181
 182     echo '### Test --replace'
 183     (echo a; echo END; echo b) | parallel -k --replace -eEND echo repl{.}ce
 184
 185     echo '### Test -t'
 186     (echo b; echo c; echo f) | parallel -k -t echo {.}ar 2>&1 >/dev/null
 187
 188     echo '### Test --verbose'
 189     (echo b; echo c; echo f) | parallel -k --verbose echo {.}ar 2>&1 >/dev/null
 190 }
 191
 192 par_pipe_float_blocksize() {
 193     echo '### Test --block <<non int>>'
 194     seq 5 | parallel --block 3.1 --pipe wc
 195 }
 196
 197 par_opt_gnu() {
 198     echo '### Test --tollef'
 199     stdout parallel -k --tollef echo -- 1 2 3 ::: a b c | LC_ALL=C sort
 200
 201     echo '### Test --tollef --gnu'
 202     stdout parallel -k --tollef --gnu echo ::: 1 2 3 -- a b c
 203
 204     echo '### Test --gnu'
 205     parallel -k --gnu echo ::: 1 2 3 -- a b c
 206 }
 207
 208 par_colsep_default() {
 209     echo "bug #37956: --colsep does not default to '\t' as specified in the man page."
 210     printf "A\tB\n1\tone" | parallel --header : echo {B} {A}
 211 }
 212
 213 par_tmux_command_not_found() {
 214     echo '### PARALLEL_TMUX not found'
 215     PARALLEL_TMUX=not-existing parallel --tmux echo ::: 1
 216 }
 217
 218 par_echo_jobseq() {
 219      echo '### bug #44995: parallel echo {#} ::: 1 2 ::: 1 2'
 220
 221      parallel -k echo {#} ::: 1 2 ::: 1 2
 222 }
 223
 224 par_no_joblog_with_dryrun() {
 225     echo 'bug #46016: --joblog should not log when --dryrun'
 226
 227     parallel --dryrun --joblog - echo ::: Only_this
 228 }
 229
 230 par_tagstring_with_d() {
 231     echo 'bug #47002: --tagstring with -d \n\n'
 232
 233     (seq 3; echo; seq 4) |
 234         parallel -k -d '\n\n' --tagstring {%} echo ABC';'echo
 235 }
 236
 237 par_xargs_nul_char_in_input() {
 238     echo 'bug #47290: xargs: Warning: a NUL character occurred in the input'
 239
 240     perl -e 'print "foo\0not printed"' | parallel echo
 241 }
 242
 243 par_maxproc() {
 244     echo '### Test --max-procs and -P: Number of processes'
 245
 246     seq 1 10 | parallel -k --max-procs +0 echo max proc
 247     seq 1 10 | parallel -k -P 200% echo 200% proc
 248 }
 249
 250 par_maxchar_s() {
 251     echo '### Test --max-chars and -s: Max number of chars in a line'
 252
 253     (echo line 1;echo line 1;echo line 2) | parallel -k --max-chars 25 -X echo
 254     (echo line 1;echo line 1;echo line 2) | parallel -k -s 25 -X echo
 255 }
 256
 257 par_no_run_if_empty() {
 258     echo '### Test --no-run-if-empty and -r: This should give no output'
 259
 260     echo "  " | parallel -r echo
 261     echo "  " | parallel --no-run-if-empty echo
 262 }
 263
 264 par_help() {
 265     echo '### Test --help and -h: Help output (just check we get the same amount of lines)'
 266
 267     echo Output from -h and --help
 268     parallel -h | wc -l
 269     parallel --help | wc -l
 270 }
 271
 272 par_version() {
 273     echo '### Test --version: Version output (just check we get the same amount of lines)'
 274
 275     parallel --version | wc -l
 276 }
 277
 278 par_verbose_t() {
 279     echo '### Test --verbose and -t'
 280
 281     (echo b; echo c; echo f) | parallel -k -t echo {}ar 2>&1 >/dev/null
 282     (echo b; echo c; echo f) | parallel -k --verbose echo {}ar 2>&1 >/dev/null
 283 }
 284
 285 par_test_zero_args() {
 286     echo '### Test 0-arguments'
 287
 288     seq 1 2 | parallel -k -n0 echo n0
 289     seq 1 2 | parallel -k -L0 echo L0
 290     seq 1 2 | parallel -k -N0 echo N0
 291 }
 292
 293 par_l0_is_l1() {
 294     echo '### Because of --tollef -l, then -l0 == -l1, sorry'
 295
 296     seq 1 2 | parallel -k -l0 echo l0
 297 }
 298
 299 par_replace_replacementstring() {
 300     echo '### Test replace {}'
 301
 302     seq 1 2 | parallel -k -N0 echo replace {} curlies
 303 }
 304
 305 par_arguments_on_cmdline() {
 306     echo '### Test arguments on commandline'
 307
 308     parallel -k -N0 echo args on cmdline ::: 1 2
 309 }
 310
 311 par_nice_locally() {
 312     echo '### Test --nice locally'
 313
 314     parallel --nice 1 -vv 'PAR=a bash -c "echo  \$PAR {}"' ::: b
 315 }
 316
 317 par_disk_full() {
 318     echo '### Disk full'
 319
 320     SMALLDISK=${SMALLDISK:-/mnt/ram}
 321     export SMALLDISK
 322     (
 323         cd /tmp
 324         sudo umount -l smalldisk.img
 325         dd if=/dev/zero of=smalldisk.img bs=100k count=1k
 326         yes|mkfs smalldisk.img
 327         sudo mkdir -p /mnt/ram
 328         sudo mount smalldisk.img /mnt/ram
 329         sudo chmod 777 /mnt/ram
 330     ) >/dev/null 2>/dev/null
 331
 332     cat /dev/zero >$SMALLDISK/out
 333     parallel --tmpdir $SMALLDISK echo ::: OK
 334
 335     rm $SMALLDISK/out
 336
 337     sudo umount -l /tmp/smalldisk.img
 338 }
 339
 340 par_delimiter() {
 341     echo '### Test --delimiter and -d: Delimiter instead of newline'
 342
 343     echo '# Yes there is supposed to be an extra newline for -d N'
 344     echo line 1Nline 2Nline 3 | parallel -k -d N echo This is
 345     echo line 1Nline 2Nline 3 | parallel -k --delimiter N echo This is
 346     printf "delimiter NUL line 1\0line 2\0line 3" | parallel -k -d '\0' echo
 347     printf "delimiter TAB line 1\tline 2\tline 3" | parallel -k --delimiter '\t' echo
 348 }
 349
 350 par_argfile() {
 351     echo '### Test -a and --arg-file: Read input from file instead of stdin'
 352
 353     tmp=$(mktemp)
 354     seq 1 10 >$tmp
 355     parallel -k -a $tmp echo
 356     parallel -k --arg-file $tmp echo
 357     rm $tmp
 358 }
 359
 360 par_pipe_unneeded_procs() {
 361     echo '### Test bug #34241: --pipe should not spawn unneeded processes'
 362     seq 3 |
 363         parallel -j30 --pipe --block-size 10 cat\;echo o 2> >(grep -Ev 'Warning: Starting|Warning: Consider')
 364 }
 365
 366 par_results_arg_256() {
 367     echo '### bug #42089: --results with arg > 256 chars (should be 1 char shorter)'
 368     parallel --results parallel_test_dir echo ::: 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456;
 369     ls parallel_test_dir/1/
 370     rm -rf parallel_test_dir
 371 }
 372
 373 par_pipe_to_func() {
 374     echo '### bug #45998: --pipe to function broken'
 375
 376     myfunc() { echo $1; cat; }
 377     export -f myfunc
 378     echo pipefunc OK | parallel --pipe myfunc {#}
 379     echo pipefunc and more OK | parallel --pipe 'myfunc {#};echo and more OK'
 380 }
 381
 382 par_roundrobin_k() {
 383     echo '### Test -k --round-robin'
 384     seq 1000000 | parallel -j4 -k --round-robin --pipe wc
 385 }
 386
 387 par_pipepart_roundrobin() {
 388     echo '### bug #45769: --round-robin --pipepart gives wrong results'
 389
 390     tmp=$(mktemp)
 391     seq 10000 >$tmp
 392     parallel -j2 --pipepart -a $tmp --block 14 --round-robin wc | wc -l
 393     rm $tmp
 394 }
 395
 396 par_pipepart_header() {
 397     echo '### bug #44614: --pipepart --header off by one'
 398
 399     tmp=$(mktemp)
 400     seq 10 >$tmp
 401     parallel --pipepart -a $tmp -k --block 5 'echo foo; cat'
 402     parallel --pipepart -a $tmp -k --block 2 --regexp --recend 3'\n' 'echo foo; cat'
 403     rm $tmp
 404 }
 405
 406 par_quote() {
 407     echo '### Test -q'
 408     parallel -kq perl -e '$ARGV[0]=~/^\S+\s+\S+$/ and print $ARGV[0],"\n"' ::: "a b" c "d e f" g "h i"
 409
 410     echo '### Test -q {#}'
 411     parallel -kq echo {#} ::: a b
 412     parallel -kq echo {\#} ::: a b
 413     parallel -kq echo {\\#} ::: a b
 414 }
 415
 416 par_read_from_stdin() {
 417     echo '### Test empty line as input'
 418     echo | parallel echo empty input line
 419
 420     echo '### Tests if (cat | sh) works'
 421     perl -e 'for(1..25) {print "echo a $_; echo b $_\n"}' | parallel 2>&1 | sort
 422
 423     echo '### Test if xargs-mode works'
 424     perl -e 'for(1..25) {print "a $_\nb $_\n"}' | parallel echo 2>&1 | sort
 425 }
 426
 427 par_total_from_joblog() {
 428     echo 'bug #47086: [PATCH] Initialize total_completed from joblog'
 429     tmp=$(mktemp)
 430     parallel -j1 --joblog $tmp --halt now,fail=1          echo '{= $_=$Global::total_completed =};exit {}' ::: 0 0 0 1 0 0
 431     parallel -j1 --joblog $tmp --halt now,fail=1 --resume echo '{= $_=$Global::total_completed =};exit {}' ::: 0 0 0 1 0 0
 432     rm $tmp
 433 }
 434
 435 par_xapply() {
 436     echo '### Test bug #43284: {%} and {#} with --xapply'
 437     parallel --xapply 'echo {1} {#} {%} {2}' ::: a ::: b
 438     parallel -N2 'echo {%}' ::: a b
 439
 440     echo '### bug #47501: --xapply for some input sources'
 441     # Wrapping does not work yet
 442     parallel -k echo ::: a b c aWRAP :::+ aa bb cc ::: A B :::+ AA BB AAwrap
 443 }
 444
 445 par_exit_val() {
 446     echo '### Test bug #45619: "--halt" erroneous error exit code (should give 0)'
 447     seq 10 | parallel --halt now,fail=1 true
 448     echo $?
 449
 450     echo '### Test exit val - true'
 451     echo true | parallel
 452     echo $?
 453
 454     echo '### Test exit val - false'
 455     echo false | parallel
 456     echo $?
 457 }
 458
 459 par_long_cmd_mem_use() {
 460     echo '### Test long commands do not take up all memory'
 461
 462     seq 1 100 |
 463         parallel -j0 -qv perl -e '$r=rand(shift); for($f = 0; $f < $r; $f++){ $a = "a"x100 } print shift,"\n"' 10000 2>/dev/null |
 464         sort
 465 }
 466
 467 par_test_L_context_replace() {
 468     echo '### Test -N context replace'
 469
 470     seq 19 | parallel -k -N 10  echo a{}b
 471
 472     echo '### Test -L context replace'
 473
 474     seq 19 | parallel -k -L 10  echo a{}b
 475 }
 476
 477 par_test_r_with_pipe() {
 478     echo '### Test of -r with --pipe - the first should give an empty line. The second should not.'
 479
 480     echo | parallel  -j2 -N1 --pipe cat | wc -l
 481     echo | parallel -r -j2 -N1 --pipe cat | wc -l
 482 }
 483
 484 par_test_tty() {
 485     echo '### Test --tty'
 486
 487     seq 0.1 0.1 0.5 | parallel -j1 --tty tty\;sleep
 488 }
 489
 490 par_no_command_given() {
 491     echo '### Test bugfix if no command given'
 492
 493     (echo echo; seq 1 5; perl -e 'print "z"x1000000'; seq 12 15) |
 494         stdout parallel -j1 -km -s 10
 495 }
 496
 497
 498 par_inefficient_L() {
 499     echo "bug #37325: Inefficiency of --pipe -L"
 500
 501     seq 2000 | parallel -k --pipe --block 1k -L 4 wc\;echo FOO | uniq
 502 }
 503
 504 par_pipe_record_size_in_lines() {
 505     echo "bug #34958: --pipe with record size measured in lines"
 506
 507     seq 10 | parallel -k --pipe -L 4 cat\;echo bug 34958-1
 508     seq 10 | parallel -k --pipe -l 4 cat\;echo bug 34958-2
 509 }
 510
 511 par_pipe_no_command() {
 512     echo '### --pipe without command'
 513
 514     seq -w 10 | stdout parallel --pipe
 515 }
 516
 517 par_expansion_in_colsep() {
 518     echo '### bug #36260: {n} expansion in --colsep files fails for empty fields if all following fields are also empty'
 519
 520     echo A,B,, | parallel --colsep , echo {1}{3}{2}
 521 }
 522
 523 par_extglob() {
 524     bash -O extglob -c '. `which env_parallel.bash`;
 525       _longopt () {
 526         case "$prev" in
 527           --+([-a-z0-9_]))
 528             echo foo;;
 529         esac;
 530       };
 531       env_parallel echo ::: env_parallel 2>&1
 532     '
 533 }
 534
 535 par_tricolonplus() {
 536     echo '### bug #48745: :::+ bug'
 537
 538     parallel -k echo ::: 11 22 33 ::::+ <(seq 3) <(seq 21 23) ::: a b c :::+ aa bb cc
 539     parallel -k echo :::: <(seq 3) <(seq 21 23) :::+ a b c ::: aa bb cc
 540     parallel -k echo :::: <(seq 3) :::: <(seq 21 23) :::+ a b c ::: aa bb cc
 541 }
 542
 543 par_colsep_0() {
 544     echo 'bug --colsep 0'
 545
 546     parallel --colsep 0 echo {2} ::: a0OK0c
 547     parallel --header : --colsep 0 echo {ok} ::: A0ok0B a0OK0b
 548 }
 549
 550 par_empty() {
 551     echo "bug #:"
 552
 553     parallel echo ::: true
 554 }
 555
 556 par_empty_line() {
 557     echo '### Test bug: empty line for | sh with -k'
 558     (echo echo a ; echo ; echo echo b) | parallel -k
 559 }
 560
 561 par_append_joblog() {
 562     echo '### can you append to a joblog using +'
 563     tmp=$(mktemp)
 564     parallel --joblog $tmp echo ::: 1
 565     parallel --joblog +$tmp echo ::: 1
 566     wc -l < $tmp
 567     rm $tmp
 568 }
 569
 570 par_file_ending_in_newline() {
 571     echo '### Hans found a bug giving unitialized variable'
 572     echo >/tmp/parallel_f1
 573     echo >/tmp/parallel_f2'
 574 '
 575     echo /tmp/parallel_f1 /tmp/parallel_f2 |
 576         stdout parallel -kv --delimiter ' ' gzip
 577     rm /tmp/parallel_f*
 578 }
 579
 580 par_python_children() {
 581     echo '### bug #49970: Python child process dies if --env is used'
 582     fu() { echo joe; }
 583     export -f fu
 584     echo foo | stdout parallel --env fu python -c \
 585     \""import os; f = os.popen('uname -p'); output = f.read(); rc = f.close()"\"
 586 }
 587
 588 par_pipepart_block_bigger_2G() {
 589     echo '### Test that --pipepart can have blocks > 2GB'
 590     tmp=$(mktemp)
 591     echo foo >$tmp
 592     parallel --pipepart -a $tmp --block 3G wc
 593     rm $tmp
 594 }
 595
 596 par_retries_replacement_string() {
 597     tmp=$(mktemp)
 598     parallel --retries {//} "echo {/} >>$tmp;exit {/}" ::: 1/11 2/22 3/33
 599     sort $tmp
 600     rm $tmp
 601 }
 602
 603 par_tee() {
 604     export PARALLEL='-k --tee --pipe --tag'
 605     seq 1000000 | parallel 'echo {%};LC_ALL=C wc' ::: {1..5} ::: {a..b}
 606     seq 300000 | parallel 'grep {1} | LC_ALL=C wc {2}' ::: {1..5} ::: -l -c
 607 }
 608
 609 par_tagstring_pipe() {
 610     echo 'bug #50228: --pipe --tagstring broken'
 611     seq 3000 | parallel -j4 --pipe -N1000 -k --tagstring {%} LC_ALL=C wc
 612 }
 613
 614 par_link_files_as_only_arg() {
 615     echo 'bug #50685: single ::::+ does not work'
 616     parallel -k echo ::::+ <(seq 10) <(seq 3) <(seq 4)
 617 }
 618
 619 par_basic_halt() {
 620     cpuburn=$(tempfile)
 621     cpuburn2=$(tempfile)
 622     (echo '#!/usr/bin/perl'
 623      echo "eval{setpriority(0,0,9)}; while(1){}") > $cpuburn
 624     chmod 700 $cpuburn
 625     cp -a $cpuburn $cpuburn2
 626
 627     parallel -j4 --halt 2 ::: 'sleep 1' $cpuburn false;
 628     killall $(basename $cpuburn) 2>/dev/null &&
 629         echo ERROR: cpuburn should already have been killed
 630     parallel -j4 --halt -2 ::: 'sleep 1' $cpuburn2 true;
 631     killall $(basename $cpuburn2) 2>/dev/null &&
 632         echo ERROR: cpuburn2 should already have been killed
 633     rm $cpuburn $cpuburn2
 634
 635     parallel --halt error echo ::: should not print
 636     parallel --halt soon echo ::: should not print
 637     parallel --halt now echo ::: should not print
 638 }
 639
 640 par_newline_in_command() {
 641     echo Command with newline and positional replacement strings
 642     parallel "
 643       echo {1
 644       } {2}
 645     " ::: O ::: K
 646 }
 647
 648 par_wd_3dot_local() {
 649     echo 'bug #45993: --wd ... should also work when run locally'
 650
 651     (
 652         parallel --wd /bi 'pwd; echo $OLDPWD; echo' ::: fail
 653         parallel --wd /bin 'pwd; echo $OLDPWD; echo' ::: OK
 654         parallel --wd / 'pwd; echo $OLDPWD; echo' ::: OK
 655         parallel --wd /tmp 'pwd; echo $OLDPWD; echo' ::: OK
 656         parallel --wd ... 'pwd; echo $OLDPWD; echo' ::: OK
 657         parallel --wd . 'pwd; echo $OLDPWD; echo' ::: OK
 658     ) |
 659         perl -pe 's:/mnt/4tb::; s:/home/tange:~:;' |
 660         perl -pe 's:parallel./:parallel/:;' |
 661         perl -pe 's/'`hostname`'/hostname/g; s/\d+/0/g'
 662 }
 663
 664 par_X_eta_div_zero() {
 665     echo '### bug #34422: parallel -X --eta crashes with div by zero'
 666
 667     # We do not care how long it took
 668     seq 2 | stdout parallel -X --eta echo |
 669         grep -E -v 'ETA:.*AVG' |
 670         perl -pe 's/\d+/0/g' |
 671         perl -pe 's/Comp.* to complete//' |
 672         perl -ne '/../ and print'
 673 }
 674
 675 par_parcat_args_stdin() {
 676     echo 'bug #51690: parcat: read args from stdin'
 677     tmp1=$(tempfile)
 678     tmp2=$(tempfile)
 679     echo OK1 > $tmp1
 680     echo OK2 > $tmp2
 681     (echo $tmp1
 682      echo $tmp2) | parcat | sort
 683     rm $tmp1 $tmp2
 684 }
 685
 686 par_parcat_rm() {
 687     echo 'bug #51691: parcat --rm remove fifo when opened'
 688     tmp1=$(tempfile)
 689     echo OK1 > $tmp1
 690     parcat --rm $tmp1
 691     rm $tmp1 2>/dev/null || echo OK file removed
 692 }
 693
 694 par_linebuffer_files() {
 695     echo '### bug #48658: --linebuffer --files'
 696
 697     stdout parallel --files --linebuffer 'sleep .1;seq {};sleep .1' ::: {1..10} | wc -l
 698 }
 699
 700 par_halt_one_job() {
 701     echo '# Halt soon if there is a single job'
 702     echo should run 0 1 = job 1 2
 703     parallel -j1 --halt now,fail=1 'echo {#};exit {}' ::: 0 1 0
 704     echo should run 1 = job 1
 705     parallel -j1 --halt now,fail=1 'echo {#};exit {}' ::: 1 0 1
 706     echo should run 0 1 = job 1 2
 707     parallel -j1 --halt soon,fail=1 'echo {#};exit {}' ::: 0 1 0
 708     echo should run 1 = job 1
 709     parallel -j1 --halt soon,fail=1 'echo {#};exit {}' ::: 1 0 1
 710 }
 711
 712 par_blocking_redir() {
 713     (
 714         echo 'bug #52740: Bash redirection with process substitution blocks'
 715         echo Test stdout
 716         echo 3 | parallel seq > >(echo stdout;wc) 2> >(echo stderr >&2; wc >&2)
 717         echo Test stderr
 718         echo nOfilE | parallel ls > >(echo stdout;wc) 2> >(echo stderr >&2; wc >&2)
 719     ) 2>&1 | LC_ALL=C sort
 720 }
 721
 722 par_pipepart_recend_recstart() {
 723     echo 'bug #52343: --recend/--recstart does wrong thing with --pipepart'
 724     tmp1=$(tempfile)
 725     seq 10 > $tmp1
 726     parallel -k --pipepart -a $tmp1 --recend '\n' --recstart '6' --block 1 'echo a; cat'
 727     parallel -k --pipe < $tmp1 --recend '\n' --recstart '6' --block 1 'echo a; cat'
 728     rm $tmp1 2>/dev/null
 729 }
 730
 731 par_pipe_tag_v() {
 732     echo 'pipe with --tag -v'
 733     seq 3 | parallel -v --pipe --tagstring foo cat
 734     # This should only give the filename
 735     seq 3 | parallel -v --pipe --tagstring foo --files cat |
 736         perl -pe 's:/tmp/par.*.par:/tmp/tmpfile.par:'
 737 }
 738
 739 par_dryrun_append_joblog() {
 740     echo '--dry-run should not append to joblog'
 741     tmp=$(mktemp)
 742     parallel -k --jl $tmp echo ::: 1 2 3
 743     parallel --dryrun -k --jl +$tmp echo ::: 1 2 3 4
 744     # Job 4 should not show up: 3 lines + header = 4
 745     wc -l < $tmp
 746     rm $tmp
 747 }
 748
 749 par_0_no_newline() {
 750     echo 'A single zero without \n should not be ignored'
 751     echo -n 0 | parallel echo
 752 }
 753
 754 par_csv() {
 755     (echo '"col1""x3""","new'
 756      echo 'line col2","new2'
 757      echo 'line col3",col 4') |
 758         parallel --csv echo {1}-{2}-{3}-{4}
 759     echo '"2""x3"" board","Value with ,",Column 3' |
 760         parallel --csv echo {1}-{2}-{3}
 761 }
 762
 763 par_csv_pipe() {
 764     echo 'Only pass full records to tail'
 765     echo 'Too small block size'
 766     perl -e 'for $b(1..10) {
 767            print join",", map {"\"$_\n$_\""} $b*1000..$b*1000+1000;
 768            print "\n"
 769          }' |
 770         stdout parallel --pipe --csv -k --block 10k tail -n1 |
 771         sort -n
 772
 773     echo 'More records in single block'
 774     perl -e 'for $b(1..10) {
 775            print join",", map {"\"$_\n$_\""} $b*1000..$b*1000+1000;
 776            print "\n"
 777          }' |
 778         stdout parallel --pipe --csv -k --block 100k tail -n1 |
 779         sort -n
 780 }
 781
 782 par_slow_pipe_regexp() {
 783     echo "### bug #53718: --pipe --regexp -N blocks"
 784     echo This should take a few ms, but took more than 2 hours
 785     seq 54000 80000 |
 786         parallel -N1000 --regexp --pipe --recstart 4 --recend 5 -k wc
 787     echo "### These should give same output"
 788     seq 54000 80000 |
 789         parallel -N1000 --regexp --pipe --recstart 4 --recend 5 -k cat |
 790         md5sum
 791     seq 54000 80000 | md5sum
 792 }
 793
 794 par_results() {
 795     echo "### --results test.csv"
 796     tmp=$(mktemp)
 797     parallel -k --results "$tmp"-dir echo ::: a b c
 798     rm -r $tmp "$tmp"-dir
 799 }
 800
 801 par_testquote() {
 802     testquote() {
 803         printf '"#&/\n()*=?'"'" |
 804             PARALLEL_SHELL=$1 parallel -0 echo
 805     }
 806     export -f testquote
 807     parallel --tag -k testquote ::: ash bash csh dash fdsh fish fizsh ksh ksh93 mksh posh rbash rc rzsh sash sh static-sh tcsh yash zsh
 808 }
 809
 810 par_locale_quoting() {
 811     echo "### quoting in different locales"
 812     printf '\243`/tmp/test\243`\n'
 813     printf '\243`/tmp/test\243`\n' | LC_ALL=zh_HK.big5hkscs xargs echo '$LC_ALL'
 814     # LC_ALL should be zh_HK.big5hkscs, but that makes quoting hard.
 815     printf '\243`/tmp/test\243`\n' | LC_ALL=zh_HK.big5hkscs parallel -v echo '$LC_ALL'
 816 }
 817
 818 par_PARALLEL_ENV() {
 819     echo '### PARALLEL_ENV as variable'
 820     PARALLEL_ENV="v='OK as variable'" parallel {} '$v' ::: echo
 821     PARALLEL_ENV=$(mktemp)
 822     echo '### PARALLEL_ENV as file'
 823     echo "v='OK as file'" > $PARALLEL_ENV
 824     PARALLEL_ENV="$PARALLEL_ENV" parallel {} '$v' ::: echo
 825     echo '### PARALLEL_ENV as fifo'
 826     rm $PARALLEL_ENV
 827     mkfifo $PARALLEL_ENV
 828     # () needed to avoid [1]+  Done
 829     (echo "v='OK as fifo'" > $PARALLEL_ENV &) 2>/dev/null
 830     PARALLEL_ENV="$PARALLEL_ENV" parallel {} '$v' ::: echo
 831     rm $PARALLEL_ENV
 832 }
 833
 834 par_pipe_recend() {
 835     echo 'bug #54328: --pipe --recend '' blocks'
 836     seq 3 | parallel -k --pipe --regexp --recend '' -n 1 xxd
 837     seq 3 | parallel -k --pipe --recend '' -n 1 xxd
 838 }
 839
 840 par_perlexpr_with_newline() {
 841     echo 'Perl expression spanning 2 lines'
 842     mkdir -p tmp
 843     cd tmp
 844     touch "Dad's \"famous\" 1' pizza"
 845     # Important with newline in perl expression:
 846     parallel mv {} '{= $a=pQ($_); $b=$_;
 847       $_=qx{date -r "$a" +%FT%T}; chomp; $_="$_ $b" =}' \
 848          ::: "Dad's \"famous\" 1' pizza"
 849     rm *"Dad's \"famous\" 1' pizza"
 850 }
 851
 852 par_empty_command() {
 853     echo 'bug #54647: parset ignores empty lines'
 854     # really due to this. Should give an empty line due to -v:
 855     parallel -v :::: <(echo)
 856     . `which env_parallel.bash`
 857     parset a,b,c :::: <(echo echo A; echo; echo echo C)
 858     echo Empty: $b
 859     parset a,b,c :::: <(echo echo A; echo echo B; echo echo C)
 860     echo B: $b
 861 }
 862
 863 par_empty_input_on_stdin() {
 864     echo 'https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=910470'
 865     echo 'This should give no output'
 866     true | stdout parallel --shuf echo
 867 }
 868
 869 par_space_envvar() {
 870     echo "### bug: --gnu was ignored if env var started with space: PARALLEL=' --gnu'"
 871     export PARALLEL=" -v" && parallel echo ::: 'space in envvar OK'
 872 }
 873
 874 par_pipe_N1_regexp() {
 875     echo 'bug #55131: --regexp --recstart hangs'
 876     echo "These should give the same"
 877     printf 'begin\n%send\n' '' a b c |
 878         parallel -kN1 --recstart 'begin\n' --pipe --regexp echo JOB{#}\;cat\;echo END
 879     printf 'begin\n%send\n' '' a b c |
 880         parallel -kN1 --recstart 'begin\n' --pipe          echo JOB{#}\;cat\;echo END
 881 }
 882
 883 par_sem_quote() {
 884     echo '### sem --quote should not add empty argument'
 885     sem --id sem_quote --fg --quote -v echo
 886 }
 887
 888 par_halt_on_error_division_by_zero() {
 889     echo '### --halt-on-error soon,fail=100% with no input should not give division by zero'
 890     stdout parallel --halt-on-error soon,fail=100% echo </dev/null
 891     echo $?
 892 }
 893
 894 par_wd_dotdotdot() {
 895     echo '### parallel --wd ... should clean up'
 896     parallel --wd ... 'pwd;true' ::: foo | parallel ls 2>/dev/null
 897     echo $? == 1
 898     echo '### $OLDPWD should be the dir in which parallel starts'
 899     cd /tmp
 900     parallel --wd ... 'echo $OLDPWD' ::: foo
 901 }
 902
 903 par_fish() {
 904     echo '### https://github.com/fish-shell/fish-shell/issues/5582'
 905     echo OK | stdout fish -c 'parallel --pipe cat'
 906 }
 907
 908 par_jobslot_jobnumber_pipe() {
 909     echo '### Test bug #43376: {%} and {#} with --pipe'
 910     echo foo | parallel -q --pipe -k echo {#}
 911     echo foo | parallel --pipe -k echo {%}
 912     echo foo | parallel -q --pipe -k echo {%}
 913     echo foo | parallel --pipe -k echo {#}
 914 }
 915
 916 par_replacement_string_as_part_of_command() {
 917     echo '### {} as part of the command'
 918     echo p /bin/ls | parallel l{= s/p/s/ =}
 919     echo /bin/ls-p | parallel --colsep '-' l{=2 s/p/s/ =} {1}
 920     echo s /bin/ls | parallel l{}
 921     echo /bin/ls | parallel ls {}
 922     echo ls /bin/ls | parallel {}
 923     echo ls /bin/ls | parallel
 924 }
 925
 926 par_japanese_chars_in_replacement_string() {
 927     echo '### bug #43817: Some JP char cause problems in positional replacement strings'
 928     parallel -k echo ::: '�<�>' '�<1 $_=2�>' 'ワ'
 929     parallel -k echo {1} ::: '�<�>' '�<1 $_=2�>' 'ワ'
 930     parallel -Xj1 echo ::: '�<�>' '�<1 $_=2�>' 'ワ'
 931     parallel -Xj1 echo {1} ::: '�<�>' '�<1 $_=2�>' 'ワ'
 932 }
 933
 934 par_rpl_that_is_substring_of_longer_rpl() {
 935     echo '### --rpl % that is a substring of longer --rpl %D'
 936     parallel --rpl '{+.} s:.*\.::' --rpl '%' \
 937              --rpl '%D $_=::shell_quote(::dirname($_));' \
 938              --rpl '%B s:.*/::;s:\.[^/.]+$::;' \
 939              --rpl '%E s:.*\.::' \
 940              'echo {}=%;echo %D={//};echo %B={/.};echo %E={+.};echo %D/%B.%E={}' ::: a.b/c.d/e.f
 941 }
 942
 943 par_unquote_replacement_string() {
 944     echo '### Can part of the replacement string be unquoted using uq()?'
 945     parallel echo '{}{=uq()=}' ::: '`echo foo`'
 946 }
 947
 948 par_delimiter_space() {
 949     echo '### Does space as delimiter work?'
 950     parallel -k -d " " echo ::: "1 done"
 951 }
 952
 953 par_recend_not_regexp() {
 954     echo '### bug #56558: --rrs with --recend that is not regexp'
 955     echo 'a+b' | parallel -k --pipe --rrs --recend '+' -N1 'cat;echo end'
 956 }
 957
 958 par_profile() {
 959     echo '### Test -J profile, -J /dir/profile, -J ./profile'
 960     echo --tag > testprofile_local
 961     parallel -J ./testprofile_local echo ::: local
 962     rm testprofile_local
 963     echo --tag > testprofile_abs
 964     parallel -J `pwd`/testprofile_abs echo ::: abs
 965     rm testprofile_abs
 966     echo --tag > ~/.parallel/testprofile_config
 967     parallel -J testprofile_config echo ::: config
 968     rm ~/.parallel/testprofile_config
 969 }
 970
 971 par_cr_newline_header() {
 972     echo '### --header : should set named replacement string if input line ends in \r\n'
 973     printf "foo\r\nbar\r\n" |
 974         parallel --colsep , --header : echo {foo}
 975 }
 976
 977 par_PARALLEL_HOME_with_+() {
 978     echo 'bug #59453: PARALLEL_HOME with plus sign causes error: config not readable'
 979     tmp=$(mktemp -d)
 980     export PARALLEL_HOME="$tmp/  space  /a+b"
 981     mkdir -p "$PARALLEL_HOME"
 982     parallel echo ::: Parallel_home_with+
 983     rm -rf "$tmp"
 984 }
 985
 986 par_group-by_colsep_space() {
 987     echo '### --colsep " " should work like ","'
 988     input() {
 989         sep="$1"
 990         printf "a\t${sep}b\n"
 991         printf "a${sep}${sep}b\n"
 992         printf "b${sep}${sep}a\n"
 993         printf "b${sep}a${sep}b\n"
 994     }
 995     input ',' | parallel --pipe --group-by 2 --colsep ',' -kN1 wc
 996     input ' ' | parallel --pipe --group-by 2 --colsep ' ' -kN1 wc
 997 }
 998
 999 par_json() {
1000     printf '"\t\\"' | parallel -k --results -.json echo :::: - ::: '"' '\\' |
1001         perl -pe 's/\d/0/g'
1002 }
1003
1004 par_hash_and_time_functions() {
1005     echo '### Functions for replacement string'
1006     parallel echo '{= $_=join(" ",
1007                               yyyy_mm_dd_hh_mm_ss(),
1008                               yyyy_mm_dd_hh_mm(),
1009                               yyyy_mm_dd(),
1010                               yyyymmddhhmmss(),
1011                               yyyymmddhhmm(),
1012                               yyyymmdd()) =}' ::: 1 |
1013         perl -pe 's/\d/9/g'
1014     parallel echo '{= $_=hash($_) =}' ::: 1 |
1015         perl -pe 's/[a-f0-9]+/X/g'
1016 }
1017
1018 export -f $(compgen -A function | grep par_)
1019 compgen -A function | grep par_ | LC_ALL=C sort |
1020     parallel --timeout 1000% -j6 --tag -k --joblog /tmp/jl-`basename $0` '{} 2>&1' |
1021     perl -pe 's:/usr/bin:/bin:g'