From bb2c22b0d477bbf45ae481eb71173654add7656b Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Thu, 22 Oct 2020 21:47:01 +0200 Subject: [PATCH] parallel: --(n)onall and --sshdelay fixed. --- doc/release_new_version | 19 +- packager/releasescripts/updateversion | 2 + src/parallel | 43 ++-- src/parallel.pod | 4 +- testsuite/Makefile | 50 ++-- testsuite/tests-to-run/parallel-centos3.sh | 14 +- testsuite/tests-to-run/parallel-local-0.3s.sh | 16 ++ testsuite/tests-to-run/parallel-local-10s.sh | 112 +++++++++ testsuite/tests-to-run/parallel-local-1s.sh | 21 ++ testsuite/tests-to-run/parallel-local-3s.sh | 127 +++++----- testsuite/tests-to-run/parallel-local-mem.sh | 25 +- testsuite/tests-to-run/parallel-local-sql01.sh | 3 +- testsuite/tests-to-run/parallel-local-ssh7.sh | 3 +- testsuite/tests-to-run/parallel-local-ssh8.sh | 8 +- testsuite/tests-to-run/parallel-local164.sh | 140 +---------- testsuite/tests-to-run/parallel-macos.sh | 10 +- testsuite/tests-to-run/parallel-virtualbox1.sh | 2 +- testsuite/tests-to-run/sql01.sh | 5 +- testsuite/tests-to-run/test60.sh | 2 +- testsuite/wanted-results/parallel-centos3 | 1 + testsuite/wanted-results/parallel-local-0.3s | 21 ++ testsuite/wanted-results/parallel-local-10s | 196 +++++++++++++++ testsuite/wanted-results/parallel-local-1s | 30 +++ testsuite/wanted-results/parallel-local-3s | 183 +++++--------- testsuite/wanted-results/parallel-local-mem | 114 +++------ testsuite/wanted-results/parallel-local-ssh1 | 2 +- testsuite/wanted-results/parallel-local-ssh7 | 18 +- testsuite/wanted-results/parallel-local-ssh8 | 4 +- testsuite/wanted-results/parallel-local164 | 317 +------------------------ testsuite/wanted-results/parallel-macos | 20 +- testsuite/wanted-results/parallel-tutorial | 7 +- testsuite/wanted-results/sql01 | 4 +- testsuite/wanted-results/test60 | 2 +- 33 files changed, 709 insertions(+), 816 deletions(-) rewrite testsuite/tests-to-run/parallel-local164.sh (92%) rewrite testsuite/wanted-results/parallel-local-mem (63%) rewrite testsuite/wanted-results/parallel-local164 (97%) diff --git a/doc/release_new_version b/doc/release_new_version index f239d208..a08cf9f1 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -192,24 +192,37 @@ from:tange@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org stable-bcc: Jesse Alama -Subject: GNU Parallel 20201022 ('CRISPR/Memphis-mumier/TrumpVirus/Belarus/Assange') released <<[stable]>> +Subject: GNU Parallel 20201022 ('Samuel Paty') released <<[stable]>> -GNU Parallel 20201022 ('') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ +GNU Parallel 20201022 ('Samuel Paty') <<[stable]>> has been released. It is available for download at: http://ftpmirror.gnu.org/parallel/ <> + +Please help spreading GNU Parallel by making a testimonial video like +Juan Sierra Pons: + +http://www.elsotanillo.net/wp-content/uploads/GnuParallel_JuanSierraPons.mp4 + +It does not have to be as detailed as Juan's. It is perfectly fine if +you just say your name, and what field you are using GNU Parallel for. + Quote of the month: New in this release: +* --termseq now works for remote running jobs. +* parsort: Performance optimized for 64-core machines. * Bug fixes and man page updates. News about GNU Parallel: -https://cloud.tencent.com/developer/article/1705499 +* Juan Sierra Pons has made a testimonial: http://www.elsotanillo.net/wp-content/uploads/GnuParallel_JuanSierraPons.mp4 + +* 「翻译」在生物信息学中使用 GNU-Parallel https://cloud.tencent.com/developer/article/1705499 Get the book: GNU Parallel 2018 http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html diff --git a/packager/releasescripts/updateversion b/packager/releasescripts/updateversion index db7df9ae..df13ff20 100755 --- a/packager/releasescripts/updateversion +++ b/packager/releasescripts/updateversion @@ -30,6 +30,8 @@ updater() { perl -i -pe "s/20\d\d\d\d\d\d/$YYYYMMDD/" configure.ac perl -i -pe "/version/ and s/20\d\d\d\d\d\d/$YYYYMMDD/" src/sql + perl -i -pe "/version/ and s/20\d\d\d\d\d\d/$YYYYMMDD/" src/parsort + perl -i -pe "/version/ and s/20\d\d\d\d\d\d/$YYYYMMDD/" src/env_parallel* perl -i -pe "/version/ and s/20\d\d\d\d\d\d/$YYYYMMDD/" src/niceload perl -i -pe "s/parallel-20\d\d\d\d\d\d/parallel-$YYYYMMDD/" README perl -i -pe ' diff --git a/src/parallel b/src/parallel index 3f719392..97682ba2 100755 --- a/src/parallel +++ b/src/parallel @@ -3906,7 +3906,9 @@ sub get_job_with_sshlogin($) { return undef; } } - $job->set_sshlogin($sshlogin); + if(not $job->suspended()) { + $job->set_sshlogin($sshlogin); + } if($opt::retries and $job->failed_here()) { # This command with these args failed for this sshlogin my ($no_of_failed_sshlogins,$min_failures) = $job->min_failed(); @@ -4529,6 +4531,7 @@ sub onall($@) { # -P should only go to the first, and -S should not be copied at all. my $options = join(" ", + ((defined $opt::sshdelay) ? "--delay ".$opt::sshdelay : ""), ((defined $opt::memfree) ? "--memfree ".$opt::memfree : ""), ((defined $opt::memsuspend) ? "--memfree ".$opt::memsuspend : ""), ((defined $opt::D) ? "-D $opt::D" : ""), @@ -4543,6 +4546,7 @@ sub onall($@) { ); my $suboptions = join(" ", + ((defined $opt::sshdelay) ? "--delay ".$opt::sshdelay : ""), ((defined $opt::D) ? "-D $opt::D" : ""), ((defined $opt::arg_file_sep) ? "--arg-file-sep ".$opt::arg_file_sep : ""), ((defined $opt::arg_sep) ? "--arg-sep ".$opt::arg_sep : ""), @@ -4813,12 +4817,8 @@ sub kill_sleep_seq(@) { # Convert pids to process groups ($processgroup = -$pid) my @pgrps = map { -$_ } @_; my @term_seq = split/,/,$opt::termseq; - if($opt::memsuspend) { - @term_seq = ("STOP",1); - } else { - if(not @term_seq) { - @term_seq = ("TERM",200,"TERM",100,"TERM",50,"KILL",25); - } + if(not @term_seq) { + @term_seq = ("TERM",200,"TERM",100,"TERM",50,"KILL",25); } while(@term_seq) { @pgrps = kill_sleep(shift @term_seq, shift @term_seq, @pgrps); @@ -5977,13 +5977,17 @@ sub kill_youngster_if_not_enough_mem() { @{$jobs_of{$sshlogin}})); ::debug("mem","\n", $job->seq(), "killed ", $sshlogin->memfree()," < ",$limit); - $job->kill(); + if($opt::memsuspend) { + $job->suspend(); + } else { + $job->kill(); + } $sshlogin->memfree_recompute(); } else { last; } } - ::debug("mem","Free mem OK ", + ::debug("mem","Free mem OK? ", $sshlogin->memfree()," > ",$limit); } } @@ -8631,11 +8635,15 @@ sub kill($) { my $self = shift; $self->set_exitstatus(-1); ::kill_sleep_seq($self->pid()); +} + +sub suspend($) { + my $self = shift; + my @pgrps = map { -$_ } $self->pid(); + kill "STOP", @pgrps; + $self->set_suspended(1); # push job onto start stack - if($opt::memsuspend) { - $self->{'suspended'} = 1; - $Global::JobQueue->{'commandlinequeue'}->unget($self); - } + $Global::JobQueue->unget($self); } sub set_suspended($$) { @@ -8648,6 +8656,13 @@ sub suspended($) { return $self->{'suspended'}; } +sub resume($) { + my $self = shift; + my @pgrps = map { -$_ } $self->pid(); + kill "CONT", @pgrps; + $self->set_suspended(0); +} + sub failed($) { # return number of times failed for this $sshlogin # Input: @@ -9662,7 +9677,7 @@ sub start($) { my $job = shift; if($job->suspended()) { # Job is kill -STOP'ped: Restart it. - kill "CONT", $job->pid(); + $job->resume(); return $job; } # Get the shell command to be executed (possibly with ssh infront). diff --git a/src/parallel.pod b/src/parallel.pod index 29e206f5..2c6a2037 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -510,7 +510,7 @@ basename of input line. Use the replacement string I instead of B<{/.}> for basename of input line without extension. -=item B<--bin> I (beta testing) +=item B<--bin> I Use I as binning key and bin input to the jobs. @@ -879,7 +879,7 @@ B<--group> is the default. Can be reversed with B<-u>. See also: B<--line-buffer> B<--ungroup> -=item B<--group-by> I +=item B<--group-by> I (beta testing) Group input by value. Combined with B<--pipe>/B<--pipepart> B<--group-by> groups lines with the same value into a record. diff --git a/testsuite/Makefile b/testsuite/Makefile index 8c13ddc7..840fe905 100644 --- a/testsuite/Makefile +++ b/testsuite/Makefile @@ -15,7 +15,8 @@ testsuite: 3 make stopvm # Mem -mem: ../src/parallel tests-to-run/*mem* wanted-results/*mem* prereqlocal +mem: ../src/parallel tests-to-run/*mem* wanted-results/*mem* + make stopvm || true time bash Start.sh mem NONE || true touch ~/.parallel/will-cite make stopvm @@ -65,33 +66,42 @@ portable: time bash Portable.sh ### Prerequisites +REQUIRE=bash -c '\ + is_installed() { \ + eval $$2 >/dev/null 2>/dev/null || (echo "$$1" is required for testsuite; /bin/false); \ + }; \ + export -f is_installed; \ + parallel -n2 --tag is_installed ::: \ + script "echo | script -c echo -q /dev/null" \ + lsh "lsh lo echo lsh is installed" \ + "ssh sh@lo" "timeout 20 ssh sh@lo true" \ + "ssh csh@lo" "timeout 5 ssh csh@lo true" \ + niceload "niceload echo niceload is installed" \ + expect "expect -c \"spawn cat; puts \\\"expect is installed\\\"\"" \ + autossh "autossh -V" \ + tcsh "tcsh -c echo tcsh installed" \ + seq "seq 1 2" \ + stdout "stdout echo stdout is installed" \ + convert "convert -version" \ + bc "echo 1+2 | bc" \ + gawk "echo | gawk \"{print \\\"gawk is installed\\\"}\"" \ + pv "echo | pv -qL 10" \ + timeout "timeout 1 echo timeout is installed" \ +' + prereqlocal: installparallel startvm - tcsh -c echo tcsh installed || (echo tcsh is required for testsuite; /bin/false) - seq 1 2 >/dev/null || (echo seq is required for testsuite; /bin/false) - stdout echo || (echo stdout is required for testsuite; /bin/false) - convert -version >/dev/null || (echo convert is required for testsuite; /bin/false) - echo 1+2 | bc >/dev/null || (echo bc is required for testsuite; /bin/false) - echo | gawk '{print "gawk is installed"}' || (echo gawk is required for testsuite; /bin/false) - expect -c 'spawn cat; puts "expect is installed"' || (echo expect is required for testsuite; /bin/false) - echo | pv -qL 10 || (echo pv is required for testsuite; /bin/false) - echo | script -c echo -q /dev/null || (echo script is required for testsuite; /bin/false) - niceload true || (echo niceload is required for testsuite; /bin/false) - which timeout || (echo timeout is required for testsuite; /bin/false) - which autossh || (echo autossh is required for testsuite; /bin/false) - which lsh || (echo lsh is required for testsuite; /bin/false) - timeout 20 ssh sh@lo true || (echo ssh sh@lo is required for testsuite; /bin/false) - timeout 5 ssh csh@lo true || (echo ssh csh@lo is required for testsuite; /bin/false) - parallel -j1 ssh-keygen -R parallel-server{} ::: 1 2 3 || true - parallel 'ssh vagrant@parallel-server{} mkdir -p bin; scp `which parallel` vagrant@parallel-server{}:bin/' ::: 1 2 3 4 + ${REQUIRE} + parallel -j1 ssh-keygen -R parallel-server{} ::: 1 2 3 4 || true + (parallel 'ssh vagrant@parallel-server{} mkdir -p bin; scp `which parallel` vagrant@parallel-server{}:bin/' ::: 1 2 3 4)& prereqremote: installparallel startvm parallel -j0 --timeout 10 --tag ssh vagrant@parallel-server{} parallel --minversion 20121021 ::: 1 2 || (echo parallel on remote required for testsuite; /bin/true) startvm: - parallel --tag -k 'ping -w 1 -c 1 {} || (cd vagrant/generic/{} && vagrant up)' ::: centos8 freebsd11 freebsd12 rhel8 + parallel --tag -k 'ping -w 1 -c 1 {} || (cd vagrant/*/{} && vagrant up)' ::: centos8 freebsd11 freebsd12 rhel8 centos3 & stopvm: - parallel --tag -k 'ping -w 1 -c 1 {} && cd vagrant/generic/{} && vagrant suspend' ::: centos8 freebsd11 freebsd12 rhel8 + parallel --tag -k 'ping -w 1 -c 1 {} && cd vagrant/*/{} && vagrant suspend' ::: centos8 freebsd11 freebsd12 rhel8 centos3 startdb: true should start Oracle in vagrant diff --git a/testsuite/tests-to-run/parallel-centos3.sh b/testsuite/tests-to-run/parallel-centos3.sh index 7cdef307..18e4188f 100644 --- a/testsuite/tests-to-run/parallel-centos3.sh +++ b/testsuite/tests-to-run/parallel-centos3.sh @@ -11,20 +11,24 @@ SERVER2=172.27.27.1 SSHUSER2=parallel export SSHLOGIN2=$SSHUSER2@$SERVER2 -( +stdout ping -w 1 -c 1 centos3 >/dev/null || ( # Vagrant does not set the IP addr cd testsuite/vagrant/tange/centos3/ 2>/dev/null cd vagrant/tange/centos3/ 2>/dev/null cd ../vagrant/tange/centos3/ 2>/dev/null - stdout ping -w 1 -c 1 centos3 >/dev/null || - stdout vagrant up >/dev/null + stdout vagrant up >/dev/null vagrant ssh -c 'sudo ifconfig eth1 172.27.27.3' - +) +( # Copy binaries to server + cd testsuite/vagrant/tange/centos3/ 2>/dev/null + cd vagrant/tange/centos3/ 2>/dev/null + cd ../vagrant/tange/centos3/ 2>/dev/null cd ../../../.. scp -q .*/src/{parallel,sem,sql,niceload,env_parallel*} $SSHLOGIN1:bin/ ssh $SSHLOGIN1 'touch .parallel/will-cite; mkdir -p bin' - ssh $SSHLOGIN1 cat .ssh/id_rsa.pub | ssh parallel@lo 'cat >>.ssh/authorized_keys' + # Allow login from centos3 to $SSHLOGIN2 (that is shellshock hardened) + ssh $SSHLOGIN1 cat .ssh/id_rsa.pub | ssh $SSHLOGIN2 'cat >>.ssh/authorized_keys' ssh $SSHLOGIN1 'cat .ssh/id_rsa.pub >>.ssh/authorized_keys; chmod 600 .ssh/authorized_keys' ssh $SSHLOGIN1 'ssh -o StrictHostKeyChecking=no localhost true; ssh -o StrictHostKeyChecking=no '$SSHLOGIN2' true;' ) & diff --git a/testsuite/tests-to-run/parallel-local-0.3s.sh b/testsuite/tests-to-run/parallel-local-0.3s.sh index 4291b4ca..de757ce8 100644 --- a/testsuite/tests-to-run/parallel-local-0.3s.sh +++ b/testsuite/tests-to-run/parallel-local-0.3s.sh @@ -12,6 +12,22 @@ export -f stdsort # Test amount of parallelization # parallel --shuf --jl /tmp/myjl -j1 'export JOBS={1};'bash tests-to-run/parallel-local-0.3s.sh ::: {1..16} ::: {1..5} +par_opt_gnu() { + echo '### Test --tollef' + stdout parallel -k --tollef echo -- 1 2 3 ::: a b c | LC_ALL=C sort + + echo '### Test --tollef --gnu' + stdout parallel -k --tollef --gnu echo ::: 1 2 3 -- a b c + + echo '### Test --gnu' + parallel -k --gnu echo ::: 1 2 3 -- a b c +} + +par_colsep_default() { + echo "bug #37956: --colsep does not default to '\t' as specified in the man page." + printf "A\tB\n1\tone" | parallel --header : echo {B} {A} +} + par_tmux_command_not_found() { echo '### PARALLEL_TMUX not found' PARALLEL_TMUX=not-existing parallel --tmux echo ::: 1 diff --git a/testsuite/tests-to-run/parallel-local-10s.sh b/testsuite/tests-to-run/parallel-local-10s.sh index 4f49abd1..84541a5f 100644 --- a/testsuite/tests-to-run/parallel-local-10s.sh +++ b/testsuite/tests-to-run/parallel-local-10s.sh @@ -4,6 +4,118 @@ # Each should be taking 10-30s and be possible to run in parallel # I.e.: No race conditions, no logins +par_compress_prg_fails() { + echo '### bug #44546: If --compress-program fails: fail' + doit() { + (parallel $* --compress-program false \ + echo \; sleep 1\; ls ::: /no-existing + echo $?) | tail -n1 + } + export -f doit + stdout parallel --tag -k doit ::: '' --line-buffer ::: '' --tag ::: '' --files | + grep -v -- -dc +} + +par_dryrun_timeout_ungroup() { + echo 'bug #51039: --dry-run --timeout 1.4m -u breaks' + seq 1000 | stdout parallel --dry-run --timeout 1.4m -u --jobs 10 echo | wc +} + +par_shard() { + echo '### --shard' + # Each of the 5 lines should match: + # ##### ##### ###### + seq 100000 | parallel --pipe --shard 1 -j5 wc | + perl -pe 's/(.*\d{5,}){3}/OK/' + # Data should be sharded to all processes + shard_on_col() { + col=$1 + seq 10 99 | shuf | perl -pe 's/(.)/$1\t/g' | + parallel --pipe --shard $col -j2 --colsep "\t" sort -k$col | + field $col | sort | uniq -c + } + shard_on_col 1 + shard_on_col 2 + + shard_on_col_name() { + colname=$1 + col=$2 + (echo AB; seq 10 99 | shuf) | perl -pe 's/(.)/$1\t/g' | + parallel --header : --pipe --shard $colname -j2 --colsep "\t" sort -k$col | + field $col | sort | uniq -c + } + shard_on_col_name A 1 + shard_on_col_name B 2 + + shard_on_col_expr() { + colexpr="$1" + col=$2 + (seq 10 99 | shuf) | perl -pe 's/(.)/$1\t/g' | + parallel --pipe --shard "$colexpr" -j2 --colsep "\t" "sort -k$col; echo c1 c2" | + field $col | sort | uniq -c + } + shard_on_col_expr '1 $_%=3' 1 + shard_on_col_expr '2 $_%=3' 2 + + shard_on_col_name_expr() { + colexpr="$1" + col=$2 + (echo AB; seq 10 99 | shuf) | perl -pe 's/(.)/$1\t/g' | + parallel --header : --pipe --shard "$colexpr" -j2 --colsep "\t" "sort -k$col; echo c1 c2" | + field $col | sort | uniq -c + } + shard_on_col_name_expr 'A $_%=3' 1 + shard_on_col_name_expr 'B $_%=3' 2 + + echo '*** broken' + # Shorthand for --pipe -j+0 + seq 100000 | parallel --shard 1 wc | + perl -pe 's/(.*\d{5,}){3}/OK/' + # Combine with arguments + seq 100000 | parallel --shard 1 echo {}\;wc ::: {1..5} ::: a b | + perl -pe 's/(.*\d{5,}){3}/OK/' +} + +par_opt_arg_eaten() { + echo 'bug #31716: Options with optional arguments may eat next argument' + echo '### Test https://savannah.gnu.org/bugs/index.php?31716' + seq 1 5 | stdout parallel -k -l echo {} OK + seq 1 5 | stdout parallel -k -l 1 echo {} OK + + echo '### -k -l -0' + printf '1\0002\0003\0004\0005\000' | stdout parallel -k -l -0 echo {} OK + + echo '### -k -0 -l' + printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -l echo {} OK + + echo '### -k -0 -l 1' + printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -l 1 echo {} OK + + echo '### -k -0 -l 0' + printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -l 0 echo {} OK + + echo '### -k -0 -L -0 - -0 is argument for -L' + printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -L -0 echo {} OK + + echo '### -k -0 -L 0 - -L always takes arg' + printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -L 0 echo {} OK + + echo '### -k -0 -L 0 - -L always takes arg' + printf '1\0002\0003\0004\0005\000' | stdout parallel -k -L 0 -0 echo {} OK + + echo '### -k -e -0' + printf '1\0002\0003\0004\0005\000' | stdout parallel -k -e -0 echo {} OK + + echo '### -k -0 -e eof' + printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -e eof echo {} OK + + echo '### -k -i -0' + printf '1\0002\0003\0004\0005\000' | stdout parallel -k -i -0 echo {} OK + + echo '### -k -0 -i repl' + printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -i repl echo repl OK +} + par_bin() { echo '### Test --bin' seq 10 | parallel --pipe --bin 1 -j4 wc | sort diff --git a/testsuite/tests-to-run/parallel-local-1s.sh b/testsuite/tests-to-run/parallel-local-1s.sh index f040c9b4..d8b309f4 100644 --- a/testsuite/tests-to-run/parallel-local-1s.sh +++ b/testsuite/tests-to-run/parallel-local-1s.sh @@ -4,6 +4,25 @@ # Each should be taking 1-3s and be possible to run in parallel # I.e.: No race conditions, no logins +par_open_files_blocks() { + echo 'bug #38439: "open files" with --files --pipe blocks after a while' + ulimit -n 28 + yes "`seq 3000`" | + head -c 20M | + stdout parallel -j10 --pipe -k echo {#} of 21 | + grep -v 'parallel: Warning: No more file handles.' | + grep -v 'Warning: Raising ulimit -n or /etc/security/limits.conf' +} + +par_pipe_unneeded_procs() { + echo 'bug #34241: --pipe should not spawn unneeded processes - part 2' + seq 500 | parallel --tmpdir . -j10 --pipe --block 1k --files wc >/dev/null + ls *.par | wc -l; rm *.par + seq 500 | parallel --tmpdir . -j10 --pipe --block 1k --files --dry-run wc >/dev/null + echo No .par should exist + stdout ls *.par +} + par_interactive() { echo '### Test -p --interactive' cat >/tmp/parallel-script-for-expect <<_EOF @@ -609,6 +628,7 @@ par_test_cpu_detection_cpuinfo() { } export -f test_one compgen -A function | grep ^cpu | sort | parallel -j0 -k test_one + rm ~/.parallel/tmp/sshlogin/*/cpuspec } par_test_cpu_detection_lscpu() { @@ -793,6 +813,7 @@ par_test_cpu_detection_lscpu() { } export -f test_one compgen -A function | grep ^cpu | sort | parallel -j0 -k test_one + rm ~/.parallel/tmp/sshlogin/*/cpuspec } par_null_resume() { diff --git a/testsuite/tests-to-run/parallel-local-3s.sh b/testsuite/tests-to-run/parallel-local-3s.sh index 8f55175d..21e2e8ad 100644 --- a/testsuite/tests-to-run/parallel-local-3s.sh +++ b/testsuite/tests-to-run/parallel-local-3s.sh @@ -4,16 +4,63 @@ # Each should be taking 3-10s and be possible to run in parallel # I.e.: No race conditions, no logins -par_compress_prg_fails() { - echo '### bug #44546: If --compress-program fails: fail' - doit() { - (parallel $* --compress-program false \ - echo \; sleep 1\; ls ::: /no-existing - echo $?) | tail -n1 - } - export -f doit - stdout parallel --tag -k doit ::: '' --line-buffer ::: '' --tag ::: '' --files | - grep -v -- -dc +par_delay() { + echo "### Test --delay" + seq 9 | /usr/bin/time -f %e parallel -j3 --delay 0.57 true {} 2>&1 | + perl -ne '$_ > 3.3 and print "More than 3.3 secs: OK\n"' +} + +par_sshdelay() { + echo '### test --sshdelay' + stdout /usr/bin/time -f %e parallel -j0 --sshdelay 0.5 -S localhost true ::: 1 2 3 | + perl -ne 'print($_ > 1.30 ? "OK\n" : "Not OK\n")' +} + +par_empty_string_quote() { + echo "bug #37694: Empty string argument skipped when using --quote" + parallel -q --nonall perl -le 'print scalar @ARGV' 'a' 'b' '' +} + +par_compute_command_len() { + echo "### Computing length of command line" + seq 1 2 | parallel -k -N2 echo {1} {2} + parallel --xapply -k -a <(seq 11 12) -a <(seq 1 3) echo + parallel -k -C %+ echo '"{1}_{3}_{2}_{4}"' ::: 'a% c %%b' 'a%c% b %d' + parallel -k -C %+ echo {4} ::: 'a% c %%b' +} + +par_replacement_slashslash() { + echo '### Test {//}' + parallel -k echo {//} {} ::: a a/b a/b/c + parallel -k echo {//} {} ::: /a /a/b /a/b/c + parallel -k echo {//} {} ::: ./a ./a/b ./a/b/c + parallel -k echo {//} {} ::: a.jpg a/b.jpg a/b/c.jpg + parallel -k echo {//} {} ::: /a.jpg /a/b.jpg /a/b/c.jpg + parallel -k echo {//} {} ::: ./a.jpg ./a/b.jpg ./a/b/c.jpg + + echo '### Test {1//}' + parallel -k echo {1//} {} ::: a a/b a/b/c + parallel -k echo {1//} {} ::: /a /a/b /a/b/c + parallel -k echo {1//} {} ::: ./a ./a/b ./a/b/c + parallel -k echo {1//} {} ::: a.jpg a/b.jpg a/b/c.jpg + parallel -k echo {1//} {} ::: /a.jpg /a/b.jpg /a/b/c.jpg + parallel -k echo {1//} {} ::: ./a.jpg ./a/b.jpg ./a/b/c.jpg +} + +par_dirnamereplace() { + echo '### Test --dnr' + parallel --dnr II -k echo II {} ::: a a/b a/b/c + + echo '### Test --dirnamereplace' + parallel --dirnamereplace II -k echo II {} ::: a a/b a/b/c +} + +par_negative_replacement() { + echo '### Negative replacement strings' + parallel -X -j1 -N 6 echo {-1}orrec{1} ::: t B X D E c + parallel -N 6 echo {-1}orrect ::: A B X D E c + parallel --colsep ' ' echo '{2} + {4} = {2} + {-1}=' '$(( {2} + {-1} ))' ::: "1 2 3 4" + parallel --colsep ' ' echo '{-3}orrect' ::: "1 c 3 4" } par_eta() { @@ -104,61 +151,6 @@ par_jobslot_repl() { rm /tmp/num10000 } -par_shard() { - echo '### --shard' - # Each of the 5 lines should match: - # ##### ##### ###### - seq 100000 | parallel --pipe --shard 1 -j5 wc | - perl -pe 's/(.*\d{5,}){3}/OK/' - # Data should be sharded to all processes - shard_on_col() { - col=$1 - seq 10 99 | shuf | perl -pe 's/(.)/$1\t/g' | - parallel --pipe --shard $col -j2 --colsep "\t" sort -k$col | - field $col | sort | uniq -c - } - shard_on_col 1 - shard_on_col 2 - - shard_on_col_name() { - colname=$1 - col=$2 - (echo AB; seq 10 99 | shuf) | perl -pe 's/(.)/$1\t/g' | - parallel --header : --pipe --shard $colname -j2 --colsep "\t" sort -k$col | - field $col | sort | uniq -c - } - shard_on_col_name A 1 - shard_on_col_name B 2 - - shard_on_col_expr() { - colexpr="$1" - col=$2 - (seq 10 99 | shuf) | perl -pe 's/(.)/$1\t/g' | - parallel --pipe --shard "$colexpr" -j2 --colsep "\t" "sort -k$col; echo c1 c2" | - field $col | sort | uniq -c - } - shard_on_col_expr '1 $_%=3' 1 - shard_on_col_expr '2 $_%=3' 2 - - shard_on_col_name_expr() { - colexpr="$1" - col=$2 - (echo AB; seq 10 99 | shuf) | perl -pe 's/(.)/$1\t/g' | - parallel --header : --pipe --shard "$colexpr" -j2 --colsep "\t" "sort -k$col; echo c1 c2" | - field $col | sort | uniq -c - } - shard_on_col_name_expr 'A $_%=3' 1 - shard_on_col_name_expr 'B $_%=3' 2 - - echo '*** broken' - # Shorthand for --pipe -j+0 - seq 100000 | parallel --shard 1 wc | - perl -pe 's/(.*\d{5,}){3}/OK/' - # Combine with arguments - seq 100000 | parallel --shard 1 echo {}\;wc ::: {1..5} ::: a b | - perl -pe 's/(.*\d{5,}){3}/OK/' -} - par_distribute_args_at_EOF() { echo '### Test distribute arguments at EOF to 2 jobslots' seq 1 92 | parallel -j2 -kX -s 100 echo @@ -282,11 +274,6 @@ echo finish {}' ::: 1 2 4 rm -f /tmp/jl.$$ } -par_dryrun_timeout_ungroup() { - echo 'bug #51039: --dry-run --timeout 1.4m -u breaks' - seq 1000 | stdout parallel --dry-run --timeout 1.4m -u --jobs 10 echo | wc -} - par_sqlworker_hostname() { echo 'bug #50901: --sqlworker should use hostname in the joblog instead of :' diff --git a/testsuite/tests-to-run/parallel-local-mem.sh b/testsuite/tests-to-run/parallel-local-mem.sh index ed16679f..aefe5040 100755 --- a/testsuite/tests-to-run/parallel-local-mem.sh +++ b/testsuite/tests-to-run/parallel-local-mem.sh @@ -1,6 +1,6 @@ #!/bin/bash -make stopvm +make stopvm >/dev/null 2>/dev/null TMPDIR=${TMPDIR:-/tmp} mkdir -p $TMPDIR # Jobs that eat more than 2 GB RAM @@ -24,22 +24,22 @@ export -f perl5.14parallel par_2gb_records_N() { echo '### bug #44358: 2 GB records cause problems for -N' echo '5 GB version: Eats 12.5 GB RAM + 4 GB Swap' - (gendata 5000MB; echo FOO; - gendata 3000MB; echo FOO; - gendata 1000MB;) | + (gendata 5000MB; echo FOO; + gendata 3000MB; echo FOO; + gendata 1000MB;) | perl5.14parallel --pipe --recend FOO -N2 --block 1g -k LANG=c wc -c echo '2 GB version: eats 10 GB' - (gendata 2300MB; echo FOO; - gendata 2300MB; echo FOO; + (gendata 2300MB; echo FOO; + gendata 2300MB; echo FOO; gendata 1000MB;) | perl5.14parallel --pipe --recend FOO -N2 --block 1g -k LANG=c wc -c echo '### -L >4GB'; echo 'Eats 12.5 GB RAM + 6 GB Swap'; - (head -c 5000MB /dev/zero; echo FOO; - head -c 3000MB /dev/zero; echo FOO; - head -c 1000MB /dev/zero;) | + (head -c 5000MB /dev/zero; echo FOO; + head -c 3000MB /dev/zero; echo FOO; + head -c 1000MB /dev/zero;) | parallel --pipe -L2 --block 1g -k LANG=c wc -c } @@ -48,18 +48,15 @@ par_2gb_record_reading() { echo '### perl -e $buf=("x"x(2**31))."x"; substr($buf,0,2**31+1)=""; print length $buf' echo 'Eats 4 GB' perl -e '$buf=("x"x(2**31))."x"; substr($buf,0,2**31+1)=""; print ((length $buf)."\n")' - + echo 'Eats 4.7 GB' (gendata 2300MB; echo ged) | perl5.14parallel -k --block 2G --pipe --recend ged md5sum echo 'Eats 4.7 GB' - (gendata 2300MB; echo ged) | + (gendata 2300MB; echo ged) | perl5.14parallel -k --block 2G --pipe --recend ged cat | wc -c } export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | sort | parallel -j1 --tag -k --joblog +/tmp/jl-`basename $0` '{} 2>&1' - - -make startvm diff --git a/testsuite/tests-to-run/parallel-local-sql01.sh b/testsuite/tests-to-run/parallel-local-sql01.sh index 0a4eab43..9e55a298 100644 --- a/testsuite/tests-to-run/parallel-local-sql01.sh +++ b/testsuite/tests-to-run/parallel-local-sql01.sh @@ -125,8 +125,9 @@ par_empty() { hostname=`hostname` export -f $(compgen -A function | egrep 'p_|par_') # Tested that -j0 in parallel is fastest (up to 15 jobs) +# -j5: SQLite complains about locked database. compgen -A function | grep par_ | sort | - stdout parallel -vj5 -k --tag --joblog /tmp/jl-`basename $0` p_wrapper \ + stdout parallel -vj4 -k --tag --joblog /tmp/jl-`basename $0` p_wrapper \ :::: - ::: \$MYSQL \$PG \$SQLITE \$CSV | perl -pe 's/tbl\d+/TBL99999/gi;' | perl -pe 's/(from TBL99999 order) .*/$1/g' | diff --git a/testsuite/tests-to-run/parallel-local-ssh7.sh b/testsuite/tests-to-run/parallel-local-ssh7.sh index a7eefcf1..4adeef2a 100755 --- a/testsuite/tests-to-run/parallel-local-ssh7.sh +++ b/testsuite/tests-to-run/parallel-local-ssh7.sh @@ -1175,7 +1175,8 @@ par_csh_funky() { echo "$funky" | parallel --shellquote _EOF ) - ssh csh@lo "$myscript" + # Sometimes the order f*cks up + stdout ssh csh@lo "$myscript" | LC_ALL=C sort } par_dash_funky() { diff --git a/testsuite/tests-to-run/parallel-local-ssh8.sh b/testsuite/tests-to-run/parallel-local-ssh8.sh index fcab89cd..d3b689da 100644 --- a/testsuite/tests-to-run/parallel-local-ssh8.sh +++ b/testsuite/tests-to-run/parallel-local-ssh8.sh @@ -17,7 +17,7 @@ par_path_remote_bash() { cp /usr/local/bin/parallel /tmp cat <<'_EOS' | - echo logged in + echo StArT echo BASH Path before: $PATH with no parallel parallel echo ::: 1 # Race condition stderr/stdout @@ -31,7 +31,7 @@ par_path_remote_bash() { env_parallel --filter --env A,PATH -Slo echo '$PATH' ::: OK _EOS stdout ssh nopathbash@lo -T | - perl -ne '/logged in/..0 and print' | + perl -ne '/StArT/..0 and print' | uniq echo } @@ -42,7 +42,7 @@ par_path_remote_csh() { cp /usr/local/bin/parallel /tmp cat <<'_EOS' | - echo logged in + echo StArT echo CSH Path before: $PATH with no parallel which parallel >& /dev/stdout echo '^^^^^^^^ Not found is OK' @@ -62,7 +62,7 @@ par_path_remote_csh() { echo Done _EOS stdout ssh nopathcsh@lo -T | - perl -ne '/Users logged in/ and next; /logged in/..0 and print' | + perl -ne '/StArT/..0 and print' | uniq } diff --git a/testsuite/tests-to-run/parallel-local164.sh b/testsuite/tests-to-run/parallel-local164.sh dissimilarity index 92% index 323420ed..7b4cba51 100644 --- a/testsuite/tests-to-run/parallel-local164.sh +++ b/testsuite/tests-to-run/parallel-local164.sh @@ -1,131 +1,9 @@ -#!/bin/bash - -# -L1 will join lines ending in ' ' -cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -r -echo "### Test --delay" -seq 9 | /usr/bin/time -f %e parallel -j3 --delay 0.57 true {} 2>&1 | - perl -ne '$_ > 3.3 and print "More than 3.3 secs: OK\n"' - -echo '### test --sshdelay' - stdout /usr/bin/time -f %e parallel -j0 --sshdelay 0.5 -S localhost true ::: 1 2 3 | perl -ne 'print($_ > 1.30 ? "OK\n" : "Not OK\n")' - -echo "bug #37694: Empty string argument skipped when using --quote" - parallel -q --nonall perl -le 'print scalar @ARGV' 'a' 'b' '' - -echo '### Test -k 5'; - sleep 5 - -echo '### Test -k 3'; - sleep 3 - -echo '### Test -k 4'; - sleep 4 - -echo '### Test -k 2'; - sleep 2 - -echo '### Test -k 1'; - sleep 1 - -echo "### Computing length of command line" - seq 1 2 | parallel -k -N2 echo {1} {2} - parallel --xapply -k -a <(seq 11 12) -a <(seq 1 3) echo - parallel -k -C %+ echo '"{1}_{3}_{2}_{4}"' ::: 'a% c %%b' 'a%c% b %d' - parallel -k -C %+ echo {4} ::: 'a% c %%b' - -echo "### test08" - cd input-files/test08; - ls | parallel -q perl -ne '/_PRE (\d+)/ and $p=$1; /hatchname> (\d+)/ and $1!=$p and print $ARGV,"\n"' | sort - -seq 1 10 | parallel -j 1 echo | sort -seq 1 10 | parallel -j 2 echo | sort -seq 1 10 | parallel -j 3 echo | sort - -echo "bug #37956: --colsep does not default to '\t' as specified in the man page." - printf "A\tB\n1\tone" | parallel --header : echo {B} {A} - -echo '### Test --tollef' - stdout parallel -k --tollef echo -- 1 2 3 ::: a b c | LC_ALL=C sort - -echo '### Test --tollef --gnu' - stdout parallel -k --tollef --gnu echo ::: 1 2 3 -- a b c - -echo '### Test --gnu' - parallel -k --gnu echo ::: 1 2 3 -- a b c - -echo '### Test {//}' - parallel -k echo {//} {} ::: a a/b a/b/c - parallel -k echo {//} {} ::: /a /a/b /a/b/c - parallel -k echo {//} {} ::: ./a ./a/b ./a/b/c - parallel -k echo {//} {} ::: a.jpg a/b.jpg a/b/c.jpg - parallel -k echo {//} {} ::: /a.jpg /a/b.jpg /a/b/c.jpg - parallel -k echo {//} {} ::: ./a.jpg ./a/b.jpg ./a/b/c.jpg - -echo '### Test {1//}' - parallel -k echo {1//} {} ::: a a/b a/b/c - parallel -k echo {1//} {} ::: /a /a/b /a/b/c - parallel -k echo {1//} {} ::: ./a ./a/b ./a/b/c - parallel -k echo {1//} {} ::: a.jpg a/b.jpg a/b/c.jpg - parallel -k echo {1//} {} ::: /a.jpg /a/b.jpg /a/b/c.jpg - parallel -k echo {1//} {} ::: ./a.jpg ./a/b.jpg ./a/b/c.jpg - -echo '### Test --dnr' - parallel --dnr II -k echo II {} ::: a a/b a/b/c - -echo '### Test --dirnamereplace' - parallel --dirnamereplace II -k echo II {} ::: a a/b a/b/c - -echo '### Test https://savannah.gnu.org/bugs/index.php?31716' - seq 1 5 | stdout parallel -k -l echo {} OK - seq 1 5 | stdout parallel -k -l 1 echo {} OK - -echo '### -k -l -0' - printf '1\0002\0003\0004\0005\000' | stdout parallel -k -l -0 echo {} OK - -echo '### -k -0 -l' - printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -l echo {} OK - -echo '### -k -0 -l 1' - printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -l 1 echo {} OK - -echo '### -k -0 -l 0' - printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -l 0 echo {} OK - -echo '### -k -0 -L -0 - -0 is argument for -L' - printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -L -0 echo {} OK - -echo '### -k -0 -L 0 - -L always takes arg' - printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -L 0 echo {} OK - -echo '### -k -0 -L 0 - -L always takes arg' - printf '1\0002\0003\0004\0005\000' | stdout parallel -k -L 0 -0 echo {} OK - -echo '### -k -e -0' - printf '1\0002\0003\0004\0005\000' | stdout parallel -k -e -0 echo {} OK - -echo '### -k -0 -e eof' - printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -e eof echo {} OK - -echo '### -k -i -0' - printf '1\0002\0003\0004\0005\000' | stdout parallel -k -i -0 echo {} OK - -echo '### -k -0 -i repl' - printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -i repl echo repl OK - -echo '### Negative replacement strings' - parallel -X -j1 -N 6 echo {-1}orrec{1} ::: t B X D E c - parallel -N 6 echo {-1}orrect ::: A B X D E c - parallel --colsep ' ' echo '{2} + {4} = {2} + {-1}=' '$(( {2} + {-1} ))' ::: "1 2 3 4" - parallel --colsep ' ' echo '{-3}orrect' ::: "1 c 3 4" - -echo 'bug #38439: "open files" with --files --pipe blocks after a while' - ulimit -n 28; yes "`seq 3000`" |head -c 20M | parallel --pipe -k echo {#} of 21 - -echo 'bug #34241: --pipe should not spawn unneeded processes - part 2' - seq 500 | parallel --tmpdir . -j10 --pipe --block 1k --files wc >/dev/null; - ls *.par | wc -l; rm *.par; - seq 500 | parallel --tmpdir . -j10 --pipe --block 1k --files --dry-run wc >/dev/null; - echo No .par should exist; - stdout ls *.par - -EOF +#!/bin/bash + +# -L1 will join lines ending in ' ' +cat <<'EOF' | sed -e 's/;$/; /;s/$SERVER1/'$SERVER1'/;s/$SERVER2/'$SERVER2'/' | stdout parallel -vj0 -k --joblog /tmp/jl-`basename $0` -L1 -r + +echo "### test08" + cd input-files/test08; + ls | parallel -q perl -ne '/_PRE (\d+)/ and $p=$1; /hatchname> (\d+)/ and $1!=$p and print $ARGV,"\n"' | sort +EOF diff --git a/testsuite/tests-to-run/parallel-macos.sh b/testsuite/tests-to-run/parallel-macos.sh index 84f49245..c78c1950 100644 --- a/testsuite/tests-to-run/parallel-macos.sh +++ b/testsuite/tests-to-run/parallel-macos.sh @@ -37,10 +37,10 @@ doit() { $envn=length join"",(keys %ENV); $envv=length join"",(values %ENV); $onechar='$onechar'; - $maxlen=5-39+262144 - $envn - $envv - $onechar*5 - $envc*10; - print("Max len = $maxlen\n"); + $maxlen=-39+262144 - $envn - $envv - $onechar*5 - $envc*10; + print("Computed max len = $maxlen\n"); $bin='$binlen'; - print("$bin=",$bin-$maxlen," $onechar $envc $envn $envv\n"); + print("Actual:$bin Diff:",$bin-$maxlen," Vars: $onechar $envc $envn $envv\n"); ' } export -f doit @@ -48,7 +48,7 @@ export -f doit val="$(seq 2 100 1000)" val="10 20 50 100 200 500 1000" val="12 103 304 506 1005" -parallel --timeout 20 --shuf --tag -k doit ::: $val ::: $val ::: $val ::: $val ::: $val ::: $val ::: $val +parallel --timeout 20 --shuf --tag -k doit ::: $val ::: $val ::: $val ::: $val ::: $val ::: $val ::: $val 2>/dev/null # Test with random data (seq 10;seq 100;seq 100;seq 100;seq 100; seq 300 ;seq 1000) | @@ -167,4 +167,4 @@ export -f $(compgen -A function | grep par_) compgen -A function | grep par_ | LC_ALL=C sort | - env_parallel --timeout 1000% --tag -k -S 6/$macsshlogin 'PATH=$HOME/bin:$PATH; {}' + env_parallel --timeout 3000% --tag -k -S 6/$macsshlogin 'PATH=$HOME/bin:$PATH; {}' diff --git a/testsuite/tests-to-run/parallel-virtualbox1.sh b/testsuite/tests-to-run/parallel-virtualbox1.sh index 61dddb4c..77c196d3 100644 --- a/testsuite/tests-to-run/parallel-virtualbox1.sh +++ b/testsuite/tests-to-run/parallel-virtualbox1.sh @@ -2,7 +2,7 @@ ( cd vagrant/tange/centos3/ - stdout vagrant up | grep -v 'default' + stdout vagrant up | grep -v 'default' | grep -v '==>' | grep -E '^$' ) par_warning_on_centos3() { diff --git a/testsuite/tests-to-run/sql01.sh b/testsuite/tests-to-run/sql01.sh index 4aa7b940..b6501981 100755 --- a/testsuite/tests-to-run/sql01.sh +++ b/testsuite/tests-to-run/sql01.sh @@ -87,10 +87,11 @@ sql --proclist :sqlunittest | wc -lw sql --listproc :sqlunittest | wc -lw echo "### Test --db-size --dbsize"; -sql --dbsize :sqlunittest | wc -sql --db-size :sqlunittest | wc +sql --dbsize :sqlunittest | wc -w +sql --db-size :sqlunittest | wc -w echo "### Test --table-size --tablesize" +sql --showtables :sqlunittest | grep TBL | parallel sql :sqlunittest drop table sql --tablesize :sqlunittest | wc -l sql --table-size :sqlunittest | wc -l diff --git a/testsuite/tests-to-run/test60.sh b/testsuite/tests-to-run/test60.sh index 53d05573..80d83960 100644 --- a/testsuite/tests-to-run/test60.sh +++ b/testsuite/tests-to-run/test60.sh @@ -27,7 +27,7 @@ echo '### Test --nonall'; parallel --nonall -k -S $SSHLOGIN1,$SSHLOGIN2 'hostname' | sort echo '### Test --nonall -u - should be interleaved x y x y'; - parallel --nonall -S $SSHLOGIN1,$SSHLOGIN2 -u 'hostname|grep -q redhat9 && sleep 3; hostname;sleep 12;hostname;' + parallel --nonall --sshdelay 2 -S $SSHLOGIN1,$SSHLOGIN2 -u 'hostname|grep -q centos && sleep 2; hostname;sleep 4;hostname;' echo '### Test read sshloginfile from STDIN'; echo $SSHLOGIN1 | parallel -S - --nonall hostname; diff --git a/testsuite/wanted-results/parallel-centos3 b/testsuite/wanted-results/parallel-centos3 index b01e8b97..0e9b72ad 100644 --- a/testsuite/wanted-results/parallel-centos3 +++ b/testsuite/wanted-results/parallel-centos3 @@ -1,5 +1,6 @@ ### These tests requires VirtualBox running with the following images vagrant@centos3 + par_shellshock_bug par_shellshock_bug 2>&1 par_shellshock_bug bug #43358: shellshock breaks exporting functions using --env name par_shellshock_bug Non-shellshock-hardened to non-shellshock-hardened diff --git a/testsuite/wanted-results/parallel-local-0.3s b/testsuite/wanted-results/parallel-local-0.3s index 591b37e3..fda22069 100644 --- a/testsuite/wanted-results/parallel-local-0.3s +++ b/testsuite/wanted-results/parallel-local-0.3s @@ -58,6 +58,8 @@ par_blocking_redir stdout par_colsep_0 bug --colsep 0 par_colsep_0 OK par_colsep_0 OK +par_colsep_default bug #37956: --colsep does not default to '\t' as specified in the man page. +par_colsep_default one 1 par_cr_newline_header ### --header : should set named replacement string if input line ends in \r\n par_cr_newline_header bar par_csv col1"x3"-new @@ -459,6 +461,25 @@ par_no_command_given parallel: Error: Command line too long (1000002 >= 10) at i par_no_joblog_with_dryrun bug #46016: --joblog should not log when --dryrun par_no_joblog_with_dryrun echo Only_this par_no_run_if_empty ### Test --no-run-if-empty and -r: This should give no output +par_opt_gnu ### Test --tollef +par_opt_gnu parallel: Error: --tollef has been retired. +par_opt_gnu parallel: Error: Remove --tollef or use --gnu to override --tollef. +par_opt_gnu ### Test --tollef --gnu +par_opt_gnu 1 +par_opt_gnu 2 +par_opt_gnu 3 +par_opt_gnu -- +par_opt_gnu a +par_opt_gnu b +par_opt_gnu c +par_opt_gnu ### Test --gnu +par_opt_gnu 1 +par_opt_gnu 2 +par_opt_gnu 3 +par_opt_gnu -- +par_opt_gnu a +par_opt_gnu b +par_opt_gnu c par_parcat_args_stdin bug #51690: parcat: read args from stdin par_parcat_args_stdin OK1 par_parcat_args_stdin OK2 diff --git a/testsuite/wanted-results/parallel-local-10s b/testsuite/wanted-results/parallel-local-10s index 778c6be7..065780f3 100644 --- a/testsuite/wanted-results/parallel-local-10s +++ b/testsuite/wanted-results/parallel-local-10s @@ -94,6 +94,33 @@ par_colsep ac par_compress_fail ### bug #41609: --compress fails par_compress_fail f1a751b1283e99e7dda40f63f1225f74 - par_compress_fail f1a751b1283e99e7dda40f63f1225f74 - +par_compress_prg_fails ### bug #44546: If --compress-program fails: fail +par_compress_prg_fails 1 +par_compress_prg_fails parallel: Error: false failed. +par_compress_prg_fails parallel: Error: false failed. +par_compress_prg_fails --files 1 +par_compress_prg_fails --files parallel: Error: false failed. +par_compress_prg_fails --files parallel: Error: false failed. +par_compress_prg_fails --tag 1 +par_compress_prg_fails --tag parallel: Error: false failed. +par_compress_prg_fails --tag parallel: Error: false failed. +par_compress_prg_fails --tag --files 1 +par_compress_prg_fails --tag --files parallel: Error: false failed. +par_compress_prg_fails --tag --files parallel: Error: false failed. +par_compress_prg_fails --line-buffer 1 +par_compress_prg_fails --line-buffer parallel: Error: false failed. +par_compress_prg_fails --line-buffer parallel: Error: false failed. +par_compress_prg_fails --line-buffer --files 1 +par_compress_prg_fails --line-buffer --files parallel: Error: false failed. +par_compress_prg_fails --line-buffer --files parallel: Error: false failed. +par_compress_prg_fails --line-buffer --tag 1 +par_compress_prg_fails --line-buffer --tag parallel: Error: false failed. +par_compress_prg_fails --line-buffer --tag parallel: Error: false failed. +par_compress_prg_fails --line-buffer --tag --files 1 +par_compress_prg_fails --line-buffer --tag --files parallel: Error: false failed. +par_compress_prg_fails --line-buffer --tag --files parallel: Error: false failed. +par_dryrun_timeout_ungroup bug #51039: --dry-run --timeout 1.4m -u breaks +par_dryrun_timeout_ungroup 1000 2000 8893 par_failing_compressor Compress with failing (de)compressor par_failing_compressor Test --tag/--line-buffer/--files in all combinations par_failing_compressor Test working/failing compressor/decompressor in all combinations @@ -497,6 +524,84 @@ par_nice Check that --nice works par_nice load_10 par_nice bzip2 18 par_nice bzip2 18 +par_opt_arg_eaten bug #31716: Options with optional arguments may eat next argument +par_opt_arg_eaten ### Test https://savannah.gnu.org/bugs/index.php?31716 +par_opt_arg_eaten 1 OK +par_opt_arg_eaten 2 OK +par_opt_arg_eaten 3 OK +par_opt_arg_eaten 4 OK +par_opt_arg_eaten 5 OK +par_opt_arg_eaten 1 OK +par_opt_arg_eaten 2 OK +par_opt_arg_eaten 3 OK +par_opt_arg_eaten 4 OK +par_opt_arg_eaten 5 OK +par_opt_arg_eaten ### -k -l -0 +par_opt_arg_eaten 1 OK +par_opt_arg_eaten 2 OK +par_opt_arg_eaten 3 OK +par_opt_arg_eaten 4 OK +par_opt_arg_eaten 5 OK +par_opt_arg_eaten ### -k -0 -l +par_opt_arg_eaten 1 OK +par_opt_arg_eaten 2 OK +par_opt_arg_eaten 3 OK +par_opt_arg_eaten 4 OK +par_opt_arg_eaten 5 OK +par_opt_arg_eaten ### -k -0 -l 1 +par_opt_arg_eaten 1 OK +par_opt_arg_eaten 2 OK +par_opt_arg_eaten 3 OK +par_opt_arg_eaten 4 OK +par_opt_arg_eaten 5 OK +par_opt_arg_eaten ### -k -0 -l 0 +par_opt_arg_eaten 1 OK +par_opt_arg_eaten 2 OK +par_opt_arg_eaten 3 OK +par_opt_arg_eaten 4 OK +par_opt_arg_eaten 5 OK +par_opt_arg_eaten ### -k -0 -L -0 - -0 is argument for -L +par_opt_arg_eaten OK +par_opt_arg_eaten OK +par_opt_arg_eaten OK +par_opt_arg_eaten OK +par_opt_arg_eaten OK +par_opt_arg_eaten ### -k -0 -L 0 - -L always takes arg +par_opt_arg_eaten OK +par_opt_arg_eaten OK +par_opt_arg_eaten OK +par_opt_arg_eaten OK +par_opt_arg_eaten OK +par_opt_arg_eaten ### -k -0 -L 0 - -L always takes arg +par_opt_arg_eaten OK +par_opt_arg_eaten OK +par_opt_arg_eaten OK +par_opt_arg_eaten OK +par_opt_arg_eaten OK +par_opt_arg_eaten ### -k -e -0 +par_opt_arg_eaten 1 OK +par_opt_arg_eaten 2 OK +par_opt_arg_eaten 3 OK +par_opt_arg_eaten 4 OK +par_opt_arg_eaten 5 OK +par_opt_arg_eaten ### -k -0 -e eof +par_opt_arg_eaten 1 OK +par_opt_arg_eaten 2 OK +par_opt_arg_eaten 3 OK +par_opt_arg_eaten 4 OK +par_opt_arg_eaten 5 OK +par_opt_arg_eaten ### -k -i -0 +par_opt_arg_eaten 1 OK +par_opt_arg_eaten 2 OK +par_opt_arg_eaten 3 OK +par_opt_arg_eaten 4 OK +par_opt_arg_eaten 5 OK +par_opt_arg_eaten ### -k -0 -i repl +par_opt_arg_eaten 1 OK +par_opt_arg_eaten 2 OK +par_opt_arg_eaten 3 OK +par_opt_arg_eaten 4 OK +par_opt_arg_eaten 5 OK par_parset ### test parset par_parset Put output into $myarray par_parset 10 @@ -825,6 +930,97 @@ par_semaphore job3a 4 par_semaphore job2b 5 par_semaphore job3b 6 par_semaphore done +par_shard ### --shard +par_shard OK +par_shard OK +par_shard OK +par_shard OK +par_shard OK +par_shard 10 1 +par_shard 10 2 +par_shard 10 3 +par_shard 10 4 +par_shard 10 5 +par_shard 10 6 +par_shard 10 7 +par_shard 10 8 +par_shard 10 9 +par_shard 9 0 +par_shard 9 1 +par_shard 9 2 +par_shard 9 3 +par_shard 9 4 +par_shard 9 5 +par_shard 9 6 +par_shard 9 7 +par_shard 9 8 +par_shard 9 9 +par_shard 10 1 +par_shard 10 2 +par_shard 10 3 +par_shard 10 4 +par_shard 10 5 +par_shard 10 6 +par_shard 10 7 +par_shard 10 8 +par_shard 10 9 +par_shard 9 0 +par_shard 9 1 +par_shard 9 2 +par_shard 9 3 +par_shard 9 4 +par_shard 9 5 +par_shard 9 6 +par_shard 9 7 +par_shard 9 8 +par_shard 9 9 +par_shard 10 1 +par_shard 10 2 +par_shard 10 3 +par_shard 10 4 +par_shard 10 5 +par_shard 10 6 +par_shard 10 7 +par_shard 10 8 +par_shard 10 9 +par_shard 2 c1 +par_shard 9 0 +par_shard 9 1 +par_shard 9 2 +par_shard 9 3 +par_shard 9 4 +par_shard 9 5 +par_shard 9 6 +par_shard 9 7 +par_shard 9 8 +par_shard 9 9 +par_shard 2 c2 +par_shard 10 1 +par_shard 10 2 +par_shard 10 3 +par_shard 10 4 +par_shard 10 5 +par_shard 10 6 +par_shard 10 7 +par_shard 10 8 +par_shard 10 9 +par_shard 2 c1 +par_shard 9 0 +par_shard 9 1 +par_shard 9 2 +par_shard 9 3 +par_shard 9 4 +par_shard 9 5 +par_shard 9 6 +par_shard 9 7 +par_shard 9 8 +par_shard 9 9 +par_shard 2 c2 +par_shard *** broken +par_shard parallel: Error: --shard requires --jobs to be higher than the number of +par_shard parallel: Error: arguments. Increase --jobs. +par_shard parallel: Error: --shard requires --jobs to be higher than the number of +par_shard parallel: Error: arguments. Increase --jobs. par_shellquote ### Test --shellquote in all shells par_shellquote ash ' par_shellquote ash par_shellquote ash  !"#$%&'"'"'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€�‚ƒ„…†‡ˆ‰Š‹Œ�Ž��‘’“”•–—˜™š›œ�žŸ ¡¢£¤¥¦§¨©ª«¬­®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ' diff --git a/testsuite/wanted-results/parallel-local-1s b/testsuite/wanted-results/parallel-local-1s index c0882814..46d486da 100644 --- a/testsuite/wanted-results/parallel-local-1s +++ b/testsuite/wanted-results/parallel-local-1s @@ -188,6 +188,32 @@ par_null_resume c par_null_resume A par_null_resume B par_null_resume C +par_open_files_blocks bug #38439: "open files" with --files --pipe blocks after a while +par_open_files_blocks parallel: Warning: Only enough file handles to run 2 jobs in parallel. +par_open_files_blocks parallel: Warning: Running 'parallel -j0 -N 2 --pipe parallel -j0' or +par_open_files_blocks parallel: Warning: raising 'ulimit -n' or 'nofile' in /etc/security/limits.conf +par_open_files_blocks parallel: Warning: or /proc/sys/fs/file-max may help. +par_open_files_blocks 1 of 21 +par_open_files_blocks 2 of 21 +par_open_files_blocks 3 of 21 +par_open_files_blocks 4 of 21 +par_open_files_blocks 5 of 21 +par_open_files_blocks 6 of 21 +par_open_files_blocks 7 of 21 +par_open_files_blocks 8 of 21 +par_open_files_blocks 9 of 21 +par_open_files_blocks 10 of 21 +par_open_files_blocks 11 of 21 +par_open_files_blocks 12 of 21 +par_open_files_blocks 13 of 21 +par_open_files_blocks 14 of 21 +par_open_files_blocks 15 of 21 +par_open_files_blocks 16 of 21 +par_open_files_blocks 17 of 21 +par_open_files_blocks 18 of 21 +par_open_files_blocks 19 of 21 +par_open_files_blocks 20 of 21 +par_open_files_blocks 21 of 21 par_pipe_compress_blocks ### bug #41482: --pipe --compress blocks at different -j/seq combinations par_pipe_compress_blocks 1 par_pipe_compress_blocks echo 1-4 + 1-4 @@ -221,6 +247,10 @@ par_pipe_compress_blocks 1 2 3 par_pipe_compress_blocks 1 par_pipe_compress_blocks 2 par_pipe_compress_blocks 3 +par_pipe_unneeded_procs bug #34241: --pipe should not spawn unneeded processes - part 2 +par_pipe_unneeded_procs 2 +par_pipe_unneeded_procs No .par should exist +par_pipe_unneeded_procs ls: cannot access '*.par': No such file or directory par_profiles_with_space ### bug #42902: profiles containing arguments with space par_profiles_with_space /bin/bash=/bin/bash par_profiles_with_space echo '/bin/bash=/bin/bash' diff --git a/testsuite/wanted-results/parallel-local-3s b/testsuite/wanted-results/parallel-local-3s index d51eb8d9..8c23bd21 100644 --- a/testsuite/wanted-results/parallel-local-3s +++ b/testsuite/wanted-results/parallel-local-3s @@ -31,37 +31,30 @@ par_commandline_with_newline . : ... ... . . . . false "commandwithwithnewlines" d par_commandline_with_newline . : ... ... . . . . false "commandwithnewlines" e par_commandline_with_newline Seq Host Starttime JobRuntime Send Receive Exitval Signal Command -par_compress_prg_fails ### bug #44546: If --compress-program fails: fail -par_compress_prg_fails 1 -par_compress_prg_fails parallel: Error: false failed. -par_compress_prg_fails parallel: Error: false failed. -par_compress_prg_fails --files 1 -par_compress_prg_fails --files parallel: Error: false failed. -par_compress_prg_fails --files parallel: Error: false failed. -par_compress_prg_fails --tag 1 -par_compress_prg_fails --tag parallel: Error: false failed. -par_compress_prg_fails --tag parallel: Error: false failed. -par_compress_prg_fails --tag --files 1 -par_compress_prg_fails --tag --files parallel: Error: false failed. -par_compress_prg_fails --tag --files parallel: Error: false failed. -par_compress_prg_fails --line-buffer 1 -par_compress_prg_fails --line-buffer parallel: Error: false failed. -par_compress_prg_fails --line-buffer parallel: Error: false failed. -par_compress_prg_fails --line-buffer --files 1 -par_compress_prg_fails --line-buffer --files parallel: Error: false failed. -par_compress_prg_fails --line-buffer --files parallel: Error: false failed. -par_compress_prg_fails --line-buffer --tag 1 -par_compress_prg_fails --line-buffer --tag parallel: Error: false failed. -par_compress_prg_fails --line-buffer --tag parallel: Error: false failed. -par_compress_prg_fails --line-buffer --tag --files 1 -par_compress_prg_fails --line-buffer --tag --files parallel: Error: false failed. -par_compress_prg_fails --line-buffer --tag --files parallel: Error: false failed. +par_compute_command_len ### Computing length of command line +par_compute_command_len 1 2 +par_compute_command_len 11 1 +par_compute_command_len 12 2 +par_compute_command_len 11 3 +par_compute_command_len a_b_c_ +par_compute_command_len a_b_c_d +par_compute_command_len +par_delay ### Test --delay +par_delay More than 3.3 secs: OK par_delay_human_readable a par_delay_human_readable b par_delay_human_readable c par_delay_human_readable a par_delay_human_readable b par_delay_human_readable c +par_dirnamereplace ### Test --dnr +par_dirnamereplace . a +par_dirnamereplace a a/b +par_dirnamereplace a/b a/b/c +par_dirnamereplace ### Test --dirnamereplace +par_dirnamereplace . a +par_dirnamereplace a a/b +par_dirnamereplace a/b a/b/c par_distribute_args_at_EOF ### Test distribute arguments at EOF to 2 jobslots par_distribute_args_at_EOF 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 par_distribute_args_at_EOF 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 @@ -115,8 +108,8 @@ par_do_not_export_PARALLEL_ENV Should be 0 par_do_not_export_PARALLEL_ENV 1 0 1 par_do_not_export_PARALLEL_ENV Should give 60k and not overflow par_do_not_export_PARALLEL_ENV 1 1 60001 -par_dryrun_timeout_ungroup bug #51039: --dry-run --timeout 1.4m -u breaks -par_dryrun_timeout_ungroup 1000 2000 8893 +par_empty_string_quote bug #37694: Empty string argument skipped when using --quote +par_empty_string_quote 3 par_eta ### Test of --eta par_eta 16 par_eta ### Test of --eta with no jobs @@ -211,6 +204,11 @@ par_multiline_commands echo finish 4 par_multiline_commands parallel: Warning: Command lines contain newline. Forcing --null. par_multiline_commands 4 par_multiline_commands finish 4 +par_negative_replacement ### Negative replacement strings +par_negative_replacement correct +par_negative_replacement correct +par_negative_replacement 2 + 4 = 2 + 4= 6 +par_negative_replacement correct par_pipepart_block ### --pipepart --block -# (# < 0) par_pipepart_block 1 par_pipepart_block 2 @@ -265,6 +263,44 @@ par_progress par_progress Computers / CPU cores / Max jobs to run par_progress 1:local / 8 / 1 par_progress par_progress 0 +par_replacement_slashslash ### Test {//} +par_replacement_slashslash . a +par_replacement_slashslash a a/b +par_replacement_slashslash a/b a/b/c +par_replacement_slashslash / /a +par_replacement_slashslash /a /a/b +par_replacement_slashslash /a/b /a/b/c +par_replacement_slashslash . ./a +par_replacement_slashslash ./a ./a/b +par_replacement_slashslash ./a/b ./a/b/c +par_replacement_slashslash . a.jpg +par_replacement_slashslash a a/b.jpg +par_replacement_slashslash a/b a/b/c.jpg +par_replacement_slashslash / /a.jpg +par_replacement_slashslash /a /a/b.jpg +par_replacement_slashslash /a/b /a/b/c.jpg +par_replacement_slashslash . ./a.jpg +par_replacement_slashslash ./a ./a/b.jpg +par_replacement_slashslash ./a/b ./a/b/c.jpg +par_replacement_slashslash ### Test {1//} +par_replacement_slashslash . a +par_replacement_slashslash a a/b +par_replacement_slashslash a/b a/b/c +par_replacement_slashslash / /a +par_replacement_slashslash /a /a/b +par_replacement_slashslash /a/b /a/b/c +par_replacement_slashslash . ./a +par_replacement_slashslash ./a ./a/b +par_replacement_slashslash ./a/b ./a/b/c +par_replacement_slashslash . a.jpg +par_replacement_slashslash a a/b.jpg +par_replacement_slashslash a/b a/b/c.jpg +par_replacement_slashslash / /a.jpg +par_replacement_slashslash /a /a/b.jpg +par_replacement_slashslash /a/b /a/b/c.jpg +par_replacement_slashslash . ./a.jpg +par_replacement_slashslash ./a ./a/b.jpg +par_replacement_slashslash ./a/b ./a/b/c.jpg par_resume_k ### --resume -k par_resume_k job0id par_resume_k job1id @@ -276,97 +312,6 @@ par_resume_k try 2 = nothing par_resume_k two extra par_resume_k job6id par_resume_k job7id -par_shard ### --shard -par_shard OK -par_shard OK -par_shard OK -par_shard OK -par_shard OK -par_shard 10 1 -par_shard 10 2 -par_shard 10 3 -par_shard 10 4 -par_shard 10 5 -par_shard 10 6 -par_shard 10 7 -par_shard 10 8 -par_shard 10 9 -par_shard 9 0 -par_shard 9 1 -par_shard 9 2 -par_shard 9 3 -par_shard 9 4 -par_shard 9 5 -par_shard 9 6 -par_shard 9 7 -par_shard 9 8 -par_shard 9 9 -par_shard 10 1 -par_shard 10 2 -par_shard 10 3 -par_shard 10 4 -par_shard 10 5 -par_shard 10 6 -par_shard 10 7 -par_shard 10 8 -par_shard 10 9 -par_shard 9 0 -par_shard 9 1 -par_shard 9 2 -par_shard 9 3 -par_shard 9 4 -par_shard 9 5 -par_shard 9 6 -par_shard 9 7 -par_shard 9 8 -par_shard 9 9 -par_shard 10 1 -par_shard 10 2 -par_shard 10 3 -par_shard 10 4 -par_shard 10 5 -par_shard 10 6 -par_shard 10 7 -par_shard 10 8 -par_shard 10 9 -par_shard 2 c1 -par_shard 9 0 -par_shard 9 1 -par_shard 9 2 -par_shard 9 3 -par_shard 9 4 -par_shard 9 5 -par_shard 9 6 -par_shard 9 7 -par_shard 9 8 -par_shard 9 9 -par_shard 2 c2 -par_shard 10 1 -par_shard 10 2 -par_shard 10 3 -par_shard 10 4 -par_shard 10 5 -par_shard 10 6 -par_shard 10 7 -par_shard 10 8 -par_shard 10 9 -par_shard 2 c1 -par_shard 9 0 -par_shard 9 1 -par_shard 9 2 -par_shard 9 3 -par_shard 9 4 -par_shard 9 5 -par_shard 9 6 -par_shard 9 7 -par_shard 9 8 -par_shard 9 9 -par_shard 2 c2 -par_shard *** broken -par_shard parallel: Error: --shard requires --jobs to be higher than the number of -par_shard parallel: Error: arguments. Increase --jobs. -par_shard parallel: Error: --shard requires --jobs to be higher than the number of -par_shard parallel: Error: arguments. Increase --jobs. par_slow_args_generation ### Test slow arguments generation - https://savannah.gnu.org/bugs/?32834 par_slow_args_generation 1 par_slow_args_generation 2 @@ -382,6 +327,8 @@ par_sqlworker_hostname host par_sqlworker_hostname par_sqlworker_hostname par_sqlworker_hostname +par_sshdelay ### test --sshdelay +par_sshdelay OK par_tee_with_premature_close --tee --pipe should send all data to all commands par_tee_with_premature_close even if a command closes stdin before reading everything par_tee_with_premature_close tee with --output-error=warn-nopipe support diff --git a/testsuite/wanted-results/parallel-local-mem b/testsuite/wanted-results/parallel-local-mem dissimilarity index 63% index b6572bb3..cd5a8e6d 100644 --- a/testsuite/wanted-results/parallel-local-mem +++ b/testsuite/wanted-results/parallel-local-mem @@ -1,84 +1,30 @@ -make[1]: Entering directory '~/privat/parallel/testsuite' -parallel --tag -k 'cd vagrant/generic/{} && vagrant suspend' ::: centos8 freebsd11 freebsd12 rhel8 -centos8 ==> default: Saving VM state and suspending execution... -freebsd11 ==> default: Saving VM state and suspending execution... -freebsd12 ==> default: Saving VM state and suspending execution... -rhel8 ==> default: Saving VM state and suspending execution... -make[1]: Leaving directory '~/privat/parallel/testsuite' -par_2gb_record_reading ### Trouble reading a record > 2 GB for certain versions of Perl (substr($a,0,2G+1)="fails") -par_2gb_record_reading ### perl -e $buf=("x"x(2**31))."x"; substr($buf,0,2**31+1)=""; print length $buf -par_2gb_record_reading Eats 4 GB -par_2gb_record_reading 0 -par_2gb_record_reading Eats 4.7 GB -par_2gb_record_reading parallel: Warning: --blocksize >= 2G causes problems. Using 2G-1. -par_2gb_record_reading 081e8cf038a5265cf6294d2131a13a93 - -par_2gb_record_reading Eats 4.7 GB -par_2gb_record_reading parallel: Warning: --blocksize >= 2G causes problems. Using 2G-1. -par_2gb_record_reading 2300000004 -par_2gb_records_N ### bug #44358: 2 GB records cause problems for -N -par_2gb_records_N 5 GB version: Eats 12.5 GB RAM + 4 GB Swap -par_2gb_records_N parallel: Warning: A record was longer than 1000000000. Increasing to --blocksize 1300000001. -par_2gb_records_N parallel: Warning: A record was longer than 1300000001. Increasing to --blocksize 1690000003. -par_2gb_records_N parallel: Warning: A record was longer than 1690000003. Increasing to --blocksize 2147483647. -par_2gb_records_N 8000000007 -par_2gb_records_N 1000000001 -par_2gb_records_N 2 GB version: eats 10 GB -par_2gb_records_N parallel: Warning: A record was longer than 1000000000. Increasing to --blocksize 1300000001. -par_2gb_records_N parallel: Warning: A record was longer than 1300000001. Increasing to --blocksize 1690000003. -par_2gb_records_N parallel: Warning: A record was longer than 1690000003. Increasing to --blocksize 2147483647. -par_2gb_records_N 4600000007 -par_2gb_records_N 1000000001 -par_2gb_records_N ### -L >4GB -par_2gb_records_N Eats 12.5 GB RAM + 6 GB Swap -par_2gb_records_N parallel: Warning: A record was longer than 1000000000. Increasing to --blocksize 1300000001. -par_2gb_records_N parallel: Warning: A record was longer than 1300000001. Increasing to --blocksize 1690000003. -par_2gb_records_N parallel: Warning: A record was longer than 1690000003. Increasing to --blocksize 2147483647. -par_2gb_records_N 8000000008 -par_2gb_records_N 1000000000 -make[1]: Entering directory '~/privat/parallel/testsuite' -parallel --tag -k 'cd vagrant/generic/{} && vagrant up' ::: centos8 freebsd11 freebsd12 rhel8 -centos8 Bringing machine 'default' up with 'virtualbox' provider... -centos8 ==> default: Checking if box 'generic/centos8' version '3.0.12' is up to date... -centos8 ==> default: Resuming suspended VM... -centos8 ==> default: Booting VM... -centos8 ==> default: Waiting for machine to boot. This may take a few minutes... -centos8 default: SSH address: 127.0.0.1:2204 -centos8 default: SSH username: vagrant -centos8 default: SSH auth method: private key -centos8 ==> default: Machine booted and ready! -centos8 ==> default: Machine already provisioned. Run `vagrant provision` or use the `--provision` -centos8 ==> default: flag to force provisioning. Provisioners marked to run always will still run. -freebsd11 Bringing machine 'default' up with 'virtualbox' provider... -freebsd11 ==> default: Checking if box 'generic/freebsd11' version '3.0.12' is up to date... -freebsd11 ==> default: Resuming suspended VM... -freebsd11 ==> default: Booting VM... -freebsd11 ==> default: Waiting for machine to boot. This may take a few minutes... -freebsd11 default: SSH address: 127.0.0.1:2206 -freebsd11 default: SSH username: vagrant -freebsd11 default: SSH auth method: private key -freebsd11 ==> default: Machine booted and ready! -freebsd11 ==> default: Machine already provisioned. Run `vagrant provision` or use the `--provision` -freebsd11 ==> default: flag to force provisioning. Provisioners marked to run always will still run. -freebsd12 Bringing machine 'default' up with 'virtualbox' provider... -freebsd12 ==> default: Checking if box 'generic/freebsd12' version '3.0.12' is up to date... -freebsd12 ==> default: Resuming suspended VM... -freebsd12 ==> default: Booting VM... -freebsd12 ==> default: Waiting for machine to boot. This may take a few minutes... -freebsd12 default: SSH address: 127.0.0.1:2205 -freebsd12 default: SSH username: vagrant -freebsd12 default: SSH auth method: private key -freebsd12 ==> default: Machine booted and ready! -freebsd12 ==> default: Machine already provisioned. Run `vagrant provision` or use the `--provision` -freebsd12 ==> default: flag to force provisioning. Provisioners marked to run always will still run. -rhel8 Bringing machine 'default' up with 'virtualbox' provider... -rhel8 ==> default: Checking if box 'generic/rhel8' version '3.0.12' is up to date... -rhel8 ==> default: Resuming suspended VM... -rhel8 ==> default: Booting VM... -rhel8 ==> default: Waiting for machine to boot. This may take a few minutes... -rhel8 default: SSH address: 127.0.0.1:2203 -rhel8 default: SSH username: vagrant -rhel8 default: SSH auth method: private key -rhel8 ==> default: Machine booted and ready! -rhel8 ==> default: Machine already provisioned. Run `vagrant provision` or use the `--provision` -rhel8 ==> default: flag to force provisioning. Provisioners marked to run always will still run. -make[1]: Leaving directory '~/privat/parallel/testsuite' +par_2gb_record_reading ### Trouble reading a record > 2 GB for certain versions of Perl (substr($a,0,2G+1)="fails") +par_2gb_record_reading ### perl -e $buf=("x"x(2**31))."x"; substr($buf,0,2**31+1)=""; print length $buf +par_2gb_record_reading Eats 4 GB +par_2gb_record_reading 0 +par_2gb_record_reading Eats 4.7 GB +par_2gb_record_reading parallel: Warning: --blocksize >= 2G causes problems. Using 2G-1. +par_2gb_record_reading 081e8cf038a5265cf6294d2131a13a93 - +par_2gb_record_reading Eats 4.7 GB +par_2gb_record_reading parallel: Warning: --blocksize >= 2G causes problems. Using 2G-1. +par_2gb_record_reading 2300000004 +par_2gb_records_N ### bug #44358: 2 GB records cause problems for -N +par_2gb_records_N 5 GB version: Eats 12.5 GB RAM + 4 GB Swap +par_2gb_records_N parallel: Warning: A record was longer than 1000000000. Increasing to --blocksize 1300000001. +par_2gb_records_N parallel: Warning: A record was longer than 1300000001. Increasing to --blocksize 1690000003. +par_2gb_records_N parallel: Warning: A record was longer than 1690000003. Increasing to --blocksize 2147483647. +par_2gb_records_N 8000000007 +par_2gb_records_N 1000000001 +par_2gb_records_N 2 GB version: eats 10 GB +par_2gb_records_N parallel: Warning: A record was longer than 1000000000. Increasing to --blocksize 1300000001. +par_2gb_records_N parallel: Warning: A record was longer than 1300000001. Increasing to --blocksize 1690000003. +par_2gb_records_N parallel: Warning: A record was longer than 1690000003. Increasing to --blocksize 2147483647. +par_2gb_records_N 4600000007 +par_2gb_records_N 1000000001 +par_2gb_records_N ### -L >4GB +par_2gb_records_N Eats 12.5 GB RAM + 6 GB Swap +par_2gb_records_N parallel: Warning: A record was longer than 1000000000. Increasing to --blocksize 1300000001. +par_2gb_records_N parallel: Warning: A record was longer than 1300000001. Increasing to --blocksize 1690000003. +par_2gb_records_N parallel: Warning: A record was longer than 1690000003. Increasing to --blocksize 2147483647. +par_2gb_records_N 8000000008 +par_2gb_records_N 1000000000 diff --git a/testsuite/wanted-results/parallel-local-ssh1 b/testsuite/wanted-results/parallel-local-ssh1 index d986ab7e..080fc320 100644 --- a/testsuite/wanted-results/parallel-local-ssh1 +++ b/testsuite/wanted-results/parallel-local-ssh1 @@ -75,7 +75,7 @@ par_remote_load parallel: Warning: This job was killed because it timed out: par_remote_load parallel: Warning: perl -e 'while(1){ }' par_remote_load OK par_remote_nice ### Test --nice remote -par_remote_nice ssh lo -- exec 'perl -e '"'"'$ENV{"PARALLEL_PID"}="XXXXX";$ENV{"PARALLEL_SEQ"}="1";$ENV{"PARALLEL_SSHLOGIN"}="lo";$ENV{"PARALLEL_SSHHOST"}="lo";$ENV{"PARALLEL_JOBSLOT"}="1";$bashfunc = "";@ARGV="PAR=a bash -c \"echo \\\$PAR b\"";$shell="$ENV{SHELL}";$tmpdir="/tmp/parallel-local-ssh1-tmpdir";$nice=1;do{$ENV{PARALLEL_TMP}=$tmpdir."/par".join"",map{(0..9,"a".."z","A".."Z")[rand(62)]}(1..5);}while(-e$ENV{PARALLEL_TMP});$SIG{CHLD}=sub{$done=1;};$pid=fork;unless($pid){eval{setpgrp};eval{setpriority(0,0,$nice)};exec$shell,"-c",($bashfunc."@ARGV");die"exec:$!\n";}do{$s=$s<1?0.001+$s*1.03:$s;select(undef,undef,undef,$s);}until($done||getppid==1);kill(SIGHUP,-${pid})unless$done;wait;exit($?&127?128+($?&127):1+$?>>8)'"'"; +par_remote_nice ssh lo -- exec 'perl -e '"'"'$ENV{"PARALLEL_PID"}="XXXXX";$ENV{"PARALLEL_SEQ"}="1";$ENV{"PARALLEL_SSHLOGIN"}="lo";$ENV{"PARALLEL_SSHHOST"}="lo";$ENV{"PARALLEL_JOBSLOT"}="1";$bashfunc = "";@ARGV="PAR=a bash -c \"echo \\\$PAR b\"";$shell="$ENV{SHELL}";$tmpdir="/tmp/parallel-local-ssh1-tmpdir";$nice=1;$termseq="";do{$ENV{PARALLEL_TMP}=$tmpdir."/par".join"",map{(0..9,"a".."z","A".."Z")[rand(62)]}(1..5);}while(-e$ENV{PARALLEL_TMP});$SIG{CHLD}=sub{$done=1;};$pid=fork;unless($pid){eval{setpgrp};eval{setpriority(0,0,$nice)};exec$shell,"-c",($bashfunc."@ARGV");die"exec:$!\n";}do{$s=$s<1?0.001+$s*1.03:$s;select(undef,undef,undef,$s);}until($done||getppid==1);if(not$done){my@term_seq=split/,/,$termseq;if(not@term_seq){@term_seq=("TERM",200,"TERM",100,"TERM",50,"KILL",25);}while(@term_seq&&kill(0,-$pid)){kill(shift@term_seq,-$pid);select(undef,undef,undef,(shift@term_seq)/1000);}}wait;exit($?&127?128+($?&127):1+$?>>8)'"'"; par_remote_nice a b par_ssh ### use --ssh par_ssh Run through BARSSH? diff --git a/testsuite/wanted-results/parallel-local-ssh7 b/testsuite/wanted-results/parallel-local-ssh7 index bcd1344e..f7872bbf 100644 --- a/testsuite/wanted-results/parallel-local-ssh7 +++ b/testsuite/wanted-results/parallel-local-ssh7 @@ -1898,20 +1898,20 @@ par_csh_man work, par_csh_man work, par_csh_man {3} {3.} {3/} {3/.} {=3 perl code =} Positional replacement strings par_csh_man {} {.} {/} {/.} {#} {%} {= perl code =} Replacement strings -par_csh_funky 3 arg alias_works par_csh_funky -par_csh_funky myvar works -par_csh_funky space special chars problem +par_csh_funky +par_csh_funky +par_csh_funky 3 arg alias_works +par_csh_funky 3 arg alias_works_over_ssh par_csh_funky Funky- par_csh_funky  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€�‚ƒ„…†‡ˆ‰Š‹Œ�Ž��‘’“”•–—˜™š›œ�žŸ ¡¢£¤¥¦§¨©ª«¬­®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ-funky alias_var_works +par_csh_funky Funky- par_csh_funky  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€�‚ƒ„…†‡ˆ‰Š‹Œ�Ž��‘’“”•–—˜™š›œ�žŸ ¡¢£¤¥¦§¨©ª«¬­®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ-funky alias_var_works_over_ssh +par_csh_funky \\\\\\\\ \ \ \ \ par_csh_funky \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~€�\‚\ƒ\„\…\†\‡\ˆ\‰\Š\‹\Œ\�\Ž\�\�\‘\’\“\”\•\–\—\˜\™\š\›\œ\�\ž\Ÿ\ \¡\¢\£\¤\¥\¦\§\¨\©\ª\«\¬\­\®\¯\°\±\²\³\´\µ\¶\·\¸\¹\º\»\¼\½\¾\¿\À\Á\Â\Ã\Ä\Å\Æ\Ç\È\É\Ê\Ë\Ì\Í\Î\Ï\Ð\Ñ\Ò\Ó\Ô\Õ\Ö\×\Ø\Ù\Ú\Û\Ü\Ý\Þ\ß\à\á\â\ã\ä\å\æ\ç\è\é\ê\ë\ì\í\î\ï\ð\ñ\ò\ó\ô\õ\ö\÷\ø\ù\ú\û\ü\ý\þ\ÿ +par_csh_funky func_echo: Command not found. par_csh_funky func_echo: Command not found. -par_csh_funky 3 arg alias_works_over_ssh -par_csh_funky +par_csh_funky myvar works par_csh_funky myvar works par_csh_funky space special chars problem -par_csh_funky Funky- par_csh_funky  !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~€�‚ƒ„…†‡ˆ‰Š‹Œ�Ž��‘’“”•–—˜™š›œ�žŸ ¡¢£¤¥¦§¨©ª«¬­®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ-funky alias_var_works_over_ssh -par_csh_funky func_echo: Command not found. -par_csh_funky -par_csh_funky \\\\\\\\ \ \ \ \ par_csh_funky \\\\\\\\\\\\\\ \!\"\#\$%\&\'\(\)\*+,-./0123456789:\;\<\=\>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\\\]\^_\`abcdefghijklmnopqrstuvwxyz\{\|\}\~€�\‚\ƒ\„\…\†\‡\ˆ\‰\Š\‹\Œ\�\Ž\�\�\‘\’\“\”\•\–\—\˜\™\š\›\œ\�\ž\Ÿ\ \¡\¢\£\¤\¥\¦\§\¨\©\ª\«\¬\­\®\¯\°\±\²\³\´\µ\¶\·\¸\¹\º\»\¼\½\¾\¿\À\Á\Â\Ã\Ä\Å\Æ\Ç\È\É\Ê\Ë\Ì\Í\Î\Ï\Ð\Ñ\Ò\Ó\Ô\Õ\Ö\×\Ø\Ù\Ú\Û\Ü\Ý\Þ\ß\à\á\â\ã\ä\å\æ\ç\è\é\ê\ë\ì\í\î\ï\ð\ñ\ò\ó\ô\õ\ö\÷\ø\ù\ú\û\ü\ý\þ\ÿ +par_csh_funky space special chars problem par_csh_environment_too_big Not implemented par_csh_env_parallel_session Not implemented par_csh_env_parallel bug #50435: Remote fifo broke in 20150522 diff --git a/testsuite/wanted-results/parallel-local-ssh8 b/testsuite/wanted-results/parallel-local-ssh8 index 24b1a73b..aa5833d8 100644 --- a/testsuite/wanted-results/parallel-local-ssh8 +++ b/testsuite/wanted-results/parallel-local-ssh8 @@ -15,14 +15,14 @@ par_keeporder job2 par_load_csh ### Gave Word too long. par_load_csh a par_path_remote_bash bug #47695: How to set $PATH on remote? Bash -par_path_remote_bash logged in +par_path_remote_bash StArT par_path_remote_bash BASH Path before: /bin:/usr/bin with no parallel par_path_remote_bash -bash: line 3: parallel: command not found par_path_remote_bash ^^^^^^^^ Not found is OK par_path_remote_bash /bin:/usr/bin:/tmp OK par_path_remote_bash par_path_remote_csh bug #47695: How to set $PATH on remote? csh -par_path_remote_csh logged in +par_path_remote_csh StArT par_path_remote_csh CSH Path before: /bin:/usr/bin with no parallel par_path_remote_csh parallel: Command not found. par_path_remote_csh ^^^^^^^^ Not found is OK diff --git a/testsuite/wanted-results/parallel-local164 b/testsuite/wanted-results/parallel-local164 dissimilarity index 97% index c5153220..29bf8e04 100644 --- a/testsuite/wanted-results/parallel-local164 +++ b/testsuite/wanted-results/parallel-local164 @@ -1,312 +1,5 @@ -echo "### Test --delay" -### Test --delay -seq 9 | /usr/bin/time -f %e parallel -j3 --delay 0.57 true {} 2>&1 | perl -ne '$_ > 3.3 and print "More than 3.3 secs: OK\n"' -More than 3.3 secs: OK -echo '### test --sshdelay' -### test --sshdelay - stdout /usr/bin/time -f %e parallel -j0 --sshdelay 0.5 -S localhost true ::: 1 2 3 | perl -ne 'print($_ > 1.30 ? "OK\n" : "Not OK\n")' -OK -echo "bug #37694: Empty string argument skipped when using --quote" -bug #37694: Empty string argument skipped when using --quote - parallel -q --nonall perl -le 'print scalar @ARGV' 'a' 'b' '' -3 -echo '### Test -k 5'; sleep 5 -### Test -k 5 -echo '### Test -k 3'; sleep 3 -### Test -k 3 -echo '### Test -k 4'; sleep 4 -### Test -k 4 -echo '### Test -k 2'; sleep 2 -### Test -k 2 -echo '### Test -k 1'; sleep 1 -### Test -k 1 -echo "### Computing length of command line" -### Computing length of command line - seq 1 2 | parallel -k -N2 echo {1} {2} -1 2 - parallel --xapply -k -a <(seq 11 12) -a <(seq 1 3) echo -11 1 -12 2 -11 3 - parallel -k -C %+ echo '"{1}_{3}_{2}_{4}"' ::: 'a% c %%b' 'a%c% b %d' -a_b_c_ -a_b_c_d - parallel -k -C %+ echo {4} ::: 'a% c %%b' - -echo "### test08" -### test08 - cd input-files/test08; ls | parallel -q perl -ne '/_PRE (\d+)/ and $p=$1; /hatchname> (\d+)/ and $1!=$p and print $ARGV,"\n"' | sort -b -d -seq 1 10 | parallel -j 1 echo | sort -1 -10 -2 -3 -4 -5 -6 -7 -8 -9 -seq 1 10 | parallel -j 2 echo | sort -1 -10 -2 -3 -4 -5 -6 -7 -8 -9 -seq 1 10 | parallel -j 3 echo | sort -1 -10 -2 -3 -4 -5 -6 -7 -8 -9 -echo "bug #37956: --colsep does not default to '\t' as specified in the man page." -bug #37956: --colsep does not default to '\t' as specified in the man page. - printf "A\tB\n1\tone" | parallel --header : echo {B} {A} -one 1 -echo '### Test --tollef' -### Test --tollef - stdout parallel -k --tollef echo -- 1 2 3 ::: a b c | LC_ALL=C sort -parallel: Error: --tollef has been retired. -parallel: Error: Remove --tollef or use --gnu to override --tollef. -echo '### Test --tollef --gnu' -### Test --tollef --gnu - stdout parallel -k --tollef --gnu echo ::: 1 2 3 -- a b c -1 -2 -3 --- -a -b -c -echo '### Test --gnu' -### Test --gnu - parallel -k --gnu echo ::: 1 2 3 -- a b c -1 -2 -3 --- -a -b -c -echo '### Test {//}' -### Test {//} - parallel -k echo {//} {} ::: a a/b a/b/c -. a -a a/b -a/b a/b/c - parallel -k echo {//} {} ::: /a /a/b /a/b/c -/ /a -/a /a/b -/a/b /a/b/c - parallel -k echo {//} {} ::: ./a ./a/b ./a/b/c -. ./a -./a ./a/b -./a/b ./a/b/c - parallel -k echo {//} {} ::: a.jpg a/b.jpg a/b/c.jpg -. a.jpg -a a/b.jpg -a/b a/b/c.jpg - parallel -k echo {//} {} ::: /a.jpg /a/b.jpg /a/b/c.jpg -/ /a.jpg -/a /a/b.jpg -/a/b /a/b/c.jpg - parallel -k echo {//} {} ::: ./a.jpg ./a/b.jpg ./a/b/c.jpg -. ./a.jpg -./a ./a/b.jpg -./a/b ./a/b/c.jpg -echo '### Test {1//}' -### Test {1//} - parallel -k echo {1//} {} ::: a a/b a/b/c -. a -a a/b -a/b a/b/c - parallel -k echo {1//} {} ::: /a /a/b /a/b/c -/ /a -/a /a/b -/a/b /a/b/c - parallel -k echo {1//} {} ::: ./a ./a/b ./a/b/c -. ./a -./a ./a/b -./a/b ./a/b/c - parallel -k echo {1//} {} ::: a.jpg a/b.jpg a/b/c.jpg -. a.jpg -a a/b.jpg -a/b a/b/c.jpg - parallel -k echo {1//} {} ::: /a.jpg /a/b.jpg /a/b/c.jpg -/ /a.jpg -/a /a/b.jpg -/a/b /a/b/c.jpg - parallel -k echo {1//} {} ::: ./a.jpg ./a/b.jpg ./a/b/c.jpg -. ./a.jpg -./a ./a/b.jpg -./a/b ./a/b/c.jpg -echo '### Test --dnr' -### Test --dnr - parallel --dnr II -k echo II {} ::: a a/b a/b/c -. a -a a/b -a/b a/b/c -echo '### Test --dirnamereplace' -### Test --dirnamereplace - parallel --dirnamereplace II -k echo II {} ::: a a/b a/b/c -. a -a a/b -a/b a/b/c -echo '### Test https://savannah.gnu.org/bugs/index.php?31716' -### Test https://savannah.gnu.org/bugs/index.php?31716 - seq 1 5 | stdout parallel -k -l echo {} OK -1 OK -2 OK -3 OK -4 OK -5 OK - seq 1 5 | stdout parallel -k -l 1 echo {} OK -1 OK -2 OK -3 OK -4 OK -5 OK -echo '### -k -l -0' -### -k -l -0 - printf '1\0002\0003\0004\0005\000' | stdout parallel -k -l -0 echo {} OK -1 OK -2 OK -3 OK -4 OK -5 OK -echo '### -k -0 -l' -### -k -0 -l - printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -l echo {} OK -1 OK -2 OK -3 OK -4 OK -5 OK -echo '### -k -0 -l 1' -### -k -0 -l 1 - printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -l 1 echo {} OK -1 OK -2 OK -3 OK -4 OK -5 OK -echo '### -k -0 -l 0' -### -k -0 -l 0 - printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -l 0 echo {} OK -1 OK -2 OK -3 OK -4 OK -5 OK -echo '### -k -0 -L -0 - -0 is argument for -L' -### -k -0 -L -0 - -0 is argument for -L - printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -L -0 echo {} OK -OK -OK -OK -OK -OK -echo '### -k -0 -L 0 - -L always takes arg' -### -k -0 -L 0 - -L always takes arg - printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -L 0 echo {} OK -OK -OK -OK -OK -OK -echo '### -k -0 -L 0 - -L always takes arg' -### -k -0 -L 0 - -L always takes arg - printf '1\0002\0003\0004\0005\000' | stdout parallel -k -L 0 -0 echo {} OK -OK -OK -OK -OK -OK -echo '### -k -e -0' -### -k -e -0 - printf '1\0002\0003\0004\0005\000' | stdout parallel -k -e -0 echo {} OK -1 OK -2 OK -3 OK -4 OK -5 OK -echo '### -k -0 -e eof' -### -k -0 -e eof - printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -e eof echo {} OK -1 OK -2 OK -3 OK -4 OK -5 OK -echo '### -k -i -0' -### -k -i -0 - printf '1\0002\0003\0004\0005\000' | stdout parallel -k -i -0 echo {} OK -1 OK -2 OK -3 OK -4 OK -5 OK -echo '### -k -0 -i repl' -### -k -0 -i repl - printf '1\0002\0003\0004\0005\000' | stdout parallel -k -0 -i repl echo repl OK -1 OK -2 OK -3 OK -4 OK -5 OK -echo '### Negative replacement strings' -### Negative replacement strings - parallel -X -j1 -N 6 echo {-1}orrec{1} ::: t B X D E c -correct - parallel -N 6 echo {-1}orrect ::: A B X D E c -correct - parallel --colsep ' ' echo '{2} + {4} = {2} + {-1}=' '$(( {2} + {-1} ))' ::: "1 2 3 4" -2 + 4 = 2 + 4= 6 - parallel --colsep ' ' echo '{-3}orrect' ::: "1 c 3 4" -correct -echo 'bug #38439: "open files" with --files --pipe blocks after a while' -bug #38439: "open files" with --files --pipe blocks after a while - ulimit -n 28; yes "`seq 3000`" |head -c 20M | parallel --pipe -k echo {#} of 21 -1 of 21 -2 of 21 -3 of 21 -4 of 21 -5 of 21 -6 of 21 -7 of 21 -8 of 21 -9 of 21 -10 of 21 -11 of 21 -12 of 21 -13 of 21 -14 of 21 -15 of 21 -16 of 21 -17 of 21 -18 of 21 -19 of 21 -20 of 21 -21 of 21 -parallel: Warning: Only enough file handles to run 2 jobs in parallel. -parallel: Warning: Running 'parallel -j0 -N 2 --pipe parallel -j0' or -parallel: Warning: raising 'ulimit -n' or 'nofile' in /etc/security/limits.conf -parallel: Warning: or /proc/sys/fs/file-max may help. -echo 'bug #34241: --pipe should not spawn unneeded processes - part 2' -bug #34241: --pipe should not spawn unneeded processes - part 2 - seq 500 | parallel --tmpdir . -j10 --pipe --block 1k --files wc >/dev/null; ls *.par | wc -l; rm *.par; seq 500 | parallel --tmpdir . -j10 --pipe --block 1k --files --dry-run wc >/dev/null; echo No .par should exist; stdout ls *.par -2 -No .par should exist -ls: cannot access '*.par': No such file or directory +echo "### test08" +### test08 + cd input-files/test08; ls | parallel -q perl -ne '/_PRE (\d+)/ and $p=$1; /hatchname> (\d+)/ and $1!=$p and print $ARGV,"\n"' | sort +b +d diff --git a/testsuite/wanted-results/parallel-macos b/testsuite/wanted-results/parallel-macos index 52cf9de3..8965edb0 100644 --- a/testsuite/wanted-results/parallel-macos +++ b/testsuite/wanted-results/parallel-macos @@ -1,17 +1,17 @@ -par_big_func 1 7968 191232 -par_big_func 1 5368 128772 -par_big_func_name 1 4964 119136 -par_big_func_name 1 36 864 +par_big_func 1 7960 191040 +par_big_func 1 5376 128964 +par_big_func_name 1 4952 118848 +par_big_func_name 1 48 1152 par_big_var_func_name 1 4960 119040 par_big_var_func_name 1 4960 119040 par_big_var_func_name 1 3416 81924 -par_many_args 1 16408 32816 -par_many_args 1 3592 7184 +par_many_args 1 16376 32752 +par_many_args 1 3624 7248 par_many_func 1 4376 105024 par_many_func 1 2292 54980 -par_many_var 1 5128 123072 -par_many_var 1 1540 36932 -par_many_var_big_func 1 4408 105792 -par_many_var_big_func 1 2260 54212 +par_many_var 1 5116 122784 +par_many_var 1 1552 37220 +par_many_var_big_func 1 4388 105312 +par_many_var_big_func 1 2280 54692 par_many_var_func 1 6628 159072 par_many_var_func 1 40 932 diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial index f4b99f41..12ef1a86 100644 --- a/testsuite/wanted-results/parallel-tutorial +++ b/testsuite/wanted-results/parallel-tutorial @@ -845,6 +845,7 @@ PARALLEL_JOBSLOT PARALLEL_PID PARALLEL_RSYNC_OPTS PARALLEL_SEQ +PARALLEL_SSH PARALLEL_SSHHOST PARALLEL_SSHLOGIN PARALLEL_TMP @@ -954,7 +955,7 @@ For details: see man env_parallel export -f my_func3 parallel -vv --workdir ... --nice 17 --env _ --trc {}.out \ -S $SERVER1 my_func3 {} ::: abc-file -ssh -l parallel lo -- mkdir -p ./.TMPWORKDIR && rsync --protocol 30 -rlDzR -e'ssh -l parallel' ./abc-file lo:./.TMPWORKDIR;ssh -l parallel lo -- exec perl -e "'"'@GNU_Parallel=split/_/,"use_IPC::Open3;_use_MIME::Base64";eval"@GNU_Parallel";$chld=$SIG{CHLD};$SIG{CHLD}="IGNORE";my$zip=(grep{-x$_}"/usr/local/bin/bzip2")[0]||"bzip2";open3($in,$out,">&STDERR",$zip,"-dc");if(my$perlpid=fork){close$in;$eval=join"",<$out>;close$out;}else{close$out;print$in(decode_base64(join"",@ARGV));close$in;exit;}wait;$SIG{CHLD}=$chld;eval$eval;'"'" BASE64;_EXIT_status=$?; mkdir -p ./. && rsync --protocol 30 --rsync-path='cd ./.TMPWORKDIR/./.; rsync' -rlDzR -e'ssh -l parallel' lo:./abc-file.out ./.;ssh -l parallel lo -- 'rm -f ./.TMPWORKDIR/abc-file; sh -c '"'"'rmdir ./.TMPWORKDIR/ ./.parallel/tmp/ ./.parallel/ 2>/dev/null;'"'"''"'"'rm -rf ./.TMPWORKDIR;'"'";ssh -l parallel lo -- 'rm -f ./.TMPWORKDIR/abc-file.out; sh -c '"'"'rmdir ./.TMPWORKDIR/ ./.parallel/tmp/ ./.parallel/ 2>/dev/null;'"'"''"'"'rm -rf ./.TMPWORKDIR;'"'";ssh -l parallel lo -- rm -rf .TMPWORKDIR;exit $_EXIT_status; +ssh -oLogLevel=quiet -l parallel lo -- mkdir -p ./.TMPWORKDIR && rsync --protocol 30 -rlDzR -e'ssh -oLogLevel=quiet -l parallel' ./abc-file lo:./.TMPWORKDIR;ssh -oLogLevel=quiet -l parallel lo -- exec perl -e "'"'@GNU_Parallel=split/_/,"use_IPC::Open3;_use_MIME::Base64";eval"@GNU_Parallel";$chld=$SIG{CHLD};$SIG{CHLD}="IGNORE";my$zip=(grep{-x$_}"/usr/local/bin/bzip2")[0]||"bzip2";open3($in,$out,">&STDERR",$zip,"-dc");if(my$perlpid=fork){close$in;$eval=join"",<$out>;close$out;}else{close$out;print$in(decode_base64(join"",@ARGV));close$in;exit;}wait;$SIG{CHLD}=$chld;eval$eval;'"'" BASE64;_EXIT_status=$?; mkdir -p ./. && rsync --protocol 30 --rsync-path='cd ./.TMPWORKDIR/./.; rsync' -rlDzR -e'ssh -oLogLevel=quiet -l parallel' lo:./abc-file.out ./.;ssh -oLogLevel=quiet -l parallel lo -- 'rm -f ./.TMPWORKDIR/abc-file; sh -c '"'"'rmdir ./.TMPWORKDIR/ ./.parallel/tmp/ ./.parallel/ 2>/dev/null;'"'"''"'"'rm -rf ./.TMPWORKDIR;'"'";ssh -oLogLevel=quiet -l parallel lo -- 'rm -f ./.TMPWORKDIR/abc-file.out; sh -c '"'"'rmdir ./.TMPWORKDIR/ ./.parallel/tmp/ ./.parallel/ 2>/dev/null;'"'"''"'"'rm -rf ./.TMPWORKDIR;'"'";ssh -oLogLevel=quiet -l parallel lo -- rm -rf .TMPWORKDIR;exit $_EXIT_status; parset myvar1,myvar2 echo ::: a b echo $myvar1 echo $myvar2 @@ -1364,7 +1365,7 @@ With --plus: {} = {+/}/{/} = {.}.{+.} = {+/}/{/.}.{+.} = {..}.{+..} = --pipe Split stdin (standard input) to multiple jobs. --recend str Record end separator for --pipe. --recstart str Record start separator for --pipe. -See 'man parallel' for details +GNU Parallel can do much more. See 'man parallel' for details Academic tradition requires you to cite works you base your article on. If you use programs that use GNU Parallel to process data for an article in a scientific publication, please cite: @@ -1424,4 +1425,4 @@ C echo A echo B echo C -7 +8 diff --git a/testsuite/wanted-results/sql01 b/testsuite/wanted-results/sql01 index ea1f3d06..4f1a090f 100644 --- a/testsuite/wanted-results/sql01 +++ b/testsuite/wanted-results/sql01 @@ -39,8 +39,8 @@ def 3 2 17 2 17 ### Test --db-size --dbsize - 2 6 48 - 2 6 48 +6 +6 ### Test --table-size --tablesize 81 81 diff --git a/testsuite/wanted-results/test60 b/testsuite/wanted-results/test60 index 8dfbe1d3..3f585418 100644 --- a/testsuite/wanted-results/test60 +++ b/testsuite/wanted-results/test60 @@ -56,7 +56,7 @@ echo '### Test --nonall'; parallel --nonall -k -S vagrant@parallel-server1,vag ### Test --nonall centos8.localdomain freebsd11.localdomain -echo '### Test --nonall -u - should be interleaved x y x y'; parallel --nonall -S vagrant@parallel-server1,vagrant@parallel-server2 -u 'hostname|grep -q redhat9 && sleep 3; hostname;sleep 12;hostname;' +echo '### Test --nonall -u - should be interleaved x y x y'; parallel --nonall --sshdelay 2 -S vagrant@parallel-server1,vagrant@parallel-server2 -u 'hostname|grep -q centos && sleep 2; hostname;sleep 4;hostname;' ### Test --nonall -u - should be interleaved x y x y freebsd11.localdomain centos8.localdomain -- 2.11.4.GIT