From 35f539696ecd41c6fce6a33d631949f051816ea7 Mon Sep 17 00:00:00 2001 From: Ole Tange Date: Tue, 22 Jun 2021 19:40:07 +0200 Subject: [PATCH] Released as 20210622 ('Protasevich') --- NEWS | 24 +++++ README | 24 ++--- configure | 20 ++-- configure.ac | 2 +- doc/haikus | 8 +- doc/release_new_version | 76 ++++++++++++--- packager/obs/home:tange/parallel/.osc/_files | 14 +-- .../obs/home:tange/parallel/.osc/parallel.spec | 2 +- src/env_parallel | 26 ++--- src/env_parallel.ash | 2 +- src/env_parallel.bash | 4 +- src/env_parallel.dash | 2 +- src/env_parallel.ksh | 2 +- src/env_parallel.mksh | 2 +- src/env_parallel.sh | 2 +- src/env_parallel.zsh | 10 +- src/niceload | 2 +- src/parallel | 35 ++++--- src/parallel.pod | 4 +- src/parallel_alternatives.pod | 16 ++- src/parset | 24 ++--- src/parset.pod | 12 +++ src/parsort | 2 +- src/sql | 2 +- testsuite/tests-to-run/parallel-android.sh | 6 +- testsuite/wanted-results/parallel-tutorial | 108 ++++++++++----------- 26 files changed, 265 insertions(+), 166 deletions(-) rewrite packager/obs/home:tange/parallel/.osc/_files (97%) diff --git a/NEWS b/NEWS index c1cdf616..c640cb88 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,27 @@ +20210622 + +New in this release: + +* Bug fixes and man page updates. + +News about GNU Parallel: + +* How to use GNU Parallel + https://techtipbits.com/linux/how-to-use-gnu-parallel/ + +* How to Speed Up Bash Scripts with Multithreading and GNU Parallel + https://adamtheautomator.com/how-to-speed-up-bash-scripts-with-multithreading-and-gnu-parallel/ + +* Use Parallel to split by line + https://madflex.de/use-parallel-to-split-by-line/ + +* Optimizing long batch processes or ETL by using buff/cache properly + II (parallelizing network operations) + http://www.elsotanillo.net/2021/06/optimizing-long-batch-processes-or-etl-by-using-buff-cache-properly-ii-parallelizing-network-operations/ + +* Parallelization 3: GNU Parallel https://www.youtube.com/watch?v=Rl06WD60afA + + 20210522 New in this release: diff --git a/README b/README index 66aab963..2bd04add 100644 --- a/README +++ b/README @@ -57,11 +57,11 @@ document. Full installation of GNU Parallel is as simple as: - wget https://ftpmirror.gnu.org/parallel/parallel-20210522.tar.bz2 - wget https://ftpmirror.gnu.org/parallel/parallel-20210522.tar.bz2.sig - gpg parallel-20210522.tar.bz2.sig - bzip2 -dc parallel-20210522.tar.bz2 | tar xvf - - cd parallel-20210522 + wget https://ftpmirror.gnu.org/parallel/parallel-20210622.tar.bz2 + wget https://ftpmirror.gnu.org/parallel/parallel-20210622.tar.bz2.sig + gpg parallel-20210622.tar.bz2.sig + bzip2 -dc parallel-20210622.tar.bz2 | tar xvf - + cd parallel-20210622 ./configure && make && sudo make install @@ -70,11 +70,11 @@ Full installation of GNU Parallel is as simple as: If you are not root you can add ~/bin to your path and install in ~/bin and ~/share: - wget https://ftpmirror.gnu.org/parallel/parallel-20210522.tar.bz2 - wget https://ftpmirror.gnu.org/parallel/parallel-20210522.tar.bz2.sig - gpg parallel-20210522.tar.bz2.sig - bzip2 -dc parallel-20210522.tar.bz2 | tar xvf - - cd parallel-20210522 + wget https://ftpmirror.gnu.org/parallel/parallel-20210622.tar.bz2 + wget https://ftpmirror.gnu.org/parallel/parallel-20210622.tar.bz2.sig + gpg parallel-20210622.tar.bz2.sig + bzip2 -dc parallel-20210622.tar.bz2 | tar xvf - + cd parallel-20210622 ./configure --prefix=$HOME && make && make install Or if your system lacks 'make' you can simply copy src/parallel @@ -122,8 +122,8 @@ will love you for it. When using programs that use GNU Parallel to process data for publication please cite: - Tange, O. (2021, May 22). GNU Parallel 20210522 ('Gaza'). - Zenodo. https://doi.org/10.5281/zenodo.4781603 + Tange, O. (2021, June 22). GNU Parallel 20210622 ('Protasevich'). + Zenodo. https://doi.org/10.5281/zenodo.5013933 Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ole Tange, http://ole.tange.dk and diff --git a/configure b/configure index 0eb0afe6..50517cb3 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for parallel 20210522. +# Generated by GNU Autoconf 2.69 for parallel 20210622. # # Report bugs to . # @@ -579,8 +579,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='parallel' PACKAGE_TARNAME='parallel' -PACKAGE_VERSION='20210522' -PACKAGE_STRING='parallel 20210522' +PACKAGE_VERSION='20210622' +PACKAGE_STRING='parallel 20210622' PACKAGE_BUGREPORT='bug-parallel@gnu.org' PACKAGE_URL='' @@ -1214,7 +1214,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures parallel 20210522 to adapt to many kinds of systems. +\`configure' configures parallel 20210622 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1281,7 +1281,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of parallel 20210522:";; + short | recursive ) echo "Configuration of parallel 20210622:";; esac cat <<\_ACEOF @@ -1357,7 +1357,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -parallel configure 20210522 +parallel configure 20210622 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1374,7 +1374,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by parallel $as_me 20210522, which was +It was created by parallel $as_me 20210622, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2237,7 +2237,7 @@ fi # Define the identity of the package. PACKAGE='parallel' - VERSION='20210522' + VERSION='20210622' cat >>confdefs.h <<_ACEOF @@ -2880,7 +2880,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by parallel $as_me 20210522, which was +This file was extended by parallel $as_me 20210622, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -2942,7 +2942,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -parallel config.status 20210522 +parallel config.status 20210622 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 472d87cb..64bfef5b 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([parallel], [20210522], [bug-parallel@gnu.org]) +AC_INIT([parallel], [20210622], [bug-parallel@gnu.org]) AM_INIT_AUTOMAKE([-Wall -Werror foreign]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_FILES([ diff --git a/doc/haikus b/doc/haikus index a9737475..cbedb69a 100644 --- a/doc/haikus +++ b/doc/haikus @@ -10,10 +10,6 @@ Quote of the month: parallel is my new kink -- python_noob_001@reddit - GNU Parallel makes my life so much easier. - I'm glad I don't have to implement multi-threaded Python scripts on the regular. - -- Fredrick Brennan @fr_brennan@twitter - @GnuParallel the best thing ever and it's not up for debate #EOchat -- Nathan Thomas @DrNASApants@twitter @@ -121,6 +117,10 @@ https://negfeedback.blogspot.com/2020/05/indispensable-command-line-tools.html === Used === + GNU Parallel makes my life so much easier. + I'm glad I don't have to implement multi-threaded Python scripts on the regular. + -- Fredrick Brennan @fr_brennan@twitter + If you work with lots of files at once Take a good look at GNU parallel Change your life for the better diff --git a/doc/release_new_version b/doc/release_new_version index 8a0d1bf8..fea59f1a 100644 --- a/doc/release_new_version +++ b/doc/release_new_version @@ -101,10 +101,53 @@ Thumbnail: https://www.gnu.org/software/parallel/logo-gray+black10000.png Tags: gnu parallel software -file_path="parallel-20210422.tar.bz2" -name="GNU Parallel $YYYYMMDD ('$SPCTAG')" -title="$name" -description="$description" +file_path="parallel-20210622.tar.bz2" +title="GNU Parallel $YYYYMMDD ('$SPCTAG') [stable]" +name="GNU-Parallel-$YYYYMMDD-$SPCTAG" +author="Ole Tange" +license="GNU GPLv3 or later" +thumbnail_url=https://www.gnu.org/software/parallel/logo-gray+black10000.png +channel_name="@GnuParallel" +tags_opt='--tag gnu --tag parallel --tag free --tag software' +description="An easy way to support GNU Parallel is to tip on LBRY. + +GNU Parallel 20210622 ('Protasevich') [stable] has been released. It is available for download at: lbry://@GnuParallel:4 + +No new functionality was introduced so this is a good candidate for a stable release. + +Please help spreading GNU Parallel by making a testimonial video like Juan Sierra Pons: http://www.elsotanillo.net/wp-content/uploads/GnuParallel_JuanSierraPons.mp4 + +It does not have to be as detailed as Juan's. It is perfectly fine if you just say your name, and what field you are using GNU Parallel for. + +Quote of the month: + + GNU Parallel makes my life so much easier. + I'm glad I don't have to implement multi-threaded Python scripts on the regular. + -- Fredrick Brennan @fr_brennan@twitter + +New in this release: + +* Bug fixes and man page updates. + +News about GNU Parallel: + +* How to use GNU Parallel https://techtipbits.com/linux/how-to-use-gnu-parallel/ + +* How to Speed Up Bash Scripts with Multithreading and GNU Parallel https://adamtheautomator.com/how-to-speed-up-bash-scripts-with-multithreading-and-gnu-parallel/ + +* Use Parallel to split by line https://madflex.de/use-parallel-to-split-by-line/ + +* Optimizing long batch processes or ETL by using buff/cache properly II (parallelizing network operations) http://www.elsotanillo.net/2021/06/optimizing-long-batch-processes-or-etl-by-using-buff-cache-properly-ii-parallelizing-network-operations/ + +* Parallelization 3: GNU Parallel https://www.youtube.com/watch?v=Rl06WD60afA + +Get the book: GNU Parallel 2018 http://www.lulu.com/shop/ole-tange/gnu-parallel-2018/paperback/product-23558902.html + +GNU Parallel - For people who live life in the parallel lane. + +If you like GNU Parallel record a video testimonial: Say who you are, what you use GNU Parallel for, how it helps you, and what you like most about it. Include a command that uses GNU Parallel if you feel like it. +" + lbrynet publish \ --bid=0.01 \ @@ -117,9 +160,9 @@ lbrynet publish \ $tags_opt \ --license="$license" \ --thumbnail_url="$thumbnail_url" \ - --release_time="$release_time" \ --channel_name="$channel_name" \ + --release_time="$release_time" \ == Update website == @@ -241,9 +284,9 @@ from:tange@gnu.org to:parallel@gnu.org, bug-parallel@gnu.org stable-bcc: Jesse Alama -Subject: GNU Parallel 20210522 ('Gaza/Lag Ba'Omer celebrations at Mt. Meron Israel/Michael Collins <<>>') released <<[stable]>> +Subject: GNU Parallel 20210622 ('Protasevich') released <<[stable]>> -GNU Parallel 20210522 ('<<>>') <<[stable]>> has been released. It is available for download at: lbry://@GnuParallel:4 +GNU Parallel 20210622 ('Protasevich') <<[stable]>> has been released. It is available for download at: lbry://@GnuParallel:4 <> @@ -253,22 +296,27 @@ It does not have to be as detailed as Juan's. It is perfectly fine if you just s Quote of the month: + GNU Parallel makes my life so much easier. + I'm glad I don't have to implement multi-threaded Python scripts on the regular. + -- Fredrick Brennan @fr_brennan@twitter <<>> New in this release: -* --plus includes {%%regexp} and {##regexp}. - * Bug fixes and man page updates. News about GNU Parallel: -* Batch Calculate and Verify MD5 Checksum With GNU Parallel https://omicx.cc/posts/2021-04-28-calculate-and-verify-md5-checksum-with-gnu-parallel/ -* HerrComp Gnu parallel, c++11 threads 2021 04 28 https://www.youtube.com/watch?v=wDd9F9nn0qA -* Distributing embarrassingly parallel tasks GNU Parallel https://ulhpc-tutorials.readthedocs.io/en/latest/sequential/gnu-parallel/ -* Job Parallelization on Niagara https://www.maryamdaryalal.com/post/job-parallelization-on-niagara +* How to use GNU Parallel https://techtipbits.com/linux/how-to-use-gnu-parallel/ + +* How to Speed Up Bash Scripts with Multithreading and GNU Parallel https://adamtheautomator.com/how-to-speed-up-bash-scripts-with-multithreading-and-gnu-parallel/ + * Use Parallel to split by line https://madflex.de/use-parallel-to-split-by-line/ -* m1 multi-core batch convert with gpu parallel + ffmpeg https://www.youtube.com/watch?v=hAuc0YsXv6A + +* Optimizing long batch processes or ETL by using buff/cache properly II (parallelizing network operations) http://www.elsotanillo.net/2021/06/optimizing-long-batch-processes-or-etl-by-using-buff-cache-properly-ii-parallelizing-network-operations/ + +* Parallelization 3: GNU Parallel https://www.youtube.com/watch?v=Rl06WD60afA + <<>> diff --git a/packager/obs/home:tange/parallel/.osc/_files b/packager/obs/home:tange/parallel/.osc/_files dissimilarity index 97% index b703f41c..20b1ffa4 100644 --- a/packager/obs/home:tange/parallel/.osc/_files +++ b/packager/obs/home:tange/parallel/.osc/_files @@ -1,7 +1,7 @@ - - - - - - - + + + + + + + diff --git a/packager/obs/home:tange/parallel/.osc/parallel.spec b/packager/obs/home:tange/parallel/.osc/parallel.spec index 58e07a7c..af6924fb 100644 --- a/packager/obs/home:tange/parallel/.osc/parallel.spec +++ b/packager/obs/home:tange/parallel/.osc/parallel.spec @@ -1,7 +1,7 @@ Summary: Shell tool for executing jobs in parallel Name: parallel -Version: 20210522 +Version: 20210622 Release: 1.3 License: GPL-3.0-or-later Group: Productivity/File utilities diff --git a/src/env_parallel b/src/env_parallel index 71c24a43..d8e9f280 100755 --- a/src/env_parallel +++ b/src/env_parallel @@ -83,45 +83,45 @@ env_parallel only works if it is a function. Do the below and restart your shell bash: Put this in $HOME/.bashrc: . `which env_parallel.bash` E.g. by doing: echo '. `which env_parallel.bash`' >> $HOME/.bashrc - Supports: aliases, functions, variables, arrays - -zsh: Put this in $HOME/.zshrc: . `which env_parallel.zsh` - E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshenv - Supports: functions, variables, arrays + Supports: variables, aliases, functions, arrays fish: Put this in $HOME/.config/fish/config.fish: . (which env_parallel.fish) E.g. by doing: echo '. (which env_parallel.fish)' >> $HOME/.config/fish/config.fish - Supports: aliases, functions, variables, arrays + Supports: variables, aliases, functions, arrays ksh: Put this in $HOME/.kshrc: source `which env_parallel.ksh` E.g. by doing: echo 'source `which env_parallel.ksh`' >> $HOME/.kshrc - Supports: aliases, functions, variables, arrays + Supports: variables, aliases, functions, arrays mksh: Put this in $HOME/.mkshrc: source `which env_parallel.mksh` E.g. by doing: echo 'source `which env_parallel.mksh`' >> $HOME/.mkshrc - Supports: aliases, functions, variables, arrays + Supports: variables, aliases, functions, arrays pdksh: Put this in $HOME/.profile: source `which env_parallel.pdksh` E.g. by doing: echo '. `which env_parallel.pdksh`' >> $HOME/.profile - Supports: aliases, functions, variables, arrays + Supports: variables, aliases, functions, arrays + +zsh: Put this in $HOME/.zshrc: . `which env_parallel.zsh` + E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshenv + Supports: variables, functions, arrays ash: Put this in $HOME/.profile: . `which env_parallel.ash` E.g. by doing: echo '. `which env_parallel.ash`' >> $HOME/.profile - Supports: aliases, variables + Supports: variables, aliases dash: Put this in $HOME/.profile: . `which env_parallel.dash` E.g. by doing: echo '. `which env_parallel.dash`' >> $HOME/.profile - Supports: aliases, variables + Supports: variables, aliases csh: Put this in $HOME/.cshrc: source `which env_parallel.csh` E.g. by doing: echo 'source `which env_parallel.csh`' >> $HOME/.cshrc - Supports: aliases, variables, arrays with no special chars + Supports: variables, aliases, arrays with no special chars tcsh: Put this in $HOME/.tcshrc: source `which env_parallel.tcsh` E.g. by doing: echo 'source `which env_parallel.tcsh`' >> $HOME/.tcshrc - Supports: aliases, variables, arrays with no special chars + Supports: variables, aliases, arrays with no special chars To install in all shells run: diff --git a/src/env_parallel.ash b/src/env_parallel.ash index 949f2af5..814bc197 100755 --- a/src/env_parallel.ash +++ b/src/env_parallel.ash @@ -385,7 +385,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20210522 (GNU parallel `parallel --minversion 1`)" + echo "parset 20210622 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.bash b/src/env_parallel.bash index 4e6e0d70..328b361a 100755 --- a/src/env_parallel.bash +++ b/src/env_parallel.bash @@ -391,7 +391,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20210522 (GNU parallel `parallel --minversion 1`)" + echo "parset 20210622 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " @@ -429,7 +429,7 @@ _parset_main() { ($_parset_PARALLEL_PRG --files -k "$@"; echo $? > "$_exit_FILE") | # var1=`cat tmpfile1; rm tmpfile1` # var2=`cat tmpfile2; rm tmpfile2` - parallel --plain -q echo '{2}=`cat {1}; rm {1}`' :::: - :::+ $( + parallel -k --plain -q echo '{2}=`cat {1}; rm {1}`' :::: - :::+ $( echo "$_parset_NAME" | perl -pe 's/,/ /g' ) ); diff --git a/src/env_parallel.dash b/src/env_parallel.dash index 78258da4..4143e85f 100755 --- a/src/env_parallel.dash +++ b/src/env_parallel.dash @@ -385,7 +385,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20210522 (GNU parallel `parallel --minversion 1`)" + echo "parset 20210622 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.ksh b/src/env_parallel.ksh index 891cc52e..4107f5a1 100755 --- a/src/env_parallel.ksh +++ b/src/env_parallel.ksh @@ -368,7 +368,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20210522 (GNU parallel `parallel --minversion 1`)" + echo "parset 20210622 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.mksh b/src/env_parallel.mksh index 196254af..ee3af514 100644 --- a/src/env_parallel.mksh +++ b/src/env_parallel.mksh @@ -371,7 +371,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20210522 (GNU parallel `parallel --minversion 1`)" + echo "parset 20210622 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.sh b/src/env_parallel.sh index a09da661..84eeb60c 100755 --- a/src/env_parallel.sh +++ b/src/env_parallel.sh @@ -390,7 +390,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20210522 (GNU parallel `parallel --minversion 1`)" + echo "parset 20210622 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " diff --git a/src/env_parallel.zsh b/src/env_parallel.zsh index 7c2acd00..1a02a178 100755 --- a/src/env_parallel.zsh +++ b/src/env_parallel.zsh @@ -362,7 +362,7 @@ _parset_main() { return 255 fi if [ "$_parset_NAME" = "--version" ] ; then - echo "parset 20210522 (GNU parallel `parallel --minversion 1`)" + echo "parset 20210622 (GNU parallel `parallel --minversion 1`)" echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software" echo "Foundation, Inc." echo "License GPLv3+: GNU GPL version 3 or later " @@ -394,11 +394,11 @@ _parset_main() { if perl -e 'exit not grep /,| /, @ARGV' "$_parset_NAME" ; then # $_parset_NAME contains , or space # Split on , or space to get the names + # Compute results into files + # var1=`cat tmpfile1; rm tmpfile1` + # var2=`cat tmpfile2; rm tmpfile2` eval "$( - # Compute results into files ($_parset_PARALLEL_PRG --files -k "$@"; echo $? > "$_exit_FILE") | - # var1=`cat tmpfile1; rm tmpfile1` - # var2=`cat tmpfile2; rm tmpfile2` parallel -q echo {2}='`cat {1}; rm {1}`' :::: - :::+ $( echo "$_parset_NAME" | perl -pe 's/,/ /g' ) @@ -409,8 +409,8 @@ _parset_main() { # => $_parset_NAME is the name of the array to put data into # Supported in: bash zsh ksh mksh # Arrays do not work in: sh ash dash + # Compute results into files. Save exit value eval "$_parset_NAME=( $( - # Compute results into files. Save exit value ($_parset_PARALLEL_PRG --files -k "$@"; echo $? > "$_exit_FILE") | perl -pe 'chop;$_="\"\`cat $_; rm $_\`\" "' ) )" diff --git a/src/niceload b/src/niceload index a0b1649e..7bb5a911 100755 --- a/src/niceload +++ b/src/niceload @@ -26,7 +26,7 @@ use strict; use Getopt::Long; $Global::progname="niceload"; -$Global::version = 20210522; +$Global::version = 20210622; Getopt::Long::Configure("bundling","require_order"); get_options_from_array(\@ARGV) || die_usage(); if($opt::version) { diff --git a/src/parallel b/src/parallel index 8ccb58d8..296a42c8 100755 --- a/src/parallel +++ b/src/parallel @@ -2187,7 +2187,7 @@ sub check_invalid_option_combinations() { sub init_globals() { # Defaults: - $Global::version = 20210522; + $Global::version = 20210622; $Global::progname = 'parallel'; $::name = "GNU Parallel"; $Global::infinity = 2**31; @@ -4812,6 +4812,8 @@ sub reaper() { $job->set_exitstatus($? >> 8); $job->set_exitsignal($? & 127); } + + debug("run", "\nseq ",$job->seq()," died (", $job->exitstatus(), ")"); if($Global::delayauto or $Global::sshdelayauto) { if($job->exitstatus()) { # Job failed: Increase delay (if $opt::(ssh)delay set) @@ -4822,9 +4824,8 @@ sub reaper() { $opt::delay &&= $opt::delay * 0.9; $opt::sshdelay &&= $opt::sshdelay * 0.9; } + debug("run", "delay:$opt::delay ssh:$opt::sshdelay "); } - - debug("run", "seq ",$job->seq()," died (", $job->exitstatus(), ")"); $job->set_endtime(::now()); my $sshlogin = $job->sshlogin(); $sshlogin->dec_jobs_running(); @@ -4992,8 +4993,8 @@ sub usage() { "If you use programs that use GNU Parallel to process data for an article in a", "scientific publication, please cite:", "", - " Tange, O. (2021, May 22). GNU Parallel 20210522 ('Gaza').", - " Zenodo. https://doi.org/10.5281/zenodo.4781603", + " Tange, O. (2021, June 22). GNU Parallel 20210622 ('Protasevich').", + " Zenodo. https://doi.org/10.5281/zenodo.5013933", "", # Before changing this line, please read # https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice @@ -5023,8 +5024,8 @@ sub citation_notice() { "If you use programs that use GNU Parallel to process data for an article in a", "scientific publication, please cite:", "", - " Tange, O. (2021, May 22). GNU Parallel 20210522 ('Gaza').", - " Zenodo. https://doi.org/10.5281/zenodo.4781603", + " Tange, O. (2021, June 22). GNU Parallel 20210622 ('Protasevich').", + " Zenodo. https://doi.org/10.5281/zenodo.5013933", "", # Before changing this line, please read # https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice and @@ -5147,20 +5148,20 @@ sub citation() { "If you use programs that use GNU Parallel to process data for an article in a", "scientific publication, please cite:", "", - "\@software{tange_2021_4781603,", + "\@software{tange_2021_5013933,", " author = {Tange, Ole},", - " title = {GNU Parallel 20210522 ('Gaza')},", - " month = May,", + " title = {GNU Parallel 20210622 ('Protasevich')},", + " month = Jun,", " year = 2021,", " note = {{GNU Parallel is a general parallelizer to run", " multiple serial command line programs in parallel", " without changing them.}},", " publisher = {Zenodo},", - " doi = {10.5281/zenodo.4781603},", - " url = {https://doi.org/10.5281/zenodo.4781603}", + " doi = {10.5281/zenodo.5013933},", + " url = {https://doi.org/10.5281/zenodo.5013933}", "}", "", - "(Feel free to use \\nocite{tange_2021_4781603})", + "(Feel free to use \\nocite{tange_2021_5013933})", "", # Before changing this line, please read # https://www.gnu.org/software/parallel/parallel_design.html#Citation-notice and @@ -6023,8 +6024,12 @@ sub reap_usleep() { $SIG{CHLD} = sub { kill "ALRM", $$ }; if($opt::delay) { # The 0.004s is approximately the time it takes for one round - usleep(1000*($Global::newest_starttime + - $opt::delay - 0.004 - ::now())); + my $next_earliest_start = + $Global::newest_starttime + $opt::delay - 0.004; + my $remaining_ms = 1000 * ($next_earliest_start - ::now()); + # The next job can only start at $next_earliest_start + # so sleep until then (but sleep at least $ms) + usleep(::max($ms,$remaining_ms)); } else { usleep($ms); } diff --git a/src/parallel.pod b/src/parallel.pod index 9a2eda8b..155a87e3 100644 --- a/src/parallel.pod +++ b/src/parallel.pod @@ -1639,9 +1639,9 @@ Activate additional replacement strings: {+/} {+.} {+..} {+...} {..} B<{##}> is the total number of jobs to be run. It is incompatible with B<-X>/B<-m>/B<--xargs>. -B<{0%}> zero-padded jobslot. (alpha testing) +B<{0%}> zero-padded jobslot. (beta testing) -B<{0#}> zero-padded sequence number. (alpha testing) +B<{0#}> zero-padded sequence number. (beta testing) B<{choose_k}> is inspired by n choose k: Given a list of n elements, choose k. k is the number of input sources and n is the number of diff --git a/src/parallel_alternatives.pod b/src/parallel_alternatives.pod index 517d8406..a31655b4 100644 --- a/src/parallel_alternatives.pod +++ b/src/parallel_alternatives.pod @@ -3162,9 +3162,17 @@ of ~0.05 ms/job compared to GNU B's ~3 ms/job). So if your jobs are extremely short lived, and you can live with the quite limited command, this may be useful. -B will not start the first job, until it has read all -input. The input can at most be 20935 lines and the lines cannot be -all be empty. +It works by making a queue for each process. Then the jobs are +distributed to the queues in a round robin fashion. Finally the queues +are started in parallel. This works fine, if you are lucky, but if +not, all the long jobs may end up in the same queue, so you may see: + + $ printf "%b\n" 1 1 1 4 1 1 1 4 1 1 1 4 | + time parallel -P4 sleep {} + (7 seconds) + $ printf "%b\n" 1 1 1 4 1 1 1 4 1 1 1 4 | + time ./parallel-bash.bash -p 4 -c sleep {} + (12 seconds) Ctrl-C does not stop spawning new jobs. Ctrl-Z does not suspend running jobs. @@ -3189,7 +3197,7 @@ running jobs. 4$ something | parallel -j 5 echo {} {} https://reposhub.com/python/command-line-tools/Akianonymus-parallel-bash.html -(Last checked: 2021-02) +(Last checked: 2021-06) =head2 DIFFERENCES BETWEEN bash-concurrent AND GNU Parallel diff --git a/src/parset b/src/parset index 13adad08..0ec4d72b 100755 --- a/src/parset +++ b/src/parset @@ -86,33 +86,33 @@ Do the below and restart your shell. bash: Put this in $HOME/.bashrc: . `which env_parallel.bash` E.g. by doing: echo '. `which env_parallel.bash`' >> $HOME/.bashrc - Supports: aliases, functions, variables, arrays - -zsh: Put this in $HOME/.zshrc: . `which env_parallel.zsh` - E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshenv - Supports: functions, variables, arrays - -fish: Unsupported + Supports: variables, aliases, functions, arrays ksh: Put this in $HOME/.kshrc: source `which env_parallel.ksh` E.g. by doing: echo 'source `which env_parallel.ksh`' >> $HOME/.kshrc - Supports: aliases, functions, variables, arrays + Supports: variables, aliases, functions, arrays mksh: Put this in $HOME/.mkshrc: source `which env_parallel.mksh` E.g. by doing: echo 'source `which env_parallel.mksh`' >> $HOME/.mkshrc - Supports: aliases, functions, variables, arrays + Supports: variables, aliases, functions, arrays pdksh: Put this in $HOME/.profile: source `which env_parallel.pdksh` E.g. by doing: echo '. `which env_parallel.pdksh`' >> $HOME/.profile - Supports: aliases, functions, variables, arrays + Supports: variables, aliases, functions, arrays + +zsh: Put this in $HOME/.zshrc: . `which env_parallel.zsh` + E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshenv + Supports: variables, functions, arrays ash: Put this in $HOME/.profile: . `which env_parallel.ash` E.g. by doing: echo '. `which env_parallel.ash`' >> $HOME/.profile - Supports: aliases, variables + Supports: variables, aliases dash: Put this in $HOME/.profile: . `which env_parallel.dash` E.g. by doing: echo '. `which env_parallel.dash`' >> $HOME/.profile - Supports: aliases, variables + Supports: variables, aliases + +fish: Unsupported csh: Unsupported diff --git a/src/parset.pod b/src/parset.pod index 3a295e39..8df65d2b 100644 --- a/src/parset.pod +++ b/src/parset.pod @@ -102,6 +102,18 @@ or Bash/Zsh/Ksh process substitution: echo "${res[1]}" echo "${res[99]}" +Put output into an associative array (Bash only): + + input=("value A" "value B") + parset res echo This is ::: "${input[@]}" + + # Zip the input and res arrays to a single associative array + declare -A myassoc + for ((i=0; $i<${#input[@]}; i++)); do + myassoc[${input[i]}]=${res[i]} + done + echo "${myassoc["value A"]}" + =head3 Installation Put this in the relevant B<$HOME/.bashrc> or B<$HOME/.zshenv> or B<$HOME/.kshrc>: diff --git a/src/parsort b/src/parsort index 1e4dcef1..eee6e05a 100755 --- a/src/parsort +++ b/src/parsort @@ -121,7 +121,7 @@ GetOptions( "help" => \$opt::dummy, ) || exit(255); $Global::progname = ($0 =~ m:(^|/)([^/]+)$:)[1]; -$Global::version = 20210522; +$Global::version = 20210622; if($opt::version) { version(); exit 0; } @Global::sortoptions = shell_quote(@ARGV_before[0..($#ARGV_before-$#ARGV-1)]); diff --git a/src/sql b/src/sql index 8c76517d..eb751f44 100755 --- a/src/sql +++ b/src/sql @@ -600,7 +600,7 @@ $Global::Initfile && unlink $Global::Initfile; exit ($err); sub parse_options { - $Global::version = 20210522; + $Global::version = 20210622; $Global::progname = 'sql'; # This must be done first as this may exec myself diff --git a/testsuite/tests-to-run/parallel-android.sh b/testsuite/tests-to-run/parallel-android.sh index 943122de..a86084fa 100755 --- a/testsuite/tests-to-run/parallel-android.sh +++ b/testsuite/tests-to-run/parallel-android.sh @@ -5,17 +5,18 @@ # SPDX-License-Identifier: GPL-3.0-or-later unset TIMEOUT +torsocks bash <<'EOF' . `which env_parallel.bash` env_parallel --session -host=$(parallel -j0 --halt now,success=1 ssh {} echo {} ::: android1 android2 2>/dev/null) +host=$(parallel -j0 --halt now,success=1 ssh {} echo {} ::: koditor huator fairtor 2>/dev/null) if [ -z "$host" ] ; then echo Error: no android host working else echo $host >&2 doit() { - export PARALLEL_SSH='ssh -p2222' + export PARALLEL_SSH='ssh -p2222 -o "StrictHostKeyChecking no"' parallel -k echo ::: Basic usage works parallel -k -S localhost echo ::: Remote usage works } @@ -23,3 +24,4 @@ else scp /usr/local/bin/parallel $host:/data/data/com.termux/files/usr/bin env_parallel -S $host doit ::: a fi +EOF diff --git a/testsuite/wanted-results/parallel-tutorial b/testsuite/wanted-results/parallel-tutorial index 13fdf03e..178b0124 100644 --- a/testsuite/wanted-results/parallel-tutorial +++ b/testsuite/wanted-results/parallel-tutorial @@ -729,36 +729,36 @@ Unknown option: green env_parallel only works if it is a function. Do the below and restart your shell. bash: Put this in $HOME/.bashrc: . `which env_parallel.bash` E.g. by doing: echo '. `which env_parallel.bash`' >> $HOME/.bashrc - Supports: aliases, functions, variables, arrays -zsh: Put this in $HOME/.zshrc: . `which env_parallel.zsh` - E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshenv - Supports: functions, variables, arrays + Supports: variables, aliases, functions, arrays fish: Put this in $HOME/.config/fish/config.fish: . (which env_parallel.fish) E.g. by doing: echo '. (which env_parallel.fish)' >> $HOME/.config/fish/config.fish - Supports: aliases, functions, variables, arrays + Supports: variables, aliases, functions, arrays ksh: Put this in $HOME/.kshrc: source `which env_parallel.ksh` E.g. by doing: echo 'source `which env_parallel.ksh`' >> $HOME/.kshrc - Supports: aliases, functions, variables, arrays + Supports: variables, aliases, functions, arrays mksh: Put this in $HOME/.mkshrc: source `which env_parallel.mksh` E.g. by doing: echo 'source `which env_parallel.mksh`' >> $HOME/.mkshrc - Supports: aliases, functions, variables, arrays + Supports: variables, aliases, functions, arrays pdksh: Put this in $HOME/.profile: source `which env_parallel.pdksh` E.g. by doing: echo '. `which env_parallel.pdksh`' >> $HOME/.profile - Supports: aliases, functions, variables, arrays + Supports: variables, aliases, functions, arrays +zsh: Put this in $HOME/.zshrc: . `which env_parallel.zsh` + E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshenv + Supports: variables, functions, arrays ash: Put this in $HOME/.profile: . `which env_parallel.ash` E.g. by doing: echo '. `which env_parallel.ash`' >> $HOME/.profile - Supports: aliases, variables + Supports: variables, aliases dash: Put this in $HOME/.profile: . `which env_parallel.dash` E.g. by doing: echo '. `which env_parallel.dash`' >> $HOME/.profile - Supports: aliases, variables + Supports: variables, aliases csh: Put this in $HOME/.cshrc: source `which env_parallel.csh` E.g. by doing: echo 'source `which env_parallel.csh`' >> $HOME/.cshrc - Supports: aliases, variables, arrays with no special chars + Supports: variables, aliases, arrays with no special chars tcsh: Put this in $HOME/.tcshrc: source `which env_parallel.tcsh` E.g. by doing: echo 'source `which env_parallel.tcsh`' >> $HOME/.tcshrc - Supports: aliases, variables, arrays with no special chars + Supports: variables, aliases, arrays with no special chars To install in all shells run: env_parallel --install For details: see man env_parallel @@ -803,36 +803,36 @@ Unknown option: bar env_parallel only works if it is a function. Do the below and restart your shell. bash: Put this in $HOME/.bashrc: . `which env_parallel.bash` E.g. by doing: echo '. `which env_parallel.bash`' >> $HOME/.bashrc - Supports: aliases, functions, variables, arrays -zsh: Put this in $HOME/.zshrc: . `which env_parallel.zsh` - E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshenv - Supports: functions, variables, arrays + Supports: variables, aliases, functions, arrays fish: Put this in $HOME/.config/fish/config.fish: . (which env_parallel.fish) E.g. by doing: echo '. (which env_parallel.fish)' >> $HOME/.config/fish/config.fish - Supports: aliases, functions, variables, arrays + Supports: variables, aliases, functions, arrays ksh: Put this in $HOME/.kshrc: source `which env_parallel.ksh` E.g. by doing: echo 'source `which env_parallel.ksh`' >> $HOME/.kshrc - Supports: aliases, functions, variables, arrays + Supports: variables, aliases, functions, arrays mksh: Put this in $HOME/.mkshrc: source `which env_parallel.mksh` E.g. by doing: echo 'source `which env_parallel.mksh`' >> $HOME/.mkshrc - Supports: aliases, functions, variables, arrays + Supports: variables, aliases, functions, arrays pdksh: Put this in $HOME/.profile: source `which env_parallel.pdksh` E.g. by doing: echo '. `which env_parallel.pdksh`' >> $HOME/.profile - Supports: aliases, functions, variables, arrays + Supports: variables, aliases, functions, arrays +zsh: Put this in $HOME/.zshrc: . `which env_parallel.zsh` + E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshenv + Supports: variables, functions, arrays ash: Put this in $HOME/.profile: . `which env_parallel.ash` E.g. by doing: echo '. `which env_parallel.ash`' >> $HOME/.profile - Supports: aliases, variables + Supports: variables, aliases dash: Put this in $HOME/.profile: . `which env_parallel.dash` E.g. by doing: echo '. `which env_parallel.dash`' >> $HOME/.profile - Supports: aliases, variables + Supports: variables, aliases csh: Put this in $HOME/.cshrc: source `which env_parallel.csh` E.g. by doing: echo 'source `which env_parallel.csh`' >> $HOME/.cshrc - Supports: aliases, variables, arrays with no special chars + Supports: variables, aliases, arrays with no special chars tcsh: Put this in $HOME/.tcshrc: source `which env_parallel.tcsh` E.g. by doing: echo 'source `which env_parallel.tcsh`' >> $HOME/.tcshrc - Supports: aliases, variables, arrays with no special chars + Supports: variables, aliases, arrays with no special chars To install in all shells run: env_parallel --install For details: see man env_parallel @@ -859,26 +859,26 @@ of env_parallel. Do the below and restart your shell. bash: Put this in $HOME/.bashrc: . `which env_parallel.bash` E.g. by doing: echo '. `which env_parallel.bash`' >> $HOME/.bashrc - Supports: aliases, functions, variables, arrays -zsh: Put this in $HOME/.zshrc: . `which env_parallel.zsh` - E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshenv - Supports: functions, variables, arrays -fish: Unsupported + Supports: variables, aliases, functions, arrays ksh: Put this in $HOME/.kshrc: source `which env_parallel.ksh` E.g. by doing: echo 'source `which env_parallel.ksh`' >> $HOME/.kshrc - Supports: aliases, functions, variables, arrays + Supports: variables, aliases, functions, arrays mksh: Put this in $HOME/.mkshrc: source `which env_parallel.mksh` E.g. by doing: echo 'source `which env_parallel.mksh`' >> $HOME/.mkshrc - Supports: aliases, functions, variables, arrays + Supports: variables, aliases, functions, arrays pdksh: Put this in $HOME/.profile: source `which env_parallel.pdksh` E.g. by doing: echo '. `which env_parallel.pdksh`' >> $HOME/.profile - Supports: aliases, functions, variables, arrays + Supports: variables, aliases, functions, arrays +zsh: Put this in $HOME/.zshrc: . `which env_parallel.zsh` + E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshenv + Supports: variables, functions, arrays ash: Put this in $HOME/.profile: . `which env_parallel.ash` E.g. by doing: echo '. `which env_parallel.ash`' >> $HOME/.profile - Supports: aliases, variables + Supports: variables, aliases dash: Put this in $HOME/.profile: . `which env_parallel.dash` E.g. by doing: echo '. `which env_parallel.dash`' >> $HOME/.profile - Supports: aliases, variables + Supports: variables, aliases +fish: Unsupported csh: Unsupported tcsh: Unsupported To install in all shells run: @@ -899,26 +899,26 @@ of env_parallel. Do the below and restart your shell. bash: Put this in $HOME/.bashrc: . `which env_parallel.bash` E.g. by doing: echo '. `which env_parallel.bash`' >> $HOME/.bashrc - Supports: aliases, functions, variables, arrays -zsh: Put this in $HOME/.zshrc: . `which env_parallel.zsh` - E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshenv - Supports: functions, variables, arrays -fish: Unsupported + Supports: variables, aliases, functions, arrays ksh: Put this in $HOME/.kshrc: source `which env_parallel.ksh` E.g. by doing: echo 'source `which env_parallel.ksh`' >> $HOME/.kshrc - Supports: aliases, functions, variables, arrays + Supports: variables, aliases, functions, arrays mksh: Put this in $HOME/.mkshrc: source `which env_parallel.mksh` E.g. by doing: echo 'source `which env_parallel.mksh`' >> $HOME/.mkshrc - Supports: aliases, functions, variables, arrays + Supports: variables, aliases, functions, arrays pdksh: Put this in $HOME/.profile: source `which env_parallel.pdksh` E.g. by doing: echo '. `which env_parallel.pdksh`' >> $HOME/.profile - Supports: aliases, functions, variables, arrays + Supports: variables, aliases, functions, arrays +zsh: Put this in $HOME/.zshrc: . `which env_parallel.zsh` + E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshenv + Supports: variables, functions, arrays ash: Put this in $HOME/.profile: . `which env_parallel.ash` E.g. by doing: echo '. `which env_parallel.ash`' >> $HOME/.profile - Supports: aliases, variables + Supports: variables, aliases dash: Put this in $HOME/.profile: . `which env_parallel.dash` E.g. by doing: echo '. `which env_parallel.dash`' >> $HOME/.profile - Supports: aliases, variables + Supports: variables, aliases +fish: Unsupported csh: Unsupported tcsh: Unsupported To install in all shells run: @@ -937,26 +937,26 @@ of env_parallel. Do the below and restart your shell. bash: Put this in $HOME/.bashrc: . `which env_parallel.bash` E.g. by doing: echo '. `which env_parallel.bash`' >> $HOME/.bashrc - Supports: aliases, functions, variables, arrays -zsh: Put this in $HOME/.zshrc: . `which env_parallel.zsh` - E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshenv - Supports: functions, variables, arrays -fish: Unsupported + Supports: variables, aliases, functions, arrays ksh: Put this in $HOME/.kshrc: source `which env_parallel.ksh` E.g. by doing: echo 'source `which env_parallel.ksh`' >> $HOME/.kshrc - Supports: aliases, functions, variables, arrays + Supports: variables, aliases, functions, arrays mksh: Put this in $HOME/.mkshrc: source `which env_parallel.mksh` E.g. by doing: echo 'source `which env_parallel.mksh`' >> $HOME/.mkshrc - Supports: aliases, functions, variables, arrays + Supports: variables, aliases, functions, arrays pdksh: Put this in $HOME/.profile: source `which env_parallel.pdksh` E.g. by doing: echo '. `which env_parallel.pdksh`' >> $HOME/.profile - Supports: aliases, functions, variables, arrays + Supports: variables, aliases, functions, arrays +zsh: Put this in $HOME/.zshrc: . `which env_parallel.zsh` + E.g. by doing: echo '. `which env_parallel.zsh`' >> $HOME/.zshenv + Supports: variables, functions, arrays ash: Put this in $HOME/.profile: . `which env_parallel.ash` E.g. by doing: echo '. `which env_parallel.ash`' >> $HOME/.profile - Supports: aliases, variables + Supports: variables, aliases dash: Put this in $HOME/.profile: . `which env_parallel.dash` E.g. by doing: echo '. `which env_parallel.dash`' >> $HOME/.profile - Supports: aliases, variables + Supports: variables, aliases +fish: Unsupported csh: Unsupported tcsh: Unsupported To install in all shells run: -- 2.11.4.GIT