src/env_parallel.bash

   1 #!/usr/bin/env bash
   2
   3 # This file must be sourced in bash:
   4 #
   5 #   source `which env_parallel.bash`
   6 #
   7 # after which 'env_parallel' works
   8 #
   9 #
  10 # Copyright (C) 2016-2021 Ole Tange, http://ole.tange.dk and Free
  11 # Software Foundation, Inc.
  12 #
  13 # This program is free software; you can redistribute it and/or modify
  14 # it under the terms of the GNU General Public License as published by
  15 # the Free Software Foundation; either version 3 of the License, or
  16 # (at your option) any later version.
  17 #
  18 # This program is distributed in the hope that it will be useful, but
  19 # WITHOUT ANY WARRANTY; without even the implied warranty of
  20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  21 # General Public License for more details.
  22 #
  23 # You should have received a copy of the GNU General Public License
  24 # along with this program; if not, see <http://www.gnu.org/licenses/>
  25 # or write to the Free Software Foundation, Inc., 51 Franklin St,
  26 # Fifth Floor, Boston, MA 02110-1301 USA
  27 #
  28 # SPDX-FileCopyrightText: 2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc.
  29 # SPDX-License-Identifier: GPL-3.0-or-later
  30
  31 env_parallel() {
  32     # env_parallel.bash
  33
  34     _names_of_ALIASES() {
  35         # No aliases will return false. This error should be ignored.
  36         compgen -a || true
  37     }
  38     _bodies_of_ALIASES() {
  39         local _i
  40         for _i in "$@"; do
  41             # shellcheck disable=SC2046
  42             if [ $(alias "$_i" | wc -l) == 1 ] ; then
  43                 true Alias is a single line. Good.
  44             else
  45                 _warning_PAR "Alias '$_i' contains newline."
  46                 _warning_PAR "Make sure the command has at least one newline after '$_i'."
  47                 _warning_PAR "See BUGS in 'man env_parallel'."
  48             fi
  49         done
  50         alias "$@"
  51     }
  52     _names_of_FUNCTIONS() {
  53         compgen -A function
  54     }
  55     _bodies_of_FUNCTIONS() {
  56         typeset -f "$@"
  57     }
  58     _names_of_VARIABLES() {
  59         compgen -A variable
  60     }
  61     _bodies_of_VARIABLES() {
  62         typeset -p "$@"
  63     }
  64     _ignore_HARDCODED() {
  65         # These names cannot be detected
  66         echo '(_|TIMEOUT|GROUPS|FUNCNAME|DIRSTACK|PIPESTATUS|USERNAME|BASHPID|BASH_[A-Z_]+)'
  67     }
  68     _ignore_READONLY() {
  69         # shellcheck disable=SC1078,SC1079,SC2026
  70         readonly | perl -e '@r = map {
  71                 chomp;
  72                 # sh on UnixWare: readonly TIMEOUT
  73                 # ash: readonly var='val'
  74                 # ksh: var='val'
  75                 s/^(readonly )?([^= ]*)(=.*|)$/$2/ or
  76                 # bash: declare -ar BASH_VERSINFO=([0]="4" [1]="4")
  77                 # zsh: typeset -r var='val'
  78                   s/^\S+\s+\S+\s+(\S[^=]*)(=.*|$)/$1/;
  79                 $_ } <>;
  80             $vars = join "|",map { quotemeta $_ } @r;
  81             print $vars ? "($vars)" : "(,,nO,,VaRs,,)";
  82             '
  83     }
  84     _remove_bad_NAMES() {
  85         # Do not transfer vars and funcs from env_parallel
  86         _ignore_RO="`_ignore_READONLY`"
  87         _ignore_HARD="`_ignore_HARDCODED`"
  88         # Macos-grep does not like long patterns
  89         # Old Solaris grep does not support -E
  90         # Perl Version of:
  91         # grep -Ev '^(...)$' |
  92         perl -ne '/^(
  93                      PARALLEL_ENV|
  94                      PARALLEL_TMP|
  95                      _alias_NAMES|
  96                      _bodies_of_ALIASES|
  97                      _bodies_of_FUNCTIONS|
  98                      _bodies_of_VARIABLES|
  99                      _error_PAR|
 100                      _function_NAMES|
 101                      _get_ignored_VARS|
 102                      _grep_REGEXP|
 103                      _ignore_HARD|
 104                      _ignore_HARDCODED|
 105                      _ignore_READONLY|
 106                      _ignore_RO|
 107                      _ignore_UNDERSCORE|
 108                      _list_alias_BODIES|
 109                      _list_function_BODIES|
 110                      _list_variable_VALUES|
 111                      _make_grep_REGEXP|
 112                      _names_of_ALIASES|
 113                      _names_of_FUNCTIONS|
 114                      _names_of_VARIABLES|
 115                      _names_of_maybe_FUNCTIONS|
 116                      _parallel_exit_CODE|
 117                      _prefix_PARALLEL_ENV|
 118                      _prefix_PARALLEL_ENV|
 119                      _remove_bad_NAMES|
 120                      _remove_readonly|
 121                      _variable_NAMES|
 122                      _warning_PAR|
 123                      _which_PAR)$/x and next;
 124             # Filter names matching --env
 125             /^'"$_grep_REGEXP"'$/ or next;
 126             /^'"$_ignore_UNDERSCORE"'$/ and next;
 127             # Remove readonly variables
 128             /^'"$_ignore_RO"'$/ and next;
 129             /^'"$_ignore_HARD"'$/ and next;
 130             print;'
 131     }
 132     _prefix_PARALLEL_ENV() {
 133         shopt 2>/dev/null |
 134         perl -pe 's:\s+off:;: and s/^/shopt -u /;
 135                   s:\s+on:;: and s/^/shopt -s /;
 136                   s:;$:&>/dev/null;:';
 137         echo 'shopt -s expand_aliases &>/dev/null';
 138     }
 139
 140     _get_ignored_VARS() {
 141         perl -e '
 142             for(@ARGV){
 143                 $next_is_env and push @envvar, split/,/, $_;
 144                 $next_is_env=/^--env$/;
 145             }
 146             if(grep { /^_$/ } @envvar) {
 147                 if(not open(IN, "<", "$ENV{HOME}/.parallel/ignored_vars")) {
 148                     print STDERR "parallel: Error: ",
 149                     "Run \"parallel --record-env\" in a clean environment first.\n";
 150                 } else {
 151                     chomp(@ignored_vars = <IN>);
 152                 }
 153             }
 154             if($ENV{PARALLEL_IGNORED_NAMES}) {
 155                 push @ignored_vars, split/\s+/, $ENV{PARALLEL_IGNORED_NAMES};
 156                 chomp @ignored_vars;
 157             }
 158             $vars = join "|",map { quotemeta $_ } @ignored_vars;
 159             print $vars ? "($vars)" : "(,,nO,,VaRs,,)";
 160             ' -- "$@"
 161     }
 162
 163     # Get the --env variables if set
 164     # --env _ should be ignored
 165     # and convert  a b c  to (a|b|c)
 166     # If --env not set: Match everything (.*)
 167     _make_grep_REGEXP() {
 168         perl -e '
 169             for(@ARGV){
 170                 /^_$/ and $next_is_env = 0;
 171                 $next_is_env and push @envvar, split/,/, $_;
 172                 $next_is_env = /^--env$/;
 173             }
 174             $vars = join "|",map { quotemeta $_ } @envvar;
 175             print $vars ? "($vars)" : "(.*)";
 176             ' -- "$@"
 177     }
 178     _which_PAR() {
 179         # type returns:
 180         #   ll is an alias for ls -l (in ash)
 181         #   bash is a tracked alias for /bin/bash
 182         #   true is a shell builtin (in bash)
 183         #   myfunc is a function (in bash)
 184         #   myfunc is a shell function (in zsh)
 185         #   which is /usr/bin/which (in sh, bash)
 186         #   which is hashed (/usr/bin/which)
 187         #   gi is aliased to `grep -i' (in bash)
 188         #   aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde'
 189         # Return 0 if found, 1 otherwise
 190         LANG=C type "$@" |
 191             perl -pe '$exit += (s/ is an alias for .*// ||
 192                                 s/ is aliased to .*// ||
 193                                 s/ is a function// ||
 194                                 s/ is a shell function// ||
 195                                 s/ is a shell builtin// ||
 196                                 s/.* is hashed .(\S+).$/$1/ ||
 197                                 s/.* is (a tracked alias for )?//);
 198                       END { exit not $exit }'
 199     }
 200     _warning_PAR() {
 201         echo "env_parallel: Warning: $*" >&2
 202     }
 203     _error_PAR() {
 204         echo "env_parallel: Error: $*" >&2
 205     }
 206
 207     # Bash is broken in version 3.2.25 and 4.2.39
 208     # The crazy '[ "`...`" == "" ]' is needed for the same reason
 209     if [ "`_which_PAR parallel`" == "" ]; then
 210         # shellcheck disable=SC2016
 211         _error_PAR 'parallel must be in $PATH.'
 212         return 255
 213     fi
 214
 215     # Grep regexp for vars given by --env
 216     _grep_REGEXP="`_make_grep_REGEXP \"$@\"`"
 217     unset _make_grep_REGEXP
 218
 219     # Deal with --env _
 220     _ignore_UNDERSCORE="`_get_ignored_VARS \"$@\"`"
 221     unset _get_ignored_VARS
 222
 223     # --record-env
 224     # Bash is broken in version 3.2.25 and 4.2.39
 225     # The crazy '[ "`...`" == 0 ]' is needed for the same reason
 226     if [ "`perl -e 'exit grep { /^--record-env$/ } @ARGV' -- "$@"; echo $?`" == 0 ] ; then
 227         true skip
 228     else
 229         (_names_of_ALIASES;
 230          _names_of_FUNCTIONS;
 231          _names_of_VARIABLES) |
 232             cat > "$HOME"/.parallel/ignored_vars
 233         return 0
 234     fi
 235
 236     # --session
 237     # Bash is broken in version 3.2.25 and 4.2.39
 238     # The crazy '[ "`...`" == 0 ]' is needed for the same reason
 239     if [ "`perl -e 'exit grep { /^--session$/ } @ARGV' -- "$@"; echo $?`" == 0 ] ; then
 240         true skip
 241     else
 242         # Insert ::: between each level of session
 243         # so you can pop off the last ::: at --end-session
 244         PARALLEL_IGNORED_NAMES="`echo \"$PARALLEL_IGNORED_NAMES\";
 245           echo :::;
 246           (_names_of_ALIASES;
 247            _names_of_FUNCTIONS;
 248            _names_of_VARIABLES) | perl -ne '
 249             BEGIN{
 250               map { $ignored_vars{$_}++ }
 251                 split/\s+/, $ENV{PARALLEL_IGNORED_NAMES};
 252             }
 253             chomp;
 254             for(split/\s+/) {
 255               if(not $ignored_vars{$_}) {
 256                 print $_,\"\\n\";
 257               }
 258             }
 259             '`"
 260         export PARALLEL_IGNORED_NAMES
 261         return 0
 262     fi
 263     # Bash is broken in version 3.2.25 and 4.2.39
 264     # The crazy '[ "`...`" == 0 ]' is needed for the same reason
 265     if [ "`perl -e 'exit grep { /^--end.?session$/ } @ARGV' -- "$@"; echo $?`" == 0 ] ; then
 266         true skip
 267     else
 268         # Pop off last ::: from PARALLEL_IGNORED_NAMES
 269         PARALLEL_IGNORED_NAMES="`perl -e '
 270           $ENV{PARALLEL_IGNORED_NAMES} =~ s/(.*):::.*?$/$1/s;
 271           print $ENV{PARALLEL_IGNORED_NAMES}
 272         '`"
 273         return 0
 274     fi
 275     # Grep alias names
 276     _alias_NAMES="`_names_of_ALIASES | _remove_bad_NAMES | xargs echo`"
 277     _list_alias_BODIES="_bodies_of_ALIASES $_alias_NAMES"
 278     if [ "$_alias_NAMES" = "" ] ; then
 279         # no aliases selected
 280         _list_alias_BODIES="true"
 281     fi
 282     unset _alias_NAMES
 283
 284     # Grep function names
 285     _function_NAMES="`_names_of_FUNCTIONS | _remove_bad_NAMES | xargs echo`"
 286     _list_function_BODIES="_bodies_of_FUNCTIONS $_function_NAMES"
 287     if [ "$_function_NAMES" = "" ] ; then
 288         # no functions selected
 289         _list_function_BODIES="true"
 290     fi
 291     unset _function_NAMES
 292
 293     # Grep variable names
 294     _variable_NAMES="`_names_of_VARIABLES | _remove_bad_NAMES | xargs echo`"
 295     _list_variable_VALUES="_bodies_of_VARIABLES $_variable_NAMES"
 296     if [ "$_variable_NAMES" = "" ] ; then
 297         # no variables selected
 298         _list_variable_VALUES="true"
 299     fi
 300     unset _variable_NAMES
 301
 302     _which_TRUE="`_which_PAR true`"
 303     # Copy shopt (so e.g. extended globbing works)
 304     # But force expand_aliases as aliases otherwise do not work
 305     PARALLEL_ENV="`
 306         _prefix_PARALLEL_ENV
 307         $_list_alias_BODIES;
 308         $_list_function_BODIES;
 309         $_list_variable_VALUES;
 310     `"
 311     export PARALLEL_ENV
 312     unset _list_alias_BODIES _list_variable_VALUES _list_function_BODIES
 313     unset _bodies_of_ALIASES _bodies_of_VARIABLES _bodies_of_FUNCTIONS
 314     unset _names_of_ALIASES _names_of_VARIABLES _names_of_FUNCTIONS
 315     unset _ignore_HARDCODED _ignore_READONLY _ignore_UNDERSCORE
 316     unset _remove_bad_NAMES _grep_REGEXP
 317     unset _prefix_PARALLEL_ENV
 318     # Test if environment is too big
 319     if [ "`_which_PAR true`" == "$_which_TRUE" ] ; then
 320         parallel "$@"
 321         _parallel_exit_CODE=$?
 322         # Clean up variables/functions
 323         unset PARALLEL_ENV
 324         unset _which_PAR _which_TRUE
 325         unset _warning_PAR _error_PAR
 326         # Unset _parallel_exit_CODE before return
 327         eval "unset _parallel_exit_CODE; return $_parallel_exit_CODE"
 328     else
 329         unset PARALLEL_ENV;
 330         _error_PAR "Your environment is too big."
 331         _error_PAR "You can try 3 different approaches:"
 332         _error_PAR "1. Run 'env_parallel --session' before you set"
 333         _error_PAR "   variables or define functions."
 334         _error_PAR "2. Use --env and only mention the names to copy."
 335         _error_PAR "3. Try running this in a clean environment once:"
 336         _error_PAR "     env_parallel --record-env"
 337         _error_PAR "   And then use '--env _'"
 338         _error_PAR "For details see: man env_parallel"
 339         return 255
 340     fi
 341 }
 342
 343 parset() {
 344     _parset_PARALLEL_PRG=parallel
 345     _parset_main "$@"
 346 }
 347
 348 env_parset() {
 349     _parset_PARALLEL_PRG=env_parallel
 350     _parset_main "$@"
 351 }
 352
 353 _parset_main() {
 354     # If $1 contains ',' or space:
 355     #   Split on , to get the destination variable names
 356     # If $1 is a single destination variable name:
 357     #   Treat it as the name of an array
 358     #
 359     #   # Create array named myvar
 360     #   parset myvar echo ::: {1..10}
 361     #   echo ${myvar[5]}
 362     #
 363     #   # Put output into $var_a $var_b $var_c
 364     #   varnames=(var_a var_b var_c)
 365     #   parset "${varnames[*]}" echo ::: {1..3}
 366     #   echo $var_c
 367     #
 368     #   # Put output into $var_a4 $var_b4 $var_c4
 369     #   parset "var_a4 var_b4 var_c4" echo ::: {1..3}
 370     #   echo $var_c4
 371
 372     _parset_NAME="$1"
 373     if [ "$_parset_NAME" = "" ] ; then
 374         echo parset: Error: No destination variable given. >&2
 375         echo parset: Error: Try: >&2
 376         echo parset: Error: ' ' parset myarray echo ::: foo bar >&2
 377         return 255
 378     fi
 379     if [ "$_parset_NAME" = "--help" ] ; then
 380         echo parset: Error: Usage: >&2
 381         echo parset: Error: ' ' parset varname GNU Parallel options and command >&2
 382         echo
 383         parallel --help
 384         return 255
 385     fi
 386     if [ "$_parset_NAME" = "--version" ] ; then
 387         echo "parset 20210922 (GNU parallel `parallel --minversion 1`)"
 388         echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software"
 389         echo "Foundation, Inc."
 390         echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
 391         echo "This is free software: you are free to change and redistribute it."
 392         echo "GNU parallel comes with no warranty."
 393         echo
 394         echo "Web site: https://www.gnu.org/software/parallel"
 395         echo
 396         echo "When using programs that use GNU Parallel to process data for publication"
 397         echo "please cite as described in 'parallel --citation'."
 398         echo
 399         return 255
 400     fi
 401     shift
 402
 403     # Bash: declare -A myassoc=( )
 404     # Zsh: typeset -A myassoc=( )
 405     # Ksh: typeset -A myassoc=( )
 406     if (typeset -p "$_parset_NAME" 2>/dev/null; echo) |
 407             perl -ne 'exit not (/^declare[^=]+-A|^typeset[^=]+-A/)' ; then
 408         # This is an associative array
 409         eval "`$_parset_PARALLEL_PRG -k --parset assoc,"$_parset_NAME" "$@"`"
 410         # The eval returns the function!
 411     else
 412         # This is a normal array or a list of variable names
 413         eval "`$_parset_PARALLEL_PRG -k --parset var,"$_parset_NAME" "$@"`"
 414         # The eval returns the function!
 415     fi
 416 }