src/env_parallel.dash

   1 #!/usr/bin/env dash
   2
   3 # This file must be sourced in dash:
   4 #
   5 #   . `which env_parallel.dash`
   6 #
   7 # after which 'env_parallel' works
   8 #
   9 #
  10 # Copyright (C) 2016-2021 Ole Tange, http://ole.tange.dk and Free
  11 # Software Foundation, Inc.
  12 #
  13 # This program is free software; you can redistribute it and/or modify
  14 # it under the terms of the GNU General Public License as published by
  15 # the Free Software Foundation; either version 3 of the License, or
  16 # (at your option) any later version.
  17 #
  18 # This program is distributed in the hope that it will be useful, but
  19 # WITHOUT ANY WARRANTY; without even the implied warranty of
  20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  21 # General Public License for more details.
  22 #
  23 # You should have received a copy of the GNU General Public License
  24 # along with this program; if not, see <http://www.gnu.org/licenses/>
  25 # or write to the Free Software Foundation, Inc., 51 Franklin St,
  26 # Fifth Floor, Boston, MA 02110-1301 USA
  27 #
  28 # SPDX-FileCopyrightText: 2021 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc.
  29 # SPDX-License-Identifier: GPL-3.0-or-later
  30
  31 env_parallel() {
  32     # based on env_parallel.sh
  33
  34     _names_of_ALIASES() {
  35         # alias fails on Unixware 5
  36         for _i in `alias 2>/dev/null | perl -ne 's/^alias //;s/^(\S+)=.*/$1/ && print' 2>/dev/null`; do
  37             # Check if this name really is an alias
  38             # or just part of a multiline alias definition
  39             if alias $_i >/dev/null 2>/dev/null; then
  40                 echo $_i
  41             fi
  42         done
  43     }
  44     _bodies_of_ALIASES() {
  45         # alias may return:
  46         #   myalias='definition' (GNU/Linux ash)
  47         #   alias myalias='definition' (FreeBSD ash)
  48         # so remove 'alias ' from first line
  49         for _i in "$@"; do
  50                 echo 'alias '"`alias $_i | perl -pe '1..1 and s/^alias //'`"
  51         done
  52     }
  53     _names_of_maybe_FUNCTIONS() {
  54         set | perl -ne '/^([A-Z_0-9]+)\s*\(\)\s*\{?$/i and print "$1\n"'
  55     }
  56     _names_of_FUNCTIONS() {
  57         # myfunc is a function
  58         LANG=C type `_names_of_maybe_FUNCTIONS` |
  59             perl -ne '/^(\S+) is a function$/ and not $seen{$1}++ and print "$1\n"'
  60     }
  61     _bodies_of_FUNCTIONS() {
  62         LANG=C type "$@" | perl -ne '/^(\S+) is a function$/ or print'
  63     }
  64     _names_of_VARIABLES() {
  65         # This may screw up if variables contain \n and =
  66         set | perl -ne 's/^(\S+?)=.*/$1/ and print;'
  67     }
  68     _bodies_of_VARIABLES() {
  69         # Crappy typeset -p
  70         for _i in "$@"
  71         do
  72             perl -e 'print @ARGV' "$_i="
  73             eval echo \"\$$_i\" | perl -e '$/=undef; $a=<>; chop($a); print $a' |
  74                 perl -pe 's/[\002-\011\013-\032\\\#\?\`\(\)\{\}\[\]\^\*\<\=\>\~\|\; \"\!\$\&\202-\377]/\\$&/go;'"s/'/\\\'/g; s/[\n]/'\\n'/go;";
  75             echo
  76         done
  77     }
  78     _ignore_HARDCODED() {
  79         # These names cannot be detected
  80         echo '(_|TIMEOUT)'
  81     }
  82     _ignore_READONLY() {
  83         readonly | perl -e '@r = map {
  84                 chomp;
  85                 # sh on UnixWare: readonly TIMEOUT
  86                 # ash: readonly var='val'
  87                 # ksh: var='val'
  88                 s/^(readonly )?([^= ]*)(=.*|)$/$2/ or
  89                 # bash: declare -ar BASH_VERSINFO=([0]="4" [1]="4")
  90                 # zsh: typeset -r var='val'
  91                   s/^\S+\s+\S+\s+(\S[^=]*)(=.*|$)/$1/;
  92                 $_ } <>;
  93             $vars = join "|",map { quotemeta $_ } @r;
  94             print $vars ? "($vars)" : "(,,nO,,VaRs,,)";
  95             '
  96     }
  97     _remove_bad_NAMES() {
  98         # Do not transfer vars and funcs from env_parallel
  99         _ignore_RO="`_ignore_READONLY`"
 100         _ignore_HARD="`_ignore_HARDCODED`"
 101         # Macos-grep does not like long patterns
 102         # Old Solaris grep does not support -E
 103         # Perl Version of:
 104         # grep -Ev '^(...)$' |
 105         perl -ne '/^(
 106                      PARALLEL_ENV|
 107                      PARALLEL_TMP|
 108                      _alias_NAMES|
 109                      _bodies_of_ALIASES|
 110                      _bodies_of_FUNCTIONS|
 111                      _bodies_of_VARIABLES|
 112                      _error_PAR|
 113                      _function_NAMES|
 114                      _get_ignored_VARS|
 115                      _grep_REGEXP|
 116                      _ignore_HARD|
 117                      _ignore_HARDCODED|
 118                      _ignore_READONLY|
 119                      _ignore_RO|
 120                      _ignore_UNDERSCORE|
 121                      _list_alias_BODIES|
 122                      _list_function_BODIES|
 123                      _list_variable_VALUES|
 124                      _make_grep_REGEXP|
 125                      _names_of_ALIASES|
 126                      _names_of_FUNCTIONS|
 127                      _names_of_VARIABLES|
 128                      _names_of_maybe_FUNCTIONS|
 129                      _parallel_exit_CODE|
 130                      _prefix_PARALLEL_ENV|
 131                      _prefix_PARALLEL_ENV|
 132                      _remove_bad_NAMES|
 133                      _remove_readonly|
 134                      _variable_NAMES|
 135                      _warning_PAR|
 136                      _which_PAR)$/x and next;
 137             # Filter names matching --env
 138             /^'"$_grep_REGEXP"'$/ or next;
 139             /^'"$_ignore_UNDERSCORE"'$/ and next;
 140             # Remove readonly variables
 141             /^'"$_ignore_RO"'$/ and next;
 142             /^'"$_ignore_HARD"'$/ and next;
 143             print;'
 144     }
 145     _get_ignored_VARS() {
 146         perl -e '
 147             for(@ARGV){
 148                 $next_is_env and push @envvar, split/,/, $_;
 149                 $next_is_env=/^--env$/;
 150             }
 151             if(grep { /^_$/ } @envvar) {
 152                 if(not open(IN, "<", "$ENV{HOME}/.parallel/ignored_vars")) {
 153                     print STDERR "parallel: Error: ",
 154                     "Run \"parallel --record-env\" in a clean environment first.\n";
 155                 } else {
 156                     chomp(@ignored_vars = <IN>);
 157                 }
 158             }
 159             if($ENV{PARALLEL_IGNORED_NAMES}) {
 160                 push @ignored_vars, split/\s+/, $ENV{PARALLEL_IGNORED_NAMES};
 161                 chomp @ignored_vars;
 162             }
 163             $vars = join "|",map { quotemeta $_ } @ignored_vars;
 164             print $vars ? "($vars)" : "(,,nO,,VaRs,,)";
 165             ' -- "$@"
 166     }
 167
 168     # Get the --env variables if set
 169     # --env _ should be ignored
 170     # and convert  a b c  to (a|b|c)
 171     # If --env not set: Match everything (.*)
 172     _make_grep_REGEXP() {
 173         perl -e '
 174             for(@ARGV){
 175                 /^_$/ and $next_is_env = 0;
 176                 $next_is_env and push @envvar, split/,/, $_;
 177                 $next_is_env = /^--env$/;
 178             }
 179             $vars = join "|",map { quotemeta $_ } @envvar;
 180             print $vars ? "($vars)" : "(.*)";
 181             ' -- "$@"
 182     }
 183     _which_PAR() {
 184         # type returns:
 185         #   ll is an alias for ls -l (in ash)
 186         #   bash is a tracked alias for /bin/bash
 187         #   true is a shell builtin (in bash)
 188         #   myfunc is a function (in bash)
 189         #   myfunc is a shell function (in zsh)
 190         #   which is /usr/bin/which (in sh, bash)
 191         #   which is hashed (/usr/bin/which)
 192         #   gi is aliased to `grep -i' (in bash)
 193         #   aliased to `alias | /usr/bin/which --tty-only --read-alias --show-dot --show-tilde'
 194         # Return 0 if found, 1 otherwise
 195         LANG=C type "$@" |
 196             perl -pe '$exit += (s/ is an alias for .*// ||
 197                                 s/ is aliased to .*// ||
 198                                 s/ is a function// ||
 199                                 s/ is a shell function// ||
 200                                 s/ is a shell builtin// ||
 201                                 s/.* is hashed .(\S+).$/$1/ ||
 202                                 s/.* is (a tracked alias for )?//);
 203                       END { exit not $exit }'
 204     }
 205     _warning_PAR() {
 206         echo "env_parallel: Warning: $@" >&2
 207     }
 208     _error_PAR() {
 209         echo "env_parallel: Error: $@" >&2
 210     }
 211
 212     if _which_PAR parallel >/dev/null; then
 213         true parallel found in path
 214     else
 215         _error_PAR 'parallel must be in $PATH.'
 216         return 255
 217     fi
 218
 219     # Grep regexp for vars given by --env
 220     _grep_REGEXP="`_make_grep_REGEXP \"$@\"`"
 221     unset _make_grep_REGEXP
 222
 223     # Deal with --env _
 224     _ignore_UNDERSCORE="`_get_ignored_VARS \"$@\"`"
 225     unset _get_ignored_VARS
 226
 227     # --record-env
 228     if perl -e 'exit grep { /^--record-env$/ } @ARGV' -- "$@"; then
 229         true skip
 230     else
 231         (_names_of_ALIASES;
 232          _names_of_FUNCTIONS;
 233          _names_of_VARIABLES) |
 234             cat > $HOME/.parallel/ignored_vars
 235         return 0
 236     fi
 237
 238     # --session
 239     if perl -e 'exit grep { /^--session$/ } @ARGV' -- "$@"; then
 240         true skip
 241     else
 242         # Insert ::: between each level of session
 243         # so you can pop off the last ::: at --end-session
 244         PARALLEL_IGNORED_NAMES="`echo \"$PARALLEL_IGNORED_NAMES\";
 245           echo :::;
 246           (_names_of_ALIASES;
 247            _names_of_FUNCTIONS;
 248            _names_of_VARIABLES) | perl -ne '
 249             BEGIN{
 250               map { $ignored_vars{$_}++ }
 251                 split/\s+/, $ENV{PARALLEL_IGNORED_NAMES};
 252             }
 253             chomp;
 254             for(split/\s+/) {
 255               if(not $ignored_vars{$_}) {
 256                 print $_,\"\\n\";
 257               }
 258             }
 259             '`"
 260         export PARALLEL_IGNORED_NAMES
 261         return 0
 262     fi
 263     if perl -e 'exit grep { /^--end.?session$/ } @ARGV' -- "$@"; then
 264         true skip
 265     else
 266         # Pop off last ::: from PARALLEL_IGNORED_NAMES
 267         PARALLEL_IGNORED_NAMES="`perl -e '
 268           $ENV{PARALLEL_IGNORED_NAMES} =~ s/(.*):::.*?$/$1/s;
 269           print $ENV{PARALLEL_IGNORED_NAMES}
 270         '`"
 271         return 0
 272     fi
 273     # Grep alias names
 274     _alias_NAMES="`_names_of_ALIASES | _remove_bad_NAMES | xargs echo`"
 275     _list_alias_BODIES="_bodies_of_ALIASES $_alias_NAMES"
 276     if [ "$_alias_NAMES" = "" ] ; then
 277         # no aliases selected
 278         _list_alias_BODIES="true"
 279     fi
 280     unset _alias_NAMES
 281
 282     # Grep function names
 283     _function_NAMES="`_names_of_FUNCTIONS | _remove_bad_NAMES | xargs echo`"
 284     _list_function_BODIES="_bodies_of_FUNCTIONS $_function_NAMES"
 285     if [ "$_function_NAMES" = "" ] ; then
 286         # no functions selected
 287         _list_function_BODIES="true"
 288     fi
 289     unset _function_NAMES
 290
 291     # Grep variable names
 292     _variable_NAMES="`_names_of_VARIABLES | _remove_bad_NAMES | xargs echo`"
 293     _list_variable_VALUES="_bodies_of_VARIABLES $_variable_NAMES"
 294     if [ "$_variable_NAMES" = "" ] ; then
 295         # no variables selected
 296         _list_variable_VALUES="true"
 297     fi
 298     unset _variable_NAMES
 299
 300     PARALLEL_ENV="`
 301         $_list_alias_BODIES;
 302         $_list_function_BODIES;
 303         $_list_variable_VALUES;
 304     `"
 305     export PARALLEL_ENV
 306     unset _list_alias_BODIES _list_variable_VALUES _list_function_BODIES
 307     unset _bodies_of_ALIASES _bodies_of_VARIABLES _bodies_of_FUNCTIONS
 308     unset _names_of_ALIASES _names_of_VARIABLES _names_of_FUNCTIONS
 309     unset _ignore_HARDCODED _ignore_READONLY _ignore_UNDERSCORE
 310     unset _remove_bad_NAMES _grep_REGEXP
 311     unset _prefix_PARALLEL_ENV
 312     # Test if environment is too big
 313     if `_which_PAR true` >/dev/null 2>/dev/null ; then
 314         parallel "$@"
 315         _parallel_exit_CODE=$?
 316         # Clean up variables/functions
 317         unset PARALLEL_ENV
 318         unset _which_PAR _which_TRUE
 319         unset _warning_PAR _error_PAR
 320         # Unset _parallel_exit_CODE before return
 321         eval "unset _parallel_exit_CODE; return $_parallel_exit_CODE"
 322     else
 323         unset PARALLEL_ENV;
 324         _error_PAR "Your environment is too big."
 325         _error_PAR "You can try 3 different approaches:"
 326         _error_PAR "1. Run 'env_parallel --session' before you set"
 327         _error_PAR "   variables or define functions."
 328         _error_PAR "2. Use --env and only mention the names to copy."
 329         _error_PAR "3. Try running this in a clean environment once:"
 330         _error_PAR "     env_parallel --record-env"
 331         _error_PAR "   And then use '--env _'"
 332         _error_PAR "For details see: man env_parallel"
 333         return 255
 334     fi
 335 }
 336
 337 parset() {
 338     _parset_PARALLEL_PRG=parallel
 339     _parset_main "$@"
 340 }
 341
 342 env_parset() {
 343     _parset_PARALLEL_PRG=env_parallel
 344     _parset_main "$@"
 345 }
 346
 347 _parset_main() {
 348     # If $1 contains ',' or space:
 349     #   Split on , to get the destination variable names
 350     # If $1 is a single destination variable name:
 351     #   Treat it as the name of an array
 352     #
 353     #   # Create array named myvar
 354     #   parset myvar echo ::: {1..10}
 355     #   echo ${myvar[5]}
 356     #
 357     #   # Put output into $var_a $var_b $var_c
 358     #   varnames=(var_a var_b var_c)
 359     #   parset "${varnames[*]}" echo ::: {1..3}
 360     #   echo $var_c
 361     #
 362     #   # Put output into $var_a4 $var_b4 $var_c4
 363     #   parset "var_a4 var_b4 var_c4" echo ::: {1..3}
 364     #   echo $var_c4
 365
 366     _make_TEMP() {
 367         # mktemp does not exist on some OS
 368         perl -e 'use File::Temp qw(tempfile);
 369                  $ENV{"TMPDIR"} ||= "/tmp";
 370                  print((tempfile(DIR=>$ENV{"TMPDIR"}, TEMPLATE => "parXXXXX"))[1])'
 371     }
 372
 373     _parset_NAME="$1"
 374     if [ "$_parset_NAME" = "" ] ; then
 375         echo parset: Error: No destination variable given. >&2
 376         echo parset: Error: Try: >&2
 377         echo parset: Error: ' ' parset myarray echo ::: foo bar >&2
 378         return 255
 379     fi
 380     if [ "$_parset_NAME" = "--help" ] ; then
 381         echo parset: Error: Usage: >&2
 382         echo parset: Error: ' ' parset varname GNU Parallel options and command >&2
 383         echo
 384         parallel --help
 385         return 255
 386     fi
 387     if [ "$_parset_NAME" = "--version" ] ; then
 388         echo "parset 20210622 (GNU parallel `parallel --minversion 1`)"
 389         echo "Copyright (C) 2007-2021 Ole Tange, http://ole.tange.dk and Free Software"
 390         echo "Foundation, Inc."
 391         echo "License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>"
 392         echo "This is free software: you are free to change and redistribute it."
 393         echo "GNU parallel comes with no warranty."
 394         echo
 395         echo "Web site: https://www.gnu.org/software/parallel"
 396         echo
 397         echo "When using programs that use GNU Parallel to process data for publication"
 398         echo "please cite as described in 'parallel --citation'."
 399         echo
 400         return 255
 401     fi
 402     shift
 403     echo "$_parset_NAME" |
 404         perl -ne 'chomp;for (split /[, ]/) {
 405             # Allow: var_32 var[3]
 406             if(not /^[a-zA-Z_][a-zA-Z_0-9]*(\[\d+\])?$/) {
 407                 print STDERR "parset: Error: $_ is an invalid variable name.\n";
 408                 print STDERR "parset: Error: Variable names must be letter followed by letters or digits.\n";
 409                 print STDERR "parset: Error: Usage:\n";
 410                 print STDERR "parset: Error:   parset varname GNU Parallel options and command\n";
 411                 $exitval = 255;
 412             }
 413         }
 414         exit $exitval;
 415         ' || return 255
 416     _exit_FILE=`_make_TEMP`
 417     if perl -e 'exit not grep /,| /, @ARGV' "$_parset_NAME" ; then
 418         # $_parset_NAME contains , or space
 419         # Split on , or space to get the names
 420         eval "`
 421             # Compute results into files
 422             ($_parset_PARALLEL_PRG --files -k "$@"; echo $? > "$_exit_FILE") |
 423                 # var1= cat tmpfile1; rm tmpfile1
 424                 # var2= cat tmpfile2; rm tmpfile2
 425                 parallel -q echo {2}='\`cat {1}; rm {1}\`' :::: - :::+ \`
 426                     echo "$_parset_NAME" |
 427                         perl -pe 's/,/ /g'
 428                          \`
 429             `"
 430     else
 431         # $_parset_NAME does not contain , or space
 432         # => $_parset_NAME is the name of the array to put data into
 433         # Supported in: bash zsh ksh mksh
 434         # Arrays do not work in: sh ash dash
 435         eval "$_parset_NAME=( $(
 436             # Compute results into files. Save exit value
 437             ($_parset_PARALLEL_PRG --files -k "$@"; echo $? > "$_exit_FILE") |
 438                 perl -pe 'chop;$_="\"\`cat $_; rm $_\`\" "'
 439             ) )"
 440     fi
 441     unset _parset_NAME _parset_PARALLEL_PRG _parallel_exit_CODE
 442     # Unset _exit_FILE before return
 443     eval "unset _exit_FILE; return \`cat $_exit_FILE; rm $_exit_FILE\`"
 444 }