From 04427e3bf236c18cc532680b957267ee70b1037d Mon Sep 17 00:00:00 2001 From: Andrew Bennett Date: Sun, 28 May 2017 18:34:50 +0000 Subject: [PATCH] 4703 would like xargs support for -P Contributions by: Robert Mustacchi Reviewed by: Gordon Ross Reviewed by: Toomas Soome Approved by: Richard Lowe --- usr/src/cmd/xargs/xargs.c | 193 +++++++++++++++++---- usr/src/man/man1/xargs.1 | 23 +-- usr/src/test/util-tests/tests/xargs/xargs_test.ksh | 37 ++++ 3 files changed, 214 insertions(+), 39 deletions(-) diff --git a/usr/src/cmd/xargs/xargs.c b/usr/src/cmd/xargs/xargs.c index 2e568d54ba..517c33c7e1 100644 --- a/usr/src/cmd/xargs/xargs.c +++ b/usr/src/cmd/xargs/xargs.c @@ -21,6 +21,7 @@ /* * Copyright 2014 Garrett D'Amore * Copyright 2012 DEY Storage Systems, Inc. + * Copyright (c) 2017, Joyent, Inc. * * Portions of this file developed by DEY Storage Systems, Inc. are licensed * under the terms of the Common Development and Distribution License (CDDL) @@ -54,6 +55,7 @@ #include #include #include +#include #include "getresponse.h" #define HEAD 0 @@ -90,6 +92,7 @@ #define MISSQUOTE "Missing quote" #define BADESCAPE "Incomplete escape" #define IBUFOVERFLOW "Insert buffer overflow" +#define NOCHILDSLOT "No free child slot available" #define _(x) gettext(x) @@ -110,6 +113,7 @@ static struct inserts { static int PROMPT = -1; static int BUFLIM = BUFSIZE; +static int MAXPROCS = 1; static int N_ARGS = 0; static int N_args = 0; static int N_lines = 0; @@ -130,15 +134,17 @@ static int exitstat = 0; /* our exit status */ static int mac; /* modified argc, after parsing */ static char **mav; /* modified argv, after parsing */ static int n_inserts; /* # of insertions. */ +static pid_t *procs; /* pids of children */ +static int n_procs; /* # of child processes. */ /* our usage message: */ #define USAGEMSG "Usage: xargs: [-t] [-p] [-0] [-e[eofstr]] [-E eofstr] "\ - "[-I replstr] [-i[replstr]] [-L #] [-l[#]] [-n # [-x]] [-s size] "\ - "[cmd [args ...]]\n" + "[-I replstr] [-i[replstr]] [-L #] [-l[#]] [-n # [-x]] [-P maxprocs] "\ + "[-s size] [cmd [args ...]]\n" static int echoargs(); static wint_t getwchr(char *, size_t *); -static int lcall(char *sub, char **subargs); +static void lcall(char *sub, char **subargs); static void addibuf(struct inserts *p); static void ermsg(char *messages, ...); static char *addarg(char *arg); @@ -147,17 +153,24 @@ static char *getarg(char *); static char *insert(char *pattern, char *subst); static void usage(); static void parseargs(); +static int procs_find(pid_t child); +static void procs_store(pid_t child); +static boolean_t procs_delete(pid_t child); +static pid_t procs_waitpid(boolean_t blocking, int *stat_loc); +static void procs_wait(boolean_t blocking); int main(int argc, char **argv) { int j; + unsigned long l; struct inserts *psave; int c; int initsize; char *cmdname, **initlist; char *arg; char *next; + char *eptr; /* initialization */ blank = wctype("blank"); @@ -176,7 +189,7 @@ main(int argc, char **argv) parseargs(argc, argv); /* handling all of xargs arguments: */ - while ((c = getopt(mac, mav, "0tpe:E:I:i:L:l:n:s:x")) != EOF) { + while ((c = getopt(mac, mav, "0tpe:E:I:i:L:l:n:P:s:x")) != EOF) { switch (c) { case '0': ZERO = TRUE; @@ -301,6 +314,25 @@ main(int argc, char **argv) } break; + case 'P': /* -P maxprocs: # of child processses */ + errno = 0; + l = strtoul(optarg, &eptr, 10); + if (*eptr != '\0' || errno != 0) { + ermsg(_("failed to parse maxprocs (-P): %s\n"), + optarg); + break; + } + + /* + * Come up with an upper bound that'll probably fit in + * memory. + */ + if (l == 0 || l > ((INT_MAX / sizeof (pid_t) >> 1))) { + l = INT_MAX / sizeof (pid_t) >> 1; + } + MAXPROCS = (int)l; + break; + case 's': /* -s size: set max size of each arg list */ BUFLIM = atoi(optarg); if (BUFLIM > BUFSIZE || BUFLIM <= 0) { @@ -341,6 +373,12 @@ main(int argc, char **argv) mac -= optind; /* dec arg count by what we've processed */ mav += optind; /* inc to current mav */ + procs = calloc(MAXPROCS, sizeof (pid_t)); + if (procs == NULL) { + PERR(MALLOCFAIL); + exit(1); + } + if (mac <= 0) { /* if there're no more args to process, */ cmdname = "/usr/bin/echo"; /* our default command */ *ARGV++ = addarg(cmdname); /* use the default cmd. */ @@ -411,6 +449,7 @@ main(int argc, char **argv) */ if (LEGAL || N_args == 0) { EMSG(LIST2LONG); + procs_wait(B_TRUE); exit(2); /* NOTREACHED */ } @@ -440,7 +479,7 @@ main(int argc, char **argv) *ARGV = NULL; if (N_args == 0) { /* Reached the end with no more work. */ - exit(exitstat); + break; } /* insert arg if requested */ @@ -469,6 +508,7 @@ main(int argc, char **argv) } if (linesize >= BUFLIM) { EMSG(LIST2LONG); + procs_wait(B_TRUE); exit(2); /* NOTREACHED */ } @@ -489,11 +529,13 @@ main(int argc, char **argv) * so if we have a non-zero status here, * quit immediately. */ - exitstat |= lcall(cmdname, arglist); + (void) lcall(cmdname, arglist); } } } + procs_wait(B_TRUE); + if (OK) return (exitstat); @@ -863,34 +905,22 @@ getwchr(char *mbc, size_t *sz) } -static int +static void lcall(char *sub, char **subargs) { - int retcode, retry = 0; - pid_t iwait, child; + int retry = 0; + pid_t child; for (;;) { - switch (child = fork()) { + switch (child = forkx(FORK_NOSIGCHLD)) { default: - while ((iwait = wait(&retcode)) != child && - iwait != (pid_t)-1) - ; - if (iwait == (pid_t)-1) { - PERR(WAITFAIL); - exit(122); - /* NOTREACHED */ - } - if (WIFSIGNALED(retcode)) { - EMSG2(CHILDSIG, WTERMSIG(retcode)); - exit(125); - /* NOTREACHED */ - } - if ((WEXITSTATUS(retcode) & 0377) == 0377) { - EMSG(CHILDFAIL); - exit(124); - /* NOTREACHED */ - } - return (WEXITSTATUS(retcode)); + procs_store(child); + /* + * Note, if we have used up all of our slots, then this + * call may end up blocking. + */ + procs_wait(B_FALSE); + return; case 0: (void) execvp(sub, subargs); PERR(EXECFAIL); @@ -908,6 +938,110 @@ lcall(char *sub, char **subargs) } } +/* + * Return the index of child in the procs array. + */ +static int +procs_find(pid_t child) +{ + int i; + + for (i = 0; i < MAXPROCS; i++) { + if (procs[i] == child) { + return (i); + } + } + + return (-1); +} + +static void +procs_store(pid_t child) +{ + int i; + + i = procs_find(0); + if (i < 0) { + EMSG(NOCHILDSLOT); + exit(1); + } + procs[i] = child; + n_procs++; +} + +static boolean_t +procs_delete(pid_t child) +{ + int i; + + i = procs_find(child); + if (i < 0) { + return (B_FALSE); + } + + procs[i] = (pid_t)0; + n_procs--; + + return (B_TRUE); +} + +static pid_t +procs_waitpid(boolean_t blocking, int *stat_loc) +{ + pid_t child; + int options; + + if (n_procs == 0) { + errno = ECHILD; + return (-1); + } + + options = 0; + if (!blocking) { + options |= WNOHANG; + } + + while ((child = waitpid((pid_t)-1, stat_loc, options)) > 0) { + if (procs_delete(child)) { + break; + } + } + + return (child); +} + +static void +procs_wait(boolean_t blocking) +{ + pid_t child; + int stat_loc; + + /* + * If we currently have filled all of our slots, then we need to block + * further execution. + */ + if (n_procs >= MAXPROCS) + blocking = B_TRUE; + while ((child = procs_waitpid(blocking, &stat_loc)) > 0) { + if (WIFSIGNALED(stat_loc)) { + EMSG2(CHILDSIG, WTERMSIG(stat_loc)); + exit(125); + /* NOTREACHED */ + } else if ((WEXITSTATUS(stat_loc) & 0377) == 0377) { + EMSG(CHILDFAIL); + exit(124); + /* NOTREACHED */ + } else { + exitstat |= WEXITSTATUS(stat_loc); + } + } + + if (child == (pid_t)(-1) && errno != ECHILD) { + EMSG(WAITFAIL); + exit(122); + /* NOTREACHED */ + } +} static void usage() @@ -1006,6 +1140,7 @@ process_special: * and the new XCU4 way of handling things are allowed. */ case 'n': /* FALLTHROUGH */ + case 'P': /* FALLTHROUGH */ case 's': /* FALLTHROUGH */ case 'E': /* FALLTHROUGH */ case 'I': /* FALLTHROUGH */ diff --git a/usr/src/man/man1/xargs.1 b/usr/src/man/man1/xargs.1 index f60b9a161c..6f4b596b4c 100644 --- a/usr/src/man/man1/xargs.1 +++ b/usr/src/man/man1/xargs.1 @@ -7,7 +7,7 @@ .\" The contents of this file are subject to the terms of the Common Development and Distribution License (the "License"). You may not use this file except in compliance with the License. .\" You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE or http://www.opensolaris.org/os/licensing. See the License for the specific language governing permissions and limitations under the License. .\" When distributing Covered Code, include this CDDL HEADER in each file and include the License file at usr/src/OPENSOLARIS.LICENSE. If applicable, add the following below this CDDL HEADER, with the fields enclosed by brackets "[]" replaced with your own identifying information: Portions Copyright [yyyy] [name of copyright owner] -.TH XARGS 1 "November 24, 2012" +.TH XARGS 1 "May 28, 2017" .SH NAME xargs \- construct argument lists and invoke utility .SH SYNOPSIS @@ -15,11 +15,11 @@ xargs \- construct argument lists and invoke utility .nf \fBxargs\fR [\fB-t\fR] [\fB-0\fR] [\fB-p\fR] [\fB-e\fR[\fIeofstr\fR]] [\fB-E\fR \fIeofstr\fR] [\fB-I\fR \fIreplstr\fR] [\fB-i\fR[\fIreplstr\fR]] [\fB-L\fR \fInumber\fR] [\fB-l\fR[\fInumber\fR]] - [\fB-n\fR \fInumber\fR [\fB-x\fR]] [\fB-s\fR \fIsize\fR] [\fIutility\fR [\fIargument\fR...]] + [\fB-n\fR \fInumber\fR [\fB-x\fR]] [\fB-P\fR \fImaxprocs\fR] [\fB-s\fR \fIsize\fR] + [\fIutility\fR [\fIargument\fR...]] .fi .SH DESCRIPTION -.sp .LP The \fBxargs\fR utility constructs a command line consisting of the \fIutility\fR and \fIargument\fR operands specified followed by as many @@ -49,7 +49,6 @@ argument and environment lists can not exceed \fB{ARG_MAX}\(mi2048\fR bytes. Within this constraint, if neither the \fB-n\fR nor the \fB-s\fR option is specified, the default command line length is at least \fB{LINE_MAX}\fR. .SH OPTIONS -.sp .LP The following options are supported: .sp @@ -171,6 +170,16 @@ otherwise, that particular invocation of \fIutility\fR is skipped. .sp .ne 2 .na +\fB\fB-P\fR \fImaxprocs\fR\fR +.ad +.RS 15n +Invokes \fIutility\fR using at most \fImaxprocs\fR (a positive decimal integer) +parallel child processes. +.RE + +.sp +.ne 2 +.na \fB\fB-s\fR \fIsize\fR\fR .ad .RS 15n @@ -236,7 +245,6 @@ the -print0 argument to \fBfind\fR(1). .RE .SH OPERANDS -.sp .LP The following operands are supported: .sp @@ -262,7 +270,6 @@ An initial option or operand for the invocation of \fIutility\fR. .RE .SH USAGE -.sp .LP The \fB255\fR exit status allows a utility being used by \fBxargs\fR to tell \fBxargs\fR to terminate if it knows no further invocations using the current @@ -371,7 +378,6 @@ example% \fBecho $* | xargs -n 2 diff\fR .sp .SH ENVIRONMENT VARIABLES -.sp .LP See \fBenviron\fR(5) for descriptions of the following environment variables that affect the execution of \fBxargs\fR: \fBLANG\fR, \fBLC_ALL\fR, @@ -396,7 +402,6 @@ in \fBLC_CTYPE\fR determines the locale for interpretation of sequences of bytes of text data a characters, the behavior of character classes used in the expression defined for the \fByesexpr\fR. See \fBlocale\fR(5). .SH EXIT STATUS -.sp .LP The following exit values are returned: .sp @@ -445,7 +450,6 @@ signal, or an invocation of the utility exits with exit status \fB255\fR, the \fBxargs\fR utility writes a diagnostic message and exit without processing any remaining input. .SH ATTRIBUTES -.sp .LP See \fBattributes\fR(5) for descriptions of the following attributes: .sp @@ -463,7 +467,6 @@ Interface Stability Standard .TE .SH SEE ALSO -.sp .LP \fBecho\fR(1), \fBshell_builtins\fR(1), \fBattributes\fR(5), \fBenviron\fR(5), \fBstandards\fR(5) diff --git a/usr/src/test/util-tests/tests/xargs/xargs_test.ksh b/usr/src/test/util-tests/tests/xargs/xargs_test.ksh index 2d6f76ce10..6e217c45e0 100644 --- a/usr/src/test/util-tests/tests/xargs/xargs_test.ksh +++ b/usr/src/test/util-tests/tests/xargs/xargs_test.ksh @@ -13,6 +13,7 @@ # # Copyright 2014 Garrett D'Amore +# Copyright (c) 2017, Joyent, Inc. # XARGS=${XARGS:=/usr/bin/xargs} @@ -242,6 +243,39 @@ test18() { test_pass $t } +test19() { + t=test19 + test_start $t "bad -P option (negative value)" + $XARGS -P -3 /dev/null + if [[ $? -eq 2 ]]; then + test_pass $t + else + test_fail $t + fi +} + +test20() { + t=test20 + test_start $t "bad -P option (bad string)" + $XARGS -P as3f /dev/null + if [[ $? -eq 2 ]]; then + test_pass $t + else + test_fail $t + fi +} + +test21() { + t=test21 + test_start $t "bad -P option (extraneous characters)" + $XARGS -P 2c /dev/null + if [[ $? -eq 2 ]]; then + test_pass $t + else + test_fail $t + fi +} + test1 test2 test3 @@ -260,3 +294,6 @@ test15 test16 test17 test18 +test19 +test20 +test21 -- 2.11.4.GIT