run-command.c

   1 #include "cache.h"
   2 #include "run-command.h"
   3 #include "environment.h"
   4 #include "exec-cmd.h"
   5 #include "gettext.h"
   6 #include "sigchain.h"
   7 #include "strvec.h"
   8 #include "symlinks.h"
   9 #include "thread-utils.h"
  10 #include "strbuf.h"
  11 #include "string-list.h"
  12 #include "trace.h"
  13 #include "trace2.h"
  14 #include "quote.h"
  15 #include "config.h"
  16 #include "packfile.h"
  17 #include "hook.h"
  18 #include "compat/nonblock.h"
  19
  20 void child_process_init(struct child_process *child)
  21 {
  22         struct child_process blank = CHILD_PROCESS_INIT;
  23         memcpy(child, &blank, sizeof(*child));
  24 }
  25
  26 void child_process_clear(struct child_process *child)
  27 {
  28         strvec_clear(&child->args);
  29         strvec_clear(&child->env);
  30 }
  31
  32 struct child_to_clean {
  33         pid_t pid;
  34         struct child_process *process;
  35         struct child_to_clean *next;
  36 };
  37 static struct child_to_clean *children_to_clean;
  38 static int installed_child_cleanup_handler;
  39
  40 static void cleanup_children(int sig, int in_signal)
  41 {
  42         struct child_to_clean *children_to_wait_for = NULL;
  43
  44         while (children_to_clean) {
  45                 struct child_to_clean *p = children_to_clean;
  46                 children_to_clean = p->next;
  47
  48                 if (p->process && !in_signal) {
  49                         struct child_process *process = p->process;
  50                         if (process->clean_on_exit_handler) {
  51                                 trace_printf(
  52                                         "trace: run_command: running exit handler for pid %"
  53                                         PRIuMAX, (uintmax_t)p->pid
  54                                 );
  55                                 process->clean_on_exit_handler(process);
  56                         }
  57                 }
  58
  59                 kill(p->pid, sig);
  60
  61                 if (p->process && p->process->wait_after_clean) {
  62                         p->next = children_to_wait_for;
  63                         children_to_wait_for = p;
  64                 } else {
  65                         if (!in_signal)
  66                                 free(p);
  67                 }
  68         }
  69
  70         while (children_to_wait_for) {
  71                 struct child_to_clean *p = children_to_wait_for;
  72                 children_to_wait_for = p->next;
  73
  74                 while (waitpid(p->pid, NULL, 0) < 0 && errno == EINTR)
  75                         ; /* spin waiting for process exit or error */
  76
  77                 if (!in_signal)
  78                         free(p);
  79         }
  80 }
  81
  82 static void cleanup_children_on_signal(int sig)
  83 {
  84         cleanup_children(sig, 1);
  85         sigchain_pop(sig);
  86         raise(sig);
  87 }
  88
  89 static void cleanup_children_on_exit(void)
  90 {
  91         cleanup_children(SIGTERM, 0);
  92 }
  93
  94 static void mark_child_for_cleanup(pid_t pid, struct child_process *process)
  95 {
  96         struct child_to_clean *p = xmalloc(sizeof(*p));
  97         p->pid = pid;
  98         p->process = process;
  99         p->next = children_to_clean;
 100         children_to_clean = p;
 101
 102         if (!installed_child_cleanup_handler) {
 103                 atexit(cleanup_children_on_exit);
 104                 sigchain_push_common(cleanup_children_on_signal);
 105                 installed_child_cleanup_handler = 1;
 106         }
 107 }
 108
 109 static void clear_child_for_cleanup(pid_t pid)
 110 {
 111         struct child_to_clean **pp;
 112
 113         for (pp = &children_to_clean; *pp; pp = &(*pp)->next) {
 114                 struct child_to_clean *clean_me = *pp;
 115
 116                 if (clean_me->pid == pid) {
 117                         *pp = clean_me->next;
 118                         free(clean_me);
 119                         return;
 120                 }
 121         }
 122 }
 123
 124 static inline void close_pair(int fd[2])
 125 {
 126         close(fd[0]);
 127         close(fd[1]);
 128 }
 129
 130 int is_executable(const char *name)
 131 {
 132         struct stat st;
 133
 134         if (stat(name, &st) || /* stat, not lstat */
 135             !S_ISREG(st.st_mode))
 136                 return 0;
 137
 138 #if defined(GIT_WINDOWS_NATIVE)
 139         /*
 140          * On Windows there is no executable bit. The file extension
 141          * indicates whether it can be run as an executable, and Git
 142          * has special-handling to detect scripts and launch them
 143          * through the indicated script interpreter. We test for the
 144          * file extension first because virus scanners may make
 145          * it quite expensive to open many files.
 146          */
 147         if (ends_with(name, ".exe"))
 148                 return S_IXUSR;
 149
 150 {
 151         /*
 152          * Now that we know it does not have an executable extension,
 153          * peek into the file instead.
 154          */
 155         char buf[3] = { 0 };
 156         int n;
 157         int fd = open(name, O_RDONLY);
 158         st.st_mode &= ~S_IXUSR;
 159         if (fd >= 0) {
 160                 n = read(fd, buf, 2);
 161                 if (n == 2)
 162                         /* look for a she-bang */
 163                         if (!strcmp(buf, "#!"))
 164                                 st.st_mode |= S_IXUSR;
 165                 close(fd);
 166         }
 167 }
 168 #endif
 169         return st.st_mode & S_IXUSR;
 170 }
 171
 172 /*
 173  * Search $PATH for a command.  This emulates the path search that
 174  * execvp would perform, without actually executing the command so it
 175  * can be used before fork() to prepare to run a command using
 176  * execve() or after execvp() to diagnose why it failed.
 177  *
 178  * The caller should ensure that file contains no directory
 179  * separators.
 180  *
 181  * Returns the path to the command, as found in $PATH or NULL if the
 182  * command could not be found.  The caller inherits ownership of the memory
 183  * used to store the resultant path.
 184  *
 185  * This should not be used on Windows, where the $PATH search rules
 186  * are more complicated (e.g., a search for "foo" should find
 187  * "foo.exe").
 188  */
 189 static char *locate_in_PATH(const char *file)
 190 {
 191         const char *p = getenv("PATH");
 192         struct strbuf buf = STRBUF_INIT;
 193
 194         if (!p || !*p)
 195                 return NULL;
 196
 197         while (1) {
 198                 const char *end = strchrnul(p, ':');
 199
 200                 strbuf_reset(&buf);
 201
 202                 /* POSIX specifies an empty entry as the current directory. */
 203                 if (end != p) {
 204                         strbuf_add(&buf, p, end - p);
 205                         strbuf_addch(&buf, '/');
 206                 }
 207                 strbuf_addstr(&buf, file);
 208
 209                 if (is_executable(buf.buf))
 210                         return strbuf_detach(&buf, NULL);
 211
 212                 if (!*end)
 213                         break;
 214                 p = end + 1;
 215         }
 216
 217         strbuf_release(&buf);
 218         return NULL;
 219 }
 220
 221 int exists_in_PATH(const char *command)
 222 {
 223         char *r = locate_in_PATH(command);
 224         int found = r != NULL;
 225         free(r);
 226         return found;
 227 }
 228
 229 int sane_execvp(const char *file, char * const argv[])
 230 {
 231 #ifndef GIT_WINDOWS_NATIVE
 232         /*
 233          * execvp() doesn't return, so we all we can do is tell trace2
 234          * what we are about to do and let it leave a hint in the log
 235          * (unless of course the execvp() fails).
 236          *
 237          * we skip this for Windows because the compat layer already
 238          * has to emulate the execvp() call anyway.
 239          */
 240         int exec_id = trace2_exec(file, (const char **)argv);
 241 #endif
 242
 243         if (!execvp(file, argv))
 244                 return 0; /* cannot happen ;-) */
 245
 246 #ifndef GIT_WINDOWS_NATIVE
 247         {
 248                 int ec = errno;
 249                 trace2_exec_result(exec_id, ec);
 250                 errno = ec;
 251         }
 252 #endif
 253
 254         /*
 255          * When a command can't be found because one of the directories
 256          * listed in $PATH is unsearchable, execvp reports EACCES, but
 257          * careful usability testing (read: analysis of occasional bug
 258          * reports) reveals that "No such file or directory" is more
 259          * intuitive.
 260          *
 261          * We avoid commands with "/", because execvp will not do $PATH
 262          * lookups in that case.
 263          *
 264          * The reassignment of EACCES to errno looks like a no-op below,
 265          * but we need to protect against exists_in_PATH overwriting errno.
 266          */
 267         if (errno == EACCES && !strchr(file, '/'))
 268                 errno = exists_in_PATH(file) ? EACCES : ENOENT;
 269         else if (errno == ENOTDIR && !strchr(file, '/'))
 270                 errno = ENOENT;
 271         return -1;
 272 }
 273
 274 static const char **prepare_shell_cmd(struct strvec *out, const char **argv)
 275 {
 276         if (!argv[0])
 277                 BUG("shell command is empty");
 278
 279         if (strcspn(argv[0], "|&;<>()$`\\\"' \t\n*?[#~=%") != strlen(argv[0])) {
 280 #ifndef GIT_WINDOWS_NATIVE
 281                 strvec_push(out, SHELL_PATH);
 282 #else
 283                 strvec_push(out, "sh");
 284 #endif
 285                 strvec_push(out, "-c");
 286
 287                 /*
 288                  * If we have no extra arguments, we do not even need to
 289                  * bother with the "$@" magic.
 290                  */
 291                 if (!argv[1])
 292                         strvec_push(out, argv[0]);
 293                 else
 294                         strvec_pushf(out, "%s \"$@\"", argv[0]);
 295         }
 296
 297         strvec_pushv(out, argv);
 298         return out->v;
 299 }
 300
 301 #ifndef GIT_WINDOWS_NATIVE
 302 static int child_notifier = -1;
 303
 304 enum child_errcode {
 305         CHILD_ERR_CHDIR,
 306         CHILD_ERR_DUP2,
 307         CHILD_ERR_CLOSE,
 308         CHILD_ERR_SIGPROCMASK,
 309         CHILD_ERR_ENOENT,
 310         CHILD_ERR_SILENT,
 311         CHILD_ERR_ERRNO
 312 };
 313
 314 struct child_err {
 315         enum child_errcode err;
 316         int syserr; /* errno */
 317 };
 318
 319 static void child_die(enum child_errcode err)
 320 {
 321         struct child_err buf;
 322
 323         buf.err = err;
 324         buf.syserr = errno;
 325
 326         /* write(2) on buf smaller than PIPE_BUF (min 512) is atomic: */
 327         xwrite(child_notifier, &buf, sizeof(buf));
 328         _exit(1);
 329 }
 330
 331 static void child_dup2(int fd, int to)
 332 {
 333         if (dup2(fd, to) < 0)
 334                 child_die(CHILD_ERR_DUP2);
 335 }
 336
 337 static void child_close(int fd)
 338 {
 339         if (close(fd))
 340                 child_die(CHILD_ERR_CLOSE);
 341 }
 342
 343 static void child_close_pair(int fd[2])
 344 {
 345         child_close(fd[0]);
 346         child_close(fd[1]);
 347 }
 348
 349 static void child_error_fn(const char *err UNUSED, va_list params UNUSED)
 350 {
 351         const char msg[] = "error() should not be called in child\n";
 352         xwrite(2, msg, sizeof(msg) - 1);
 353 }
 354
 355 static void child_warn_fn(const char *err UNUSED, va_list params UNUSED)
 356 {
 357         const char msg[] = "warn() should not be called in child\n";
 358         xwrite(2, msg, sizeof(msg) - 1);
 359 }
 360
 361 static void NORETURN child_die_fn(const char *err UNUSED, va_list params UNUSED)
 362 {
 363         const char msg[] = "die() should not be called in child\n";
 364         xwrite(2, msg, sizeof(msg) - 1);
 365         _exit(2);
 366 }
 367
 368 /* this runs in the parent process */
 369 static void child_err_spew(struct child_process *cmd, struct child_err *cerr)
 370 {
 371         static void (*old_errfn)(const char *err, va_list params);
 372         report_fn die_message_routine = get_die_message_routine();
 373
 374         old_errfn = get_error_routine();
 375         set_error_routine(die_message_routine);
 376         errno = cerr->syserr;
 377
 378         switch (cerr->err) {
 379         case CHILD_ERR_CHDIR:
 380                 error_errno("exec '%s': cd to '%s' failed",
 381                             cmd->args.v[0], cmd->dir);
 382                 break;
 383         case CHILD_ERR_DUP2:
 384                 error_errno("dup2() in child failed");
 385                 break;
 386         case CHILD_ERR_CLOSE:
 387                 error_errno("close() in child failed");
 388                 break;
 389         case CHILD_ERR_SIGPROCMASK:
 390                 error_errno("sigprocmask failed restoring signals");
 391                 break;
 392         case CHILD_ERR_ENOENT:
 393                 error_errno("cannot run %s", cmd->args.v[0]);
 394                 break;
 395         case CHILD_ERR_SILENT:
 396                 break;
 397         case CHILD_ERR_ERRNO:
 398                 error_errno("cannot exec '%s'", cmd->args.v[0]);
 399                 break;
 400         }
 401         set_error_routine(old_errfn);
 402 }
 403
 404 static int prepare_cmd(struct strvec *out, const struct child_process *cmd)
 405 {
 406         if (!cmd->args.v[0])
 407                 BUG("command is empty");
 408
 409         /*
 410          * Add SHELL_PATH so in the event exec fails with ENOEXEC we can
 411          * attempt to interpret the command with 'sh'.
 412          */
 413         strvec_push(out, SHELL_PATH);
 414
 415         if (cmd->git_cmd) {
 416                 prepare_git_cmd(out, cmd->args.v);
 417         } else if (cmd->use_shell) {
 418                 prepare_shell_cmd(out, cmd->args.v);
 419         } else {
 420                 strvec_pushv(out, cmd->args.v);
 421         }
 422
 423         /*
 424          * If there are no dir separator characters in the command then perform
 425          * a path lookup and use the resolved path as the command to exec. If
 426          * there are dir separator characters, we have exec attempt to invoke
 427          * the command directly.
 428          */
 429         if (!has_dir_sep(out->v[1])) {
 430                 char *program = locate_in_PATH(out->v[1]);
 431                 if (program) {
 432                         free((char *)out->v[1]);
 433                         out->v[1] = program;
 434                 } else {
 435                         strvec_clear(out);
 436                         errno = ENOENT;
 437                         return -1;
 438                 }
 439         }
 440
 441         return 0;
 442 }
 443
 444 static char **prep_childenv(const char *const *deltaenv)
 445 {
 446         extern char **environ;
 447         char **childenv;
 448         struct string_list env = STRING_LIST_INIT_DUP;
 449         struct strbuf key = STRBUF_INIT;
 450         const char *const *p;
 451         int i;
 452
 453         /* Construct a sorted string list consisting of the current environ */
 454         for (p = (const char *const *) environ; p && *p; p++) {
 455                 const char *equals = strchr(*p, '=');
 456
 457                 if (equals) {
 458                         strbuf_reset(&key);
 459                         strbuf_add(&key, *p, equals - *p);
 460                         string_list_append(&env, key.buf)->util = (void *) *p;
 461                 } else {
 462                         string_list_append(&env, *p)->util = (void *) *p;
 463                 }
 464         }
 465         string_list_sort(&env);
 466
 467         /* Merge in 'deltaenv' with the current environ */
 468         for (p = deltaenv; p && *p; p++) {
 469                 const char *equals = strchr(*p, '=');
 470
 471                 if (equals) {
 472                         /* ('key=value'), insert or replace entry */
 473                         strbuf_reset(&key);
 474                         strbuf_add(&key, *p, equals - *p);
 475                         string_list_insert(&env, key.buf)->util = (void *) *p;
 476                 } else {
 477                         /* otherwise ('key') remove existing entry */
 478                         string_list_remove(&env, *p, 0);
 479                 }
 480         }
 481
 482         /* Create an array of 'char *' to be used as the childenv */
 483         ALLOC_ARRAY(childenv, env.nr + 1);
 484         for (i = 0; i < env.nr; i++)
 485                 childenv[i] = env.items[i].util;
 486         childenv[env.nr] = NULL;
 487
 488         string_list_clear(&env, 0);
 489         strbuf_release(&key);
 490         return childenv;
 491 }
 492
 493 struct atfork_state {
 494 #ifndef NO_PTHREADS
 495         int cs;
 496 #endif
 497         sigset_t old;
 498 };
 499
 500 #define CHECK_BUG(err, msg) \
 501         do { \
 502                 int e = (err); \
 503                 if (e) \
 504                         BUG("%s: %s", msg, strerror(e)); \
 505         } while(0)
 506
 507 static void atfork_prepare(struct atfork_state *as)
 508 {
 509         sigset_t all;
 510
 511         if (sigfillset(&all))
 512                 die_errno("sigfillset");
 513 #ifdef NO_PTHREADS
 514         if (sigprocmask(SIG_SETMASK, &all, &as->old))
 515                 die_errno("sigprocmask");
 516 #else
 517         CHECK_BUG(pthread_sigmask(SIG_SETMASK, &all, &as->old),
 518                 "blocking all signals");
 519         CHECK_BUG(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &as->cs),
 520                 "disabling cancellation");
 521 #endif
 522 }
 523
 524 static void atfork_parent(struct atfork_state *as)
 525 {
 526 #ifdef NO_PTHREADS
 527         if (sigprocmask(SIG_SETMASK, &as->old, NULL))
 528                 die_errno("sigprocmask");
 529 #else
 530         CHECK_BUG(pthread_setcancelstate(as->cs, NULL),
 531                 "re-enabling cancellation");
 532         CHECK_BUG(pthread_sigmask(SIG_SETMASK, &as->old, NULL),
 533                 "restoring signal mask");
 534 #endif
 535 }
 536 #endif /* GIT_WINDOWS_NATIVE */
 537
 538 static inline void set_cloexec(int fd)
 539 {
 540         int flags = fcntl(fd, F_GETFD);
 541         if (flags >= 0)
 542                 fcntl(fd, F_SETFD, flags | FD_CLOEXEC);
 543 }
 544
 545 static int wait_or_whine(pid_t pid, const char *argv0, int in_signal)
 546 {
 547         int status, code = -1;
 548         pid_t waiting;
 549         int failed_errno = 0;
 550
 551         while ((waiting = waitpid(pid, &status, 0)) < 0 && errno == EINTR)
 552                 ;       /* nothing */
 553
 554         if (waiting < 0) {
 555                 failed_errno = errno;
 556                 if (!in_signal)
 557                         error_errno("waitpid for %s failed", argv0);
 558         } else if (waiting != pid) {
 559                 if (!in_signal)
 560                         error("waitpid is confused (%s)", argv0);
 561         } else if (WIFSIGNALED(status)) {
 562                 code = WTERMSIG(status);
 563                 if (!in_signal && code != SIGINT && code != SIGQUIT && code != SIGPIPE)
 564                         error("%s died of signal %d", argv0, code);
 565                 /*
 566                  * This return value is chosen so that code & 0xff
 567                  * mimics the exit code that a POSIX shell would report for
 568                  * a program that died from this signal.
 569                  */
 570                 code += 128;
 571         } else if (WIFEXITED(status)) {
 572                 code = WEXITSTATUS(status);
 573         } else {
 574                 if (!in_signal)
 575                         error("waitpid is confused (%s)", argv0);
 576         }
 577
 578         if (!in_signal)
 579                 clear_child_for_cleanup(pid);
 580
 581         errno = failed_errno;
 582         return code;
 583 }
 584
 585 static void trace_add_env(struct strbuf *dst, const char *const *deltaenv)
 586 {
 587         struct string_list envs = STRING_LIST_INIT_DUP;
 588         const char *const *e;
 589         int i;
 590         int printed_unset = 0;
 591
 592         /* Last one wins, see run-command.c:prep_childenv() for context */
 593         for (e = deltaenv; e && *e; e++) {
 594                 struct strbuf key = STRBUF_INIT;
 595                 char *equals = strchr(*e, '=');
 596
 597                 if (equals) {
 598                         strbuf_add(&key, *e, equals - *e);
 599                         string_list_insert(&envs, key.buf)->util = equals + 1;
 600                 } else {
 601                         string_list_insert(&envs, *e)->util = NULL;
 602                 }
 603                 strbuf_release(&key);
 604         }
 605
 606         /* "unset X Y...;" */
 607         for (i = 0; i < envs.nr; i++) {
 608                 const char *var = envs.items[i].string;
 609                 const char *val = envs.items[i].util;
 610
 611                 if (val || !getenv(var))
 612                         continue;
 613
 614                 if (!printed_unset) {
 615                         strbuf_addstr(dst, " unset");
 616                         printed_unset = 1;
 617                 }
 618                 strbuf_addf(dst, " %s", var);
 619         }
 620         if (printed_unset)
 621                 strbuf_addch(dst, ';');
 622
 623         /* ... followed by "A=B C=D ..." */
 624         for (i = 0; i < envs.nr; i++) {
 625                 const char *var = envs.items[i].string;
 626                 const char *val = envs.items[i].util;
 627                 const char *oldval;
 628
 629                 if (!val)
 630                         continue;
 631
 632                 oldval = getenv(var);
 633                 if (oldval && !strcmp(val, oldval))
 634                         continue;
 635
 636                 strbuf_addf(dst, " %s=", var);
 637                 sq_quote_buf_pretty(dst, val);
 638         }
 639         string_list_clear(&envs, 0);
 640 }
 641
 642 static void trace_run_command(const struct child_process *cp)
 643 {
 644         struct strbuf buf = STRBUF_INIT;
 645
 646         if (!trace_want(&trace_default_key))
 647                 return;
 648
 649         strbuf_addstr(&buf, "trace: run_command:");
 650         if (cp->dir) {
 651                 strbuf_addstr(&buf, " cd ");
 652                 sq_quote_buf_pretty(&buf, cp->dir);
 653                 strbuf_addch(&buf, ';');
 654         }
 655         trace_add_env(&buf, cp->env.v);
 656         if (cp->git_cmd)
 657                 strbuf_addstr(&buf, " git");
 658         sq_quote_argv_pretty(&buf, cp->args.v);
 659
 660         trace_printf("%s", buf.buf);
 661         strbuf_release(&buf);
 662 }
 663
 664 int start_command(struct child_process *cmd)
 665 {
 666         int need_in, need_out, need_err;
 667         int fdin[2], fdout[2], fderr[2];
 668         int failed_errno;
 669         char *str;
 670
 671         /*
 672          * In case of errors we must keep the promise to close FDs
 673          * that have been passed in via ->in and ->out.
 674          */
 675
 676         need_in = !cmd->no_stdin && cmd->in < 0;
 677         if (need_in) {
 678                 if (pipe(fdin) < 0) {
 679                         failed_errno = errno;
 680                         if (cmd->out > 0)
 681                                 close(cmd->out);
 682                         str = "standard input";
 683                         goto fail_pipe;
 684                 }
 685                 cmd->in = fdin[1];
 686         }
 687
 688         need_out = !cmd->no_stdout
 689                 && !cmd->stdout_to_stderr
 690                 && cmd->out < 0;
 691         if (need_out) {
 692                 if (pipe(fdout) < 0) {
 693                         failed_errno = errno;
 694                         if (need_in)
 695                                 close_pair(fdin);
 696                         else if (cmd->in)
 697                                 close(cmd->in);
 698                         str = "standard output";
 699                         goto fail_pipe;
 700                 }
 701                 cmd->out = fdout[0];
 702         }
 703
 704         need_err = !cmd->no_stderr && cmd->err < 0;
 705         if (need_err) {
 706                 if (pipe(fderr) < 0) {
 707                         failed_errno = errno;
 708                         if (need_in)
 709                                 close_pair(fdin);
 710                         else if (cmd->in)
 711                                 close(cmd->in);
 712                         if (need_out)
 713                                 close_pair(fdout);
 714                         else if (cmd->out)
 715                                 close(cmd->out);
 716                         str = "standard error";
 717 fail_pipe:
 718                         error("cannot create %s pipe for %s: %s",
 719                                 str, cmd->args.v[0], strerror(failed_errno));
 720                         child_process_clear(cmd);
 721                         errno = failed_errno;
 722                         return -1;
 723                 }
 724                 cmd->err = fderr[0];
 725         }
 726
 727         trace2_child_start(cmd);
 728         trace_run_command(cmd);
 729
 730         fflush(NULL);
 731
 732         if (cmd->close_object_store)
 733                 close_object_store(the_repository->objects);
 734
 735 #ifndef GIT_WINDOWS_NATIVE
 736 {
 737         int notify_pipe[2];
 738         int null_fd = -1;
 739         char **childenv;
 740         struct strvec argv = STRVEC_INIT;
 741         struct child_err cerr;
 742         struct atfork_state as;
 743
 744         if (prepare_cmd(&argv, cmd) < 0) {
 745                 failed_errno = errno;
 746                 cmd->pid = -1;
 747                 if (!cmd->silent_exec_failure)
 748                         error_errno("cannot run %s", cmd->args.v[0]);
 749                 goto end_of_spawn;
 750         }
 751
 752         if (pipe(notify_pipe))
 753                 notify_pipe[0] = notify_pipe[1] = -1;
 754
 755         if (cmd->no_stdin || cmd->no_stdout || cmd->no_stderr) {
 756                 null_fd = xopen("/dev/null", O_RDWR | O_CLOEXEC);
 757                 set_cloexec(null_fd);
 758         }
 759
 760         childenv = prep_childenv(cmd->env.v);
 761         atfork_prepare(&as);
 762
 763         /*
 764          * NOTE: In order to prevent deadlocking when using threads special
 765          * care should be taken with the function calls made in between the
 766          * fork() and exec() calls.  No calls should be made to functions which
 767          * require acquiring a lock (e.g. malloc) as the lock could have been
 768          * held by another thread at the time of forking, causing the lock to
 769          * never be released in the child process.  This means only
 770          * Async-Signal-Safe functions are permitted in the child.
 771          */
 772         cmd->pid = fork();
 773         failed_errno = errno;
 774         if (!cmd->pid) {
 775                 int sig;
 776                 /*
 777                  * Ensure the default die/error/warn routines do not get
 778                  * called, they can take stdio locks and malloc.
 779                  */
 780                 set_die_routine(child_die_fn);
 781                 set_error_routine(child_error_fn);
 782                 set_warn_routine(child_warn_fn);
 783
 784                 close(notify_pipe[0]);
 785                 set_cloexec(notify_pipe[1]);
 786                 child_notifier = notify_pipe[1];
 787
 788                 if (cmd->no_stdin)
 789                         child_dup2(null_fd, 0);
 790                 else if (need_in) {
 791                         child_dup2(fdin[0], 0);
 792                         child_close_pair(fdin);
 793                 } else if (cmd->in) {
 794                         child_dup2(cmd->in, 0);
 795                         child_close(cmd->in);
 796                 }
 797
 798                 if (cmd->no_stderr)
 799                         child_dup2(null_fd, 2);
 800                 else if (need_err) {
 801                         child_dup2(fderr[1], 2);
 802                         child_close_pair(fderr);
 803                 } else if (cmd->err > 1) {
 804                         child_dup2(cmd->err, 2);
 805                         child_close(cmd->err);
 806                 }
 807
 808                 if (cmd->no_stdout)
 809                         child_dup2(null_fd, 1);
 810                 else if (cmd->stdout_to_stderr)
 811                         child_dup2(2, 1);
 812                 else if (need_out) {
 813                         child_dup2(fdout[1], 1);
 814                         child_close_pair(fdout);
 815                 } else if (cmd->out > 1) {
 816                         child_dup2(cmd->out, 1);
 817                         child_close(cmd->out);
 818                 }
 819
 820                 if (cmd->dir && chdir(cmd->dir))
 821                         child_die(CHILD_ERR_CHDIR);
 822
 823                 /*
 824                  * restore default signal handlers here, in case
 825                  * we catch a signal right before execve below
 826                  */
 827                 for (sig = 1; sig < NSIG; sig++) {
 828                         /* ignored signals get reset to SIG_DFL on execve */
 829                         if (signal(sig, SIG_DFL) == SIG_IGN)
 830                                 signal(sig, SIG_IGN);
 831                 }
 832
 833                 if (sigprocmask(SIG_SETMASK, &as.old, NULL) != 0)
 834                         child_die(CHILD_ERR_SIGPROCMASK);
 835
 836                 /*
 837                  * Attempt to exec using the command and arguments starting at
 838                  * argv.argv[1].  argv.argv[0] contains SHELL_PATH which will
 839                  * be used in the event exec failed with ENOEXEC at which point
 840                  * we will try to interpret the command using 'sh'.
 841                  */
 842                 execve(argv.v[1], (char *const *) argv.v + 1,
 843                        (char *const *) childenv);
 844                 if (errno == ENOEXEC)
 845                         execve(argv.v[0], (char *const *) argv.v,
 846                                (char *const *) childenv);
 847
 848                 if (errno == ENOENT) {
 849                         if (cmd->silent_exec_failure)
 850                                 child_die(CHILD_ERR_SILENT);
 851                         child_die(CHILD_ERR_ENOENT);
 852                 } else {
 853                         child_die(CHILD_ERR_ERRNO);
 854                 }
 855         }
 856         atfork_parent(&as);
 857         if (cmd->pid < 0)
 858                 error_errno("cannot fork() for %s", cmd->args.v[0]);
 859         else if (cmd->clean_on_exit)
 860                 mark_child_for_cleanup(cmd->pid, cmd);
 861
 862         /*
 863          * Wait for child's exec. If the exec succeeds (or if fork()
 864          * failed), EOF is seen immediately by the parent. Otherwise, the
 865          * child process sends a child_err struct.
 866          * Note that use of this infrastructure is completely advisory,
 867          * therefore, we keep error checks minimal.
 868          */
 869         close(notify_pipe[1]);
 870         if (xread(notify_pipe[0], &cerr, sizeof(cerr)) == sizeof(cerr)) {
 871                 /*
 872                  * At this point we know that fork() succeeded, but exec()
 873                  * failed. Errors have been reported to our stderr.
 874                  */
 875                 wait_or_whine(cmd->pid, cmd->args.v[0], 0);
 876                 child_err_spew(cmd, &cerr);
 877                 failed_errno = errno;
 878                 cmd->pid = -1;
 879         }
 880         close(notify_pipe[0]);
 881
 882         if (null_fd >= 0)
 883                 close(null_fd);
 884         strvec_clear(&argv);
 885         free(childenv);
 886 }
 887 end_of_spawn:
 888
 889 #else
 890 {
 891         int fhin = 0, fhout = 1, fherr = 2;
 892         const char **sargv = cmd->args.v;
 893         struct strvec nargv = STRVEC_INIT;
 894
 895         if (cmd->no_stdin)
 896                 fhin = open("/dev/null", O_RDWR);
 897         else if (need_in)
 898                 fhin = dup(fdin[0]);
 899         else if (cmd->in)
 900                 fhin = dup(cmd->in);
 901
 902         if (cmd->no_stderr)
 903                 fherr = open("/dev/null", O_RDWR);
 904         else if (need_err)
 905                 fherr = dup(fderr[1]);
 906         else if (cmd->err > 2)
 907                 fherr = dup(cmd->err);
 908
 909         if (cmd->no_stdout)
 910                 fhout = open("/dev/null", O_RDWR);
 911         else if (cmd->stdout_to_stderr)
 912                 fhout = dup(fherr);
 913         else if (need_out)
 914                 fhout = dup(fdout[1]);
 915         else if (cmd->out > 1)
 916                 fhout = dup(cmd->out);
 917
 918         if (cmd->git_cmd)
 919                 cmd->args.v = prepare_git_cmd(&nargv, sargv);
 920         else if (cmd->use_shell)
 921                 cmd->args.v = prepare_shell_cmd(&nargv, sargv);
 922
 923         cmd->pid = mingw_spawnvpe(cmd->args.v[0], cmd->args.v,
 924                                   (char**) cmd->env.v,
 925                                   cmd->dir, fhin, fhout, fherr);
 926         failed_errno = errno;
 927         if (cmd->pid < 0 && (!cmd->silent_exec_failure || errno != ENOENT))
 928                 error_errno("cannot spawn %s", cmd->args.v[0]);
 929         if (cmd->clean_on_exit && cmd->pid >= 0)
 930                 mark_child_for_cleanup(cmd->pid, cmd);
 931
 932         strvec_clear(&nargv);
 933         cmd->args.v = sargv;
 934         if (fhin != 0)
 935                 close(fhin);
 936         if (fhout != 1)
 937                 close(fhout);
 938         if (fherr != 2)
 939                 close(fherr);
 940 }
 941 #endif
 942
 943         if (cmd->pid < 0) {
 944                 trace2_child_exit(cmd, -1);
 945
 946                 if (need_in)
 947                         close_pair(fdin);
 948                 else if (cmd->in)
 949                         close(cmd->in);
 950                 if (need_out)
 951                         close_pair(fdout);
 952                 else if (cmd->out)
 953                         close(cmd->out);
 954                 if (need_err)
 955                         close_pair(fderr);
 956                 else if (cmd->err)
 957                         close(cmd->err);
 958                 child_process_clear(cmd);
 959                 errno = failed_errno;
 960                 return -1;
 961         }
 962
 963         if (need_in)
 964                 close(fdin[0]);
 965         else if (cmd->in)
 966                 close(cmd->in);
 967
 968         if (need_out)
 969                 close(fdout[1]);
 970         else if (cmd->out)
 971                 close(cmd->out);
 972
 973         if (need_err)
 974                 close(fderr[1]);
 975         else if (cmd->err)
 976                 close(cmd->err);
 977
 978         return 0;
 979 }
 980
 981 int finish_command(struct child_process *cmd)
 982 {
 983         int ret = wait_or_whine(cmd->pid, cmd->args.v[0], 0);
 984         trace2_child_exit(cmd, ret);
 985         child_process_clear(cmd);
 986         invalidate_lstat_cache();
 987         return ret;
 988 }
 989
 990 int finish_command_in_signal(struct child_process *cmd)
 991 {
 992         int ret = wait_or_whine(cmd->pid, cmd->args.v[0], 1);
 993         if (ret != -1)
 994                 trace2_child_exit(cmd, ret);
 995         return ret;
 996 }
 997
 998
 999 int run_command(struct child_process *cmd)
1000 {
1001         int code;
1002
1003         if (cmd->out < 0 || cmd->err < 0)
1004                 BUG("run_command with a pipe can cause deadlock");
1005
1006         code = start_command(cmd);
1007         if (code)
1008                 return code;
1009         return finish_command(cmd);
1010 }
1011
1012 #ifndef NO_PTHREADS
1013 static pthread_t main_thread;
1014 static int main_thread_set;
1015 static pthread_key_t async_key;
1016 static pthread_key_t async_die_counter;
1017
1018 static void *run_thread(void *data)
1019 {
1020         struct async *async = data;
1021         intptr_t ret;
1022
1023         if (async->isolate_sigpipe) {
1024                 sigset_t mask;
1025                 sigemptyset(&mask);
1026                 sigaddset(&mask, SIGPIPE);
1027                 if (pthread_sigmask(SIG_BLOCK, &mask, NULL)) {
1028                         ret = error("unable to block SIGPIPE in async thread");
1029                         return (void *)ret;
1030                 }
1031         }
1032
1033         pthread_setspecific(async_key, async);
1034         ret = async->proc(async->proc_in, async->proc_out, async->data);
1035         return (void *)ret;
1036 }
1037
1038 static NORETURN void die_async(const char *err, va_list params)
1039 {
1040         report_fn die_message_fn = get_die_message_routine();
1041
1042         die_message_fn(err, params);
1043
1044         if (in_async()) {
1045                 struct async *async = pthread_getspecific(async_key);
1046                 if (async->proc_in >= 0)
1047                         close(async->proc_in);
1048                 if (async->proc_out >= 0)
1049                         close(async->proc_out);
1050                 pthread_exit((void *)128);
1051         }
1052
1053         exit(128);
1054 }
1055
1056 static int async_die_is_recursing(void)
1057 {
1058         void *ret = pthread_getspecific(async_die_counter);
1059         pthread_setspecific(async_die_counter, &async_die_counter); /* set to any non-NULL valid pointer */
1060         return ret != NULL;
1061 }
1062
1063 int in_async(void)
1064 {
1065         if (!main_thread_set)
1066                 return 0; /* no asyncs started yet */
1067         return !pthread_equal(main_thread, pthread_self());
1068 }
1069
1070 static void NORETURN async_exit(int code)
1071 {
1072         pthread_exit((void *)(intptr_t)code);
1073 }
1074
1075 #else
1076
1077 static struct {
1078         void (**handlers)(void);
1079         size_t nr;
1080         size_t alloc;
1081 } git_atexit_hdlrs;
1082
1083 static int git_atexit_installed;
1084
1085 static void git_atexit_dispatch(void)
1086 {
1087         size_t i;
1088
1089         for (i=git_atexit_hdlrs.nr ; i ; i--)
1090                 git_atexit_hdlrs.handlers[i-1]();
1091 }
1092
1093 static void git_atexit_clear(void)
1094 {
1095         free(git_atexit_hdlrs.handlers);
1096         memset(&git_atexit_hdlrs, 0, sizeof(git_atexit_hdlrs));
1097         git_atexit_installed = 0;
1098 }
1099
1100 #undef atexit
1101 int git_atexit(void (*handler)(void))
1102 {
1103         ALLOC_GROW(git_atexit_hdlrs.handlers, git_atexit_hdlrs.nr + 1, git_atexit_hdlrs.alloc);
1104         git_atexit_hdlrs.handlers[git_atexit_hdlrs.nr++] = handler;
1105         if (!git_atexit_installed) {
1106                 if (atexit(&git_atexit_dispatch))
1107                         return -1;
1108                 git_atexit_installed = 1;
1109         }
1110         return 0;
1111 }
1112 #define atexit git_atexit
1113
1114 static int process_is_async;
1115 int in_async(void)
1116 {
1117         return process_is_async;
1118 }
1119
1120 static void NORETURN async_exit(int code)
1121 {
1122         exit(code);
1123 }
1124
1125 #endif
1126
1127 void check_pipe(int err)
1128 {
1129         if (err == EPIPE) {
1130                 if (in_async())
1131                         async_exit(141);
1132
1133                 signal(SIGPIPE, SIG_DFL);
1134                 raise(SIGPIPE);
1135                 /* Should never happen, but just in case... */
1136                 exit(141);
1137         }
1138 }
1139
1140 int start_async(struct async *async)
1141 {
1142         int need_in, need_out;
1143         int fdin[2], fdout[2];
1144         int proc_in, proc_out;
1145
1146         need_in = async->in < 0;
1147         if (need_in) {
1148                 if (pipe(fdin) < 0) {
1149                         if (async->out > 0)
1150                                 close(async->out);
1151                         return error_errno("cannot create pipe");
1152                 }
1153                 async->in = fdin[1];
1154         }
1155
1156         need_out = async->out < 0;
1157         if (need_out) {
1158                 if (pipe(fdout) < 0) {
1159                         if (need_in)
1160                                 close_pair(fdin);
1161                         else if (async->in)
1162                                 close(async->in);
1163                         return error_errno("cannot create pipe");
1164                 }
1165                 async->out = fdout[0];
1166         }
1167
1168         if (need_in)
1169                 proc_in = fdin[0];
1170         else if (async->in)
1171                 proc_in = async->in;
1172         else
1173                 proc_in = -1;
1174
1175         if (need_out)
1176                 proc_out = fdout[1];
1177         else if (async->out)
1178                 proc_out = async->out;
1179         else
1180                 proc_out = -1;
1181
1182 #ifdef NO_PTHREADS
1183         /* Flush stdio before fork() to avoid cloning buffers */
1184         fflush(NULL);
1185
1186         async->pid = fork();
1187         if (async->pid < 0) {
1188                 error_errno("fork (async) failed");
1189                 goto error;
1190         }
1191         if (!async->pid) {
1192                 if (need_in)
1193                         close(fdin[1]);
1194                 if (need_out)
1195                         close(fdout[0]);
1196                 git_atexit_clear();
1197                 process_is_async = 1;
1198                 exit(!!async->proc(proc_in, proc_out, async->data));
1199         }
1200
1201         mark_child_for_cleanup(async->pid, NULL);
1202
1203         if (need_in)
1204                 close(fdin[0]);
1205         else if (async->in)
1206                 close(async->in);
1207
1208         if (need_out)
1209                 close(fdout[1]);
1210         else if (async->out)
1211                 close(async->out);
1212 #else
1213         if (!main_thread_set) {
1214                 /*
1215                  * We assume that the first time that start_async is called
1216                  * it is from the main thread.
1217                  */
1218                 main_thread_set = 1;
1219                 main_thread = pthread_self();
1220                 pthread_key_create(&async_key, NULL);
1221                 pthread_key_create(&async_die_counter, NULL);
1222                 set_die_routine(die_async);
1223                 set_die_is_recursing_routine(async_die_is_recursing);
1224         }
1225
1226         if (proc_in >= 0)
1227                 set_cloexec(proc_in);
1228         if (proc_out >= 0)
1229                 set_cloexec(proc_out);
1230         async->proc_in = proc_in;
1231         async->proc_out = proc_out;
1232         {
1233                 int err = pthread_create(&async->tid, NULL, run_thread, async);
1234                 if (err) {
1235                         error(_("cannot create async thread: %s"), strerror(err));
1236                         goto error;
1237                 }
1238         }
1239 #endif
1240         return 0;
1241
1242 error:
1243         if (need_in)
1244                 close_pair(fdin);
1245         else if (async->in)
1246                 close(async->in);
1247
1248         if (need_out)
1249                 close_pair(fdout);
1250         else if (async->out)
1251                 close(async->out);
1252         return -1;
1253 }
1254
1255 int finish_async(struct async *async)
1256 {
1257 #ifdef NO_PTHREADS
1258         int ret = wait_or_whine(async->pid, "child process", 0);
1259
1260         invalidate_lstat_cache();
1261
1262         return ret;
1263 #else
1264         void *ret = (void *)(intptr_t)(-1);
1265
1266         if (pthread_join(async->tid, &ret))
1267                 error("pthread_join failed");
1268         invalidate_lstat_cache();
1269         return (int)(intptr_t)ret;
1270
1271 #endif
1272 }
1273
1274 int async_with_fork(void)
1275 {
1276 #ifdef NO_PTHREADS
1277         return 1;
1278 #else
1279         return 0;
1280 #endif
1281 }
1282
1283 struct io_pump {
1284         /* initialized by caller */
1285         int fd;
1286         int type; /* POLLOUT or POLLIN */
1287         union {
1288                 struct {
1289                         const char *buf;
1290                         size_t len;
1291                 } out;
1292                 struct {
1293                         struct strbuf *buf;
1294                         size_t hint;
1295                 } in;
1296         } u;
1297
1298         /* returned by pump_io */
1299         int error; /* 0 for success, otherwise errno */
1300
1301         /* internal use */
1302         struct pollfd *pfd;
1303 };
1304
1305 static int pump_io_round(struct io_pump *slots, int nr, struct pollfd *pfd)
1306 {
1307         int pollsize = 0;
1308         int i;
1309
1310         for (i = 0; i < nr; i++) {
1311                 struct io_pump *io = &slots[i];
1312                 if (io->fd < 0)
1313                         continue;
1314                 pfd[pollsize].fd = io->fd;
1315                 pfd[pollsize].events = io->type;
1316                 io->pfd = &pfd[pollsize++];
1317         }
1318
1319         if (!pollsize)
1320                 return 0;
1321
1322         if (poll(pfd, pollsize, -1) < 0) {
1323                 if (errno == EINTR)
1324                         return 1;
1325                 die_errno("poll failed");
1326         }
1327
1328         for (i = 0; i < nr; i++) {
1329                 struct io_pump *io = &slots[i];
1330
1331                 if (io->fd < 0)
1332                         continue;
1333
1334                 if (!(io->pfd->revents & (POLLOUT|POLLIN|POLLHUP|POLLERR|POLLNVAL)))
1335                         continue;
1336
1337                 if (io->type == POLLOUT) {
1338                         ssize_t len;
1339
1340                         /*
1341                          * Don't use xwrite() here. It loops forever on EAGAIN,
1342                          * and we're in our own poll() loop here.
1343                          *
1344                          * Note that we lose xwrite()'s handling of MAX_IO_SIZE
1345                          * and EINTR, so we have to implement those ourselves.
1346                          */
1347                         len = write(io->fd, io->u.out.buf,
1348                                     io->u.out.len <= MAX_IO_SIZE ?
1349                                     io->u.out.len : MAX_IO_SIZE);
1350                         if (len < 0) {
1351                                 if (errno != EINTR && errno != EAGAIN &&
1352                                     errno != ENOSPC) {
1353                                         io->error = errno;
1354                                         close(io->fd);
1355                                         io->fd = -1;
1356                                 }
1357                         } else {
1358                                 io->u.out.buf += len;
1359                                 io->u.out.len -= len;
1360                                 if (!io->u.out.len) {
1361                                         close(io->fd);
1362                                         io->fd = -1;
1363                                 }
1364                         }
1365                 }
1366
1367                 if (io->type == POLLIN) {
1368                         ssize_t len = strbuf_read_once(io->u.in.buf,
1369                                                        io->fd, io->u.in.hint);
1370                         if (len < 0)
1371                                 io->error = errno;
1372                         if (len <= 0) {
1373                                 close(io->fd);
1374                                 io->fd = -1;
1375                         }
1376                 }
1377         }
1378
1379         return 1;
1380 }
1381
1382 static int pump_io(struct io_pump *slots, int nr)
1383 {
1384         struct pollfd *pfd;
1385         int i;
1386
1387         for (i = 0; i < nr; i++)
1388                 slots[i].error = 0;
1389
1390         ALLOC_ARRAY(pfd, nr);
1391         while (pump_io_round(slots, nr, pfd))
1392                 ; /* nothing */
1393         free(pfd);
1394
1395         /* There may be multiple errno values, so just pick the first. */
1396         for (i = 0; i < nr; i++) {
1397                 if (slots[i].error) {
1398                         errno = slots[i].error;
1399                         return -1;
1400                 }
1401         }
1402         return 0;
1403 }
1404
1405
1406 int pipe_command(struct child_process *cmd,
1407                  const char *in, size_t in_len,
1408                  struct strbuf *out, size_t out_hint,
1409                  struct strbuf *err, size_t err_hint)
1410 {
1411         struct io_pump io[3];
1412         int nr = 0;
1413
1414         if (in)
1415                 cmd->in = -1;
1416         if (out)
1417                 cmd->out = -1;
1418         if (err)
1419                 cmd->err = -1;
1420
1421         if (start_command(cmd) < 0)
1422                 return -1;
1423
1424         if (in) {
1425                 if (enable_pipe_nonblock(cmd->in) < 0) {
1426                         error_errno("unable to make pipe non-blocking");
1427                         close(cmd->in);
1428                         if (out)
1429                                 close(cmd->out);
1430                         if (err)
1431                                 close(cmd->err);
1432                         return -1;
1433                 }
1434                 io[nr].fd = cmd->in;
1435                 io[nr].type = POLLOUT;
1436                 io[nr].u.out.buf = in;
1437                 io[nr].u.out.len = in_len;
1438                 nr++;
1439         }
1440         if (out) {
1441                 io[nr].fd = cmd->out;
1442                 io[nr].type = POLLIN;
1443                 io[nr].u.in.buf = out;
1444                 io[nr].u.in.hint = out_hint;
1445                 nr++;
1446         }
1447         if (err) {
1448                 io[nr].fd = cmd->err;
1449                 io[nr].type = POLLIN;
1450                 io[nr].u.in.buf = err;
1451                 io[nr].u.in.hint = err_hint;
1452                 nr++;
1453         }
1454
1455         if (pump_io(io, nr) < 0) {
1456                 finish_command(cmd); /* throw away exit code */
1457                 return -1;
1458         }
1459
1460         return finish_command(cmd);
1461 }
1462
1463 enum child_state {
1464         GIT_CP_FREE,
1465         GIT_CP_WORKING,
1466         GIT_CP_WAIT_CLEANUP,
1467 };
1468
1469 struct parallel_processes {
1470         size_t nr_processes;
1471
1472         struct {
1473                 enum child_state state;
1474                 struct child_process process;
1475                 struct strbuf err;
1476                 void *data;
1477         } *children;
1478         /*
1479          * The struct pollfd is logically part of *children,
1480          * but the system call expects it as its own array.
1481          */
1482         struct pollfd *pfd;
1483
1484         unsigned shutdown : 1;
1485
1486         size_t output_owner;
1487         struct strbuf buffered_output; /* of finished children */
1488 };
1489
1490 struct parallel_processes_for_signal {
1491         const struct run_process_parallel_opts *opts;
1492         const struct parallel_processes *pp;
1493 };
1494
1495 static void kill_children(const struct parallel_processes *pp,
1496                           const struct run_process_parallel_opts *opts,
1497                           int signo)
1498 {
1499         for (size_t i = 0; i < opts->processes; i++)
1500                 if (pp->children[i].state == GIT_CP_WORKING)
1501                         kill(pp->children[i].process.pid, signo);
1502 }
1503
1504 static void kill_children_signal(const struct parallel_processes_for_signal *pp_sig,
1505                                  int signo)
1506 {
1507         kill_children(pp_sig->pp, pp_sig->opts, signo);
1508 }
1509
1510 static struct parallel_processes_for_signal *pp_for_signal;
1511
1512 static void handle_children_on_signal(int signo)
1513 {
1514         kill_children_signal(pp_for_signal, signo);
1515         sigchain_pop(signo);
1516         raise(signo);
1517 }
1518
1519 static void pp_init(struct parallel_processes *pp,
1520                     const struct run_process_parallel_opts *opts,
1521                     struct parallel_processes_for_signal *pp_sig)
1522 {
1523         const size_t n = opts->processes;
1524
1525         if (!n)
1526                 BUG("you must provide a non-zero number of processes!");
1527
1528         trace_printf("run_processes_parallel: preparing to run up to %"PRIuMAX" tasks",
1529                      (uintmax_t)n);
1530
1531         if (!opts->get_next_task)
1532                 BUG("you need to specify a get_next_task function");
1533
1534         CALLOC_ARRAY(pp->children, n);
1535         if (!opts->ungroup)
1536                 CALLOC_ARRAY(pp->pfd, n);
1537
1538         for (size_t i = 0; i < n; i++) {
1539                 strbuf_init(&pp->children[i].err, 0);
1540                 child_process_init(&pp->children[i].process);
1541                 if (pp->pfd) {
1542                         pp->pfd[i].events = POLLIN | POLLHUP;
1543                         pp->pfd[i].fd = -1;
1544                 }
1545         }
1546
1547         pp_sig->pp = pp;
1548         pp_sig->opts = opts;
1549         pp_for_signal = pp_sig;
1550         sigchain_push_common(handle_children_on_signal);
1551 }
1552
1553 static void pp_cleanup(struct parallel_processes *pp,
1554                        const struct run_process_parallel_opts *opts)
1555 {
1556         trace_printf("run_processes_parallel: done");
1557         for (size_t i = 0; i < opts->processes; i++) {
1558                 strbuf_release(&pp->children[i].err);
1559                 child_process_clear(&pp->children[i].process);
1560         }
1561
1562         free(pp->children);
1563         free(pp->pfd);
1564
1565         /*
1566          * When get_next_task added messages to the buffer in its last
1567          * iteration, the buffered output is non empty.
1568          */
1569         strbuf_write(&pp->buffered_output, stderr);
1570         strbuf_release(&pp->buffered_output);
1571
1572         sigchain_pop_common();
1573 }
1574
1575 /* returns
1576  *  0 if a new task was started.
1577  *  1 if no new jobs was started (get_next_task ran out of work, non critical
1578  *    problem with starting a new command)
1579  * <0 no new job was started, user wishes to shutdown early. Use negative code
1580  *    to signal the children.
1581  */
1582 static int pp_start_one(struct parallel_processes *pp,
1583                         const struct run_process_parallel_opts *opts)
1584 {
1585         size_t i;
1586         int code;
1587
1588         for (i = 0; i < opts->processes; i++)
1589                 if (pp->children[i].state == GIT_CP_FREE)
1590                         break;
1591         if (i == opts->processes)
1592                 BUG("bookkeeping is hard");
1593
1594         /*
1595          * By default, do not inherit stdin from the parent process - otherwise,
1596          * all children would share stdin! Users may overwrite this to provide
1597          * something to the child's stdin by having their 'get_next_task'
1598          * callback assign 0 to .no_stdin and an appropriate integer to .in.
1599          */
1600         pp->children[i].process.no_stdin = 1;
1601
1602         code = opts->get_next_task(&pp->children[i].process,
1603                                    opts->ungroup ? NULL : &pp->children[i].err,
1604                                    opts->data,
1605                                    &pp->children[i].data);
1606         if (!code) {
1607                 if (!opts->ungroup) {
1608                         strbuf_addbuf(&pp->buffered_output, &pp->children[i].err);
1609                         strbuf_reset(&pp->children[i].err);
1610                 }
1611                 return 1;
1612         }
1613         if (!opts->ungroup) {
1614                 pp->children[i].process.err = -1;
1615                 pp->children[i].process.stdout_to_stderr = 1;
1616         }
1617
1618         if (start_command(&pp->children[i].process)) {
1619                 if (opts->start_failure)
1620                         code = opts->start_failure(opts->ungroup ? NULL :
1621                                                    &pp->children[i].err,
1622                                                    opts->data,
1623                                                    pp->children[i].data);
1624                 else
1625                         code = 0;
1626
1627                 if (!opts->ungroup) {
1628                         strbuf_addbuf(&pp->buffered_output, &pp->children[i].err);
1629                         strbuf_reset(&pp->children[i].err);
1630                 }
1631                 if (code)
1632                         pp->shutdown = 1;
1633                 return code;
1634         }
1635
1636         pp->nr_processes++;
1637         pp->children[i].state = GIT_CP_WORKING;
1638         if (pp->pfd)
1639                 pp->pfd[i].fd = pp->children[i].process.err;
1640         return 0;
1641 }
1642
1643 static void pp_buffer_stderr(struct parallel_processes *pp,
1644                              const struct run_process_parallel_opts *opts,
1645                              int output_timeout)
1646 {
1647         while (poll(pp->pfd, opts->processes, output_timeout) < 0) {
1648                 if (errno == EINTR)
1649                         continue;
1650                 pp_cleanup(pp, opts);
1651                 die_errno("poll");
1652         }
1653
1654         /* Buffer output from all pipes. */
1655         for (size_t i = 0; i < opts->processes; i++) {
1656                 if (pp->children[i].state == GIT_CP_WORKING &&
1657                     pp->pfd[i].revents & (POLLIN | POLLHUP)) {
1658                         int n = strbuf_read_once(&pp->children[i].err,
1659                                                  pp->children[i].process.err, 0);
1660                         if (n == 0) {
1661                                 close(pp->children[i].process.err);
1662                                 pp->children[i].state = GIT_CP_WAIT_CLEANUP;
1663                         } else if (n < 0)
1664                                 if (errno != EAGAIN)
1665                                         die_errno("read");
1666                 }
1667         }
1668 }
1669
1670 static void pp_output(const struct parallel_processes *pp)
1671 {
1672         size_t i = pp->output_owner;
1673
1674         if (pp->children[i].state == GIT_CP_WORKING &&
1675             pp->children[i].err.len) {
1676                 strbuf_write(&pp->children[i].err, stderr);
1677                 strbuf_reset(&pp->children[i].err);
1678         }
1679 }
1680
1681 static int pp_collect_finished(struct parallel_processes *pp,
1682                                const struct run_process_parallel_opts *opts)
1683 {
1684         int code;
1685         size_t i;
1686         int result = 0;
1687
1688         while (pp->nr_processes > 0) {
1689                 for (i = 0; i < opts->processes; i++)
1690                         if (pp->children[i].state == GIT_CP_WAIT_CLEANUP)
1691                                 break;
1692                 if (i == opts->processes)
1693                         break;
1694
1695                 code = finish_command(&pp->children[i].process);
1696
1697                 if (opts->task_finished)
1698                         code = opts->task_finished(code, opts->ungroup ? NULL :
1699                                                    &pp->children[i].err, opts->data,
1700                                                    pp->children[i].data);
1701                 else
1702                         code = 0;
1703
1704                 if (code)
1705                         result = code;
1706                 if (code < 0)
1707                         break;
1708
1709                 pp->nr_processes--;
1710                 pp->children[i].state = GIT_CP_FREE;
1711                 if (pp->pfd)
1712                         pp->pfd[i].fd = -1;
1713                 child_process_init(&pp->children[i].process);
1714
1715                 if (opts->ungroup) {
1716                         ; /* no strbuf_*() work to do here */
1717                 } else if (i != pp->output_owner) {
1718                         strbuf_addbuf(&pp->buffered_output, &pp->children[i].err);
1719                         strbuf_reset(&pp->children[i].err);
1720                 } else {
1721                         const size_t n = opts->processes;
1722
1723                         strbuf_write(&pp->children[i].err, stderr);
1724                         strbuf_reset(&pp->children[i].err);
1725
1726                         /* Output all other finished child processes */
1727                         strbuf_write(&pp->buffered_output, stderr);
1728                         strbuf_reset(&pp->buffered_output);
1729
1730                         /*
1731                          * Pick next process to output live.
1732                          * NEEDSWORK:
1733                          * For now we pick it randomly by doing a round
1734                          * robin. Later we may want to pick the one with
1735                          * the most output or the longest or shortest
1736                          * running process time.
1737                          */
1738                         for (i = 0; i < n; i++)
1739                                 if (pp->children[(pp->output_owner + i) % n].state == GIT_CP_WORKING)
1740                                         break;
1741                         pp->output_owner = (pp->output_owner + i) % n;
1742                 }
1743         }
1744         return result;
1745 }
1746
1747 void run_processes_parallel(const struct run_process_parallel_opts *opts)
1748 {
1749         int i, code;
1750         int output_timeout = 100;
1751         int spawn_cap = 4;
1752         struct parallel_processes_for_signal pp_sig;
1753         struct parallel_processes pp = {
1754                 .buffered_output = STRBUF_INIT,
1755         };
1756         /* options */
1757         const char *tr2_category = opts->tr2_category;
1758         const char *tr2_label = opts->tr2_label;
1759         const int do_trace2 = tr2_category && tr2_label;
1760
1761         if (do_trace2)
1762                 trace2_region_enter_printf(tr2_category, tr2_label, NULL,
1763                                            "max:%d", opts->processes);
1764
1765         pp_init(&pp, opts, &pp_sig);
1766         while (1) {
1767                 for (i = 0;
1768                     i < spawn_cap && !pp.shutdown &&
1769                     pp.nr_processes < opts->processes;
1770                     i++) {
1771                         code = pp_start_one(&pp, opts);
1772                         if (!code)
1773                                 continue;
1774                         if (code < 0) {
1775                                 pp.shutdown = 1;
1776                                 kill_children(&pp, opts, -code);
1777                         }
1778                         break;
1779                 }
1780                 if (!pp.nr_processes)
1781                         break;
1782                 if (opts->ungroup) {
1783                         for (size_t i = 0; i < opts->processes; i++)
1784                                 pp.children[i].state = GIT_CP_WAIT_CLEANUP;
1785                 } else {
1786                         pp_buffer_stderr(&pp, opts, output_timeout);
1787                         pp_output(&pp);
1788                 }
1789                 code = pp_collect_finished(&pp, opts);
1790                 if (code) {
1791                         pp.shutdown = 1;
1792                         if (code < 0)
1793                                 kill_children(&pp, opts,-code);
1794                 }
1795         }
1796
1797         pp_cleanup(&pp, opts);
1798
1799         if (do_trace2)
1800                 trace2_region_leave(tr2_category, tr2_label, NULL);
1801 }
1802
1803 int run_auto_maintenance(int quiet)
1804 {
1805         int enabled;
1806         struct child_process maint = CHILD_PROCESS_INIT;
1807
1808         if (!git_config_get_bool("maintenance.auto", &enabled) &&
1809             !enabled)
1810                 return 0;
1811
1812         maint.git_cmd = 1;
1813         maint.close_object_store = 1;
1814         strvec_pushl(&maint.args, "maintenance", "run", "--auto", NULL);
1815         strvec_push(&maint.args, quiet ? "--quiet" : "--no-quiet");
1816
1817         return run_command(&maint);
1818 }
1819
1820 void prepare_other_repo_env(struct strvec *env, const char *new_git_dir)
1821 {
1822         const char * const *var;
1823
1824         for (var = local_repo_env; *var; var++) {
1825                 if (strcmp(*var, CONFIG_DATA_ENVIRONMENT) &&
1826                     strcmp(*var, CONFIG_COUNT_ENVIRONMENT))
1827                         strvec_push(env, *var);
1828         }
1829         strvec_pushf(env, "%s=%s", GIT_DIR_ENVIRONMENT, new_git_dir);
1830 }
1831
1832 enum start_bg_result start_bg_command(struct child_process *cmd,
1833                                       start_bg_wait_cb *wait_cb,
1834                                       void *cb_data,
1835                                       unsigned int timeout_sec)
1836 {
1837         enum start_bg_result sbgr = SBGR_ERROR;
1838         int ret;
1839         int wait_status;
1840         pid_t pid_seen;
1841         time_t time_limit;
1842
1843         /*
1844          * We do not allow clean-on-exit because the child process
1845          * should persist in the background and possibly/probably
1846          * after this process exits.  So we don't want to kill the
1847          * child during our atexit routine.
1848          */
1849         if (cmd->clean_on_exit)
1850                 BUG("start_bg_command() does not allow non-zero clean_on_exit");
1851
1852         if (!cmd->trace2_child_class)
1853                 cmd->trace2_child_class = "background";
1854
1855         ret = start_command(cmd);
1856         if (ret) {
1857                 /*
1858                  * We assume that if `start_command()` fails, we
1859                  * either get a complete `trace2_child_start() /
1860                  * trace2_child_exit()` pair or it fails before the
1861                  * `trace2_child_start()` is emitted, so we do not
1862                  * need to worry about it here.
1863                  *
1864                  * We also assume that `start_command()` does not add
1865                  * us to the cleanup list.  And that it calls
1866                  * `child_process_clear()`.
1867                  */
1868                 sbgr = SBGR_ERROR;
1869                 goto done;
1870         }
1871
1872         time(&time_limit);
1873         time_limit += timeout_sec;
1874
1875 wait:
1876         pid_seen = waitpid(cmd->pid, &wait_status, WNOHANG);
1877
1878         if (!pid_seen) {
1879                 /*
1880                  * The child is currently running.  Ask the callback
1881                  * if the child is ready to do work or whether we
1882                  * should keep waiting for it to boot up.
1883                  */
1884                 ret = (*wait_cb)(cmd, cb_data);
1885                 if (!ret) {
1886                         /*
1887                          * The child is running and "ready".
1888                          */
1889                         trace2_child_ready(cmd, "ready");
1890                         sbgr = SBGR_READY;
1891                         goto done;
1892                 } else if (ret > 0) {
1893                         /*
1894                          * The callback said to give it more time to boot up
1895                          * (subject to our timeout limit).
1896                          */
1897                         time_t now;
1898
1899                         time(&now);
1900                         if (now < time_limit)
1901                                 goto wait;
1902
1903                         /*
1904                          * Our timeout has expired.  We don't try to
1905                          * kill the child, but rather let it continue
1906                          * (hopefully) trying to startup.
1907                          */
1908                         trace2_child_ready(cmd, "timeout");
1909                         sbgr = SBGR_TIMEOUT;
1910                         goto done;
1911                 } else {
1912                         /*
1913                          * The cb gave up on this child.  It is still running,
1914                          * but our cb got an error trying to probe it.
1915                          */
1916                         trace2_child_ready(cmd, "error");
1917                         sbgr = SBGR_CB_ERROR;
1918                         goto done;
1919                 }
1920         }
1921
1922         else if (pid_seen == cmd->pid) {
1923                 int child_code = -1;
1924
1925                 /*
1926                  * The child started, but exited or was terminated
1927                  * before becoming "ready".
1928                  *
1929                  * We try to match the behavior of `wait_or_whine()`
1930                  * WRT the handling of WIFSIGNALED() and WIFEXITED()
1931                  * and convert the child's status to a return code for
1932                  * tracing purposes and emit the `trace2_child_exit()`
1933                  * event.
1934                  *
1935                  * We do not want the wait_or_whine() error message
1936                  * because we will be called by client-side library
1937                  * routines.
1938                  */
1939                 if (WIFEXITED(wait_status))
1940                         child_code = WEXITSTATUS(wait_status);
1941                 else if (WIFSIGNALED(wait_status))
1942                         child_code = WTERMSIG(wait_status) + 128;
1943                 trace2_child_exit(cmd, child_code);
1944
1945                 sbgr = SBGR_DIED;
1946                 goto done;
1947         }
1948
1949         else if (pid_seen < 0 && errno == EINTR)
1950                 goto wait;
1951
1952         trace2_child_exit(cmd, -1);
1953         sbgr = SBGR_ERROR;
1954
1955 done:
1956         child_process_clear(cmd);
1957         invalidate_lstat_cache();
1958         return sbgr;
1959 }