pan/ltp-pan.c

   1 /*
   2  * Copyright (c) 2000 Silicon Graphics, Inc.  All Rights Reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms of version 2 of the GNU General Public License as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it would be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  11  *
  12  * Further, this software is distributed without any warranty that it is
  13  * free of the rightful claim of any third person regarding infringement
  14  * or the like.  Any license provided herein, whether implied or
  15  * otherwise, applies only to this software file.  Patent licenses, if
  16  * any, provided herein do not apply to combinations of this program with
  17  * other software, or any other product whatsoever.
  18  *
  19  * You should have received a copy of the GNU General Public License along
  20  * with this program; if not, write the Free Software Foundation, Inc., 59
  21  * Temple Place - Suite 330, Boston MA 02111-1307, USA.
  22  *
  23  * Contact information: Silicon Graphics, Inc., 1600 Amphitheatre Pkwy,
  24  * Mountain View, CA  94043, or:
  25  *
  26  * http://www.sgi.com
  27  *
  28  * For further information regarding this notice, see:
  29  *
  30  * http://oss.sgi.com/projects/GenInfo/NoticeExplan/
  31  *
  32  * Changelog:
  33  *
  34  *      Added timer options: William Jay Huie, IBM
  35  *      01/27/03 - Added: Manoj Iyer, manjo@mail.utexas.edu
  36  *                         - option '-p' (pretty printing)i to enabled formatted printing
  37  *                           of results.
  38  *
  39  *      01/27/03 - Added: Manoj Iyer, manjo@mail.utexas.edu
  40  *                         - added code to print system information
  41  *
  42  *      01/28/03 - Added: Manoj Iyer, manjo@mail.utexas.edu
  43  *                         - added code to print test exit value.
  44  *
  45  *      01/29/03 - Added: Manoj Iyer, manjo@mail.utexas.edu
  46  *                         - added code supresses test start and test end tags.
  47  *
  48  *      07/22/07 - Added: Ricardo Salveti de Araujo, rsalveti@linux.vnet.ibm.com
  49  *                         - added option to create a command file with all failed tests.
  50  *
  51  */
  52 /* $Id: ltp-pan.c,v 1.4 2009/10/15 18:45:55 yaberauneya Exp $ */
  53
  54 #include <errno.h>
  55 #include <string.h>
  56 #include <sys/param.h>
  57 #include <sys/types.h>
  58 #include <sys/times.h>
  59 #include <sys/wait.h>
  60 #include <sys/stat.h>
  61 #include <time.h>
  62 #include <stdlib.h>
  63 #include <limits.h>
  64 #include <sys/utsname.h>
  65
  66 #include "splitstr.h"
  67 #include "zoolib.h"
  68
  69 /* One entry in the command line collection.  */
  70 struct coll_entry
  71 {
  72     char *name;         /* tag name */
  73     char *cmdline;      /* command line */
  74     char *pcnt_f;       /* location of %f in the command line args, flag */
  75     struct coll_entry *next;
  76 };
  77
  78 struct collection
  79 {
  80     int cnt;
  81     struct coll_entry **ary;
  82 };
  83
  84 struct tag_pgrp
  85 {
  86     int pgrp;
  87     int stopping;
  88     time_t mystime;
  89     struct coll_entry *cmd;
  90     char output[PATH_MAX];
  91 };
  92
  93 struct orphan_pgrp
  94 {
  95     int pgrp;
  96     struct orphan_pgrp *next;
  97 };
  98
  99 static pid_t run_child(struct coll_entry *colle, struct tag_pgrp *active,
 100                                 int quiet_mode);
 101 static char *slurp(char *file);
 102 static struct collection *get_collection(char *file, int optind, int argc,
 103                                          char **argv);
 104 static void pids_running(struct tag_pgrp *running, int keep_active);
 105 static int check_pids(struct tag_pgrp *running, int *num_active,
 106                       int keep_active, FILE * logfile, FILE * failcmdfile,
 107                       struct orphan_pgrp *orphans, int fmt_print,
 108                       int *failcnt, int quiet_mode);
 109 static void propagate_signal(struct tag_pgrp *running, int keep_active,
 110                              struct orphan_pgrp *orphans);
 111 static void dump_coll(struct collection *coll);
 112 static char *subst_pcnt_f(struct coll_entry *colle);
 113 static void mark_orphan(struct orphan_pgrp *orphans, pid_t cpid);
 114 static void orphans_running(struct orphan_pgrp *orphans);
 115 static void check_orphans(struct orphan_pgrp *orphans, int sig);
 116
 117 static void copy_buffered_output(struct tag_pgrp *running);
 118 static void write_test_start(struct tag_pgrp *running);
 119 static void write_test_end(struct tag_pgrp *running, const char *init_status,
 120                            time_t exit_time, char *term_type, int stat_loc,
 121                            int term_id, struct tms *tms1, struct tms *tms2);
 122
 123 //wjh
 124 static char PAN_STOP_FILE[] = "PAN_STOP_FILE";
 125
 126 static char *panname = NULL;
 127 static char *test_out_dir = NULL;       /* dir to buffer output to */
 128 zoo_t zoofile;
 129 static char *reporttype = NULL;
 130
 131 /* zoolib */
 132 int rec_signal;         /* received signal */
 133 int send_signal;        /* signal to send */
 134
 135 /* Debug Bits */
 136 int Debug = 0;
 137 #define Dbuffile        0x000400        /* buffer file use */
 138 #define Dsetup          0x000200        /* one-time set-up */
 139 #define Dshutdown       0x000100        /* killed by signal */
 140 #define Dexit           0x000020        /* exit status */
 141 #define Drunning        0x000010        /* current pids running */
 142 #define Dstartup        0x000004        /* started command */
 143 #define Dstart          0x000002        /* started command */
 144 #define Dwait           0x000001        /* wait interrupted */
 145
 146 int
 147 main(int argc, char **argv)
 148 {
 149     extern char *optarg;
 150     extern int optind;
 151     char *zooname = NULL;       /* name of the zoo file to use */
 152     char *filename = "/dev/null";       /* filename to read test tags from */
 153     char *logfilename = NULL;
 154     char *failcmdfilename = NULL;
 155     char *outputfilename = NULL;
 156     struct collection *coll = NULL;
 157     struct tag_pgrp *running;
 158     struct orphan_pgrp *orphans, *orph;
 159         struct utsname unamebuf;
 160     FILE *logfile = NULL;
 161     FILE *failcmdfile = NULL;
 162     int keep_active = 1;
 163     int num_active = 0;
 164         int failcnt = 0;           /* count of total testcases that failed. */
 165     int err, i;
 166     int starts = -1;
 167     int timed = 0;
 168     int run_time = -1; char modifier = 'm'; int ret = 0;
 169     int stop;
 170     int go_idle;
 171     int has_brakes = 0;         /* stop everything if a test case fails */
 172     int sequential = 0;         /* run tests sequentially */
 173     int fork_in_road = 0;
 174     int exit_stat;
 175     int track_exit_stats = 0;   /* exit non-zero if any test exits non-zero */
 176         int fmt_print = 0;          /* enables formatted printing of logfiles. */
 177         int quiet_mode = 0;                     /* supresses test start and test end tags. */
 178     int c;
 179     pid_t cpid;
 180     struct sigaction sa;
 181
 182     while ((c = getopt(argc, argv, "AO:Sa:C:d:ef:hl:n:o:pqr:s:t:x:y")) != -1) {
 183         switch (c) {
 184         case 'A':       /* all-stop flag */
 185             has_brakes = 1;
 186             track_exit_stats = 1;
 187             break;
 188         case 'O':       /* output buffering directory */
 189             test_out_dir = strdup(optarg);
 190             break;
 191         case 'S':       /* run tests sequentially */
 192             sequential = 1;
 193             break;
 194         case 'a':       /* name of the zoo file to use */
 195             zooname = strdup(optarg);
 196             break;
 197         case 'C':       /* name of the file where all failed commands will be */
 198             failcmdfilename = strdup(optarg);
 199             break;
 200         case 'd':       /* debug options */
 201             sscanf(optarg, "%i", &Debug);
 202             break;
 203         case 'e':       /* exit non-zero if any test exists non-zero */
 204             track_exit_stats = 1;
 205             break;
 206         case 'f':       /* filename to read test tags from */
 207             filename = strdup(optarg);
 208             break;
 209         case 'h':       /* help */
 210             fprintf(stdout, "Usage: pan -n name [ -SyAehpq ] [ -s starts ]"
 211                                  " [-t time[s|m|h|d] [ -x nactive ] [ -l logfile ]\n\t"
 212                                  "[ -a active-file ] [ -f command-file ] "
 213                                  "[ -C fail-command-file ] "
 214                                  "[ -d debug-level ]\n\t[-o output-file] "
 215                                  "[-O output-buffer-directory] [cmd]\n");
 216             exit(0);
 217         case 'l':       /* log file */
 218             logfilename = strdup(optarg);
 219             break;
 220         case 'n':       /* tag given to pan */
 221             panname = strdup(optarg);
 222             break;
 223         case 'o':       /* send test output here */
 224             outputfilename = strdup(optarg);
 225             break;
 226         case 'p':       /* formatted printing. */
 227                 fmt_print = 1;
 228                 break;
 229         case 'q':       /* supress test start and test end messages */
 230                 quiet_mode = 1;
 231                 break;
 232         case 'r':       /* reporting type: none, rts */
 233             reporttype = strdup(optarg);
 234             break;
 235         case 's':       /* number of tags to run */
 236             starts = atoi(optarg);
 237             break;
 238         case 't':       /* run_time to run */
 239             ret = sscanf(optarg, "%d%c", &run_time, &modifier);
 240             if (ret == 0) { fprintf(stderr, "Need proper time input: ####x where"
 241                                     "x is one of s,m,h,d\n"); break; }
 242             else if (ret == 1) { fprintf(stderr, "Only got a time value of %d "
 243                                  "modifiers need to come immediately after #"
 244                                  " assuming %c\n", run_time, modifier); }
 245             else
 246             {
 247                switch (modifier)
 248                {
 249                   case 's': run_time = run_time; break;
 250                   case 'm': run_time = run_time * 60; break;
 251                   case 'h': run_time = run_time * 60 * 60; break;
 252                   case 'd': run_time = run_time * 60 * 60 * 24; break;
 253                   default:
 254                      fprintf(stderr, "Invalid time modifier, try: s|h|m|d\n"); exit(-1);
 255                }
 256                if (!quiet_mode)
 257                   printf("PAN will run for %d seconds\n", run_time);
 258             }
 259             timed = 1; //-t implies run as many starts as possible, by default
 260             break;
 261         case 'x':       /* number of tags to keep running */
 262             keep_active = atoi(optarg);
 263             break;
 264         case 'y':       /* restart on failure or signal */
 265             fork_in_road = 1;
 266             break;
 267         }
 268     }
 269
 270     if (panname == NULL) {
 271         fprintf(stderr, "pan: Must supply -n\n");
 272         exit(1);
 273     }
 274     if (zooname == NULL) {
 275         zooname = zoo_getname();
 276         if (zooname == NULL) {
 277             fprintf(stderr,
 278                     "pan(%s): Must supply -a or set ZOO env variable\n",
 279                     panname);
 280             exit(1);
 281         }
 282     }
 283     if (reporttype) {
 284         /* make sure we understand the report type */
 285         if (strcasecmp(reporttype, "rts")
 286                         && strcasecmp(reporttype, "none")
 287                         /* && strcasecmp(reporttype, "xml")*/)
 288             reporttype = "rts";
 289     } else {
 290         /* set the default */
 291         reporttype = "rts";
 292     }
 293
 294     if (logfilename != NULL) {
 295         time_t startup;
 296         char *s;
 297
 298         if (!strcmp(logfilename, "-")) {
 299             logfile = stdout;
 300         } else {
 301             if ((logfile = fopen(logfilename, "a+")) == NULL) {
 302                 fprintf(stderr,
 303                         "pan(%s): Error %s (%d) opening log file '%s'\n",
 304                         panname, strerror(errno), errno, logfilename);
 305                 exit(1);
 306             }
 307         }
 308
 309         time(&startup);
 310         s = ctime(&startup);
 311         *(s + strlen(s) - 1) = '\0';
 312         if (!fmt_print)
 313                 fprintf(logfile, "startup='%s'\n", s);
 314         else
 315         {
 316                 fprintf(logfile, "Test Start Time: %s\n", s);
 317                 fprintf(logfile, "-----------------------------------------\n");
 318                 fprintf(logfile, "%-30.20s %-10.10s %-10.10s\n",
 319                                 "Testcase", "Result", "Exit Value");
 320                 fprintf(logfile, "%-30.20s %-10.10s %-10.10s\n",
 321                                 "--------", "------", "------------");
 322         }
 323     }
 324
 325     coll = get_collection(filename, optind, argc, argv);
 326     if(!coll)
 327         exit(1);
 328     if (coll->cnt == 0) {
 329         fprintf(stderr,
 330                 "pan(%s): Must supply a file collection or a command\n",
 331                 panname);
 332         exit(1);
 333     }
 334
 335     if (Debug & Dsetup)
 336         dump_coll(coll);
 337
 338     /* a place to store the pgrps we're watching */
 339     running = (struct tag_pgrp *)malloc((keep_active + 1) * sizeof(struct tag_pgrp));
 340     if (running == NULL) {
 341         fprintf(stderr, "pan(%s): Failed to allocate memory: %s\n", panname,
 342                 strerror(errno));
 343         exit(2);
 344     }
 345     memset(running, 0, keep_active * sizeof(struct tag_pgrp));
 346     running[keep_active].pgrp = -1;     /* end sentinel */
 347
 348     /* a head to the orphaned pgrp list */
 349     orphans = (struct orphan_pgrp *) malloc(sizeof(struct orphan_pgrp));
 350     memset(orphans, 0, sizeof(struct orphan_pgrp));
 351
 352     srand48(time(NULL) ^ (getpid() + (getpid() << 15)));
 353
 354     /* Supply a default for starts.  If we are in sequential mode, use
 355      * the number of commands available; otherwise 1.
 356      */
 357     if (timed == 1 && starts == -1) {   /* timed, infinite by default */
 358         starts = -1;
 359     } else if (starts == -1) {
 360         if (sequential) {
 361             starts = coll->cnt;
 362         } else {
 363             starts = 1;
 364         }
 365     } else if (starts == 0) {   /* if the user specified infinite, set it */
 366         starts = -1;
 367     } else {                    /* else, make sure we are starting at least keep_active processes */
 368         if (starts < keep_active)
 369             starts = keep_active;
 370     }
 371
 372     /* if we're buffering output, but we're only running on process at a time,
 373      * then essentially "turn off buffering"
 374      */
 375     if (test_out_dir && (keep_active == 1)) {
 376         free(test_out_dir);
 377         test_out_dir = NULL;
 378     }
 379
 380     if (test_out_dir) {
 381         struct stat sbuf;
 382
 383         if (stat(test_out_dir, &sbuf) < 0) {
 384             fprintf(stderr,
 385                     "pan(%s): stat of -O arg '%s' failed.  errno: %d  %s\n",
 386                     panname, test_out_dir, errno, strerror(errno));
 387             exit(1);
 388         }
 389         if (!S_ISDIR(sbuf.st_mode)) {
 390             fprintf(stderr, "pan(%s): -O arg '%s' must be a directory.\n",
 391                     panname, test_out_dir);
 392             exit(1);
 393         }
 394         if (access(test_out_dir, W_OK | R_OK | X_OK) < 0) {
 395             fprintf(stderr,
 396                     "pan(%s): permission denied on -O arg '%s'.  errno: %d  %s\n",
 397                     panname, test_out_dir, errno, strerror(errno));
 398             exit(1);
 399         }
 400     }
 401
 402     if (outputfilename) {
 403         if (!freopen(outputfilename, "a+", stdout)) {
 404             fprintf(stderr,
 405                     "pan(%s): Error %s (%d) opening output file '%s'\n",
 406                     panname, strerror(errno), errno, outputfilename);
 407             exit(1);
 408         }
 409     }
 410
 411     if (failcmdfilename) {
 412         if (!(failcmdfile = fopen(failcmdfilename, "a+"))) {
 413             fprintf(stderr,
 414                     "pan(%s): Error %s (%d) opening fail cmd file '%s'\n",
 415                     panname, strerror(errno), errno, failcmdfilename);
 416             exit(1);
 417         }
 418     }
 419
 420     if ((zoofile = zoo_open(zooname)) == NULL) {
 421         fprintf(stderr, "pan(%s): %s\n", panname, zoo_error);
 422         exit(1);
 423     }
 424     if (zoo_mark_args(zoofile, getpid(), panname, argc, argv)) {
 425         fprintf(stderr, "pan(%s): %s\n", panname, zoo_error);
 426         exit(1);
 427     }
 428
 429     /* Allocate N spaces for max-arg commands.
 430      * this is an "active file cleanliness" thing
 431      */
 432     {
 433         char *av[2], bigarg[82];
 434
 435         memset(bigarg, '.', 81);
 436         bigarg[81] = '\0';
 437         av[0] = bigarg;
 438         av[1] = NULL;
 439
 440         for (c = 0; c < keep_active; c++) {
 441             if (zoo_mark_cmdline(zoofile, c, panname, "")) {
 442                 fprintf(stderr, "pan(%s): %s\n", panname, zoo_error);
 443                 exit(1);
 444             }
 445         }
 446         for (c = 0; c < keep_active; c++) {
 447             if (zoo_clear(zoofile, c)) {
 448                 fprintf(stderr, "pan(%s): %s\n", panname, zoo_error);
 449                 exit(1);
 450             }
 451         }
 452     }
 453
 454     rec_signal = send_signal = 0;
 455     if (run_time != -1) { alarm(run_time); }
 456
 457     sigemptyset(&sa.sa_mask);
 458     sa.sa_flags = 0;
 459     sa.sa_handler = wait_handler;
 460
 461     sigaction(SIGALRM, &sa, NULL);
 462     sigaction(SIGINT, &sa, NULL);
 463     sigaction(SIGTERM, &sa, NULL);
 464     sigaction(SIGHUP, &sa, NULL);
 465     sigaction(SIGUSR1, &sa, NULL);      /* ignore fork_in_road */
 466     sigaction(SIGUSR2, &sa, NULL);      /* stop the scheduler */
 467
 468     c = 0;                      /* in this loop, c is the command index */
 469     stop = 0;
 470     exit_stat = 0;
 471     go_idle = 0;
 472     while (1) {
 473
 474         while ((num_active < keep_active) && (starts != 0)) {
 475             if (stop || rec_signal || go_idle)
 476                 break;
 477
 478             if (!sequential)
 479                 c = lrand48() % coll->cnt;
 480
 481             /* find a slot for the child */
 482             for (i = 0; i < keep_active; ++i) {
 483                 if (running[i].pgrp == 0)
 484                     break;
 485             }
 486             if (i == keep_active) {
 487                 fprintf(stderr, "pan(%s): Aborting: i == keep_active = %d\n",
 488                         panname, i);
 489                 wait_handler(SIGINT);
 490                 exit_stat++;
 491                 break;
 492             }
 493
 494             cpid = run_child(coll->ary[c], running + i, quiet_mode);
 495             if (cpid != -1)
 496                 ++num_active;
 497             if ((cpid != -1 || sequential) && starts > 0)
 498                 --starts;
 499
 500             if (sequential)
 501                 if (++c >= coll->cnt)
 502                     c = 0;
 503
 504         } /* while( (num_active < keep_active) && (starts != 0) ) */
 505
 506         if (starts == 0)
 507         {
 508                 if (!quiet_mode)
 509                         printf("incrementing stop\n");
 510                 ++stop;
 511         }
 512         else if (starts == -1) //wjh
 513         {
 514            FILE *f = (FILE*)-1;
 515            if ((f = fopen(PAN_STOP_FILE, "r")) != 0)
 516            {  printf("Got %s Stopping!\n", PAN_STOP_FILE);
 517                   fclose(f); unlink(PAN_STOP_FILE); stop++;
 518            }
 519         }
 520
 521         if (rec_signal) {
 522             /* propagate everything except sigusr2 */
 523
 524             if (rec_signal == SIGUSR2) {
 525                 if (fork_in_road)
 526                     ++go_idle;
 527                 else
 528                     ++stop;
 529                 rec_signal = send_signal = 0;
 530             } else {
 531                 if (rec_signal == SIGUSR1)
 532                     fork_in_road = 0;
 533                 propagate_signal(running, keep_active, orphans);
 534                 if (fork_in_road)
 535                     ++go_idle;
 536                 else
 537                     ++stop;
 538             }
 539         }
 540
 541         err = check_pids(running, &num_active, keep_active, logfile,
 542                          failcmdfile, orphans, fmt_print, &failcnt, quiet_mode);
 543         if (Debug & Drunning) {
 544             pids_running(running, keep_active);
 545             orphans_running(orphans);
 546         }
 547         if (err) {
 548             if (fork_in_road)
 549                 ++go_idle;
 550             if (track_exit_stats)
 551                 exit_stat++;
 552             if (has_brakes) {
 553                 fprintf(stderr, "pan(%s): All stop!%s\n", panname,
 554                         go_idle ? " (idling)" : "");
 555                 wait_handler(SIGINT);
 556             }
 557         }
 558
 559         if (stop && (num_active == 0))
 560             break;
 561
 562         if (go_idle && (num_active == 0)) {
 563             go_idle = 0;        /* It is idle, now resume scheduling. */
 564             wait_handler(0);    /* Reset the signal ratchet. */
 565         }
 566     }
 567
 568     /* Wait for orphaned pgrps */
 569     while (1) {
 570         for (orph = orphans; orph != NULL; orph = orph->next) {
 571             if (orph->pgrp == 0)
 572                 continue;
 573             /* Yes, we have orphaned pgrps */
 574             sleep(5);
 575             if (!rec_signal) {
 576                 /* force an artificial signal, move us
 577                  * through the signal ratchet.
 578                  */
 579                 wait_handler(SIGINT);
 580             }
 581             propagate_signal(running, keep_active, orphans);
 582             if (Debug & Drunning)
 583                 orphans_running(orphans);
 584             break;
 585         }
 586         if (orph == NULL)
 587             break;
 588     }
 589
 590     if (zoo_clear(zoofile, getpid())) {
 591         fprintf(stderr, "pan(%s): %s\n", panname, zoo_error);
 592         ++exit_stat;
 593     }
 594     fclose(zoofile);
 595         if (logfile && fmt_print)
 596         {
 597                 if (uname(&unamebuf) == -1)
 598                         fprintf(stderr, "ERROR: uname(): %s\n", strerror(errno));
 599                 fprintf(logfile, "\n-----------------------------------------------\n");
 600                 fprintf(logfile, "Total Tests: %d\n", coll->cnt);
 601                 fprintf(logfile, "Total Failures: %d\n", failcnt);
 602                 fprintf(logfile, "Kernel Version: %s\n", unamebuf.release);
 603                 fprintf(logfile, "Machine Architecture: %s\n", unamebuf.machine);
 604                 fprintf(logfile, "Hostname: %s\n\n", unamebuf.nodename);
 605         }
 606     if (logfile && (logfile != stdout))
 607         fclose(logfile);
 608
 609     exit(exit_stat);
 610 }
 611
 612
 613
 614 static void
 615 propagate_signal(struct tag_pgrp *running, int keep_active,
 616                  struct orphan_pgrp *orphans)
 617 {
 618     int i;
 619
 620     if (Debug & Dshutdown)
 621         fprintf(stderr, "pan was signaled with sig %d...\n", rec_signal);
 622
 623     if (rec_signal == SIGALRM)
 624     {
 625        printf("PAN stop Alarm was received\n");
 626        rec_signal = SIGTERM;
 627     }
 628
 629     for (i = 0; i < keep_active; ++i) {
 630         if (running[i].pgrp == 0)
 631             continue;
 632
 633         if (Debug & Dshutdown)
 634             fprintf(stderr, "  propagating sig %d to %d\n",
 635                     send_signal, -running[i].pgrp);
 636         if (kill(-running[i].pgrp, send_signal) != 0) {
 637             fprintf(stderr,
 638                     "pan(%s): kill(%d,%d) failed on tag (%s).  errno:%d  %s\n",
 639                     panname, -running[i].pgrp, send_signal,
 640                     running[i].cmd->name, errno, strerror(errno));
 641         }
 642         running[i].stopping = 1;
 643     }
 644
 645     check_orphans(orphans, send_signal);
 646
 647     rec_signal = send_signal = 0;
 648 }
 649
 650
 651 static int
 652 check_pids(struct tag_pgrp *running, int *num_active, int keep_active,
 653            FILE * logfile, FILE * failcmdfile, struct orphan_pgrp *orphans,
 654            int fmt_print, int *failcnt, int quiet_mode)
 655 {
 656     int w;
 657     pid_t cpid;
 658     int stat_loc;
 659     int ret = 0;
 660     int i;
 661     time_t t;
 662     char *status;
 663     int signaled = 0;
 664     struct tms tms1, tms2;
 665     clock_t tck;
 666
 667     check_orphans(orphans, 0);
 668
 669     tck = times(&tms1);
 670     if (tck == -1) {
 671         fprintf(stderr, "pan(%s): times(&tms1) failed.  errno:%d  %s\n",
 672                 panname, errno, strerror(errno));
 673     }
 674     cpid = wait(&stat_loc);
 675     tck = times(&tms2);
 676     if (tck == -1) {
 677         fprintf(stderr, "pan(%s): times(&tms2) failed.  errno:%d  %s\n",
 678                 panname, errno, strerror(errno));
 679     }
 680
 681     if (cpid < 0) {
 682         if (errno == EINTR) {
 683             if (Debug)
 684                 fprintf(stderr, "pan(%s): wait() interrupted\n", panname);
 685         } else if (errno != ECHILD) {
 686             fprintf(stderr, "pan(%s): wait() failed.  errno:%d  %s\n",
 687                     panname, errno, strerror(errno));
 688         }
 689     } else if (cpid > 0) {
 690
 691         if (WIFSIGNALED(stat_loc)) {
 692             w = WTERMSIG(stat_loc);
 693             status = "signaled";
 694             if (Debug & Dexit)
 695                 fprintf(stderr, "child %d terminated with signal %d\n", cpid,
 696                         w);
 697             --*num_active;
 698             signaled = 1;
 699         } else if (WIFEXITED(stat_loc)) {
 700             w = WEXITSTATUS(stat_loc);
 701             status = "exited";
 702             if (Debug & Dexit)
 703                 fprintf(stderr, "child %d exited with status %d\n", cpid, w);
 704             --*num_active;
 705             if (w != 0)
 706                 ret++;
 707         } else if (WIFSTOPPED(stat_loc)) {      /* should never happen */
 708             w = WSTOPSIG(stat_loc);
 709             status = "stopped";
 710             ret++;
 711         } else {                /* should never happen */
 712             w = 0;
 713             status = "unknown";
 714             ret++;
 715         }
 716
 717         for (i = 0; i < keep_active; ++i) {
 718             if (running[i].pgrp == cpid) {
 719                 if ((w == 130) && running[i].stopping &&
 720                     (strcmp(status, "exited") == 0)) {
 721                     /* The child received sigint, but
 722                      * did not trap for it?  Compensate
 723                      * for it here.
 724                      */
 725                     w = 0;
 726                     ret--;      /* undo */
 727                     if (Debug & Drunning)
 728                         fprintf(stderr,
 729                                 "pan(%s): tag=%s exited 130, known to be signaled; will give it an exit 0.\n",
 730                                 panname, running[i].cmd->name);
 731                 }
 732                 time(&t);
 733                 if (logfile != NULL) {
 734                         if (!fmt_print)
 735                                 fprintf(logfile,
 736                                  "tag=%s stime=%d dur=%d exit=%s stat=%d core=%s cu=%d cs=%d\n",
 737                                         running[i].cmd->name, (int) (running[i].mystime),
 738                                         (int) (t - running[i].mystime), status, w,
 739                                         (stat_loc & 0200) ? "yes" : "no",
 740                                         (int) (tms2.tms_cutime - tms1.tms_cutime),
 741                                         (int) (tms2.tms_cstime - tms1.tms_cstime));
 742                         else
 743                         {
 744                                         if (w != 0)
 745                                                 ++*failcnt;
 746                                         fprintf(logfile, "%-30.30s %-10.10s %-5d\n",
 747                                                         running[i].cmd->name, ((w != 0) ? "FAIL" : "PASS"),
 748                                                         w);
 749                         }
 750
 751                         fflush(logfile);
 752                 }
 753
 754                 if ((failcmdfile != NULL) && (w !=0)) {
 755                         fprintf(failcmdfile, "%s %s\n", running[i].cmd->name, running[i].cmd->cmdline);
 756                 }
 757
 758                 if (running[i].stopping)
 759                     status = "driver_interrupt";
 760
 761                 if (test_out_dir) {
 762                         if (!quiet_mode)
 763                                 write_test_start(running+i);
 764                         copy_buffered_output(running + i);
 765                         unlink(running[i].output);
 766                 }
 767                 if (!quiet_mode)
 768                         write_test_end(running+i, "ok", t, status,
 769                            stat_loc, w, &tms1, &tms2);
 770
 771                 /* If signaled and we weren't expecting
 772                  * this to be stopped then the proc
 773                  * had a problem.
 774                  */
 775                 if (signaled && !running[i].stopping)
 776                     ret++;
 777
 778                 running[i].pgrp = 0;
 779                 if (zoo_clear(zoofile, cpid)) {
 780                     fprintf(stderr, "pan(%s): %s\n", panname, zoo_error);
 781                     exit(1);
 782                 }
 783
 784                 /* Check for orphaned pgrps */
 785                 if ((kill(-cpid, 0) == 0) || (errno == EPERM)) {
 786                     if (zoo_mark_cmdline(zoofile, cpid, "panorphan",
 787                                           running[i].cmd->cmdline)) {
 788                         fprintf(stderr, "pan(%s): %s\n", panname, zoo_error);
 789                         exit(1);
 790                     }
 791                     mark_orphan(orphans, cpid);
 792                     /* status of kill doesn't matter */
 793                     kill(-cpid, SIGTERM);
 794                 }
 795
 796                 break;
 797             }
 798         }
 799     }
 800     return ret;
 801 }
 802
 803
 804 static pid_t
 805 run_child(struct coll_entry *colle, struct tag_pgrp *active, int quiet_mode)
 806 {
 807     int cpid;
 808     int c_stdout = -1;          /* child's stdout, stderr */
 809     int capturing = 0;          /* output is going to a file instead of stdout */
 810     char *c_cmdline;
 811     static long cmdno = 0;
 812     int errpipe[2];             /* way to communicate to parent that the tag  */
 813     char errbuf[1024];          /* didn't actually start */
 814     int errlen;
 815
 816     /* Try to open the file that will be stdout for the test */
 817     if (test_out_dir) {
 818         capturing = 1;
 819         do {
 820             sprintf(active->output, "%s/%s.%ld",
 821                     test_out_dir, colle->name, cmdno++);
 822             c_stdout = open(active->output, O_CREAT | O_RDWR | O_EXCL | O_SYNC, 0666);
 823         } while (c_stdout < 0 && errno == EEXIST);
 824         if (c_stdout < 0) {
 825             fprintf(stderr,
 826                     "pan(%s): open of stdout file failed (tag %s).  errno: %d  %s\n  file: %s\n",
 827                     panname, colle->name, errno, strerror(errno),
 828                     active->output);
 829             return -1;
 830         }
 831     }
 832
 833     /* get the tag's command line arguments ready.  subst_pcnt_f() uses a
 834      * static counter, that's why we do it here instead of after we fork.
 835      */
 836     if (colle->pcnt_f) {
 837         c_cmdline = subst_pcnt_f(colle);
 838     } else {
 839         c_cmdline = colle->cmdline;
 840     }
 841
 842     if (pipe(errpipe) < 0) {
 843         fprintf(stderr, "pan(%s): pipe() failed. errno:%d %s\n",
 844                         panname, errno, strerror(errno));
 845         if (capturing) {
 846             close(c_stdout);
 847             unlink(active->output);
 848         }
 849         return -1;
 850     }
 851
 852     time(&active->mystime);
 853     active->cmd = colle;
 854
 855     if (!test_out_dir)
 856         if (!quiet_mode)
 857                 write_test_start(active);
 858
 859     if ((cpid = fork()) < 0) {
 860         fprintf(stderr, "pan(%s): fork failed (tag %s).  errno:%d  %s\n",
 861                 panname, colle->name, errno, strerror(errno));
 862         if (capturing) {
 863             unlink(active->output);
 864             close(c_stdout);
 865         }
 866         close(errpipe[0]);
 867         close(errpipe[1]);
 868         return -1;
 869     } else if (cpid == 0) {
 870         /* child */
 871
 872         fclose(zoofile);
 873         close(errpipe[0]);
 874         fcntl(errpipe[1], F_SETFD, 1);  /* close the pipe if we succeed */
 875         setpgrp();
 876
 877         umask(0);
 878
 879         /* if we're putting output into a buffer file, we need to do the
 880          * redirection now.  If we fail
 881          */
 882         if (capturing) {
 883             if (dup2(c_stdout, fileno(stdout)) == -1) {
 884                 errlen = sprintf(errbuf, "pan(%s): couldn't redirect stdout for tag %s.  errno:%d  %s",
 885                                 panname, colle->name, errno, strerror(errno));
 886                 write(errpipe[1], &errlen, sizeof(errlen));
 887                 write(errpipe[1], errbuf, errlen);
 888                 exit(2);
 889             }
 890             if (dup2(c_stdout, fileno(stderr)) == -1) {
 891                 errlen = sprintf(errbuf, "pan(%s): couldn't redirect stderr for tag %s.  errno:%d  %s",
 892                                 panname, colle->name, errno, strerror(errno));
 893                 write(errpipe[1], &errlen, sizeof(errlen));
 894                 write(errpipe[1], errbuf, errlen);
 895                 exit(2);
 896             }
 897         } else { /* stderr still needs to be redirected */
 898             if (dup2(fileno(stdout), fileno(stderr)) == -1) {
 899                 errlen = sprintf(errbuf, "pan(%s): couldn't redirect stderr for tag %s.  errno:%d  %s",
 900                                 panname, colle->name, errno, strerror(errno));
 901                 write(errpipe[1], &errlen, sizeof(errlen));
 902                 write(errpipe[1], errbuf, errlen);
 903                 exit(2);
 904             }
 905         }
 906         /* If there are any shell-type characters in the cmdline
 907          * such as '>', '<', '$', '|', etc, then we exec a shell and
 908          * run the cmd under a shell.
 909          *
 910          * Otherwise, break the cmdline at white space and exec the
 911          * cmd directly.
 912          */
 913         if (strpbrk(c_cmdline, "\"';|<>$\\")) {
 914             execlp("sh", "sh", "-c", c_cmdline, (char*)0);
 915             errlen = sprintf(errbuf,
 916                     "pan(%s): execlp of '%s' (tag %s) failed.  errno:%d %s",
 917                     panname, c_cmdline, colle->name, errno, strerror(errno));
 918         } else {
 919             char **arg_v;
 920
 921             arg_v = (char **)splitstr(c_cmdline, NULL, NULL);
 922
 923             execvp(arg_v[0], arg_v);
 924             errlen = sprintf(errbuf,
 925                     "pan(%s): execvp of '%s' (tag %s) failed.  errno:%d  %s",
 926                     panname, arg_v[0], colle->name, errno, strerror(errno));
 927         }
 928         write(errpipe[1], &errlen, sizeof(errlen));
 929         write(errpipe[1], errbuf, errlen);
 930         exit(errno);
 931     }
 932
 933     /* parent */
 934
 935     /* subst_pcnt_f() allocates the command line dynamically
 936      * free the malloc to prevent a memory leak
 937      */
 938     if (colle->pcnt_f) free(c_cmdline);
 939
 940     close(errpipe[1]);
 941
 942     /* if the child couldn't go through with the exec,
 943      * clean up the mess, note it, and move on
 944      */
 945     if(read(errpipe[0], &errlen, sizeof(errlen))) {
 946         int status;
 947         time_t end_time;
 948         int termid;
 949         char *termtype;
 950         struct tms notime = {0, 0, 0, 0};
 951
 952         read(errpipe[0], errbuf, errlen);
 953         close(errpipe[0]);
 954         errbuf[errlen] = '\0';
 955         /* fprintf(stderr, "%s", errbuf); */
 956         waitpid(cpid, &status, 0);
 957         if (WIFSIGNALED(status)) {
 958             termid = WTERMSIG(status);
 959             termtype = "signaled";
 960         } else if (WIFEXITED(status)) {
 961             termid = WEXITSTATUS(status);
 962             termtype = "exited";
 963         } else if (WIFSTOPPED(status)) {
 964             termid = WSTOPSIG(status);
 965             termtype = "stopped";
 966         } else {
 967             termid = 0;
 968             termtype = "unknown";
 969         }
 970         time(&end_time);
 971         if (!quiet_mode)
 972         {
 973                 //write_test_start(active, errbuf);
 974                 write_test_end(active, errbuf, end_time, termtype, status,
 975                         termid, &notime, &notime);
 976         }
 977         if (capturing) {
 978             close(c_stdout);
 979             unlink(active->output);
 980         }
 981         return -1;
 982     }
 983
 984     close(errpipe[0]);
 985     if (capturing) close(c_stdout);
 986
 987     active->pgrp = cpid;
 988     active->stopping = 0;
 989
 990     if (zoo_mark_cmdline(zoofile, cpid, colle->name, colle->cmdline)) {
 991         fprintf(stderr, "pan(%s): %s\n", panname, zoo_error);
 992         exit(1);
 993     }
 994
 995     if (Debug & Dstartup)
 996         fprintf(stderr, "started %s cpid=%d at %s",
 997                 colle->name, cpid, ctime(&active->mystime));
 998
 999     if (Debug & Dstart) {
1000         fprintf(stderr, "Executing test = %s as %s", colle->name, colle->cmdline);
1001         if (capturing)
1002             fprintf(stderr, "with output file = %s\n", active->output);
1003         else
1004             fprintf(stderr, "\n");
1005     }
1006
1007     return cpid;
1008 }
1009
1010
1011 static char *
1012 subst_pcnt_f(struct coll_entry *colle)
1013 {
1014     static int counter = 1;
1015     char pid_and_counter[20];
1016     char new_cmdline[1024];
1017
1018     /* if we get called falsely, do the right thing anyway */
1019     if (!colle->pcnt_f)
1020         return colle->cmdline;
1021
1022     snprintf(pid_and_counter, 20, "%d_%d", getpid(), counter++);
1023     snprintf(new_cmdline, 1024, colle->cmdline, pid_and_counter);
1024     return strdup(new_cmdline);
1025 }
1026
1027 static struct collection *
1028 get_collection(char *file, int optind, int argc, char **argv)
1029 {
1030     char *buf, *a, *b;
1031     struct coll_entry *head, *p, *n;
1032     struct collection *coll;
1033     int i;
1034
1035     buf = slurp(file);
1036     if(!buf)
1037         return NULL;
1038
1039     coll = (struct collection *) malloc(sizeof(struct collection));
1040     coll->cnt = 0;
1041
1042     head = p = n = NULL;
1043     a = b = buf;
1044     while (a) {
1045         /* set b to the start of the next line and add a NULL character
1046          * to separate the two lines */
1047         if ((b = strchr(a, '\n')) != NULL)
1048             *b++ = '\0';
1049
1050         /* If this is line isn't a comment */
1051         if ((*a != '#') && (*a != '\0') && (*a != ' ')) {
1052             n = (struct coll_entry *) malloc(sizeof(struct coll_entry));
1053             if ((n->pcnt_f = strstr(a, "%f"))) {
1054                 n->pcnt_f[1] = 's';
1055             }
1056             n->name = strdup(strsep(&a, " \t"));
1057             n->cmdline = strdup(a);
1058             n->next = NULL;
1059
1060             if (p) {
1061                 p->next = n;
1062             }
1063             if (head == NULL) {
1064                 head = n;
1065             }
1066             p = n;
1067             coll->cnt++;
1068         }
1069         a = b;
1070     }
1071     free(buf);
1072
1073     /* is there something on the commandline to be counted? */
1074     if (optind < argc) {
1075         char workstr[1024] = "";
1076         int workstr_left = 1023;
1077
1078         /* fill arg list */
1079         for (i = 0; optind < argc; ++optind, ++i) {
1080             strncat(workstr, argv[optind], workstr_left);
1081             workstr_left = workstr_left - strlen(argv[optind]);
1082             strncat(workstr, " ", workstr_left);
1083             workstr_left--;
1084         }
1085
1086         n = (struct coll_entry *) malloc(sizeof(struct coll_entry));
1087         if ((n->pcnt_f = strstr(workstr, "%f"))) {
1088             n->pcnt_f[1] = 's';
1089         }
1090         n->cmdline = strdup(workstr);
1091         n->name = "cmdln";
1092         n->next = NULL;
1093         if (p) {
1094             p->next = n;
1095         }
1096         if (head == NULL) {
1097             head = n;
1098         }
1099         coll->cnt++;
1100     }
1101
1102     /* get an array */
1103     coll->ary = (struct coll_entry **) malloc(coll->cnt *
1104                                               sizeof(struct coll_entry *));
1105
1106     /* fill the array */
1107     i = 0;
1108     n = head;
1109     while (n != NULL) {
1110         coll->ary[i] = n;
1111         n = n->next;
1112         ++i;
1113     }
1114     if (i != coll->cnt)
1115         fprintf(stderr, "pan(%s): i doesn't match cnt\n", panname);
1116
1117     return coll;
1118 }
1119
1120
1121 static char *
1122 slurp(char *file)
1123 {
1124     char *buf;
1125     int fd;
1126     struct stat sbuf;
1127
1128     if ((fd = open(file, O_RDONLY)) < 0) {
1129         fprintf(stderr, "pan(%s): open(%s,O_RDONLY) failed.  errno:%d  %s\n",
1130                 panname, file, errno, strerror(errno));
1131         return NULL;
1132     }
1133
1134     if (fstat(fd, &sbuf) < 0) {
1135         fprintf(stderr, "pan(%s): fstat(%s) failed.  errno:%d  %s\n",
1136                 panname, file, errno, strerror(errno));
1137         return NULL;
1138     }
1139
1140     buf = (char *) malloc(sbuf.st_size + 1);
1141     if (read(fd, buf, sbuf.st_size) != sbuf.st_size) {
1142         fprintf(stderr, "pan(%s): slurp failed.  errno:%d  %s\n",
1143                 panname, errno, strerror(errno));
1144         return NULL;
1145     }
1146     buf[sbuf.st_size] = '\0';
1147
1148     close(fd);
1149     return buf;
1150 }
1151
1152 static void
1153 check_orphans(struct orphan_pgrp *orphans, int sig)
1154 {
1155     struct orphan_pgrp *orph;
1156
1157     for (orph = orphans; orph != NULL; orph = orph->next) {
1158         if (orph->pgrp == 0)
1159             continue;
1160
1161         if (Debug & Dshutdown)
1162             fprintf(stderr, "  propagating sig %d to orphaned pgrp %d\n",
1163                     sig, -(orph->pgrp));
1164         if (kill(-(orph->pgrp), sig) != 0) {
1165             if (errno == ESRCH) {
1166                 /* This pgrp is now empty */
1167                 if (zoo_clear(zoofile, orph->pgrp)) {
1168                     fprintf(stderr, "pan(%s): %s\n", panname, zoo_error);
1169                 }
1170                 orph->pgrp = 0;
1171             } else {
1172                 fprintf(stderr,
1173                         "pan(%s): kill(%d,%d) on orphaned pgrp failed.  errno:%d  %s\n",
1174                         panname, -(orph->pgrp), sig, errno, strerror(errno));
1175             }
1176         }
1177     }
1178 }
1179
1180
1181 static void
1182 mark_orphan(struct orphan_pgrp *orphans, pid_t cpid)
1183 {
1184     struct orphan_pgrp *orph;
1185
1186     for (orph = orphans; orph != NULL; orph = orph->next) {
1187         if (orph->pgrp == 0)
1188             break;
1189     }
1190     if (orph == NULL) {
1191         /* make a new struct */
1192         orph = (struct orphan_pgrp *) malloc(sizeof(struct orphan_pgrp));
1193
1194         /* plug in the new struct just after the head */
1195         orph->next = orphans->next;
1196         orphans->next = orph;
1197     }
1198     orph->pgrp = cpid;
1199 }
1200
1201
1202
1203 static void
1204 copy_buffered_output(struct tag_pgrp *running)
1205 {
1206     char *tag_output;
1207
1208     tag_output = slurp(running->output);
1209     if (tag_output) {
1210         printf("%s", tag_output);
1211         /* make sure the output ends with a newline */
1212         if (tag_output[strlen(tag_output) - 1] != '\n')
1213             printf("\n");
1214         fflush(stdout);
1215         free(tag_output);
1216     }
1217 }
1218
1219
1220 static void
1221 write_test_start(struct tag_pgrp *running)
1222 {
1223     if (!strcmp(reporttype, "rts")) {
1224
1225         printf("%s\ntag=%s stime=%ld\ncmdline=\"%s\"\ncontacts=\"%s\"\nanalysis=%s\n%s\n",
1226                         "<<<test_start>>>",
1227                         running->cmd->name, running->mystime, running->cmd->cmdline, "",
1228                         "exit",
1229                         "<<<test_output>>>");
1230     }
1231     fflush(stdout);
1232 }
1233
1234
1235 static void
1236 write_test_end(struct tag_pgrp *running, const char *init_status,
1237                 time_t exit_time, char *term_type, int stat_loc,
1238                 int term_id, struct tms *tms1, struct tms *tms2)
1239 {
1240     if (!strcmp(reporttype, "rts")) {
1241         printf("%s\ninitiation_status=\"%s\"\nduration=%ld termination_type=%s "
1242                 "termination_id=%d corefile=%s\ncutime=%d cstime=%d\n%s\n",
1243                         "<<<execution_status>>>", init_status,
1244                         (long) (exit_time - running->mystime),
1245                         term_type, term_id, (stat_loc & 0200) ? "yes" : "no",
1246                         (int) (tms2->tms_cutime - tms1->tms_cutime),
1247                         (int) (tms2->tms_cstime - tms1->tms_cstime),
1248                         "<<<test_end>>>");
1249     }
1250     fflush(stdout);
1251 }
1252
1253 /* The functions below are all debugging related */
1254
1255 static void
1256 pids_running(struct tag_pgrp *running, int keep_active)
1257 {
1258     int i;
1259
1260     fprintf(stderr, "pids still running: ");
1261     for (i = 0; i < keep_active; ++i) {
1262         if (running[i].pgrp != 0)
1263             fprintf(stderr, "%d ", running[i].pgrp);
1264     }
1265     fprintf(stderr, "\n");
1266 }
1267
1268 static void
1269 orphans_running(struct orphan_pgrp *orphans)
1270 {
1271     struct orphan_pgrp *orph;
1272
1273     fprintf(stderr, "orphans still running: ");
1274     for (orph = orphans; orph != NULL; orph = orph->next) {
1275         if (orph->pgrp != 0)
1276             fprintf(stderr, "%d ", -(orph->pgrp));
1277     }
1278     fprintf(stderr, "\n");
1279 }
1280
1281 static void
1282 dump_coll(struct collection *coll)
1283 {
1284     int i;
1285
1286     for (i = 0; i < coll->cnt; ++i) {
1287         fprintf(stderr, "coll %d\n", i);
1288         fprintf(stderr, "  name=%s cmdline=%s\n", coll->ary[i]->name,
1289                 coll->ary[i]->cmdline);
1290     }
1291 }
1292
1293 void
1294 wait_handler( int sig )
1295 {
1296     static int lastsent = 0;
1297
1298     if( sig == 0 ){
1299         lastsent = 0;
1300     } else {
1301         rec_signal = sig;
1302         if( sig == SIGUSR2 )
1303             return;
1304         if( lastsent == 0 )
1305             send_signal = sig;
1306         else if( lastsent == SIGUSR1 )
1307             send_signal = SIGINT;
1308         else if( lastsent == sig )
1309             send_signal = SIGTERM;
1310         else if( lastsent == SIGTERM )
1311             send_signal = SIGHUP;
1312         else if( lastsent == SIGHUP )
1313             send_signal = SIGKILL;
1314         lastsent = send_signal;
1315     }
1316 }
1317