t/helper/test-run-command.c

   1 /*
   2  * test-run-command.c: test run command API.
   3  *
   4  * (C) 2009 Ilari Liusvaara <ilari.liusvaara@elisanet.fi>
   5  *
   6  * This code is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  */
  10
  11 #include "test-tool.h"
  12 #include "run-command.h"
  13 #include "strvec.h"
  14 #include "strbuf.h"
  15 #include "parse-options.h"
  16 #include "string-list.h"
  17 #include "thread-utils.h"
  18 #include "wildmatch.h"
  19
  20 static int number_callbacks;
  21 static int parallel_next(struct child_process *cp,
  22                          struct strbuf *err,
  23                          void *cb,
  24                          void **task_cb UNUSED)
  25 {
  26         struct child_process *d = cb;
  27         if (number_callbacks >= 4)
  28                 return 0;
  29
  30         strvec_pushv(&cp->args, d->args.v);
  31         if (err)
  32                 strbuf_addstr(err, "preloaded output of a child\n");
  33         else
  34                 fprintf(stderr, "preloaded output of a child\n");
  35
  36         number_callbacks++;
  37         return 1;
  38 }
  39
  40 static int no_job(struct child_process *cp UNUSED,
  41                   struct strbuf *err,
  42                   void *cb UNUSED,
  43                   void **task_cb UNUSED)
  44 {
  45         if (err)
  46                 strbuf_addstr(err, "no further jobs available\n");
  47         else
  48                 fprintf(stderr, "no further jobs available\n");
  49         return 0;
  50 }
  51
  52 static int task_finished(int result UNUSED,
  53                          struct strbuf *err,
  54                          void *pp_cb UNUSED,
  55                          void *pp_task_cb UNUSED)
  56 {
  57         if (err)
  58                 strbuf_addstr(err, "asking for a quick stop\n");
  59         else
  60                 fprintf(stderr, "asking for a quick stop\n");
  61         return 1;
  62 }
  63
  64 struct testsuite {
  65         struct string_list tests, failed;
  66         int next;
  67         int quiet, immediate, verbose, verbose_log, trace, write_junit_xml;
  68 };
  69 #define TESTSUITE_INIT { \
  70         .tests = STRING_LIST_INIT_DUP, \
  71         .failed = STRING_LIST_INIT_DUP, \
  72 }
  73
  74 static int next_test(struct child_process *cp, struct strbuf *err, void *cb,
  75                      void **task_cb)
  76 {
  77         struct testsuite *suite = cb;
  78         const char *test;
  79         if (suite->next >= suite->tests.nr)
  80                 return 0;
  81
  82         test = suite->tests.items[suite->next++].string;
  83         strvec_pushl(&cp->args, "sh", test, NULL);
  84         if (suite->quiet)
  85                 strvec_push(&cp->args, "--quiet");
  86         if (suite->immediate)
  87                 strvec_push(&cp->args, "-i");
  88         if (suite->verbose)
  89                 strvec_push(&cp->args, "-v");
  90         if (suite->verbose_log)
  91                 strvec_push(&cp->args, "-V");
  92         if (suite->trace)
  93                 strvec_push(&cp->args, "-x");
  94         if (suite->write_junit_xml)
  95                 strvec_push(&cp->args, "--write-junit-xml");
  96
  97         strbuf_addf(err, "Output of '%s':\n", test);
  98         *task_cb = (void *)test;
  99
 100         return 1;
 101 }
 102
 103 static int test_finished(int result, struct strbuf *err, void *cb,
 104                          void *task_cb)
 105 {
 106         struct testsuite *suite = cb;
 107         const char *name = (const char *)task_cb;
 108
 109         if (result)
 110                 string_list_append(&suite->failed, name);
 111
 112         strbuf_addf(err, "%s: '%s'\n", result ? "FAIL" : "SUCCESS", name);
 113
 114         return 0;
 115 }
 116
 117 static int test_failed(struct strbuf *out, void *cb, void *task_cb)
 118 {
 119         struct testsuite *suite = cb;
 120         const char *name = (const char *)task_cb;
 121
 122         string_list_append(&suite->failed, name);
 123         strbuf_addf(out, "FAILED TO START: '%s'\n", name);
 124
 125         return 0;
 126 }
 127
 128 static const char * const testsuite_usage[] = {
 129         "test-run-command testsuite [<options>] [<pattern>...]",
 130         NULL
 131 };
 132
 133 static int testsuite(int argc, const char **argv)
 134 {
 135         struct testsuite suite = TESTSUITE_INIT;
 136         int max_jobs = 1, i, ret = 0;
 137         DIR *dir;
 138         struct dirent *d;
 139         struct option options[] = {
 140                 OPT_BOOL('i', "immediate", &suite.immediate,
 141                          "stop at first failed test case(s)"),
 142                 OPT_INTEGER('j', "jobs", &max_jobs, "run <N> jobs in parallel"),
 143                 OPT_BOOL('q', "quiet", &suite.quiet, "be terse"),
 144                 OPT_BOOL('v', "verbose", &suite.verbose, "be verbose"),
 145                 OPT_BOOL('V', "verbose-log", &suite.verbose_log,
 146                          "be verbose, redirected to a file"),
 147                 OPT_BOOL('x', "trace", &suite.trace, "trace shell commands"),
 148                 OPT_BOOL(0, "write-junit-xml", &suite.write_junit_xml,
 149                          "write JUnit-style XML files"),
 150                 OPT_END()
 151         };
 152         struct run_process_parallel_opts opts = {
 153                 .get_next_task = next_test,
 154                 .start_failure = test_failed,
 155                 .task_finished = test_finished,
 156                 .data = &suite,
 157         };
 158
 159         argc = parse_options(argc, argv, NULL, options,
 160                         testsuite_usage, PARSE_OPT_STOP_AT_NON_OPTION);
 161
 162         if (max_jobs <= 0)
 163                 max_jobs = online_cpus();
 164
 165         dir = opendir(".");
 166         if (!dir)
 167                 die("Could not open the current directory");
 168         while ((d = readdir(dir))) {
 169                 const char *p = d->d_name;
 170
 171                 if (*p != 't' || !isdigit(p[1]) || !isdigit(p[2]) ||
 172                     !isdigit(p[3]) || !isdigit(p[4]) || p[5] != '-' ||
 173                     !ends_with(p, ".sh"))
 174                         continue;
 175
 176                 /* No pattern: match all */
 177                 if (!argc) {
 178                         string_list_append(&suite.tests, p);
 179                         continue;
 180                 }
 181
 182                 for (i = 0; i < argc; i++)
 183                         if (!wildmatch(argv[i], p, 0)) {
 184                                 string_list_append(&suite.tests, p);
 185                                 break;
 186                         }
 187         }
 188         closedir(dir);
 189
 190         if (!suite.tests.nr)
 191                 die("No tests match!");
 192         if (max_jobs > suite.tests.nr)
 193                 max_jobs = suite.tests.nr;
 194
 195         fprintf(stderr, "Running %"PRIuMAX" tests (%d at a time)\n",
 196                 (uintmax_t)suite.tests.nr, max_jobs);
 197
 198         opts.processes = max_jobs;
 199         run_processes_parallel(&opts);
 200
 201         if (suite.failed.nr > 0) {
 202                 ret = 1;
 203                 fprintf(stderr, "%"PRIuMAX" tests failed:\n\n",
 204                         (uintmax_t)suite.failed.nr);
 205                 for (i = 0; i < suite.failed.nr; i++)
 206                         fprintf(stderr, "\t%s\n", suite.failed.items[i].string);
 207         }
 208
 209         string_list_clear(&suite.tests, 0);
 210         string_list_clear(&suite.failed, 0);
 211
 212         return ret;
 213 }
 214
 215 static uint64_t my_random_next = 1234;
 216
 217 static uint64_t my_random(void)
 218 {
 219         uint64_t res = my_random_next;
 220         my_random_next = my_random_next * 1103515245 + 12345;
 221         return res;
 222 }
 223
 224 static int quote_stress_test(int argc, const char **argv)
 225 {
 226         /*
 227          * We are running a quote-stress test.
 228          * spawn a subprocess that runs quote-stress with a
 229          * special option that echoes back the arguments that
 230          * were passed in.
 231          */
 232         char special[] = ".?*\\^_\"'`{}()[]<>@~&+:;$%"; // \t\r\n\a";
 233         int i, j, k, trials = 100, skip = 0, msys2 = 0;
 234         struct strbuf out = STRBUF_INIT;
 235         struct strvec args = STRVEC_INIT;
 236         struct option options[] = {
 237                 OPT_INTEGER('n', "trials", &trials, "number of trials"),
 238                 OPT_INTEGER('s', "skip", &skip, "skip <n> trials"),
 239                 OPT_BOOL('m', "msys2", &msys2, "test quoting for MSYS2's sh"),
 240                 OPT_END()
 241         };
 242         const char * const usage[] = {
 243                 "test-tool run-command quote-stress-test <options>",
 244                 NULL
 245         };
 246
 247         argc = parse_options(argc, argv, NULL, options, usage, 0);
 248
 249         setenv("MSYS_NO_PATHCONV", "1", 0);
 250
 251         for (i = 0; i < trials; i++) {
 252                 struct child_process cp = CHILD_PROCESS_INIT;
 253                 size_t arg_count, arg_offset;
 254                 int ret = 0;
 255
 256                 strvec_clear(&args);
 257                 if (msys2)
 258                         strvec_pushl(&args, "sh", "-c",
 259                                      "printf %s\\\\0 \"$@\"", "skip", NULL);
 260                 else
 261                         strvec_pushl(&args, "test-tool", "run-command",
 262                                      "quote-echo", NULL);
 263                 arg_offset = args.nr;
 264
 265                 if (argc > 0) {
 266                         trials = 1;
 267                         arg_count = argc;
 268                         for (j = 0; j < arg_count; j++)
 269                                 strvec_push(&args, argv[j]);
 270                 } else {
 271                         arg_count = 1 + (my_random() % 5);
 272                         for (j = 0; j < arg_count; j++) {
 273                                 char buf[20];
 274                                 size_t min_len = 1;
 275                                 size_t arg_len = min_len +
 276                                         (my_random() % (ARRAY_SIZE(buf) - min_len));
 277
 278                                 for (k = 0; k < arg_len; k++)
 279                                         buf[k] = special[my_random() %
 280                                                 ARRAY_SIZE(special)];
 281                                 buf[arg_len] = '\0';
 282
 283                                 strvec_push(&args, buf);
 284                         }
 285                 }
 286
 287                 if (i < skip)
 288                         continue;
 289
 290                 strvec_pushv(&cp.args, args.v);
 291                 strbuf_reset(&out);
 292                 if (pipe_command(&cp, NULL, 0, &out, 0, NULL, 0) < 0)
 293                         return error("Failed to spawn child process");
 294
 295                 for (j = 0, k = 0; j < arg_count; j++) {
 296                         const char *arg = args.v[j + arg_offset];
 297
 298                         if (strcmp(arg, out.buf + k))
 299                                 ret = error("incorrectly quoted arg: '%s', "
 300                                             "echoed back as '%s'",
 301                                              arg, out.buf + k);
 302                         k += strlen(out.buf + k) + 1;
 303                 }
 304
 305                 if (k != out.len)
 306                         ret = error("got %d bytes, but consumed only %d",
 307                                      (int)out.len, (int)k);
 308
 309                 if (ret) {
 310                         fprintf(stderr, "Trial #%d failed. Arguments:\n", i);
 311                         for (j = 0; j < arg_count; j++)
 312                                 fprintf(stderr, "arg #%d: '%s'\n",
 313                                         (int)j, args.v[j + arg_offset]);
 314
 315                         strbuf_release(&out);
 316                         strvec_clear(&args);
 317
 318                         return ret;
 319                 }
 320
 321                 if (i && (i % 100) == 0)
 322                         fprintf(stderr, "Trials completed: %d\n", (int)i);
 323         }
 324
 325         strbuf_release(&out);
 326         strvec_clear(&args);
 327
 328         return 0;
 329 }
 330
 331 static int quote_echo(int argc, const char **argv)
 332 {
 333         while (argc > 1) {
 334                 fwrite(argv[1], strlen(argv[1]), 1, stdout);
 335                 fputc('\0', stdout);
 336                 argv++;
 337                 argc--;
 338         }
 339
 340         return 0;
 341 }
 342
 343 static int inherit_handle(const char *argv0)
 344 {
 345         struct child_process cp = CHILD_PROCESS_INIT;
 346         char path[PATH_MAX];
 347         int tmp;
 348
 349         /* First, open an inheritable handle */
 350         xsnprintf(path, sizeof(path), "out-XXXXXX");
 351         tmp = xmkstemp(path);
 352
 353         strvec_pushl(&cp.args,
 354                      "test-tool", argv0, "inherited-handle-child", NULL);
 355         cp.in = -1;
 356         cp.no_stdout = cp.no_stderr = 1;
 357         if (start_command(&cp) < 0)
 358                 die("Could not start child process");
 359
 360         /* Then close it, and try to delete it. */
 361         close(tmp);
 362         if (unlink(path))
 363                 die("Could not delete '%s'", path);
 364
 365         if (close(cp.in) < 0 || finish_command(&cp) < 0)
 366                 die("Child did not finish");
 367
 368         return 0;
 369 }
 370
 371 static int inherit_handle_child(void)
 372 {
 373         struct strbuf buf = STRBUF_INIT;
 374
 375         if (strbuf_read(&buf, 0, 0) < 0)
 376                 die("Could not read stdin");
 377         printf("Received %s\n", buf.buf);
 378         strbuf_release(&buf);
 379
 380         return 0;
 381 }
 382
 383 int cmd__run_command(int argc, const char **argv)
 384 {
 385         struct child_process proc = CHILD_PROCESS_INIT;
 386         int jobs;
 387         int ret;
 388         struct run_process_parallel_opts opts = {
 389                 .data = &proc,
 390         };
 391
 392         if (argc > 1 && !strcmp(argv[1], "testsuite"))
 393                 return testsuite(argc - 1, argv + 1);
 394         if (!strcmp(argv[1], "inherited-handle"))
 395                 return inherit_handle(argv[0]);
 396         if (!strcmp(argv[1], "inherited-handle-child"))
 397                 return inherit_handle_child();
 398
 399         if (argc >= 2 && !strcmp(argv[1], "quote-stress-test"))
 400                 return !!quote_stress_test(argc - 1, argv + 1);
 401
 402         if (argc >= 2 && !strcmp(argv[1], "quote-echo"))
 403                 return !!quote_echo(argc - 1, argv + 1);
 404
 405         if (argc < 3)
 406                 return 1;
 407         while (!strcmp(argv[1], "env")) {
 408                 if (!argv[2])
 409                         die("env specifier without a value");
 410                 strvec_push(&proc.env, argv[2]);
 411                 argv += 2;
 412                 argc -= 2;
 413         }
 414         if (argc < 3) {
 415                 ret = 1;
 416                 goto cleanup;
 417         }
 418         strvec_pushv(&proc.args, (const char **)argv + 2);
 419
 420         if (!strcmp(argv[1], "start-command-ENOENT")) {
 421                 if (start_command(&proc) < 0 && errno == ENOENT) {
 422                         ret = 0;
 423                         goto cleanup;
 424                 }
 425                 fprintf(stderr, "FAIL %s\n", argv[1]);
 426                 return 1;
 427         }
 428         if (!strcmp(argv[1], "run-command")) {
 429                 ret = run_command(&proc);
 430                 goto cleanup;
 431         }
 432
 433         if (!strcmp(argv[1], "--ungroup")) {
 434                 argv += 1;
 435                 argc -= 1;
 436                 opts.ungroup = 1;
 437         }
 438
 439         jobs = atoi(argv[2]);
 440         strvec_clear(&proc.args);
 441         strvec_pushv(&proc.args, (const char **)argv + 3);
 442
 443         if (!strcmp(argv[1], "run-command-parallel")) {
 444                 opts.get_next_task = parallel_next;
 445         } else if (!strcmp(argv[1], "run-command-abort")) {
 446                 opts.get_next_task = parallel_next;
 447                 opts.task_finished = task_finished;
 448         } else if (!strcmp(argv[1], "run-command-no-jobs")) {
 449                 opts.get_next_task = no_job;
 450                 opts.task_finished = task_finished;
 451         } else {
 452                 ret = 1;
 453                 fprintf(stderr, "check usage\n");
 454                 goto cleanup;
 455         }
 456         opts.processes = jobs;
 457         run_processes_parallel(&opts);
 458         ret = 0;
 459 cleanup:
 460         child_process_clear(&proc);
 461         return ret;
 462 }