src/test/tinytest.c

   1 /* tinytest.c -- Copyright 2009-2012 Nick Mathewson
   2  *
   3  * Redistribution and use in source and binary forms, with or without
   4  * modification, are permitted provided that the following conditions
   5  * are met:
   6  * 1. Redistributions of source code must retain the above copyright
   7  *    notice, this list of conditions and the following disclaimer.
   8  * 2. Redistributions in binary form must reproduce the above copyright
   9  *    notice, this list of conditions and the following disclaimer in the
  10  *    documentation and/or other materials provided with the distribution.
  11  * 3. The name of the author may not be used to endorse or promote products
  12  *    derived from this software without specific prior written permission.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  15  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  16  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  17  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  18  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  19  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  20  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  21  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  23  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25 #ifdef TINYTEST_LOCAL
  26 #include "tinytest_local.h"
  27 #endif
  28
  29 #include <stdio.h>
  30 #include <stdlib.h>
  31 #include <string.h>
  32 #include <assert.h>
  33
  34 #ifdef _WIN32
  35 #include <windows.h>
  36 #else
  37 #include <sys/types.h>
  38 #include <sys/wait.h>
  39 #include <unistd.h>
  40 #endif
  41
  42 #ifndef __GNUC__
  43 #define __attribute__(x)
  44 #endif
  45
  46 #include "tinytest.h"
  47 #include "tinytest_macros.h"
  48
  49 #define LONGEST_TEST_NAME 16384
  50
  51 static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/
  52 static int n_ok = 0; /**< Number of tests that have passed */
  53 static int n_bad = 0; /**< Number of tests that have failed. */
  54 static int n_skipped = 0; /**< Number of tests that have been skipped. */
  55
  56 static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/
  57 static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */
  58 static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */
  59 const char *verbosity_flag = "";
  60
  61 enum outcome { SKIP=2, OK=1, FAIL=0 };
  62 static enum outcome cur_test_outcome = 0;
  63 const char *cur_test_prefix = NULL; /**< prefix of the current test group */
  64 /** Name of the current test, if we haven't logged is yet. Used for --quiet */
  65 const char *cur_test_name = NULL;
  66
  67 #ifdef _WIN32
  68 /* Copy of argv[0] for win32. */
  69 static char commandname[MAX_PATH+1];
  70 #endif
  71
  72 static void usage(struct testgroup_t *groups, int list_groups)
  73   __attribute__((noreturn));
  74
  75 static enum outcome
  76 testcase_run_bare_(const struct testcase_t *testcase)
  77 {
  78         void *env = NULL;
  79         int outcome;
  80         if (testcase->setup) {
  81                 env = testcase->setup->setup_fn(testcase);
  82                 if (!env)
  83                         return FAIL;
  84                 else if (env == (void*)TT_SKIP)
  85                         return SKIP;
  86         }
  87
  88         cur_test_outcome = OK;
  89         testcase->fn(env);
  90         outcome = cur_test_outcome;
  91
  92         if (testcase->setup) {
  93                 if (testcase->setup->cleanup_fn(testcase, env) == 0)
  94                         outcome = FAIL;
  95         }
  96
  97         return outcome;
  98 }
  99
 100 #define MAGIC_EXITCODE 42
 101
 102 static enum outcome
 103 testcase_run_forked_(const struct testgroup_t *group,
 104                      const struct testcase_t *testcase)
 105 {
 106 #ifdef _WIN32
 107         /* Fork? On Win32?  How primitive!  We'll do what the smart kids do:
 108            we'll invoke our own exe (whose name we recall from the command
 109            line) with a command line that tells it to run just the test we
 110            want, and this time without forking.
 111
 112            (No, threads aren't an option.  The whole point of forking is to
 113            share no state between tests.)
 114          */
 115         int ok;
 116         char buffer[LONGEST_TEST_NAME+256];
 117         STARTUPINFOA si;
 118         PROCESS_INFORMATION info;
 119         DWORD exitcode;
 120
 121         if (!in_tinytest_main) {
 122                 printf("\nERROR.  On Windows, testcase_run_forked_ must be"
 123                        " called from within tinytest_main.\n");
 124                 abort();
 125         }
 126         if (opt_verbosity>0)
 127                 printf("[forking] ");
 128
 129         snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s",
 130                  commandname, verbosity_flag, group->prefix, testcase->name);
 131
 132         memset(&si, 0, sizeof(si));
 133         memset(&info, 0, sizeof(info));
 134         si.cb = sizeof(si);
 135
 136         ok = CreateProcessA(commandname, buffer, NULL, NULL, 0,
 137                            0, NULL, NULL, &si, &info);
 138         if (!ok) {
 139                 printf("CreateProcess failed!\n");
 140                 return 0;
 141         }
 142         WaitForSingleObject(info.hProcess, INFINITE);
 143         GetExitCodeProcess(info.hProcess, &exitcode);
 144         CloseHandle(info.hProcess);
 145         CloseHandle(info.hThread);
 146         if (exitcode == 0)
 147                 return OK;
 148         else if (exitcode == MAGIC_EXITCODE)
 149                 return SKIP;
 150         else
 151                 return FAIL;
 152 #else
 153         int outcome_pipe[2];
 154         pid_t pid;
 155         (void)group;
 156
 157         if (pipe(outcome_pipe))
 158                 perror("opening pipe");
 159
 160         if (opt_verbosity>0)
 161                 printf("[forking] ");
 162         pid = fork();
 163         if (!pid) {
 164                 /* child. */
 165                 int test_r, write_r;
 166                 char b[1];
 167                 close(outcome_pipe[0]);
 168                 test_r = testcase_run_bare_(testcase);
 169                 assert(0<=(int)test_r && (int)test_r<=2);
 170                 b[0] = "NYS"[test_r];
 171                 write_r = (int)write(outcome_pipe[1], b, 1);
 172                 if (write_r != 1) {
 173                         perror("write outcome to pipe");
 174                         exit(1);
 175                 }
 176                 exit(0);
 177                 return FAIL; /* unreachable */
 178         } else {
 179                 /* parent */
 180                 int status, r;
 181                 char b[1];
 182                 /* Close this now, so that if the other side closes it,
 183                  * our read fails. */
 184                 close(outcome_pipe[1]);
 185                 r = (int)read(outcome_pipe[0], b, 1);
 186                 if (r == 0) {
 187                         printf("[Lost connection!] ");
 188                         return 0;
 189                 } else if (r != 1) {
 190                         perror("read outcome from pipe");
 191                 }
 192                 waitpid(pid, &status, 0);
 193                 close(outcome_pipe[0]);
 194                 return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL);
 195         }
 196 #endif
 197 }
 198
 199 int
 200 testcase_run_one(const struct testgroup_t *group,
 201                  const struct testcase_t *testcase)
 202 {
 203         enum outcome outcome;
 204
 205         if (testcase->flags & TT_SKIP) {
 206                 if (opt_verbosity>0)
 207                         printf("%s%s: SKIPPED\n",
 208                             group->prefix, testcase->name);
 209                 ++n_skipped;
 210                 return SKIP;
 211         }
 212
 213         if (opt_verbosity>0 && !opt_forked) {
 214                 printf("%s%s: ", group->prefix, testcase->name);
 215         } else {
 216                 if (opt_verbosity==0) printf(".");
 217                 cur_test_prefix = group->prefix;
 218                 cur_test_name = testcase->name;
 219         }
 220
 221         if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) {
 222                 outcome = testcase_run_forked_(group, testcase);
 223         } else {
 224                 outcome = testcase_run_bare_(testcase);
 225         }
 226
 227         if (outcome == OK) {
 228                 ++n_ok;
 229                 if (opt_verbosity>0 && !opt_forked)
 230                         puts(opt_verbosity==1?"OK":"");
 231         } else if (outcome == SKIP) {
 232                 ++n_skipped;
 233                 if (opt_verbosity>0 && !opt_forked)
 234                         puts("SKIPPED");
 235         } else {
 236                 ++n_bad;
 237                 if (!opt_forked)
 238                         printf("\n  [%s FAILED]\n", testcase->name);
 239         }
 240
 241         if (opt_forked) {
 242                 exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1));
 243                 return 1; /* unreachable */
 244         } else {
 245                 return (int)outcome;
 246         }
 247 }
 248
 249 int
 250 tinytest_set_flag_(struct testgroup_t *groups, const char *arg, unsigned long flag)
 251 {
 252         int i, j;
 253         size_t length = LONGEST_TEST_NAME;
 254         char fullname[LONGEST_TEST_NAME];
 255         int found=0;
 256         if (strstr(arg, ".."))
 257                 length = strstr(arg,"..")-arg;
 258         for (i=0; groups[i].prefix; ++i) {
 259                 for (j=0; groups[i].cases[j].name; ++j) {
 260                         snprintf(fullname, sizeof(fullname), "%s%s",
 261                                  groups[i].prefix, groups[i].cases[j].name);
 262                         if (!flag) /* Hack! */
 263                                 printf("    %s\n", fullname);
 264                         if (!strncmp(fullname, arg, length)) {
 265                                 groups[i].cases[j].flags |= flag;
 266                                 ++found;
 267                         }
 268                 }
 269         }
 270         return found;
 271 }
 272
 273 static void
 274 usage(struct testgroup_t *groups, int list_groups)
 275 {
 276         puts("Options are: [--verbose|--quiet|--terse] [--no-fork]");
 277         puts("  Specify tests by name, or using a prefix ending with '..'");
 278         puts("  To skip a test, list give its name prefixed with a colon.");
 279         puts("  Use --list-tests for a list of tests.");
 280         if (list_groups) {
 281                 puts("Known tests are:");
 282                 tinytest_set_flag_(groups, "..", 0);
 283         }
 284         exit(0);
 285 }
 286
 287 int
 288 tinytest_main(int c, const char **v, struct testgroup_t *groups)
 289 {
 290         int i, j, n=0;
 291
 292 #ifdef _WIN32
 293         const char *sp = strrchr(v[0], '.');
 294         const char *extension = "";
 295         if (!sp || stricmp(sp, ".exe"))
 296                 extension = ".exe"; /* Add an exe so CreateProcess will work */
 297         snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension);
 298         commandname[MAX_PATH]='\0';
 299 #endif
 300         for (i=1; i<c; ++i) {
 301                 if (v[i][0] == '-') {
 302                         if (!strcmp(v[i], "--RUNNING-FORKED")) {
 303                                 opt_forked = 1;
 304                         } else if (!strcmp(v[i], "--no-fork")) {
 305                                 opt_nofork = 1;
 306                         } else if (!strcmp(v[i], "--quiet")) {
 307                                 opt_verbosity = -1;
 308                                 verbosity_flag = "--quiet";
 309                         } else if (!strcmp(v[i], "--verbose")) {
 310                                 opt_verbosity = 2;
 311                                 verbosity_flag = "--verbose";
 312                         } else if (!strcmp(v[i], "--terse")) {
 313                                 opt_verbosity = 0;
 314                                 verbosity_flag = "--terse";
 315                         } else if (!strcmp(v[i], "--help")) {
 316                                 usage(groups, 0);
 317                         } else if (!strcmp(v[i], "--list-tests")) {
 318                                 usage(groups, 1);
 319                         } else {
 320                                 printf("Unknown option %s.  Try --help\n",v[i]);
 321                                 return -1;
 322                         }
 323                 } else {
 324                         const char *test = v[i];
 325                         int flag = TT_ENABLED_;
 326                         if (test[0] == ':') {
 327                                 ++test;
 328                                 flag = TT_SKIP;
 329                         } else {
 330                                 ++n;
 331                         }
 332                         if (!tinytest_set_flag_(groups, test, flag)) {
 333                                 printf("No such test as %s!\n", v[i]);
 334                                 return -1;
 335                         }
 336                 }
 337         }
 338         if (!n)
 339                 tinytest_set_flag_(groups, "..", TT_ENABLED_);
 340
 341         setvbuf(stdout, NULL, _IONBF, 0);
 342
 343         ++in_tinytest_main;
 344         for (i=0; groups[i].prefix; ++i)
 345                 for (j=0; groups[i].cases[j].name; ++j)
 346                         if (groups[i].cases[j].flags & TT_ENABLED_)
 347                                 testcase_run_one(&groups[i],
 348                                                  &groups[i].cases[j]);
 349
 350         --in_tinytest_main;
 351
 352         if (opt_verbosity==0)
 353                 puts("");
 354
 355         if (n_bad)
 356                 printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad,
 357                        n_bad+n_ok,n_skipped);
 358         else if (opt_verbosity >= 1)
 359                 printf("%d tests ok.  (%d skipped)\n", n_ok, n_skipped);
 360
 361         return (n_bad == 0) ? 0 : 1;
 362 }
 363
 364 int
 365 tinytest_get_verbosity_(void)
 366 {
 367         return opt_verbosity;
 368 }
 369
 370 void
 371 tinytest_set_test_failed_(void)
 372 {
 373         if (opt_verbosity <= 0 && cur_test_name) {
 374                 if (opt_verbosity==0) puts("");
 375                 printf("%s%s: ", cur_test_prefix, cur_test_name);
 376                 cur_test_name = NULL;
 377         }
 378         cur_test_outcome = 0;
 379 }
 380
 381 void
 382 tinytest_set_test_skipped_(void)
 383 {
 384         if (cur_test_outcome==OK)
 385                 cur_test_outcome = SKIP;
 386 }
 387