src/test/tinytest.c

   1 /* tinytest.c -- Copyright 2009-2010 Nick Mathewson
   2  *
   3  * Redistribution and use in source and binary forms, with or without
   4  * modification, are permitted provided that the following conditions
   5  * are met:
   6  * 1. Redistributions of source code must retain the above copyright
   7  *    notice, this list of conditions and the following disclaimer.
   8  * 2. Redistributions in binary form must reproduce the above copyright
   9  *    notice, this list of conditions and the following disclaimer in the
  10  *    documentation and/or other materials provided with the distribution.
  11  * 3. The name of the author may not be used to endorse or promote products
  12  *    derived from this software without specific prior written permission.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  15  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  16  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  17  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  18  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  19  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  20  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  21  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  23  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25
  26 #include <stdio.h>
  27 #include <stdlib.h>
  28 #include <string.h>
  29 #include <assert.h>
  30
  31 #ifdef WIN32
  32 #include <windows.h>
  33 #else
  34 #include <sys/types.h>
  35 #include <sys/wait.h>
  36 #include <unistd.h>
  37 #endif
  38
  39 #ifndef __GNUC__
  40 #define __attribute__(x)
  41 #endif
  42
  43 #ifdef TINYTEST_LOCAL
  44 #include "tinytest_local.h"
  45 #endif
  46 #include "tinytest.h"
  47 #include "tinytest_macros.h"
  48
  49 #define LONGEST_TEST_NAME 16384
  50
  51 static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/
  52 static int n_ok = 0; /**< Number of tests that have passed */
  53 static int n_bad = 0; /**< Number of tests that have failed. */
  54 static int n_skipped = 0; /**< Number of tests that have been skipped. */
  55
  56 static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/
  57 static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */
  58 static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */
  59 const char *verbosity_flag = "";
  60
  61 enum outcome { SKIP=2, OK=1, FAIL=0 };
  62 static enum outcome cur_test_outcome = 0;
  63 const char *cur_test_prefix = NULL; /**< prefix of the current test group */
  64 /** Name of the current test, if we haven't logged is yet. Used for --quiet */
  65 const char *cur_test_name = NULL;
  66
  67 #ifdef WIN32
  68 /** Pointer to argv[0] for win32. */
  69 static const char *commandname = NULL;
  70 #endif
  71
  72 static void usage(struct testgroup_t *groups, int list_groups)
  73   __attribute__((noreturn));
  74
  75 static enum outcome
  76 _testcase_run_bare(const struct testcase_t *testcase)
  77 {
  78         void *env = NULL;
  79         int outcome;
  80         if (testcase->setup) {
  81                 env = testcase->setup->setup_fn(testcase);
  82                 if (!env)
  83                         return FAIL;
  84                 else if (env == (void*)TT_SKIP)
  85                         return SKIP;
  86         }
  87
  88         cur_test_outcome = OK;
  89         testcase->fn(env);
  90         outcome = cur_test_outcome;
  91
  92         if (testcase->setup) {
  93                 if (testcase->setup->cleanup_fn(testcase, env) == 0)
  94                         outcome = FAIL;
  95         }
  96
  97         return outcome;
  98 }
  99
 100 #define MAGIC_EXITCODE 42
 101
 102 static enum outcome
 103 _testcase_run_forked(const struct testgroup_t *group,
 104                      const struct testcase_t *testcase)
 105 {
 106 #ifdef WIN32
 107         /* Fork? On Win32?  How primitive!  We'll do what the smart kids do:
 108            we'll invoke our own exe (whose name we recall from the command
 109            line) with a command line that tells it to run just the test we
 110            want, and this time without forking.
 111
 112            (No, threads aren't an option.  The whole point of forking is to
 113            share no state between tests.)
 114          */
 115         int ok;
 116         char buffer[LONGEST_TEST_NAME+256];
 117         STARTUPINFOA si;
 118         PROCESS_INFORMATION info;
 119         DWORD exitcode;
 120
 121         if (!in_tinytest_main) {
 122                 printf("\nERROR.  On Windows, _testcase_run_forked must be"
 123                        " called from within tinytest_main.\n");
 124                 abort();
 125         }
 126         if (opt_verbosity>0)
 127                 printf("[forking] ");
 128
 129         snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s",
 130                  commandname, verbosity_flag, group->prefix, testcase->name);
 131
 132         memset(&si, 0, sizeof(si));
 133         memset(&info, 0, sizeof(info));
 134         si.cb = sizeof(si);
 135
 136         ok = CreateProcessA(commandname, buffer, NULL, NULL, 0,
 137                            0, NULL, NULL, &si, &info);
 138         if (!ok) {
 139                 printf("CreateProcess failed!\n");
 140                 return 0;
 141         }
 142         WaitForSingleObject(info.hProcess, INFINITE);
 143         GetExitCodeProcess(info.hProcess, &exitcode);
 144         CloseHandle(info.hProcess);
 145         CloseHandle(info.hThread);
 146         if (exitcode == 0)
 147                 return OK;
 148         else if (exitcode == MAGIC_EXITCODE)
 149                 return SKIP;
 150         else
 151                 return FAIL;
 152 #else
 153         int outcome_pipe[2];
 154         pid_t pid;
 155         (void)group;
 156
 157         if (pipe(outcome_pipe))
 158                 perror("opening pipe");
 159
 160         if (opt_verbosity>0)
 161                 printf("[forking] ");
 162         pid = fork();
 163         if (!pid) {
 164                 /* child. */
 165                 int test_r, write_r;
 166                 char b[1];
 167                 close(outcome_pipe[0]);
 168                 test_r = _testcase_run_bare(testcase);
 169                 assert(0<=(int)test_r && (int)test_r<=2);
 170                 b[0] = "NYS"[test_r];
 171                 write_r = (int)write(outcome_pipe[1], b, 1);
 172                 if (write_r != 1) {
 173                         perror("write outcome to pipe");
 174                         exit(1);
 175                 }
 176                 exit(0);
 177         } else {
 178                 /* parent */
 179                 int status, r;
 180                 char b[1];
 181                 /* Close this now, so that if the other side closes it,
 182                  * our read fails. */
 183                 close(outcome_pipe[1]);
 184                 r = (int)read(outcome_pipe[0], b, 1);
 185                 if (r == 0) {
 186                         printf("[Lost connection!] ");
 187                         return 0;
 188                 } else if (r != 1) {
 189                         perror("read outcome from pipe");
 190                 }
 191                 waitpid(pid, &status, 0);
 192                 close(outcome_pipe[0]);
 193                 return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL);
 194         }
 195 #endif
 196 }
 197
 198 int
 199 testcase_run_one(const struct testgroup_t *group,
 200                  const struct testcase_t *testcase)
 201 {
 202         enum outcome outcome;
 203
 204         if (testcase->flags & TT_SKIP) {
 205                 if (opt_verbosity>0)
 206                         printf("%s%s: SKIPPED\n",
 207                             group->prefix, testcase->name);
 208                 ++n_skipped;
 209                 return SKIP;
 210         }
 211
 212         if (opt_verbosity>0 && !opt_forked) {
 213                 printf("%s%s: ", group->prefix, testcase->name);
 214         } else {
 215                 if (opt_verbosity==0) printf(".");
 216                 cur_test_prefix = group->prefix;
 217                 cur_test_name = testcase->name;
 218         }
 219
 220         if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) {
 221                 outcome = _testcase_run_forked(group, testcase);
 222         } else {
 223                 outcome = _testcase_run_bare(testcase);
 224         }
 225
 226         if (outcome == OK) {
 227                 ++n_ok;
 228                 if (opt_verbosity>0 && !opt_forked)
 229                         puts(opt_verbosity==1?"OK":"");
 230         } else if (outcome == SKIP) {
 231                 ++n_skipped;
 232                 if (opt_verbosity>0 && !opt_forked)
 233                         puts("SKIPPED");
 234         } else {
 235                 ++n_bad;
 236                 if (!opt_forked)
 237                         printf("\n  [%s FAILED]\n", testcase->name);
 238         }
 239
 240         if (opt_forked) {
 241                 exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1));
 242         } else {
 243                 return (int)outcome;
 244         }
 245 }
 246
 247 int
 248 _tinytest_set_flag(struct testgroup_t *groups, const char *arg, unsigned long flag)
 249 {
 250         int i, j;
 251         size_t length = LONGEST_TEST_NAME;
 252         char fullname[LONGEST_TEST_NAME];
 253         int found=0;
 254         if (strstr(arg, ".."))
 255                 length = strstr(arg,"..")-arg;
 256         for (i=0; groups[i].prefix; ++i) {
 257                 for (j=0; groups[i].cases[j].name; ++j) {
 258                         snprintf(fullname, sizeof(fullname), "%s%s",
 259                                  groups[i].prefix, groups[i].cases[j].name);
 260                         if (!flag) /* Hack! */
 261                                 printf("    %s\n", fullname);
 262                         if (!strncmp(fullname, arg, length)) {
 263                                 groups[i].cases[j].flags |= flag;
 264                                 ++found;
 265                         }
 266                 }
 267         }
 268         return found;
 269 }
 270
 271 static void
 272 usage(struct testgroup_t *groups, int list_groups)
 273 {
 274         puts("Options are: [--verbose|--quiet|--terse] [--no-fork]");
 275         puts("  Specify tests by name, or using a prefix ending with '..'");
 276         puts("  To skip a test, list give its name prefixed with a colon.");
 277         puts("  Use --list-tests for a list of tests.");
 278         if (list_groups) {
 279                 puts("Known tests are:");
 280                 _tinytest_set_flag(groups, "..", 0);
 281         }
 282         exit(0);
 283 }
 284
 285 int
 286 tinytest_main(int c, const char **v, struct testgroup_t *groups)
 287 {
 288         int i, j, n=0;
 289
 290 #ifdef WIN32
 291         commandname = v[0];
 292 #endif
 293         for (i=1; i<c; ++i) {
 294                 if (v[i][0] == '-') {
 295                         if (!strcmp(v[i], "--RUNNING-FORKED")) {
 296                                 opt_forked = 1;
 297                         } else if (!strcmp(v[i], "--no-fork")) {
 298                                 opt_nofork = 1;
 299                         } else if (!strcmp(v[i], "--quiet")) {
 300                                 opt_verbosity = -1;
 301                                 verbosity_flag = "--quiet";
 302                         } else if (!strcmp(v[i], "--verbose")) {
 303                                 opt_verbosity = 2;
 304                                 verbosity_flag = "--verbose";
 305                         } else if (!strcmp(v[i], "--terse")) {
 306                                 opt_verbosity = 0;
 307                                 verbosity_flag = "--terse";
 308                         } else if (!strcmp(v[i], "--help")) {
 309                                 usage(groups, 0);
 310                         } else if (!strcmp(v[i], "--list-tests")) {
 311                                 usage(groups, 1);
 312                         } else {
 313                                 printf("Unknown option %s.  Try --help\n",v[i]);
 314                                 return -1;
 315                         }
 316                 } else {
 317                         const char *test = v[i];
 318                         int flag = _TT_ENABLED;
 319                         if (test[0] == ':') {
 320                                 ++test;
 321                                 flag = TT_SKIP;
 322                         } else {
 323                                 ++n;
 324                         }
 325                         if (!_tinytest_set_flag(groups, test, flag)) {
 326                                 printf("No such test as %s!\n", v[i]);
 327                                 return -1;
 328                         }
 329                 }
 330         }
 331         if (!n)
 332                 _tinytest_set_flag(groups, "..", _TT_ENABLED);
 333
 334         setvbuf(stdout, NULL, _IONBF, 0);
 335
 336         ++in_tinytest_main;
 337         for (i=0; groups[i].prefix; ++i)
 338                 for (j=0; groups[i].cases[j].name; ++j)
 339                         if (groups[i].cases[j].flags & _TT_ENABLED)
 340                                 testcase_run_one(&groups[i],
 341                                                  &groups[i].cases[j]);
 342
 343         --in_tinytest_main;
 344
 345         if (opt_verbosity==0)
 346                 puts("");
 347
 348         if (n_bad)
 349                 printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad,
 350                        n_bad+n_ok,n_skipped);
 351         else if (opt_verbosity >= 1)
 352                 printf("%d tests ok.  (%d skipped)\n", n_ok, n_skipped);
 353
 354         return (n_bad == 0) ? 0 : 1;
 355 }
 356
 357 int
 358 _tinytest_get_verbosity(void)
 359 {
 360         return opt_verbosity;
 361 }
 362
 363 void
 364 _tinytest_set_test_failed(void)
 365 {
 366         if (opt_verbosity <= 0 && cur_test_name) {
 367                 if (opt_verbosity==0) puts("");
 368                 printf("%s%s: ", cur_test_prefix, cur_test_name);
 369                 cur_test_name = NULL;
 370         }
 371         cur_test_outcome = 0;
 372 }
 373
 374 void
 375 _tinytest_set_test_skipped(void)
 376 {
 377         if (cur_test_outcome==OK)
 378                 cur_test_outcome = SKIP;
 379 }
 380