2 * Run a set of tests, reporting results.
6 * runtests [-hv] [-b <build-dir>] [-s <source-dir>] -l <test-list>
7 * runtests [-hv] [-b <build-dir>] [-s <source-dir>] <test> [<test> ...]
8 * runtests -o [-h] [-b <build-dir>] [-s <source-dir>] <test>
10 * In the first case, expects a list of executables located in the given file,
11 * one line per executable. For each one, runs it as part of a test suite,
12 * reporting results. In the second case, use the same infrastructure, but
13 * run only the tests listed on the command line.
15 * Test output should start with a line containing the number of tests
16 * (numbered from 1 to this number), optionally preceded by "1..", although
17 * that line may be given anywhere in the output. Each additional line should
18 * be in the following format:
23 * not ok <number> # todo
25 * where <number> is the number of the test. An optional comment is permitted
26 * after the number if preceded by whitespace. ok indicates success, not ok
27 * indicates failure. "# skip" and "# todo" are a special cases of a comment,
28 * and must start with exactly that formatting. They indicate the test was
29 * skipped for some reason (maybe because it doesn't apply to this platform)
30 * or is testing something known to currently fail. The text following either
31 * "# skip" or "# todo" and whitespace is the reason.
33 * As a special case, the first line of the output may be in the form:
35 * 1..0 # skip some reason
37 * which indicates that this entire test case should be skipped and gives a
40 * Any other lines are ignored, although for compliance with the TAP protocol
41 * all lines other than the ones in the above format should be sent to
42 * standard error rather than standard output and start with #.
44 * This is a subset of TAP as documented in Test::Harness::TAP or
45 * TAP::Parser::Grammar, which comes with Perl.
47 * If the -o option is given, instead run a single test and display all of its
48 * output. This is intended for use with failing tests so that the person
49 * running the test suite can get more details about what failed.
51 * If built with the C preprocessor symbols SOURCE and BUILD defined, C TAP
52 * Harness will export those values in the environment so that tests can find
53 * the source and build directory and will look for tests under both
54 * directories. These paths can also be set with the -b and -s command-line
55 * options, which will override anything set at build time.
57 * If the -v option is given, or the C_TAP_VERBOSE environment variable is set,
58 * display the full output of each test as it runs rather than showing a
59 * summary of the results of each test.
61 * Any bug reports, bug fixes, and improvements are very much welcome and
62 * should be sent to the e-mail address below. This program is part of C TAP
63 * Harness <http://www.eyrie.org/~eagle/software/c-tap-harness/>.
65 * Copyright 2000, 2001, 2004, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013,
66 * 2014, 2015 Russ Allbery <eagle@eyrie.org>
68 * Permission is hereby granted, free of charge, to any person obtaining a
69 * copy of this software and associated documentation files (the "Software"),
70 * to deal in the Software without restriction, including without limitation
71 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
72 * and/or sell copies of the Software, and to permit persons to whom the
73 * Software is furnished to do so, subject to the following conditions:
75 * The above copyright notice and this permission notice shall be included in
76 * all copies or substantial portions of the Software.
78 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
79 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
80 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
81 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
82 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
83 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
84 * DEALINGS IN THE SOFTWARE.
87 /* Required for fdopen(), getopt(), and putenv(). */
88 #if defined(__STRICT_ANSI__) || defined(PEDANTIC)
89 # ifndef _XOPEN_SOURCE
90 # define _XOPEN_SOURCE 500
104 #include <sys/stat.h>
105 #include <sys/time.h>
106 #include <sys/types.h>
107 #include <sys/wait.h>
111 /* sys/time.h must be included before sys/resource.h on some platforms. */
112 #include <sys/resource.h>
114 /* AIX 6.1 (and possibly later) doesn't have WCOREDUMP. */
116 # define WCOREDUMP(status) ((unsigned)(status) & 0x80)
120 * POSIX requires that these be defined in <unistd.h>, but they're not always
121 * available. If one of them has been defined, all the rest almost certainly
125 # define STDIN_FILENO 0
126 # define STDOUT_FILENO 1
127 # define STDERR_FILENO 2
131 * Used for iterating through arrays. Returns the number of elements in the
132 * array (useful for a < upper bound in a for loop).
134 #define ARRAY_SIZE(array) (sizeof(array) / sizeof((array)[0]))
137 * The source and build versions of the tests directory. This is used to set
138 * the SOURCE and BUILD environment variables and find test programs, if set.
139 * Normally, this should be set as part of the build process to the test
140 * subdirectories of $(abs_top_srcdir) and $(abs_top_builddir) respectively.
149 /* Test status codes. */
157 /* Really, just a boolean, but this is more self-documenting. */
163 /* Indicates the state of our plan. */
165 PLAN_INIT
, /* Nothing seen yet. */
166 PLAN_FIRST
, /* Plan seen before any tests. */
167 PLAN_PENDING
, /* Test seen and no plan yet. */
168 PLAN_FINAL
/* Plan seen after some tests. */
171 /* Error exit statuses for test processes. */
172 #define CHILDERR_DUP 100 /* Couldn't redirect stderr or stdout. */
173 #define CHILDERR_EXEC 101 /* Couldn't exec child process. */
174 #define CHILDERR_STDIN 102 /* Couldn't open stdin file. */
175 #define CHILDERR_STDERR 103 /* Couldn't open stderr file. */
177 /* Structure to hold data for a set of tests. */
179 char *file
; /* The file name of the test. */
180 char *path
; /* The path to the test program. */
181 enum plan_status plan
; /* The status of our plan. */
182 unsigned long count
; /* Expected count of tests. */
183 unsigned long current
; /* The last seen test number. */
184 unsigned int length
; /* The length of the last status message. */
185 unsigned long passed
; /* Count of passing tests. */
186 unsigned long failed
; /* Count of failing lists. */
187 unsigned long skipped
; /* Count of skipped tests (passed). */
188 unsigned long allocated
; /* The size of the results table. */
189 enum test_status
*results
; /* Table of results by test number. */
190 unsigned int aborted
; /* Whether the set was aborted. */
191 int reported
; /* Whether the results were reported. */
192 int status
; /* The exit status of the test. */
193 unsigned int all_skipped
; /* Whether all tests were skipped. */
194 char *reason
; /* Why all tests were skipped. */
197 /* Structure to hold a linked list of test sets. */
200 struct testlist
*next
;
204 * Usage message. Should be used as a printf format with four arguments: the
205 * path to runtests, given three times, and the usage_description. This is
206 * split into variables to satisfy the pedantic ISO C90 limit on strings.
208 static const char usage_message
[] = "\
209 Usage: %s [-hv] [-b <build-dir>] [-s <source-dir>] <test> ...\n\
210 %s [-hv] [-b <build-dir>] [-s <source-dir>] -l <test-list>\n\
211 %s -o [-h] [-b <build-dir>] [-s <source-dir>] <test>\n\
214 -b <build-dir> Set the build directory to <build-dir>\n\
216 static const char usage_extra
[] = "\
217 -l <list> Take the list of tests to run from <test-list>\n\
218 -o Run a single test rather than a list of tests\n\
219 -s <source-dir> Set the source directory to <source-dir>\n\
220 -v Show the full output of each test\n\
222 runtests normally runs each test listed on the command line. With the -l\n\
223 option, it instead runs every test listed in a file. With the -o option,\n\
224 it instead runs a single test and shows its complete output.\n";
227 * Header used for test output. %s is replaced by the file name of the list
230 static const char banner
[] = "\n\
231 Running all tests listed in %s. If any tests fail, run the failing\n\
232 test program with runtests -o to see more details.\n\n";
234 /* Header for reports of failed tests. */
235 static const char header
[] = "\n\
236 Failed Set Fail/Total (%) Skip Stat Failing Tests\n\
237 -------------------------- -------------- ---- ---- ------------------------";
239 /* Include the file name and line number in malloc failures. */
240 #define xcalloc(n, size) x_calloc((n), (size), __FILE__, __LINE__)
241 #define xmalloc(size) x_malloc((size), __FILE__, __LINE__)
242 #define xstrdup(p) x_strdup((p), __FILE__, __LINE__)
243 #define xreallocarray(p, n, size) \
244 x_reallocarray((p), (n), (size), __FILE__, __LINE__)
247 * __attribute__ is available in gcc 2.5 and later, but only with gcc 2.7
248 * could you use the __format__ form of the attributes, which is what we use
249 * (to avoid confusion with other macros).
251 #ifndef __attribute__
252 # if __GNUC__ < 2 || (__GNUC__ == 2 && __GNUC_MINOR__ < 7)
253 # define __attribute__(spec) /* empty */
258 * We use __alloc_size__, but it was only available in fairly recent versions
259 * of GCC. Suppress warnings about the unknown attribute if GCC is too old.
260 * We know that we're GCC at this point, so we can use the GCC variadic macro
261 * extension, which will still work with versions of GCC too old to have C99
262 * variadic macro support.
264 #if !defined(__attribute__) && !defined(__alloc_size__)
265 # if defined(__GNUC__) && !defined(__clang__)
266 # if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 3)
267 # define __alloc_size__(spec, args...) /* empty */
273 * LLVM and Clang pretend to be GCC but don't support all of the __attribute__
274 * settings that GCC does. For them, suppress warnings about unknown
275 * attributes on declarations. This unfortunately will affect the entire
276 * compilation context, but there's no push and pop available.
278 #if !defined(__attribute__) && (defined(__llvm__) || defined(__clang__))
279 # pragma GCC diagnostic ignored "-Wattributes"
282 /* Declare internal functions that benefit from compiler attributes. */
283 static void sysdie(const char *, ...)
284 __attribute__((__nonnull__
, __noreturn__
, __format__(printf
, 1, 2)));
285 static void *x_calloc(size_t, size_t, const char *, int)
286 __attribute__((__alloc_size__(1, 2), __malloc__
, __nonnull__
));
287 static void *x_malloc(size_t, const char *, int)
288 __attribute__((__alloc_size__(1), __malloc__
, __nonnull__
));
289 static void *x_reallocarray(void *, size_t, size_t, const char *, int)
290 __attribute__((__alloc_size__(2, 3), __malloc__
, __nonnull__(4)));
291 static char *x_strdup(const char *, const char *, int)
292 __attribute__((__malloc__
, __nonnull__
));
296 * Report a fatal error, including the results of strerror, and exit.
299 sysdie(const char *format
, ...)
306 fprintf(stderr
, "runtests: ");
307 va_start(args
, format
);
308 vfprintf(stderr
, format
, args
);
310 fprintf(stderr
, ": %s\n", strerror(oerrno
));
316 * Allocate zeroed memory, reporting a fatal error and exiting on failure.
319 x_calloc(size_t n
, size_t size
, const char *file
, int line
)
324 size
= (size
> 0) ? size
: 1;
327 sysdie("failed to calloc %lu bytes at %s line %d",
328 (unsigned long) size
, file
, line
);
334 * Allocate memory, reporting a fatal error and exiting on failure.
337 x_malloc(size_t size
, const char *file
, int line
)
343 sysdie("failed to malloc %lu bytes at %s line %d",
344 (unsigned long) size
, file
, line
);
350 * Reallocate memory, reporting a fatal error and exiting on failure.
352 * We should technically use SIZE_MAX here for the overflow check, but
353 * SIZE_MAX is C99 and we're only assuming C89 + SUSv3, which does not
354 * guarantee that it exists. They do guarantee that UINT_MAX exists, and we
355 * can assume that UINT_MAX <= SIZE_MAX. And we should not be allocating
356 * anything anywhere near that large.
358 * (In theory, C89 and C99 permit size_t to be smaller than unsigned int, but
359 * I disbelieve in the existence of such systems and they will have to cope
360 * without overflow checks.)
363 x_reallocarray(void *p
, size_t n
, size_t size
, const char *file
, int line
)
365 if (n
> 0 && UINT_MAX
/ n
<= size
)
366 sysdie("realloc too large at %s line %d", file
, line
);
367 p
= realloc(p
, n
* size
);
369 sysdie("failed to realloc %lu bytes at %s line %d",
370 (unsigned long) (n
* size
), file
, line
);
376 * Copy a string, reporting a fatal error and exiting on failure.
379 x_strdup(const char *s
, const char *file
, int line
)
387 sysdie("failed to strdup %lu bytes at %s line %d",
388 (unsigned long) len
, file
, line
);
395 * Form a new string by concatenating multiple strings. The arguments must be
396 * terminated by (const char *) 0.
398 * This function only exists because we can't assume asprintf. We can't
399 * simulate asprintf with snprintf because we're only assuming SUSv3, which
400 * does not require that snprintf with a NULL buffer return the required
401 * length. When those constraints are relaxed, this should be ripped out and
402 * replaced with asprintf or a more trivial replacement with snprintf.
405 concat(const char *first
, ...)
414 * Find the total memory required. Ensure we don't overflow length. We
415 * aren't guaranteed to have SIZE_MAX, so use UINT_MAX as an acceptable
416 * substitute (see the x_nrealloc comments).
418 va_start(args
, first
);
419 for (string
= first
; string
!= NULL
; string
= va_arg(args
, const char *)) {
420 if (length
>= UINT_MAX
- strlen(string
)) {
422 sysdie("strings too long in concat");
424 length
+= strlen(string
);
429 /* Create the string. */
430 result
= xmalloc(length
);
431 va_start(args
, first
);
433 for (string
= first
; string
!= NULL
; string
= va_arg(args
, const char *)) {
434 memcpy(result
+ offset
, string
, strlen(string
));
435 offset
+= strlen(string
);
438 result
[offset
] = '\0';
444 * Given a struct timeval, return the number of seconds it represents as a
445 * double. Use difftime() to convert a time_t to a double.
448 tv_seconds(const struct timeval
*tv
)
450 return difftime(tv
->tv_sec
, 0) + tv
->tv_usec
* 1e-6;
455 * Given two struct timevals, return the difference in seconds.
458 tv_diff(const struct timeval
*tv1
, const struct timeval
*tv0
)
460 return tv_seconds(tv1
) - tv_seconds(tv0
);
465 * Given two struct timevals, return the sum in seconds as a double.
468 tv_sum(const struct timeval
*tv1
, const struct timeval
*tv2
)
470 return tv_seconds(tv1
) + tv_seconds(tv2
);
475 * Given a pointer to a string, skip any leading whitespace and return a
476 * pointer to the first non-whitespace character.
479 skip_whitespace(const char *p
)
481 while (isspace((unsigned char)(*p
)))
488 * Start a program, connecting its stdout to a pipe on our end and its stderr
489 * to /dev/null, and storing the file descriptor to read from in the two
490 * argument. Returns the PID of the new process. Errors are fatal.
493 test_start(const char *path
, int *fd
)
495 int fds
[2], infd
, errfd
;
498 /* Create a pipe used to capture the output from the test program. */
499 if (pipe(fds
) == -1) {
502 sysdie("can't create pipe");
505 /* Fork a child process, massage the file descriptors, and exec. */
511 sysdie("can't fork");
513 /* In the child. Set up our standard output. */
516 close(STDOUT_FILENO
);
517 if (dup2(fds
[1], STDOUT_FILENO
) < 0)
521 /* Point standard input at /dev/null. */
523 infd
= open("/dev/null", O_RDONLY
);
525 _exit(CHILDERR_STDIN
);
526 if (infd
!= STDIN_FILENO
) {
527 if (dup2(infd
, STDIN_FILENO
) < 0)
532 /* Point standard error at /dev/null. */
533 close(STDERR_FILENO
);
534 errfd
= open("/dev/null", O_WRONLY
);
536 _exit(CHILDERR_STDERR
);
537 if (errfd
!= STDERR_FILENO
) {
538 if (dup2(errfd
, STDERR_FILENO
) < 0)
543 /* Now, exec our process. */
544 if (execl(path
, path
, (char *) 0) == -1)
545 _exit(CHILDERR_EXEC
);
547 /* In parent. Close the extra file descriptor. */
558 * Back up over the output saying what test we were executing.
561 test_backspace(struct testset
*ts
)
565 if (!isatty(STDOUT_FILENO
))
567 for (i
= 0; i
< ts
->length
; i
++)
569 for (i
= 0; i
< ts
->length
; i
++)
571 for (i
= 0; i
< ts
->length
; i
++)
578 * Allocate or resize the array of test results to be large enough to contain
579 * the test number in.
582 resize_results(struct testset
*ts
, unsigned long n
)
587 /* If there's already enough space, return quickly. */
588 if (n
<= ts
->allocated
)
592 * If no space has been allocated, do the initial allocation. Otherwise,
593 * resize. Start with 32 test cases and then add 1024 with each resize to
594 * try to reduce the number of reallocations.
596 if (ts
->allocated
== 0) {
597 s
= (n
> 32) ? n
: 32;
598 ts
->results
= xcalloc(s
, sizeof(enum test_status
));
600 s
= (n
> ts
->allocated
+ 1024) ? n
: ts
->allocated
+ 1024;
601 ts
->results
= xreallocarray(ts
->results
, s
, sizeof(enum test_status
));
604 /* Set the results for the newly-allocated test array. */
605 for (i
= ts
->allocated
; i
< s
; i
++)
606 ts
->results
[i
] = TEST_INVALID
;
612 * Report an invalid test number and set the appropriate flags. Pulled into a
613 * separate function since we do this in several places.
616 invalid_test_number(struct testset
*ts
, long n
, enum test_verbose verbose
)
620 printf("ABORTED (invalid test number %ld)\n", n
);
627 * Read the plan line of test output, which should contain the range of test
628 * numbers. We may initialize the testset structure here if we haven't yet
629 * seen a test. Return true if initialization succeeded and the test should
630 * continue, false otherwise.
633 test_plan(const char *line
, struct testset
*ts
, enum test_verbose verbose
)
638 * Accept a plan without the leading 1.. for compatibility with older
639 * versions of runtests. This will only be allowed if we've not yet seen
642 line
= skip_whitespace(line
);
643 if (strncmp(line
, "1..", 3) == 0)
647 * Get the count and check it for validity.
649 * If we have something of the form "1..0 # skip foo", the whole file was
650 * skipped; record that. If we do skip the whole file, zero out all of
651 * our statistics, since they're no longer relevant.
653 * strtol is called with a second argument to advance the line pointer
654 * past the count to make it simpler to detect the # skip case.
656 n
= strtol(line
, (char **) &line
, 10);
658 line
= skip_whitespace(line
);
660 line
= skip_whitespace(line
+ 1);
661 if (strncasecmp(line
, "skip", 4) == 0) {
662 line
= skip_whitespace(line
+ 4);
664 ts
->reason
= xstrdup(line
);
665 ts
->reason
[strlen(ts
->reason
) - 1] = '\0';
678 puts("ABORTED (invalid test count)");
685 * If we are doing lazy planning, check the plan against the largest test
686 * number that we saw and fail now if we saw a check outside the plan
689 if (ts
->plan
== PLAN_PENDING
&& (unsigned long) n
< ts
->count
) {
690 invalid_test_number(ts
, (long) ts
->count
, verbose
);
695 * Otherwise, allocated or resize the results if needed and update count,
696 * and then record that we've seen a plan.
698 resize_results(ts
, (unsigned long) n
);
699 ts
->count
= (unsigned long) n
;
700 if (ts
->plan
== PLAN_INIT
)
701 ts
->plan
= PLAN_FIRST
;
702 else if (ts
->plan
== PLAN_PENDING
)
703 ts
->plan
= PLAN_FINAL
;
709 * Given a single line of output from a test, parse it and return the success
710 * status of that test. Anything printed to stdout not matching the form
711 * /^(not )?ok \d+/ is ignored. Sets ts->current to the test number that just
715 test_checkline(const char *line
, struct testset
*ts
,
716 enum test_verbose verbose
)
718 enum test_status status
= TEST_PASS
;
722 unsigned long current
;
725 /* Before anything, check for a test abort. */
726 bail
= strstr(line
, "Bail out!");
728 bail
= skip_whitespace(bail
+ strlen("Bail out!"));
732 length
= strlen(bail
);
733 if (bail
[length
- 1] == '\n')
737 printf("ABORTED (%.*s)\n", (int) length
, bail
);
745 * If the given line isn't newline-terminated, it was too big for an
746 * fgets(), which means ignore it.
748 if (line
[strlen(line
) - 1] != '\n')
751 /* If the line begins with a hash mark, ignore it. */
755 /* If we haven't yet seen a plan, look for one. */
756 if (ts
->plan
== PLAN_INIT
&& isdigit((unsigned char)(*line
))) {
757 if (!test_plan(line
, ts
, verbose
))
759 } else if (strncmp(line
, "1..", 3) == 0) {
760 if (ts
->plan
== PLAN_PENDING
) {
761 if (!test_plan(line
, ts
, verbose
))
766 puts("ABORTED (multiple plans)");
773 /* Parse the line, ignoring something we can't parse. */
774 if (strncmp(line
, "not ", 4) == 0) {
778 if (strncmp(line
, "ok", 2) != 0)
780 line
= skip_whitespace(line
+ 2);
782 number
= strtol(line
, &end
, 10);
783 if (errno
!= 0 || end
== line
)
784 current
= ts
->current
+ 1;
785 else if (number
<= 0) {
786 invalid_test_number(ts
, number
, verbose
);
789 current
= (unsigned long) number
;
790 if (current
> ts
->count
&& ts
->plan
== PLAN_FIRST
) {
791 invalid_test_number(ts
, (long) current
, verbose
);
795 /* We have a valid test result. Tweak the results array if needed. */
796 if (ts
->plan
== PLAN_INIT
|| ts
->plan
== PLAN_PENDING
) {
797 ts
->plan
= PLAN_PENDING
;
798 resize_results(ts
, current
);
799 if (current
> ts
->count
)
804 * Handle directives. We should probably do something more interesting
805 * with unexpected passes of todo tests.
807 while (isdigit((unsigned char)(*line
)))
809 line
= skip_whitespace(line
);
811 line
= skip_whitespace(line
+ 1);
812 if (strncasecmp(line
, "skip", 4) == 0)
814 if (strncasecmp(line
, "todo", 4) == 0)
815 status
= (status
== TEST_FAIL
) ? TEST_SKIP
: TEST_FAIL
;
818 /* Make sure that the test number is in range and not a duplicate. */
819 if (ts
->results
[current
- 1] != TEST_INVALID
) {
822 printf("ABORTED (duplicate test number %lu)\n", current
);
828 /* Good results. Increment our various counters. */
830 case TEST_PASS
: ts
->passed
++; break;
831 case TEST_FAIL
: ts
->failed
++; break;
832 case TEST_SKIP
: ts
->skipped
++; break;
833 case TEST_INVALID
: break;
835 ts
->current
= current
;
836 ts
->results
[current
- 1] = status
;
837 if (!verbose
&& isatty(STDOUT_FILENO
)) {
839 if (ts
->plan
== PLAN_PENDING
)
840 outlen
= printf("%lu/?", current
);
842 outlen
= printf("%lu/%lu", current
, ts
->count
);
843 ts
->length
= (outlen
>= 0) ? (unsigned int) outlen
: 0;
850 * Print out a range of test numbers, returning the number of characters it
851 * took up. Takes the first number, the last number, the number of characters
852 * already printed on the line, and the limit of number of characters the line
853 * can hold. Add a comma and a space before the range if chars indicates that
854 * something has already been printed on the line, and print ... instead if
855 * chars plus the space needed would go over the limit (use a limit of 0 to
859 test_print_range(unsigned long first
, unsigned long last
, unsigned long chars
,
862 unsigned int needed
= 0;
865 for (n
= first
; n
> 0; n
/= 10)
868 for (n
= last
; n
> 0; n
/= 10)
874 if (limit
> 0 && chars
+ needed
> limit
) {
876 if (chars
<= limit
) {
888 printf("%lu-", first
);
896 * Summarize a single test set. The second argument is 0 if the set exited
897 * cleanly, a positive integer representing the exit status if it exited
898 * with a non-zero status, and a negative integer representing the signal
899 * that terminated it if it was killed by a signal.
902 test_summarize(struct testset
*ts
, int status
)
905 unsigned long missing
= 0;
906 unsigned long failed
= 0;
907 unsigned long first
= 0;
908 unsigned long last
= 0;
911 fputs("ABORTED", stdout
);
913 printf(" (passed %lu/%lu)", ts
->passed
, ts
->count
- ts
->skipped
);
915 for (i
= 0; i
< ts
->count
; i
++) {
916 if (ts
->results
[i
] == TEST_INVALID
) {
918 fputs("MISSED ", stdout
);
919 if (first
&& i
== last
)
923 test_print_range(first
, last
, missing
- 1, 0);
931 test_print_range(first
, last
, missing
- 1, 0);
934 for (i
= 0; i
< ts
->count
; i
++) {
935 if (ts
->results
[i
] == TEST_FAIL
) {
936 if (missing
&& !failed
)
939 fputs("FAILED ", stdout
);
940 if (first
&& i
== last
)
944 test_print_range(first
, last
, failed
- 1, 0);
952 test_print_range(first
, last
, failed
- 1, 0);
953 if (!missing
&& !failed
) {
954 fputs(!status
? "ok" : "dubious", stdout
);
955 if (ts
->skipped
> 0) {
956 if (ts
->skipped
== 1)
957 printf(" (skipped %lu test)", ts
->skipped
);
959 printf(" (skipped %lu tests)", ts
->skipped
);
964 printf(" (exit status %d)", status
);
966 printf(" (killed by signal %d%s)", -status
,
967 WCOREDUMP(ts
->status
) ? ", core dumped" : "");
973 * Given a test set, analyze the results, classify the exit status, handle a
974 * few special error messages, and then pass it along to test_summarize() for
975 * the regular output. Returns true if the test set ran successfully and all
976 * tests passed or were skipped, false otherwise.
979 test_analyze(struct testset
*ts
)
983 if (ts
->all_skipped
) {
984 if (ts
->reason
== NULL
)
987 printf("skipped (%s)\n", ts
->reason
);
989 } else if (WIFEXITED(ts
->status
) && WEXITSTATUS(ts
->status
) != 0) {
990 switch (WEXITSTATUS(ts
->status
)) {
993 puts("ABORTED (can't dup file descriptors)");
997 puts("ABORTED (execution failed -- not found?)");
1000 case CHILDERR_STDERR
:
1002 puts("ABORTED (can't open /dev/null)");
1005 test_summarize(ts
, WEXITSTATUS(ts
->status
));
1009 } else if (WIFSIGNALED(ts
->status
)) {
1010 test_summarize(ts
, -WTERMSIG(ts
->status
));
1012 } else if (ts
->plan
!= PLAN_FIRST
&& ts
->plan
!= PLAN_FINAL
) {
1013 puts("ABORTED (no valid test plan)");
1017 test_summarize(ts
, 0);
1018 return (ts
->failed
== 0);
1024 * Runs a single test set, accumulating and then reporting the results.
1025 * Returns true if the test set was successfully run and all tests passed,
1029 test_run(struct testset
*ts
, enum test_verbose verbose
)
1031 pid_t testpid
, child
;
1035 char buffer
[BUFSIZ
];
1037 /* Run the test program. */
1038 testpid
= test_start(ts
->path
, &outfd
);
1039 output
= fdopen(outfd
, "r");
1043 sysdie("fdopen failed");
1047 * Pass each line of output to test_checkline(), and print the line if
1048 * verbosity is requested.
1050 while (!ts
->aborted
&& fgets(buffer
, sizeof(buffer
), output
)) {
1052 printf("%s", buffer
);
1053 test_checkline(buffer
, ts
, verbose
);
1055 if (ferror(output
) || ts
->plan
== PLAN_INIT
)
1061 * Consume the rest of the test output, close the output descriptor,
1062 * retrieve the exit status, and pass that information to test_analyze()
1063 * for eventual output.
1065 while (fgets(buffer
, sizeof(buffer
), output
))
1067 printf("%s", buffer
);
1069 child
= waitpid(testpid
, &ts
->status
, 0);
1070 if (child
== (pid_t
) -1) {
1071 if (!ts
->reported
) {
1075 sysdie("waitpid for %u failed", (unsigned int) testpid
);
1077 if (ts
->all_skipped
)
1079 status
= test_analyze(ts
);
1081 /* Convert missing tests to failed tests. */
1082 for (i
= 0; i
< ts
->count
; i
++) {
1083 if (ts
->results
[i
] == TEST_INVALID
) {
1085 ts
->results
[i
] = TEST_FAIL
;
1093 /* Summarize a list of test failures. */
1095 test_fail_summary(const struct testlist
*fails
)
1099 unsigned long i
, first
, last
, total
;
1103 /* Failed Set Fail/Total (%) Skip Stat Failing (25)
1104 -------------------------- -------------- ---- ---- -------------- */
1105 for (; fails
; fails
= fails
->next
) {
1107 total
= ts
->count
- ts
->skipped
;
1108 printf("%-26.26s %4lu/%-4lu %3.0f%% %4lu ", ts
->file
, ts
->failed
,
1109 total
, total
? (ts
->failed
* 100.0) / total
: 0,
1111 if (WIFEXITED(ts
->status
))
1112 printf("%4d ", WEXITSTATUS(ts
->status
));
1122 for (i
= 0; i
< ts
->count
; i
++) {
1123 if (ts
->results
[i
] == TEST_FAIL
) {
1124 if (first
!= 0 && i
== last
)
1128 chars
+= test_print_range(first
, last
, chars
, 19);
1135 test_print_range(first
, last
, chars
, 19);
1142 * Check whether a given file path is a valid test. Currently, this checks
1143 * whether it is executable and is a regular file. Returns true or false.
1146 is_valid_test(const char *path
)
1150 if (access(path
, X_OK
) < 0)
1152 if (stat(path
, &st
) < 0)
1154 if (!S_ISREG(st
.st_mode
))
1161 * Given the name of a test, a pointer to the testset struct, and the source
1162 * and build directories, find the test. We try first relative to the current
1163 * directory, then in the build directory (if not NULL), then in the source
1164 * directory. In each of those directories, we first try a "-t" extension and
1165 * then a ".t" extension. When we find an executable program, we return the
1166 * path to that program. If none of those paths are executable, just fill in
1167 * the name of the test as is.
1169 * The caller is responsible for freeing the path member of the testset
1173 find_test(const char *name
, const char *source
, const char *build
)
1176 const char *bases
[3], *suffix
, *base
;
1178 const char *suffixes
[3] = { "-t", ".t", "" };
1180 /* Possible base directories. */
1185 /* Try each suffix with each base. */
1186 for (i
= 0; i
< ARRAY_SIZE(suffixes
); i
++) {
1187 suffix
= suffixes
[i
];
1188 for (j
= 0; j
< ARRAY_SIZE(bases
); j
++) {
1192 path
= concat(base
, "/", name
, suffix
, (const char *) 0);
1193 if (is_valid_test(path
))
1200 path
= xstrdup(name
);
1206 * Read a list of tests from a file, returning the list of tests as a struct
1207 * testlist. Reports an error to standard error and exits if the list of
1208 * tests cannot be read.
1210 static struct testlist
*
1211 read_test_list(const char *filename
)
1216 char buffer
[BUFSIZ
];
1217 const char *testname
;
1218 struct testlist
*listhead
, *current
;
1220 /* Create the initial container list that will hold our results. */
1221 listhead
= xcalloc(1, sizeof(struct testlist
));
1225 * Open our file of tests to run and read it line by line, creating a new
1226 * struct testlist and struct testset for each line.
1228 file
= fopen(filename
, "r");
1230 sysdie("can't open %s", filename
);
1232 while (fgets(buffer
, sizeof(buffer
), file
)) {
1234 length
= strlen(buffer
) - 1;
1235 if (buffer
[length
] != '\n') {
1236 fprintf(stderr
, "%s:%u: line too long\n", filename
, line
);
1239 buffer
[length
] = '\0';
1241 /* Skip comments, leading spaces, and blank lines. */
1242 testname
= skip_whitespace(buffer
);
1243 if (strlen(testname
) == 0)
1245 if (testname
[0] == '#')
1248 /* Allocate the new testset structure. */
1249 if (current
== NULL
)
1252 current
->next
= xcalloc(1, sizeof(struct testlist
));
1253 current
= current
->next
;
1255 current
->ts
= xcalloc(1, sizeof(struct testset
));
1256 current
->ts
->plan
= PLAN_INIT
;
1257 current
->ts
->file
= xstrdup(testname
);
1261 /* Return the results. */
1267 * Build a list of tests from command line arguments. Takes the argv and argc
1268 * representing the command line arguments and returns a newly allocated test
1269 * list. The caller is responsible for freeing.
1271 static struct testlist
*
1272 build_test_list(char *argv
[], int argc
)
1275 struct testlist
*listhead
, *current
;
1277 /* Create the initial container list that will hold our results. */
1278 listhead
= xcalloc(1, sizeof(struct testlist
));
1281 /* Walk the list of arguments and create test sets for them. */
1282 for (i
= 0; i
< argc
; i
++) {
1283 if (current
== NULL
)
1286 current
->next
= xcalloc(1, sizeof(struct testlist
));
1287 current
= current
->next
;
1289 current
->ts
= xcalloc(1, sizeof(struct testset
));
1290 current
->ts
->plan
= PLAN_INIT
;
1291 current
->ts
->file
= xstrdup(argv
[i
]);
1294 /* Return the results. */
1299 /* Free a struct testset. */
1301 free_testset(struct testset
*ts
)
1312 * Run a batch of tests. Takes two additional parameters: the root of the
1313 * source directory and the root of the build directory. Test programs will
1314 * be first searched for in the current directory, then the build directory,
1315 * then the source directory. Returns true iff all tests passed, and always
1316 * frees the test list that's passed in.
1319 test_batch(struct testlist
*tests
, const char *source
, const char *build
,
1320 enum test_verbose verbose
)
1324 unsigned int count
= 0;
1326 struct timeval start
, end
;
1327 struct rusage stats
;
1328 struct testlist
*failhead
= NULL
;
1329 struct testlist
*failtail
= NULL
;
1330 struct testlist
*current
, *next
;
1332 unsigned long total
= 0;
1333 unsigned long passed
= 0;
1334 unsigned long skipped
= 0;
1335 unsigned long failed
= 0;
1336 unsigned long aborted
= 0;
1338 /* Walk the list of tests to find the longest name. */
1339 for (current
= tests
; current
!= NULL
; current
= current
->next
) {
1340 length
= strlen(current
->ts
->file
);
1341 if (length
> longest
)
1346 * Add two to longest and round up to the nearest tab stop. This is how
1347 * wide the column for printing the current test name will be.
1351 longest
+= 8 - (longest
% 8);
1353 /* Start the wall clock timer. */
1354 gettimeofday(&start
, NULL
);
1356 /* Now, plow through our tests again, running each one. */
1357 for (current
= tests
; current
!= NULL
; current
= current
->next
) {
1360 /* Print out the name of the test file. */
1361 fputs(ts
->file
, stdout
);
1363 fputs("\n\n", stdout
);
1365 for (i
= strlen(ts
->file
); i
< longest
; i
++)
1367 if (isatty(STDOUT_FILENO
))
1371 ts
->path
= find_test(ts
->file
, source
, build
);
1372 succeeded
= test_run(ts
, verbose
);
1377 /* Record cumulative statistics. */
1378 aborted
+= ts
->aborted
;
1379 total
+= ts
->count
+ ts
->all_skipped
;
1380 passed
+= ts
->passed
;
1381 skipped
+= ts
->skipped
+ ts
->all_skipped
;
1382 failed
+= ts
->failed
;
1385 /* If the test fails, we shuffle it over to the fail list. */
1387 if (failhead
== NULL
) {
1388 failhead
= xmalloc(sizeof(struct testset
));
1389 failtail
= failhead
;
1391 failtail
->next
= xmalloc(sizeof(struct testset
));
1392 failtail
= failtail
->next
;
1395 failtail
->next
= NULL
;
1400 /* Stop the timer and get our child resource statistics. */
1401 gettimeofday(&end
, NULL
);
1402 getrusage(RUSAGE_CHILDREN
, &stats
);
1404 /* Summarize the failures and free the failure list. */
1405 if (failhead
!= NULL
) {
1406 test_fail_summary(failhead
);
1407 while (failhead
!= NULL
) {
1408 next
= failhead
->next
;
1414 /* Free the memory used by the test lists. */
1415 while (tests
!= NULL
) {
1417 free_testset(tests
->ts
);
1422 /* Print out the final test summary. */
1426 printf("Aborted %lu test set", aborted
);
1428 printf("Aborted %lu test sets", aborted
);
1429 printf(", passed %lu/%lu tests", passed
, total
);
1431 else if (failed
== 0)
1432 fputs("All tests successful", stdout
);
1434 printf("Failed %lu/%lu tests, %.2f%% okay", failed
, total
,
1435 (total
- failed
) * 100.0 / total
);
1438 printf(", %lu test skipped", skipped
);
1440 printf(", %lu tests skipped", skipped
);
1443 printf("Files=%u, Tests=%lu", count
, total
);
1444 printf(", %.2f seconds", tv_diff(&end
, &start
));
1445 printf(" (%.2f usr + %.2f sys = %.2f CPU)\n",
1446 tv_seconds(&stats
.ru_utime
), tv_seconds(&stats
.ru_stime
),
1447 tv_sum(&stats
.ru_utime
, &stats
.ru_stime
));
1448 return (failed
== 0 && aborted
== 0);
1453 * Run a single test case. This involves just running the test program after
1454 * having done the environment setup and finding the test program.
1457 test_single(const char *program
, const char *source
, const char *build
)
1461 path
= find_test(program
, source
, build
);
1462 if (execl(path
, path
, (char *) 0) == -1)
1463 sysdie("cannot exec %s", path
);
1468 * Main routine. Set the SOURCE and BUILD environment variables and then,
1469 * given a file listing tests, run each test listed.
1472 main(int argc
, char *argv
[])
1477 enum test_verbose verbose
= CONCISE
;
1478 char *source_env
= NULL
;
1479 char *build_env
= NULL
;
1480 const char *program
;
1481 const char *shortlist
;
1482 const char *list
= NULL
;
1483 const char *source
= SOURCE
;
1484 const char *build
= BUILD
;
1485 struct testlist
*tests
;
1488 while ((option
= getopt(argc
, argv
, "b:hl:os:v")) != EOF
) {
1494 printf(usage_message
, program
, program
, program
, usage_extra
);
1514 if ((list
== NULL
&& argc
< 1) || (list
!= NULL
&& argc
> 0)) {
1515 fprintf(stderr
, usage_message
, program
, program
, program
, usage_extra
);
1520 * If C_TAP_VERBOSE is set in the environment, that also turns on verbose
1523 if (getenv("C_TAP_VERBOSE") != NULL
)
1526 /* Set SOURCE and BUILD environment variables. */
1527 if (source
!= NULL
) {
1528 source_env
= concat("SOURCE=", source
, (const char *) 0);
1529 if (putenv(source_env
) != 0)
1530 sysdie("cannot set SOURCE in the environment");
1532 if (build
!= NULL
) {
1533 build_env
= concat("BUILD=", build
, (const char *) 0);
1534 if (putenv(build_env
) != 0)
1535 sysdie("cannot set BUILD in the environment");
1538 /* Run the tests as instructed. */
1540 test_single(argv
[0], source
, build
);
1541 else if (list
!= NULL
) {
1542 shortlist
= strrchr(list
, '/');
1543 if (shortlist
== NULL
)
1547 printf(banner
, shortlist
);
1548 tests
= read_test_list(list
);
1549 status
= test_batch(tests
, source
, build
, verbose
) ? 0 : 1;
1551 tests
= build_test_list(argv
, argc
);
1552 status
= test_batch(tests
, source
, build
, verbose
) ? 0 : 1;
1555 /* For valgrind cleanliness, free all our memory. */
1556 if (source_env
!= NULL
) {
1557 putenv((char *) "SOURCE=");
1560 if (build_env
!= NULL
) {
1561 putenv((char *) "BUILD=");