r19381: Print out the specific tests that failed after a smbtorture run.
[Samba.git] / source4 / torture / smbtorture.c
blobfdd44ddc0818cfc00685d589c44805f614e48657
1 /*
2 Unix SMB/CIFS implementation.
3 SMB torture tester
4 Copyright (C) Andrew Tridgell 1997-2003
5 Copyright (C) Jelmer Vernooij 2006
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
12 This program is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with this program; if not, write to the Free Software
19 Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
22 #include "includes.h"
23 #include "lib/cmdline/popt_common.h"
24 #include "system/time.h"
25 #include "system/wait.h"
26 #include "system/filesys.h"
27 #include "libcli/libcli.h"
28 #include "lib/ldb/include/ldb.h"
29 #include "lib/events/events.h"
30 #include "dynconfig.h"
32 #include "torture/torture.h"
33 #include "build.h"
34 #include "lib/util/dlinklist.h"
35 #include "librpc/rpc/dcerpc.h"
37 static bool run_matching(struct torture_context *torture,
38 const char *prefix,
39 const char *expr,
40 struct torture_suite *suite,
41 bool *matched)
43 bool ret = true;
45 if (suite == NULL) {
46 struct torture_suite *o;
48 for (o = torture_root->children; o; o = o->next) {
49 if (gen_fnmatch(expr, o->name) == 0) {
50 *matched = true;
51 init_iconv();
52 ret &= torture_run_suite(torture, o);
53 continue;
56 ret &= run_matching(torture, o->name, expr, o, matched);
58 } else {
59 char *name;
60 struct torture_suite *c;
61 struct torture_tcase *t;
63 for (c = suite->children; c; c = c->next) {
64 asprintf(&name, "%s-%s", prefix, c->name);
65 if (gen_fnmatch(expr, name) == 0) {
66 *matched = true;
67 init_iconv();
68 ret &= torture_run_suite(torture, c);
69 free(name);
70 continue;
73 ret &= run_matching(torture, name, expr, c, matched);
75 free(name);
78 for (t = suite->testcases; t; t = t->next) {
79 asprintf(&name, "%s-%s", prefix, t->name);
80 if (gen_fnmatch(expr, name) == 0) {
81 *matched = true;
82 init_iconv();
83 ret &= torture_run_tcase(torture, t);
85 free(name);
89 return ret;
92 #define MAX_COLS 80 /* FIXME: Determine this at run-time */
94 /****************************************************************************
95 run a specified test or "ALL"
96 ****************************************************************************/
97 static bool run_test(struct torture_context *torture, const char *name)
99 bool ret = true;
100 bool matched = false;
101 struct torture_suite *o;
103 if (strequal(name, "ALL")) {
104 for (o = torture_root->children; o; o = o->next) {
105 ret &= torture_run_suite(torture, o);
107 return ret;
110 ret = run_matching(torture, NULL, name, NULL, &matched);
112 if (!matched) {
113 printf("Unknown torture operation '%s'\n", name);
114 return false;
117 return ret;
120 static void parse_dns(const char *dns)
122 char *userdn, *basedn, *secret;
123 char *p, *d;
125 /* retrievieng the userdn */
126 p = strchr_m(dns, '#');
127 if (!p) {
128 lp_set_cmdline("torture:ldap_userdn", "");
129 lp_set_cmdline("torture:ldap_basedn", "");
130 lp_set_cmdline("torture:ldap_secret", "");
131 return;
133 userdn = strndup(dns, p - dns);
134 lp_set_cmdline("torture:ldap_userdn", userdn);
136 /* retrieve the basedn */
137 d = p + 1;
138 p = strchr_m(d, '#');
139 if (!p) {
140 lp_set_cmdline("torture:ldap_basedn", "");
141 lp_set_cmdline("torture:ldap_secret", "");
142 return;
144 basedn = strndup(d, p - d);
145 lp_set_cmdline("torture:ldap_basedn", basedn);
147 /* retrieve the secret */
148 p = p + 1;
149 if (!p) {
150 lp_set_cmdline("torture:ldap_secret", "");
151 return;
153 secret = strdup(p);
154 lp_set_cmdline("torture:ldap_secret", secret);
156 printf ("%s - %s - %s\n", userdn, basedn, secret);
160 static void usage(poptContext pc)
162 struct torture_suite *o;
163 struct torture_suite *s;
164 struct torture_tcase *t;
165 int i;
167 poptPrintUsage(pc, stdout, 0);
168 printf("\n");
170 printf("The binding format is:\n\n");
172 printf(" TRANSPORT:host[flags]\n\n");
174 printf(" where TRANSPORT is either ncacn_np for SMB, ncacn_ip_tcp for RPC/TCP\n");
175 printf(" or ncalrpc for local connections.\n\n");
177 printf(" 'host' is an IP or hostname or netbios name. If the binding string\n");
178 printf(" identifies the server side of an endpoint, 'host' may be an empty\n");
179 printf(" string.\n\n");
181 printf(" 'flags' can include a SMB pipe name if using the ncacn_np transport or\n");
182 printf(" a TCP port number if using the ncacn_ip_tcp transport, otherwise they\n");
183 printf(" will be auto-determined.\n\n");
185 printf(" other recognised flags are:\n\n");
187 printf(" sign : enable ntlmssp signing\n");
188 printf(" seal : enable ntlmssp sealing\n");
189 printf(" connect : enable rpc connect level auth (auth, but no sign or seal)\n");
190 printf(" validate: enable the NDR validator\n");
191 printf(" print: enable debugging of the packets\n");
192 printf(" bigendian: use bigendian RPC\n");
193 printf(" padcheck: check reply data for non-zero pad bytes\n\n");
195 printf(" For example, these all connect to the samr pipe:\n\n");
197 printf(" ncacn_np:myserver\n");
198 printf(" ncacn_np:myserver[samr]\n");
199 printf(" ncacn_np:myserver[\\pipe\\samr]\n");
200 printf(" ncacn_np:myserver[/pipe/samr]\n");
201 printf(" ncacn_np:myserver[samr,sign,print]\n");
202 printf(" ncacn_np:myserver[\\pipe\\samr,sign,seal,bigendian]\n");
203 printf(" ncacn_np:myserver[/pipe/samr,seal,validate]\n");
204 printf(" ncacn_np:\n");
205 printf(" ncacn_np:[/pipe/samr]\n\n");
207 printf(" ncacn_ip_tcp:myserver\n");
208 printf(" ncacn_ip_tcp:myserver[1024]\n");
209 printf(" ncacn_ip_tcp:myserver[1024,sign,seal]\n\n");
211 printf(" ncalrpc:\n\n");
213 printf("The UNC format is:\n\n");
215 printf(" //server/share\n\n");
217 printf("Tests are:");
219 for (o = torture_root->children; o; o = o->next) {
220 printf("\n%s (%s):\n ", o->description, o->name);
222 i = 0;
223 for (s = o->children; s; s = s->next) {
224 if (i + strlen(o->name) + strlen(s->name) >= (MAX_COLS - 3)) {
225 printf("\n ");
226 i = 0;
228 i+=printf("%s-%s ", o->name, s->name);
231 for (t = o->testcases; t; t = t->next) {
232 if (i + strlen(o->name) + strlen(t->name) >= (MAX_COLS - 3)) {
233 printf("\n ");
234 i = 0;
236 i+=printf("%s-%s ", o->name, t->name);
239 if (i) printf("\n");
242 printf("\nThe default test is ALL.\n");
244 exit(1);
247 static bool is_binding_string(const char *binding_string)
249 TALLOC_CTX *mem_ctx = talloc_named_const(NULL, 0, "is_binding_string");
250 struct dcerpc_binding *binding_struct;
251 NTSTATUS status;
253 status = dcerpc_parse_binding(mem_ctx, binding_string, &binding_struct);
255 talloc_free(mem_ctx);
256 return NT_STATUS_IS_OK(status);
259 static void max_runtime_handler(int sig)
261 DEBUG(0,("maximum runtime exceeded for smbtorture - terminating\n"));
262 exit(1);
265 struct timeval last_suite_started;
267 static void simple_suite_start(struct torture_context *ctx,
268 struct torture_suite *suite)
270 last_suite_started = timeval_current();
271 printf("Running %s\n", suite->name);
274 static void simple_suite_finish(struct torture_context *ctx,
275 struct torture_suite *suite)
278 printf("%s took %g secs\n\n", suite->name,
279 timeval_elapsed(&last_suite_started));
282 static void simple_test_result (struct torture_context *context,
283 enum torture_result res, const char *reason)
285 switch (res) {
286 case TORTURE_OK:
287 if (reason)
288 printf("OK: %s\n", reason);
289 break;
290 case TORTURE_FAIL:
291 printf("TEST %s FAILED! - %s\n", context->active_test->name, reason);
292 break;
293 case TORTURE_ERROR:
294 printf("ERROR IN TEST %s! - %s\n", context->active_test->name, reason);
295 break;
296 case TORTURE_SKIP:
297 printf("SKIP: %s - %s\n", context->active_test->name, reason);
298 break;
302 static void simple_comment (struct torture_context *test,
303 const char *comment)
305 printf("%s", comment);
308 const static struct torture_ui_ops std_ui_ops = {
309 .comment = simple_comment,
310 .suite_start = simple_suite_start,
311 .suite_finish = simple_suite_finish,
312 .test_result = simple_test_result
316 static void subunit_test_start (struct torture_context *ctx,
317 struct torture_tcase *tcase,
318 struct torture_test *test)
320 printf("test: %s\n", test->name);
323 static void subunit_test_result (struct torture_context *context,
324 enum torture_result res, const char *reason)
326 switch (res) {
327 case TORTURE_OK:
328 printf("success: %s", context->active_test->name);
329 break;
330 case TORTURE_FAIL:
331 printf("failure: %s", context->active_test->name);
332 break;
333 case TORTURE_ERROR:
334 printf("error: %s", context->active_test->name);
335 break;
336 case TORTURE_SKIP:
337 printf("skip: %s", context->active_test->name);
338 break;
340 if (reason)
341 printf(" [ %s ]", reason);
342 printf("\n");
345 static void subunit_comment (struct torture_context *test,
346 const char *comment)
348 fprintf(stderr, "%s", comment);
351 const static struct torture_ui_ops subunit_ui_ops = {
352 .comment = subunit_comment,
353 .test_start = subunit_test_start,
354 .test_result = subunit_test_result
357 static void harness_test_start (struct torture_context *ctx,
358 struct torture_tcase *tcase,
359 struct torture_test *test)
363 static void harness_test_result (struct torture_context *context,
364 enum torture_result res, const char *reason)
366 switch (res) {
367 case TORTURE_OK:
368 printf("ok %s - %s\n", context->active_test->name, reason);
369 break;
370 case TORTURE_FAIL:
371 case TORTURE_ERROR:
372 printf("not ok %s - %s\n", context->active_test->name, reason);
373 break;
374 case TORTURE_SKIP:
375 printf("skip %s - %s\n", context->active_test->name, reason);
376 break;
380 static void harness_comment (struct torture_context *test,
381 const char *comment)
383 printf("# %s\n", comment);
386 const static struct torture_ui_ops harness_ui_ops = {
387 .comment = harness_comment,
388 .test_start = harness_test_start,
389 .test_result = harness_test_result
392 static void quiet_suite_start(struct torture_context *ctx,
393 struct torture_suite *suite)
395 int i;
396 ctx->quiet = true;
397 for (i = 1; i < ctx->level; i++) putchar('\t');
398 printf("%s: ", suite->name);
399 fflush(stdout);
402 static void quiet_suite_finish(struct torture_context *ctx,
403 struct torture_suite *suite)
405 putchar('\n');
408 static void quiet_test_result (struct torture_context *context,
409 enum torture_result res, const char *reason)
411 fflush(stdout);
412 switch (res) {
413 case TORTURE_OK: putchar('.'); break;
414 case TORTURE_FAIL: putchar('F'); break;
415 case TORTURE_ERROR: putchar('E'); break;
416 case TORTURE_SKIP: putchar('I'); break;
420 const static struct torture_ui_ops quiet_ui_ops = {
421 .suite_start = quiet_suite_start,
422 .suite_finish = quiet_suite_finish,
423 .test_result = quiet_test_result
427 /****************************************************************************
428 main program
429 ****************************************************************************/
430 int main(int argc,char *argv[])
432 int opt, i;
433 bool correct = true;
434 int max_runtime=0;
435 int argc_new;
436 struct torture_context *torture;
437 const struct torture_ui_ops *ui_ops;
438 char **argv_new;
439 poptContext pc;
440 static const char *target = "other";
441 const char **subunit_dir;
442 static const char *ui_ops_name = "simple";
443 enum {OPT_LOADFILE=1000,OPT_UNCLIST,OPT_TIMELIMIT,OPT_DNS,
444 OPT_DANGEROUS,OPT_SMB_PORTS,OPT_ASYNC};
446 struct poptOption long_options[] = {
447 POPT_AUTOHELP
448 {"format", 0, POPT_ARG_STRING, &ui_ops_name, 0, "Output format (one of: simple, subunit, harness)", NULL },
449 {"smb-ports", 'p', POPT_ARG_STRING, NULL, OPT_SMB_PORTS, "SMB ports", NULL},
450 {"seed", 0, POPT_ARG_INT, &torture_seed, 0, "seed", NULL},
451 {"num-progs", 0, POPT_ARG_INT, &torture_nprocs, 0, "num progs", NULL},
452 {"num-ops", 0, POPT_ARG_INT, &torture_numops, 0, "num ops", NULL},
453 {"entries", 0, POPT_ARG_INT, &torture_entries, 0, "entries", NULL},
454 {"show-all", 0, POPT_ARG_NONE, &torture_showall, 0, "show all", NULL},
455 {"loadfile", 0, POPT_ARG_STRING, NULL, OPT_LOADFILE, "loadfile", NULL},
456 {"unclist", 0, POPT_ARG_STRING, NULL, OPT_UNCLIST, "unclist", NULL},
457 {"timelimit", 't', POPT_ARG_STRING, NULL, OPT_TIMELIMIT, "timelimit", NULL},
458 {"failures", 'f', POPT_ARG_INT, &torture_failures, 0, "failures", NULL},
459 {"parse-dns", 'D', POPT_ARG_STRING, NULL, OPT_DNS, "parse-dns", NULL},
460 {"dangerous", 'X', POPT_ARG_NONE, NULL, OPT_DANGEROUS,
461 "run dangerous tests (eg. wiping out password database)", NULL},
462 {"target", 'T', POPT_ARG_STRING, &target, 0, "samba3|samba4|other", NULL},
463 {"async", 'a', POPT_ARG_NONE, NULL, OPT_ASYNC,
464 "run async tests", NULL},
465 {"num-async", 0, POPT_ARG_INT, &torture_numasync, 0,
466 "number of simultaneous async requests", NULL},
467 {"maximum-runtime", 0, POPT_ARG_INT, &max_runtime, 0,
468 "set maximum time for smbtorture to live", "seconds"},
469 POPT_COMMON_SAMBA
470 POPT_COMMON_CONNECTION
471 POPT_COMMON_CREDENTIALS
472 POPT_COMMON_VERSION
473 { NULL }
476 setlinebuf(stdout);
478 /* we are never interested in SIGPIPE */
479 BlockSignals(true,SIGPIPE);
481 pc = poptGetContext("smbtorture", argc, (const char **) argv, long_options,
482 POPT_CONTEXT_KEEP_FIRST);
484 poptSetOtherOptionHelp(pc, "<binding>|<unc> TEST1 TEST2 ...");
486 while((opt = poptGetNextOpt(pc)) != -1) {
487 switch (opt) {
488 case OPT_LOADFILE:
489 lp_set_cmdline("torture:loadfile", poptGetOptArg(pc));
490 break;
491 case OPT_UNCLIST:
492 lp_set_cmdline("torture:unclist", poptGetOptArg(pc));
493 break;
494 case OPT_TIMELIMIT:
495 lp_set_cmdline("torture:timelimit", poptGetOptArg(pc));
496 break;
497 case OPT_DNS:
498 parse_dns(poptGetOptArg(pc));
499 break;
500 case OPT_DANGEROUS:
501 lp_set_cmdline("torture:dangerous", "Yes");
502 break;
503 case OPT_ASYNC:
504 lp_set_cmdline("torture:async", "Yes");
505 break;
506 case OPT_SMB_PORTS:
507 lp_set_cmdline("smb ports", poptGetOptArg(pc));
508 break;
509 default:
510 d_printf("Invalid option %s: %s\n",
511 poptBadOption(pc, 0), poptStrerror(opt));
512 torture_init();
513 usage(pc);
514 exit(1);
518 if (strcmp(target, "samba3") == 0) {
519 lp_set_cmdline("target:samba3", "true");
520 } else if (strcmp(target, "samba4") == 0) {
521 lp_set_cmdline("target:samba4", "true");
524 if (max_runtime) {
525 /* this will only work if nobody else uses alarm(),
526 which means it won't work for some tests, but we
527 can't use the event context method we use for smbd
528 as so many tests create their own event
529 context. This will at least catch most cases. */
530 signal(SIGALRM, max_runtime_handler);
531 alarm(max_runtime);
534 torture_init();
535 ldb_global_init();
537 subunit_dir = lp_parm_string_list(-1, "torture", "subunitdir", ":");
538 if (subunit_dir == NULL)
539 torture_subunit_load_testsuites(dyn_TORTUREDIR, true, NULL);
540 else {
541 for (i = 0; subunit_dir[i]; i++)
542 torture_subunit_load_testsuites(subunit_dir[i], true, NULL);
545 if (torture_seed == 0) {
546 torture_seed = time(NULL);
548 printf("Using seed %d\n", torture_seed);
549 srandom(torture_seed);
551 argv_new = discard_const_p(char *, poptGetArgs(pc));
553 argc_new = argc;
554 for (i=0; i<argc; i++) {
555 if (argv_new[i] == NULL) {
556 argc_new = i;
557 break;
561 if (argc_new < 3) {
562 usage(pc);
563 exit(1);
566 /* see if its a RPC transport specifier */
567 if (is_binding_string(argv_new[1])) {
568 lp_set_cmdline("torture:binding", argv_new[1]);
569 } else {
570 char *binding = NULL;
571 char *host = NULL, *share = NULL;
573 if (!smbcli_parse_unc(argv_new[1], NULL, &host, &share)) {
574 d_printf("Invalid option: %s is not a valid torture target (share or binding string)\n\n", argv_new[1]);
575 usage(pc);
578 lp_set_cmdline("torture:host", host);
579 lp_set_cmdline("torture:share", share);
580 asprintf(&binding, "ncacn_np:%s", host);
581 lp_set_cmdline("torture:binding", binding);
584 if (!strcmp(ui_ops_name, "simple")) {
585 ui_ops = &std_ui_ops;
586 } else if (!strcmp(ui_ops_name, "subunit")) {
587 ui_ops = &subunit_ui_ops;
588 } else if (!strcmp(ui_ops_name, "harness")) {
589 ui_ops = &harness_ui_ops;
590 } else if (!strcmp(ui_ops_name, "quiet")) {
591 ui_ops = &quiet_ui_ops;
592 } else {
593 printf("Unknown output format '%s'\n", ui_ops_name);
594 exit(1);
597 torture = torture_context_init(talloc_autofree_context(), "KNOWN_FAILURES",
598 ui_ops);
600 if (argc_new == 0) {
601 printf("You must specify a test to run, or 'ALL'\n");
602 } else {
603 int total;
604 double rate;
605 int unexpected_failures;
606 for (i=2;i<argc_new;i++) {
607 if (!run_test(torture, argv_new[i])) {
608 correct = false;
613 unexpected_failures = str_list_length(torture->results.unexpected_failures);
615 total = torture->results.skipped+torture->results.success+torture->results.failed+torture->results.errors;
616 if (total == 0) {
617 printf("No tests run.\n");
618 } else {
619 rate = ((total - unexpected_failures - torture->results.errors) * (100.0 / total));
621 printf("Tests: %d, Failures: %d", total, torture->results.failed);
622 if (torture->results.failed - unexpected_failures) {
623 printf(" (%d expected)", torture->results.failed - unexpected_failures);
625 printf(", Errors: %d, Skipped: %d. Success rate: %.2f%%\n",
626 torture->results.errors, torture->results.skipped, rate);
629 if (unexpected_failures) {
630 printf("The following tests failed:\n");
631 for (i = 0; torture->results.unexpected_failures[i]; i++) {
632 printf(" %s\n", torture->results.unexpected_failures[i]);
634 printf("\n");
637 if (str_list_length(torture->results.unexpected_errors)) {
638 printf("Errors occurred while running the following tests:\n");
639 for (i = 0; torture->results.unexpected_errors[i]; i++) {
640 printf(" %s\n", torture->results.unexpected_errors[i]);
642 printf("\n");
645 if (str_list_length(torture->results.unexpected_successes)) {
646 printf("The following tests were expected to fail but succeeded:\n");
647 for (i = 0; torture->results.unexpected_successes[i]; i++) {
648 printf(" %s\n", torture->results.unexpected_successes[i]);
650 printf("\n");
654 talloc_free(torture);
656 if (correct) {
657 return(0);
658 } else {
659 return(1);