2 # Bootstrap Samba and run a number of tests against it.
3 # Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
4 # Published under the GNU GPL, v3 or later.
10 selftest - Samba test runner
16 selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win] [--socket-wrapper] [--quick] [--one] [--prefix=prefix] [--immediate] [TESTS]
20 A simple test runner. TESTS is a regular expression with tests to run.
28 Show list of available options.
34 =item I<--builddir=DIR>
40 Change directory to run tests in. Default is 'st'.
44 Show errors as soon as they happen rather than at the end of the test run.
46 =item I<--target samba4|samba3|win>
48 Specify test target against which to run. Default is 'samba4'.
52 Run only a limited number of tests. Intended to run in about 30 seconds on
53 moderately recent systems.
55 =item I<--socket-wrapper>
57 Use socket wrapper library for communication with server. Only works
58 when the server is running locally.
60 Will prevent TCP and UDP ports being opened on the local host but
61 (transparently) redirects these calls to use unix domain sockets.
63 =item I<--expected-failures>
65 Specify a file containing a list of tests that are expected to fail. Failures for
66 these tests will be counted as successes, successes will be counted as failures.
68 The format for the file is, one entry per line:
70 TESTSUITE-NAME/TEST-NAME
74 Specify a file containing a list of tests that should be skipped. Possible candidates are
75 tests that segfault the server, flip or don't end.
79 Abort as soon as one test fails.
87 =item I<SMBD_VALGRIND>
89 =item I<TORTURE_MAXTIME>
103 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
114 use FindBin
qw($RealBin $Script);
124 my $opt_target = "samba4";
126 my $opt_socket_wrapper = 0;
127 my $opt_socket_wrapper_pcap = undef;
129 my $opt_immediate = 0;
130 my $opt_expected_failures = undef;
131 my $opt_skip = undef;
139 my $suitesfailed = [];
141 my @expected_failures = ();
149 TESTS_UNEXPECTED_OK => 0,
150 TESTS_EXPECTED_OK => 0,
151 TESTS_UNEXPECTED_FAIL => 0,
152 TESTS_EXPECTED_FAIL => 0,
156 sub expecting_failure($)
158 my $fullname = shift;
160 return 1 if (grep(/^$fullname$/, @expected_failures));
167 my $fullname = shift;
169 return 1 if (grep(/^$fullname$/, @skips));
173 sub run_test_buildfarm($$$$)
175 my ($name, $cmd, $i, $suitestotal) = @_;
176 print "--==--==--==--==--==--==--==--==--==--==--\n";
177 print "Running test $name (level 0 stdout)\n";
178 print "--==--==--==--==--==--==--==--==--==--==--\n";
181 my $expected_ret = 1;
183 open(RESULT, "$cmd 2>&1|");
186 if (/^test: (.+)\n/) {
187 $open_tests->{$1} = 1;
188 } elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
190 if ($1 eq "success") {
191 delete $open_tests->{$2};
192 if (expecting_failure("$name/$2")) {
193 $statistics->{TESTS_UNEXPECTED_OK}++;
195 $statistics->{TESTS_EXPECTED_OK}++;
197 } elsif ($1 eq "failure") {
198 delete $open_tests->{$2};
199 if (expecting_failure("$name/$2")) {
200 $statistics->{TESTS_EXPECTED_FAIL}++;
203 $statistics->{TESTS_UNEXPECTED_FAIL}++;
205 } elsif ($1 eq "skip") {
206 delete $open_tests->{$2};
207 } elsif ($1 eq "error") {
208 $statistics->{TESTS_ERROR}++;
209 delete $open_tests->{$2};
213 print "COMMAND: $cmd\n";
214 foreach (keys %$open_tests) {
215 print "$_ was started but never finished!\n";
216 $statistics->{TESTS_ERROR}++;
218 my $ret = close(RESULT);
220 print "==========================================\n";
221 if ($ret == $expected_ret) {
222 print "TEST PASSED: $name\n";
224 print "TEST FAILED: $name (status $ret)\n";
226 print "==========================================\n";
229 my $test_output = {};
230 sub run_test_plain($$$$)
232 my ($name, $cmd, $i, $totalsuites) = @_;
234 if ($#$suitesfailed+1 > 0) { $err = ", ".($#$suitesfailed+1)." errors"; }
235 printf "[$i/$totalsuites in " . (time() - $start)."s$err] $name\n";
236 open(RESULT, "$cmd 2>&1|");
237 my $expected_ret = 1;
239 $test_output->{$name} = "";
241 $test_output->{$name}.=$_;
242 print if ($opt_verbose);
243 if (/^test: (.+)\n/) {
244 $open_tests->{$1} = 1;
245 } elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
247 if ($1 eq "success") {
248 delete $open_tests->{$2};
249 if (expecting_failure("$name/$2")) {
250 $statistics->{TESTS_UNEXPECTED_OK}++;
252 $statistics->{TESTS_EXPECTED_OK}++;
254 } elsif ($1 eq "failure") {
255 delete $open_tests->{$2};
256 if (expecting_failure("$name/$2")) {
257 $statistics->{TESTS_EXPECTED_FAIL}++;
260 $statistics->{TESTS_UNEXPECTED_FAIL}++;
262 } elsif ($1 eq "skip") {
263 delete $open_tests->{$2};
264 } elsif ($1 eq "error") {
265 $statistics->{TESTS_ERROR}++;
266 delete $open_tests->{$2};
270 $test_output->{$name}.="COMMAND: $cmd\n";
271 foreach (keys %$open_tests) {
272 $test_output->{$name}.="$_ was started but never finished!\n";
273 $statistics->{TESTS_ERROR}++;
275 my $ret = close(RESULT);
276 if ($ret != $expected_ret and ($opt_immediate or $opt_one) and not $opt_verbose) {
277 print "$test_output->{$name}\n";
279 if ($ret != $expected_ret) {
280 push(@$suitesfailed, $name);
281 $statistics->{SUITES_FAIL}++;
282 exit(1) if ($opt_one);
284 $statistics->{SUITES_OK}++;
290 print "Samba test runner
291 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
293 Usage: $Script [OPTIONS] PREFIX
296 --help this help page
299 --prefix=DIR prefix to run tests in [st]
300 --srcdir=DIR source directory [.]
301 --builddir=DIR output directory [.]
304 --target=samba4|samba3|win Samba version to target
305 --socket-wrapper-pcap=FILE save traffic to pcap file
306 --socket-wrapper enable socket wrapper
307 --expected-failures=FILE specify list of tests that is guaranteed to fail
310 --quick run quick overall test
311 --one abort when the first test fails
312 --immediate print test output for failed tests during run
318 my $result = GetOptions (
319 'help|h|?' => \$opt_help,
320 'target=s' => \$opt_target,
321 'prefix=s' => \$prefix,
322 'socket-wrapper' => \$opt_socket_wrapper,
323 'socket-wrapper-pcap=s' => \$opt_socket_wrapper_pcap,
324 'quick' => \$opt_quick,
326 'immediate' => \$opt_immediate,
327 'expected-failures=s' => \$opt_expected_failures,
328 'skip=s' => \$opt_skip,
329 'srcdir=s' => \$srcdir,
330 'builddir=s' => \$builddir,
331 'verbose' => \$opt_verbose,
332 'testenv' => \$opt_testenv
335 exit(1) if (not $result);
337 ShowHelp() if ($opt_help);
341 my $torture_maxtime = $ENV{TORTURE_MAXTIME};
342 unless (defined($torture_maxtime)) {
343 $torture_maxtime = 1200;
346 # quick hack to disable rpc validation when using valgrind - its way too slow
347 unless (defined($ENV{VALGRIND})) {
348 $ENV{VALIDATE} = "validate";
351 my $old_pwd = "$RealBin/../..";
352 my $ldap = (defined($ENV{TEST_LDAP}) and ($ENV{TEST_LDAP} eq "yes"))?1:0;
355 $ENV{PREFIX} = $prefix;
357 $ENV{SRCDIR} = $srcdir;
359 my $bindir = "$srcdir/bin";
360 my $setupdir = "$srcdir/setup";
361 my $testsdir = "$srcdir/script/tests";
363 my $tls_enabled = not $opt_quick;
364 my $from_build_farm = (defined($ENV{RUN_FROM_BUILD_FARM}) and
365 ($ENV{RUN_FROM_BUILD_FARM} eq "yes"));
367 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
368 $ENV{LD_LDB_MODULE_PATH} = "$old_pwd/bin/modules/ldb";
369 $ENV{LD_SAMBA_MODULE_PATH} = "$old_pwd/bin/modules";
370 if (defined($ENV{LD_LIBRARY_PATH})) {
371 $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared:$ENV{LD_LIBRARY_PATH}";
373 $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared";
375 $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig:$ENV{PKG_CONFIG_PATH}";
376 $ENV{PATH} = "$old_pwd/bin:$ENV{PATH}";
378 my @torture_options = ();
380 my $testenv_vars = {};
382 if ($opt_target eq "samba4") {
383 $testenv_vars = Samba4::provision($prefix);
384 } elsif ($opt_target eq "win") {
385 die ("Windows tests will not run without root privileges.")
386 if (`whoami` ne "root");
388 die("Windows tests will not run with socket wrapper enabled.")
389 if ($opt_socket_wrapper);
391 die("Windows tests will not run quickly.") if ($opt_quick);
393 die("Environment variable WINTESTCONF has not been defined.\n".
394 "Windows tests will not run unconfigured.") if (not defined($ENV{WINTESTCONF}));
396 die ("$ENV{WINTESTCONF} could not be read.") if (! -r $ENV{WINTESTCONF});
398 $ENV{WINTEST_DIR}="$ENV{SRCDIR}/script/tests/win";
399 } elsif ($opt_target eq "none") {
401 die("unknown target `$opt_target'");
404 foreach (keys %$testenv_vars) { $ENV{$_} = $testenv_vars->{$_}; }
406 if ($opt_socket_wrapper_pcap) {
407 $ENV{SOCKET_WRAPPER_PCAP_FILE} = $opt_socket_wrapper_pcap;
408 # Socket wrapper pcap implies socket wrapper
409 $opt_socket_wrapper = 1;
412 my $socket_wrapper_dir;
413 if ($opt_socket_wrapper)
415 $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w");
416 print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
419 # Start slapd before smbd
421 Samba4::slapd_start($bindir, $ENV{SLAPD_CONF}, $ENV{LDAP_URI}) or die("couldn't start slapd");
423 print "LDAP PROVISIONING...";
424 Samba4::provision_ldap($bindir, $setupdir);
427 $torture_maxtime *= 2;
430 if (defined($opt_expected_failures)) {
431 open(KNOWN, "<$opt_expected_failures") or die("unable to read known failures file: $!");
435 push (@expected_failures, $_); }
439 if (defined($opt_skip)) {
440 open(SKIP, "<$opt_skip") or die("unable to read skip file: $!");
448 my $test_fifo = "$prefix/smbd_test.fifo";
450 $ENV{SMBD_TEST_FIFO} = $test_fifo;
451 $ENV{SMBD_TEST_LOG} = "$prefix/smbd_test.log";
453 SocketWrapper::set_default_iface(1);
455 if (defined($ENV{SMBD_MAX_TIME})) {
456 $max_time = $ENV{SMBD_MAX_TIME};
458 Samba4::smbd_check_or_start($bindir, $test_fifo, $ENV{SMBD_TEST_LOG},
459 $socket_wrapper_dir, $max_time, $ENV{CONFFILE});
461 SocketWrapper::set_default_iface(6);
463 my $interfaces = join(',', ("127.0.0.6/8",
470 push (@torture_options, "--option=interfaces=$interfaces");
471 push (@torture_options, $ENV{CONFIGURATION});
472 # ensure any one smbtorture call doesn't run too long
473 push (@torture_options, "--maximum-runtime=$torture_maxtime");
474 push (@torture_options, "--target=$opt_target");
475 push (@torture_options, "--option=torture:progress=no") if ($from_build_farm);
476 push (@torture_options, "--format=subunit");
477 push (@torture_options, "--option=torture:quick=yes") if ($opt_quick);
479 $ENV{TORTURE_OPTIONS} = join(' ', @torture_options);
480 print "OPTIONS $ENV{TORTURE_OPTIONS}\n";
482 open(DATA, ">$test_fifo");
486 if ($opt_target eq "win") {
487 system("$testsdir/test_win.sh");
490 open(IN, "$testsdir/tests_quick.sh|");
492 open(IN, "$testsdir/tests_all.sh|");
495 if ($_ eq "-- TEST --\n") {
500 push (@todo, [$name, $cmdline])
501 if (not defined($tests) or $name =~ /$tests/);
506 close(IN) or die("Error creating recipe");
509 Samba4::wait_for_start();
511 # start off with 0 failures
514 my $suitestotal = $#todo + 1;
521 my $term = $ENV{TERM} or "xterm";
522 system("$term -e 'echo -e \"Welcome to the Samba4 Test environment
523 This matches the client environment used in make test
524 smbd is pid `cat \$PIDDIR/smbd.pid`
526 Some useful environment variables:
528 TORTURE_OPTIONS=\$TORTURE_OPTIONS
529 CONFIGURATION=\$CONFIGURATION
531 NETBIOSNAME=\$NETBIOSNAME\" && bash'");
536 $cmd =~ s/([\(\)])/\\$1/g;
540 print "SKIPPED: $name\n";
541 $statistics->{SUITES_SKIPPED}++;
545 if ($from_build_farm) {
546 run_test_buildfarm($name, $cmd, $i, $suitestotal);
548 run_test_plain($name, $cmd, $i, $suitestotal);
559 my $failed = $? >> 8;
561 if (-f "$ENV{PIDDIR}/smbd.pid" ) {
562 open(IN, "<$ENV{PIDDIR}/smbd.pid") or die("unable to open smbd pid file");
567 Samba4::slapd_stop() if ($ldap);
570 my $duration = ($end-$start);
571 my $numfailed = $#$suitesfailed+1;
572 if ($numfailed == 0) {
573 my $ok = $statistics->{TESTS_EXPECTED_OK} + $statistics->{TESTS_EXPECTED_FAIL};
574 print "ALL OK ($ok tests in $statistics->{SUITES_OK} testsuites)\n";
577 unless ($from_build_farm) {
578 if (not $opt_immediate and not $opt_verbose) {
579 foreach (@$suitesfailed) {
580 print "===============================================================================\n";
582 print $test_output->{$_};
587 print "FAILED ($statistics->{TESTS_UNEXPECTED_FAIL} failures and $statistics->{TESTS_ERROR} errors in $statistics->{SUITES_FAIL} testsuites)\n";
590 ************************
591 *** TESTSUITE FAILED ***
592 ************************
597 print "DURATION: $duration seconds\n";
599 # if there were any valgrind failures, show them
600 foreach (<$prefix/valgrind
.log*>) {
602 system("grep DWARF2.CFI.reader $_ > /dev/null");
604 print "VALGRIND FAILURE\n";
610 if ($from_build_farm) {
611 print "TEST STATUS: $numfailed\n";