2 # Bootstrap Samba and run a number of tests against it.
3 # Copyright (C) 2005-2008 Jelmer Vernooij <jelmer@samba.org>
4 # Published under the GNU GPL, v3 or later.
10 selftest - Samba test runner
16 selftest [--srcdir=DIR] [--builddir=DIR] [--exeext=EXT][--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--immediate] [--testlist=FILE] [TESTS]
20 A simple test runner. TESTS is a regular expression with tests to run.
28 Show list of available options.
34 =item I<--builddir=DIR>
44 Change directory to run tests in. Default is 'st'.
48 Show errors as soon as they happen rather than at the end of the test run.
50 =item I<--target samba4|samba3|win|kvm>
52 Specify test target against which to run. Default is 'samba4'.
56 Run only a limited number of tests. Intended to run in about 30 seconds on
57 moderately recent systems.
59 =item I<--socket-wrapper>
61 Use socket wrapper library for communication with server. Only works
62 when the server is running locally.
64 Will prevent TCP and UDP ports being opened on the local host but
65 (transparently) redirects these calls to use unix domain sockets.
67 =item I<--expected-failures>
69 Specify a file containing a list of tests that are expected to fail. Failures for
70 these tests will be counted as successes, successes will be counted as failures.
72 The format for the file is, one entry per line:
74 TESTSUITE-NAME.TEST-NAME
76 The reason for a test can also be specified, by adding a hash sign (#) and the reason
81 Specify a file containing a list of tests that should be skipped. Possible
82 candidates are tests that segfault the server, flip or don't end. The format of this file is the same as
83 for the --expected-failures flag.
87 Specify a file containing a list of tests that should be run. Same format
88 as the --exclude flag.
90 Not includes specified means all tests will be run.
94 Abort as soon as one test fails.
98 Load a list of tests from the specified location.
106 =item I<SMBD_VALGRIND>
108 =item I<TORTURE_MAXTIME>
120 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
130 use FindBin
qw($RealBin $Script);
134 use Cwd qw(abs_path);
136 use Subunit
qw(parse_results);
140 my $opt_target = "samba4";
142 my $opt_socket_wrapper = 0;
143 my $opt_socket_wrapper_pcap = undef;
144 my $opt_socket_wrapper_keep_pcap = undef;
146 my $opt_immediate = 0;
147 my $opt_expected_failures = undef;
148 my @opt_exclude = ();
149 my @opt_include = ();
151 my $opt_image = undef;
154 my $opt_analyse_cmd = undef;
155 my $opt_resetup_env = undef;
156 my $opt_bindir = undef;
157 my $opt_no_lazy_setup = undef;
158 my $opt_format = "plain";
166 my @expected_failures = ();
173 TESTS_UNEXPECTED_OK
=> 0,
174 TESTS_EXPECTED_OK
=> 0,
175 TESTS_UNEXPECTED_FAIL
=> 0,
176 TESTS_EXPECTED_FAIL
=> 0,
183 my ($list, $fullname) = @_;
186 if ($fullname =~ /$$_[0]/) {
187 return ($$_[1]) if ($$_[1]);
188 return "NO REASON SPECIFIED";
195 sub expecting_failure
($)
198 return find_in_list
(\
@expected_failures, $name);
205 return find_in_list
(\
@excludes, $name);
214 return unless ($opt_socket_wrapper_pcap);
215 return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR
});
218 $fname =~ s
%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\
-]%_%g;
220 my $pcap_file = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
222 SocketWrapper
::setup_pcap
($pcap_file);
227 sub cleanup_pcap
($$$)
229 my ($pcap_file, $expected_ret, $ret) = @_;
231 return unless ($opt_socket_wrapper_pcap);
232 return if ($opt_socket_wrapper_keep_pcap);
233 return unless ($expected_ret == $ret);
234 return unless defined($pcap_file);
239 sub run_testsuite
($$$$$$)
241 my ($envname, $name, $cmd, $i, $totalsuites, $msg_ops) = @_;
242 my $pcap_file = setup_pcap
($name);
244 $msg_ops->start_test([], $name);
246 unless (open(RESULT
, "$cmd 2>&1|")) {
247 $statistics->{TESTS_ERROR
}++;
248 $msg_ops->end_test([], $name, "error", 1, "Unable to run $cmd: $!");
249 $statistics->{SUITES_FAIL
}++;
253 my $expected_ret = parse_results
(
254 $msg_ops, $statistics, *RESULT
, \
&expecting_failure
, [$name]);
256 my $envlog = getlog_env
($envname);
257 $msg_ops->output_msg("ENVLOG: $envlog\n") if ($envlog ne "");
259 $msg_ops->output_msg("CMD: $cmd\n");
261 my $ret = close(RESULT
);
262 $ret = 0 unless $ret == 1;
264 my $exitcode = $?
>> 8;
267 $msg_ops->end_test([], $name, "success", $expected_ret != $ret, undef);
269 $msg_ops->end_test([], $name, "failure", $expected_ret != $ret, "Exit code was $exitcode");
272 cleanup_pcap
($pcap_file, $expected_ret, $ret);
274 if (not $opt_socket_wrapper_keep_pcap and defined($pcap_file)) {
275 $msg_ops->output_msg("PCAP FILE: $pcap_file\n");
278 if ($ret != $expected_ret) {
279 $statistics->{SUITES_FAIL
}++;
280 exit(1) if ($opt_one);
283 return ($ret == $expected_ret);
288 print "Samba test runner
289 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
291 Usage: $Script [OPTIONS] TESTNAME-REGEX
294 --help this help page
295 --target=samba[34]|win|kvm Samba version to target
296 --testlist=FILE file to read available tests from
299 --prefix=DIR prefix to run tests in [st]
300 --srcdir=DIR source directory [.]
301 --builddir=DIR output directory [.]
302 --exeext=EXT executable extention []
305 --socket-wrapper-pcap save traffic to pcap directories
306 --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
308 --socket-wrapper enable socket wrapper
309 --bindir=PATH path to target binaries
310 --expected-failures=FILE specify list of tests that is guaranteed to fail
313 --ldap=openldap|fedora-ds back samba onto specified ldap server
316 --image=PATH path to KVM image
319 --quick run quick overall test
320 --one abort when the first test fails
321 --immediate print test output for failed tests during run
323 --analyse-cmd CMD command to run after each test
328 my $result = GetOptions
(
329 'help|h|?' => \
$opt_help,
330 'target=s' => \
$opt_target,
331 'prefix=s' => \
$prefix,
332 'socket-wrapper' => \
$opt_socket_wrapper,
333 'socket-wrapper-pcap' => \
$opt_socket_wrapper_pcap,
334 'socket-wrapper-keep-pcap' => \
$opt_socket_wrapper_keep_pcap,
335 'quick' => \
$opt_quick,
337 'immediate' => \
$opt_immediate,
338 'expected-failures=s' => \
$opt_expected_failures,
339 'exclude=s' => \
@opt_exclude,
340 'include=s' => \
@opt_include,
341 'srcdir=s' => \
$srcdir,
342 'builddir=s' => \
$builddir,
343 'exeext=s' => \
$exeext,
344 'verbose' => \
$opt_verbose,
345 'testenv' => \
$opt_testenv,
347 'analyse-cmd=s' => \
$opt_analyse_cmd,
348 'no-lazy-setup' => \
$opt_no_lazy_setup,
349 'resetup-environment' => \
$opt_resetup_env,
350 'bindir:s' => \
$opt_bindir,
351 'format=s' => \
$opt_format,
352 'image=s' => \
$opt_image,
353 'testlist=s' => \
@testlists
356 exit(1) if (not $result);
358 ShowHelp
() if ($opt_help);
362 # quick hack to disable rpc validation when using valgrind - its way too slow
363 unless (defined($ENV{VALGRIND
})) {
364 $ENV{VALIDATE
} = "validate";
365 $ENV{MALLOC_CHECK_
} = 2;
368 my $old_pwd = "$RealBin/..";
370 # Backwards compatibility:
371 if (defined($ENV{TEST_LDAP
}) and $ENV{TEST_LDAP
} eq "yes") {
372 if (defined($ENV{FEDORA_DS_ROOT
})) {
379 my $torture_maxtime = ($ENV{TORTURE_MAXTIME
} or 1200);
382 $torture_maxtime *= 2;
389 die("using an empty prefix isn't allowed") unless $prefix ne "";
391 #Ensure we have the test prefix around
392 mkdir($prefix, 0777) unless -d
$prefix;
394 my $prefix_abs = abs_path
($prefix);
395 my $srcdir_abs = abs_path
($srcdir);
397 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
398 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
400 $ENV{PREFIX
} = $prefix;
401 $ENV{KRB5CCNAME
} = "$prefix/krb5ticket";
402 $ENV{PREFIX_ABS
} = $prefix_abs;
403 $ENV{SRCDIR
} = $srcdir;
404 $ENV{SRCDIR_ABS
} = $srcdir_abs;
406 if (defined($ENV{RUN_FROM_BUILD_FARM
}) and
407 ($ENV{RUN_FROM_BUILD_FARM
} eq "yes")) {
408 $opt_format = "buildfarm";
411 my $tls_enabled = not $opt_quick;
412 $ENV{TLS_ENABLED
} = ($tls_enabled?
"yes":"no");
413 $ENV{LDB_MODULES_PATH
} = "$old_pwd/source4/bin/modules/ldb";
414 $ENV{LD_SAMBA_MODULE_PATH
} = "$old_pwd/source4/bin/modules";
415 sub prefix_pathvar
($$)
417 my ($name, $newpath) = @_;
418 if (defined($ENV{$name})) {
419 $ENV{$name} = "$newpath:$ENV{$name}";
421 $ENV{$name} = $newpath;
424 prefix_pathvar
("PKG_CONFIG_PATH", "$old_pwd/source4/bin/pkgconfig");
425 prefix_pathvar
("PYTHONPATH", "$old_pwd/source4/bin/python");
427 if ($opt_socket_wrapper_keep_pcap) {
428 # Socket wrapper keep pcap implies socket wrapper pcap
429 $opt_socket_wrapper_pcap = 1;
432 if ($opt_socket_wrapper_pcap) {
433 # Socket wrapper pcap implies socket wrapper
434 $opt_socket_wrapper = 1;
437 my $socket_wrapper_dir;
438 if ($opt_socket_wrapper) {
439 $socket_wrapper_dir = SocketWrapper
::setup_dir
("$prefix/w", $opt_socket_wrapper_pcap);
440 print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
443 print "WARNING: Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports\n";
448 my $testenv_default = "none";
450 if ($opt_target eq "samba4") {
451 $testenv_default = "member";
452 require target
::Samba4
;
453 $target = new Samba4
($opt_bindir or "$builddir/bin",
454 $ldap, "$srcdir/setup", $exeext);
455 } elsif ($opt_target eq "samba3") {
456 my $bindir = ($opt_bindir or "$builddir/bin");
457 if ($opt_socket_wrapper and `$bindir/smbd -b | grep SOCKET_WRAPPER` eq "") {
458 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'. Exiting....");
460 $testenv_default = "member";
461 require target
::Samba3
;
462 $target = new Samba3
($bindir);
463 } elsif ($opt_target eq "win") {
464 die("Windows tests will not run with socket wrapper enabled.")
465 if ($opt_socket_wrapper);
466 $testenv_default = "dc";
467 require target
::Windows
;
468 $target = new Windows
();
469 } elsif ($opt_target eq "kvm") {
470 die("Kvm tests will not run with socket wrapper enabled.")
471 if ($opt_socket_wrapper);
473 die("No image specified") unless ($opt_image);
474 $target = new Kvm
($opt_image, undef);
478 # Start a Virtual Distributed Ethernet Switch
479 # Returns the pid of the switch.
481 sub start_vde_switch
($)
485 system("vde_switch --pidfile $path/vde.pid --sock $path/vde.sock --daemon");
487 open(PID
, "$path/vde.pid");
495 # Stop a Virtual Distributed Ethernet Switch
496 sub stop_vde_switch
($)
502 sub read_test_regexes
($)
506 open(LF
, "<$name") or die("unable to read $name: $!");
510 if (/^(.*?)([ \t]+)\#([\t ]*)(.*?)$/) {
511 push (@ret, [$1, $4]);
513 s/^(.*?)([ \t]+)\#([\t ]*)(.*?)$//;
514 push (@ret, [$_, undef]);
521 if (defined($opt_expected_failures)) {
522 @expected_failures = read_test_regexes
($opt_expected_failures);
525 foreach (@opt_exclude) {
526 push (@excludes, read_test_regexes
($_));
529 foreach (@opt_include) {
530 push (@includes, read_test_regexes
($_));
533 my $interfaces = join(',', ("127.0.0.6/8",
540 my $conffile = "$prefix_abs/client/client.conf";
541 $ENV{SMB_CONF_PATH
} = $conffile;
543 sub write_clientconf
($$)
545 my ($conffile, $vars) = @_;
547 mkdir("$prefix/client", 0777) unless -d
"$prefix/client";
549 if ( -d
"$prefix/client/private" ) {
550 unlink <$prefix/client/private
/*>;
552 mkdir("$prefix/client/private", 0777);
555 if ( -d
"$prefix/client/lock" ) {
556 unlink <$prefix/client/lockdir
/*>;
558 mkdir("$prefix/client/lockdir", 0777);
561 open(CF
, ">$conffile");
562 print CF
"[global]\n";
563 if (defined($ENV{VALGRIND
})) {
564 print CF
"\ticonv:native = true\n";
566 print CF
"\ticonv:native = false\n";
568 print CF
"\tnetbios name = client\n";
569 if (defined($vars->{DOMAIN
})) {
570 print CF
"\tworkgroup = $vars->{DOMAIN}\n";
572 if (defined($vars->{REALM
})) {
573 print CF
"\trealm = $vars->{REALM}\n";
575 if (defined($vars->{NCALRPCDIR
})) {
576 print CF
"\tncalrpc dir = $vars->{NCALRPCDIR}\n";
578 if (defined($vars->{PIDDIR
})) {
579 print CF
"\tpid directory = $vars->{PIDDIR}\n";
581 if (defined($vars->{WINBINDD_SOCKET_DIR
})) {
582 print CF
"\twinbindd socket directory = $vars->{WINBINDD_SOCKET_DIR}\n";
583 print CF
"\twinbindd:socket dir = $vars->{WINBINDD_SOCKET_DIR}\n";
585 if ($opt_socket_wrapper) {
586 print CF
"\tinterfaces = $interfaces\n";
589 private dir = $prefix_abs/client/private
590 lock dir = $prefix_abs/client/lockdir
591 name resolve order = bcast
592 panic action = $RealBin/gdb_backtrace \%PID\% \%PROG\%
594 notify:inotify = false
596 system:anonymous = true
597 client lanman auth = Yes
598 torture:basedir = $prefix_abs/client
599 #We don't want to pass our self-tests if the PAC code is wrong
600 gensec:require_pac = true
601 modules dir = $ENV{LD_SAMBA_MODULE_PATH}
608 my $testsdir = "$srcdir/selftest";
610 my %required_envs = ();
617 open(IN
, $filename) or die("Unable to open $filename: $!");
620 if ($_ eq "-- TEST --\n") {
627 if (not defined($tests) or $name =~ /$tests/) {
628 $required_envs{$env} = 1;
629 push (@ret, [$name, $env, $cmdline]);
635 close(IN
) or die("Error creating recipe");
639 if ($#testlists == -1) {
640 die("No testlists specified");
643 $ENV{SELFTEST_PREFIX
} = "$prefix_abs";
644 if ($opt_socket_wrapper) {
645 $ENV{SELFTEST_INTERFACES
} = $interfaces;
647 $ENV{SELFTEST_INTERFACES
} = "";
650 $ENV{SELFTEST_VERBOSE
} = "1";
652 $ENV{SELFTEST_VERBOSE
} = "";
655 $ENV{SELFTEST_QUICK
} = "1";
657 $ENV{SELFTEST_QUICK
} = "";
659 $ENV{SELFTEST_TARGET
} = $opt_target;
660 $ENV{SELFTEST_MAXTIME
} = $torture_maxtime;
661 $ENV{SELFTEST_CONFFILE
} = $conffile;
664 foreach my $fn (@testlists) {
665 foreach (read_testlist
($fn)) {
667 next if (@includes and not find_in_list
(\
@includes, $name));
668 push (@available, $_);
673 if ($opt_format eq "buildfarm") {
674 require output
::buildfarm
;
675 $msg_ops = new output
::buildfarm
($statistics);
676 } elsif ($opt_format eq "plain") {
677 require output
::plain
;
678 $msg_ops = new output
::plain
("$prefix/summary", $opt_verbose, $opt_immediate, $statistics, $#available+1);
679 } elsif ($opt_format eq "html") {
680 require output
::html
;
681 mkdir("test-results", 0777);
682 $msg_ops = new output
::html
("test-results", $statistics);
684 die("Invalid output format '$opt_format'");
688 foreach (@available) {
690 my $skipreason = skip
($name);
692 $msg_ops->skip_testsuite($name, $skipreason);
699 print STDERR
"No tests to run\n";
703 my $suitestotal = $#todo + 1;
707 my %running_envs = ();
709 my @exported_envvars = (
714 # domain controller stuff
734 "WINBINDD_SOCKET_DIR",
735 "WINBINDD_PRIV_PIPE_DIR"
738 $SIG{INT
} = $SIG{QUIT
} = $SIG{TERM
} = sub {
740 teardown_env
($_) foreach(keys %running_envs);
741 die("Received signal $signame");
749 if ($envname eq "none") {
751 } elsif (defined($running_envs{$envname})) {
752 $testenv_vars = $running_envs{$envname};
753 if (not $target->check_env($testenv_vars)) {
754 $testenv_vars = undef;
757 $testenv_vars = $target->setup_env($envname, $prefix);
760 return undef unless defined($testenv_vars);
762 $running_envs{$envname} = $testenv_vars;
764 SocketWrapper
::set_default_iface
(6);
765 write_clientconf
($conffile, $testenv_vars);
767 foreach (@exported_envvars) {
768 if (defined($testenv_vars->{$_})) {
769 $ENV{$_} = $testenv_vars->{$_};
775 return $testenv_vars;
778 sub exported_envvars_str
($)
780 my ($testenv_vars) = @_;
783 foreach (@exported_envvars) {
784 next unless defined($testenv_vars->{$_});
785 $out .= $_."=".$testenv_vars->{$_}."\n";
794 return "" if ($envname eq "none");
795 return $target->getlog_env($running_envs{$envname});
801 return 1 if ($envname eq "none");
802 return $target->check_env($running_envs{$envname});
808 return if ($envname eq "none");
809 $target->teardown_env($running_envs{$envname});
810 delete $running_envs{$envname};
813 if ($opt_no_lazy_setup) {
814 setup_env
($_) foreach (keys %required_envs);
818 my $testenv_name = $ENV{SELFTEST_TESTENV
};
819 $testenv_name = $testenv_default unless defined($testenv_name);
821 my $testenv_vars = setup_env
($testenv_name);
823 $ENV{PIDDIR
} = $testenv_vars->{PIDDIR
};
825 my $envvarstr = exported_envvars_str
($testenv_vars);
827 my $term = ($ENV{TERM
} or "xterm");
828 system("$term -e 'echo -e \"
829 Welcome to the Samba4 Test environment '$testenv_name'
831 This matches the client environment used in make test
832 server is pid `cat \$PIDDIR/samba.pid`
834 Some useful environment variables:
835 TORTURE_OPTIONS=\$TORTURE_OPTIONS
836 CONFIGURATION=\$CONFIGURATION
839 \" && LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH} bash'");
840 teardown_env
($testenv_name);
845 $cmd =~ s/([\(\)])/\\$1/g;
847 my $envname = $$_[1];
849 my $envvars = setup_env
($envname);
850 if (not defined($envvars)) {
851 $msg_ops->skip_testsuite($name, "unable to set up environment $envname");
855 run_testsuite
($envname, $name, $cmd, $i, $suitestotal,
858 if (defined($opt_analyse_cmd)) {
859 system("$opt_analyse_cmd \"$name\"");
862 teardown_env
($envname) if ($opt_resetup_env);
868 teardown_env
($_) foreach (keys %running_envs);
876 # if there were any valgrind failures, show them
877 foreach (<$prefix/valgrind
.log*>) {
879 system("grep DWARF2.CFI.reader $_ > /dev/null");
881 print "VALGRIND FAILURE\n";
887 if ($opt_format eq "buildfarm") {
888 print "TEST STATUS: $statistics->{SUITES_FAIL}\n";
891 exit $statistics->{SUITES_FAIL
};