2 # Bootstrap Samba and run a number of tests against it.
3 # Copyright (C) 2005-2010 Jelmer Vernooij <jelmer@samba.org>
4 # Copyright (C) 2007-2009 Stefan Metzmacher <metze@samba.org>
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
23 selftest - Samba test runner
29 selftest [--srcdir=DIR] [--builddir=DIR] [--exeext=EXT][--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--testlist=FILE] [TESTS]
33 A simple test runner. TESTS is a regular expression with tests to run.
41 Show list of available options.
47 =item I<--builddir=DIR>
57 Change directory to run tests in. Default is 'st'.
59 =item I<--target samba4|samba3|win|kvm>
61 Specify test target against which to run. Default is 'samba4'.
65 Run only a limited number of tests. Intended to run in about 30 seconds on
66 moderately recent systems.
68 =item I<--socket-wrapper>
70 Use socket wrapper library for communication with server. Only works
71 when the server is running locally.
73 Will prevent TCP and UDP ports being opened on the local host but
74 (transparently) redirects these calls to use unix domain sockets.
78 Specify a file containing a list of tests that should be skipped. Possible
79 candidates are tests that segfault the server, flip or don't end.
83 Specify a file containing a list of tests that should be run. Same format
84 as the --exclude flag.
86 Not includes specified means all tests will be run.
90 Abort as soon as one test fails.
94 Load a list of tests from the specified location.
102 =item I<SMBD_VALGRIND>
104 =item I<TORTURE_MAXTIME>
116 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
126 use FindBin
qw($RealBin $Script);
128 use File::Temp qw(tempfile);
131 use Cwd
qw(abs_path);
140 use Time
::HiRes
qw(time);
144 my $opt_target = "samba4";
146 my $opt_socket_wrapper = 0;
147 my $opt_socket_wrapper_pcap = undef;
148 my $opt_socket_wrapper_keep_pcap = undef;
150 my @opt_exclude = ();
151 my @opt_include = ();
153 my $opt_image = undef;
156 my $opt_resetup_env = undef;
157 my $opt_bindir = undef;
158 my $opt_load_list = undef;
171 print STDERR
"Exiting early because of SIGPIPE.\n";
175 $SIG{PIPE
} = \
&pipe_handler
;
179 my ($list, $fullname) = @_;
182 if ($fullname =~ /$$_[0]/) {
183 return ($$_[1]) if ($$_[1]);
195 return find_in_list
(\
@excludes, $name);
204 return unless ($opt_socket_wrapper_pcap);
205 return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR
});
208 $fname =~ s
%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\
-]%_%g;
210 my $pcap_file = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
212 SocketWrapper
::setup_pcap
($pcap_file);
219 my ($pcap_file, $exitcode) = @_;
221 return unless ($opt_socket_wrapper_pcap);
222 return if ($opt_socket_wrapper_keep_pcap);
223 return unless ($exitcode == 0);
224 return unless defined($pcap_file);
229 # expand strings from %ENV
230 sub expand_environment_strings
($)
233 # we use a reverse sort so we do the longer ones first
234 foreach my $k (sort { $b cmp $a } keys %ENV) {
235 $s =~ s/\$$k/$ENV{$k}/g;
240 sub run_testsuite
($$$$$)
242 my ($envname, $name, $cmd, $i, $totalsuites) = @_;
243 my $pcap_file = setup_pcap
($name);
245 Subunit
::start_testsuite
($name);
246 Subunit
::progress_push
();
247 Subunit
::report_time
(time());
249 Subunit
::report_time
(time());
250 Subunit
::progress_pop
();
253 Subunit
::progress_pop
();
254 Subunit
::end_testsuite
($name, "error", "Unable to run $cmd: $!");
257 Subunit
::end_testsuite
($name, "error",
258 sprintf("%s died with signal %d, %s coredump\n", $cmd, ($?
& 127), ($?
& 128) ?
'with' : 'without'));
262 my $exitcode = $?
>> 8;
264 my $envlog = getlog_env
($envname);
266 print "envlog: $envlog\n";
269 print "command: $cmd\n";
270 printf "expanded command: %s\n", expand_environment_strings
($cmd);
272 if ($exitcode == 0) {
273 Subunit
::end_testsuite
($name, "success");
275 Subunit
::end_testsuite
($name, "failure", "Exit code was $exitcode");
278 cleanup_pcap
($pcap_file, $exitcode);
280 if (not $opt_socket_wrapper_keep_pcap and defined($pcap_file)) {
281 print "PCAP FILE: $pcap_file\n";
284 if ($exitcode != 0) {
285 exit(1) if ($opt_one);
293 print "Samba test runner
294 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
295 Copyright (C) Stefan Metzmacher <metze\@samba.org>
297 Usage: $Script [OPTIONS] TESTNAME-REGEX
300 --help this help page
301 --target=samba[34]|win|kvm Samba version to target
302 --testlist=FILE file to read available tests from
305 --prefix=DIR prefix to run tests in [st]
306 --srcdir=DIR source directory [.]
307 --builddir=DIR output directory [.]
308 --exeext=EXT executable extention []
311 --socket-wrapper-pcap save traffic to pcap directories
312 --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
314 --socket-wrapper enable socket wrapper
315 --bindir=PATH path to target binaries
318 --ldap=openldap|fedora-ds back samba onto specified ldap server
321 --image=PATH path to KVM image
324 --quick run quick overall test
325 --one abort when the first test fails
327 --analyse-cmd CMD command to run after each test
332 my $result = GetOptions
(
333 'help|h|?' => \
$opt_help,
334 'target=s' => \
$opt_target,
335 'prefix=s' => \
$prefix,
336 'socket-wrapper' => \
$opt_socket_wrapper,
337 'socket-wrapper-pcap' => \
$opt_socket_wrapper_pcap,
338 'socket-wrapper-keep-pcap' => \
$opt_socket_wrapper_keep_pcap,
339 'quick' => \
$opt_quick,
341 'exclude=s' => \
@opt_exclude,
342 'include=s' => \
@opt_include,
343 'srcdir=s' => \
$srcdir,
344 'builddir=s' => \
$builddir,
345 'exeext=s' => \
$exeext,
346 'verbose' => \
$opt_verbose,
347 'testenv' => \
$opt_testenv,
349 'resetup-environment' => \
$opt_resetup_env,
350 'bindir:s' => \
$opt_bindir,
351 'image=s' => \
$opt_image,
352 'testlist=s' => \
@testlists,
353 'load-list=s' => \
$opt_load_list,
356 exit(1) if (not $result);
358 ShowHelp
() if ($opt_help);
360 # we want unbuffered output
365 # quick hack to disable rpc validation when using valgrind - its way too slow
366 unless (defined($ENV{VALGRIND
})) {
367 $ENV{VALIDATE
} = "validate";
368 $ENV{MALLOC_CHECK_
} = 2;
371 # make all our python scripts unbuffered
372 $ENV{PYTHONUNBUFFERED
} = 1;
374 my $bindir = ($opt_bindir or "$builddir/bin");
375 my $bindir_abs = abs_path
($bindir);
377 # Backwards compatibility:
378 if (defined($ENV{TEST_LDAP
}) and $ENV{TEST_LDAP
} eq "yes") {
379 if (defined($ENV{FEDORA_DS_ROOT
})) {
386 my $torture_maxtime = ($ENV{TORTURE_MAXTIME
} or 1200);
389 $torture_maxtime *= 2;
396 die("using an empty prefix isn't allowed") unless $prefix ne "";
398 #Ensure we have the test prefix around
399 mkdir($prefix, 0777) unless -d
$prefix;
401 my $prefix_abs = abs_path
($prefix);
402 my $srcdir_abs = abs_path
($srcdir);
403 my $builddir_abs = abs_path
($builddir);
405 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
406 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
408 $ENV{PREFIX
} = $prefix;
409 $ENV{KRB5CCNAME
} = "$prefix/krb5ticket";
410 $ENV{PREFIX_ABS
} = $prefix_abs;
411 $ENV{SRCDIR
} = $srcdir;
412 $ENV{SRCDIR_ABS
} = $srcdir_abs;
413 $ENV{BUILDDIR
} = $builddir;
414 $ENV{BUILDDIR_ABS
} = $builddir_abs;
415 $ENV{EXEEXT
} = $exeext;
417 my $tls_enabled = not $opt_quick;
418 $ENV{TLS_ENABLED
} = ($tls_enabled?
"yes":"no");
419 $ENV{LDB_MODULES_PATH
} = "$bindir_abs/modules/ldb";
420 $ENV{LD_SAMBA_MODULE_PATH
} = "$bindir_abs/modules";
421 sub prefix_pathvar
($$)
423 my ($name, $newpath) = @_;
424 if (defined($ENV{$name})) {
425 $ENV{$name} = "$newpath:$ENV{$name}";
427 $ENV{$name} = $newpath;
430 prefix_pathvar
("PKG_CONFIG_PATH", "$bindir_abs/pkgconfig");
431 prefix_pathvar
("PYTHONPATH", "$bindir_abs/python");
433 if ($opt_socket_wrapper_keep_pcap) {
434 # Socket wrapper keep pcap implies socket wrapper pcap
435 $opt_socket_wrapper_pcap = 1;
438 if ($opt_socket_wrapper_pcap) {
439 # Socket wrapper pcap implies socket wrapper
440 $opt_socket_wrapper = 1;
443 my $socket_wrapper_dir;
444 if ($opt_socket_wrapper) {
445 $socket_wrapper_dir = SocketWrapper
::setup_dir
("$prefix_abs/w", $opt_socket_wrapper_pcap);
446 print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
449 print "WARNING: Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports\n";
454 my $testenv_default = "none";
456 if ($opt_target eq "samba4") {
457 $testenv_default = "all";
458 require target
::Samba4
;
459 $target = new Samba4
($bindir, $ldap, "$srcdir/setup", $exeext);
460 } elsif ($opt_target eq "samba3") {
461 if ($opt_socket_wrapper and `$bindir/smbd -b | grep SOCKET_WRAPPER` eq "") {
462 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'. Exiting....");
464 $testenv_default = "member";
465 require target
::Samba3
;
466 $target = new Samba3
($bindir);
467 } elsif ($opt_target eq "win") {
468 die("Windows tests will not run with socket wrapper enabled.")
469 if ($opt_socket_wrapper);
470 $testenv_default = "dc";
471 require target
::Windows
;
472 $target = new Windows
();
473 } elsif ($opt_target eq "kvm") {
474 die("Kvm tests will not run with socket wrapper enabled.")
475 if ($opt_socket_wrapper);
477 die("No image specified") unless ($opt_image);
478 $target = new Kvm
($opt_image, undef);
482 # Start a Virtual Distributed Ethernet Switch
483 # Returns the pid of the switch.
485 sub start_vde_switch
($)
489 system("vde_switch --pidfile $path/vde.pid --sock $path/vde.sock --daemon");
491 open(PID
, "$path/vde.pid");
499 # Stop a Virtual Distributed Ethernet Switch
500 sub stop_vde_switch
($)
506 sub read_test_regexes
($)
510 open(LF
, "<$name") or die("unable to read $name: $!");
514 if (/^(.*?)([ \t]+)\#([\t ]*)(.*?)$/) {
515 push (@ret, [$1, $4]);
517 s/^(.*?)([ \t]+)\#([\t ]*)(.*?)$//;
518 push (@ret, [$_, undef]);
525 foreach (@opt_exclude) {
526 push (@excludes, read_test_regexes
($_));
529 foreach (@opt_include) {
530 push (@includes, read_test_regexes
($_));
533 my $interfaces = join(',', ("127.0.0.11/8",
540 my $clientdir = "$prefix_abs/client";
542 my $conffile = "$clientdir/client.conf";
543 $ENV{SMB_CONF_PATH
} = $conffile;
545 sub write_clientconf
($$$)
547 my ($conffile, $clientdir, $vars) = @_;
549 mkdir("$clientdir", 0777) unless -d
"$clientdir";
551 if ( -d
"$clientdir/private" ) {
552 unlink <$clientdir/private/*>;
554 mkdir("$clientdir/private", 0777);
557 if ( -d
"$clientdir/lockdir" ) {
558 unlink <$clientdir/lockdir/*>;
560 mkdir("$clientdir/lockdir", 0777);
563 if ( -d
"$clientdir/ncalrpcdir" ) {
564 unlink <$clientdir/ncalrpcdir/*>;
566 mkdir("$clientdir/ncalrpcdir", 0777);
569 open(CF
, ">$conffile");
570 print CF
"[global]\n";
571 if (defined($ENV{VALGRIND
})) {
572 print CF
"\ticonv:native = true\n";
574 print CF
"\ticonv:native = false\n";
576 print CF
"\tnetbios name = client\n";
577 if (defined($vars->{DOMAIN
})) {
578 print CF
"\tworkgroup = $vars->{DOMAIN}\n";
580 if (defined($vars->{REALM
})) {
581 print CF
"\trealm = $vars->{REALM}\n";
583 if ($opt_socket_wrapper) {
584 print CF
"\tinterfaces = $interfaces\n";
587 private dir = $clientdir/private
588 lock dir = $clientdir/lockdir
589 ncalrpc dir = $clientdir/ncalrpcdir
590 name resolve order = bcast file
591 panic action = $RealBin/gdb_backtrace \%PID\% \%PROG\%
593 notify:inotify = false
595 system:anonymous = true
596 client lanman auth = Yes
598 torture:basedir = $clientdir
599 #We don't want to pass our self-tests if the PAC code is wrong
600 gensec:require_pac = true
601 modules dir = $ENV{LD_SAMBA_MODULE_PATH}
602 setup directory = ./setup
603 resolv:host file = $prefix_abs/dns_host_file
604 #We don't want to run 'speed' tests for very long
605 torture:timelimit = 1
612 my $testsdir = "$srcdir/selftest";
614 sub should_run_test
($)
620 for (my $i=0; $i <= $#tests; $i++) {
621 if ($name =~ /$tests[$i]/i) {
633 open(IN
, $filename) or die("Unable to open $filename: $!");
636 if (/-- TEST(-LOADLIST|-IDLIST|) --\n/) {
637 my $supports_loadlist = (defined($1) and $1 eq "-LOADLIST");
638 my $supports_idlist = (defined($1) and $1 eq "-IDLIST");
645 if (should_run_test
($name) == 1) {
646 push (@ret, [$name, $env, $cmdline, $supports_loadlist, $supports_idlist]);
652 close(IN
) or die("Error creating recipe");
656 if ($#testlists == -1) {
657 die("No testlists specified");
660 $ENV{SELFTEST_PREFIX
} = "$prefix_abs";
661 if ($opt_socket_wrapper) {
662 $ENV{SELFTEST_INTERFACES
} = $interfaces;
664 $ENV{SELFTEST_INTERFACES
} = "";
667 $ENV{SELFTEST_VERBOSE
} = "1";
669 $ENV{SELFTEST_VERBOSE
} = "";
672 $ENV{SELFTEST_QUICK
} = "1";
674 $ENV{SELFTEST_QUICK
} = "";
676 $ENV{SELFTEST_TARGET
} = $opt_target;
677 $ENV{SELFTEST_MAXTIME
} = $torture_maxtime;
680 foreach my $fn (@testlists) {
681 foreach (read_testlist
($fn)) {
683 next if (@includes and not defined(find_in_list
(\
@includes, $name)));
684 push (@available, $_);
688 my $restricted = undef;
689 my $restricted_used = {};
691 if ($opt_load_list) {
693 open(LOAD_LIST
, "<$opt_load_list") or die("Unable to open $opt_load_list");
694 while (<LOAD_LIST
>) {
696 push (@
$restricted, $_);
701 my $individual_tests = undef;
702 $individual_tests = {};
704 foreach my $testsuite (@available) {
705 my $name = $$testsuite[0];
706 my $skipreason = skip
($name);
707 if (defined($restricted)) {
708 # Find the testsuite for this test
710 foreach my $r (@
$restricted) {
712 $individual_tests->{$name} = [];
714 $restricted_used->{$r} = 1;
715 } elsif (substr($r, 0, length($name)+1) eq "$name.") {
716 push(@
{$individual_tests->{$name}}, $r);
718 $restricted_used->{$r} = 1;
722 if (defined($skipreason)) {
723 Subunit
::skip_testsuite
($name, $skipreason);
725 push(@todo, $testsuite);
728 } elsif (defined($skipreason)) {
729 Subunit
::skip_testsuite
($name, $skipreason);
731 push(@todo, $testsuite);
735 if (defined($restricted)) {
736 foreach (@
$restricted) {
737 unless (defined($restricted_used->{$_})) {
738 print "No test or testsuite found matching $_\n";
741 } elsif ($#todo == -1) {
742 print STDERR
"No tests to run\n";
746 my $suitestotal = $#todo + 1;
748 Subunit
::progress
($suitestotal);
749 Subunit
::report_time
(time());
754 my %running_envs = ();
756 sub get_running_env
($)
764 return $running_envs{$envname};
767 my @exported_envvars = (
772 # domain controller stuff
781 "MEMBER_NETBIOSNAME",
782 "MEMBER_NETBIOSALIAS",
784 # rpc proxy controller stuff
786 "RPC_PROXY_SERVER_IP",
787 "RPC_PROXY_NETBIOSNAME",
788 "RPC_PROXY_NETBIOSALIAS",
790 # domain controller stuff for Vampired DC
792 "VAMPIRE_DC_SERVER_IP",
793 "VAMPIRE_DC_NETBIOSNAME",
794 "VAMPIRE_DC_NETBIOSALIAS",
811 "WINBINDD_SOCKET_DIR",
812 "WINBINDD_PRIV_PIPE_DIR",
816 $SIG{INT
} = $SIG{QUIT
} = $SIG{TERM
} = sub {
818 teardown_env
($_) foreach(keys %running_envs);
819 die("Received signal $signame");
824 my ($name, $prefix) = @_;
826 my $testenv_vars = undef;
832 $option =~ s/^[^:]*//;
835 $option = "client" if $option eq "";
837 if ($envname eq "none") {
839 } elsif (defined(get_running_env
($envname))) {
840 $testenv_vars = get_running_env
($envname);
841 if (not $target->check_env($testenv_vars)) {
842 print $target->getlog_env($testenv_vars);
843 $testenv_vars = undef;
846 $testenv_vars = $target->setup_env($envname, $prefix);
849 return undef unless defined($testenv_vars);
851 $running_envs{$envname} = $testenv_vars;
853 if ($option eq "local") {
854 SocketWrapper
::set_default_iface
($testenv_vars->{SOCKET_WRAPPER_DEFAULT_IFACE
});
855 $ENV{SMB_CONF_PATH
} = $testenv_vars->{SERVERCONFFILE
};
856 } elsif ($option eq "client") {
857 SocketWrapper
::set_default_iface
(11);
858 write_clientconf
($conffile, $clientdir, $testenv_vars);
859 $ENV{SMB_CONF_PATH
} = $conffile;
861 die("Unknown option[$option] for envname[$envname]");
864 foreach (@exported_envvars) {
865 if (defined($testenv_vars->{$_})) {
866 $ENV{$_} = $testenv_vars->{$_};
872 return $testenv_vars;
875 sub exported_envvars_str
($)
877 my ($testenv_vars) = @_;
880 foreach (@exported_envvars) {
881 next unless defined($testenv_vars->{$_});
882 $out .= $_."=".$testenv_vars->{$_}."\n";
891 return "" if ($envname eq "none");
892 return $target->getlog_env(get_running_env
($envname));
898 return 1 if ($envname eq "none");
899 return $target->check_env(get_running_env
($envname));
905 return if ($envname eq "none");
906 $target->teardown_env(get_running_env
($envname));
907 delete $running_envs{$envname};
910 # This 'global' file needs to be empty when we start
911 unlink("$prefix_abs/dns_host_file");
914 my $testenv_name = $ENV{SELFTEST_TESTENV
};
915 $testenv_name = $testenv_default unless defined($testenv_name);
917 my $testenv_vars = setup_env
($testenv_name, $prefix);
919 die("Unable to setup environment $testenv_name") unless ($testenv_vars);
921 $ENV{PIDDIR
} = $testenv_vars->{PIDDIR
};
923 my $envvarstr = exported_envvars_str
($testenv_vars);
925 my $term = ($ENV{TERMINAL
} or "xterm");
926 system("$term -e 'echo -e \"
927 Welcome to the Samba4 Test environment '$testenv_name'
929 This matches the client environment used in make test
930 server is pid `cat \$PIDDIR/samba.pid`
932 Some useful environment variables:
933 TORTURE_OPTIONS=\$TORTURE_OPTIONS
934 SMB_CONF_PATH=\$SMB_CONF_PATH
937 \" && LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH} bash'");
938 teardown_env
($testenv_name);
944 my $envname = $$_[1];
946 my $envvars = setup_env
($envname, $prefix);
947 if (not defined($envvars)) {
948 Subunit
::start_testsuite
($name);
949 Subunit
::end_testsuite
($name, "error",
950 "unable to set up environment $envname");
954 # Generate a file with the individual tests to run, if the
955 # test runner for this test suite supports it.
956 if ($individual_tests and $individual_tests->{$name}) {
958 my ($fh, $listid_file) = tempfile
(UNLINK
=> 0);
959 foreach my $test (@
{$individual_tests->{$name}}) {
960 print $fh substr($test, length($name)+1) . "\n";
962 $cmd =~ s/\$LOADLIST/--load-list=$listid_file/g;
964 $cmd =~ s/\s+[^\s]+\s*$//;
965 $cmd .= " " . join(' ', @
{$individual_tests->{$name}});
969 run_testsuite
($envname, $name, $cmd, $i, $suitestotal);
971 teardown_env
($envname) if ($opt_resetup_env);
977 teardown_env
($_) foreach (keys %running_envs);
981 # if there were any valgrind failures, show them
982 foreach (<$prefix/valgrind
.log*>) {
984 print "VALGRIND FAILURE\n";