2 # Bootstrap Samba and run a number of tests against it.
3 # Copyright (C) 2005-2009 Jelmer Vernooij <jelmer@samba.org>
4 # Copyright (C) 2007-2009 Stefan Metzmacher <metze@samba.org>
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
23 selftest - Samba test runner
29 selftest [--srcdir=DIR] [--builddir=DIR] [--exeext=EXT][--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--testlist=FILE] [TESTS]
33 A simple test runner. TESTS is a regular expression with tests to run.
41 Show list of available options.
47 =item I<--builddir=DIR>
57 Change directory to run tests in. Default is 'st'.
59 =item I<--target samba4|samba3|win|kvm>
61 Specify test target against which to run. Default is 'samba4'.
65 Run only a limited number of tests. Intended to run in about 30 seconds on
66 moderately recent systems.
68 =item I<--socket-wrapper>
70 Use socket wrapper library for communication with server. Only works
71 when the server is running locally.
73 Will prevent TCP and UDP ports being opened on the local host but
74 (transparently) redirects these calls to use unix domain sockets.
78 Specify a file containing a list of tests that should be skipped. Possible
79 candidates are tests that segfault the server, flip or don't end.
83 Specify a file containing a list of tests that should be run. Same format
84 as the --exclude flag.
86 Not includes specified means all tests will be run.
90 Abort as soon as one test fails.
94 Load a list of tests from the specified location.
102 =item I<SMBD_VALGRIND>
104 =item I<TORTURE_MAXTIME>
116 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
126 use FindBin
qw($RealBin $Script);
128 use File::Temp qw(tempfile);
131 use Cwd
qw(abs_path);
140 use Time
::HiRes
qw(time);
144 my $opt_target = "samba4";
146 my $opt_socket_wrapper = 0;
147 my $opt_socket_wrapper_pcap = undef;
148 my $opt_socket_wrapper_keep_pcap = undef;
150 my @opt_exclude = ();
151 my @opt_include = ();
153 my $opt_image = undef;
156 my $opt_analyse_cmd = undef;
157 my $opt_resetup_env = undef;
158 my $opt_bindir = undef;
159 my $opt_load_list = undef;
172 my ($list, $fullname) = @_;
175 if ($fullname =~ /$$_[0]/) {
176 return ($$_[1]) if ($$_[1]);
188 return find_in_list
(\
@excludes, $name);
197 return unless ($opt_socket_wrapper_pcap);
198 return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR
});
201 $fname =~ s
%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\
-]%_%g;
203 my $pcap_file = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
205 SocketWrapper
::setup_pcap
($pcap_file);
212 my ($pcap_file, $exitcode) = @_;
214 return unless ($opt_socket_wrapper_pcap);
215 return if ($opt_socket_wrapper_keep_pcap);
216 return unless ($exitcode == 0);
217 return unless defined($pcap_file);
222 # expand strings from %ENV
223 sub expand_environment_strings
($)
226 # we use a reverse sort so we do the longer ones first
227 foreach my $k (sort { $b cmp $a } keys %ENV) {
228 $s =~ s/\$$k/$ENV{$k}/g;
233 sub run_testsuite
($$$$$)
235 my ($envname, $name, $cmd, $i, $totalsuites) = @_;
236 my $pcap_file = setup_pcap
($name);
238 Subunit
::start_testsuite
($name);
239 Subunit
::progress_push
();
240 Subunit
::report_time
(time());
242 open(RESULTS
, "$cmd 2>&1|");
244 Subunit
::filter_add_prefix
("$name\.", *RESULTS
);
248 unless (close(RESULTS
)) {
250 Subunit
::progress_pop
();
251 Subunit
::end_testsuite
($name, "error", "Unable to run $cmd: $!");
259 Subunit
::progress_pop
();
260 Subunit
::end_testsuite
($name, "error", sprintf("Testsuite died with signal %d, %s coredump", ($ret & 127), ($ret & 128) ?
"with": "without"));
263 my $envlog = getlog_env
($envname);
265 print "envlog: $envlog\n";
268 print "command: $cmd\n";
269 printf "expanded command: %s\n", expand_environment_strings
($cmd);
271 my $exitcode = $ret >> 8;
273 Subunit
::report_time
(time());
274 Subunit
::progress_pop
();
275 if ($exitcode == 0) {
276 Subunit
::end_testsuite
($name, "success");
278 Subunit
::end_testsuite
($name, "failure", "Exit code was $exitcode");
281 cleanup_pcap
($pcap_file, $exitcode);
283 if (not $opt_socket_wrapper_keep_pcap and defined($pcap_file)) {
284 print "PCAP FILE: $pcap_file\n";
287 if ($exitcode != 0) {
288 exit(1) if ($opt_one);
296 print "Samba test runner
297 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
298 Copyright (C) Stefan Metzmacher <metze\@samba.org>
300 Usage: $Script [OPTIONS] TESTNAME-REGEX
303 --help this help page
304 --target=samba[34]|win|kvm Samba version to target
305 --testlist=FILE file to read available tests from
308 --prefix=DIR prefix to run tests in [st]
309 --srcdir=DIR source directory [.]
310 --builddir=DIR output directory [.]
311 --exeext=EXT executable extention []
314 --socket-wrapper-pcap save traffic to pcap directories
315 --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
317 --socket-wrapper enable socket wrapper
318 --bindir=PATH path to target binaries
321 --ldap=openldap|fedora-ds back samba onto specified ldap server
324 --image=PATH path to KVM image
327 --quick run quick overall test
328 --one abort when the first test fails
330 --analyse-cmd CMD command to run after each test
335 my $result = GetOptions
(
336 'help|h|?' => \
$opt_help,
337 'target=s' => \
$opt_target,
338 'prefix=s' => \
$prefix,
339 'socket-wrapper' => \
$opt_socket_wrapper,
340 'socket-wrapper-pcap' => \
$opt_socket_wrapper_pcap,
341 'socket-wrapper-keep-pcap' => \
$opt_socket_wrapper_keep_pcap,
342 'quick' => \
$opt_quick,
344 'exclude=s' => \
@opt_exclude,
345 'include=s' => \
@opt_include,
346 'srcdir=s' => \
$srcdir,
347 'builddir=s' => \
$builddir,
348 'exeext=s' => \
$exeext,
349 'verbose' => \
$opt_verbose,
350 'testenv' => \
$opt_testenv,
352 'analyse-cmd=s' => \
$opt_analyse_cmd,
353 'resetup-environment' => \
$opt_resetup_env,
354 'bindir:s' => \
$opt_bindir,
355 'image=s' => \
$opt_image,
356 'testlist=s' => \
@testlists,
357 'load-list=s' => \
$opt_load_list,
360 exit(1) if (not $result);
362 ShowHelp
() if ($opt_help);
364 # we want unbuffered output
369 # quick hack to disable rpc validation when using valgrind - its way too slow
370 unless (defined($ENV{VALGRIND
})) {
371 $ENV{VALIDATE
} = "validate";
372 $ENV{MALLOC_CHECK_
} = 2;
375 # make all our python scripts unbuffered
376 $ENV{PYTHONUNBUFFERED
} = 1;
378 my $bindir = ($opt_bindir or "$builddir/bin");
379 my $bindir_abs = abs_path
($bindir);
381 # Backwards compatibility:
382 if (defined($ENV{TEST_LDAP
}) and $ENV{TEST_LDAP
} eq "yes") {
383 if (defined($ENV{FEDORA_DS_ROOT
})) {
390 my $torture_maxtime = ($ENV{TORTURE_MAXTIME
} or 1200);
393 $torture_maxtime *= 2;
400 die("using an empty prefix isn't allowed") unless $prefix ne "";
402 #Ensure we have the test prefix around
403 mkdir($prefix, 0777) unless -d
$prefix;
405 my $prefix_abs = abs_path
($prefix);
406 my $srcdir_abs = abs_path
($srcdir);
407 my $builddir_abs = abs_path
($builddir);
409 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
410 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
412 $ENV{PREFIX
} = $prefix;
413 $ENV{KRB5CCNAME
} = "$prefix/krb5ticket";
414 $ENV{PREFIX_ABS
} = $prefix_abs;
415 $ENV{SRCDIR
} = $srcdir;
416 $ENV{SRCDIR_ABS
} = $srcdir_abs;
417 $ENV{BUILDDIR
} = $builddir;
418 $ENV{BUILDDIR_ABS
} = $builddir_abs;
419 $ENV{EXEEXT
} = $exeext;
421 my $tls_enabled = not $opt_quick;
422 $ENV{TLS_ENABLED
} = ($tls_enabled?
"yes":"no");
423 $ENV{LDB_MODULES_PATH
} = "$bindir_abs/modules/ldb";
424 $ENV{LD_SAMBA_MODULE_PATH
} = "$bindir_abs/modules";
425 sub prefix_pathvar
($$)
427 my ($name, $newpath) = @_;
428 if (defined($ENV{$name})) {
429 $ENV{$name} = "$newpath:$ENV{$name}";
431 $ENV{$name} = $newpath;
434 prefix_pathvar
("PKG_CONFIG_PATH", "$bindir_abs/pkgconfig");
435 prefix_pathvar
("PYTHONPATH", "$bindir_abs/python");
437 if ($opt_socket_wrapper_keep_pcap) {
438 # Socket wrapper keep pcap implies socket wrapper pcap
439 $opt_socket_wrapper_pcap = 1;
442 if ($opt_socket_wrapper_pcap) {
443 # Socket wrapper pcap implies socket wrapper
444 $opt_socket_wrapper = 1;
447 my $socket_wrapper_dir;
448 if ($opt_socket_wrapper) {
449 $socket_wrapper_dir = SocketWrapper
::setup_dir
("$prefix_abs/w", $opt_socket_wrapper_pcap);
450 print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
453 print "WARNING: Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports\n";
458 my $testenv_default = "none";
460 if ($opt_target eq "samba4") {
461 $testenv_default = "all";
462 require target
::Samba4
;
463 $target = new Samba4
($bindir, $ldap, "$srcdir/setup", $exeext);
464 } elsif ($opt_target eq "samba3") {
465 if ($opt_socket_wrapper and `$bindir/smbd -b | grep SOCKET_WRAPPER` eq "") {
466 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'. Exiting....");
468 $testenv_default = "member";
469 require target
::Samba3
;
470 $target = new Samba3
($bindir);
471 } elsif ($opt_target eq "win") {
472 die("Windows tests will not run with socket wrapper enabled.")
473 if ($opt_socket_wrapper);
474 $testenv_default = "dc";
475 require target
::Windows
;
476 $target = new Windows
();
477 } elsif ($opt_target eq "kvm") {
478 die("Kvm tests will not run with socket wrapper enabled.")
479 if ($opt_socket_wrapper);
481 die("No image specified") unless ($opt_image);
482 $target = new Kvm
($opt_image, undef);
486 # Start a Virtual Distributed Ethernet Switch
487 # Returns the pid of the switch.
489 sub start_vde_switch
($)
493 system("vde_switch --pidfile $path/vde.pid --sock $path/vde.sock --daemon");
495 open(PID
, "$path/vde.pid");
503 # Stop a Virtual Distributed Ethernet Switch
504 sub stop_vde_switch
($)
510 sub read_test_regexes
($)
514 open(LF
, "<$name") or die("unable to read $name: $!");
518 if (/^(.*?)([ \t]+)\#([\t ]*)(.*?)$/) {
519 push (@ret, [$1, $4]);
521 s/^(.*?)([ \t]+)\#([\t ]*)(.*?)$//;
522 push (@ret, [$_, undef]);
529 foreach (@opt_exclude) {
530 push (@excludes, read_test_regexes
($_));
533 foreach (@opt_include) {
534 push (@includes, read_test_regexes
($_));
537 my $interfaces = join(',', ("127.0.0.11/8",
544 my $conffile = "$prefix_abs/client/client.conf";
545 $ENV{SMB_CONF_PATH
} = $conffile;
547 sub write_clientconf
($$)
549 my ($conffile, $vars) = @_;
551 mkdir("$prefix/client", 0777) unless -d
"$prefix/client";
553 if ( -d
"$prefix/client/private" ) {
554 unlink <$prefix/client/private
/*>;
556 mkdir("$prefix/client/private", 0777);
559 if ( -d
"$prefix/client/lockdir" ) {
560 unlink <$prefix/client/lockdir
/*>;
562 mkdir("$prefix/client/lockdir", 0777);
565 if ( -d
"$prefix_abs/client/ncalrpcdir" ) {
566 unlink <$prefix/client/ncalrpcdir
/*>;
568 mkdir("$prefix/client/ncalrpcdir", 0777);
571 open(CF
, ">$conffile");
572 print CF
"[global]\n";
573 if (defined($ENV{VALGRIND
})) {
574 print CF
"\ticonv:native = true\n";
576 print CF
"\ticonv:native = false\n";
578 print CF
"\tnetbios name = client\n";
579 if (defined($vars->{DOMAIN
})) {
580 print CF
"\tworkgroup = $vars->{DOMAIN}\n";
582 if (defined($vars->{REALM
})) {
583 print CF
"\trealm = $vars->{REALM}\n";
585 if ($opt_socket_wrapper) {
586 print CF
"\tinterfaces = $interfaces\n";
589 private dir = $prefix_abs/client/private
590 lock dir = $prefix_abs/client/lockdir
591 ncalrpc dir = $prefix_abs/client/ncalrpcdir
592 name resolve order = bcast file
593 panic action = $RealBin/gdb_backtrace \%PID\% \%PROG\%
595 notify:inotify = false
597 system:anonymous = true
598 client lanman auth = Yes
600 torture:basedir = $prefix_abs/client
601 #We don't want to pass our self-tests if the PAC code is wrong
602 gensec:require_pac = true
603 modules dir = $ENV{LD_SAMBA_MODULE_PATH}
604 setup directory = ./setup
605 resolv:host file = $prefix_abs/dns_host_file
612 my $testsdir = "$srcdir/selftest";
614 sub should_run_test
($)
620 for (my $i=0; $i <= $#tests; $i++) {
621 if ($name =~ /$tests[$i]/i) {
633 open(IN
, $filename) or die("Unable to open $filename: $!");
636 if (/-- TEST(-LOADLIST)? --\n/) {
637 my $supports_loadlist = (defined($1) and $1 eq "-LOADLIST");
644 if (should_run_test
($name) == 1) {
645 push (@ret, [$name, $env, $cmdline, $supports_loadlist]);
651 close(IN
) or die("Error creating recipe");
655 if ($#testlists == -1) {
656 die("No testlists specified");
659 $ENV{SELFTEST_PREFIX
} = "$prefix_abs";
660 if ($opt_socket_wrapper) {
661 $ENV{SELFTEST_INTERFACES
} = $interfaces;
663 $ENV{SELFTEST_INTERFACES
} = "";
666 $ENV{SELFTEST_VERBOSE
} = "1";
668 $ENV{SELFTEST_VERBOSE
} = "";
671 $ENV{SELFTEST_QUICK
} = "1";
673 $ENV{SELFTEST_QUICK
} = "";
675 $ENV{SELFTEST_TARGET
} = $opt_target;
676 $ENV{SELFTEST_MAXTIME
} = $torture_maxtime;
679 foreach my $fn (@testlists) {
680 foreach (read_testlist
($fn)) {
682 next if (@includes and not defined(find_in_list
(\
@includes, $name)));
683 push (@available, $_);
687 my $restricted = undef;
688 my $restricted_used = {};
690 if ($opt_load_list) {
692 open(LOAD_LIST
, "<$opt_load_list") or die("Unable to open $opt_load_list");
693 while (<LOAD_LIST
>) {
695 push (@
$restricted, $_);
700 Subunit
::progress
($#available+1);
701 Subunit
::report_time
(time());
703 my $individual_tests = undef;
704 $individual_tests = {};
706 foreach my $testsuite (@available) {
707 my $name = $$testsuite[0];
708 my $skipreason = skip
($name);
709 if (defined($skipreason)) {
710 Subunit
::skip_testsuite
($name, $skipreason);
711 } elsif (defined($restricted)) {
712 # Find the testsuite for this test
714 foreach my $r (@
$restricted) {
716 $individual_tests->{$name} = [];
718 $restricted_used->{$r} = 1;
719 } elsif (substr($r, $name, length($name)+1) eq "$name.") {
720 push(@
{$individual_tests->{$name}}, $1);
722 $restricted_used->{$r} = 1;
725 push(@todo, $testsuite) if ($match);
727 push(@todo, $testsuite);
731 if (defined($restricted)) {
732 foreach (@
$restricted) {
733 unless (defined($restricted_used->{$_})) {
734 print "No test or testsuite found matching $_\n";
737 } elsif ($#todo == -1) {
738 print STDERR
"No tests to run\n";
742 my $suitestotal = $#todo + 1;
746 my %running_envs = ();
748 sub get_running_env
($)
756 return $running_envs{$envname};
759 my @exported_envvars = (
764 # domain controller stuff
773 "MEMBER_NETBIOSNAME",
774 "MEMBER_NETBIOSALIAS",
776 # rpc proxy controller stuff
778 "RPC_PROXY_SERVER_IP",
779 "RPC_PROXY_NETBIOSNAME",
780 "RPC_PROXY_NETBIOSALIAS",
782 # domain controller stuff for Vampired DC
784 "VAMPIRE_DC_SERVER_IP",
785 "VAMPIRE_DC_NETBIOSNAME",
786 "VAMPIRE_DC_NETBIOSALIAS",
803 "WINBINDD_SOCKET_DIR",
804 "WINBINDD_PRIV_PIPE_DIR",
808 $SIG{INT
} = $SIG{QUIT
} = $SIG{TERM
} = sub {
810 teardown_env
($_) foreach(keys %running_envs);
811 die("Received signal $signame");
818 my $testenv_vars = undef;
824 $option =~ s/^[^:]*//;
827 $option = "client" if $option eq "";
829 if ($envname eq "none") {
831 } elsif (defined(get_running_env
($envname))) {
832 $testenv_vars = get_running_env
($envname);
833 if (not $target->check_env($testenv_vars)) {
834 print $target->getlog_env($testenv_vars);
835 $testenv_vars = undef;
838 $testenv_vars = $target->setup_env($envname, $prefix);
841 return undef unless defined($testenv_vars);
843 $running_envs{$envname} = $testenv_vars;
845 if ($option eq "local") {
846 SocketWrapper
::set_default_iface
($testenv_vars->{SOCKET_WRAPPER_DEFAULT_IFACE
});
847 $ENV{SMB_CONF_PATH
} = $testenv_vars->{SERVERCONFFILE
};
848 } elsif ($option eq "client") {
849 SocketWrapper
::set_default_iface
(11);
850 write_clientconf
($conffile, $testenv_vars);
851 $ENV{SMB_CONF_PATH
} = $conffile;
853 die("Unknown option[$option] for envname[$envname]");
856 foreach (@exported_envvars) {
857 if (defined($testenv_vars->{$_})) {
858 $ENV{$_} = $testenv_vars->{$_};
864 return $testenv_vars;
867 sub exported_envvars_str
($)
869 my ($testenv_vars) = @_;
872 foreach (@exported_envvars) {
873 next unless defined($testenv_vars->{$_});
874 $out .= $_."=".$testenv_vars->{$_}."\n";
883 return "" if ($envname eq "none");
884 return $target->getlog_env(get_running_env
($envname));
890 return 1 if ($envname eq "none");
891 return $target->check_env(get_running_env
($envname));
897 return if ($envname eq "none");
898 $target->teardown_env(get_running_env
($envname));
899 delete $running_envs{$envname};
902 # This 'global' file needs to be empty when we start
903 unlink("$prefix_abs/dns_host_file");
906 my $testenv_name = $ENV{SELFTEST_TESTENV
};
907 $testenv_name = $testenv_default unless defined($testenv_name);
909 my $testenv_vars = setup_env
($testenv_name);
911 die("Unable to setup environment $testenv_name") unless ($testenv_vars);
913 $ENV{PIDDIR
} = $testenv_vars->{PIDDIR
};
915 my $envvarstr = exported_envvars_str
($testenv_vars);
917 my $term = ($ENV{TERMINAL
} or "xterm");
918 system("$term -e 'echo -e \"
919 Welcome to the Samba4 Test environment '$testenv_name'
921 This matches the client environment used in make test
922 server is pid `cat \$PIDDIR/samba.pid`
924 Some useful environment variables:
925 TORTURE_OPTIONS=\$TORTURE_OPTIONS
926 SMB_CONF_PATH=\$SMB_CONF_PATH
929 \" && LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH} bash'");
930 teardown_env
($testenv_name);
935 $cmd =~ s/([\(\)])/\\$1/g;
937 my $envname = $$_[1];
939 my $envvars = setup_env
($envname);
940 if (not defined($envvars)) {
941 Subunit
::start_testsuite
($name);
942 Subunit
::end_testsuite
($name, "error",
943 "unable to set up environment $envname");
947 # Generate a file with the individual tests to run, if the
948 # test runner for this test suite supports it.
949 if ($$_[3] and $individual_tests and $individual_tests->{$name}) {
950 my ($fh, $listid_file) = tempfile
(UNLINK
=> 0);
951 foreach (@
{$individual_tests->{$name}}) {
954 $cmd .= " --load-list=$listid_file";
957 run_testsuite
($envname, $name, $cmd, $i, $suitestotal);
959 if (defined($opt_analyse_cmd)) {
960 system("$opt_analyse_cmd \"$name\"");
963 teardown_env
($envname) if ($opt_resetup_env);
969 teardown_env
($_) foreach (keys %running_envs);
973 # if there were any valgrind failures, show them
974 foreach (<$prefix/valgrind
.log*>) {
976 print "VALGRIND FAILURE\n";