2 # Bootstrap Samba and run a number of tests against it.
3 # Copyright (C) 2005-2008 Jelmer Vernooij <jelmer@samba.org>
4 # Published under the GNU GPL, v3 or later.
10 selftest - Samba test runner
16 selftest [--srcdir=DIR] [--builddir=DIR] [--exeext=EXT][--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--immediate] [--testlist=FILE] [TESTS]
20 A simple test runner. TESTS is a regular expression with tests to run.
28 Show list of available options.
34 =item I<--builddir=DIR>
44 Change directory to run tests in. Default is 'st'.
48 Show errors as soon as they happen rather than at the end of the test run.
50 =item I<--target samba4|samba3|win|kvm>
52 Specify test target against which to run. Default is 'samba4'.
56 Run only a limited number of tests. Intended to run in about 30 seconds on
57 moderately recent systems.
59 =item I<--socket-wrapper>
61 Use socket wrapper library for communication with server. Only works
62 when the server is running locally.
64 Will prevent TCP and UDP ports being opened on the local host but
65 (transparently) redirects these calls to use unix domain sockets.
67 =item I<--expected-failures>
69 Specify a file containing a list of tests that are expected to fail. Failures for
70 these tests will be counted as successes, successes will be counted as failures.
72 The format for the file is, one entry per line:
74 TESTSUITE-NAME.TEST-NAME
76 The reason for a test can also be specified, by adding a hash sign (#) and the reason
81 Specify a file containing a list of tests that should be skipped. Possible
82 candidates are tests that segfault the server, flip or don't end. The format of this file is the same as
83 for the --expected-failures flag.
87 Specify a file containing a list of tests that should be run. Same format
88 as the --exclude flag.
90 Not includes specified means all tests will be run.
94 Abort as soon as one test fails.
98 Load a list of tests from the specified location.
106 =item I<SMBD_VALGRIND>
108 =item I<TORTURE_MAXTIME>
120 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
130 use FindBin
qw($RealBin $Script);
134 use Cwd qw(abs_path);
136 use Subunit
qw(parse_results);
140 my $opt_target = "samba4";
142 my $opt_socket_wrapper = 0;
143 my $opt_socket_wrapper_pcap = undef;
144 my $opt_socket_wrapper_keep_pcap = undef;
146 my $opt_immediate = 0;
147 my $opt_expected_failures = undef;
148 my @opt_exclude = ();
149 my @opt_include = ();
151 my $opt_image = undef;
154 my $opt_analyse_cmd = undef;
155 my $opt_resetup_env = undef;
156 my $opt_bindir = undef;
157 my $opt_no_lazy_setup = undef;
158 my $opt_format = "plain";
166 my @expected_failures = ();
173 TESTS_UNEXPECTED_OK
=> 0,
174 TESTS_EXPECTED_OK
=> 0,
175 TESTS_UNEXPECTED_FAIL
=> 0,
176 TESTS_EXPECTED_FAIL
=> 0,
183 my ($list, $fullname) = @_;
186 if ($fullname =~ /$$_[0]/) {
187 return ($$_[1]) if ($$_[1]);
188 return "NO REASON SPECIFIED";
195 sub expecting_failure
($)
198 return find_in_list
(\
@expected_failures, $name);
205 return find_in_list
(\
@excludes, $name);
214 return unless ($opt_socket_wrapper_pcap);
215 return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR
});
218 $fname =~ s
%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\
-]%_%g;
220 my $pcap_file = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
222 SocketWrapper
::setup_pcap
($pcap_file);
227 sub cleanup_pcap
($$$)
229 my ($pcap_file, $expected_ret, $ret) = @_;
231 return unless ($opt_socket_wrapper_pcap);
232 return if ($opt_socket_wrapper_keep_pcap);
233 return unless ($expected_ret == $ret);
234 return unless defined($pcap_file);
239 sub run_testsuite
($$$$$$)
241 my ($envname, $name, $cmd, $i, $totalsuites, $msg_ops) = @_;
242 my $pcap_file = setup_pcap
($name);
244 $msg_ops->start_test([], $name);
246 unless (open(RESULT
, "$cmd 2>&1|")) {
247 $statistics->{TESTS_ERROR
}++;
248 $msg_ops->end_test([], $name, "error", 1, "Unable to run $cmd: $!");
249 $statistics->{SUITES_FAIL
}++;
253 my $expected_ret = parse_results
(
254 $msg_ops, $statistics, *RESULT
, \
&expecting_failure
, [$name]);
256 my $envlog = getlog_env
($envname);
257 $msg_ops->output_msg("ENVLOG: $envlog\n") if ($envlog ne "");
259 $msg_ops->output_msg("CMD: $cmd\n");
261 my $ret = close(RESULT
);
262 $ret = 0 unless $ret == 1;
264 my $exitcode = $?
>> 8;
267 $msg_ops->end_test([], $name, "success", $expected_ret != $ret, undef);
269 $msg_ops->end_test([], $name, "failure", $expected_ret != $ret, "Exit code was $exitcode");
272 cleanup_pcap
($pcap_file, $expected_ret, $ret);
274 if (not $opt_socket_wrapper_keep_pcap and defined($pcap_file)) {
275 $msg_ops->output_msg("PCAP FILE: $pcap_file\n");
278 if ($ret != $expected_ret) {
279 $statistics->{SUITES_FAIL
}++;
280 exit(1) if ($opt_one);
283 return ($ret == $expected_ret);
288 print "Samba test runner
289 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
291 Usage: $Script [OPTIONS] TESTNAME-REGEX
294 --help this help page
295 --target=samba[34]|win|kvm Samba version to target
296 --testlist=FILE file to read available tests from
299 --prefix=DIR prefix to run tests in [st]
300 --srcdir=DIR source directory [.]
301 --builddir=DIR output directory [.]
302 --exeext=EXT executable extention []
305 --socket-wrapper-pcap save traffic to pcap directories
306 --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
308 --socket-wrapper enable socket wrapper
309 --bindir=PATH path to target binaries
310 --expected-failures=FILE specify list of tests that is guaranteed to fail
313 --ldap=openldap|fedora-ds back samba onto specified ldap server
316 --image=PATH path to KVM image
319 --quick run quick overall test
320 --one abort when the first test fails
321 --immediate print test output for failed tests during run
323 --analyse-cmd CMD command to run after each test
328 my $result = GetOptions
(
329 'help|h|?' => \
$opt_help,
330 'target=s' => \
$opt_target,
331 'prefix=s' => \
$prefix,
332 'socket-wrapper' => \
$opt_socket_wrapper,
333 'socket-wrapper-pcap' => \
$opt_socket_wrapper_pcap,
334 'socket-wrapper-keep-pcap' => \
$opt_socket_wrapper_keep_pcap,
335 'quick' => \
$opt_quick,
337 'immediate' => \
$opt_immediate,
338 'expected-failures=s' => \
$opt_expected_failures,
339 'exclude=s' => \
@opt_exclude,
340 'include=s' => \
@opt_include,
341 'srcdir=s' => \
$srcdir,
342 'builddir=s' => \
$builddir,
343 'exeext=s' => \
$exeext,
344 'verbose' => \
$opt_verbose,
345 'testenv' => \
$opt_testenv,
347 'analyse-cmd=s' => \
$opt_analyse_cmd,
348 'no-lazy-setup' => \
$opt_no_lazy_setup,
349 'resetup-environment' => \
$opt_resetup_env,
350 'bindir:s' => \
$opt_bindir,
351 'format=s' => \
$opt_format,
352 'image=s' => \
$opt_image,
353 'testlist=s' => \
@testlists
356 exit(1) if (not $result);
358 ShowHelp
() if ($opt_help);
362 # quick hack to disable rpc validation when using valgrind - its way too slow
363 unless (defined($ENV{VALGRIND
})) {
364 $ENV{VALIDATE
} = "validate";
365 $ENV{MALLOC_CHECK_
} = 2;
368 my $old_pwd = "$RealBin/..";
370 # Backwards compatibility:
371 if (defined($ENV{TEST_LDAP
}) and $ENV{TEST_LDAP
} eq "yes") {
372 if (defined($ENV{FEDORA_DS_ROOT
})) {
379 my $torture_maxtime = ($ENV{TORTURE_MAXTIME
} or 1200);
382 $torture_maxtime *= 2;
389 die("using an empty prefix isn't allowed") unless $prefix ne "";
391 #Ensure we have the test prefix around
392 mkdir($prefix, 0777) unless -d
$prefix;
394 my $prefix_abs = abs_path
($prefix);
395 my $srcdir_abs = abs_path
($srcdir);
397 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
398 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
400 $ENV{PREFIX
} = $prefix;
401 $ENV{KRB5CCNAME
} = "$prefix/krb5ticket";
402 $ENV{PREFIX_ABS
} = $prefix_abs;
403 $ENV{SRCDIR
} = $srcdir;
404 $ENV{SRCDIR_ABS
} = $srcdir_abs;
406 if (defined($ENV{RUN_FROM_BUILD_FARM
}) and
407 ($ENV{RUN_FROM_BUILD_FARM
} eq "yes")) {
408 $opt_format = "buildfarm";
411 my $tls_enabled = not $opt_quick;
412 $ENV{TLS_ENABLED
} = ($tls_enabled?
"yes":"no");
413 $ENV{LDB_MODULES_PATH
} = "$old_pwd/source4/bin/modules/ldb";
414 $ENV{LD_SAMBA_MODULE_PATH
} = "$old_pwd/source4/bin/modules";
415 sub prefix_pathvar
($$)
417 my ($name, $newpath) = @_;
418 if (defined($ENV{$name})) {
419 $ENV{$name} = "$newpath:$ENV{$name}";
421 $ENV{$name} = $newpath;
424 prefix_pathvar
("PKG_CONFIG_PATH", "$old_pwd/source4/bin/pkgconfig");
425 prefix_pathvar
("PYTHONPATH", "$old_pwd/source4/bin/python");
427 if ($opt_socket_wrapper_keep_pcap) {
428 # Socket wrapper keep pcap implies socket wrapper pcap
429 $opt_socket_wrapper_pcap = 1;
432 if ($opt_socket_wrapper_pcap) {
433 # Socket wrapper pcap implies socket wrapper
434 $opt_socket_wrapper = 1;
437 my $socket_wrapper_dir;
438 if ($opt_socket_wrapper) {
439 $socket_wrapper_dir = SocketWrapper
::setup_dir
("$prefix/w", $opt_socket_wrapper_pcap);
440 print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
443 print "WARNING: Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports\n";
448 my $testenv_default = "none";
450 if ($opt_target eq "samba4") {
451 $testenv_default = "member";
452 require target
::Samba4
;
453 $target = new Samba4
($opt_bindir or "$builddir/bin",
454 $ldap, "$srcdir/setup", $exeext);
455 } elsif ($opt_target eq "samba3") {
456 my $bindir = ($opt_bindir or "$builddir/bin");
457 if ($opt_socket_wrapper and `$bindir/smbd -b | grep SOCKET_WRAPPER` eq "") {
458 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'. Exiting....");
460 $testenv_default = "member";
461 require target
::Samba3
;
462 $target = new Samba3
($bindir);
463 } elsif ($opt_target eq "win") {
464 die("Windows tests will not run with socket wrapper enabled.")
465 if ($opt_socket_wrapper);
466 $testenv_default = "dc";
467 require target
::Windows
;
468 $target = new Windows
();
469 } elsif ($opt_target eq "kvm") {
470 die("Kvm tests will not run with socket wrapper enabled.")
471 if ($opt_socket_wrapper);
473 die("No image specified") unless ($opt_image);
474 $target = new Kvm
($opt_image, undef);
478 # Start a Virtual Distributed Ethernet Switch
479 # Returns the pid of the switch.
481 sub start_vde_switch
($)
485 system("vde_switch --pidfile $path/vde.pid --sock $path/vde.sock --daemon");
487 open(PID
, "$path/vde.pid");
495 # Stop a Virtual Distributed Ethernet Switch
496 sub stop_vde_switch
($)
502 sub read_test_regexes
($)
506 open(LF
, "<$name") or die("unable to read $name: $!");
510 if (/^(.*?)([ \t]+)\#([\t ]*)(.*?)$/) {
511 push (@ret, [$1, $4]);
513 s/^(.*?)([ \t]+)\#([\t ]*)(.*?)$//;
514 push (@ret, [$_, undef]);
521 if (defined($opt_expected_failures)) {
522 @expected_failures = read_test_regexes
($opt_expected_failures);
525 foreach (@opt_exclude) {
526 push (@excludes, read_test_regexes
($_));
529 foreach (@opt_include) {
530 push (@includes, read_test_regexes
($_));
533 my $interfaces = join(',', ("127.0.0.6/8",
540 my $conffile = "$prefix_abs/client/client.conf";
541 $ENV{SMB_CONF_PATH
} = $conffile;
543 sub write_clientconf
($$)
545 my ($conffile, $vars) = @_;
547 mkdir("$prefix/client", 0777) unless -d
"$prefix/client";
549 if ( -d
"$prefix/client/private" ) {
550 unlink <$prefix/client/private
/*>;
552 mkdir("$prefix/client/private", 0777);
555 if ( -d
"$prefix/client/lock" ) {
556 unlink <$prefix/client/lockdir
/*>;
558 mkdir("$prefix/client/lockdir", 0777);
561 open(CF
, ">$conffile");
562 print CF
"[global]\n";
563 if (defined($ENV{VALGRIND
})) {
564 print CF
"\ticonv:native = true\n";
566 print CF
"\ticonv:native = false\n";
568 print CF
"\tnetbios name = client\n";
569 if (defined($vars->{DOMAIN
})) {
570 print CF
"\tworkgroup = $vars->{DOMAIN}\n";
572 if (defined($vars->{REALM
})) {
573 print CF
"\trealm = $vars->{REALM}\n";
575 if ($opt_socket_wrapper) {
576 print CF
"\tinterfaces = $interfaces\n";
579 private dir = $prefix_abs/client/private
580 lock dir = $prefix_abs/client/lockdir
581 name resolve order = bcast
582 panic action = $RealBin/gdb_backtrace \%PID\% \%PROG\%
584 notify:inotify = false
586 system:anonymous = true
587 client lanman auth = Yes
588 torture:basedir = $prefix_abs/client
589 #We don't want to pass our self-tests if the PAC code is wrong
590 gensec:require_pac = true
591 modules dir = $ENV{LD_SAMBA_MODULE_PATH}
598 my $testsdir = "$srcdir/selftest";
600 my %required_envs = ();
607 open(IN
, $filename) or die("Unable to open $filename: $!");
610 if ($_ eq "-- TEST --\n") {
617 if (not defined($tests) or $name =~ /$tests/) {
618 $required_envs{$env} = 1;
619 push (@ret, [$name, $env, $cmdline]);
625 close(IN
) or die("Error creating recipe");
629 if ($#testlists == -1) {
630 die("No testlists specified");
633 $ENV{SELFTEST_PREFIX
} = "$prefix_abs";
634 if ($opt_socket_wrapper) {
635 $ENV{SELFTEST_INTERFACES
} = $interfaces;
637 $ENV{SELFTEST_INTERFACES
} = "";
640 $ENV{SELFTEST_VERBOSE
} = "1";
642 $ENV{SELFTEST_VERBOSE
} = "";
645 $ENV{SELFTEST_QUICK
} = "1";
647 $ENV{SELFTEST_QUICK
} = "";
649 $ENV{SELFTEST_TARGET
} = $opt_target;
650 $ENV{SELFTEST_MAXTIME
} = $torture_maxtime;
653 foreach my $fn (@testlists) {
654 foreach (read_testlist
($fn)) {
656 next if (@includes and not find_in_list
(\
@includes, $name));
657 push (@available, $_);
662 if ($opt_format eq "buildfarm") {
663 require output
::buildfarm
;
664 $msg_ops = new output
::buildfarm
($statistics);
665 } elsif ($opt_format eq "plain") {
666 require output
::plain
;
667 $msg_ops = new output
::plain
("$prefix/summary", $opt_verbose, $opt_immediate, $statistics, $#available+1);
668 } elsif ($opt_format eq "html") {
669 require output
::html
;
670 mkdir("test-results", 0777);
671 $msg_ops = new output
::html
("test-results", $statistics);
673 die("Invalid output format '$opt_format'");
677 foreach (@available) {
679 my $skipreason = skip
($name);
681 $msg_ops->skip_testsuite($name, $skipreason);
688 print STDERR
"No tests to run\n";
692 my $suitestotal = $#todo + 1;
696 my %running_envs = ();
698 sub get_running_env
($)
706 return $running_envs{$envname};
709 my @exported_envvars = (
714 # domain controller stuff
734 "WINBINDD_SOCKET_DIR",
735 "WINBINDD_PRIV_PIPE_DIR"
738 $SIG{INT
} = $SIG{QUIT
} = $SIG{TERM
} = sub {
740 teardown_env
($_) foreach(keys %running_envs);
741 die("Received signal $signame");
748 my $testenv_vars = undef;
754 $option =~ s/^[^:]*//;
757 $option = "client" if $option eq "";
759 if ($envname eq "none") {
761 } elsif (defined(get_running_env
($envname))) {
762 $testenv_vars = get_running_env
($envname);
763 if (not $target->check_env($testenv_vars)) {
764 $testenv_vars = undef;
767 $testenv_vars = $target->setup_env($envname, $prefix);
770 return undef unless defined($testenv_vars);
772 $running_envs{$envname} = $testenv_vars;
774 if ($option eq "local") {
775 SocketWrapper
::set_default_iface
($testenv_vars->{SOCKET_WRAPPER_DEFAULT_IFACE
});
776 $ENV{SMB_CONF_PATH
} = $testenv_vars->{SERVERCONFFILE
};
777 } elsif ($option eq "client") {
778 SocketWrapper
::set_default_iface
(6);
779 write_clientconf
($conffile, $testenv_vars);
780 $ENV{SMB_CONF_PATH
} = $conffile;
782 die("Unknown option[$option] for envname[$envname]");
785 foreach (@exported_envvars) {
786 if (defined($testenv_vars->{$_})) {
787 $ENV{$_} = $testenv_vars->{$_};
793 return $testenv_vars;
796 sub exported_envvars_str
($)
798 my ($testenv_vars) = @_;
801 foreach (@exported_envvars) {
802 next unless defined($testenv_vars->{$_});
803 $out .= $_."=".$testenv_vars->{$_}."\n";
812 return "" if ($envname eq "none");
813 return $target->getlog_env(get_running_env
($envname));
819 return 1 if ($envname eq "none");
820 return $target->check_env(get_running_env
($envname));
826 return if ($envname eq "none");
827 $target->teardown_env(get_running_env
($envname));
828 delete $running_envs{$envname};
831 if ($opt_no_lazy_setup) {
832 setup_env
($_) foreach (keys %required_envs);
836 my $testenv_name = $ENV{SELFTEST_TESTENV
};
837 $testenv_name = $testenv_default unless defined($testenv_name);
839 my $testenv_vars = setup_env
($testenv_name);
841 $ENV{PIDDIR
} = $testenv_vars->{PIDDIR
};
843 my $envvarstr = exported_envvars_str
($testenv_vars);
845 my $term = ($ENV{TERM
} or "xterm");
846 system("$term -e 'echo -e \"
847 Welcome to the Samba4 Test environment '$testenv_name'
849 This matches the client environment used in make test
850 server is pid `cat \$PIDDIR/samba.pid`
852 Some useful environment variables:
853 TORTURE_OPTIONS=\$TORTURE_OPTIONS
854 SMB_CONF_PATH=\$SMB_CONF_PATH
857 \" && LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH} bash'");
858 teardown_env
($testenv_name);
863 $cmd =~ s/([\(\)])/\\$1/g;
865 my $envname = $$_[1];
867 my $envvars = setup_env
($envname);
868 if (not defined($envvars)) {
869 $msg_ops->skip_testsuite($name, "unable to set up environment $envname");
873 run_testsuite
($envname, $name, $cmd, $i, $suitestotal,
876 if (defined($opt_analyse_cmd)) {
877 system("$opt_analyse_cmd \"$name\"");
880 teardown_env
($envname) if ($opt_resetup_env);
886 teardown_env
($_) foreach (keys %running_envs);
894 # if there were any valgrind failures, show them
895 foreach (<$prefix/valgrind
.log*>) {
897 system("grep DWARF2.CFI.reader $_ > /dev/null");
899 print "VALGRIND FAILURE\n";
905 if ($opt_format eq "buildfarm") {
906 print "TEST STATUS: $statistics->{SUITES_FAIL}\n";
909 exit $statistics->{SUITES_FAIL
};