2 # Bootstrap Samba and run a number of tests against it.
3 # Copyright (C) 2005-2009 Jelmer Vernooij <jelmer@samba.org>
4 # Copyright (C) 2007-2009 Stefan Metzmacher <metze@samba.org>
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
23 selftest - Samba test runner
29 selftest [--srcdir=DIR] [--builddir=DIR] [--exeext=EXT][--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--immediate] [--testlist=FILE] [TESTS]
33 A simple test runner. TESTS is a regular expression with tests to run.
41 Show list of available options.
47 =item I<--builddir=DIR>
57 Change directory to run tests in. Default is 'st'.
61 Show errors as soon as they happen rather than at the end of the test run.
63 =item I<--target samba4|samba3|win|kvm>
65 Specify test target against which to run. Default is 'samba4'.
69 Run only a limited number of tests. Intended to run in about 30 seconds on
70 moderately recent systems.
72 =item I<--socket-wrapper>
74 Use socket wrapper library for communication with server. Only works
75 when the server is running locally.
77 Will prevent TCP and UDP ports being opened on the local host but
78 (transparently) redirects these calls to use unix domain sockets.
80 =item I<--expected-failures>
82 Specify a file containing a list of tests that are expected to fail. Failures for
83 these tests will be counted as successes, successes will be counted as failures.
85 The format for the file is, one entry per line:
87 TESTSUITE-NAME.TEST-NAME
89 The reason for a test can also be specified, by adding a hash sign (#) and the reason
94 Specify a file containing a list of tests that should be skipped. Possible
95 candidates are tests that segfault the server, flip or don't end. The format of this file is the same as
96 for the --expected-failures flag.
100 Specify a file containing a list of tests that should be run. Same format
101 as the --exclude flag.
103 Not includes specified means all tests will be run.
107 Abort as soon as one test fails.
111 Load a list of tests from the specified location.
119 =item I<SMBD_VALGRIND>
121 =item I<TORTURE_MAXTIME>
133 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
143 use FindBin
qw($RealBin $Script);
147 use Cwd qw(abs_path);
149 use Subunit
qw(parse_results);
153 my $opt_target = "samba4";
155 my $opt_socket_wrapper = 0;
156 my $opt_socket_wrapper_pcap = undef;
157 my $opt_socket_wrapper_keep_pcap = undef;
159 my $opt_immediate = 0;
160 my $opt_expected_failures = undef;
161 my @opt_exclude = ();
162 my @opt_include = ();
164 my $opt_image = undef;
167 my $opt_analyse_cmd = undef;
168 my $opt_resetup_env = undef;
169 my $opt_bindir = undef;
170 my $opt_no_lazy_setup = undef;
171 my $opt_format = "plain";
179 my @expected_failures = ();
186 TESTS_UNEXPECTED_OK
=> 0,
187 TESTS_EXPECTED_OK
=> 0,
188 TESTS_UNEXPECTED_FAIL
=> 0,
189 TESTS_EXPECTED_FAIL
=> 0,
196 my ($list, $fullname) = @_;
199 if ($fullname =~ /$$_[0]/) {
200 return ($$_[1]) if ($$_[1]);
201 return "NO REASON SPECIFIED";
208 sub expecting_failure
($)
211 return find_in_list
(\
@expected_failures, $name);
218 return find_in_list
(\
@excludes, $name);
227 return unless ($opt_socket_wrapper_pcap);
228 return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR
});
231 $fname =~ s
%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\
-]%_%g;
233 my $pcap_file = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
235 SocketWrapper
::setup_pcap
($pcap_file);
240 sub cleanup_pcap
($$$)
242 my ($pcap_file, $expected_ret, $ret) = @_;
244 return unless ($opt_socket_wrapper_pcap);
245 return if ($opt_socket_wrapper_keep_pcap);
246 return unless ($expected_ret == $ret);
247 return unless defined($pcap_file);
252 sub run_testsuite
($$$$$$)
254 my ($envname, $name, $cmd, $i, $totalsuites, $msg_ops) = @_;
255 my $pcap_file = setup_pcap
($name);
257 $msg_ops->report_time(time());
258 $msg_ops->start_test([], $name);
260 unless (open(RESULT
, "$cmd 2>&1|")) {
261 $statistics->{TESTS_ERROR
}++;
262 $msg_ops->end_test([], $name, "error", 1, "Unable to run $cmd: $!");
263 $statistics->{SUITES_FAIL
}++;
267 my $expected_ret = parse_results
(
268 $msg_ops, $statistics, *RESULT
, \
&expecting_failure
, [$name]);
270 my $envlog = getlog_env
($envname);
271 $msg_ops->output_msg("ENVLOG: $envlog\n") if ($envlog ne "");
273 $msg_ops->output_msg("CMD: $cmd\n");
275 my $ret = close(RESULT
);
276 $ret = 0 unless $ret == 1;
278 my $exitcode = $?
>> 8;
280 $msg_ops->report_time(time());
282 $msg_ops->end_test([], $name, "success", $expected_ret != $ret, undef);
284 $msg_ops->end_test([], $name, "failure", $expected_ret != $ret, "Exit code was $exitcode");
287 cleanup_pcap
($pcap_file, $expected_ret, $ret);
289 if (not $opt_socket_wrapper_keep_pcap and defined($pcap_file)) {
290 $msg_ops->output_msg("PCAP FILE: $pcap_file\n");
293 if ($ret != $expected_ret) {
294 $statistics->{SUITES_FAIL
}++;
295 exit(1) if ($opt_one);
298 return ($ret == $expected_ret);
303 print "Samba test runner
304 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
305 Copyright (C) Stefan Metzmacher <metze\@samba.org>
307 Usage: $Script [OPTIONS] TESTNAME-REGEX
310 --help this help page
311 --target=samba[34]|win|kvm Samba version to target
312 --testlist=FILE file to read available tests from
315 --prefix=DIR prefix to run tests in [st]
316 --srcdir=DIR source directory [.]
317 --builddir=DIR output directory [.]
318 --exeext=EXT executable extention []
321 --socket-wrapper-pcap save traffic to pcap directories
322 --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
324 --socket-wrapper enable socket wrapper
325 --bindir=PATH path to target binaries
326 --expected-failures=FILE specify list of tests that is guaranteed to fail
329 --ldap=openldap|fedora-ds back samba onto specified ldap server
332 --image=PATH path to KVM image
335 --quick run quick overall test
336 --one abort when the first test fails
337 --immediate print test output for failed tests during run
339 --analyse-cmd CMD command to run after each test
344 my $result = GetOptions
(
345 'help|h|?' => \
$opt_help,
346 'target=s' => \
$opt_target,
347 'prefix=s' => \
$prefix,
348 'socket-wrapper' => \
$opt_socket_wrapper,
349 'socket-wrapper-pcap' => \
$opt_socket_wrapper_pcap,
350 'socket-wrapper-keep-pcap' => \
$opt_socket_wrapper_keep_pcap,
351 'quick' => \
$opt_quick,
353 'immediate' => \
$opt_immediate,
354 'expected-failures=s' => \
$opt_expected_failures,
355 'exclude=s' => \
@opt_exclude,
356 'include=s' => \
@opt_include,
357 'srcdir=s' => \
$srcdir,
358 'builddir=s' => \
$builddir,
359 'exeext=s' => \
$exeext,
360 'verbose' => \
$opt_verbose,
361 'testenv' => \
$opt_testenv,
363 'analyse-cmd=s' => \
$opt_analyse_cmd,
364 'no-lazy-setup' => \
$opt_no_lazy_setup,
365 'resetup-environment' => \
$opt_resetup_env,
366 'bindir:s' => \
$opt_bindir,
367 'format=s' => \
$opt_format,
368 'image=s' => \
$opt_image,
369 'testlist=s' => \
@testlists
372 exit(1) if (not $result);
374 ShowHelp
() if ($opt_help);
378 # quick hack to disable rpc validation when using valgrind - its way too slow
379 unless (defined($ENV{VALGRIND
})) {
380 $ENV{VALIDATE
} = "validate";
381 $ENV{MALLOC_CHECK_
} = 2;
384 my $bindir = ($opt_bindir or "$builddir/bin");
385 my $bindir_abs = abs_path
($bindir);
387 # Backwards compatibility:
388 if (defined($ENV{TEST_LDAP
}) and $ENV{TEST_LDAP
} eq "yes") {
389 if (defined($ENV{FEDORA_DS_ROOT
})) {
396 my $torture_maxtime = ($ENV{TORTURE_MAXTIME
} or 1200);
399 $torture_maxtime *= 2;
406 die("using an empty prefix isn't allowed") unless $prefix ne "";
408 #Ensure we have the test prefix around
409 mkdir($prefix, 0777) unless -d
$prefix;
411 my $prefix_abs = abs_path
($prefix);
412 my $srcdir_abs = abs_path
($srcdir);
413 my $builddir_abs = abs_path
($builddir);
415 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
416 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
418 $ENV{PREFIX
} = $prefix;
419 $ENV{KRB5CCNAME
} = "$prefix/krb5ticket";
420 $ENV{PREFIX_ABS
} = $prefix_abs;
421 $ENV{SRCDIR
} = $srcdir;
422 $ENV{SRCDIR_ABS
} = $srcdir_abs;
423 $ENV{BUILDDIR
} = $builddir;
424 $ENV{BUILDDIR_ABS
} = $builddir_abs;
425 $ENV{EXEEXT
} = $exeext;
427 if (defined($ENV{RUN_FROM_BUILD_FARM
}) and
428 ($ENV{RUN_FROM_BUILD_FARM
} eq "yes")) {
429 $opt_format = "buildfarm";
432 my $tls_enabled = not $opt_quick;
433 $ENV{TLS_ENABLED
} = ($tls_enabled?
"yes":"no");
434 $ENV{LDB_MODULES_PATH
} = "$bindir_abs/modules/ldb";
435 $ENV{LD_SAMBA_MODULE_PATH
} = "$bindir_abs/modules";
436 sub prefix_pathvar
($$)
438 my ($name, $newpath) = @_;
439 if (defined($ENV{$name})) {
440 $ENV{$name} = "$newpath:$ENV{$name}";
442 $ENV{$name} = $newpath;
445 prefix_pathvar
("PKG_CONFIG_PATH", "$bindir_abs/pkgconfig");
446 prefix_pathvar
("PYTHONPATH", "$bindir_abs/python");
448 if ($opt_socket_wrapper_keep_pcap) {
449 # Socket wrapper keep pcap implies socket wrapper pcap
450 $opt_socket_wrapper_pcap = 1;
453 if ($opt_socket_wrapper_pcap) {
454 # Socket wrapper pcap implies socket wrapper
455 $opt_socket_wrapper = 1;
458 my $socket_wrapper_dir;
459 if ($opt_socket_wrapper) {
460 $socket_wrapper_dir = SocketWrapper
::setup_dir
("$prefix/w", $opt_socket_wrapper_pcap);
461 print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
464 print "WARNING: Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports\n";
469 my $testenv_default = "none";
471 if ($opt_target eq "samba4") {
472 $testenv_default = "member";
473 require target
::Samba4
;
474 $target = new Samba4
($bindir, $ldap, "$srcdir/setup", $exeext);
475 } elsif ($opt_target eq "samba3") {
476 if ($opt_socket_wrapper and `$bindir/smbd -b | grep SOCKET_WRAPPER` eq "") {
477 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'. Exiting....");
479 $testenv_default = "member";
480 require target
::Samba3
;
481 $target = new Samba3
($bindir);
482 } elsif ($opt_target eq "win") {
483 die("Windows tests will not run with socket wrapper enabled.")
484 if ($opt_socket_wrapper);
485 $testenv_default = "dc";
486 require target
::Windows
;
487 $target = new Windows
();
488 } elsif ($opt_target eq "kvm") {
489 die("Kvm tests will not run with socket wrapper enabled.")
490 if ($opt_socket_wrapper);
492 die("No image specified") unless ($opt_image);
493 $target = new Kvm
($opt_image, undef);
497 # Start a Virtual Distributed Ethernet Switch
498 # Returns the pid of the switch.
500 sub start_vde_switch
($)
504 system("vde_switch --pidfile $path/vde.pid --sock $path/vde.sock --daemon");
506 open(PID
, "$path/vde.pid");
514 # Stop a Virtual Distributed Ethernet Switch
515 sub stop_vde_switch
($)
521 sub read_test_regexes
($)
525 open(LF
, "<$name") or die("unable to read $name: $!");
529 if (/^(.*?)([ \t]+)\#([\t ]*)(.*?)$/) {
530 push (@ret, [$1, $4]);
532 s/^(.*?)([ \t]+)\#([\t ]*)(.*?)$//;
533 push (@ret, [$_, undef]);
540 if (defined($opt_expected_failures)) {
541 @expected_failures = read_test_regexes
($opt_expected_failures);
544 foreach (@opt_exclude) {
545 push (@excludes, read_test_regexes
($_));
548 foreach (@opt_include) {
549 push (@includes, read_test_regexes
($_));
552 my $interfaces = join(',', ("127.0.0.6/8",
559 my $conffile = "$prefix_abs/client/client.conf";
560 $ENV{SMB_CONF_PATH
} = $conffile;
562 sub write_clientconf
($$)
564 my ($conffile, $vars) = @_;
566 mkdir("$prefix/client", 0777) unless -d
"$prefix/client";
568 if ( -d
"$prefix/client/private" ) {
569 unlink <$prefix/client/private
/*>;
571 mkdir("$prefix/client/private", 0777);
574 if ( -d
"$prefix/client/lock" ) {
575 unlink <$prefix/client/lockdir
/*>;
577 mkdir("$prefix/client/lockdir", 0777);
580 open(CF
, ">$conffile");
581 print CF
"[global]\n";
582 if (defined($ENV{VALGRIND
})) {
583 print CF
"\ticonv:native = true\n";
585 print CF
"\ticonv:native = false\n";
587 print CF
"\tnetbios name = client\n";
588 if (defined($vars->{DOMAIN
})) {
589 print CF
"\tworkgroup = $vars->{DOMAIN}\n";
591 if (defined($vars->{REALM
})) {
592 print CF
"\trealm = $vars->{REALM}\n";
594 if ($opt_socket_wrapper) {
595 print CF
"\tinterfaces = $interfaces\n";
598 private dir = $prefix_abs/client/private
599 lock dir = $prefix_abs/client/lockdir
600 name resolve order = bcast
601 panic action = $RealBin/gdb_backtrace \%PID\% \%PROG\%
603 notify:inotify = false
605 system:anonymous = true
606 client lanman auth = Yes
607 torture:basedir = $prefix_abs/client
608 #We don't want to pass our self-tests if the PAC code is wrong
609 gensec:require_pac = true
610 modules dir = $ENV{LD_SAMBA_MODULE_PATH}
617 my $testsdir = "$srcdir/selftest";
619 my %required_envs = ();
626 open(IN
, $filename) or die("Unable to open $filename: $!");
629 if ($_ eq "-- TEST --\n") {
636 if (not defined($tests) or $name =~ /$tests/) {
637 $required_envs{$env} = 1;
638 push (@ret, [$name, $env, $cmdline]);
644 close(IN
) or die("Error creating recipe");
648 if ($#testlists == -1) {
649 die("No testlists specified");
652 $ENV{SELFTEST_PREFIX
} = "$prefix_abs";
653 if ($opt_socket_wrapper) {
654 $ENV{SELFTEST_INTERFACES
} = $interfaces;
656 $ENV{SELFTEST_INTERFACES
} = "";
659 $ENV{SELFTEST_VERBOSE
} = "1";
661 $ENV{SELFTEST_VERBOSE
} = "";
664 $ENV{SELFTEST_QUICK
} = "1";
666 $ENV{SELFTEST_QUICK
} = "";
668 $ENV{SELFTEST_TARGET
} = $opt_target;
669 $ENV{SELFTEST_MAXTIME
} = $torture_maxtime;
672 foreach my $fn (@testlists) {
673 foreach (read_testlist
($fn)) {
675 next if (@includes and not find_in_list
(\
@includes, $name));
676 push (@available, $_);
681 if ($opt_format eq "buildfarm") {
682 require output
::buildfarm
;
683 $msg_ops = new output
::buildfarm
($statistics);
684 } elsif ($opt_format eq "plain") {
685 require output
::plain
;
686 $msg_ops = new output
::plain
("$prefix/summary", $opt_verbose, $opt_immediate, $statistics, $#available+1);
687 } elsif ($opt_format eq "html") {
688 require output
::html
;
689 mkdir("test-results", 0777);
690 $msg_ops = new output
::html
("test-results", $statistics);
691 } elsif ($opt_format eq "subunit") {
692 require output
::subunit
;
693 $msg_ops = new output
::subunit
();
695 die("Invalid output format '$opt_format'");
697 $msg_ops->report_time(time());
699 foreach (@available) {
701 my $skipreason = skip
($name);
703 $msg_ops->skip_testsuite($name, $skipreason);
710 print STDERR
"No tests to run\n";
714 my $suitestotal = $#todo + 1;
718 my %running_envs = ();
720 sub get_running_env
($)
728 return $running_envs{$envname};
731 my @exported_envvars = (
736 # domain controller stuff
756 "WINBINDD_SOCKET_DIR",
757 "WINBINDD_PRIV_PIPE_DIR"
760 $SIG{INT
} = $SIG{QUIT
} = $SIG{TERM
} = sub {
762 teardown_env
($_) foreach(keys %running_envs);
763 die("Received signal $signame");
770 my $testenv_vars = undef;
776 $option =~ s/^[^:]*//;
779 $option = "client" if $option eq "";
781 if ($envname eq "none") {
783 } elsif (defined(get_running_env
($envname))) {
784 $testenv_vars = get_running_env
($envname);
785 if (not $target->check_env($testenv_vars)) {
786 $testenv_vars = undef;
789 $testenv_vars = $target->setup_env($envname, $prefix);
792 return undef unless defined($testenv_vars);
794 $running_envs{$envname} = $testenv_vars;
796 if ($option eq "local") {
797 SocketWrapper
::set_default_iface
($testenv_vars->{SOCKET_WRAPPER_DEFAULT_IFACE
});
798 $ENV{SMB_CONF_PATH
} = $testenv_vars->{SERVERCONFFILE
};
799 } elsif ($option eq "client") {
800 SocketWrapper
::set_default_iface
(6);
801 write_clientconf
($conffile, $testenv_vars);
802 $ENV{SMB_CONF_PATH
} = $conffile;
804 die("Unknown option[$option] for envname[$envname]");
807 foreach (@exported_envvars) {
808 if (defined($testenv_vars->{$_})) {
809 $ENV{$_} = $testenv_vars->{$_};
815 return $testenv_vars;
818 sub exported_envvars_str
($)
820 my ($testenv_vars) = @_;
823 foreach (@exported_envvars) {
824 next unless defined($testenv_vars->{$_});
825 $out .= $_."=".$testenv_vars->{$_}."\n";
834 return "" if ($envname eq "none");
835 return $target->getlog_env(get_running_env
($envname));
841 return 1 if ($envname eq "none");
842 return $target->check_env(get_running_env
($envname));
848 return if ($envname eq "none");
849 $target->teardown_env(get_running_env
($envname));
850 delete $running_envs{$envname};
853 if ($opt_no_lazy_setup) {
854 setup_env
($_) foreach (keys %required_envs);
858 my $testenv_name = $ENV{SELFTEST_TESTENV
};
859 $testenv_name = $testenv_default unless defined($testenv_name);
861 my $testenv_vars = setup_env
($testenv_name);
863 $ENV{PIDDIR
} = $testenv_vars->{PIDDIR
};
865 my $envvarstr = exported_envvars_str
($testenv_vars);
867 my $term = ($ENV{TERM
} or "xterm");
868 system("$term -e 'echo -e \"
869 Welcome to the Samba4 Test environment '$testenv_name'
871 This matches the client environment used in make test
872 server is pid `cat \$PIDDIR/samba.pid`
874 Some useful environment variables:
875 TORTURE_OPTIONS=\$TORTURE_OPTIONS
876 SMB_CONF_PATH=\$SMB_CONF_PATH
879 \" && LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH} bash'");
880 teardown_env
($testenv_name);
885 $cmd =~ s/([\(\)])/\\$1/g;
887 my $envname = $$_[1];
889 my $envvars = setup_env
($envname);
890 if (not defined($envvars)) {
891 $msg_ops->skip_testsuite($name,
892 "unable to set up environment $envname");
896 run_testsuite
($envname, $name, $cmd, $i, $suitestotal,
899 if (defined($opt_analyse_cmd)) {
900 system("$opt_analyse_cmd \"$name\"");
903 teardown_env
($envname) if ($opt_resetup_env);
909 teardown_env
($_) foreach (keys %running_envs);
917 # if there were any valgrind failures, show them
918 foreach (<$prefix/valgrind
.log*>) {
920 system("grep DWARF2.CFI.reader $_ > /dev/null");
922 print "VALGRIND FAILURE\n";
928 if ($opt_format eq "buildfarm") {
929 print "TEST STATUS: $statistics->{SUITES_FAIL}\n";
932 exit $statistics->{SUITES_FAIL
};