source4/script/tests/selftest.pl

   1 #!/usr/bin/perl
   2 # Bootstrap Samba and run a number of tests against it.
   3 # Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
   4 # Published under the GNU GPL, v3 or later.
   5
   6 =pod
   7
   8 =head1 NAME
   9
  10 selftest - Samba test runner
  11
  12 =head1 SYNOPSIS
  13
  14 selftest --help
  15
  16 selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win] [--socket-wrapper] [--quick] [--one] [--prefix=prefix] [--immediate] [TESTS]
  17
  18 =head1 DESCRIPTION
  19
  20 A simple test runner. TESTS is a regular expression with tests to run.
  21
  22 =head1 OPTIONS
  23
  24 =over 4
  25
  26 =item I<--help>
  27
  28 Show list of available options.
  29
  30 =item I<--srcdir=DIR>
  31
  32 Source directory.
  33
  34 =item I<--builddir=DIR>
  35
  36 Build directory.
  37
  38 =item I<--prefix=DIR>
  39
  40 Change directory to run tests in. Default is 'st'.
  41
  42 =item I<--immediate>
  43
  44 Show errors as soon as they happen rather than at the end of the test run.
  45
  46 =item I<--target samba4|samba3|win>
  47
  48 Specify test target against which to run. Default is 'samba4'.
  49
  50 =item I<--quick>
  51
  52 Run only a limited number of tests. Intended to run in about 30 seconds on
  53 moderately recent systems.
  54
  55 =item I<--socket-wrapper>
  56
  57 Use socket wrapper library for communication with server. Only works
  58 when the server is running locally.
  59
  60 Will prevent TCP and UDP ports being opened on the local host but
  61 (transparently) redirects these calls to use unix domain sockets.
  62
  63 =item I<--expected-failures>
  64
  65 Specify a file containing a list of tests that are expected to fail. Failures for
  66 these tests will be counted as successes, successes will be counted as failures.
  67
  68 The format for the file is, one entry per line:
  69
  70 TESTSUITE-NAME/TEST-NAME
  71
  72 =item I<--skip>
  73
  74 Specify a file containing a list of tests that should be skipped. Possible candidates are
  75 tests that segfault the server, flip or don't end.
  76
  77 =item I<--one>
  78
  79 Abort as soon as one test fails.
  80
  81 =back
  82
  83 =head1 ENVIRONMENT
  84
  85 =over 4
  86
  87 =item I<SMBD_VALGRIND>
  88
  89 =item I<TORTURE_MAXTIME>
  90
  91 =item I<VALGRIND>
  92
  93 =item I<TEST_LDAP>
  94
  95 =item I<TLS_ENABLED>
  96
  97 =item I<srcdir>
  98
  99 =back
 100
 101 =head1 LICENSE
 102
 103 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
 104
 105 =head1 AUTHOR
 106
 107 Jelmer Vernooij
 108
 109 =cut
 110
 111 use strict;
 112 use warnings;
 113
 114 use FindBin qw($RealBin $Script);
 115 use File::Spec;
 116 use Getopt::Long;
 117 use POSIX;
 118 use Cwd;
 119 use lib "$RealBin";
 120 use Samba4;
 121 use SocketWrapper;
 122
 123 my $opt_help = 0;
 124 my $opt_target = "samba4";
 125 my $opt_quick = 0;
 126 my $opt_socket_wrapper = 0;
 127 my $opt_socket_wrapper_pcap = undef;
 128 my $opt_one = 0;
 129 my $opt_immediate = 0;
 130 my $opt_expected_failures = undef;
 131 my $opt_skip = undef;
 132 my $opt_verbose = 0;
 133 my $opt_testenv = 0;
 134
 135 my $srcdir = ".";
 136 my $builddir = ".";
 137 my $prefix = "st";
 138
 139 my $suitesfailed = [];
 140 my $start = time();
 141 my @expected_failures = ();
 142 my @skips = ();
 143
 144 my $statistics = {
 145         SUITES_FAIL => 0,
 146         SUITES_OK => 0,
 147         SUITES_SKIPPED => 0,
 148
 149         TESTS_UNEXPECTED_OK => 0,
 150         TESTS_EXPECTED_OK => 0,
 151         TESTS_UNEXPECTED_FAIL => 0,
 152         TESTS_EXPECTED_FAIL => 0,
 153         TESTS_ERROR => 0
 154 };
 155
 156 sub expecting_failure($)
 157 {
 158         my $fullname = shift;
 159
 160         return 1 if (grep(/^$fullname$/, @expected_failures));
 161
 162         return 0;
 163 }
 164
 165 sub skip($)
 166 {
 167         my $fullname = shift;
 168
 169         return 1 if (grep(/^$fullname$/, @skips));
 170         return 0;
 171 }
 172
 173 sub run_test_buildfarm($$$$)
 174 {
 175         my ($name, $cmd, $i, $suitestotal) = @_;
 176         print "--==--==--==--==--==--==--==--==--==--==--\n";
 177         print "Running test $name (level 0 stdout)\n";
 178         print "--==--==--==--==--==--==--==--==--==--==--\n";
 179         system("date");
 180
 181         my $expected_ret = 1;
 182         my $open_tests = {};
 183         open(RESULT, "$cmd 2>&1|");
 184         while (<RESULT>) {
 185                 print;
 186                 if (/^test: (.+)\n/) {
 187                         $open_tests->{$1} = 1;
 188                 } elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
 189                         my $result = $1;
 190                         if ($1 eq "success") {
 191                                 delete $open_tests->{$2};
 192                                 if (expecting_failure("$name/$2")) {
 193                                         $statistics->{TESTS_UNEXPECTED_OK}++;
 194                                 } else {
 195                                         $statistics->{TESTS_EXPECTED_OK}++;
 196                                 }
 197                         } elsif ($1 eq "failure") {
 198                                 delete $open_tests->{$2};
 199                                 if (expecting_failure("$name/$2")) {
 200                                         $statistics->{TESTS_EXPECTED_FAIL}++;
 201                                         $expected_ret = 0;
 202                                 } else {
 203                                         $statistics->{TESTS_UNEXPECTED_FAIL}++;
 204                                 }
 205                         } elsif ($1 eq "skip") {
 206                                 delete $open_tests->{$2};
 207                         } elsif ($1 eq "error") {
 208                                 $statistics->{TESTS_ERROR}++;
 209                                 delete $open_tests->{$2};
 210                         }
 211                 }
 212         }
 213         print "COMMAND: $cmd\n";
 214         foreach (keys %$open_tests) {
 215                 print "$_ was started but never finished!\n";
 216                 $statistics->{TESTS_ERROR}++;
 217         }
 218         my $ret = close(RESULT);
 219
 220         print "==========================================\n";
 221         if ($ret == $expected_ret) {
 222                 print "TEST PASSED: $name\n";
 223         } else {
 224                 print "TEST FAILED: $name (status $ret)\n";
 225         }
 226         print "==========================================\n";
 227 }
 228
 229 my $test_output = {};
 230 sub run_test_plain($$$$)
 231 {
 232         my ($name, $cmd, $i, $totalsuites) = @_;
 233         my $err = "";
 234         if ($#$suitesfailed+1 > 0) { $err = ", ".($#$suitesfailed+1)." errors"; }
 235         printf "[$i/$totalsuites in " . (time() - $start)."s$err] $name\n";
 236         open(RESULT, "$cmd 2>&1|");
 237         my $expected_ret = 1;
 238         my $open_tests = {};
 239         $test_output->{$name} = "";
 240         while (<RESULT>) {
 241                 $test_output->{$name}.=$_;
 242                 print if ($opt_verbose);
 243                 if (/^test: (.+)\n/) {
 244                         $open_tests->{$1} = 1;
 245                 } elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
 246                         my $result = $1;
 247                         if ($1 eq "success") {
 248                                 delete $open_tests->{$2};
 249                                 if (expecting_failure("$name/$2")) {
 250                                         $statistics->{TESTS_UNEXPECTED_OK}++;
 251                                 } else {
 252                                         $statistics->{TESTS_EXPECTED_OK}++;
 253                                 }
 254                         } elsif ($1 eq "failure") {
 255                                 delete $open_tests->{$2};
 256                                 if (expecting_failure("$name/$2")) {
 257                                         $statistics->{TESTS_EXPECTED_FAIL}++;
 258                                         $expected_ret = 0;
 259                                 } else {
 260                                         $statistics->{TESTS_UNEXPECTED_FAIL}++;
 261                                 }
 262                         } elsif ($1 eq "skip") {
 263                                 delete $open_tests->{$2};
 264                         } elsif ($1 eq "error") {
 265                                 $statistics->{TESTS_ERROR}++;
 266                                 delete $open_tests->{$2};
 267                         }
 268                 }
 269         }
 270         $test_output->{$name}.="COMMAND: $cmd\n";
 271         foreach (keys %$open_tests) {
 272                 $test_output->{$name}.="$_ was started but never finished!\n";
 273                 $statistics->{TESTS_ERROR}++;
 274         }
 275         my $ret = close(RESULT);
 276         if ($ret != $expected_ret and ($opt_immediate or $opt_one) and not $opt_verbose) {
 277                 print "$test_output->{$name}\n";
 278         }
 279         if ($ret != $expected_ret) {
 280                 push(@$suitesfailed, $name);
 281                 $statistics->{SUITES_FAIL}++;
 282                 exit(1) if ($opt_one);
 283         } else {
 284                 $statistics->{SUITES_OK}++;
 285         }
 286 }
 287
 288 sub ShowHelp()
 289 {
 290         print "Samba test runner
 291 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
 292
 293 Usage: $Script [OPTIONS] PREFIX
 294
 295 Generic options:
 296  --help                     this help page
 297
 298 Paths:
 299  --prefix=DIR               prefix to run tests in [st]
 300  --srcdir=DIR               source directory [.]
 301  --builddir=DIR             output directory [.]
 302
 303 Target Specific:
 304  --target=samba4|samba3|win Samba version to target
 305  --socket-wrapper-pcap=FILE save traffic to pcap file
 306  --socket-wrapper           enable socket wrapper
 307  --expected-failures=FILE   specify list of tests that is guaranteed to fail
 308
 309 Behaviour:
 310  --quick                    run quick overall test
 311  --one                      abort when the first test fails
 312  --immediate                print test output for failed tests during run
 313  --verbose                  be verbose
 314 ";
 315         exit(0);
 316 }
 317
 318 my $result = GetOptions (
 319             'help|h|?' => \$opt_help,
 320                 'target=s' => \$opt_target,
 321                 'prefix=s' => \$prefix,
 322                 'socket-wrapper' => \$opt_socket_wrapper,
 323                 'socket-wrapper-pcap=s' => \$opt_socket_wrapper_pcap,
 324                 'quick' => \$opt_quick,
 325                 'one' => \$opt_one,
 326                 'immediate' => \$opt_immediate,
 327                 'expected-failures=s' => \$opt_expected_failures,
 328                 'skip=s' => \$opt_skip,
 329                 'srcdir=s' => \$srcdir,
 330                 'builddir=s' => \$builddir,
 331                 'verbose' => \$opt_verbose,
 332                 'testenv' => \$opt_testenv
 333             );
 334
 335 exit(1) if (not $result);
 336
 337 ShowHelp() if ($opt_help);
 338
 339 my $tests = shift;
 340
 341 my $torture_maxtime = $ENV{TORTURE_MAXTIME};
 342 unless (defined($torture_maxtime)) {
 343         $torture_maxtime = 1200;
 344 }
 345
 346 # quick hack to disable rpc validation when using valgrind - its way too slow
 347 unless (defined($ENV{VALGRIND})) {
 348         $ENV{VALIDATE} = "validate";
 349 }
 350
 351 my $old_pwd = "$RealBin/../..";
 352 my $ldap = (defined($ENV{TEST_LDAP}) and ($ENV{TEST_LDAP} eq "yes"))?1:0;
 353
 354 $prefix =~ s+//+/+;
 355 $ENV{PREFIX} = $prefix;
 356
 357 $ENV{SRCDIR} = $srcdir;
 358
 359 my $bindir = "$srcdir/bin";
 360 my $setupdir = "$srcdir/setup";
 361 my $testsdir = "$srcdir/script/tests";
 362
 363 my $tls_enabled = not $opt_quick;
 364 my $from_build_farm = (defined($ENV{RUN_FROM_BUILD_FARM}) and
 365                       ($ENV{RUN_FROM_BUILD_FARM} eq "yes"));
 366
 367 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
 368 $ENV{LD_LDB_MODULE_PATH} = "$old_pwd/bin/modules/ldb";
 369 $ENV{LD_SAMBA_MODULE_PATH} = "$old_pwd/bin/modules";
 370 if (defined($ENV{LD_LIBRARY_PATH})) {
 371         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared:$ENV{LD_LIBRARY_PATH}";
 372 } else {
 373         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared";
 374 }
 375 $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig:$ENV{PKG_CONFIG_PATH}";
 376 $ENV{PATH} = "$old_pwd/bin:$ENV{PATH}";
 377
 378 my @torture_options = ();
 379
 380 my $testenv_vars = {};
 381
 382 if ($opt_target eq "samba4") {
 383         $testenv_vars = Samba4::provision($prefix);
 384 } elsif ($opt_target eq "win") {
 385         die ("Windows tests will not run without root privileges.")
 386                 if (`whoami` ne "root");
 387
 388         die("Windows tests will not run with socket wrapper enabled.")
 389         if ($opt_socket_wrapper);
 390
 391         die("Windows tests will not run quickly.") if ($opt_quick);
 392
 393         die("Environment variable WINTESTCONF has not been defined.\n".
 394                 "Windows tests will not run unconfigured.") if (not defined($ENV{WINTESTCONF}));
 395
 396         die ("$ENV{WINTESTCONF} could not be read.") if (! -r $ENV{WINTESTCONF});
 397
 398         $ENV{WINTEST_DIR}="$ENV{SRCDIR}/script/tests/win";
 399 } elsif ($opt_target eq "none") {
 400 } else {
 401         die("unknown target `$opt_target'");
 402 }
 403
 404 foreach (keys %$testenv_vars) { $ENV{$_} = $testenv_vars->{$_}; }
 405
 406 if ($opt_socket_wrapper_pcap) {
 407         $ENV{SOCKET_WRAPPER_PCAP_FILE} = $opt_socket_wrapper_pcap;
 408         # Socket wrapper pcap implies socket wrapper
 409         $opt_socket_wrapper = 1;
 410 }
 411
 412 my $socket_wrapper_dir;
 413 if ($opt_socket_wrapper)
 414 {
 415         $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w");
 416         print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
 417 }
 418
 419 # Start slapd before smbd
 420 if ($ldap) {
 421         Samba4::slapd_start($ENV{SLAPD_CONF}, $ENV{LDAP_URI_ESCAPE}) or die("couldn't start slapd");
 422
 423     print "LDAP PROVISIONING...";
 424         Samba4::provision_ldap($bindir, $setupdir);
 425
 426     # LDAP is slow
 427         $torture_maxtime *= 2;
 428 }
 429
 430 if (defined($opt_expected_failures)) {
 431         open(KNOWN, "<$opt_expected_failures") or die("unable to read known failures file: $!");
 432         while (<KNOWN>) {
 433                 chomp;
 434                 s/([ \t]+)\#(.*)$//;
 435                 push (@expected_failures, $_); }
 436         close(KNOWN);
 437 }
 438
 439 if (defined($opt_skip)) {
 440         open(SKIP, "<$opt_skip") or die("unable to read skip file: $!");
 441         while (<SKIP>) {
 442                 chomp;
 443                 s/([ \t]+)\#(.*)$//;
 444                 push (@skips, $_); }
 445         close(SKIP);
 446 }
 447
 448 my $test_fifo = "$prefix/smbd_test.fifo";
 449
 450 $ENV{SMBD_TEST_FIFO} = $test_fifo;
 451 $ENV{SMBD_TEST_LOG} = "$prefix/smbd_test.log";
 452
 453 SocketWrapper::set_default_iface(1);
 454 my $max_time = 5400;
 455 if (defined($ENV{SMBD_MAX_TIME})) {
 456         $max_time = $ENV{SMBD_MAX_TIME};
 457 }
 458 Samba4::smbd_check_or_start($bindir, $test_fifo, $ENV{SMBD_TEST_LOG},
 459                         $socket_wrapper_dir, $max_time, $ENV{CONFFILE});
 460
 461 SocketWrapper::set_default_iface(6);
 462
 463 my $interfaces = join(',', ("127.0.0.6/8",
 464                                  "127.0.0.7/8",
 465                                                  "127.0.0.8/8",
 466                                                  "127.0.0.9/8",
 467                                                  "127.0.0.10/8",
 468                                                  "127.0.0.11/8"));
 469
 470 push (@torture_options, "--option=interfaces=$interfaces");
 471 push (@torture_options, $ENV{CONFIGURATION});
 472 # ensure any one smbtorture call doesn't run too long
 473 push (@torture_options, "--maximum-runtime=$torture_maxtime");
 474 push (@torture_options, "--target=$opt_target");
 475 push (@torture_options, "--option=torture:progress=no") if ($from_build_farm);
 476 push (@torture_options, "--format=subunit");
 477 push (@torture_options, "--option=torture:quick=yes") if ($opt_quick);
 478
 479 $ENV{TORTURE_OPTIONS} = join(' ', @torture_options);
 480 print "OPTIONS $ENV{TORTURE_OPTIONS}\n";
 481
 482 open(DATA, ">$test_fifo");
 483
 484 my @todo = ();
 485
 486 if ($opt_target eq "win") {
 487         system("$testsdir/test_win.sh");
 488 } else {
 489         if ($opt_quick) {
 490                 open(IN, "$testsdir/tests_quick.sh|");
 491         } else {
 492                 open(IN, "$testsdir/tests_all.sh|");
 493         }
 494         while (<IN>) {
 495                 if ($_ eq "-- TEST --\n") {
 496                         my $name = <IN>;
 497                         $name =~ s/\n//g;
 498                         my $cmdline = <IN>;
 499                         $cmdline =~ s/\n//g;
 500                         push (@todo, [$name, $cmdline])
 501                                 if (not defined($tests) or $name =~ /$tests/);
 502                 } else {
 503                         print;
 504                 }
 505         }
 506         close(IN) or die("Error creating recipe");
 507 }
 508
 509 Samba4::wait_for_start();
 510
 511 # start off with 0 failures
 512 $ENV{failed} = 0;
 513
 514 my $suitestotal = $#todo + 1;
 515 my $i = 0;
 516 $| = 1;
 517
 518 delete $ENV{DOMAIN};
 519
 520 if ($opt_testenv) {
 521         my $term = $ENV{TERM} or "xterm";
 522         system("$term -e 'echo -e \"Welcome to the Samba4 Test environment
 523 This matches the client environment used in make test
 524 smbd is pid `cat \$PIDDIR/smbd.pid`
 525
 526 Some useful environment variables:
 527 AUTH=\$AUTH
 528 TORTURE_OPTIONS=\$TORTURE_OPTIONS
 529 CONFIGURATION=\$CONFIGURATION
 530 SERVER=\$SERVER
 531 NETBIOSNAME=\$NETBIOSNAME\" && bash'");
 532 } else {
 533         foreach (@todo) {
 534                 $i++;
 535                 my $cmd = $$_[1];
 536                 $cmd =~ s/([\(\)])/\\$1/g;
 537                 my $name = $$_[0];
 538
 539                 if (skip($name)) {
 540                         print "SKIPPED: $name\n";
 541                         $statistics->{SUITES_SKIPPED}++;
 542                         next;
 543                 }
 544
 545                 if ($from_build_farm) {
 546                         run_test_buildfarm($name, $cmd, $i, $suitestotal);
 547                 } else {
 548                         run_test_plain($name, $cmd, $i, $suitestotal);
 549                 }
 550         }
 551 }
 552
 553 print "\n";
 554
 555 close(DATA);
 556
 557 sleep(2);
 558
 559 my $failed = $? >> 8;
 560
 561 if (-f "$ENV{PIDDIR}/smbd.pid" ) {
 562         open(IN, "<$ENV{PIDDIR}/smbd.pid") or die("unable to open smbd pid file");
 563         kill 9, <IN>;
 564         close(IN);
 565 }
 566
 567 Samba4::slapd_stop() if ($ldap);
 568
 569 my $end = time();
 570 my $duration = ($end-$start);
 571 my $numfailed = $#$suitesfailed+1;
 572 if ($numfailed == 0) {
 573         my $ok = $statistics->{TESTS_EXPECTED_OK} + $statistics->{TESTS_EXPECTED_FAIL};
 574         print "ALL OK ($ok tests in $statistics->{SUITES_OK} testsuites)\n";
 575 } else {
 576
 577         unless ($from_build_farm) {
 578                 if (not $opt_immediate and not $opt_verbose) {
 579                         foreach (@$suitesfailed) {
 580                                 print "===============================================================================\n";
 581                                 print "FAIL: $_\n";
 582                                 print $test_output->{$_};
 583                                 print "\n";
 584                         }
 585                 }
 586
 587                 print "FAILED ($statistics->{TESTS_UNEXPECTED_FAIL} failures and $statistics->{TESTS_ERROR} errors in $statistics->{SUITES_FAIL} testsuites)\n";
 588         } else {
 589                 print <<EOF
 590 ************************
 591 *** TESTSUITE FAILED ***
 592 ************************
 593 EOF
 594 ;
 595         }
 596 }
 597 print "DURATION: $duration seconds\n";
 598
 599 # if there were any valgrind failures, show them
 600 foreach (<$prefix/valgrind.log*>) {
 601         next unless (-s $_);
 602         system("grep DWARF2.CFI.reader $_ > /dev/null");
 603         if ($? >> 8 == 0) {
 604             print "VALGRIND FAILURE\n";
 605             $failed++;
 606             system("cat $_");
 607         }
 608 }
 609
 610 if ($from_build_farm) {
 611         print "TEST STATUS: $numfailed\n";
 612 }
 613
 614 exit $numfailed;