source4/script/tests/selftest.pl

   1 #!/usr/bin/perl
   2 # Bootstrap Samba and run a number of tests against it.
   3 # Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
   4 # Published under the GNU GPL, v3 or later.
   5
   6 =pod
   7
   8 =head1 NAME
   9
  10 selftest - Samba test runner
  11
  12 =head1 SYNOPSIS
  13
  14 selftest --help
  15
  16 selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win] [--socket-wrapper] [--quick] [--one] [--prefix=prefix] [--immediate] [TESTS]
  17
  18 =head1 DESCRIPTION
  19
  20 A simple test runner. TESTS is a regular expression with tests to run.
  21
  22 =head1 OPTIONS
  23
  24 =over 4
  25
  26 =item I<--help>
  27
  28 Show list of available options.
  29
  30 =item I<--srcdir=DIR>
  31
  32 Source directory.
  33
  34 =item I<--builddir=DIR>
  35
  36 Build directory.
  37
  38 =item I<--prefix=DIR>
  39
  40 Change directory to run tests in. Default is 'st'.
  41
  42 =item I<--immediate>
  43
  44 Show errors as soon as they happen rather than at the end of the test run.
  45
  46 =item I<--target samba4|samba3|win>
  47
  48 Specify test target against which to run. Default is 'samba4'.
  49
  50 =item I<--quick>
  51
  52 Run only a limited number of tests. Intended to run in about 30 seconds on
  53 moderately recent systems.
  54
  55 =item I<--socket-wrapper>
  56
  57 Use socket wrapper library for communication with server. Only works
  58 when the server is running locally.
  59
  60 Will prevent TCP and UDP ports being opened on the local host but
  61 (transparently) redirects these calls to use unix domain sockets.
  62
  63 =item I<--expected-failures>
  64
  65 Specify a file containing a list of tests that are expected to fail. Failures for
  66 these tests will be counted as successes, successes will be counted as failures.
  67
  68 The format for the file is, one entry per line:
  69
  70 TESTSUITE-NAME/TEST-NAME
  71
  72 =item I<--skip>
  73
  74 Specify a file containing a list of tests that should be skipped. Possible candidates are
  75 tests that segfault the server, flip or don't end.
  76
  77 =item I<--one>
  78
  79 Abort as soon as one test fails.
  80
  81 =back
  82
  83 =head1 ENVIRONMENT
  84
  85 =over 4
  86
  87 =item I<SMBD_VALGRIND>
  88
  89 =item I<TORTURE_MAXTIME>
  90
  91 =item I<VALGRIND>
  92
  93 =item I<TEST_LDAP>
  94
  95 =item I<TLS_ENABLED>
  96
  97 =item I<srcdir>
  98
  99 =back
 100
 101 =head1 LICENSE
 102
 103 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
 104
 105 =head1 AUTHOR
 106
 107 Jelmer Vernooij
 108
 109 =cut
 110
 111 use strict;
 112 use warnings;
 113
 114 use FindBin qw($RealBin $Script);
 115 use File::Spec;
 116 use Getopt::Long;
 117 use POSIX;
 118 use Cwd;
 119 use lib "$RealBin";
 120 use Samba4;
 121 use SocketWrapper;
 122
 123 my $opt_help = 0;
 124 my $opt_target = "samba4";
 125 my $opt_quick = 0;
 126 my $opt_socket_wrapper = 0;
 127 my $opt_socket_wrapper_pcap = undef;
 128 my $opt_one = 0;
 129 my $opt_immediate = 0;
 130 my $opt_expected_failures = undef;
 131 my $opt_skip = undef;
 132 my $opt_verbose = 0;
 133
 134 my $srcdir = ".";
 135 my $builddir = ".";
 136 my $prefix = "st";
 137
 138 my $suitesfailed = [];
 139 my $start = time();
 140 my @expected_failures = ();
 141 my @skips = ();
 142
 143 my $statistics = {
 144         SUITES_FAIL => 0,
 145         SUITES_OK => 0,
 146         SUITES_SKIPPED => 0,
 147
 148         TESTS_UNEXPECTED_OK => 0,
 149         TESTS_EXPECTED_OK => 0,
 150         TESTS_UNEXPECTED_FAIL => 0,
 151         TESTS_EXPECTED_FAIL => 0,
 152         TESTS_ERROR => 0
 153 };
 154
 155 sub expecting_failure($)
 156 {
 157         my $fullname = shift;
 158
 159         return 1 if (grep(/^$fullname$/, @expected_failures));
 160
 161         return 0;
 162 }
 163
 164 sub skip($)
 165 {
 166         my $fullname = shift;
 167
 168         return 1 if (grep(/^$fullname$/, @skips));
 169         return 0;
 170 }
 171
 172 sub run_test_buildfarm($$$$)
 173 {
 174         my ($name, $cmd, $i, $suitestotal) = @_;
 175         print "--==--==--==--==--==--==--==--==--==--==--\n";
 176         print "Running test $name (level 0 stdout)\n";
 177         print "--==--==--==--==--==--==--==--==--==--==--\n";
 178         system("date");
 179
 180         my $expected_ret = 1;
 181         my $open_tests = {};
 182         open(RESULT, "$cmd 2>&1|");
 183         while (<RESULT>) {
 184                 print;
 185                 if (/^test: (.+)\n/) {
 186                         $open_tests->{$1} = 1;
 187                 } elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
 188                         my $result = $1;
 189                         if ($1 eq "success") {
 190                                 delete $open_tests->{$2};
 191                                 if (expecting_failure("$name/$2")) {
 192                                         $statistics->{TESTS_UNEXPECTED_OK}++;
 193                                 } else {
 194                                         $statistics->{TESTS_EXPECTED_OK}++;
 195                                 }
 196                         } elsif ($1 eq "failure") {
 197                                 delete $open_tests->{$2};
 198                                 if (expecting_failure("$name/$2")) {
 199                                         $statistics->{TESTS_EXPECTED_FAIL}++;
 200                                         $expected_ret = 0;
 201                                 } else {
 202                                         $statistics->{TESTS_UNEXPECTED_FAIL}++;
 203                                 }
 204                         } elsif ($1 eq "skip") {
 205                                 delete $open_tests->{$2};
 206                         } elsif ($1 eq "error") {
 207                                 $statistics->{TESTS_ERROR}++;
 208                                 delete $open_tests->{$2};
 209                         }
 210                 }
 211         }
 212         print "COMMAND: $cmd\n";
 213         foreach (keys %$open_tests) {
 214                 print "$_ was started but never finished!\n";
 215                 $statistics->{TESTS_ERROR}++;
 216         }
 217         my $ret = close(RESULT);
 218
 219         print "==========================================\n";
 220         if ($ret == $expected_ret) {
 221                 print "TEST PASSED: $name\n";
 222         } else {
 223                 print "TEST FAILED: $name (status $ret)\n";
 224         }
 225         print "==========================================\n";
 226 }
 227
 228 my $test_output = {};
 229 sub run_test_plain($$$$)
 230 {
 231         my ($name, $cmd, $i, $totalsuites) = @_;
 232         my $err = "";
 233         if ($#$suitesfailed+1 > 0) { $err = ", ".($#$suitesfailed+1)." errors"; }
 234         printf "[$i/$totalsuites in " . (time() - $start)."s$err] $name\n";
 235         open(RESULT, "$cmd 2>&1|");
 236         my $expected_ret = 1;
 237         my $open_tests = {};
 238         $test_output->{$name} = "";
 239         while (<RESULT>) {
 240                 $test_output->{$name}.=$_;
 241                 print if ($opt_verbose);
 242                 if (/^test: (.+)\n/) {
 243                         $open_tests->{$1} = 1;
 244                 } elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
 245                         my $result = $1;
 246                         if ($1 eq "success") {
 247                                 delete $open_tests->{$2};
 248                                 if (expecting_failure("$name/$2")) {
 249                                         $statistics->{TESTS_UNEXPECTED_OK}++;
 250                                 } else {
 251                                         $statistics->{TESTS_EXPECTED_OK}++;
 252                                 }
 253                         } elsif ($1 eq "failure") {
 254                                 delete $open_tests->{$2};
 255                                 if (expecting_failure("$name/$2")) {
 256                                         $statistics->{TESTS_EXPECTED_FAIL}++;
 257                                         $expected_ret = 0;
 258                                 } else {
 259                                         $statistics->{TESTS_UNEXPECTED_FAIL}++;
 260                                 }
 261                         } elsif ($1 eq "skip") {
 262                                 delete $open_tests->{$2};
 263                         } elsif ($1 eq "error") {
 264                                 $statistics->{TESTS_ERROR}++;
 265                                 delete $open_tests->{$2};
 266                         }
 267                 }
 268         }
 269         $test_output->{$name}.="COMMAND: $cmd\n";
 270         foreach (keys %$open_tests) {
 271                 $test_output->{$name}.="$_ was started but never finished!\n";
 272                 $statistics->{TESTS_ERROR}++;
 273         }
 274         my $ret = close(RESULT);
 275         if ($ret != $expected_ret and ($opt_immediate or $opt_one) and not $opt_verbose) {
 276                 print "$test_output->{$name}\n";
 277         }
 278         if ($ret != $expected_ret) {
 279                 push(@$suitesfailed, $name);
 280                 $statistics->{SUITES_FAIL}++;
 281                 exit(1) if ($opt_one);
 282         } else {
 283                 $statistics->{SUITES_OK}++;
 284         }
 285 }
 286
 287 sub ShowHelp()
 288 {
 289         print "Samba test runner
 290 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
 291
 292 Usage: $Script [OPTIONS] PREFIX
 293
 294 Generic options:
 295  --help                     this help page
 296
 297 Paths:
 298  --prefix=DIR               prefix to run tests in [st]
 299  --srcdir=DIR               source directory [.]
 300  --builddir=DIR             output directory [.]
 301
 302 Target Specific:
 303  --target=samba4|samba3|win Samba version to target
 304  --socket-wrapper-pcap=FILE save traffic to pcap file
 305  --socket-wrapper           enable socket wrapper
 306  --expected-failures=FILE   specify list of tests that is guaranteed to fail
 307
 308 Behaviour:
 309  --quick                    run quick overall test
 310  --one                      abort when the first test fails
 311  --immediate                print test output for failed tests during run
 312  --verbose                  be verbose
 313 ";
 314         exit(0);
 315 }
 316
 317 my $result = GetOptions (
 318             'help|h|?' => \$opt_help,
 319                 'target=s' => \$opt_target,
 320                 'prefix=s' => \$prefix,
 321                 'socket-wrapper' => \$opt_socket_wrapper,
 322                 'socket-wrapper-pcap=s' => \$opt_socket_wrapper_pcap,
 323                 'quick' => \$opt_quick,
 324                 'one' => \$opt_one,
 325                 'immediate' => \$opt_immediate,
 326                 'expected-failures=s' => \$opt_expected_failures,
 327                 'skip=s' => \$opt_skip,
 328                 'srcdir=s' => \$srcdir,
 329                 'builddir=s' => \$builddir,
 330                 'verbose' => \$opt_verbose
 331             );
 332
 333 exit(1) if (not $result);
 334
 335 ShowHelp() if ($opt_help);
 336
 337 my $tests = shift;
 338
 339 my $torture_maxtime = $ENV{TORTURE_MAXTIME};
 340 unless (defined($torture_maxtime)) {
 341         $torture_maxtime = 1200;
 342 }
 343
 344 # quick hack to disable rpc validation when using valgrind - its way too slow
 345 unless (defined($ENV{VALGRIND})) {
 346         $ENV{VALIDATE} = "validate";
 347 }
 348
 349 my $old_pwd = "$RealBin/../..";
 350 my $ldap = (defined($ENV{TEST_LDAP}) and ($ENV{TEST_LDAP} eq "yes"))?1:0;
 351
 352 $prefix =~ s+//+/+;
 353 $ENV{PREFIX} = $prefix;
 354
 355 $ENV{SRCDIR} = $srcdir;
 356
 357 my $bindir = "$srcdir/bin";
 358 my $setupdir = "$srcdir/setup";
 359 my $testsdir = "$srcdir/script/tests";
 360
 361 my $tls_enabled = not $opt_quick;
 362 my $from_build_farm = (defined($ENV{RUN_FROM_BUILD_FARM}) and
 363                       ($ENV{RUN_FROM_BUILD_FARM} eq "yes"));
 364
 365 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
 366 $ENV{LD_LDB_MODULE_PATH} = "$old_pwd/bin/modules/ldb";
 367 $ENV{LD_SAMBA_MODULE_PATH} = "$old_pwd/bin/modules";
 368 if (defined($ENV{LD_LIBRARY_PATH})) {
 369         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared:$ENV{LD_LIBRARY_PATH}";
 370 } else {
 371         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared";
 372 }
 373 $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig:$ENV{PKG_CONFIG_PATH}";
 374 $ENV{PATH} = "$old_pwd/bin:$ENV{PATH}";
 375
 376 my @torture_options = ();
 377
 378 my $testenv_vars = {};
 379
 380 if ($opt_target eq "samba4") {
 381         $testenv_vars = Samba4::provision($prefix);
 382 } elsif ($opt_target eq "win") {
 383         die ("Windows tests will not run without root privileges.")
 384                 if (`whoami` ne "root");
 385
 386         die("Windows tests will not run with socket wrapper enabled.")
 387         if ($opt_socket_wrapper);
 388
 389         die("Windows tests will not run quickly.") if ($opt_quick);
 390
 391         die("Environment variable WINTESTCONF has not been defined.\n".
 392                 "Windows tests will not run unconfigured.") if (not defined($ENV{WINTESTCONF}));
 393
 394         die ("$ENV{WINTESTCONF} could not be read.") if (! -r $ENV{WINTESTCONF});
 395
 396         $ENV{WINTEST_DIR}="$ENV{SRCDIR}/script/tests/win";
 397 } elsif ($opt_target eq "none") {
 398 } else {
 399         die("unknown target `$opt_target'");
 400 }
 401
 402 foreach (keys %$testenv_vars) { $ENV{$_} = $testenv_vars->{$_}; }
 403
 404 if ($opt_socket_wrapper_pcap) {
 405         $ENV{SOCKET_WRAPPER_PCAP_FILE} = $opt_socket_wrapper_pcap;
 406         # Socket wrapper pcap implies socket wrapper
 407         $opt_socket_wrapper = 1;
 408 }
 409
 410 my $socket_wrapper_dir;
 411 if ($opt_socket_wrapper)
 412 {
 413         $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w");
 414         print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
 415 }
 416
 417 # Start slapd before smbd
 418 if ($ldap) {
 419         Samba4::slapd_start($ENV{SLAPD_CONF}, $ENV{LDAPI_ESCAPE}) or die("couldn't start slapd");
 420
 421     print "LDAP PROVISIONING...";
 422         Samba4::provision_ldap($bindir, $setupdir);
 423
 424     # LDAP is slow
 425         $torture_maxtime *= 2;
 426 }
 427
 428 if (defined($opt_expected_failures)) {
 429         open(KNOWN, "<$opt_expected_failures") or die("unable to read known failures file: $!");
 430         while (<KNOWN>) {
 431                 chomp;
 432                 s/([ \t]+)\#(.*)$//;
 433                 push (@expected_failures, $_); }
 434         close(KNOWN);
 435 }
 436
 437 if (defined($opt_skip)) {
 438         open(SKIP, "<$opt_skip") or die("unable to read skip file: $!");
 439         while (<SKIP>) {
 440                 chomp;
 441                 s/([ \t]+)\#(.*)$//;
 442                 push (@skips, $_); }
 443         close(SKIP);
 444 }
 445
 446 my $test_fifo = "$prefix/smbd_test.fifo";
 447
 448 $ENV{SMBD_TEST_FIFO} = $test_fifo;
 449 $ENV{SMBD_TEST_LOG} = "$prefix/smbd_test.log";
 450
 451 SocketWrapper::set_default_iface(1);
 452 my $max_time = 5400;
 453 if (defined($ENV{SMBD_MAX_TIME})) {
 454         $max_time = $ENV{SMBD_MAX_TIME};
 455 }
 456 Samba4::smbd_check_or_start($bindir, $test_fifo, $ENV{SMBD_TEST_LOG},
 457                         $socket_wrapper_dir, $max_time, $ENV{CONFFILE});
 458
 459 SocketWrapper::set_default_iface(6);
 460
 461 my $interfaces = join(',', ("127.0.0.6/8",
 462                                  "127.0.0.7/8",
 463                                                  "127.0.0.8/8",
 464                                                  "127.0.0.9/8",
 465                                                  "127.0.0.10/8",
 466                                                  "127.0.0.11/8"));
 467
 468 push (@torture_options, "--option=interfaces=$interfaces");
 469 push (@torture_options, $ENV{CONFIGURATION});
 470 # ensure any one smbtorture call doesn't run too long
 471 push (@torture_options, "--maximum-runtime=$torture_maxtime");
 472 push (@torture_options, "--target=$opt_target");
 473 push (@torture_options, "--option=torture:progress=no") if ($from_build_farm);
 474 push (@torture_options, "--format=subunit");
 475 push (@torture_options, "--option=torture:quick=yes") if ($opt_quick);
 476
 477 $ENV{TORTURE_OPTIONS} = join(' ', @torture_options);
 478 print "OPTIONS $ENV{TORTURE_OPTIONS}\n";
 479
 480 open(DATA, ">$test_fifo");
 481
 482 my @todo = ();
 483
 484 if ($opt_target eq "win") {
 485         system("$testsdir/test_win.sh");
 486 } else {
 487         if ($opt_quick) {
 488                 open(IN, "$testsdir/tests_quick.sh|");
 489         } else {
 490                 open(IN, "$testsdir/tests_all.sh|");
 491         }
 492         while (<IN>) {
 493                 if ($_ eq "-- TEST --\n") {
 494                         my $name = <IN>;
 495                         $name =~ s/\n//g;
 496                         my $cmdline = <IN>;
 497                         $cmdline =~ s/\n//g;
 498                         push (@todo, [$name, $cmdline])
 499                                 if (not defined($tests) or $name =~ /$tests/);
 500                 } else {
 501                         print;
 502                 }
 503         }
 504         close(IN) or die("Error creating recipe");
 505 }
 506
 507 Samba4::wait_for_start();
 508
 509 # start off with 0 failures
 510 $ENV{failed} = 0;
 511
 512 my $suitestotal = $#todo + 1;
 513 my $i = 0;
 514 $| = 1;
 515
 516 delete $ENV{DOMAIN};
 517
 518 foreach (@todo) {
 519         $i++;
 520         my $cmd = $$_[1];
 521         $cmd =~ s/([\(\)])/\\$1/g;
 522         my $name = $$_[0];
 523
 524         if (skip($name)) {
 525                 print "SKIPPED: $name\n";
 526                 $statistics->{SUITES_SKIPPED}++;
 527                 next;
 528         }
 529
 530         if ($from_build_farm) {
 531                 run_test_buildfarm($name, $cmd, $i, $suitestotal);
 532         } else {
 533                 run_test_plain($name, $cmd, $i, $suitestotal);
 534         }
 535 }
 536
 537 print "\n";
 538
 539 close(DATA);
 540
 541 sleep(2);
 542
 543 my $failed = $? >> 8;
 544
 545 if (-f "$ENV{PIDDIR}/smbd.pid" ) {
 546         open(IN, "<$ENV{PIDDIR}/smbd.pid") or die("unable to open smbd pid file");
 547         kill 9, <IN>;
 548         close(IN);
 549 }
 550
 551 Samba4::slapd_stop() if ($ldap);
 552
 553 my $end = time();
 554 my $duration = ($end-$start);
 555 my $numfailed = $#$suitesfailed+1;
 556 if ($numfailed == 0) {
 557         my $ok = $statistics->{TESTS_EXPECTED_OK} + $statistics->{TESTS_EXPECTED_FAIL};
 558         print "ALL OK ($ok tests in $statistics->{SUITES_OK} testsuites)\n";
 559 } else {
 560
 561         unless ($from_build_farm) {
 562                 if (not $opt_immediate and not $opt_verbose) {
 563                         foreach (@$suitesfailed) {
 564                                 print "===============================================================================\n";
 565                                 print "FAIL: $_\n";
 566                                 print $test_output->{$_};
 567                                 print "\n";
 568                         }
 569                 }
 570
 571                 print "FAILED ($statistics->{TESTS_UNEXPECTED_FAIL} failures and $statistics->{TESTS_ERROR} errors in $statistics->{SUITES_FAIL} testsuites)\n";
 572         } else {
 573                 print <<EOF
 574 ************************
 575 *** TESTSUITE FAILED ***
 576 ************************
 577 EOF
 578 ;
 579         }
 580 }
 581 print "DURATION: $duration seconds\n";
 582
 583 # if there were any valgrind failures, show them
 584 foreach (<$prefix/valgrind.log*>) {
 585         next unless (-s $_);
 586         system("grep DWARF2.CFI.reader $_ > /dev/null");
 587         if ($? >> 8 == 0) {
 588             print "VALGRIND FAILURE\n";
 589             $failed++;
 590             system("cat $_");
 591         }
 592 }
 593
 594 if ($from_build_farm) {
 595         print "TEST STATUS: $numfailed\n";
 596 }
 597
 598 exit $numfailed;