source/selftest/selftest.pl

   1 #!/usr/bin/perl
   2 # Bootstrap Samba and run a number of tests against it.
   3 # Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
   4 # Published under the GNU GPL, v3 or later.
   5
   6 =pod
   7
   8 =head1 NAME
   9
  10 selftest - Samba test runner
  11
  12 =head1 SYNOPSIS
  13
  14 selftest --help
  15
  16 selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win] [--socket-wrapper] [--quick] [--one] [--prefix=prefix] [--immediate] [TESTS]
  17
  18 =head1 DESCRIPTION
  19
  20 A simple test runner. TESTS is a regular expression with tests to run.
  21
  22 =head1 OPTIONS
  23
  24 =over 4
  25
  26 =item I<--help>
  27
  28 Show list of available options.
  29
  30 =item I<--srcdir=DIR>
  31
  32 Source directory.
  33
  34 =item I<--builddir=DIR>
  35
  36 Build directory.
  37
  38 =item I<--prefix=DIR>
  39
  40 Change directory to run tests in. Default is 'st'.
  41
  42 =item I<--immediate>
  43
  44 Show errors as soon as they happen rather than at the end of the test run.
  45
  46 =item I<--target samba4|samba3|win>
  47
  48 Specify test target against which to run. Default is 'samba4'.
  49
  50 =item I<--quick>
  51
  52 Run only a limited number of tests. Intended to run in about 30 seconds on
  53 moderately recent systems.
  54
  55 =item I<--socket-wrapper>
  56
  57 Use socket wrapper library for communication with server. Only works
  58 when the server is running locally.
  59
  60 Will prevent TCP and UDP ports being opened on the local host but
  61 (transparently) redirects these calls to use unix domain sockets.
  62
  63 =item I<--expected-failures>
  64
  65 Specify a file containing a list of tests that are expected to fail. Failures for
  66 these tests will be counted as successes, successes will be counted as failures.
  67
  68 The format for the file is, one entry per line:
  69
  70 TESTSUITE-NAME/TEST-NAME
  71
  72 =item I<--skip>
  73
  74 Specify a file containing a list of tests that should be skipped. Possible candidates are
  75 tests that segfault the server, flip or don't end.
  76
  77 =item I<--one>
  78
  79 Abort as soon as one test fails.
  80
  81 =back
  82
  83 =head1 ENVIRONMENT
  84
  85 =over 4
  86
  87 =item I<SMBD_VALGRIND>
  88
  89 =item I<TORTURE_MAXTIME>
  90
  91 =item I<VALGRIND>
  92
  93 =item I<TLS_ENABLED>
  94
  95 =item I<srcdir>
  96
  97 =back
  98
  99 =head1 LICENSE
 100
 101 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
 102
 103 =head1 AUTHOR
 104
 105 Jelmer Vernooij
 106
 107 =cut
 108
 109 use strict;
 110
 111 use FindBin qw($RealBin $Script);
 112 use File::Spec;
 113 use Getopt::Long;
 114 use POSIX;
 115 use Cwd qw(abs_path);
 116 use lib "$RealBin";
 117 use Samba3;
 118 use Samba4;
 119 use SocketWrapper;
 120
 121 my $opt_help = 0;
 122 my $opt_target = "samba4";
 123 my $opt_quick = 0;
 124 my $opt_socket_wrapper = 0;
 125 my $opt_socket_wrapper_pcap = undef;
 126 my $opt_socket_wrapper_keep_pcap = undef;
 127 my $opt_one = 0;
 128 my $opt_immediate = 0;
 129 my $opt_expected_failures = undef;
 130 my $opt_skip = undef;
 131 my $opt_verbose = 0;
 132 my $opt_testenv = 0;
 133 my $ldap = undef;
 134 my $opt_analyse_cmd = undef;
 135 my $opt_resetup_env = undef;
 136 my $opt_bindir = undef;
 137 my $opt_no_lazy_setup = undef;
 138
 139 my $srcdir = ".";
 140 my $builddir = ".";
 141 my $prefix = "./st";
 142
 143 my $suitesfailed = [];
 144 my $start = time();
 145 my @expected_failures = ();
 146 my @skips = ();
 147
 148 my $statistics = {
 149         SUITES_FAIL => 0,
 150         SUITES_OK => 0,
 151         SUITES_SKIPPED => 0,
 152
 153         TESTS_UNEXPECTED_OK => 0,
 154         TESTS_EXPECTED_OK => 0,
 155         TESTS_UNEXPECTED_FAIL => 0,
 156         TESTS_EXPECTED_FAIL => 0,
 157         TESTS_ERROR => 0
 158 };
 159
 160 sub expecting_failure($)
 161 {
 162         my $fullname = shift;
 163
 164         foreach (@expected_failures) {
 165                 return 1 if ($fullname =~ /$_/);
 166         }
 167
 168         return 0;
 169 }
 170
 171 sub skip($)
 172 {
 173         my $fullname = shift;
 174
 175         foreach (@skips) {
 176                 return 1 if ($fullname =~ /$_/);
 177         }
 178
 179         return 0;
 180 }
 181
 182 sub getlog_env($);
 183
 184 my $test_output = {};
 185
 186 sub buildfarm_start_msg($)
 187 {
 188         my ($state) = @_;
 189         my $out = "";
 190
 191         $out .= "--==--==--==--==--==--==--==--==--==--==--\n";
 192         $out .= "Running test $state->{NAME} (level 0 stdout)\n";
 193         $out .= "--==--==--==--==--==--==--==--==--==--==--\n";
 194         $out .= scalar(localtime())."\n";
 195         $out .= "SELFTEST RUNTIME: " . ($state->{START} - $start) . "s\n";
 196         $out .= "NAME: $state->{NAME}\n";
 197         $out .= "CMD: $state->{CMD}\n";
 198
 199         $test_output->{$state->{NAME}} = "";
 200
 201         print $out;
 202 }
 203
 204 sub buildfarm_output_msg($$)
 205 {
 206         my ($state, $output) = @_;
 207
 208         $test_output->{$state->{NAME}} .= $output;
 209 }
 210
 211 sub buildfarm_end_msg($$$)
 212 {
 213         my ($state, $expected_ret, $ret) = @_;
 214         my $out = "";
 215
 216         $out .= "TEST RUNTIME: " . (time() - $state->{START}) . "s\n";
 217
 218         if ($ret == $expected_ret) {
 219                 $out .= "ALL OK\n";
 220         } else {
 221                 $out .= "ERROR: $ret";
 222                 $out .= $test_output->{$state->{NAME}};
 223         }
 224
 225         $out .= "PCAP FILE: $state->{PCAP_FILE}\n" if defined($state->{PCAP_FILE});
 226
 227         $out .= getlog_env($state->{ENVNAME});
 228
 229         $out .= "==========================================\n";
 230         if ($ret == $expected_ret) {
 231                 $out .= "TEST PASSED: $state->{NAME}\n";
 232         } else {
 233                 $out .= "TEST FAILED: $state->{NAME} (status $ret)\n";
 234         }
 235         $out .= "==========================================\n";
 236
 237         print $out;
 238 }
 239
 240 my $buildfarm_msg_ops = {
 241         start_msg       => \&buildfarm_start_msg,
 242         output_msg      => \&buildfarm_output_msg,
 243         end_msg         => \&buildfarm_end_msg
 244 };
 245
 246 sub plain_output_msg($$);
 247
 248 sub plain_start_msg($)
 249 {
 250         my ($state) = @_;
 251         my $out = "";
 252
 253         $out .= "[$state->{INDEX}/$state->{TOTAL} in " . ($state->{START} - $start) . "s";
 254         $out .= ", ".($#$suitesfailed+1)." errors" if ($#$suitesfailed+1 > 0);
 255         $out .= "] $state->{NAME}\n";
 256
 257         $test_output->{$state->{NAME}} = "" unless $opt_verbose;
 258
 259         plain_output_msg($state, "CMD: $state->{CMD}\n");
 260
 261         print $out;
 262 }
 263
 264 sub plain_output_msg($$)
 265 {
 266         my ($state, $output) = @_;
 267
 268         if ($opt_verbose) {
 269                 print $output;
 270         } else {
 271                 $test_output->{$state->{NAME}} .= $output;
 272         }
 273 }
 274
 275 sub plain_end_msg($$$)
 276 {
 277         my ($state, $expected_ret, $ret) = @_;
 278         my $out = "";
 279
 280         if ($ret != $expected_ret) {
 281                 plain_output_msg($state, "ERROR: $ret\n");
 282         }
 283
 284         if ($ret != $expected_ret and ($opt_immediate or $opt_one) and not $opt_verbose) {
 285                 $out .= $test_output->{$state->{NAME}};
 286         }
 287
 288         if (not $opt_socket_wrapper_keep_pcap and defined($state->{PCAP_FILE})) {
 289                 $out .= "PCAP FILE: $state->{PCAP_FILE}\n";
 290         }
 291
 292         $out .= getlog_env($state->{ENVNAME});
 293
 294         print $out;
 295 }
 296
 297 my $plain_msg_ops = {
 298         start_msg       => \&plain_start_msg,
 299         output_msg      => \&plain_output_msg,
 300         end_msg         => \&plain_end_msg
 301 };
 302
 303 sub setup_pcap($)
 304 {
 305         my ($state) = @_;
 306
 307         return unless ($opt_socket_wrapper_pcap);
 308         return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
 309
 310         my $fname = sprintf("t%03u_%s", $state->{INDEX}, $state->{NAME});
 311         $fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
 312
 313         $state->{PCAP_FILE} = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
 314
 315         SocketWrapper::setup_pcap($state->{PCAP_FILE});
 316 }
 317
 318 sub cleanup_pcap($$$)
 319 {
 320         my ($state, $expected_ret, $ret) = @_;
 321
 322         return unless ($opt_socket_wrapper_pcap);
 323         return if ($opt_socket_wrapper_keep_pcap);
 324         return unless ($expected_ret == $ret);
 325         return unless defined($state->{PCAP_FILE});
 326
 327         unlink($state->{PCAP_FILE});
 328         $state->{PCAP_FILE} = undef;
 329 }
 330
 331 sub run_test($$$$$$)
 332 {
 333         my ($envname, $name, $cmd, $i, $totalsuites, $msg_ops) = @_;
 334         my $expected_ret = 1;
 335         my $open_tests = {};
 336         my $msg_state = {
 337                 ENVNAME => $envname,
 338                 NAME    => $name,
 339                 CMD     => $cmd,
 340                 INDEX   => $i,
 341                 TOTAL   => $totalsuites,
 342                 START   => time()
 343         };
 344
 345         setup_pcap($msg_state);
 346
 347         $msg_ops->{start_msg}->($msg_state);
 348
 349         open(RESULT, "$cmd 2>&1|");
 350         while (<RESULT>) {
 351                 $msg_ops->{output_msg}->($msg_state, $_);
 352                 if (/^test: (.+)\n/) {
 353                         $open_tests->{$1} = 1;
 354                 } elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
 355                         my $result = $1;
 356                         if ($1 eq "success") {
 357                                 delete $open_tests->{$2};
 358                                 if (expecting_failure("$name/$2")) {
 359                                         $statistics->{TESTS_UNEXPECTED_OK}++;
 360                                 } else {
 361                                         $statistics->{TESTS_EXPECTED_OK}++;
 362                                 }
 363                         } elsif ($1 eq "failure") {
 364                                 delete $open_tests->{$2};
 365                                 if (expecting_failure("$name/$2")) {
 366                                         $statistics->{TESTS_EXPECTED_FAIL}++;
 367                                         $expected_ret = 0;
 368                                 } else {
 369                                         print "n:$name/$2l\n";
 370                                         $statistics->{TESTS_UNEXPECTED_FAIL}++;
 371                                 }
 372                         } elsif ($1 eq "skip") {
 373                                 delete $open_tests->{$2};
 374                         } elsif ($1 eq "error") {
 375                                 $statistics->{TESTS_ERROR}++;
 376                                 delete $open_tests->{$2};
 377                         }
 378                 }
 379         }
 380         foreach (keys %$open_tests) {
 381                 $msg_ops->{output_msg}->($msg_state, "$_ was started but never finished!\n");
 382                 $statistics->{TESTS_ERROR}++;
 383         }
 384         my $ret = close(RESULT);
 385
 386         cleanup_pcap($msg_state,  $expected_ret, $ret);
 387
 388         $msg_ops->{end_msg}->($msg_state, $expected_ret, $ret);
 389
 390         if ($ret != $expected_ret) {
 391                 push(@$suitesfailed, $name);
 392                 $statistics->{SUITES_FAIL}++;
 393                 exit(1) if ($opt_one);
 394         } else {
 395                 $statistics->{SUITES_OK}++;
 396         }
 397
 398         return ($ret == $expected_ret);
 399 }
 400
 401 sub ShowHelp()
 402 {
 403         print "Samba test runner
 404 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
 405
 406 Usage: $Script [OPTIONS] PREFIX
 407
 408 Generic options:
 409  --help                     this help page
 410  --target=samba4|samba3|win Samba version to target
 411
 412 Paths:
 413  --prefix=DIR               prefix to run tests in [st]
 414  --srcdir=DIR               source directory [.]
 415  --builddir=DIR             output directory [.]
 416
 417 Target Specific:
 418  --socket-wrapper-pcap=DIR      save traffic to pcap directories
 419  --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
 420                             failed
 421  --socket-wrapper           enable socket wrapper
 422  --expected-failures=FILE   specify list of tests that is guaranteed to fail
 423
 424 Samba4 Specific:
 425  --ldap=openldap|fedora     back smbd onto specified ldap server
 426
 427 Samba3 Specific:
 428  --bindir=PATH              path to binaries
 429
 430 Behaviour:
 431  --quick                    run quick overall test
 432  --one                      abort when the first test fails
 433  --immediate                print test output for failed tests during run
 434  --verbose                  be verbose
 435  --analyse-cmd CMD          command to run after each test
 436 ";
 437         exit(0);
 438 }
 439
 440 my $result = GetOptions (
 441                 'help|h|?' => \$opt_help,
 442                 'target=s' => \$opt_target,
 443                 'prefix=s' => \$prefix,
 444                 'socket-wrapper' => \$opt_socket_wrapper,
 445                 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
 446                 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
 447                 'quick' => \$opt_quick,
 448                 'one' => \$opt_one,
 449                 'immediate' => \$opt_immediate,
 450                 'expected-failures=s' => \$opt_expected_failures,
 451                 'skip=s' => \$opt_skip,
 452                 'srcdir=s' => \$srcdir,
 453                 'builddir=s' => \$builddir,
 454                 'verbose' => \$opt_verbose,
 455                 'testenv' => \$opt_testenv,
 456                 'ldap:s' => \$ldap,
 457                 'analyse-cmd=s' => \$opt_analyse_cmd,
 458                 'no-lazy-setup' => \$opt_no_lazy_setup,
 459                 'resetup-environment' => \$opt_resetup_env,
 460                 'bindir:s' => \$opt_bindir,
 461             );
 462
 463 exit(1) if (not $result);
 464
 465 ShowHelp() if ($opt_help);
 466
 467 my $tests = shift;
 468
 469 # quick hack to disable rpc validation when using valgrind - its way too slow
 470 unless (defined($ENV{VALGRIND})) {
 471         $ENV{VALIDATE} = "validate";
 472         $ENV{MALLOC_CHECK_} = 2;
 473 }
 474
 475 my $old_pwd = "$RealBin/..";
 476
 477 # Backwards compatibility:
 478 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
 479         if (defined($ENV{FEDORA_DS_PREFIX})) {
 480                 $ldap = "fedora";
 481         } else {
 482                 $ldap = "openldap";
 483         }
 484 }
 485
 486 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
 487 if ($ldap) {
 488         # LDAP is slow
 489         $torture_maxtime *= 2;
 490 }
 491
 492 $prefix =~ s+//+/+;
 493 $prefix =~ s+/./+/+;
 494 $prefix =~ s+/$++;
 495
 496 die("using an empty prefix isn't allowed") unless $prefix ne "";
 497
 498 #Ensure we have the test prefix around
 499 mkdir($prefix, 0777) unless -d $prefix;
 500
 501 my $prefix_abs = abs_path($prefix);
 502 my $srcdir_abs = abs_path($srcdir);
 503
 504 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
 505 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
 506
 507 $ENV{PREFIX} = $prefix;
 508 $ENV{SRCDIR} = $srcdir;
 509
 510 my $tls_enabled = not $opt_quick;
 511 my $from_build_farm = (defined($ENV{RUN_FROM_BUILD_FARM}) and
 512                       ($ENV{RUN_FROM_BUILD_FARM} eq "yes"));
 513
 514 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
 515 $ENV{LD_LDB_MODULE_PATH} = "$old_pwd/bin/modules/ldb";
 516 $ENV{LD_SAMBA_MODULE_PATH} = "$old_pwd/bin/modules";
 517 if (defined($ENV{LD_LIBRARY_PATH})) {
 518         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared:$ENV{LD_LIBRARY_PATH}";
 519 } else {
 520         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared";
 521 }
 522 $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig:$ENV{PKG_CONFIG_PATH}";
 523 $ENV{PATH} = "$old_pwd/bin:$ENV{PATH}";
 524
 525
 526 if ($opt_socket_wrapper_pcap) {
 527         # Socket wrapper pcap implies socket wrapper
 528         $opt_socket_wrapper = 1;
 529 }
 530
 531 my $socket_wrapper_dir;
 532 if ($opt_socket_wrapper) {
 533         $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w", $opt_socket_wrapper_pcap);
 534         print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
 535 } else {
 536         warn("Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports") unless $< == 0;
 537 }
 538
 539 my $target;
 540
 541 if ($opt_target eq "samba4") {
 542         $target = new Samba4("$srcdir/bin", $ldap, "$srcdir/setup");
 543 } elsif ($opt_target eq "samba3") {
 544         if ($opt_socket_wrapper and `smbd -b | grep SOCKET_WRAPPER` eq "") {
 545                 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'.  Exiting....");
 546         }
 547
 548         $target = new Samba3($opt_bindir);
 549 } elsif ($opt_target eq "win") {
 550         die("Windows tests will not run with socket wrapper enabled.")
 551                 if ($opt_socket_wrapper);
 552         $target = new Windows();
 553 }
 554
 555 if (defined($opt_expected_failures)) {
 556         open(KNOWN, "<$opt_expected_failures") or die("unable to read known failures file: $!");
 557         while (<KNOWN>) {
 558                 chomp;
 559                 s/([ \t]+)\#(.*)$//;
 560                 push (@expected_failures, $_); }
 561         close(KNOWN);
 562 }
 563
 564 if (defined($opt_skip)) {
 565         open(SKIP, "<$opt_skip") or die("unable to read skip file: $!");
 566         while (<SKIP>) {
 567                 chomp;
 568                 s/([ \t]+)\#(.*)$//;
 569                 push (@skips, $_); }
 570         close(SKIP);
 571 }
 572
 573 my $interfaces = join(',', ("127.0.0.6/8",
 574                             "127.0.0.7/8",
 575                             "127.0.0.8/8",
 576                             "127.0.0.9/8",
 577                             "127.0.0.10/8",
 578                             "127.0.0.11/8"));
 579
 580 my $conffile = "$prefix_abs/client/client.conf";
 581
 582 sub write_clientconf($$)
 583 {
 584         my ($conffile, $vars) = @_;
 585
 586         mkdir("$prefix/client", 0777) unless -d "$prefix/client";
 587
 588         if ( -d "$prefix/client/private" ) {
 589                 unlink <$prefix/client/private/*>;
 590         } else {
 591                 mkdir("$prefix/client/private", 0777);
 592         }
 593
 594         open(CF, ">$conffile");
 595         print CF "[global]\n";
 596         if (defined($ENV{VALGRIND})) {
 597                 print CF "\ticonv:native = true\n";
 598         } else {
 599                 print CF "\ticonv:native = false\n";
 600         }
 601         print CF "\tnetbios name = client\n";
 602         if (defined($vars->{DOMAIN})) {
 603                 print CF "\tworkgroup = $vars->{DOMAIN}\n";
 604         }
 605         if (defined($vars->{REALM})) {
 606                 print CF "\trealm = $vars->{REALM}\n";
 607         }
 608         if (defined($vars->{NCALRPCDIR})) {
 609                 print CF "\tncalrpc dir = $vars->{NCALRPCDIR}\n";
 610         }
 611         if (defined($vars->{PIDDIR})) {
 612                 print CF "\tpid directory = $vars->{PIDDIR}\n";
 613         }
 614         if (defined($vars->{WINBINDD_SOCKET_DIR})) {
 615                 print CF "\twinbindd socket directory = $vars->{WINBINDD_SOCKET_DIR}\n";
 616         }
 617         print CF "
 618         private dir = $prefix_abs/client/private
 619         js include = $srcdir_abs/scripting/libjs
 620         name resolve order = bcast
 621         interfaces = $interfaces
 622         panic action = $srcdir_abs/script/gdb_backtrace \%PID\% \%PROG\%
 623         max xmit = 32K
 624         notify:inotify = false
 625         ldb:nosync = true
 626         system:anonymous = true
 627         torture:basedir = $prefix_abs/client
 628 #We don't want to pass our self-tests if the PAC code is wrong
 629         gensec:require_pac = true
 630 ";
 631         close(CF);
 632 }
 633
 634
 635 my @torture_options = ();
 636 push (@torture_options, "--configfile=$conffile");
 637 # ensure any one smbtorture call doesn't run too long
 638 push (@torture_options, "--maximum-runtime=$torture_maxtime");
 639 push (@torture_options, "--target=$opt_target");
 640 push (@torture_options, "--option=torture:progress=no") if ($from_build_farm);
 641 push (@torture_options, "--format=subunit");
 642 push (@torture_options, "--option=torture:quick=yes") if ($opt_quick);
 643
 644 $ENV{TORTURE_OPTIONS} = join(' ', @torture_options);
 645 print "OPTIONS $ENV{TORTURE_OPTIONS}\n";
 646
 647 my @todo = ();
 648
 649 my $testsdir = "$srcdir/selftest";
 650 $ENV{SMB_CONF_PATH} = "$conffile";
 651 $ENV{CONFIGURATION} = "--configfile=$conffile";
 652
 653 my %required_envs = ();
 654
 655 if ($opt_quick) {
 656         open(IN, "$testsdir/tests_quick.sh|");
 657 } else {
 658         open(IN, "$testsdir/tests_all.sh|");
 659 }
 660 while (<IN>) {
 661         if ($_ eq "-- TEST --\n") {
 662                 my $name = <IN>;
 663                 $name =~ s/\n//g;
 664                 my $env = <IN>;
 665                 $env =~ s/\n//g;
 666                 my $cmdline = <IN>;
 667                 $cmdline =~ s/\n//g;
 668                 if (not defined($tests) or $name =~ /$tests/) {
 669                         $required_envs{$env} = 1;
 670                         push (@todo, [$name, $env, $cmdline]);
 671                 }
 672         } else {
 673                 print;
 674         }
 675 }
 676 close(IN) or die("Error creating recipe");
 677
 678 my $suitestotal = $#todo + 1;
 679 my $i = 0;
 680 $| = 1;
 681
 682 my %running_envs = ();
 683
 684 my @exported_envvars = (
 685         # domain stuff
 686         "DOMAIN",
 687         "REALM",
 688
 689         # domain controller stuff
 690         "DC_SERVER",
 691         "DC_SERVER_IP",
 692         "DC_NETBIOSNAME",
 693         "DC_NETBIOSALIAS",
 694
 695         # server stuff
 696         "SERVER",
 697         "SERVER_IP",
 698         "NETBIOSNAME",
 699         "NETBIOSALIAS",
 700
 701         # user stuff
 702         "USERNAME",
 703         "PASSWORD",
 704         "DC_USERNAME",
 705         "DC_PASSWORD",
 706
 707         # misc stuff
 708         "KRB5_CONFIG"
 709 );
 710
 711 sub setup_env($)
 712 {
 713         my ($envname) = @_;
 714
 715         my $testenv_vars;
 716         if ($envname eq "none") {
 717                 $testenv_vars = {};
 718         } elsif (defined($running_envs{$envname})) {
 719                 $testenv_vars = $running_envs{$envname};
 720                 if (not $target->check_env($testenv_vars)) {
 721                         $testenv_vars = undef;
 722                 }
 723         } else {
 724                 $testenv_vars = $target->setup_env($envname, $prefix);
 725         }
 726
 727         return undef unless defined($testenv_vars);
 728
 729         SocketWrapper::set_default_iface(6);
 730         write_clientconf($conffile, $testenv_vars);
 731
 732         foreach (@exported_envvars) {
 733                 if (defined($testenv_vars->{$_})) {
 734                         $ENV{$_} = $testenv_vars->{$_};
 735                 } else {
 736                         delete $ENV{$_};
 737                 }
 738         }
 739
 740         $running_envs{$envname} = $testenv_vars;
 741         return $testenv_vars;
 742 }
 743
 744 sub exported_envvars_str($)
 745 {
 746         my ($testenv_vars) = @_;
 747         my $out = "";
 748
 749         foreach (@exported_envvars) {
 750                 next unless defined($testenv_vars->{$_});
 751                 $out .= $_."=".$testenv_vars->{$_}."\n";
 752         }
 753
 754         return $out;
 755 }
 756
 757 sub getlog_env($)
 758 {
 759         my ($envname) = @_;
 760         return "" if ($envname eq "none");
 761         return $target->getlog_env($running_envs{$envname});
 762 }
 763
 764 sub check_env($)
 765 {
 766         my ($envname) = @_;
 767         return 1 if ($envname eq "none");
 768         return $target->check_env($running_envs{$envname});
 769 }
 770
 771 sub teardown_env($)
 772 {
 773         my ($envname) = @_;
 774         return if ($envname eq "none");
 775         $target->teardown_env($running_envs{$envname});
 776         delete $running_envs{$envname};
 777 }
 778
 779 my $msg_ops;
 780 if ($from_build_farm) {
 781         $msg_ops = $buildfarm_msg_ops;
 782 } else {
 783         $msg_ops = $plain_msg_ops;
 784 }
 785
 786 if ($opt_no_lazy_setup) {
 787         setup_env($_) foreach (keys %required_envs);
 788 }
 789
 790 if ($opt_testenv) {
 791         my $testenv_name = $ENV{SELFTEST_TESTENV};
 792         $testenv_name = "dc" unless defined($testenv_name);
 793
 794         my $testenv_vars = setup_env($testenv_name);
 795
 796         $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
 797
 798         my $envvarstr = exported_envvars_str($testenv_vars);
 799
 800         my $term = ($ENV{TERM} or "xterm");
 801         system("$term -e 'echo -e \"
 802 Welcome to the Samba4 Test environment '$testenv_name'
 803
 804 This matches the client environment used in make test
 805 smbd is pid `cat \$PIDDIR/smbd.pid`
 806
 807 Some useful environment variables:
 808 TORTURE_OPTIONS=\$TORTURE_OPTIONS
 809 CONFIGURATION=\$CONFIGURATION
 810
 811 $envvarstr
 812 \" && bash'");
 813         teardown_env($testenv_name);
 814 } else {
 815         foreach (@todo) {
 816                 $i++;
 817                 my $cmd = $$_[2];
 818                 $cmd =~ s/([\(\)])/\\$1/g;
 819                 my $name = $$_[0];
 820                 my $envname = $$_[1];
 821
 822                 if (skip($name)) {
 823                         print "SKIPPED: $name\n";
 824                         $statistics->{SUITES_SKIPPED}++;
 825                         next;
 826                 }
 827
 828                 my $envvars = setup_env($envname);
 829                 if (not defined($envvars)) {
 830                         push(@$suitesfailed, $name);
 831                         $statistics->{SUITES_FAIL}++;
 832                         $statistics->{TESTS_ERROR}++;
 833                         print "FAIL: $name (ENV[$envname] not available!)\n";
 834                         next;
 835                 }
 836
 837                 run_test($envname, $name, $cmd, $i, $suitestotal, $msg_ops);
 838
 839                 if (defined($opt_analyse_cmd)) {
 840                         system("$opt_analyse_cmd \"$name\"");
 841                 }
 842
 843                 teardown_env($envname) if ($opt_resetup_env);
 844         }
 845 }
 846
 847 print "\n";
 848
 849 teardown_env($_) foreach (keys %running_envs);
 850
 851 $target->stop();
 852
 853 my $end = time();
 854 my $duration = ($end-$start);
 855 my $numfailed = $#$suitesfailed+1;
 856 if ($numfailed == 0) {
 857         my $ok = $statistics->{TESTS_EXPECTED_OK} +
 858                  $statistics->{TESTS_EXPECTED_FAIL};
 859         print "ALL OK ($ok tests in $statistics->{SUITES_OK} testsuites)\n";
 860 } else {
 861         unless ($from_build_farm) {
 862                 if (not $opt_immediate and not $opt_verbose) {
 863                         foreach (@$suitesfailed) {
 864                                 print "===============================================================================\n";
 865                                 print "FAIL: $_\n";
 866                                 print $test_output->{$_};
 867                                 print "\n";
 868                         }
 869                 }
 870
 871                 print "FAILED ($statistics->{TESTS_UNEXPECTED_FAIL} failures and $statistics->{TESTS_ERROR} errors in $statistics->{SUITES_FAIL} testsuites)\n";
 872         }
 873 }
 874 print "DURATION: $duration seconds\n";
 875
 876 my $failed = 0;
 877
 878 # if there were any valgrind failures, show them
 879 foreach (<$prefix/valgrind.log*>) {
 880         next unless (-s $_);
 881         system("grep DWARF2.CFI.reader $_ > /dev/null");
 882         if ($? >> 8 == 0) {
 883             print "VALGRIND FAILURE\n";
 884             $failed++;
 885             system("cat $_");
 886         }
 887 }
 888
 889 if ($from_build_farm) {
 890         print "TEST STATUS: $numfailed\n";
 891 }
 892
 893 exit $numfailed;