source/script/tests/selftest.pl

   1 #!/usr/bin/perl
   2 # Bootstrap Samba and run a number of tests against it.
   3 # Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
   4 # Published under the GNU GPL, v3 or later.
   5
   6 =pod
   7
   8 =head1 NAME
   9
  10 selftest - Samba test runner
  11
  12 =head1 SYNOPSIS
  13
  14 selftest --help
  15
  16 selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win] [--socket-wrapper] [--quick] [--one] [--prefix=prefix] [--immediate] [TESTS]
  17
  18 =head1 DESCRIPTION
  19
  20 A simple test runner. TESTS is a regular expression with tests to run.
  21
  22 =head1 OPTIONS
  23
  24 =over 4
  25
  26 =item I<--help>
  27
  28 Show list of available options.
  29
  30 =item I<--srcdir=DIR>
  31
  32 Source directory.
  33
  34 =item I<--builddir=DIR>
  35
  36 Build directory.
  37
  38 =item I<--prefix=DIR>
  39
  40 Change directory to run tests in. Default is 'st'.
  41
  42 =item I<--immediate>
  43
  44 Show errors as soon as they happen rather than at the end of the test run.
  45
  46 =item I<--target samba4|samba3|win>
  47
  48 Specify test target against which to run. Default is 'samba4'.
  49
  50 =item I<--quick>
  51
  52 Run only a limited number of tests. Intended to run in about 30 seconds on
  53 moderately recent systems.
  54
  55 =item I<--socket-wrapper>
  56
  57 Use socket wrapper library for communication with server. Only works
  58 when the server is running locally.
  59
  60 Will prevent TCP and UDP ports being opened on the local host but
  61 (transparently) redirects these calls to use unix domain sockets.
  62
  63 =item I<--expected-failures>
  64
  65 Specify a file containing a list of tests that are expected to fail. Failures for
  66 these tests will be counted as successes, successes will be counted as failures.
  67
  68 The format for the file is, one entry per line:
  69
  70 TESTSUITE-NAME/TEST-NAME
  71
  72 =item I<--skip>
  73
  74 Specify a file containing a list of tests that should be skipped. Possible candidates are
  75 tests that segfault the server, flip or don't end.
  76
  77 =item I<--one>
  78
  79 Abort as soon as one test fails.
  80
  81 =back
  82
  83 =head1 ENVIRONMENT
  84
  85 =over 4
  86
  87 =item I<SMBD_VALGRIND>
  88
  89 =item I<TORTURE_MAXTIME>
  90
  91 =item I<VALGRIND>
  92
  93 =item I<TLS_ENABLED>
  94
  95 =item I<srcdir>
  96
  97 =back
  98
  99 =head1 LICENSE
 100
 101 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
 102
 103 =head1 AUTHOR
 104
 105 Jelmer Vernooij
 106
 107 =cut
 108
 109 use strict;
 110
 111 use FindBin qw($RealBin $Script);
 112 use File::Spec;
 113 use Getopt::Long;
 114 use POSIX;
 115 use Cwd qw(abs_path);
 116 use lib "$RealBin";
 117 use Samba3;
 118 use Samba4;
 119 use SocketWrapper;
 120
 121 my $opt_help = 0;
 122 my $opt_target = "samba4";
 123 my $opt_quick = 0;
 124 my $opt_socket_wrapper = 0;
 125 my $opt_socket_wrapper_pcap = undef;
 126 my $opt_socket_wrapper_keep_pcap = undef;
 127 my $opt_one = 0;
 128 my $opt_immediate = 0;
 129 my $opt_expected_failures = undef;
 130 my $opt_skip = undef;
 131 my $opt_verbose = 0;
 132 my $opt_testenv = 0;
 133 my $ldap = undef;
 134 my $opt_analyse_cmd = undef;
 135 my $opt_resetup_env = undef;
 136 my $opt_bindir = undef;
 137
 138 my $srcdir = ".";
 139 my $builddir = ".";
 140 my $prefix = "./st";
 141
 142 my $suitesfailed = [];
 143 my $start = time();
 144 my @expected_failures = ();
 145 my @skips = ();
 146
 147 my $statistics = {
 148         SUITES_FAIL => 0,
 149         SUITES_OK => 0,
 150         SUITES_SKIPPED => 0,
 151
 152         TESTS_UNEXPECTED_OK => 0,
 153         TESTS_EXPECTED_OK => 0,
 154         TESTS_UNEXPECTED_FAIL => 0,
 155         TESTS_EXPECTED_FAIL => 0,
 156         TESTS_ERROR => 0
 157 };
 158
 159 sub expecting_failure($)
 160 {
 161         my $fullname = shift;
 162
 163         foreach (@expected_failures) {
 164                 return 1 if ($fullname =~ /$_/);
 165         }
 166
 167         return 0;
 168 }
 169
 170 sub skip($)
 171 {
 172         my $fullname = shift;
 173
 174         foreach (@skips) {
 175                 return 1 if ($fullname =~ /$_/);
 176         }
 177
 178         return 0;
 179 }
 180
 181 sub getlog_env($);
 182
 183 my $test_output = {};
 184
 185 sub buildfarm_start_msg($)
 186 {
 187         my ($state) = @_;
 188         my $out = "";
 189
 190         $out .= "--==--==--==--==--==--==--==--==--==--==--\n";
 191         $out .= "Running test $state->{NAME} (level 0 stdout)\n";
 192         $out .= "--==--==--==--==--==--==--==--==--==--==--\n";
 193         $out .= scalar(localtime())."\n";
 194         $out .= "SELFTEST RUNTIME: " . ($state->{START} - $start) . "s\n";
 195         $out .= "NAME: $state->{NAME}\n";
 196         $out .= "CMD: $state->{CMD}\n";
 197
 198         $test_output->{$state->{NAME}} = "";
 199
 200         print $out;
 201 }
 202
 203 sub buildfarm_output_msg($$)
 204 {
 205         my ($state, $output) = @_;
 206
 207         $test_output->{$state->{NAME}} .= $output;
 208 }
 209
 210 sub buildfarm_end_msg($$$)
 211 {
 212         my ($state, $expected_ret, $ret) = @_;
 213         my $out = "";
 214
 215         $out .= "TEST RUNTIME: " . (time() - $state->{START}) . "s\n";
 216
 217         if ($ret == $expected_ret) {
 218                 $out .= "ALL OK\n";
 219         } else {
 220                 $out .= "ERROR: $ret";
 221                 $out .= $test_output->{$state->{NAME}};
 222         }
 223
 224         $out .= "PCAP FILE: $state->{PCAP_FILE}\n" if defined($state->{PCAP_FILE});
 225
 226         $out .= getlog_env($state->{ENVNAME});
 227
 228         $out .= "==========================================\n";
 229         if ($ret == $expected_ret) {
 230                 $out .= "TEST PASSED: $state->{NAME}\n";
 231         } else {
 232                 $out .= "TEST FAILED: $state->{NAME} (status $ret)\n";
 233         }
 234         $out .= "==========================================\n";
 235
 236         print $out;
 237 }
 238
 239 my $buildfarm_msg_ops = {
 240         start_msg       => \&buildfarm_start_msg,
 241         output_msg      => \&buildfarm_output_msg,
 242         end_msg         => \&buildfarm_end_msg
 243 };
 244
 245 sub plain_output_msg($$);
 246
 247 sub plain_start_msg($)
 248 {
 249         my ($state) = @_;
 250         my $out = "";
 251
 252         $out .= "[$state->{INDEX}/$state->{TOTAL} in " . ($state->{START} - $start) . "s";
 253         $out .= ", ".($#$suitesfailed+1)." errors" if ($#$suitesfailed+1 > 0);
 254         $out .= "] $state->{NAME}\n";
 255
 256         $test_output->{$state->{NAME}} = "" unless $opt_verbose;
 257
 258         plain_output_msg($state, "CMD: $state->{CMD}\n");
 259
 260         print $out;
 261 }
 262
 263 sub plain_output_msg($$)
 264 {
 265         my ($state, $output) = @_;
 266
 267         if ($opt_verbose) {
 268                 print $output;
 269         } else {
 270                 $test_output->{$state->{NAME}} .= $output;
 271         }
 272 }
 273
 274 sub plain_end_msg($$$)
 275 {
 276         my ($state, $expected_ret, $ret) = @_;
 277         my $out = "";
 278
 279         if ($ret != $expected_ret) {
 280                 plain_output_msg($state, "ERROR: $ret\n");
 281         }
 282
 283         if ($ret != $expected_ret and ($opt_immediate or $opt_one) and not $opt_verbose) {
 284                 $out .= $test_output->{$state->{NAME}};
 285         }
 286
 287         if (not $opt_socket_wrapper_keep_pcap and defined($state->{PCAP_FILE})) {
 288                 $out .= "PCAP FILE: $state->{PCAP_FILE}\n";
 289         }
 290
 291         $out .= getlog_env($state->{ENVNAME});
 292
 293         print $out;
 294 }
 295
 296 my $plain_msg_ops = {
 297         start_msg       => \&plain_start_msg,
 298         output_msg      => \&plain_output_msg,
 299         end_msg         => \&plain_end_msg
 300 };
 301
 302 sub setup_pcap($)
 303 {
 304         my ($state) = @_;
 305
 306         return unless ($opt_socket_wrapper_pcap);
 307         return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
 308
 309         my $fname = sprintf("t%03u_%s", $state->{INDEX}, $state->{NAME});
 310         $fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
 311
 312         $state->{PCAP_FILE} = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
 313
 314         SocketWrapper::setup_pcap($state->{PCAP_FILE});
 315 }
 316
 317 sub cleanup_pcap($$$)
 318 {
 319         my ($state, $expected_ret, $ret) = @_;
 320
 321         return unless ($opt_socket_wrapper_pcap);
 322         return if ($opt_socket_wrapper_keep_pcap);
 323         return unless ($expected_ret == $ret);
 324         return unless defined($state->{PCAP_FILE});
 325
 326         unlink($state->{PCAP_FILE});
 327         $state->{PCAP_FILE} = undef;
 328 }
 329
 330 sub run_test($$$$$$)
 331 {
 332         my ($envname, $name, $cmd, $i, $totalsuites, $msg_ops) = @_;
 333         my $expected_ret = 1;
 334         my $open_tests = {};
 335         my $msg_state = {
 336                 ENVNAME => $envname,
 337                 NAME    => $name,
 338                 CMD     => $cmd,
 339                 INDEX   => $i,
 340                 TOTAL   => $totalsuites,
 341                 START   => time()
 342         };
 343
 344         setup_pcap($msg_state);
 345
 346         $msg_ops->{start_msg}($msg_state);
 347
 348         open(RESULT, "$cmd 2>&1|");
 349         while (<RESULT>) {
 350                 $msg_ops->{output_msg}($msg_state, $_);
 351                 if (/^test: (.+)\n/) {
 352                         $open_tests->{$1} = 1;
 353                 } elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
 354                         my $result = $1;
 355                         if ($1 eq "success") {
 356                                 delete $open_tests->{$2};
 357                                 if (expecting_failure("$name/$2")) {
 358                                         $statistics->{TESTS_UNEXPECTED_OK}++;
 359                                 } else {
 360                                         $statistics->{TESTS_EXPECTED_OK}++;
 361                                 }
 362                         } elsif ($1 eq "failure") {
 363                                 delete $open_tests->{$2};
 364                                 if (expecting_failure("$name/$2")) {
 365                                         $statistics->{TESTS_EXPECTED_FAIL}++;
 366                                         $expected_ret = 0;
 367                                 } else {
 368                                         print "n:$name/$2l\n";
 369                                         $statistics->{TESTS_UNEXPECTED_FAIL}++;
 370                                 }
 371                         } elsif ($1 eq "skip") {
 372                                 delete $open_tests->{$2};
 373                         } elsif ($1 eq "error") {
 374                                 $statistics->{TESTS_ERROR}++;
 375                                 delete $open_tests->{$2};
 376                         }
 377                 }
 378         }
 379         foreach (keys %$open_tests) {
 380                 $msg_ops->{output_msg}($msg_state, "$_ was started but never finished!\n");
 381                 $statistics->{TESTS_ERROR}++;
 382         }
 383         my $ret = close(RESULT);
 384
 385         cleanup_pcap($msg_state,  $expected_ret, $ret);
 386
 387         $msg_ops->{end_msg}($msg_state, $expected_ret, $ret);
 388
 389         if ($ret != $expected_ret) {
 390                 push(@$suitesfailed, $name);
 391                 $statistics->{SUITES_FAIL}++;
 392                 exit(1) if ($opt_one);
 393         } else {
 394                 $statistics->{SUITES_OK}++;
 395         }
 396
 397         return ($ret == $expected_ret);
 398 }
 399
 400 sub ShowHelp()
 401 {
 402         print "Samba test runner
 403 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
 404
 405 Usage: $Script [OPTIONS] PREFIX
 406
 407 Generic options:
 408  --help                     this help page
 409  --target=samba4|samba3|win Samba version to target
 410
 411 Paths:
 412  --prefix=DIR               prefix to run tests in [st]
 413  --srcdir=DIR               source directory [.]
 414  --builddir=DIR             output directory [.]
 415
 416 Target Specific:
 417  --socket-wrapper-pcap=DIR      save traffic to pcap directories
 418  --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
 419                             failed
 420  --socket-wrapper           enable socket wrapper
 421  --expected-failures=FILE   specify list of tests that is guaranteed to fail
 422
 423 Samba4 Specific:
 424  --ldap=openldap|fedora     back smbd onto specified ldap server
 425
 426 Samba3 Specific:
 427  --bindir=PATH              path to binaries
 428
 429 Behaviour:
 430  --quick                    run quick overall test
 431  --one                      abort when the first test fails
 432  --immediate                print test output for failed tests during run
 433  --verbose                  be verbose
 434  --analyse-cmd CMD          command to run after each test
 435 ";
 436         exit(0);
 437 }
 438
 439 my $result = GetOptions (
 440                 'help|h|?' => \$opt_help,
 441                 'target=s' => \$opt_target,
 442                 'prefix=s' => \$prefix,
 443                 'socket-wrapper' => \$opt_socket_wrapper,
 444                 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
 445                 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
 446                 'quick' => \$opt_quick,
 447                 'one' => \$opt_one,
 448                 'immediate' => \$opt_immediate,
 449                 'expected-failures=s' => \$opt_expected_failures,
 450                 'skip=s' => \$opt_skip,
 451                 'srcdir=s' => \$srcdir,
 452                 'builddir=s' => \$builddir,
 453                 'verbose' => \$opt_verbose,
 454                 'testenv' => \$opt_testenv,
 455                 'ldap:s' => \$ldap,
 456                 'analyse-cmd=s' => \$opt_analyse_cmd,
 457                 'resetup-environment' => \$opt_resetup_env,
 458                 'bindir:s' => \$opt_bindir,
 459             );
 460
 461 exit(1) if (not $result);
 462
 463 ShowHelp() if ($opt_help);
 464
 465 my $tests = shift;
 466
 467 # quick hack to disable rpc validation when using valgrind - its way too slow
 468 unless (defined($ENV{VALGRIND})) {
 469         $ENV{VALIDATE} = "validate";
 470         $ENV{MALLOC_CHECK_} = 2;
 471 }
 472
 473 my $old_pwd = "$RealBin/../..";
 474
 475 # Backwards compatibility:
 476 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
 477         if (defined($ENV{FEDORA_DS_PREFIX})) {
 478                 $ldap = "fedora";
 479         } else {
 480                 $ldap = "openldap";
 481         }
 482 }
 483
 484 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
 485 if ($ldap) {
 486         # LDAP is slow
 487         $torture_maxtime *= 2;
 488 }
 489
 490 $prefix =~ s+//+/+;
 491 $prefix =~ s+/./+/+;
 492 $prefix =~ s+/$++;
 493
 494 die("using an empty prefix isn't allowed") unless $prefix ne "";
 495
 496 #Ensure we have the test prefix around
 497 mkdir($prefix, 0777) unless -d $prefix;
 498
 499 my $prefix_abs = abs_path($prefix);
 500 my $srcdir_abs = abs_path($srcdir);
 501
 502 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
 503 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
 504
 505 $ENV{PREFIX} = $prefix;
 506 $ENV{SRCDIR} = $srcdir;
 507
 508 my $tls_enabled = not $opt_quick;
 509 my $from_build_farm = (defined($ENV{RUN_FROM_BUILD_FARM}) and
 510                       ($ENV{RUN_FROM_BUILD_FARM} eq "yes"));
 511
 512 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
 513 $ENV{LD_LDB_MODULE_PATH} = "$old_pwd/bin/modules/ldb";
 514 $ENV{LD_SAMBA_MODULE_PATH} = "$old_pwd/bin/modules";
 515 if (defined($ENV{LD_LIBRARY_PATH})) {
 516         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared:$ENV{LD_LIBRARY_PATH}";
 517 } else {
 518         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared";
 519 }
 520 $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig:$ENV{PKG_CONFIG_PATH}";
 521 $ENV{PATH} = "$old_pwd/bin:$ENV{PATH}";
 522
 523
 524 if ($opt_socket_wrapper_pcap) {
 525         # Socket wrapper pcap implies socket wrapper
 526         $opt_socket_wrapper = 1;
 527 }
 528
 529 my $socket_wrapper_dir;
 530 if ($opt_socket_wrapper) {
 531         $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w", $opt_socket_wrapper_pcap);
 532         print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
 533 } else {
 534         warn("Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports") unless $< == 0;
 535 }
 536
 537 my $target;
 538
 539 if ($opt_target eq "samba4") {
 540         $target = new Samba4("$srcdir/bin", $ldap, "$srcdir/setup");
 541 } elsif ($opt_target eq "samba3") {
 542         if ($opt_socket_wrapper and `smbd -b | grep SOCKET_WRAPPER` eq "") {
 543                 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'.  Exiting....");
 544         }
 545
 546         $target = new Samba3($opt_bindir);
 547 } elsif ($opt_target eq "win") {
 548         die("Windows tests will not run with socket wrapper enabled.")
 549                 if ($opt_socket_wrapper);
 550         $target = new Windows();
 551 }
 552
 553 if (defined($opt_expected_failures)) {
 554         open(KNOWN, "<$opt_expected_failures") or die("unable to read known failures file: $!");
 555         while (<KNOWN>) {
 556                 chomp;
 557                 s/([ \t]+)\#(.*)$//;
 558                 push (@expected_failures, $_); }
 559         close(KNOWN);
 560 }
 561
 562 if (defined($opt_skip)) {
 563         open(SKIP, "<$opt_skip") or die("unable to read skip file: $!");
 564         while (<SKIP>) {
 565                 chomp;
 566                 s/([ \t]+)\#(.*)$//;
 567                 push (@skips, $_); }
 568         close(SKIP);
 569 }
 570
 571 my $interfaces = join(',', ("127.0.0.6/8",
 572                             "127.0.0.7/8",
 573                             "127.0.0.8/8",
 574                             "127.0.0.9/8",
 575                             "127.0.0.10/8",
 576                             "127.0.0.11/8"));
 577
 578 my $conffile = "$prefix_abs/client/client.conf";
 579
 580 sub write_clientconf($$)
 581 {
 582         my ($conffile, $vars) = @_;
 583
 584         mkdir("$prefix/client", 0777) unless -d "$prefix/client";
 585
 586         if ( -d "$prefix/client/private" ) {
 587                 unlink <$prefix/client/private/*>;
 588         } else {
 589                 mkdir("$prefix/client/private", 0777);
 590         }
 591
 592         open(CF, ">$conffile");
 593         print CF "[global]\n";
 594         if (defined($ENV{VALGRIND})) {
 595                 print CF "\ticonv:native = true\n";
 596         } else {
 597                 print CF "\ticonv:native = false\n";
 598         }
 599         print CF "\tnetbios name = client\n";
 600         if (defined($vars->{DOMAIN})) {
 601                 print CF "\tworkgroup = $vars->{DOMAIN}\n";
 602         }
 603         if (defined($vars->{REALM})) {
 604                 print CF "\trealm = $vars->{REALM}\n";
 605         }
 606         if (defined($vars->{NCALRPCDIR})) {
 607                 print CF "\tncalrpc dir = $vars->{NCALRPCDIR}\n";
 608         }
 609         if (defined($vars->{PIDDIR})) {
 610                 print CF "\tpid directory = $vars->{PIDDIR}\n";
 611         }
 612         if (defined($vars->{WINBINDD_SOCKET_DIR})) {
 613                 print CF "\twinbindd socket directory = $vars->{WINBINDD_SOCKET_DIR}\n";
 614         }
 615         print CF "
 616         private dir = $srcdir_abs/$prefix/client/private
 617         js include = $srcdir_abs/scripting/libjs
 618         name resolve order = bcast
 619         interfaces = $interfaces
 620         panic action = $srcdir_abs/script/gdb_backtrace \%PID\% \%PROG\%
 621         max xmit = 32K
 622         notify:inotify = false
 623         ldb:nosync = true
 624         system:anonymous = true
 625         torture:basedir = $prefix_abs
 626 #We don't want to pass our self-tests if the PAC code is wrong
 627         gensec:require_pac = true
 628 ";
 629         close(CF);
 630 }
 631
 632
 633 my @torture_options = ();
 634 push (@torture_options, "--configfile=$conffile");
 635 # ensure any one smbtorture call doesn't run too long
 636 push (@torture_options, "--maximum-runtime=$torture_maxtime");
 637 push (@torture_options, "--target=$opt_target");
 638 push (@torture_options, "--option=torture:progress=no") if ($from_build_farm);
 639 push (@torture_options, "--format=subunit");
 640 push (@torture_options, "--option=torture:quick=yes") if ($opt_quick);
 641
 642 $ENV{TORTURE_OPTIONS} = join(' ', @torture_options);
 643 print "OPTIONS $ENV{TORTURE_OPTIONS}\n";
 644
 645 my @todo = ();
 646
 647 my $testsdir = "$srcdir/script/tests";
 648 $ENV{CONFIGURATION} = "--configfile=$conffile";
 649
 650
 651 if ($opt_quick) {
 652         open(IN, "$testsdir/tests_quick.sh|");
 653 } else {
 654         open(IN, "$testsdir/tests_all.sh|");
 655 }
 656 while (<IN>) {
 657         if ($_ eq "-- TEST --\n") {
 658                 my $name = <IN>;
 659                 $name =~ s/\n//g;
 660                 my $env = <IN>;
 661                 $env =~ s/\n//g;
 662                 my $cmdline = <IN>;
 663                 $cmdline =~ s/\n//g;
 664                 push (@todo, [$name, $env, $cmdline])
 665                         if (not defined($tests) or $name =~ /$tests/);
 666         } else {
 667                 print;
 668         }
 669 }
 670 close(IN) or die("Error creating recipe");
 671
 672 my $suitestotal = $#todo + 1;
 673 my $i = 0;
 674 $| = 1;
 675
 676 my %running_envs = ();
 677
 678 sub setup_env($)
 679 {
 680         my ($envname) = @_;
 681
 682         my $testenv_vars;
 683         if ($envname eq "none") {
 684                 $testenv_vars = {};
 685         } elsif (defined($running_envs{$envname})) {
 686                 $testenv_vars = $running_envs{$envname};
 687                 if (not $target->check_env($testenv_vars)) {
 688                         $testenv_vars = undef;
 689                 }
 690         } else {
 691                 $testenv_vars = $target->setup_env($envname, $prefix);
 692         }
 693
 694         return undef unless defined($testenv_vars);
 695
 696         write_clientconf($conffile, $testenv_vars);
 697         foreach ("PASSWORD", "DOMAIN", "SERVER", "USERNAME", "NETBIOSNAME",
 698                          "KRB5_CONFIG", "REALM") {
 699                 if (defined($testenv_vars->{$_})) {
 700                         $ENV{$_} = $testenv_vars->{$_};
 701                 } else {
 702                         delete $ENV{$_};
 703                 }
 704         }
 705
 706         SocketWrapper::set_default_iface(6);
 707
 708         $running_envs{$envname} = $testenv_vars;
 709         return $testenv_vars;
 710 }
 711
 712 sub getlog_env($)
 713 {
 714         my ($envname) = @_;
 715         return "" if ($envname eq "none");
 716         return $target->getlog_env($running_envs{$envname});
 717 }
 718
 719 sub check_env($)
 720 {
 721         my ($envname) = @_;
 722         return 1 if ($envname eq "none");
 723         return $target->check_env($running_envs{$envname});
 724 }
 725
 726 sub teardown_env($)
 727 {
 728         my ($envname) = @_;
 729         return if ($envname eq "none");
 730         $target->teardown_env($running_envs{$envname});
 731         delete $running_envs{$envname};
 732 }
 733
 734 my $msg_ops;
 735 if ($from_build_farm) {
 736         $msg_ops = $buildfarm_msg_ops;
 737 } else {
 738         $msg_ops = $plain_msg_ops;
 739 }
 740
 741 if ($opt_testenv) {
 742         my $testenv_vars = setup_env("dc");
 743         $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
 744         my $term = ($ENV{TERM} or "xterm");
 745         system("$term -e 'echo -e \"Welcome to the Samba4 Test environment
 746 This matches the client environment used in make test
 747 smbd is pid `cat \$PIDDIR/smbd.pid`
 748
 749 Some useful environment variables:
 750 TORTURE_OPTIONS=\$TORTURE_OPTIONS
 751 CONFIGURATION=\$CONFIGURATION
 752 SERVER=\$SERVER
 753 NETBIOSNAME=\$NETBIOSNAME\" && bash'");
 754         teardown_env("dc");
 755 } else {
 756         foreach (@todo) {
 757                 $i++;
 758                 my $cmd = $$_[2];
 759                 $cmd =~ s/([\(\)])/\\$1/g;
 760                 my $name = $$_[0];
 761                 my $envname = $$_[1];
 762
 763                 if (skip($name)) {
 764                         print "SKIPPED: $name\n";
 765                         $statistics->{SUITES_SKIPPED}++;
 766                         next;
 767                 }
 768
 769                 my $envvars = setup_env($envname);
 770                 if (not defined($envvars)) {
 771                         push(@$suitesfailed, $name);
 772                         $statistics->{SUITES_FAIL}++;
 773                         $statistics->{TESTS_ERROR}++;
 774                         print "FAIL: $name (ENV[$envname] not available!)\n";
 775                         next;
 776                 }
 777
 778                 run_test($envname, $name, $cmd, $i, $suitestotal, $msg_ops);
 779
 780                 if (defined($opt_analyse_cmd)) {
 781                         system("$opt_analyse_cmd \"$name\"");
 782                 }
 783
 784                 teardown_env($envname) if ($opt_resetup_env);
 785         }
 786 }
 787
 788 print "\n";
 789
 790 teardown_env($_) foreach (keys %running_envs);
 791
 792 $target->stop();
 793
 794 my $end = time();
 795 my $duration = ($end-$start);
 796 my $numfailed = $#$suitesfailed+1;
 797 if ($numfailed == 0) {
 798         my $ok = $statistics->{TESTS_EXPECTED_OK} +
 799                  $statistics->{TESTS_EXPECTED_FAIL};
 800         print "ALL OK ($ok tests in $statistics->{SUITES_OK} testsuites)\n";
 801 } else {
 802         unless ($from_build_farm) {
 803                 if (not $opt_immediate and not $opt_verbose) {
 804                         foreach (@$suitesfailed) {
 805                                 print "===============================================================================\n";
 806                                 print "FAIL: $_\n";
 807                                 print $test_output->{$_};
 808                                 print "\n";
 809                         }
 810                 }
 811
 812                 print "FAILED ($statistics->{TESTS_UNEXPECTED_FAIL} failures and $statistics->{TESTS_ERROR} errors in $statistics->{SUITES_FAIL} testsuites)\n";
 813         }
 814 }
 815 print "DURATION: $duration seconds\n";
 816
 817 my $failed = 0;
 818
 819 # if there were any valgrind failures, show them
 820 foreach (<$prefix/valgrind.log*>) {
 821         next unless (-s $_);
 822         system("grep DWARF2.CFI.reader $_ > /dev/null");
 823         if ($? >> 8 == 0) {
 824             print "VALGRIND FAILURE\n";
 825             $failed++;
 826             system("cat $_");
 827         }
 828 }
 829
 830 if ($from_build_farm) {
 831         print "TEST STATUS: $numfailed\n";
 832 }
 833
 834 exit $numfailed;