selftest/selftest.pl

   1 #!/usr/bin/perl
   2 # Bootstrap Samba and run a number of tests against it.
   3 # Copyright (C) 2005-2009 Jelmer Vernooij <jelmer@samba.org>
   4 # Copyright (C) 2007-2009 Stefan Metzmacher <metze@samba.org>
   5
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 3 of the License, or
   9 # (at your option) any later version.
  10
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 =pod
  20
  21 =head1 NAME
  22
  23 selftest - Samba test runner
  24
  25 =head1 SYNOPSIS
  26
  27 selftest --help
  28
  29 selftest [--srcdir=DIR] [--builddir=DIR] [--exeext=EXT][--target=samba4|samba3|win|kvm] [--socket-wrapper] [--quick] [--exclude=FILE] [--include=FILE] [--one] [--prefix=prefix] [--immediate] [--testlist=FILE] [TESTS]
  30
  31 =head1 DESCRIPTION
  32
  33 A simple test runner. TESTS is a regular expression with tests to run.
  34
  35 =head1 OPTIONS
  36
  37 =over 4
  38
  39 =item I<--help>
  40
  41 Show list of available options.
  42
  43 =item I<--srcdir=DIR>
  44
  45 Source directory.
  46
  47 =item I<--builddir=DIR>
  48
  49 Build directory.
  50
  51 =item I<--exeext=EXT>
  52
  53 Executable extention
  54
  55 =item I<--prefix=DIR>
  56
  57 Change directory to run tests in. Default is 'st'.
  58
  59 =item I<--immediate>
  60
  61 Show errors as soon as they happen rather than at the end of the test run.
  62
  63 =item I<--target samba4|samba3|win|kvm>
  64
  65 Specify test target against which to run. Default is 'samba4'.
  66
  67 =item I<--quick>
  68
  69 Run only a limited number of tests. Intended to run in about 30 seconds on
  70 moderately recent systems.
  71
  72 =item I<--socket-wrapper>
  73
  74 Use socket wrapper library for communication with server. Only works
  75 when the server is running locally.
  76
  77 Will prevent TCP and UDP ports being opened on the local host but
  78 (transparently) redirects these calls to use unix domain sockets.
  79
  80 =item I<--expected-failures>
  81
  82 Specify a file containing a list of tests that are expected to fail. Failures for
  83 these tests will be counted as successes, successes will be counted as failures.
  84
  85 The format for the file is, one entry per line:
  86
  87 TESTSUITE-NAME.TEST-NAME
  88
  89 The reason for a test can also be specified, by adding a hash sign (#) and the reason
  90 after the test name.
  91
  92 =item I<--exclude>
  93
  94 Specify a file containing a list of tests that should be skipped. Possible
  95 candidates are tests that segfault the server, flip or don't end. The format of this file is the same as
  96 for the --expected-failures flag.
  97
  98 =item I<--include>
  99
 100 Specify a file containing a list of tests that should be run. Same format
 101 as the --exclude flag.
 102
 103 Not includes specified means all tests will be run.
 104
 105 =item I<--one>
 106
 107 Abort as soon as one test fails.
 108
 109 =item I<--testlist>
 110
 111 Load a list of tests from the specified location.
 112
 113 =back
 114
 115 =head1 ENVIRONMENT
 116
 117 =over 4
 118
 119 =item I<SMBD_VALGRIND>
 120
 121 =item I<TORTURE_MAXTIME>
 122
 123 =item I<VALGRIND>
 124
 125 =item I<TLS_ENABLED>
 126
 127 =item I<srcdir>
 128
 129 =back
 130
 131 =head1 LICENSE
 132
 133 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
 134
 135 =head1 AUTHOR
 136
 137 Jelmer Vernooij
 138
 139 =cut
 140
 141 use strict;
 142
 143 use FindBin qw($RealBin $Script);
 144 use File::Spec;
 145 use Getopt::Long;
 146 use POSIX;
 147 use Cwd qw(abs_path);
 148 use lib "$RealBin";
 149 use Subunit qw(parse_results);
 150 use SocketWrapper;
 151
 152 my $opt_help = 0;
 153 my $opt_target = "samba4";
 154 my $opt_quick = 0;
 155 my $opt_socket_wrapper = 0;
 156 my $opt_socket_wrapper_pcap = undef;
 157 my $opt_socket_wrapper_keep_pcap = undef;
 158 my $opt_one = 0;
 159 my $opt_immediate = 0;
 160 my $opt_expected_failures = undef;
 161 my @opt_exclude = ();
 162 my @opt_include = ();
 163 my $opt_verbose = 0;
 164 my $opt_image = undef;
 165 my $opt_testenv = 0;
 166 my $ldap = undef;
 167 my $opt_analyse_cmd = undef;
 168 my $opt_resetup_env = undef;
 169 my $opt_bindir = undef;
 170 my $opt_no_lazy_setup = undef;
 171 my $opt_format = "plain";
 172 my @testlists = ();
 173
 174 my $srcdir = ".";
 175 my $builddir = ".";
 176 my $exeext = "";
 177 my $prefix = "./st";
 178
 179 my @expected_failures = ();
 180 my @includes = ();
 181 my @excludes = ();
 182
 183 my $statistics = {
 184         SUITES_FAIL => 0,
 185
 186         TESTS_UNEXPECTED_OK => 0,
 187         TESTS_EXPECTED_OK => 0,
 188         TESTS_UNEXPECTED_FAIL => 0,
 189         TESTS_EXPECTED_FAIL => 0,
 190         TESTS_ERROR => 0,
 191         TESTS_SKIP => 0,
 192 };
 193
 194 sub find_in_list($$)
 195 {
 196         my ($list, $fullname) = @_;
 197
 198         foreach (@$list) {
 199                 if ($fullname =~ /$$_[0]/) {
 200                          return ($$_[1]) if ($$_[1]);
 201                          return "NO REASON SPECIFIED";
 202                 }
 203         }
 204
 205         return undef;
 206 }
 207
 208 sub expecting_failure($)
 209 {
 210         my ($name) = @_;
 211         return find_in_list(\@expected_failures, $name);
 212 }
 213
 214 sub skip($)
 215 {
 216         my ($name) = @_;
 217
 218         return find_in_list(\@excludes, $name);
 219 }
 220
 221 sub getlog_env($);
 222
 223 sub setup_pcap($)
 224 {
 225         my ($name) = @_;
 226
 227         return unless ($opt_socket_wrapper_pcap);
 228         return unless defined($ENV{SOCKET_WRAPPER_PCAP_DIR});
 229
 230         my $fname = $name;
 231         $fname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
 232
 233         my $pcap_file = "$ENV{SOCKET_WRAPPER_PCAP_DIR}/$fname.pcap";
 234
 235         SocketWrapper::setup_pcap($pcap_file);
 236
 237         return $pcap_file;
 238 }
 239
 240 sub cleanup_pcap($$$)
 241 {
 242         my ($pcap_file, $expected_ret, $ret) = @_;
 243
 244         return unless ($opt_socket_wrapper_pcap);
 245         return if ($opt_socket_wrapper_keep_pcap);
 246         return unless ($expected_ret == $ret);
 247         return unless defined($pcap_file);
 248
 249         unlink($pcap_file);
 250 }
 251
 252 sub run_testsuite($$$$$$)
 253 {
 254         my ($envname, $name, $cmd, $i, $totalsuites, $msg_ops) = @_;
 255         my $pcap_file = setup_pcap($name);
 256
 257         $msg_ops->report_time(time());
 258         $msg_ops->start_test([], $name);
 259
 260         unless (open(RESULT, "$cmd 2>&1|")) {
 261                 $statistics->{TESTS_ERROR}++;
 262                 $msg_ops->end_test([], $name, "error", 1, "Unable to run $cmd: $!");
 263                 $statistics->{SUITES_FAIL}++;
 264                 return 0;
 265         }
 266
 267         my $expected_ret = parse_results(
 268                 $msg_ops, $statistics, *RESULT, \&expecting_failure, [$name]);
 269
 270         my $envlog = getlog_env($envname);
 271         $msg_ops->output_msg("ENVLOG: $envlog\n") if ($envlog ne "");
 272
 273         $msg_ops->output_msg("CMD: $cmd\n");
 274
 275         my $ret = close(RESULT);
 276         $ret = 0 unless $ret == 1;
 277
 278         my $exitcode = $? >> 8;
 279
 280         $msg_ops->report_time(time());
 281         if ($ret == 1) {
 282                 $msg_ops->end_test([], $name, "success", $expected_ret != $ret, undef);
 283         } else {
 284                 $msg_ops->end_test([], $name, "failure", $expected_ret != $ret, "Exit code was $exitcode");
 285         }
 286
 287         cleanup_pcap($pcap_file, $expected_ret, $ret);
 288
 289         if (not $opt_socket_wrapper_keep_pcap and defined($pcap_file)) {
 290                 $msg_ops->output_msg("PCAP FILE: $pcap_file\n");
 291         }
 292
 293         if ($ret != $expected_ret) {
 294                 $statistics->{SUITES_FAIL}++;
 295                 exit(1) if ($opt_one);
 296         }
 297
 298         return ($ret == $expected_ret);
 299 }
 300
 301 sub ShowHelp()
 302 {
 303         print "Samba test runner
 304 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
 305 Copyright (C) Stefan Metzmacher <metze\@samba.org>
 306
 307 Usage: $Script [OPTIONS] TESTNAME-REGEX
 308
 309 Generic options:
 310  --help                     this help page
 311  --target=samba[34]|win|kvm Samba version to target
 312  --testlist=FILE            file to read available tests from
 313
 314 Paths:
 315  --prefix=DIR               prefix to run tests in [st]
 316  --srcdir=DIR               source directory [.]
 317  --builddir=DIR             output directory [.]
 318  --exeext=EXT               executable extention []
 319
 320 Target Specific:
 321  --socket-wrapper-pcap      save traffic to pcap directories
 322  --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
 323                             failed
 324  --socket-wrapper           enable socket wrapper
 325  --bindir=PATH              path to target binaries
 326  --expected-failures=FILE   specify list of tests that is guaranteed to fail
 327
 328 Samba4 Specific:
 329  --ldap=openldap|fedora-ds  back samba onto specified ldap server
 330
 331 Kvm Specific:
 332  --image=PATH               path to KVM image
 333
 334 Behaviour:
 335  --quick                    run quick overall test
 336  --one                      abort when the first test fails
 337  --immediate                print test output for failed tests during run
 338  --verbose                  be verbose
 339  --analyse-cmd CMD          command to run after each test
 340 ";
 341         exit(0);
 342 }
 343
 344 my $result = GetOptions (
 345                 'help|h|?' => \$opt_help,
 346                 'target=s' => \$opt_target,
 347                 'prefix=s' => \$prefix,
 348                 'socket-wrapper' => \$opt_socket_wrapper,
 349                 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
 350                 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
 351                 'quick' => \$opt_quick,
 352                 'one' => \$opt_one,
 353                 'immediate' => \$opt_immediate,
 354                 'expected-failures=s' => \$opt_expected_failures,
 355                 'exclude=s' => \@opt_exclude,
 356                 'include=s' => \@opt_include,
 357                 'srcdir=s' => \$srcdir,
 358                 'builddir=s' => \$builddir,
 359                 'exeext=s' => \$exeext,
 360                 'verbose' => \$opt_verbose,
 361                 'testenv' => \$opt_testenv,
 362                 'ldap:s' => \$ldap,
 363                 'analyse-cmd=s' => \$opt_analyse_cmd,
 364                 'no-lazy-setup' => \$opt_no_lazy_setup,
 365                 'resetup-environment' => \$opt_resetup_env,
 366                 'bindir:s' => \$opt_bindir,
 367                 'format=s' => \$opt_format,
 368                 'image=s' => \$opt_image,
 369                 'testlist=s' => \@testlists
 370             );
 371
 372 exit(1) if (not $result);
 373
 374 ShowHelp() if ($opt_help);
 375
 376 my $tests = shift;
 377
 378 # quick hack to disable rpc validation when using valgrind - its way too slow
 379 unless (defined($ENV{VALGRIND})) {
 380         $ENV{VALIDATE} = "validate";
 381         $ENV{MALLOC_CHECK_} = 2;
 382 }
 383
 384 my $bindir = ($opt_bindir or "$builddir/bin");
 385 my $bindir_abs = abs_path($bindir);
 386
 387 # Backwards compatibility:
 388 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
 389         if (defined($ENV{FEDORA_DS_ROOT})) {
 390                 $ldap = "fedora-ds";
 391         } else {
 392                 $ldap = "openldap";
 393         }
 394 }
 395
 396 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
 397 if ($ldap) {
 398         # LDAP is slow
 399         $torture_maxtime *= 2;
 400 }
 401
 402 $prefix =~ s+//+/+;
 403 $prefix =~ s+/./+/+;
 404 $prefix =~ s+/$++;
 405
 406 die("using an empty prefix isn't allowed") unless $prefix ne "";
 407
 408 #Ensure we have the test prefix around
 409 mkdir($prefix, 0777) unless -d $prefix;
 410
 411 my $prefix_abs = abs_path($prefix);
 412 my $srcdir_abs = abs_path($srcdir);
 413 my $builddir_abs = abs_path($builddir);
 414
 415 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
 416 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
 417
 418 $ENV{PREFIX} = $prefix;
 419 $ENV{KRB5CCNAME} = "$prefix/krb5ticket";
 420 $ENV{PREFIX_ABS} = $prefix_abs;
 421 $ENV{SRCDIR} = $srcdir;
 422 $ENV{SRCDIR_ABS} = $srcdir_abs;
 423 $ENV{BUILDDIR} = $builddir;
 424 $ENV{BUILDDIR_ABS} = $builddir_abs;
 425 $ENV{EXEEXT} = $exeext;
 426
 427 if (defined($ENV{RUN_FROM_BUILD_FARM}) and
 428         ($ENV{RUN_FROM_BUILD_FARM} eq "yes")) {
 429         $opt_format = "buildfarm";
 430 }
 431
 432 my $tls_enabled = not $opt_quick;
 433 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
 434 $ENV{LDB_MODULES_PATH} = "$bindir_abs/modules/ldb";
 435 $ENV{LD_SAMBA_MODULE_PATH} = "$bindir_abs/modules";
 436 sub prefix_pathvar($$)
 437 {
 438         my ($name, $newpath) = @_;
 439         if (defined($ENV{$name})) {
 440                 $ENV{$name} = "$newpath:$ENV{$name}";
 441         } else {
 442                 $ENV{$name} = $newpath;
 443         }
 444 }
 445 prefix_pathvar("PKG_CONFIG_PATH", "$bindir_abs/pkgconfig");
 446 prefix_pathvar("PYTHONPATH", "$bindir_abs/python");
 447
 448 if ($opt_socket_wrapper_keep_pcap) {
 449         # Socket wrapper keep pcap implies socket wrapper pcap
 450         $opt_socket_wrapper_pcap = 1;
 451 }
 452
 453 if ($opt_socket_wrapper_pcap) {
 454         # Socket wrapper pcap implies socket wrapper
 455         $opt_socket_wrapper = 1;
 456 }
 457
 458 my $socket_wrapper_dir;
 459 if ($opt_socket_wrapper) {
 460         $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w", $opt_socket_wrapper_pcap);
 461         print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
 462 } else {
 463          unless ($< == 0) {
 464                  print "WARNING: Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports\n";
 465          }
 466 }
 467
 468 my $target;
 469 my $testenv_default = "none";
 470
 471 if ($opt_target eq "samba4") {
 472         $testenv_default = "member";
 473         require target::Samba4;
 474         $target = new Samba4($bindir, $ldap, "$srcdir/setup", $exeext);
 475 } elsif ($opt_target eq "samba3") {
 476         if ($opt_socket_wrapper and `$bindir/smbd -b | grep SOCKET_WRAPPER` eq "") {
 477                 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'.  Exiting....");
 478         }
 479         $testenv_default = "member";
 480         require target::Samba3;
 481         $target = new Samba3($bindir);
 482 } elsif ($opt_target eq "win") {
 483         die("Windows tests will not run with socket wrapper enabled.")
 484                 if ($opt_socket_wrapper);
 485         $testenv_default = "dc";
 486         require target::Windows;
 487         $target = new Windows();
 488 } elsif ($opt_target eq "kvm") {
 489         die("Kvm tests will not run with socket wrapper enabled.")
 490                 if ($opt_socket_wrapper);
 491         require target::Kvm;
 492         die("No image specified") unless ($opt_image);
 493         $target = new Kvm($opt_image, undef);
 494 }
 495
 496 #
 497 # Start a Virtual Distributed Ethernet Switch
 498 # Returns the pid of the switch.
 499 #
 500 sub start_vde_switch($)
 501 {
 502         my ($path) = @_;
 503
 504         system("vde_switch --pidfile $path/vde.pid --sock $path/vde.sock --daemon");
 505
 506         open(PID, "$path/vde.pid");
 507         <PID> =~ /([0-9]+)/;
 508         my $pid = $1;
 509         close(PID);
 510
 511         return $pid;
 512 }
 513
 514 # Stop a Virtual Distributed Ethernet Switch
 515 sub stop_vde_switch($)
 516 {
 517         my ($pid) = @_;
 518         kill 9, $pid;
 519 }
 520
 521 sub read_test_regexes($)
 522 {
 523         my ($name) = @_;
 524         my @ret = ();
 525         open(LF, "<$name") or die("unable to read $name: $!");
 526         while (<LF>) {
 527                 chomp;
 528                 next if (/^#/);
 529                 if (/^(.*?)([ \t]+)\#([\t ]*)(.*?)$/) {
 530                         push (@ret, [$1, $4]);
 531                 } else {
 532                         s/^(.*?)([ \t]+)\#([\t ]*)(.*?)$//;
 533                         push (@ret, [$_, undef]);
 534                 }
 535         }
 536         close(LF);
 537         return @ret;
 538 }
 539
 540 if (defined($opt_expected_failures)) {
 541         @expected_failures = read_test_regexes($opt_expected_failures);
 542 }
 543
 544 foreach (@opt_exclude) {
 545         push (@excludes, read_test_regexes($_));
 546 }
 547
 548 foreach (@opt_include) {
 549         push (@includes, read_test_regexes($_));
 550 }
 551
 552 my $interfaces = join(',', ("127.0.0.6/8",
 553                             "127.0.0.7/8",
 554                             "127.0.0.8/8",
 555                             "127.0.0.9/8",
 556                             "127.0.0.10/8",
 557                             "127.0.0.11/8"));
 558
 559 my $conffile = "$prefix_abs/client/client.conf";
 560 $ENV{SMB_CONF_PATH} = $conffile;
 561
 562 sub write_clientconf($$)
 563 {
 564         my ($conffile, $vars) = @_;
 565
 566         mkdir("$prefix/client", 0777) unless -d "$prefix/client";
 567
 568         if ( -d "$prefix/client/private" ) {
 569                 unlink <$prefix/client/private/*>;
 570         } else {
 571                 mkdir("$prefix/client/private", 0777);
 572         }
 573
 574         if ( -d "$prefix/client/lock" ) {
 575                 unlink <$prefix/client/lockdir/*>;
 576         } else {
 577                 mkdir("$prefix/client/lockdir", 0777);
 578         }
 579
 580         open(CF, ">$conffile");
 581         print CF "[global]\n";
 582         if (defined($ENV{VALGRIND})) {
 583                 print CF "\ticonv:native = true\n";
 584         } else {
 585                 print CF "\ticonv:native = false\n";
 586         }
 587         print CF "\tnetbios name = client\n";
 588         if (defined($vars->{DOMAIN})) {
 589                 print CF "\tworkgroup = $vars->{DOMAIN}\n";
 590         }
 591         if (defined($vars->{REALM})) {
 592                 print CF "\trealm = $vars->{REALM}\n";
 593         }
 594         if ($opt_socket_wrapper) {
 595                 print CF "\tinterfaces = $interfaces\n";
 596         }
 597         print CF "
 598         private dir = $prefix_abs/client/private
 599         lock dir = $prefix_abs/client/lockdir
 600         name resolve order = bcast
 601         panic action = $RealBin/gdb_backtrace \%PID\% \%PROG\%
 602         max xmit = 32K
 603         notify:inotify = false
 604         ldb:nosync = true
 605         system:anonymous = true
 606         client lanman auth = Yes
 607         torture:basedir = $prefix_abs/client
 608 #We don't want to pass our self-tests if the PAC code is wrong
 609         gensec:require_pac = true
 610         modules dir = $ENV{LD_SAMBA_MODULE_PATH}
 611 ";
 612         close(CF);
 613 }
 614
 615 my @todo = ();
 616
 617 my $testsdir = "$srcdir/selftest";
 618
 619 my %required_envs = ();
 620
 621 sub read_testlist($)
 622 {
 623         my ($filename) = @_;
 624
 625         my @ret = ();
 626         open(IN, $filename) or die("Unable to open $filename: $!");
 627
 628         while (<IN>) {
 629                 if ($_ eq "-- TEST --\n") {
 630                         my $name = <IN>;
 631                         $name =~ s/\n//g;
 632                         my $env = <IN>;
 633                         $env =~ s/\n//g;
 634                         my $cmdline = <IN>;
 635                         $cmdline =~ s/\n//g;
 636                         if (not defined($tests) or $name =~ /$tests/) {
 637                                 $required_envs{$env} = 1;
 638                                 push (@ret, [$name, $env, $cmdline]);
 639                         }
 640                 } else {
 641                         print;
 642                 }
 643         }
 644         close(IN) or die("Error creating recipe");
 645         return @ret;
 646 }
 647
 648 if ($#testlists == -1) {
 649         die("No testlists specified");
 650 }
 651
 652 $ENV{SELFTEST_PREFIX} = "$prefix_abs";
 653 if ($opt_socket_wrapper) {
 654         $ENV{SELFTEST_INTERFACES} = $interfaces;
 655 } else {
 656         $ENV{SELFTEST_INTERFACES} = "";
 657 }
 658 if ($opt_verbose) {
 659         $ENV{SELFTEST_VERBOSE} = "1";
 660 } else {
 661         $ENV{SELFTEST_VERBOSE} = "";
 662 }
 663 if ($opt_quick) {
 664         $ENV{SELFTEST_QUICK} = "1";
 665 } else {
 666         $ENV{SELFTEST_QUICK} = "";
 667 }
 668 $ENV{SELFTEST_TARGET} = $opt_target;
 669 $ENV{SELFTEST_MAXTIME} = $torture_maxtime;
 670
 671 my @available = ();
 672 foreach my $fn (@testlists) {
 673         foreach (read_testlist($fn)) {
 674                 my $name = $$_[0];
 675                 next if (@includes and not find_in_list(\@includes, $name));
 676                 push (@available, $_);
 677         }
 678 }
 679
 680 my $msg_ops;
 681 if ($opt_format eq "buildfarm") {
 682         require output::buildfarm;
 683         $msg_ops = new output::buildfarm($statistics);
 684 } elsif ($opt_format eq "plain") {
 685         require output::plain;
 686         $msg_ops = new output::plain("$prefix/summary", $opt_verbose, $opt_immediate, $statistics, $#available+1);
 687 } elsif ($opt_format eq "html") {
 688         require output::html;
 689         mkdir("test-results", 0777);
 690         $msg_ops = new output::html("test-results", $statistics);
 691 } elsif ($opt_format eq "subunit") {
 692         require output::subunit;
 693         $msg_ops = new output::subunit();
 694 } else {
 695         die("Invalid output format '$opt_format'");
 696 }
 697 $msg_ops->report_time(time());
 698
 699 foreach (@available) {
 700         my $name = $$_[0];
 701         my $skipreason = skip($name);
 702         if ($skipreason) {
 703                 $msg_ops->skip_testsuite($name, $skipreason);
 704         } else {
 705                 push(@todo, $_);
 706         }
 707 }
 708
 709 if ($#todo == -1) {
 710         print STDERR "No tests to run\n";
 711         exit(1);
 712         }
 713
 714 my $suitestotal = $#todo + 1;
 715 my $i = 0;
 716 $| = 1;
 717
 718 my %running_envs = ();
 719
 720 sub get_running_env($)
 721 {
 722         my ($name) = @_;
 723
 724         my $envname = $name;
 725
 726         $envname =~ s/:.*//;
 727
 728         return $running_envs{$envname};
 729 }
 730
 731 my @exported_envvars = (
 732         # domain stuff
 733         "DOMAIN",
 734         "REALM",
 735
 736         # domain controller stuff
 737         "DC_SERVER",
 738         "DC_SERVER_IP",
 739         "DC_NETBIOSNAME",
 740         "DC_NETBIOSALIAS",
 741
 742         # server stuff
 743         "SERVER",
 744         "SERVER_IP",
 745         "NETBIOSNAME",
 746         "NETBIOSALIAS",
 747
 748         # user stuff
 749         "USERNAME",
 750         "PASSWORD",
 751         "DC_USERNAME",
 752         "DC_PASSWORD",
 753
 754         # misc stuff
 755         "KRB5_CONFIG",
 756         "WINBINDD_SOCKET_DIR",
 757         "WINBINDD_PRIV_PIPE_DIR"
 758 );
 759
 760 $SIG{INT} = $SIG{QUIT} = $SIG{TERM} = sub {
 761         my $signame = shift;
 762         teardown_env($_) foreach(keys %running_envs);
 763         die("Received signal $signame");
 764 };
 765
 766 sub setup_env($)
 767 {
 768         my ($name) = @_;
 769
 770         my $testenv_vars = undef;
 771
 772         my $envname = $name;
 773         my $option = $name;
 774
 775         $envname =~ s/:.*//;
 776         $option =~ s/^[^:]*//;
 777         $option =~ s/^://;
 778
 779         $option = "client" if $option eq "";
 780
 781         if ($envname eq "none") {
 782                 $testenv_vars = {};
 783         } elsif (defined(get_running_env($envname))) {
 784                 $testenv_vars = get_running_env($envname);
 785                 if (not $target->check_env($testenv_vars)) {
 786                         $testenv_vars = undef;
 787                 }
 788         } else {
 789                 $testenv_vars = $target->setup_env($envname, $prefix);
 790         }
 791
 792         return undef unless defined($testenv_vars);
 793
 794         $running_envs{$envname} = $testenv_vars;
 795
 796         if ($option eq "local") {
 797                 SocketWrapper::set_default_iface($testenv_vars->{SOCKET_WRAPPER_DEFAULT_IFACE});
 798                 $ENV{SMB_CONF_PATH} = $testenv_vars->{SERVERCONFFILE};
 799         } elsif ($option eq "client") {
 800                 SocketWrapper::set_default_iface(6);
 801                 write_clientconf($conffile, $testenv_vars);
 802                 $ENV{SMB_CONF_PATH} = $conffile;
 803         } else {
 804                 die("Unknown option[$option] for envname[$envname]");
 805         }
 806
 807         foreach (@exported_envvars) {
 808                 if (defined($testenv_vars->{$_})) {
 809                         $ENV{$_} = $testenv_vars->{$_};
 810                 } else {
 811                         delete $ENV{$_};
 812                 }
 813         }
 814
 815         return $testenv_vars;
 816 }
 817
 818 sub exported_envvars_str($)
 819 {
 820         my ($testenv_vars) = @_;
 821         my $out = "";
 822
 823         foreach (@exported_envvars) {
 824                 next unless defined($testenv_vars->{$_});
 825                 $out .= $_."=".$testenv_vars->{$_}."\n";
 826         }
 827
 828         return $out;
 829 }
 830
 831 sub getlog_env($)
 832 {
 833         my ($envname) = @_;
 834         return "" if ($envname eq "none");
 835         return $target->getlog_env(get_running_env($envname));
 836 }
 837
 838 sub check_env($)
 839 {
 840         my ($envname) = @_;
 841         return 1 if ($envname eq "none");
 842         return $target->check_env(get_running_env($envname));
 843 }
 844
 845 sub teardown_env($)
 846 {
 847         my ($envname) = @_;
 848         return if ($envname eq "none");
 849         $target->teardown_env(get_running_env($envname));
 850         delete $running_envs{$envname};
 851 }
 852
 853 if ($opt_no_lazy_setup) {
 854         setup_env($_) foreach (keys %required_envs);
 855 }
 856
 857 if ($opt_testenv) {
 858         my $testenv_name = $ENV{SELFTEST_TESTENV};
 859         $testenv_name = $testenv_default unless defined($testenv_name);
 860
 861         my $testenv_vars = setup_env($testenv_name);
 862
 863         $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
 864
 865         my $envvarstr = exported_envvars_str($testenv_vars);
 866
 867         my $term = ($ENV{TERM} or "xterm");
 868         system("$term -e 'echo -e \"
 869 Welcome to the Samba4 Test environment '$testenv_name'
 870
 871 This matches the client environment used in make test
 872 server is pid `cat \$PIDDIR/samba.pid`
 873
 874 Some useful environment variables:
 875 TORTURE_OPTIONS=\$TORTURE_OPTIONS
 876 SMB_CONF_PATH=\$SMB_CONF_PATH
 877
 878 $envvarstr
 879 \" && LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH} bash'");
 880         teardown_env($testenv_name);
 881 } else {
 882         foreach (@todo) {
 883                 $i++;
 884                 my $cmd = $$_[2];
 885                 $cmd =~ s/([\(\)])/\\$1/g;
 886                 my $name = $$_[0];
 887                 my $envname = $$_[1];
 888
 889                 my $envvars = setup_env($envname);
 890                 if (not defined($envvars)) {
 891                         $msg_ops->skip_testsuite($name,
 892                                 "unable to set up environment $envname");
 893                         next;
 894                 }
 895
 896                 run_testsuite($envname, $name, $cmd, $i, $suitestotal,
 897                               $msg_ops);
 898
 899                 if (defined($opt_analyse_cmd)) {
 900                         system("$opt_analyse_cmd \"$name\"");
 901                 }
 902
 903                 teardown_env($envname) if ($opt_resetup_env);
 904         }
 905 }
 906
 907 print "\n";
 908
 909 teardown_env($_) foreach (keys %running_envs);
 910
 911 $target->stop();
 912
 913 $msg_ops->summary();
 914
 915 my $failed = 0;
 916
 917 # if there were any valgrind failures, show them
 918 foreach (<$prefix/valgrind.log*>) {
 919         next unless (-s $_);
 920         system("grep DWARF2.CFI.reader $_ > /dev/null");
 921         if ($? >> 8 == 0) {
 922             print "VALGRIND FAILURE\n";
 923             $failed++;
 924             system("cat $_");
 925         }
 926 }
 927
 928 if ($opt_format eq "buildfarm") {
 929         print "TEST STATUS: $statistics->{SUITES_FAIL}\n";
 930 }
 931
 932 exit $statistics->{SUITES_FAIL};