source/script/tests/selftest.pl

   1 #!/usr/bin/perl
   2 # Bootstrap Samba and run a number of tests against it.
   3 # Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
   4 # Published under the GNU GPL, v3 or later.
   5
   6 =pod
   7
   8 =head1 NAME
   9
  10 selftest - Samba test runner
  11
  12 =head1 SYNOPSIS
  13
  14 selftest --help
  15
  16 selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win] [--socket-wrapper] [--quick] [--one] [--prefix=prefix] [--immediate] [TESTS]
  17
  18 =head1 DESCRIPTION
  19
  20 A simple test runner. TESTS is a regular expression with tests to run.
  21
  22 =head1 OPTIONS
  23
  24 =over 4
  25
  26 =item I<--help>
  27
  28 Show list of available options.
  29
  30 =item I<--srcdir=DIR>
  31
  32 Source directory.
  33
  34 =item I<--builddir=DIR>
  35
  36 Build directory.
  37
  38 =item I<--prefix=DIR>
  39
  40 Change directory to run tests in. Default is 'st'.
  41
  42 =item I<--immediate>
  43
  44 Show errors as soon as they happen rather than at the end of the test run.
  45
  46 =item I<--target samba4|samba3|win>
  47
  48 Specify test target against which to run. Default is 'samba4'.
  49
  50 =item I<--quick>
  51
  52 Run only a limited number of tests. Intended to run in about 30 seconds on
  53 moderately recent systems.
  54
  55 =item I<--socket-wrapper>
  56
  57 Use socket wrapper library for communication with server. Only works
  58 when the server is running locally.
  59
  60 Will prevent TCP and UDP ports being opened on the local host but
  61 (transparently) redirects these calls to use unix domain sockets.
  62
  63 =item I<--expected-failures>
  64
  65 Specify a file containing a list of tests that are expected to fail. Failures for
  66 these tests will be counted as successes, successes will be counted as failures.
  67
  68 The format for the file is, one entry per line:
  69
  70 TESTSUITE-NAME/TEST-NAME
  71
  72 =item I<--skip>
  73
  74 Specify a file containing a list of tests that should be skipped. Possible candidates are
  75 tests that segfault the server, flip or don't end.
  76
  77 =item I<--one>
  78
  79 Abort as soon as one test fails.
  80
  81 =back
  82
  83 =head1 ENVIRONMENT
  84
  85 =over 4
  86
  87 =item I<SMBD_VALGRIND>
  88
  89 =item I<TORTURE_MAXTIME>
  90
  91 =item I<VALGRIND>
  92
  93 =item I<TLS_ENABLED>
  94
  95 =item I<srcdir>
  96
  97 =back
  98
  99 =head1 LICENSE
 100
 101 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
 102
 103 =head1 AUTHOR
 104
 105 Jelmer Vernooij
 106
 107 =cut
 108
 109 use strict;
 110 use warnings;
 111
 112 use FindBin qw($RealBin $Script);
 113 use File::Spec;
 114 use Getopt::Long;
 115 use POSIX;
 116 use Cwd qw(abs_path);
 117 use lib "$RealBin";
 118 use Samba3;
 119 use Samba4;
 120 use SocketWrapper;
 121
 122 my $opt_help = 0;
 123 my $opt_target = "samba4";
 124 my $opt_quick = 0;
 125 my $opt_socket_wrapper = 0;
 126 my $opt_socket_wrapper_pcap = undef;
 127 my $opt_socket_wrapper_keep_pcap = undef;
 128 my $opt_one = 0;
 129 my $opt_immediate = 0;
 130 my $opt_expected_failures = undef;
 131 my $opt_skip = undef;
 132 my $opt_verbose = 0;
 133 my $opt_testenv = 0;
 134 my $ldap = undef;
 135 my $opt_analyse_cmd = undef;
 136 my $opt_resetup_env = undef;
 137 my $opt_bindir = undef;
 138
 139 my $srcdir = ".";
 140 my $builddir = ".";
 141 my $prefix = "./st";
 142
 143 my $suitesfailed = [];
 144 my $start = time();
 145 my @expected_failures = ();
 146 my @skips = ();
 147
 148 my $statistics = {
 149         SUITES_FAIL => 0,
 150         SUITES_OK => 0,
 151         SUITES_SKIPPED => 0,
 152
 153         TESTS_UNEXPECTED_OK => 0,
 154         TESTS_EXPECTED_OK => 0,
 155         TESTS_UNEXPECTED_FAIL => 0,
 156         TESTS_EXPECTED_FAIL => 0,
 157         TESTS_ERROR => 0
 158 };
 159
 160 sub expecting_failure($)
 161 {
 162         my $fullname = shift;
 163
 164         return 1 if (grep(/^$fullname$/, @expected_failures));
 165
 166         return 0;
 167 }
 168
 169 sub skip($)
 170 {
 171         my $fullname = shift;
 172
 173         return 1 if (grep(/^$fullname$/, @skips));
 174         return 0;
 175 }
 176
 177 sub run_test_buildfarm($$$$)
 178 {
 179         my ($name, $cmd, $i, $suitestotal) = @_;
 180         print "--==--==--==--==--==--==--==--==--==--==--\n";
 181         print "Running test $name (level 0 stdout)\n";
 182         print "--==--==--==--==--==--==--==--==--==--==--\n";
 183         system("date");
 184
 185         my $expected_ret = 1;
 186         my $open_tests = {};
 187         open(RESULT, "$cmd 2>&1|");
 188         while (<RESULT>) {
 189                 print;
 190                 if (/^test: (.+)\n/) {
 191                         $open_tests->{$1} = 1;
 192                 } elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
 193                         my $result = $1;
 194                         if ($1 eq "success") {
 195                                 delete $open_tests->{$2};
 196                                 if (expecting_failure("$name/$2")) {
 197                                         $statistics->{TESTS_UNEXPECTED_OK}++;
 198                                 } else {
 199                                         $statistics->{TESTS_EXPECTED_OK}++;
 200                                 }
 201                         } elsif ($1 eq "failure") {
 202                                 delete $open_tests->{$2};
 203                                 if (expecting_failure("$name/$2")) {
 204                                         $statistics->{TESTS_EXPECTED_FAIL}++;
 205                                         $expected_ret = 0;
 206                                 } else {
 207                                         $statistics->{TESTS_UNEXPECTED_FAIL}++;
 208                                 }
 209                         } elsif ($1 eq "skip") {
 210                                 delete $open_tests->{$2};
 211                         } elsif ($1 eq "error") {
 212                                 $statistics->{TESTS_ERROR}++;
 213                                 delete $open_tests->{$2};
 214                         }
 215                 }
 216         }
 217         print "COMMAND: $cmd\n";
 218         foreach (keys %$open_tests) {
 219                 print "$_ was started but never finished!\n";
 220                 $statistics->{TESTS_ERROR}++;
 221         }
 222         my $ret = close(RESULT);
 223
 224         print "==========================================\n";
 225         if ($ret == $expected_ret) {
 226                 print "TEST PASSED: $name\n";
 227         } else {
 228                 push(@$suitesfailed, $name);
 229                 print "TEST FAILED: $name (status $ret)\n";
 230         }
 231         print "==========================================\n";
 232
 233         return ($ret == $expected_ret);
 234 }
 235
 236 my $test_output = {};
 237 sub run_test_plain($$$$)
 238 {
 239         my ($name, $cmd, $i, $totalsuites) = @_;
 240         my $err = "";
 241         if ($#$suitesfailed+1 > 0) { $err = ", ".($#$suitesfailed+1)." errors"; }
 242         print "[$i/$totalsuites in " . (time() - $start)."s$err] $name\n";
 243         open(RESULT, "$cmd 2>&1|");
 244         my $expected_ret = 1;
 245         my $open_tests = {};
 246         $test_output->{$name} = "";
 247         while (<RESULT>) {
 248                 $test_output->{$name}.=$_;
 249                 print if ($opt_verbose);
 250                 if (/^test: (.+)\n/) {
 251                         $open_tests->{$1} = 1;
 252                 } elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
 253                         my $result = $1;
 254                         if ($1 eq "success") {
 255                                 delete $open_tests->{$2};
 256                                 if (expecting_failure("$name/$2")) {
 257                                         $statistics->{TESTS_UNEXPECTED_OK}++;
 258                                 } else {
 259                                         $statistics->{TESTS_EXPECTED_OK}++;
 260                                 }
 261                         } elsif ($1 eq "failure") {
 262                                 delete $open_tests->{$2};
 263                                 if (expecting_failure("$name/$2")) {
 264                                         $statistics->{TESTS_EXPECTED_FAIL}++;
 265                                         $expected_ret = 0;
 266                                 } else {
 267                                         $statistics->{TESTS_UNEXPECTED_FAIL}++;
 268                                 }
 269                         } elsif ($1 eq "skip") {
 270                                 delete $open_tests->{$2};
 271                         } elsif ($1 eq "error") {
 272                                 $statistics->{TESTS_ERROR}++;
 273                                 delete $open_tests->{$2};
 274                         }
 275                 }
 276         }
 277         $test_output->{$name}.="COMMAND: $cmd\n";
 278         foreach (keys %$open_tests) {
 279                 $test_output->{$name}.="$_ was started but never finished!\n";
 280                 $statistics->{TESTS_ERROR}++;
 281         }
 282         my $ret = close(RESULT);
 283         if ($ret != $expected_ret and ($opt_immediate or $opt_one) and not $opt_verbose) {
 284                 print "$test_output->{$name}\n";
 285         }
 286         if ($ret != $expected_ret) {
 287                 push(@$suitesfailed, $name);
 288                 $statistics->{SUITES_FAIL}++;
 289                 exit(1) if ($opt_one);
 290         } else {
 291                 $statistics->{SUITES_OK}++;
 292         }
 293
 294         return ($ret == $expected_ret);
 295 }
 296
 297 sub ShowHelp()
 298 {
 299         print "Samba test runner
 300 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
 301
 302 Usage: $Script [OPTIONS] PREFIX
 303
 304 Generic options:
 305  --help                     this help page
 306  --target=samba4|samba3|win Samba version to target
 307
 308 Paths:
 309  --prefix=DIR               prefix to run tests in [st]
 310  --srcdir=DIR               source directory [.]
 311  --builddir=DIR             output directory [.]
 312
 313 Target Specific:
 314  --socket-wrapper-pcap=DIR      save traffic to pcap directories
 315  --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
 316                             failed
 317  --socket-wrapper           enable socket wrapper
 318  --expected-failures=FILE   specify list of tests that is guaranteed to fail
 319
 320 Samba4 Specific:
 321  --ldap=openldap|fedora     back smbd onto specified ldap server
 322
 323 Samba3 Specific:
 324  --bindir=PATH              path to binaries
 325
 326 Behaviour:
 327  --quick                    run quick overall test
 328  --one                      abort when the first test fails
 329  --immediate                print test output for failed tests during run
 330  --verbose                  be verbose
 331  --analyse-cmd CMD          command to run after each test
 332 ";
 333         exit(0);
 334 }
 335
 336 my $result = GetOptions (
 337                 'help|h|?' => \$opt_help,
 338                 'target=s' => \$opt_target,
 339                 'prefix=s' => \$prefix,
 340                 'socket-wrapper' => \$opt_socket_wrapper,
 341                 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
 342                 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
 343                 'quick' => \$opt_quick,
 344                 'one' => \$opt_one,
 345                 'immediate' => \$opt_immediate,
 346                 'expected-failures=s' => \$opt_expected_failures,
 347                 'skip=s' => \$opt_skip,
 348                 'srcdir=s' => \$srcdir,
 349                 'builddir=s' => \$builddir,
 350                 'verbose' => \$opt_verbose,
 351                 'testenv' => \$opt_testenv,
 352                 'ldap:s' => \$ldap,
 353                 'analyse-cmd=s' => \$opt_analyse_cmd,
 354                 'resetup-environment' => \$opt_resetup_env,
 355                 'bindir:s' => \$opt_bindir,
 356             );
 357
 358 exit(1) if (not $result);
 359
 360 ShowHelp() if ($opt_help);
 361
 362 my $tests = shift;
 363
 364 # quick hack to disable rpc validation when using valgrind - its way too slow
 365 unless (defined($ENV{VALGRIND})) {
 366         $ENV{VALIDATE} = "validate";
 367         $ENV{MALLOC_CHECK_} = 2;
 368 }
 369
 370 my $old_pwd = "$RealBin/../..";
 371
 372 # Backwards compatibility:
 373 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
 374         if (defined($ENV{FEDORA_DS_PREFIX})) {
 375                 $ldap = "fedora";
 376         } else {
 377                 $ldap = "openldap";
 378         }
 379 }
 380
 381 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
 382 if ($ldap) {
 383         # LDAP is slow
 384         $torture_maxtime *= 2;
 385 }
 386
 387 $prefix =~ s+//+/+;
 388 $prefix =~ s+/./+/+;
 389 $prefix =~ s+/$++;
 390
 391 my $prefix_abs = abs_path($prefix);
 392 my $srcdir_abs = abs_path($srcdir);
 393
 394 die("using an empty prefix isn't allowed") unless $prefix ne "";
 395 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
 396 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
 397
 398 $ENV{PREFIX} = $prefix;
 399 $ENV{SRCDIR} = $srcdir;
 400
 401 #Ensure we have the test prefix around
 402 mkdir $prefix unless -d $prefix;
 403
 404 my $tls_enabled = not $opt_quick;
 405 my $from_build_farm = (defined($ENV{RUN_FROM_BUILD_FARM}) and
 406                       ($ENV{RUN_FROM_BUILD_FARM} eq "yes"));
 407
 408 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
 409 $ENV{LD_LDB_MODULE_PATH} = "$old_pwd/bin/modules/ldb";
 410 $ENV{LD_SAMBA_MODULE_PATH} = "$old_pwd/bin/modules";
 411 if (defined($ENV{LD_LIBRARY_PATH})) {
 412         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared:$ENV{LD_LIBRARY_PATH}";
 413 } else {
 414         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared";
 415 }
 416 $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig:$ENV{PKG_CONFIG_PATH}";
 417 $ENV{PATH} = "$old_pwd/bin:$ENV{PATH}";
 418
 419 my $pcap_dir = "$prefix/pcap";
 420
 421 if ($opt_socket_wrapper_pcap) {
 422         mkdir($pcap_dir);
 423         # Socket wrapper pcap implies socket wrapper
 424         $opt_socket_wrapper = 1;
 425 }
 426
 427 my $socket_wrapper_dir;
 428 if ($opt_socket_wrapper) {
 429         $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w");
 430         print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
 431 } else {
 432         warn("Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports") unless $< == 0;
 433 }
 434
 435 my $target;
 436
 437 if ($opt_target eq "samba4") {
 438         $target = new Samba4("$srcdir/bin", $ldap, "$srcdir/setup");
 439 } elsif ($opt_target eq "samba3") {
 440         if ($opt_socket_wrapper and `smbd -b | grep SOCKET_WRAPPER` eq "") {
 441                 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'.  Exiting....");
 442         }
 443
 444         $target = new Samba3($opt_bindir);
 445 } elsif ($opt_target eq "win") {
 446         die("Windows tests will not run with socket wrapper enabled.")
 447                 if ($opt_socket_wrapper);
 448         $target = new Windows();
 449 }
 450
 451 if (defined($opt_expected_failures)) {
 452         open(KNOWN, "<$opt_expected_failures") or die("unable to read known failures file: $!");
 453         while (<KNOWN>) {
 454                 chomp;
 455                 s/([ \t]+)\#(.*)$//;
 456                 push (@expected_failures, $_); }
 457         close(KNOWN);
 458 }
 459
 460 if (defined($opt_skip)) {
 461         open(SKIP, "<$opt_skip") or die("unable to read skip file: $!");
 462         while (<SKIP>) {
 463                 chomp;
 464                 s/([ \t]+)\#(.*)$//;
 465                 push (@skips, $_); }
 466         close(SKIP);
 467 }
 468
 469 my $interfaces = join(',', ("127.0.0.6/8",
 470                             "127.0.0.7/8",
 471                             "127.0.0.8/8",
 472                             "127.0.0.9/8",
 473                             "127.0.0.10/8",
 474                             "127.0.0.11/8"));
 475
 476 my $conffile = "$prefix_abs/client/client.conf";
 477
 478 sub write_clientconf($$)
 479 {
 480         my ($conffile, $vars) = @_;
 481
 482         mkdir "$prefix/client" unless -d "$prefix/client";
 483
 484         if ( -d "$prefix/client/private" ) {
 485                 unlink <$prefix/client/private/*>;
 486         } else {
 487                 mkdir("$prefix/client/private");
 488         }
 489
 490         open(CF, ">$conffile");
 491         print CF "[global]\n";
 492         if (defined($ENV{VALGRIND})) {
 493                 print CF "\ticonv:native = true\n";
 494         } else {
 495                 print CF "\ticonv:native = false\n";
 496         }
 497         print CF "\tnetbios name = client\n";
 498         if (defined($vars->{DOMAIN})) {
 499                 print CF "\tworkgroup = $vars->{DOMAIN}\n";
 500         }
 501         if (defined($vars->{REALM})) {
 502                 print CF "\trealm = $vars->{REALM}\n";
 503         }
 504         if (defined($vars->{NCALRPCDIR})) {
 505                 print CF "\tncalrpc dir = $vars->{NCALRPCDIR}\n";
 506         }
 507         if (defined($vars->{PIDDIR})) {
 508                 print CF "\tpid directory = $vars->{PIDDIR}\n";
 509         }
 510         if (defined($vars->{WINBINDD_SOCKET_DIR})) {
 511                 print CF "\twinbindd socket directory = $vars->{WINBINDD_SOCKET_DIR}\n";
 512         }
 513         print CF "
 514         private dir = $srcdir_abs/$prefix/client/private
 515         js include = $srcdir_abs/scripting/libjs
 516         name resolve order = bcast
 517         interfaces = $interfaces
 518         panic action = $srcdir_abs/script/gdb_backtrace \%PID\% \%PROG\%
 519         max xmit = 32K
 520         notify:inotify = false
 521         ldb:nosync = true
 522         system:anonymous = true
 523         torture:basedir = $prefix_abs
 524 #We don't want to pass our self-tests if the PAC code is wrong
 525         gensec:require_pac = true
 526 ";
 527         close(CF);
 528 }
 529
 530
 531 my @torture_options = ();
 532 push (@torture_options, "--configfile=$conffile");
 533 # ensure any one smbtorture call doesn't run too long
 534 push (@torture_options, "--maximum-runtime=$torture_maxtime");
 535 push (@torture_options, "--target=$opt_target");
 536 push (@torture_options, "--option=torture:progress=no") if ($from_build_farm);
 537 push (@torture_options, "--format=subunit");
 538 push (@torture_options, "--option=torture:quick=yes") if ($opt_quick);
 539
 540 $ENV{TORTURE_OPTIONS} = join(' ', @torture_options);
 541 print "OPTIONS $ENV{TORTURE_OPTIONS}\n";
 542
 543 my @todo = ();
 544
 545 my $testsdir = "$srcdir/script/tests";
 546 $ENV{CONFIGURATION} = "--configfile=$conffile";
 547
 548
 549 if ($opt_quick) {
 550         open(IN, "$testsdir/tests_quick.sh|");
 551 } else {
 552         open(IN, "$testsdir/tests_all.sh|");
 553 }
 554 while (<IN>) {
 555         if ($_ eq "-- TEST --\n") {
 556                 my $name = <IN>;
 557                 $name =~ s/\n//g;
 558                 my $env = <IN>;
 559                 $env =~ s/\n//g;
 560                 my $cmdline = <IN>;
 561                 $cmdline =~ s/\n//g;
 562                 push (@todo, [$name, $env, $cmdline])
 563                         if (not defined($tests) or $name =~ /$tests/);
 564         } else {
 565                 print;
 566         }
 567 }
 568 close(IN) or die("Error creating recipe");
 569
 570 my $suitestotal = $#todo + 1;
 571 my $i = 0;
 572 $| = 1;
 573
 574 my %running_envs = ();
 575
 576 sub setup_env($)
 577 {
 578         my ($envname) = @_;
 579
 580         my $testenv_vars;
 581         if (defined($running_envs{$envname})) {
 582                 $testenv_vars = $running_envs{$envname};
 583         } elsif ($envname eq "none") {
 584                 $testenv_vars = {};
 585         } else {
 586                 $testenv_vars = $target->setup_env($envname, $prefix);
 587         }
 588         write_clientconf($conffile, $testenv_vars);
 589         foreach ("PASSWORD", "DOMAIN", "SERVER", "USERNAME", "NETBIOSNAME",
 590                          "KRB5_CONFIG", "REALM") {
 591                 if (defined($testenv_vars->{$_})) {
 592                         $ENV{$_} = $testenv_vars->{$_};
 593                 } else {
 594                         delete $ENV{$_};
 595                 }
 596         }
 597
 598         SocketWrapper::set_default_iface(6);
 599
 600         $running_envs{$envname} = $testenv_vars;
 601         return $testenv_vars;
 602 }
 603
 604 sub teardown_env($)
 605 {
 606         my ($envname) = @_;
 607         return if ($envname eq "none");
 608         $target->teardown_env($running_envs{$envname});
 609         delete $running_envs{$envname};
 610 }
 611
 612
 613 if ($opt_testenv) {
 614         my $testenv_vars = setup_env("dc");
 615         $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
 616         my $term = ($ENV{TERM} or "xterm");
 617         system("$term -e 'echo -e \"Welcome to the Samba4 Test environment
 618 This matches the client environment used in make test
 619 smbd is pid `cat \$PIDDIR/smbd.pid`
 620
 621 Some useful environment variables:
 622 TORTURE_OPTIONS=\$TORTURE_OPTIONS
 623 CONFIGURATION=\$CONFIGURATION
 624 SERVER=\$SERVER
 625 NETBIOSNAME=\$NETBIOSNAME\" && bash'");
 626         teardown_env("dc");
 627 } else {
 628         foreach (@todo) {
 629                 $i++;
 630                 my $cmd = $$_[2];
 631                 $cmd =~ s/([\(\)])/\\$1/g;
 632                 my $name = $$_[0];
 633                 my $envname = $$_[1];
 634
 635                 if (skip($name)) {
 636                         print "SKIPPED: $name\n";
 637                         $statistics->{SUITES_SKIPPED}++;
 638                         next;
 639                 }
 640
 641                 setup_env($envname);
 642
 643                 my $pcap_file = "$pcap_dir/$name.cap";
 644
 645                 SocketWrapper::setup_pcap($pcap_file) if ($opt_socket_wrapper_pcap);
 646                 my $result;
 647                 if ($from_build_farm) {
 648                         $result = run_test_buildfarm($name, $cmd, $i, $suitestotal);
 649                 } else {
 650                         $result = run_test_plain($name, $cmd, $i, $suitestotal);
 651                 }
 652
 653                 if ($opt_socket_wrapper_pcap and $result and
 654                         not $opt_socket_wrapper_keep_pcap) {
 655                         unlink($pcap_file);
 656                 }
 657
 658                 if (defined($opt_analyse_cmd)) {
 659                         system("$opt_analyse_cmd \"$name\"");
 660                 }
 661
 662                 teardown_env($envname) if ($opt_resetup_env);
 663         }
 664 }
 665
 666 print "\n";
 667
 668 teardown_env($_) foreach (keys %running_envs);
 669
 670 $target->stop();
 671
 672 my $end = time();
 673 my $duration = ($end-$start);
 674 my $numfailed = $#$suitesfailed+1;
 675 if ($numfailed == 0) {
 676         my $ok = $statistics->{TESTS_EXPECTED_OK} + $statistics->{TESTS_EXPECTED_FAIL};
 677         print "ALL OK ($ok tests in $statistics->{SUITES_OK} testsuites)\n";
 678 } else {
 679         unless ($from_build_farm) {
 680                 if (not $opt_immediate and not $opt_verbose) {
 681                         foreach (@$suitesfailed) {
 682                                 print "===============================================================================\n";
 683                                 print "FAIL: $_\n";
 684                                 print $test_output->{$_};
 685                                 print "\n";
 686                         }
 687                 }
 688
 689                 print "FAILED ($statistics->{TESTS_UNEXPECTED_FAIL} failures and $statistics->{TESTS_ERROR} errors in $statistics->{SUITES_FAIL} testsuites)\n";
 690         }
 691 }
 692 print "DURATION: $duration seconds\n";
 693
 694 my $failed = 0;
 695
 696 # if there were any valgrind failures, show them
 697 foreach (<$prefix/valgrind.log*>) {
 698         next unless (-s $_);
 699         system("grep DWARF2.CFI.reader $_ > /dev/null");
 700         if ($? >> 8 == 0) {
 701             print "VALGRIND FAILURE\n";
 702             $failed++;
 703             system("cat $_");
 704         }
 705 }
 706
 707 if ($from_build_farm) {
 708         print "TEST STATUS: $numfailed\n";
 709 }
 710
 711 exit $numfailed;