source/script/tests/selftest.pl

   1 #!/usr/bin/perl
   2 # Bootstrap Samba and run a number of tests against it.
   3 # Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
   4 # Published under the GNU GPL, v3 or later.
   5
   6 =pod
   7
   8 =head1 NAME
   9
  10 selftest - Samba test runner
  11
  12 =head1 SYNOPSIS
  13
  14 selftest --help
  15
  16 selftest [--srcdir=DIR] [--builddir=DIR] [--target=samba4|samba3|win] [--socket-wrapper] [--quick] [--one] [--prefix=prefix] [--immediate] [TESTS]
  17
  18 =head1 DESCRIPTION
  19
  20 A simple test runner. TESTS is a regular expression with tests to run.
  21
  22 =head1 OPTIONS
  23
  24 =over 4
  25
  26 =item I<--help>
  27
  28 Show list of available options.
  29
  30 =item I<--srcdir=DIR>
  31
  32 Source directory.
  33
  34 =item I<--builddir=DIR>
  35
  36 Build directory.
  37
  38 =item I<--prefix=DIR>
  39
  40 Change directory to run tests in. Default is 'st'.
  41
  42 =item I<--immediate>
  43
  44 Show errors as soon as they happen rather than at the end of the test run.
  45
  46 =item I<--target samba4|samba3|win>
  47
  48 Specify test target against which to run. Default is 'samba4'.
  49
  50 =item I<--quick>
  51
  52 Run only a limited number of tests. Intended to run in about 30 seconds on
  53 moderately recent systems.
  54
  55 =item I<--socket-wrapper>
  56
  57 Use socket wrapper library for communication with server. Only works
  58 when the server is running locally.
  59
  60 Will prevent TCP and UDP ports being opened on the local host but
  61 (transparently) redirects these calls to use unix domain sockets.
  62
  63 =item I<--expected-failures>
  64
  65 Specify a file containing a list of tests that are expected to fail. Failures for
  66 these tests will be counted as successes, successes will be counted as failures.
  67
  68 The format for the file is, one entry per line:
  69
  70 TESTSUITE-NAME/TEST-NAME
  71
  72 =item I<--skip>
  73
  74 Specify a file containing a list of tests that should be skipped. Possible candidates are
  75 tests that segfault the server, flip or don't end.
  76
  77 =item I<--one>
  78
  79 Abort as soon as one test fails.
  80
  81 =back
  82
  83 =head1 ENVIRONMENT
  84
  85 =over 4
  86
  87 =item I<SMBD_VALGRIND>
  88
  89 =item I<TORTURE_MAXTIME>
  90
  91 =item I<VALGRIND>
  92
  93 =item I<TLS_ENABLED>
  94
  95 =item I<srcdir>
  96
  97 =back
  98
  99 =head1 LICENSE
 100
 101 selftest is licensed under the GNU General Public License L<http://www.gnu.org/licenses/gpl.html>.
 102
 103 =head1 AUTHOR
 104
 105 Jelmer Vernooij
 106
 107 =cut
 108
 109 use strict;
 110 use warnings;
 111
 112 use FindBin qw($RealBin $Script);
 113 use File::Spec;
 114 use Getopt::Long;
 115 use POSIX;
 116 use Cwd qw(abs_path);
 117 use lib "$RealBin";
 118 use Samba3;
 119 use Samba4;
 120 use SocketWrapper;
 121
 122 my $opt_help = 0;
 123 my $opt_target = "samba4";
 124 my $opt_quick = 0;
 125 my $opt_socket_wrapper = 0;
 126 my $opt_socket_wrapper_pcap = undef;
 127 my $opt_socket_wrapper_keep_pcap = undef;
 128 my $opt_one = 0;
 129 my $opt_immediate = 0;
 130 my $opt_expected_failures = undef;
 131 my $opt_skip = undef;
 132 my $opt_verbose = 0;
 133 my $opt_testenv = 0;
 134 my $ldap = undef;
 135 my $opt_analyse_cmd = undef;
 136 my $opt_resetup_env = undef;
 137 my $opt_bindir = undef;
 138
 139 my $srcdir = ".";
 140 my $builddir = ".";
 141 my $prefix = "./st";
 142
 143 my $suitesfailed = [];
 144 my $start = time();
 145 my @expected_failures = ();
 146 my @skips = ();
 147
 148 my $statistics = {
 149         SUITES_FAIL => 0,
 150         SUITES_OK => 0,
 151         SUITES_SKIPPED => 0,
 152
 153         TESTS_UNEXPECTED_OK => 0,
 154         TESTS_EXPECTED_OK => 0,
 155         TESTS_UNEXPECTED_FAIL => 0,
 156         TESTS_EXPECTED_FAIL => 0,
 157         TESTS_ERROR => 0
 158 };
 159
 160 sub expecting_failure($)
 161 {
 162         my $fullname = shift;
 163
 164         return 1 if (grep(/^$fullname$/, @expected_failures));
 165
 166         return 0;
 167 }
 168
 169 sub skip($)
 170 {
 171         my $fullname = shift;
 172
 173         return 1 if (grep(/^$fullname$/, @skips));
 174         return 0;
 175 }
 176
 177 my $test_output = {};
 178
 179 sub buildfarm_start_msg($)
 180 {
 181         my ($state) = @_;
 182         my $out = "";
 183
 184         $out .= "--==--==--==--==--==--==--==--==--==--==--\n";
 185         $out .= "Running test $state->{NAME} (level 0 stdout)\n";
 186         $out .= "--==--==--==--==--==--==--==--==--==--==--\n";
 187         $out .= scalar(localtime())."\n";
 188         $out .= "NAME: $state->{NAME}\n";
 189         $out .= "CMD: $state->{CMD}\n";
 190
 191         $test_output->{$state->{NAME}} = "";
 192
 193         print $out;
 194 }
 195
 196 sub buildfarm_output_msg($$)
 197 {
 198         my ($state, $output) = @_;
 199
 200         $test_output->{$state->{NAME}} .= $output;
 201 }
 202
 203 sub buildfarm_end_msg($$$)
 204 {
 205         my ($state, $expected_ret, $ret) = @_;
 206         my $out = "";
 207
 208         if ($ret == $expected_ret) {
 209                 $out .= "ALL OK\n";
 210         } else {
 211                 $out .= "ERROR: $ret";
 212                 $out .= $test_output->{$state->{NAME}};
 213         }
 214
 215         $out .= "==========================================\n";
 216         if ($ret == $expected_ret) {
 217                 $out .= "TEST PASSED: $state->{NAME}\n";
 218         } else {
 219                 $out .= "TEST FAILED: $state->{NAME} (status $ret)\n";
 220         }
 221         $out .= "==========================================\n";
 222
 223         print $out;
 224 }
 225
 226 my $buildfarm_msg_ops = {
 227         start_msg       => \&buildfarm_start_msg,
 228         output_msg      => \&buildfarm_output_msg,
 229         end_msg         => \&buildfarm_end_msg
 230 };
 231
 232 sub plain_output_msg($$);
 233
 234 sub plain_start_msg($)
 235 {
 236         my ($state) = @_;
 237         my $out = "";
 238
 239         $out .= "[$state->{INDEX}/$state->{TOTAL} in " . ($state->{START} - $start) . "s";
 240         $out .= ", ".($#$suitesfailed+1)." errors" if ($#$suitesfailed+1 > 0);
 241         $out .= "] $state->{NAME}\n";
 242
 243         $test_output->{$state->{NAME}} = "" unless $opt_verbose;
 244
 245         plain_output_msg($state, "CMD: $state->{CMD}\n");
 246
 247         print $out;
 248 }
 249
 250 sub plain_output_msg($$)
 251 {
 252         my ($state, $output) = @_;
 253
 254         if ($opt_verbose) {
 255                 print $output;
 256         } else {
 257                 $test_output->{$state->{NAME}} .= $output;
 258         }
 259 }
 260
 261 sub plain_end_msg($$$)
 262 {
 263         my ($state, $expected_ret, $ret) = @_;
 264
 265         if ($ret != $expected_ret and ($opt_immediate or $opt_one) and not $opt_verbose) {
 266                 print $test_output->{$state->{NAME}}."\n";
 267         }
 268 }
 269
 270 my $plain_msg_ops = {
 271         start_msg       => \&plain_start_msg,
 272         output_msg      => \&plain_output_msg,
 273         end_msg         => \&plain_end_msg
 274 };
 275
 276 sub run_test($$$$$)
 277 {
 278         my ($name, $cmd, $i, $totalsuites, $msg_ops) = @_;
 279         my $msg_state = {
 280                 NAME    => $name,
 281                 CMD     => $cmd,
 282                 INDEX   => $i,
 283                 TOTAL   => $totalsuites,
 284                 START   => time()
 285         };
 286         $msg_ops->{start_msg}($msg_state);
 287         open(RESULT, "$cmd 2>&1|");
 288         my $expected_ret = 1;
 289         my $open_tests = {};
 290         while (<RESULT>) {
 291                 $msg_ops->{output_msg}($msg_state, $_);
 292                 if (/^test: (.+)\n/) {
 293                         $open_tests->{$1} = 1;
 294                 } elsif (/^(success|failure|skip|error): (.*?)( \[)?\n/) {
 295                         my $result = $1;
 296                         if ($1 eq "success") {
 297                                 delete $open_tests->{$2};
 298                                 if (expecting_failure("$name/$2")) {
 299                                         $statistics->{TESTS_UNEXPECTED_OK}++;
 300                                 } else {
 301                                         $statistics->{TESTS_EXPECTED_OK}++;
 302                                 }
 303                         } elsif ($1 eq "failure") {
 304                                 delete $open_tests->{$2};
 305                                 if (expecting_failure("$name/$2")) {
 306                                         $statistics->{TESTS_EXPECTED_FAIL}++;
 307                                         $expected_ret = 0;
 308                                 } else {
 309                                         $statistics->{TESTS_UNEXPECTED_FAIL}++;
 310                                 }
 311                         } elsif ($1 eq "skip") {
 312                                 delete $open_tests->{$2};
 313                         } elsif ($1 eq "error") {
 314                                 $statistics->{TESTS_ERROR}++;
 315                                 delete $open_tests->{$2};
 316                         }
 317                 }
 318         }
 319         foreach (keys %$open_tests) {
 320                 $msg_ops->{output_msg}($msg_state, "$_ was started but never finished!\n");
 321                 $statistics->{TESTS_ERROR}++;
 322         }
 323         my $ret = close(RESULT);
 324         $msg_ops->{end_msg}($msg_state, $expected_ret, $ret);
 325         if ($ret != $expected_ret) {
 326                 push(@$suitesfailed, $name);
 327                 $statistics->{SUITES_FAIL}++;
 328                 exit(1) if ($opt_one);
 329         } else {
 330                 $statistics->{SUITES_OK}++;
 331         }
 332
 333         return ($ret == $expected_ret);
 334 }
 335
 336 sub ShowHelp()
 337 {
 338         print "Samba test runner
 339 Copyright (C) Jelmer Vernooij <jelmer\@samba.org>
 340
 341 Usage: $Script [OPTIONS] PREFIX
 342
 343 Generic options:
 344  --help                     this help page
 345  --target=samba4|samba3|win Samba version to target
 346
 347 Paths:
 348  --prefix=DIR               prefix to run tests in [st]
 349  --srcdir=DIR               source directory [.]
 350  --builddir=DIR             output directory [.]
 351
 352 Target Specific:
 353  --socket-wrapper-pcap=DIR      save traffic to pcap directories
 354  --socket-wrapper-keep-pcap keep all pcap files, not just those for tests that
 355                             failed
 356  --socket-wrapper           enable socket wrapper
 357  --expected-failures=FILE   specify list of tests that is guaranteed to fail
 358
 359 Samba4 Specific:
 360  --ldap=openldap|fedora     back smbd onto specified ldap server
 361
 362 Samba3 Specific:
 363  --bindir=PATH              path to binaries
 364
 365 Behaviour:
 366  --quick                    run quick overall test
 367  --one                      abort when the first test fails
 368  --immediate                print test output for failed tests during run
 369  --verbose                  be verbose
 370  --analyse-cmd CMD          command to run after each test
 371 ";
 372         exit(0);
 373 }
 374
 375 my $result = GetOptions (
 376                 'help|h|?' => \$opt_help,
 377                 'target=s' => \$opt_target,
 378                 'prefix=s' => \$prefix,
 379                 'socket-wrapper' => \$opt_socket_wrapper,
 380                 'socket-wrapper-pcap' => \$opt_socket_wrapper_pcap,
 381                 'socket-wrapper-keep-pcap' => \$opt_socket_wrapper_keep_pcap,
 382                 'quick' => \$opt_quick,
 383                 'one' => \$opt_one,
 384                 'immediate' => \$opt_immediate,
 385                 'expected-failures=s' => \$opt_expected_failures,
 386                 'skip=s' => \$opt_skip,
 387                 'srcdir=s' => \$srcdir,
 388                 'builddir=s' => \$builddir,
 389                 'verbose' => \$opt_verbose,
 390                 'testenv' => \$opt_testenv,
 391                 'ldap:s' => \$ldap,
 392                 'analyse-cmd=s' => \$opt_analyse_cmd,
 393                 'resetup-environment' => \$opt_resetup_env,
 394                 'bindir:s' => \$opt_bindir,
 395             );
 396
 397 exit(1) if (not $result);
 398
 399 ShowHelp() if ($opt_help);
 400
 401 my $tests = shift;
 402
 403 # quick hack to disable rpc validation when using valgrind - its way too slow
 404 unless (defined($ENV{VALGRIND})) {
 405         $ENV{VALIDATE} = "validate";
 406         $ENV{MALLOC_CHECK_} = 2;
 407 }
 408
 409 my $old_pwd = "$RealBin/../..";
 410
 411 # Backwards compatibility:
 412 if (defined($ENV{TEST_LDAP}) and $ENV{TEST_LDAP} eq "yes") {
 413         if (defined($ENV{FEDORA_DS_PREFIX})) {
 414                 $ldap = "fedora";
 415         } else {
 416                 $ldap = "openldap";
 417         }
 418 }
 419
 420 my $torture_maxtime = ($ENV{TORTURE_MAXTIME} or 1200);
 421 if ($ldap) {
 422         # LDAP is slow
 423         $torture_maxtime *= 2;
 424 }
 425
 426 $prefix =~ s+//+/+;
 427 $prefix =~ s+/./+/+;
 428 $prefix =~ s+/$++;
 429
 430 my $prefix_abs = abs_path($prefix);
 431 my $srcdir_abs = abs_path($srcdir);
 432
 433 die("using an empty prefix isn't allowed") unless $prefix ne "";
 434 die("using an empty absolute prefix isn't allowed") unless $prefix_abs ne "";
 435 die("using '/' as absolute prefix isn't allowed") unless $prefix_abs ne "/";
 436
 437 $ENV{PREFIX} = $prefix;
 438 $ENV{SRCDIR} = $srcdir;
 439
 440 #Ensure we have the test prefix around
 441 mkdir $prefix unless -d $prefix;
 442
 443 my $tls_enabled = not $opt_quick;
 444 my $from_build_farm = (defined($ENV{RUN_FROM_BUILD_FARM}) and
 445                       ($ENV{RUN_FROM_BUILD_FARM} eq "yes"));
 446
 447 $ENV{TLS_ENABLED} = ($tls_enabled?"yes":"no");
 448 $ENV{LD_LDB_MODULE_PATH} = "$old_pwd/bin/modules/ldb";
 449 $ENV{LD_SAMBA_MODULE_PATH} = "$old_pwd/bin/modules";
 450 if (defined($ENV{LD_LIBRARY_PATH})) {
 451         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared:$ENV{LD_LIBRARY_PATH}";
 452 } else {
 453         $ENV{LD_LIBRARY_PATH} = "$old_pwd/bin/shared";
 454 }
 455 $ENV{PKG_CONFIG_PATH} = "$old_pwd/bin/pkgconfig:$ENV{PKG_CONFIG_PATH}";
 456 $ENV{PATH} = "$old_pwd/bin:$ENV{PATH}";
 457
 458 my $pcap_dir = "$prefix/pcap";
 459
 460 if ($opt_socket_wrapper_pcap) {
 461         mkdir($pcap_dir);
 462         # Socket wrapper pcap implies socket wrapper
 463         $opt_socket_wrapper = 1;
 464 }
 465
 466 my $socket_wrapper_dir;
 467 if ($opt_socket_wrapper) {
 468         $socket_wrapper_dir = SocketWrapper::setup_dir("$prefix/w");
 469         print "SOCKET_WRAPPER_DIR=$socket_wrapper_dir\n";
 470 } else {
 471         warn("Not using socket wrapper, but also not running as root. Will not be able to listen on proper ports") unless $< == 0;
 472 }
 473
 474 my $target;
 475
 476 if ($opt_target eq "samba4") {
 477         $target = new Samba4("$srcdir/bin", $ldap, "$srcdir/setup");
 478 } elsif ($opt_target eq "samba3") {
 479         if ($opt_socket_wrapper and `smbd -b | grep SOCKET_WRAPPER` eq "") {
 480                 die("You must include --enable-socket-wrapper when compiling Samba in order to execute 'make test'.  Exiting....");
 481         }
 482
 483         $target = new Samba3($opt_bindir);
 484 } elsif ($opt_target eq "win") {
 485         die("Windows tests will not run with socket wrapper enabled.")
 486                 if ($opt_socket_wrapper);
 487         $target = new Windows();
 488 }
 489
 490 if (defined($opt_expected_failures)) {
 491         open(KNOWN, "<$opt_expected_failures") or die("unable to read known failures file: $!");
 492         while (<KNOWN>) {
 493                 chomp;
 494                 s/([ \t]+)\#(.*)$//;
 495                 push (@expected_failures, $_); }
 496         close(KNOWN);
 497 }
 498
 499 if (defined($opt_skip)) {
 500         open(SKIP, "<$opt_skip") or die("unable to read skip file: $!");
 501         while (<SKIP>) {
 502                 chomp;
 503                 s/([ \t]+)\#(.*)$//;
 504                 push (@skips, $_); }
 505         close(SKIP);
 506 }
 507
 508 my $interfaces = join(',', ("127.0.0.6/8",
 509                             "127.0.0.7/8",
 510                             "127.0.0.8/8",
 511                             "127.0.0.9/8",
 512                             "127.0.0.10/8",
 513                             "127.0.0.11/8"));
 514
 515 my $conffile = "$prefix_abs/client/client.conf";
 516
 517 sub write_clientconf($$)
 518 {
 519         my ($conffile, $vars) = @_;
 520
 521         mkdir "$prefix/client" unless -d "$prefix/client";
 522
 523         if ( -d "$prefix/client/private" ) {
 524                 unlink <$prefix/client/private/*>;
 525         } else {
 526                 mkdir("$prefix/client/private");
 527         }
 528
 529         open(CF, ">$conffile");
 530         print CF "[global]\n";
 531         if (defined($ENV{VALGRIND})) {
 532                 print CF "\ticonv:native = true\n";
 533         } else {
 534                 print CF "\ticonv:native = false\n";
 535         }
 536         print CF "\tnetbios name = client\n";
 537         if (defined($vars->{DOMAIN})) {
 538                 print CF "\tworkgroup = $vars->{DOMAIN}\n";
 539         }
 540         if (defined($vars->{REALM})) {
 541                 print CF "\trealm = $vars->{REALM}\n";
 542         }
 543         if (defined($vars->{NCALRPCDIR})) {
 544                 print CF "\tncalrpc dir = $vars->{NCALRPCDIR}\n";
 545         }
 546         if (defined($vars->{PIDDIR})) {
 547                 print CF "\tpid directory = $vars->{PIDDIR}\n";
 548         }
 549         if (defined($vars->{WINBINDD_SOCKET_DIR})) {
 550                 print CF "\twinbindd socket directory = $vars->{WINBINDD_SOCKET_DIR}\n";
 551         }
 552         print CF "
 553         private dir = $srcdir_abs/$prefix/client/private
 554         js include = $srcdir_abs/scripting/libjs
 555         name resolve order = bcast
 556         interfaces = $interfaces
 557         panic action = $srcdir_abs/script/gdb_backtrace \%PID\% \%PROG\%
 558         max xmit = 32K
 559         notify:inotify = false
 560         ldb:nosync = true
 561         system:anonymous = true
 562         torture:basedir = $prefix_abs
 563 #We don't want to pass our self-tests if the PAC code is wrong
 564         gensec:require_pac = true
 565 ";
 566         close(CF);
 567 }
 568
 569
 570 my @torture_options = ();
 571 push (@torture_options, "--configfile=$conffile");
 572 # ensure any one smbtorture call doesn't run too long
 573 push (@torture_options, "--maximum-runtime=$torture_maxtime");
 574 push (@torture_options, "--target=$opt_target");
 575 push (@torture_options, "--option=torture:progress=no") if ($from_build_farm);
 576 push (@torture_options, "--format=subunit");
 577 push (@torture_options, "--option=torture:quick=yes") if ($opt_quick);
 578
 579 $ENV{TORTURE_OPTIONS} = join(' ', @torture_options);
 580 print "OPTIONS $ENV{TORTURE_OPTIONS}\n";
 581
 582 my @todo = ();
 583
 584 my $testsdir = "$srcdir/script/tests";
 585 $ENV{CONFIGURATION} = "--configfile=$conffile";
 586
 587
 588 if ($opt_quick) {
 589         open(IN, "$testsdir/tests_quick.sh|");
 590 } else {
 591         open(IN, "$testsdir/tests_all.sh|");
 592 }
 593 while (<IN>) {
 594         if ($_ eq "-- TEST --\n") {
 595                 my $name = <IN>;
 596                 $name =~ s/\n//g;
 597                 my $env = <IN>;
 598                 $env =~ s/\n//g;
 599                 my $cmdline = <IN>;
 600                 $cmdline =~ s/\n//g;
 601                 push (@todo, [$name, $env, $cmdline])
 602                         if (not defined($tests) or $name =~ /$tests/);
 603         } else {
 604                 print;
 605         }
 606 }
 607 close(IN) or die("Error creating recipe");
 608
 609 my $suitestotal = $#todo + 1;
 610 my $i = 0;
 611 $| = 1;
 612
 613 my %running_envs = ();
 614
 615 sub setup_env($)
 616 {
 617         my ($envname) = @_;
 618
 619         my $testenv_vars;
 620         if (defined($running_envs{$envname})) {
 621                 $testenv_vars = $running_envs{$envname};
 622         } elsif ($envname eq "none") {
 623                 $testenv_vars = {};
 624         } else {
 625                 $testenv_vars = $target->setup_env($envname, $prefix);
 626         }
 627         write_clientconf($conffile, $testenv_vars);
 628         foreach ("PASSWORD", "DOMAIN", "SERVER", "USERNAME", "NETBIOSNAME",
 629                          "KRB5_CONFIG", "REALM") {
 630                 if (defined($testenv_vars->{$_})) {
 631                         $ENV{$_} = $testenv_vars->{$_};
 632                 } else {
 633                         delete $ENV{$_};
 634                 }
 635         }
 636
 637         SocketWrapper::set_default_iface(6);
 638
 639         $running_envs{$envname} = $testenv_vars;
 640         return $testenv_vars;
 641 }
 642
 643 sub teardown_env($)
 644 {
 645         my ($envname) = @_;
 646         return if ($envname eq "none");
 647         $target->teardown_env($running_envs{$envname});
 648         delete $running_envs{$envname};
 649 }
 650
 651
 652 if ($opt_testenv) {
 653         my $testenv_vars = setup_env("dc");
 654         $ENV{PIDDIR} = $testenv_vars->{PIDDIR};
 655         my $term = ($ENV{TERM} or "xterm");
 656         system("$term -e 'echo -e \"Welcome to the Samba4 Test environment
 657 This matches the client environment used in make test
 658 smbd is pid `cat \$PIDDIR/smbd.pid`
 659
 660 Some useful environment variables:
 661 TORTURE_OPTIONS=\$TORTURE_OPTIONS
 662 CONFIGURATION=\$CONFIGURATION
 663 SERVER=\$SERVER
 664 NETBIOSNAME=\$NETBIOSNAME\" && bash'");
 665         teardown_env("dc");
 666 } else {
 667         foreach (@todo) {
 668                 $i++;
 669                 my $cmd = $$_[2];
 670                 $cmd =~ s/([\(\)])/\\$1/g;
 671                 my $name = $$_[0];
 672                 my $envname = $$_[1];
 673
 674                 if (skip($name)) {
 675                         print "SKIPPED: $name\n";
 676                         $statistics->{SUITES_SKIPPED}++;
 677                         next;
 678                 }
 679
 680                 setup_env($envname);
 681
 682                 my $shname = $name;
 683                 $shname =~ s%[^abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789\-]%_%g;
 684                 my $pcap_file = "$pcap_dir/$shname.cap";
 685
 686                 SocketWrapper::setup_pcap($pcap_file) if ($opt_socket_wrapper_pcap);
 687                 my $result;
 688                 if ($from_build_farm) {
 689                         $result = run_test($name, $cmd, $i, $suitestotal, $buildfarm_msg_ops);
 690                 } else {
 691                         $result = run_test($name, $cmd, $i, $suitestotal, $plain_msg_ops);
 692                 }
 693
 694                 if ($opt_socket_wrapper_pcap and $result and
 695                         not $opt_socket_wrapper_keep_pcap) {
 696                         unlink($pcap_file);
 697                 }
 698
 699                 if (defined($opt_analyse_cmd)) {
 700                         system("$opt_analyse_cmd \"$name\"");
 701                 }
 702
 703                 teardown_env($envname) if ($opt_resetup_env);
 704         }
 705 }
 706
 707 print "\n";
 708
 709 teardown_env($_) foreach (keys %running_envs);
 710
 711 $target->stop();
 712
 713 my $end = time();
 714 my $duration = ($end-$start);
 715 my $numfailed = $#$suitesfailed+1;
 716 if ($numfailed == 0) {
 717         my $ok = $statistics->{TESTS_EXPECTED_OK} + $statistics->{TESTS_EXPECTED_FAIL};
 718         print "ALL OK ($ok tests in $statistics->{SUITES_OK} testsuites)\n";
 719 } else {
 720         unless ($from_build_farm) {
 721                 if (not $opt_immediate and not $opt_verbose) {
 722                         foreach (@$suitesfailed) {
 723                                 print "===============================================================================\n";
 724                                 print "FAIL: $_\n";
 725                                 print $test_output->{$_};
 726                                 print "\n";
 727                         }
 728                 }
 729
 730                 print "FAILED ($statistics->{TESTS_UNEXPECTED_FAIL} failures and $statistics->{TESTS_ERROR} errors in $statistics->{SUITES_FAIL} testsuites)\n";
 731         }
 732 }
 733 print "DURATION: $duration seconds\n";
 734
 735 my $failed = 0;
 736
 737 # if there were any valgrind failures, show them
 738 foreach (<$prefix/valgrind.log*>) {
 739         next unless (-s $_);
 740         system("grep DWARF2.CFI.reader $_ > /dev/null");
 741         if ($? >> 8 == 0) {
 742             print "VALGRIND FAILURE\n";
 743             $failed++;
 744             system("cat $_");
 745         }
 746 }
 747
 748 if ($from_build_farm) {
 749         print "TEST STATUS: $numfailed\n";
 750 }
 751
 752 exit $numfailed;