release/src/router/mysql/mysql-test/mysql-stress-test.pl

   1 #!/usr/bin/perl
   2
   3 # Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
   4 #
   5 # This program is free software; you can redistribute it and/or
   6 # modify it under the terms of the GNU Library General Public
   7 # License as published by the Free Software Foundation; version 2
   8 # of the License.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13 # Library General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program; if not, write to the Free Software
  17 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  18
  19 # ======================================================================
  20 #                     MySQL server stress test system
  21 # ======================================================================
  22 #
  23 ##########################################################################
  24 #
  25 #                       SCENARIOS AND REQUIREMENTS
  26 #
  27 #   The system should perform stress testing of MySQL server with
  28 # following requirements and basic scenarios:
  29 #
  30 # Basic requirements:
  31 #
  32 # Design of stress script should allow one:
  33 #
  34 #   - to use for stress testing mysqltest binary as test engine
  35 #   - to use for stress testing both regular test suite and any
  36 #     additional test suites (e.g. mysql-test-extra-5.0)
  37 #   - to specify files with lists of tests both for initialization of
  38 #     stress db and for further testing itself
  39 #   - to define number of threads that will be concurrently used in testing
  40 #   - to define limitations for test run. e.g. number of tests or loops
  41 #     for execution or duration of testing, delay between test executions, etc.
  42 #   - to get readable log file which can be used for identification of
  43 #     errors arose during testing
  44 #
  45 # Basic scenarios:
  46 #
  47 #     * It should be possible to run stress script in standalone mode
  48 #       which will allow to create various scenarios of stress workloads:
  49 #
  50 #       simple ones:
  51 #
  52 #         box #1:
  53 #           - one instance of script with list of tests #1
  54 #
  55 #       and more advanced ones:
  56 #
  57 #         box #1:
  58 #           - one instance of script with list of tests #1
  59 #           - another instance of script with list of tests #2
  60 #         box #2:
  61 #           - one instance of script with list of tests #3
  62 #           - another instance of script with list of tests #4
  63 #             that will recreate whole database to back it to clean
  64 #             state
  65 #
  66 #       One kind of such complex scenarios maybe continued testing
  67 #       when we want to run stress tests from many boxes with various
  68 #       lists of tests that will last very long time. And in such case
  69 #       we need some wrapper for MySQL server that will restart it in
  70 #       case of crashes.
  71 #
  72 #     * It should be possible to run stress script in ad-hoc mode from
  73 #       shell or perl versions of mysql-test-run. This allows developers
  74 #       to reproduce and debug errors that was found in continued stress
  75 #       testing
  76 #
  77 # 2009-01-28 OBN Additions and modifications per WL#4685
  78 #
  79 ########################################################################
  80
  81 use Config;
  82
  83 if (!defined($Config{useithreads}))
  84 {
  85   die <<EOF;
  86 It is unable to run threaded version of stress test on this system
  87 due to disabled ithreads. Please check that installed perl binary
  88 was built with support of ithreads.
  89 EOF
  90 }
  91
  92 use threads;
  93 use threads::shared;
  94
  95 use IO::Socket;
  96 use Sys::Hostname;
  97 use File::Copy;
  98 use File::Spec;
  99 use File::Find;
 100 use File::Basename;
 101 use File::Path;
 102 use Cwd;
 103
 104 use Data::Dumper;
 105 use Getopt::Long;
 106
 107 my $stress_suite_version="1.0";
 108
 109 $|=1;
 110
 111 $opt_server_host="";
 112 $opt_server_logs_dir="";
 113 $opt_help="";
 114 $opt_server_port="";
 115 $opt_server_socket="";
 116 $opt_server_user="";
 117 $opt_server_password="";
 118 $opt_server_database="";
 119 $opt_cleanup="";
 120 $opt_verbose="";
 121 $opt_log_error_details="";
 122
 123
 124 $opt_suite="main";
 125 $opt_stress_suite_basedir="";
 126 $opt_stress_basedir="";
 127 $opt_stress_datadir="";
 128 $opt_test_suffix="";
 129
 130 $opt_stress_mode="random";
 131
 132 $opt_loop_count=0;
 133 $opt_test_count=0;
 134 $opt_test_duration=0;
 135 # OBN: Changing abort-on-error default to -1 (for WL-4626/4685): -1 means no abort
 136 $opt_abort_on_error=-1;
 137 $opt_sleep_time = 0;
 138 $opt_threads=1;
 139 $pid_file="mysql_stress_test.pid";
 140 $opt_mysqltest= ($^O =~ /mswin32/i) ? "mysqltest.exe" : "mysqltest";
 141 $opt_check_tests_file="";
 142 # OBM adding a setting for 'max-connect-retries=20' the default of 500 is to high
 143 @mysqltest_args=("--silent", "-v", "--skip-safemalloc", "--max-connect-retries=20");
 144
 145 # Client ip address
 146 $client_ip=inet_ntoa((gethostbyname(hostname()))[4]);
 147 $client_ip=~ s/\.//g;
 148
 149 %tests_files=(client => {mtime => 0, data => []},
 150               initdb => {mtime => 0, data => []});
 151
 152 # Error codes and sub-strings with corresponding severity
 153 #
 154 # S1 - Critical errors - cause immediately abort of testing. These errors
 155 #                        could be caused by server crash or impossibility
 156 #                        of test execution.
 157 #
 158 # S2 - Serious errors - these errors are bugs for sure as it knowns that
 159 #                       they shouldn't appear during stress testing
 160 #
 161 # S3 - Unknown errors - Errors were returned but we don't know what they are
 162 #                       so script can't determine if they are OK or not
 163 #
 164 # S4 - Non-seriuos errros - these errors could be caused by fact that
 165 #                           we execute simultaneously statements that
 166 #                           affect tests executed by other threads
 167
 168 %error_strings = ( 'Failed in mysql_real_connect()' => S1,
 169                    'Can\'t connect' => S1,
 170                    'not found (Errcode: 2)' => S1,
 171                    'does not exist' => S1,
 172                    'Could not open connection \'default\' after \d+ attempts' => S1,
 173                    'wrong errno ' => S3,
 174                    'Result length mismatch' => S4,
 175                    'Result content mismatch' => S4);
 176
 177 %error_codes = ( 1012 => S2, 1015 => S2, 1021 => S2,
 178                  1027 => S2, 1037 => S2, 1038 => S2,
 179                  1039 => S2, 1040 => S2, 1046 => S2,
 180                  1053 => S2, 1180 => S2, 1181 => S2,
 181                  1203 => S2, 1205 => S4, 1206 => S2,
 182                  1207 => S2, 1213 => S4, 1223 => S2,
 183                  2002 => S1, 2003 => S1, 2006 => S1,
 184                  2013 => S1
 185                  );
 186
 187 share(%test_counters);
 188 %test_counters=( loop_count => 0, test_count=>0);
 189
 190 share($exiting);
 191 $exiting=0;
 192
 193 # OBN Code and 'set_exit_code' function added by ES to set an exit code based on the error category returned
 194 #     in combination with the --abort-on-error value see WL#4685)
 195 use constant ABORT_MAKEWEIGHT => 20;
 196 share($gExitCode);
 197 $gExitCode = 0;   # global exit code
 198 sub set_exit_code {
 199         my $severity = shift;
 200         my $code = 0;
 201         if ( $severity =~ /^S(\d+)/ ) {
 202                 $severity = $1;
 203                 $code = 11 - $severity; # S1=10, S2=9, ... -- as per WL
 204         }
 205         else {
 206         # we know how we call the sub: severity should be S<num>; so, we should never be here...
 207                 print STDERR "Unknown severity format: $severity; setting to S1\n";
 208                 $severity = 1;
 209         }
 210         $abort = 0;
 211         if ( $severity <= $opt_abort_on_error ) {
 212                 # the test finished with a failure severe enough to abort. We are adding the 'abort flag' to the exit code
 213                 $code += ABORT_MAKEWEIGHT;
 214                 # but are not exiting just yet -- we need to update global exit code first
 215                 $abort = 1;
 216         }
 217         lock $gExitCode; # we can use lock here because the script uses threads anyway
 218         $gExitCode = $code if $code > $gExitCode;
 219         kill INT, $$ if $abort; # this is just a way to call sig_INT_handler: it will set exiting flag, which should do the rest
 220 }
 221
 222 share($test_counters_lock);
 223 $test_counters_lock=0;
 224 share($log_file_lock);
 225 $log_file_lock=0;
 226
 227 $SIG{INT}= \&sig_INT_handler;
 228 $SIG{TERM}= \&sig_TERM_handler;
 229
 230
 231 GetOptions("server-host=s", "server-logs-dir=s", "server-port=s",
 232            "server-socket=s", "server-user=s", "server-password=s",
 233            "server-database=s",
 234            "stress-suite-basedir=s", "suite=s", "stress-init-file:s",
 235            "stress-tests-file:s", "stress-basedir=s", "stress-mode=s",
 236            "stress-datadir=s",
 237            "threads=s", "sleep-time=s", "loop-count=i", "test-count=i",
 238            "test-duration=i", "test-suffix=s", "check-tests-file",
 239            "verbose", "log-error-details", "cleanup", "mysqltest=s",
 240            # OBN: (changing 'abort-on-error' to numberic for WL-4626/4685)
 241            "abort-on-error=i" => \$opt_abort_on_error, "help") || usage();
 242
 243 usage() if ($opt_help);
 244
 245 #$opt_abort_on_error=1;
 246
 247 $test_dirname=get_timestamp();
 248 $test_dirname.="-$opt_test_suffix" if ($opt_test_suffix ne '');
 249
 250 print <<EOF;
 251 #############################################################
 252                   CONFIGURATION STAGE
 253 #############################################################
 254 EOF
 255
 256 if ($opt_stress_basedir eq '' || $opt_stress_suite_basedir eq '' ||
 257     $opt_server_logs_dir eq '')
 258 {
 259   die <<EOF;
 260
 261 Options --stress-basedir, --stress-suite-basedir and --server-logs-dir are
 262 required. Please use these options to specify proper basedir for
 263 client, test suite and location of server logs.
 264
 265 stress-basedir: '$opt_stress_basedir'
 266 stress-suite-basedir: '$opt_stress_suite_basedir'
 267 server-logs-dir: '$opt_server_logs_dir'
 268
 269 EOF
 270 }
 271
 272 #Workaround for case when we got relative but not absolute path
 273 $opt_stress_basedir=File::Spec->rel2abs($opt_stress_basedir);
 274 $opt_stress_suite_basedir=File::Spec->rel2abs($opt_stress_suite_basedir);
 275 $opt_server_logs_dir=File::Spec->rel2abs($opt_server_logs_dir);
 276
 277 if ($opt_stress_datadir ne '')
 278 {
 279   $opt_stress_datadir=File::Spec->rel2abs($opt_stress_datadir);
 280 }
 281
 282 if (! -d "$opt_stress_basedir")
 283 {
 284   die <<EOF;
 285
 286 Directory '$opt_stress_basedir' does not exist.
 287 Use --stress-basedir option to specify proper basedir for client
 288
 289 EOF
 290 }
 291
 292 if (!-d $opt_stress_suite_basedir)
 293 {
 294   die <<EOF;
 295
 296 Directory '$opt_stress_suite_basedir' does not exist.
 297 Use --stress-suite-basedir option to specify proper basedir for test suite
 298
 299 EOF
 300 }
 301
 302 $test_dataset_dir=$opt_stress_suite_basedir;
 303 if ($opt_stress_datadir ne '')
 304 {
 305   if (-d $opt_stress_datadir)
 306   {
 307     $test_dataset_dir=$opt_stress_datadir;
 308
 309   }
 310   else
 311   {
 312     die <<EOF;
 313 Directory '$opt_stress_datadir' not exists. Please specify proper one
 314 with --stress-datadir option.
 315 EOF
 316   }
 317 }
 318
 319 if ($^O =~ /mswin32/i)
 320 {
 321   $test_dataset_dir=~ s/\\/\\\\/g;
 322 }
 323 else
 324 {
 325   $test_dataset_dir.="/";
 326 }
 327
 328
 329
 330 if (!-d $opt_server_logs_dir)
 331 {
 332   die <<EOF;
 333
 334 Directory server-logs-dir '$opt_server_logs_dir' does not exist.
 335 Use --server-logs-dir option to specify proper directory for storing
 336 logs
 337
 338 EOF
 339 }
 340 else
 341 {
 342   #Create sub-directory for test session logs
 343   mkpath(File::Spec->catdir($opt_server_logs_dir, $test_dirname), 0, 0755);
 344   #Define filename of global session log file
 345   $stress_log_file=File::Spec->catfile($opt_server_logs_dir, $test_dirname,
 346                                        "mysql-stress-test.log");
 347 }
 348
 349 if ($opt_suite ne '' && $opt_suite ne 'main' && $opt_suite ne 'default')
 350 {
 351   $test_suite_dir=File::Spec->catdir($opt_stress_suite_basedir, "suite", $opt_suite);
 352 }
 353 else
 354 {
 355   $test_suite_dir= $opt_stress_suite_basedir;
 356 }
 357
 358 if (!-d $test_suite_dir)
 359 {
 360   die <<EOF
 361
 362 Directory '$test_suite_dir' does not exist.
 363 Use --suite options to specify proper dir for test suite
 364
 365 EOF
 366 }
 367
 368 $test_suite_t_path=File::Spec->catdir($test_suite_dir,'t');
 369 $test_suite_r_path=File::Spec->catdir($test_suite_dir,'r');
 370
 371 foreach my $suite_dir ($test_suite_t_path, $test_suite_r_path)
 372 {
 373   if (!-d $suite_dir)
 374   {
 375     die <<EOF;
 376
 377 Directory '$suite_dir' does not exist.
 378 Please ensure that you specified proper source location for
 379 test/result files with --stress-suite-basedir option and name
 380 of test suite with --suite option
 381
 382 EOF
 383   }
 384 }
 385
 386 $test_t_path=File::Spec->catdir($opt_stress_basedir,'t');
 387 $test_r_path=File::Spec->catdir($opt_stress_basedir,'r');
 388
 389 foreach $test_dir ($test_t_path, $test_r_path)
 390 {
 391   if (-d $test_dir)
 392   {
 393     if ($opt_cleanup)
 394     {
 395       #Delete existing 't', 'r', 'r/*' subfolders in $stress_basedir
 396       rmtree("$test_dir", 0, 0);
 397       print "Cleanup $test_dir\n";
 398     }
 399     else
 400     {
 401       die <<EOF;
 402 Directory '$test_dir' already exist.
 403 Please ensure that you specified proper location of working dir
 404 for current test run with --stress-basedir option or in case of staled
 405 directories use --cleanup option to remove ones
 406 EOF
 407     }
 408   }
 409   #Create empty 't', 'r' subfolders that will be filled later
 410   mkpath("$test_dir", 0, 0777);
 411 }
 412
 413 if (!defined($opt_stress_tests_file) && !defined($opt_stress_init_file))
 414 {
 415   die <<EOF;
 416 You should run stress script either with --stress-tests-file or with
 417 --stress-init-file otions. See help for details.
 418 EOF
 419 }
 420
 421 if (defined($opt_stress_tests_file))
 422 {
 423   if ($opt_stress_tests_file eq '')
 424   {
 425     #Default location of file with set of tests for current test run
 426     $tests_files{client}->{filename}= File::Spec->catfile($opt_stress_suite_basedir,
 427                                       "testslist_client.txt");
 428   }
 429   else
 430   {
 431     $tests_files{client}->{filename}= $opt_stress_tests_file;
 432   }
 433
 434   if (!-f $tests_files{client}->{filename})
 435   {
 436     die <<EOF;
 437
 438 File '$tests_files{client}->{filename}' with list of tests not exists.
 439 Please ensure that this file exists, readable or specify another one with
 440 --stress-tests-file option.
 441
 442 EOF
 443   }
 444 }
 445
 446 if (defined($opt_stress_init_file))
 447 {
 448   if ($opt_stress_init_file eq '')
 449   {
 450     #Default location of file with set of tests for current test run
 451     $tests_files{initdb}->{filename}= File::Spec->catfile($opt_stress_suite_basedir,
 452                                       "testslist_initdb.txt");
 453   }
 454   else
 455   {
 456     $tests_files{initdb}->{filename}= $opt_stress_init_file;
 457   }
 458
 459   if (!-f $tests_files{initdb}->{filename})
 460   {
 461     die <<EOF;
 462
 463 File '$tests_files{initdb}->{filename}' with list of tests for initialization of database
 464 for stress test not exists.
 465 Please ensure that this file exists, readable or specify another one with
 466 --stress-init-file option.
 467
 468 EOF
 469   }
 470 }
 471
 472 if ($opt_stress_mode !~ /^(random|seq)$/)
 473 {
 474   die <<EOF
 475 Was specified wrong --stress-mode. Correct values 'random' and 'seq'.
 476 EOF
 477 }
 478
 479 if (open(TEST, "$opt_mysqltest -V |"))
 480 {
 481   $mysqltest_version=join("",<TEST>);
 482   close(TEST);
 483   print "FOUND MYSQLTEST BINARY: ", $mysqltest_version,"\n";
 484 }
 485 else
 486 {
 487   die <<EOF;
 488 ERROR: mysqltest binary $opt_mysqltest not found $!.
 489 You must either specify file location explicitly using --mysqltest
 490 option, or make sure path to mysqltest binary is listed
 491 in your PATH environment variable.
 492 EOF
 493 }
 494
 495 #
 496 #Adding mysql server specific command line options for mysqltest binary
 497 #
 498 $opt_server_host= $opt_server_host ? $opt_server_host : "localhost";
 499 $opt_server_port= $opt_server_port ? $opt_server_port : "3306";
 500 $opt_server_user= $opt_server_user ? $opt_server_user : "root";
 501 $opt_server_socket= $opt_server_socket ? $opt_server_socket : "/tmp/mysql.sock";
 502 $opt_server_database= $opt_server_database ? $opt_server_database : "test";
 503
 504 unshift @mysqltest_args, "--host=$opt_server_host";
 505 unshift @mysqltest_args, "--port=$opt_server_port";
 506 unshift @mysqltest_args, "--user=$opt_server_user";
 507 unshift @mysqltest_args, "--password=$opt_server_password";
 508 unshift @mysqltest_args, "--socket=$opt_server_socket";
 509 unshift @mysqltest_args, "--database=$opt_server_database";
 510
 511 #Export variables that could be used in tests
 512 $ENV{MYSQL_TEST_DIR}=$test_dataset_dir;
 513 $ENV{MASTER_MYPORT}=$opt_server_port;
 514 $ENV{MASTER_MYSOCK}=$opt_server_socket;
 515
 516 print <<EOF;
 517 TEST-SUITE-BASEDIR: $opt_stress_suite_basedir
 518 SUITE:              $opt_suite
 519 TEST-BASE-DIR:      $opt_stress_basedir
 520 TEST-DATADIR:       $test_dataset_dir
 521 SERVER-LOGS-DIR:    $opt_server_logs_dir
 522
 523 THREADS:            $opt_threads
 524 TEST-MODE:          $opt_stress_mode
 525
 526 EOF
 527
 528 #-------------------------------------------------------------------------------
 529 #At this stage we've already checked all needed pathes/files
 530 #and ready to start the test
 531 #-------------------------------------------------------------------------------
 532
 533 if (defined($opt_stress_tests_file) || defined($opt_stress_init_file))
 534 {
 535   print <<EOF;
 536 #############################################################
 537                   PREPARATION STAGE
 538 #############################################################
 539 EOF
 540
 541   #Copy Test files from network share to 't' folder
 542   print "\nCopying Test files from $test_suite_t_path to $test_t_path folder...";
 543   find({wanted=>\&copy_test_files, bydepth=>1}, "$test_suite_t_path");
 544   print "Done\n";
 545
 546   #$test_r_path/r0 dir reserved for initdb
 547   $count_start= defined($opt_stress_init_file) ? 0 : 1;
 548
 549   our $r_folder='';
 550   print "\nCreating 'r' folder and copying Protocol files to each 'r#' sub-folder...";
 551   for($count=$count_start; $count <= $opt_threads; $count++)
 552   {
 553     $r_folder = File::Spec->catdir($test_r_path, "r".$count);
 554     mkpath("$r_folder", 0, 0777);
 555
 556     find(\&copy_result_files,"$test_suite_r_path");
 557   }
 558   print "Done\n\n";
 559 }
 560
 561 if (defined($opt_stress_init_file))
 562 {
 563   print <<EOF;
 564 #############################################################
 565                   INITIALIZATION STAGE
 566 #############################################################
 567 EOF
 568
 569   #Set limits for stress db initialization
 570   %limits=(loop_count => 1, test_count => undef);
 571
 572   #Read list of tests from $opt_stress_init_file
 573   read_tests_names($tests_files{initdb});
 574   test_loop($client_ip, 0, 'seq', $tests_files{initdb});
 575   #print Dumper($tests_files{initdb}),"\n";
 576   print <<EOF;
 577
 578 Done initialization of stress database by tests from
 579 $tests_files{initdb}->{filename} file.
 580
 581 EOF
 582 }
 583
 584 if (defined($opt_stress_tests_file))
 585 {
 586   print <<EOF;
 587 #############################################################
 588                   STRESS TEST RUNNING STAGE
 589 #############################################################
 590 EOF
 591
 592   $exiting=0;
 593   #Read list of tests from $opt_stress_tests_file
 594   read_tests_names($tests_files{client});
 595
 596   #Reset current counter and set limits
 597   %test_counters=( loop_count => 0, test_count=>0);
 598   %limits=(loop_count => $opt_loop_count, test_count => $opt_test_count);
 599
 600   if (($opt_loop_count && $opt_threads > $opt_loop_count) ||
 601       ($opt_test_count && $opt_threads > $opt_test_count))
 602   {
 603     warn <<EOF;
 604
 605 WARNING: Possible inaccuracies in number of executed loops or
 606          tests because number of threads bigger than number of
 607          loops or tests:
 608
 609          Threads will be started: $opt_threads
 610          Loops will be executed:  $opt_loop_count
 611          Tests will be executed:  $opt_test_count
 612
 613 EOF
 614   }
 615
 616   #Create threads (number depending on the variable )
 617   for ($id=1; $id<=$opt_threads && !$exiting; $id++)
 618   {
 619     $thrd[$id] = threads->create("test_loop", $client_ip, $id,
 620                                  $opt_stress_mode, $tests_files{client});
 621
 622     print "main: Thread ID $id TID ",$thrd[$id]->tid," started\n";
 623     select(undef, undef, undef, 0.5);
 624   }
 625
 626   if ($opt_test_duration)
 627   {
 628   # OBN - At this point we need to wait for the duration of the test, hoever
 629   #       we need to be able to quit if an 'abort-on-error' condition has happend
 630   #       with one of the children (WL#4685). Using solution by ES and replacing
 631   #       the 'sleep' command with a loop checking the abort condition every second
 632
 633         foreach ( 1..$opt_test_duration ) {
 634                 last if $exiting;
 635                 sleep 1;
 636         }
 637     kill INT, $$;                             #Interrupt child threads
 638   }
 639
 640   #Let other threads to process INT signal
 641   sleep(1);
 642
 643   for ($id=1; $id<=$opt_threads;$id++)
 644   {
 645     if (defined($thrd[$id]))
 646     {
 647       $thrd[$id]->join();
 648     }
 649   }
 650   print "EXIT\n";
 651 }
 652
 653 exit $gExitCode; # ES WL#4685: script should return a meaningful exit code
 654
 655 sub test_init
 656 {
 657   my ($env)=@_;
 658
 659   $env->{session_id}=$env->{ip}."_".$env->{thread_id};
 660   $env->{r_folder}='r'.$env->{thread_id};
 661   $env->{screen_logs}=File::Spec->catdir($opt_server_logs_dir, $test_dirname,
 662                                          "screen_logs", $env->{session_id});
 663   $env->{reject_logs}=File::Spec->catdir($opt_server_logs_dir, $test_dirname,
 664                                          "reject_logs", $env->{session_id});
 665
 666   mkpath($env->{screen_logs}, 0, 0755) unless (-d $env->{screen_logs});
 667   mkpath($env->{reject_logs}, 0, 0755) unless (-d $env->{reject_logs});
 668
 669   $env->{session_log}= File::Spec->catfile($env->{screen_logs}, $env->{session_id}.".log");
 670 }
 671
 672 sub test_execute
 673 {
 674   my $env= shift;
 675   my $test_name= shift;
 676
 677   my $g_start= "";
 678   my $g_end= "";
 679   my $mysqltest_cmd= "";
 680   my @mysqltest_test_args=();
 681   my @stderr=();
 682
 683   #Get time stamp
 684   $g_start = get_timestamp();
 685   $env->{errors}={};
 686   @{$env->{test_status}}=();
 687
 688   my $test_file= $test_name.".test";
 689   my $result_file= $test_name.".result";
 690   my $reject_file = $test_name.'.reject';
 691   my $output_file = $env->{session_id}.'_'.$test_name.'_'.$g_start."_".$env->{test_count}.'.txt';
 692
 693   my $test_filename = File::Spec->catfile($test_t_path, $test_file);
 694   my $result_filename = File::Spec->catdir($test_r_path, $env->{r_folder}, $result_file);
 695   my $reject_filename = File::Spec->catdir($test_r_path, $env->{r_folder}, $reject_file);
 696   my $output_filename = File::Spec->catfile($env->{screen_logs}, $output_file);
 697
 698
 699   push @mysqltest_test_args, "--basedir=$opt_stress_suite_basedir/",
 700                              "--tmpdir=$opt_stress_basedir",
 701                              "-x $test_filename",
 702                              "-R $result_filename",
 703                              "2>$output_filename";
 704
 705   $cmd= "$opt_mysqltest --no-defaults ".join(" ", @mysqltest_args)." ".
 706                                         join(" ", @mysqltest_test_args);
 707
 708   system($cmd);
 709
 710   $exit_value  = $? >> 8;
 711   $signal_num  = $? & 127;
 712   $dumped_core = $? & 128;
 713
 714   my $tid= threads->self->tid;
 715
 716   if (-s $output_filename > 0)
 717   {
 718     #Read stderr for further analysis
 719     open (STDERR_LOG, $output_filename) or
 720                              warn "Can't open file $output_filename";
 721     @stderr=<STDERR_LOG>;
 722     close(STDERR_LOG);
 723
 724     if ($opt_verbose)
 725     {
 726       $session_debug_file="$opt_stress_basedir/error$tid.txt";
 727
 728       stress_log($session_debug_file,
 729                 "Something wrong happened during execution of this command line:");
 730       stress_log($session_debug_file, "MYSQLTEST CMD - $cmd");
 731       stress_log($session_debug_file, "STDERR:".join("",@stderr));
 732
 733       stress_log($session_debug_file, "EXIT STATUS:\n1. EXIT: $exit_value \n".
 734                                       "2. SIGNAL: $signal_num\n".
 735                                       "3. CORE: $dumped_core\n");
 736     }
 737   }
 738
 739   #If something wrong trying to analyse stderr
 740   if ($exit_value || $signal_num)
 741   {
 742     if (@stderr)
 743     {
 744       foreach my $line (@stderr)
 745       {
 746         #FIXME: we should handle case when for one sub-string/code
 747         #       we have several different error messages
 748         #       Now for both codes/substrings we assume that
 749         #       first found message will represent error
 750
 751         #Check line for error codes
 752         if (($err_msg, $err_code)= $line=~/failed: ((\d+):.+?$)/)
 753         {
 754           if (!exists($error_codes{$err_code}))
 755           {
 756             # OBN Changing severity level to S4 from S3 as S3 now reserved
 757             #     for the case where the error is unknown (for WL#4626/4685
 758             $severity="S4";
 759             $err_code=0;
 760           }
 761           else
 762           {
 763             $severity=$error_codes{$err_code};
 764           }
 765
 766           if (!exists($env->{errors}->{$severity}->{$err_code}))
 767           {
 768             $env->{errors}->{$severity}->{$err_code}=[0, $err_msg];
 769           }
 770           $env->{errors}->{$severity}->{$err_code}->[0]++;
 771           $env->{errors}->{$severity}->{total}++;
 772         }
 773
 774         #Check line for error patterns
 775         foreach $err_string (keys %error_strings)
 776         {
 777           $pattern= quotemeta $err_string;
 778           if ($line =~ /$pattern/i)
 779           {
 780             my $severity= $error_strings{$err_string};
 781             if (!exists($env->{errors}->{$severity}->{$err_string}))
 782             {
 783               $env->{errors}->{$severity}->{$err_string}=[0, $line];
 784             }
 785             $env->{errors}->{$severity}->{$err_string}->[0]++;
 786             $env->{errors}->{$severity}->{total}++;
 787           }
 788         }
 789       }
 790     }
 791     else
 792     {
 793       $env->{errors}->{S3}->{'Unknown error'}=
 794                               [1,"Unknown error. Nothing was output to STDERR"];
 795       $env->{errors}->{S3}->{total}=1;
 796     }
 797   }
 798
 799   #
 800   #FIXME: Here we can perform further analysis of recognized
 801   #       error codes
 802   #
 803
 804   foreach my $severity (sort {$a cmp $b} keys %{$env->{errors}})
 805   {
 806     my $total=$env->{errors}->{$severity}->{total};
 807     if ($total)
 808     {
 809       push @{$env->{test_status}}, "Severity $severity: $total";
 810       $env->{errors}->{total}=+$total;
 811       set_exit_code($severity);
 812     }
 813   }
 814
 815   #FIXME: Should we take into account $exit_value here?
 816   #       Now we assume that all stringified errors(i.e. errors without
 817   #       error codes) which are not exist in %error_string structure
 818   #       are OK
 819   if (!$env->{errors}->{total})
 820   {
 821     push @{$env->{test_status}},"No Errors. Test Passed OK";
 822   }
 823
 824   log_session_errors($env, $test_file);
 825
 826   #OBN Removing the case of S1 and abort-on-error as that is now set
 827   #     inside the set_exit_code function (for WL#4626/4685)
 828   #if (!$exiting && ($signal_num == 2 || $signal_num == 15 ||
 829   #       ($opt_abort_on_error && $env->{errors}->{S1} > 0)))
 830   if (!$exiting && ($signal_num == 2 || $signal_num == 15))
 831   {
 832     #mysqltest was interrupted with INT or TERM signals
 833     #so we assume that we should cancel testing and exit
 834     $exiting=1;
 835     # OBN - Adjusted text to exclude case of S1 and abort-on-error that
 836     #       was mentioned (for WL#4626/4685)
 837     print STDERR<<EOF;
 838 WARNING:
 839    mysqltest was interrupted with INT or TERM signals  so we assume that
 840    we should cancel testing and exit. Please check log file for this thread
 841    in $stress_log_file or
 842    inspect below output of the last test case executed with mysqltest to
 843    find out cause of error.
 844
 845    Output of mysqltest:
 846    @stderr
 847
 848 EOF
 849   }
 850
 851   if (-e $reject_filename)
 852   {
 853     move_to_logs($env->{reject_logs}, $reject_filename, $reject_file);
 854   }
 855
 856   if (-e $output_filename)
 857   {
 858     move_to_logs($env->{screen_logs}, $output_filename, $output_file);
 859   }
 860
 861 }
 862
 863 sub test_loop
 864 {
 865   my %client_env=();
 866   my $test_name="";
 867
 868   # KEY for session identification: IP-THREAD_ID
 869   $client_env{ip} = shift;
 870   $client_env{thread_id} = shift;
 871
 872   $client_env{mode} = shift;
 873   $client_env{tests_file}=shift;
 874
 875   $client_env{test_seq_idx}=0;
 876
 877   #Initialize session variables
 878   test_init(\%client_env);
 879
 880 LOOP:
 881
 882   while(!$exiting)
 883   {
 884     if ($opt_check_tests_file)
 885     {
 886       #Check if tests_file was modified and reread it in this case
 887       read_tests_names($client_env{tests_file}, 0);
 888     }
 889
 890     {
 891       lock($test_counters_lock);
 892
 893       if (($limits{loop_count} && $limits{loop_count} <= $test_counters{loop_count}*1) ||
 894           ($limits{test_count} && $limits{test_count} <= $test_counters{test_count}*1) )
 895       {
 896         $exiting=1;
 897         next LOOP;
 898       }
 899     }
 900
 901     #Get random file name
 902     if (($test_name = get_test(\%client_env)) ne '')
 903     {
 904       {
 905         lock($test_counters_lock);
 906
 907         #Save current counters values
 908         $client_env{loop_count}=$test_counters{loop_count};
 909         $client_env{test_count}=$test_counters{test_count};
 910       }
 911       #Run test and analyze results
 912       test_execute(\%client_env, $test_name);
 913
 914       print "test_loop[".$limits{loop_count}.":".
 915              $limits{test_count}." ".
 916              $client_env{loop_count}.":".
 917              $client_env{test_count}."]:".
 918              " TID ".$client_env{thread_id}.
 919              " test: '$test_name' ".
 920              " Errors: ".join(" ",@{$client_env{test_status}}).
 921                 ( $exiting ? " (thread aborting)" : "" )."\n";
 922     }
 923
 924     # OBN - At this point we need to wait until the 'wait' time between test
 925     #       executions passes (in case it is specifed) passes, hoever we need
 926     #       to be able to quit and break out of the test if an 'abort-on-error'
 927     #       condition has happend with one of the other children (WL#4685).
 928     #       Using solution by ES and replacing the 'sleep' command with a loop
 929     #       checking the abort condition every second
 930
 931         if ( $opt_sleep_time ) {
 932                 foreach ( 1..$opt_sleep_time ) {
 933                         last if $exiting;
 934                         sleep 1;
 935                 }
 936         }
 937   }
 938 }
 939
 940 sub move_to_logs ($$$)
 941 {
 942   my $path_to_logs = shift;
 943   my $src_file = shift;
 944   my $random_filename = shift;
 945
 946   my $dst_file = File::Spec->catfile($path_to_logs, $random_filename);
 947
 948   move ($src_file, $dst_file) or warn<<EOF;
 949 ERROR: move_to_logs: File $src_file cannot be moved to $dst_file: $!
 950 EOF
 951 }
 952
 953 sub copy_test_files ()
 954 {
 955   if (/\.test$/)
 956   {
 957     $src_file = $File::Find::name;
 958     #print "## $File::Find::topdir - $File::Find::dir - $src_file\n";
 959
 960     if ($File::Find::topdir eq $File::Find::dir && $src_file !~ /SCCS/)
 961     {
 962       $test_filename = basename($src_file);
 963       $dst_file = File::Spec->catfile($test_t_path, $test_filename);
 964
 965       copy($src_file, $dst_file) or die "ERROR: copy_test_files: File cannot be copied. $!";
 966     }
 967   }
 968 }
 969
 970 sub copy_result_files ()
 971 {
 972   if (/\.result$/)
 973   {
 974     $src_file = $File::Find::name;
 975
 976     if ($File::Find::topdir eq $File::Find::dir && $src_file !~ /SCCS/)
 977     {
 978       $result_filename = basename($src_file) ;
 979       $dst_file = File::Spec->catfile($r_folder, $result_filename);
 980
 981       copy($src_file, $dst_file) or die "ERROR: copy_result_files: File cannot be copied. $!";
 982     }
 983   }
 984 }
 985
 986 sub get_timestamp
 987 {
 988   my ($sec,$min,$hour,$mday,$mon,$year,$wday,$ydat,$isdst) = localtime();
 989
 990   return sprintf("%04d%02d%02d%02d%02d%02d", $year+1900, $mon+1, $mday, $hour, $min, $sec);
 991 }
 992
 993 sub read_tests_names
 994 {
 995   my $tests_file = shift;
 996   my $force_load = shift;
 997
 998   if ($force_load || ( (stat($tests_file->{filename}))[9] != $tests_file->{mtime}) )
 999   {
1000     open (TEST, $tests_file->{filename}) || die ("Could not open file <".
1001                                                   $tests_file->{filename}."> $!");
1002     @{$tests_file->{data}}= grep {!/^[#\r\n]|^$/} map { s/[\r\n]//g; $_ } <TEST>;
1003
1004     close (TEST);
1005     $tests_file->{mtime}=(stat(_))[9];
1006   }
1007 }
1008
1009 sub get_random_test
1010 {
1011   my $envt=shift;
1012   my $tests= $envt->{tests_file}->{data};
1013
1014   my $random = int(rand(@{$tests}));
1015   my $test = $tests->[$random];
1016
1017   return $test;
1018 }
1019
1020 sub get_next_test
1021 {
1022   my $envt=shift;
1023   my $test;
1024
1025   if (@{$envt->{tests_file}->{data}})
1026   {
1027     $test=${$envt->{tests_file}->{data}}[$envt->{test_seq_idx}];
1028     $envt->{test_seq_idx}++;
1029   }
1030
1031   #If we reach bound of array, reset seq index and increment loop counter
1032   if ($envt->{test_seq_idx} == scalar(@{$envt->{tests_file}->{data}}))
1033   {
1034     $envt->{test_seq_idx}=0;
1035     {
1036       lock($test_counters_lock);
1037       $test_counters{loop_count}++;
1038     }
1039   }
1040
1041   return $test;
1042 }
1043
1044 sub get_test
1045 {
1046    my $envt=shift;
1047
1048    {
1049      lock($test_counters_lock);
1050      $test_counters{test_count}++;
1051    }
1052
1053    if ($envt->{mode} eq 'seq')
1054    {
1055      return get_next_test($envt);
1056    }
1057    elsif ($envt->{mode} eq 'random')
1058    {
1059      return get_random_test($envt);
1060    }
1061 }
1062
1063 sub stress_log
1064 {
1065   my ($log_file, $line)=@_;
1066
1067   {
1068     open(SLOG,">>$log_file") or warn "Error during opening log file $log_file";
1069     print SLOG $line,"\n";
1070     close(SLOG);
1071   }
1072 }
1073
1074 sub log_session_errors
1075 {
1076   my ($env, $test_name) = @_;
1077   my $line='';
1078
1079   {
1080     lock ($log_file_lock);
1081
1082     #header in the begining of log file
1083     if (!-e $stress_log_file)
1084     {
1085       stress_log($stress_log_file,
1086                    "TestID TID      Suite         TestFileName Found Errors");
1087       stress_log($stress_log_file,
1088                    "=======================================================");
1089     }
1090
1091     $line=sprintf('%6d %3d %10s %20s %s', $env->{test_count}, threads->self->tid,
1092                                           $opt_suite, $test_name,
1093                                           join(",", @{$env->{test_status}}));
1094
1095     stress_log($stress_log_file, $line);
1096     #stress_log_with_lock($stress_log_file, "\n");
1097
1098     if ($opt_log_error_details)
1099     {
1100       foreach $severity (sort {$a cmp $b} keys %{$env->{errors}})
1101       {
1102         stress_log($stress_log_file, "");
1103         foreach $error (keys %{$env->{errors}->{$severity}})
1104         {
1105           if ($error ne 'total')
1106           {
1107             stress_log($stress_log_file, "$severity: Count:".
1108                       $env->{errors}->{$severity}->{$error}->[0].
1109                       " Error:". $env->{errors}->{$severity}->{$error}->[1]);
1110           }
1111         }
1112       }
1113     }
1114   }
1115 }
1116
1117 sub sig_INT_handler
1118 {
1119   $SIG{INT}= \&sig_INT_handler;
1120   $exiting=1;
1121   print STDERR "$$: Got INT signal-------------------------------------------\n";
1122
1123 }
1124
1125 sub sig_TERM_handler
1126 {
1127   $SIG{TERM}= \&sig_TERM_handler;
1128   $exiting=1;
1129   print STDERR "$$: Got TERM signal\n";
1130 }
1131
1132 sub usage
1133 {
1134   print <<EOF;
1135
1136 The MySQL Stress suite Ver $stress_suite_version
1137
1138 mysql-stress-test.pl --stress-basedir=<dir> --stress-suite-basedir=<dir> --server-logs-dir=<dir>
1139
1140 --server-host
1141 --server-port
1142 --server-socket
1143 --server-user
1144 --server-password
1145 --server-logs-dir
1146   Directory where all clients session logs will be stored. Usually
1147   this is shared directory associated with server that used
1148   in testing
1149
1150   Required option.
1151
1152 --stress-suite-basedir=<dir>
1153   Directory that has r/ t/ subfolders with test/result files
1154   which will be used for testing. Also by default we are looking
1155   in this directory for 'stress-tests.txt' file which contains
1156   list of tests.  It is possible to specify other location of this
1157   file with --stress-tests-file option.
1158
1159   Required option.
1160
1161 --stress-basedir=<dir>
1162   Working directory for this test run. This directory will be used
1163   as temporary location for results tracking during testing
1164
1165   Required option.
1166
1167 --stress-datadir=<dir>
1168   Location of data files used which will be used in testing.
1169   By default we search for these files in <dir>/data where dir
1170   is value of --stress-suite-basedir option.
1171
1172 --stress-init-file[=/path/to/file with tests for initialization of stress db]
1173   Using of this option allows to perform initialization of database
1174   by execution of test files. List of tests will be taken either from
1175   specified file or if it omited from default file 'stress-init.txt'
1176   located in <--stress-suite-basedir/--suite> dir
1177
1178 --stress-tests-file[=/path/to/file with tests]
1179   Using of this option allows to run stress test itself. Tests for testing
1180   will be taken either from specified file or if it omited from default
1181   file 'stress-tests.txt' located in <--stress-suite-basedir/--suite> dir
1182
1183 --stress-mode= [random|seq]
1184   There are two possible modes which affect order of selecting tests
1185   from the list:
1186     - in random mode tests will be selected in random order
1187     - in seq mode each thread will execute tests in the loop one by one as
1188       they specified in the list file.
1189
1190 --sleep-time=<time in seconds>
1191   Delay between test execution. Could be usefull in continued testsing
1192   when one of instance of stress script perform periodical cleanup or
1193   recreating of some database objects
1194
1195 --threads=#number of threads
1196   Define number of threads
1197
1198 --check-tests-file
1199   Check file with list of tests. If file was modified it will force to
1200   reread list of tests. Could be usefull in continued testing for
1201   adding/removing tests without script interruption
1202
1203 --mysqltest=/path/to/mysqltest binary
1204
1205 --verbose
1206
1207 --cleanup
1208   Force to clean up working directory (specified with --stress-basedir)
1209
1210 --abort-on-error=<number>
1211   Causes the script to abort if an error with severity <= number was encounterd
1212
1213 --log-error-details
1214   Enable errors details in the global error log file. (Default: off)
1215
1216 --test-count=<number of executed tests before we have to exit>
1217 --loop-count=<number of executed loops in sequential mode before we have to exit>
1218 --test-duration=<number of seconds that stress test should run>
1219
1220 Example of tool usage:
1221
1222 perl mysql-stress-test.pl \
1223 --stress-suite-basedir=/opt/qa/mysql-test-extra-5.0/mysql-test \
1224 --stress-basedir=/opt/qa/test \
1225 --server-logs-dir=/opt/qa/logs \
1226 --test-count=20  \
1227 --stress-tests-file=innodb-tests.txt \
1228 --stress-init-file=innodb-init.txt \
1229 --threads=5 \
1230 --suite=funcs_1  \
1231 --mysqltest=/opt/mysql/mysql-5.0/client/mysqltest \
1232 --server-user=root \
1233 --server-database=test \
1234 --cleanup \
1235
1236 EOF
1237 exit(0);
1238 }
1239
1240