release/src/router/mysql/storage/ndb/test/run-test/main.cpp

   1 /* Copyright (c) 2003-2005, 2007 MySQL AB
   2
   3    This program is free software; you can redistribute it and/or modify
   4    it under the terms of the GNU General Public License as published by
   5    the Free Software Foundation; version 2 of the License.
   6
   7    This program is distributed in the hope that it will be useful,
   8    but WITHOUT ANY WARRANTY; without even the implied warranty of
   9    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10    GNU General Public License for more details.
  11
  12    You should have received a copy of the GNU General Public License
  13    along with this program; if not, write to the Free Software
  14    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA */
  15
  16
  17 #include "atrt.hpp"
  18 #include <my_sys.h>
  19 #include <my_getopt.h>
  20
  21 #include <NdbOut.hpp>
  22 #include <NdbAutoPtr.hpp>
  23
  24 #include <SysLogHandler.hpp>
  25 #include <FileLogHandler.hpp>
  26
  27 #include <NdbSleep.h>
  28
  29 #define PATH_SEPARATOR "/"
  30
  31 /** Global variables */
  32 static const char progname[] = "ndb_atrt";
  33 static const char * g_gather_progname = "atrt-gather-result.sh";
  34 static const char * g_analyze_progname = "atrt-analyze-result.sh";
  35 static const char * g_clear_progname = "atrt-clear-result.sh";
  36 static const char * g_setup_progname = "atrt-setup.sh";
  37
  38 static const char * g_log_filename = 0;
  39 static const char * g_test_case_filename = 0;
  40 static const char * g_report_filename = 0;
  41
  42 static int g_do_setup = 0;
  43 static int g_do_deploy = 0;
  44 static int g_do_sshx = 0;
  45 static int g_do_start = 0;
  46 static int g_do_quit = 0;
  47
  48 static int g_help = 0;
  49 static int g_verbosity = 1;
  50 static FILE * g_report_file = 0;
  51 static FILE * g_test_case_file = stdin;
  52 static int g_mode = 0;
  53
  54 Logger g_logger;
  55 atrt_config g_config;
  56 const char * g_user = 0;
  57 int          g_baseport = 10000;
  58 int          g_fqpn = 0;
  59 int          g_default_ports = 0;
  60
  61 const char * g_cwd = 0;
  62 const char * g_basedir = 0;
  63 const char * g_my_cnf = 0;
  64 const char * g_prefix = 0;
  65 const char * g_clusters = 0;
  66 BaseString g_replicate;
  67 const char *save_file = 0;
  68 char *save_extra_file = 0;
  69 const char *save_group_suffix = 0;
  70 const char * g_dummy;
  71 char * g_env_path = 0;
  72
  73 /** Dummy, extern declared in ndb_opts.h */
  74 int g_print_full_config = 0, opt_ndb_shm;
  75 my_bool opt_core;
  76
  77 static struct my_option g_options[] =
  78 {
  79   { "help", '?', "Display this help and exit.",
  80     (uchar **) &g_help, (uchar **) &g_help,
  81     0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0 },
  82   { "version", 'V', "Output version information and exit.", 0, 0, 0,
  83     GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0 },
  84   { "clusters", 256, "Cluster",
  85     (uchar **) &g_clusters, (uchar **) &g_clusters,
  86     0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  87   { "replicate", 1024, "replicate",
  88     (uchar **) &g_dummy, (uchar **) &g_dummy,
  89     0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  90   { "log-file", 256, "log-file",
  91     (uchar **) &g_log_filename, (uchar **) &g_log_filename,
  92     0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  93   { "testcase-file", 'f', "testcase-file",
  94     (uchar **) &g_test_case_filename, (uchar **) &g_test_case_filename,
  95     0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  96   { "report-file", 'r', "report-file",
  97     (uchar **) &g_report_filename, (uchar **) &g_report_filename,
  98     0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
  99   { "basedir", 256, "Base path",
 100     (uchar **) &g_basedir, (uchar **) &g_basedir,
 101     0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
 102   { "baseport", 256, "Base port",
 103     (uchar **) &g_baseport, (uchar **) &g_baseport,
 104     0, GET_INT, REQUIRED_ARG, g_baseport, 0, 0, 0, 0, 0},
 105   { "prefix", 256, "mysql install dir",
 106     (uchar **) &g_prefix, (uchar **) &g_prefix,
 107     0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
 108   { "verbose", 'v', "Verbosity",
 109     (uchar **) &g_verbosity, (uchar **) &g_verbosity,
 110     0, GET_INT, REQUIRED_ARG, g_verbosity, 0, 0, 0, 0, 0},
 111   { "configure", 256, "configure",
 112     (uchar **) &g_do_setup, (uchar **) &g_do_setup,
 113     0, GET_INT, REQUIRED_ARG, g_do_setup, 0, 0, 0, 0, 0 },
 114   { "deploy", 256, "deploy",
 115     (uchar **) &g_do_deploy, (uchar **) &g_do_deploy,
 116     0, GET_INT, REQUIRED_ARG, g_do_deploy, 0, 0, 0, 0, 0 },
 117   { "sshx", 256, "sshx",
 118     (uchar **) &g_do_sshx, (uchar **) &g_do_sshx,
 119     0, GET_INT, REQUIRED_ARG, g_do_sshx, 0, 0, 0, 0, 0 },
 120   { "start", 256, "start",
 121     (uchar **) &g_do_start, (uchar **) &g_do_start,
 122     0, GET_INT, REQUIRED_ARG, g_do_start, 0, 0, 0, 0, 0 },
 123   { "fqpn", 256, "Fully qualified path-names ",
 124     (uchar **) &g_fqpn, (uchar **) &g_fqpn,
 125     0, GET_INT, REQUIRED_ARG, g_fqpn, 0, 0, 0, 0, 0 },
 126   { "default-ports", 256, "Use default ports when possible",
 127     (uchar **) &g_default_ports, (uchar **) &g_default_ports,
 128     0, GET_INT, REQUIRED_ARG, g_default_ports, 0, 0, 0, 0, 0 },
 129   { "mode", 256, "Mode 0=interactive 1=regression 2=bench",
 130     (uchar **) &g_mode, (uchar **) &g_mode,
 131     0, GET_INT, REQUIRED_ARG, g_mode, 0, 0, 0, 0, 0 },
 132   { "quit", 256, "Quit before starting tests",
 133     (uchar **) &g_mode, (uchar **) &g_do_quit,
 134     0, GET_BOOL, NO_ARG, g_do_quit, 0, 0, 0, 0, 0 },
 135   { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
 136 };
 137
 138 const int p_ndb     = atrt_process::AP_NDB_MGMD | atrt_process::AP_NDBD;
 139 const int p_servers = atrt_process::AP_MYSQLD;
 140 const int p_clients = atrt_process::AP_CLIENT | atrt_process::AP_NDB_API;
 141
 142 int
 143 main(int argc, char ** argv)
 144 {
 145   ndb_init();
 146
 147   bool restart = true;
 148   int lineno = 1;
 149   int test_no = 1;
 150   int return_code = 1;
 151
 152   g_logger.setCategory(progname);
 153   g_logger.enable(Logger::LL_ALL);
 154   g_logger.createConsoleHandler();
 155
 156   if(!parse_args(argc, argv))
 157     goto end;
 158
 159   g_logger.info("Starting...");
 160   g_config.m_generated = false;
 161   g_config.m_replication = g_replicate;
 162   if (!setup_config(g_config))
 163     goto end;
 164
 165   if (!configure(g_config, g_do_setup))
 166     goto end;
 167
 168   g_logger.info("Setting up directories");
 169   if (!setup_directories(g_config, g_do_setup))
 170     goto end;
 171
 172   if (g_do_setup)
 173   {
 174     g_logger.info("Setting up files");
 175     if (!setup_files(g_config, g_do_setup, g_do_sshx))
 176       goto end;
 177   }
 178
 179   if (g_do_deploy)
 180   {
 181     if (!deploy(g_config))
 182       goto end;
 183   }
 184
 185   if (g_do_quit)
 186   {
 187     return_code = 0;
 188     goto end;
 189   }
 190
 191   if(!setup_hosts(g_config))
 192     goto end;
 193
 194   if (g_do_sshx)
 195   {
 196     g_logger.info("Starting xterm-ssh");
 197     if (!sshx(g_config, g_do_sshx))
 198       goto end;
 199
 200     g_logger.info("Done...sleeping");
 201     while(true)
 202     {
 203       NdbSleep_SecSleep(1);
 204     }
 205     return_code = 0;
 206     goto end;
 207   }
 208
 209   g_logger.info("Connecting to hosts");
 210   if(!connect_hosts(g_config))
 211     goto end;
 212
 213   if (g_do_start && !g_test_case_filename)
 214   {
 215     g_logger.info("Starting server processes: %x", g_do_start);
 216     if (!start(g_config, g_do_start))
 217       goto end;
 218
 219     g_logger.info("Done...sleeping");
 220     while(true)
 221     {
 222       NdbSleep_SecSleep(1);
 223     }
 224     return_code = 0;
 225     goto end;
 226   }
 227
 228   return_code = 0;
 229
 230   /**
 231    * Main loop
 232    */
 233   while(!feof(g_test_case_file)){
 234     /**
 235      * Do we need to restart ndb
 236      */
 237     if(restart){
 238       g_logger.info("(Re)starting server processes processes");
 239       if(!stop_processes(g_config, ~0))
 240         goto end;
 241
 242       if (!setup_directories(g_config, 2))
 243         goto end;
 244
 245       if (!setup_files(g_config, 2, 1))
 246         goto end;
 247
 248       if(!setup_hosts(g_config))
 249         goto end;
 250
 251       if (!start(g_config, p_ndb | p_servers))
 252         goto end;
 253       g_logger.info("All servers start completed");
 254     }
 255
 256     // const int start_line = lineno;
 257     atrt_testcase test_case;
 258     if(!read_test_case(g_test_case_file, test_case, lineno))
 259       goto end;
 260
 261     g_logger.info("#%d - %s %s",
 262                   test_no,
 263                   test_case.m_command.c_str(), test_case.m_args.c_str());
 264
 265     // Assign processes to programs
 266     if(!setup_test_case(g_config, test_case))
 267       goto end;
 268
 269     if(!start_processes(g_config, p_clients))
 270       goto end;
 271
 272     int result = 0;
 273
 274     const time_t start = time(0);
 275     time_t now = start;
 276     do {
 277       if(!update_status(g_config, atrt_process::AP_ALL))
 278         goto end;
 279
 280       int count = 0;
 281
 282       if((count = is_running(g_config, p_ndb)) != 2){
 283         result = ERR_NDB_FAILED;
 284         break;
 285       }
 286
 287       if((count = is_running(g_config, p_servers)) != 2){
 288         result = ERR_SERVERS_FAILED;
 289         break;
 290       }
 291
 292       if((count = is_running(g_config, p_clients)) == 0){
 293         break;
 294       }
 295
 296       now = time(0);
 297       if(now  > (start + test_case.m_max_time)){
 298         result = ERR_MAX_TIME_ELAPSED;
 299         break;
 300       }
 301       NdbSleep_SecSleep(1);
 302     } while(true);
 303
 304     const time_t elapsed = time(0) - start;
 305
 306     if(!stop_processes(g_config, p_clients))
 307       goto end;
 308
 309     int tmp, *rp = result ? &tmp : &result;
 310     if(!gather_result(g_config, rp))
 311       goto end;
 312
 313     g_logger.info("#%d %s(%d)",
 314                   test_no,
 315                   (result == 0 ? "OK" : "FAILED"), result);
 316
 317     if(g_report_file != 0){
 318       fprintf(g_report_file, "%s ; %d ; %d ; %ld\n",
 319               test_case.m_name.c_str(), test_no, result, elapsed);
 320       fflush(g_report_file);
 321     }
 322
 323     if(g_mode == 0 && result){
 324       g_logger.info
 325         ("Encountered failed test in interactive mode - terminating");
 326       break;
 327     }
 328
 329     BaseString resdir;
 330     resdir.assfmt("result.%d", test_no);
 331     remove_dir(resdir.c_str(), true);
 332
 333     if(test_case.m_report || g_mode == 2 || (g_mode && result))
 334     {
 335       if(rename("result", resdir.c_str()) != 0)
 336       {
 337         g_logger.critical("Failed to rename %s as %s",
 338                           "result", resdir.c_str());
 339         goto end;
 340       }
 341     }
 342     else
 343     {
 344       remove_dir("result", true);
 345     }
 346
 347     if(result != 0){
 348       restart = true;
 349     } else {
 350       restart = false;
 351     }
 352     test_no++;
 353   }
 354
 355  end:
 356   if(g_report_file != 0){
 357     fclose(g_report_file);
 358     g_report_file = 0;
 359   }
 360
 361   if(g_test_case_file != 0 && g_test_case_file != stdin){
 362     fclose(g_test_case_file);
 363     g_test_case_file = 0;
 364   }
 365
 366   stop_processes(g_config, atrt_process::AP_ALL);
 367   return return_code;
 368 }
 369
 370 static
 371 my_bool
 372 get_one_option(int arg, const struct my_option * opt, char * value)
 373 {
 374   if (arg == 1024)
 375   {
 376     if (g_replicate.length())
 377       g_replicate.append(";");
 378     g_replicate.append(value);
 379     return 1;
 380   }
 381   return 0;
 382 }
 383
 384 bool
 385 parse_args(int argc, char** argv)
 386 {
 387   char buf[2048];
 388   if (getcwd(buf, sizeof(buf)) == 0)
 389   {
 390     g_logger.error("Unable to get current working directory");
 391     return false;
 392   }
 393   g_cwd = strdup(buf);
 394
 395   struct stat sbuf;
 396   BaseString mycnf;
 397   if (argc > 1 && lstat(argv[argc-1], &sbuf) == 0)
 398   {
 399     mycnf.append(g_cwd);
 400     mycnf.append(PATH_SEPARATOR);
 401     mycnf.append(argv[argc-1]);
 402   }
 403   else
 404   {
 405     mycnf.append(g_cwd);
 406     mycnf.append(PATH_SEPARATOR);
 407     mycnf.append("my.cnf");
 408     if (lstat(mycnf.c_str(), &sbuf) != 0)
 409     {
 410       g_logger.error("Unable to stat %s", mycnf.c_str());
 411       return false;
 412     }
 413   }
 414
 415   g_logger.info("Bootstrapping using %s", mycnf.c_str());
 416
 417   const char *groups[] = { "atrt", 0 };
 418   int ret = load_defaults(mycnf.c_str(), groups, &argc, &argv);
 419
 420   save_file = my_defaults_file;
 421   save_extra_file = my_defaults_extra_file;
 422   save_group_suffix = my_defaults_group_suffix;
 423
 424   if (save_extra_file)
 425   {
 426     g_logger.error("--defaults-extra-file(%s) is not supported...",
 427                    save_extra_file);
 428     return false;
 429   }
 430
 431   if (ret || handle_options(&argc, &argv, g_options, get_one_option))
 432   {
 433     g_logger.error("Failed to load defaults/handle_options");
 434     return false;
 435   }
 436
 437   if (argc >= 2)
 438   {
 439     const char * arg = argv[argc-2];
 440     while(* arg)
 441     {
 442       switch(* arg){
 443       case 'c':
 444         g_do_setup = (g_do_setup == 0) ? 1 : g_do_setup;
 445         break;
 446       case 'C':
 447         g_do_setup = 2;
 448         break;
 449       case 'd':
 450         g_do_deploy = 1;
 451         break;
 452       case 'x':
 453         g_do_sshx = atrt_process::AP_CLIENT | atrt_process::AP_NDB_API;
 454         break;
 455       case 'X':
 456         g_do_sshx = atrt_process::AP_ALL;
 457         break;
 458       case 's':
 459         g_do_start = p_ndb;
 460         break;
 461       case 'S':
 462         g_do_start = p_ndb | p_servers;
 463         break;
 464       case 'f':
 465         g_fqpn = 1;
 466         break;
 467       case 'q':
 468         g_do_quit = 1;
 469         break;
 470       default:
 471         g_logger.error("Unknown switch '%c'", *arg);
 472         return false;
 473       }
 474       arg++;
 475     }
 476   }
 477
 478   if(g_log_filename != 0)
 479   {
 480     g_logger.removeConsoleHandler();
 481     g_logger.addHandler(new FileLogHandler(g_log_filename));
 482   }
 483
 484   {
 485     int tmp = Logger::LL_WARNING - g_verbosity;
 486     tmp = (tmp < Logger::LL_DEBUG ? Logger::LL_DEBUG : tmp);
 487     g_logger.disable(Logger::LL_ALL);
 488     g_logger.enable(Logger::LL_ON);
 489     g_logger.enable((Logger::LoggerLevel)tmp, Logger::LL_ALERT);
 490   }
 491
 492   if(!g_basedir)
 493   {
 494     g_basedir = g_cwd;
 495     g_logger.info("basedir not specified, using %s", g_basedir);
 496   }
 497
 498   if (!g_prefix)
 499   {
 500     g_prefix = DEFAULT_PREFIX;
 501   }
 502
 503   /**
 504    * Add path to atrt-*.sh
 505    */
 506   {
 507     BaseString tmp;
 508     const char* env = getenv("PATH");
 509     if (env && strlen(env))
 510     {
 511       tmp.assfmt("PATH=%s:%s/mysql-test/ndb",
 512                  env, g_prefix);
 513     }
 514     else
 515     {
 516       tmp.assfmt("PATH=%s/mysql-test/ndb", g_prefix);
 517     }
 518     g_env_path = strdup(tmp.c_str());
 519     putenv(g_env_path);
 520   }
 521
 522   if (g_help)
 523   {
 524     my_print_help(g_options);
 525     my_print_variables(g_options);
 526     return 0;
 527   }
 528
 529   if(g_test_case_filename)
 530   {
 531     g_test_case_file = fopen(g_test_case_filename, "r");
 532     if(g_test_case_file == 0)
 533     {
 534       g_logger.critical("Unable to open file: %s", g_test_case_filename);
 535       return false;
 536     }
 537     if (g_do_setup == 0)
 538       g_do_setup = 2;
 539
 540     if (g_do_start == 0)
 541       g_do_start = p_ndb | p_servers;
 542
 543     if (g_mode == 0)
 544       g_mode = 1;
 545
 546     if (g_do_sshx)
 547     {
 548       g_logger.critical("ssx specified...not possible with testfile");
 549       return false;
 550     }
 551   }
 552
 553   if (g_do_setup == 0)
 554   {
 555     BaseString tmp;
 556     tmp.append(g_basedir);
 557     tmp.append(PATH_SEPARATOR);
 558     tmp.append("my.cnf");
 559     if (lstat(tmp.c_str(), &sbuf) != 0)
 560     {
 561       g_logger.error("Unable to stat %s", tmp.c_str());
 562       return false;
 563     }
 564
 565     if (!S_ISREG(sbuf.st_mode))
 566     {
 567       g_logger.error("%s is not a regular file", tmp.c_str());
 568       return false;
 569     }
 570
 571     g_my_cnf = strdup(tmp.c_str());
 572     g_logger.info("Using %s", tmp.c_str());
 573   }
 574   else
 575   {
 576     g_my_cnf = strdup(mycnf.c_str());
 577   }
 578
 579   g_logger.info("Using --prefix=\"%s\"", g_prefix);
 580
 581   if(g_report_filename)
 582   {
 583     g_report_file = fopen(g_report_filename, "w");
 584     if(g_report_file == 0)
 585     {
 586       g_logger.critical("Unable to create report file: %s", g_report_filename);
 587       return false;
 588     }
 589   }
 590
 591   if (g_clusters == 0)
 592   {
 593     g_logger.critical("No clusters specified");
 594     return false;
 595   }
 596
 597   g_user = strdup(getenv("LOGNAME"));
 598
 599   return true;
 600 }
 601
 602 bool
 603 connect_hosts(atrt_config& config){
 604   for(size_t i = 0; i<config.m_hosts.size(); i++){
 605     if(config.m_hosts[i]->m_cpcd->connect() != 0){
 606       g_logger.error("Unable to connect to cpc %s:%d",
 607                      config.m_hosts[i]->m_cpcd->getHost(),
 608                      config.m_hosts[i]->m_cpcd->getPort());
 609       return false;
 610     }
 611     g_logger.debug("Connected to %s:%d",
 612                    config.m_hosts[i]->m_cpcd->getHost(),
 613                    config.m_hosts[i]->m_cpcd->getPort());
 614   }
 615
 616   return true;
 617 }
 618
 619 bool
 620 connect_ndb_mgm(atrt_process & proc){
 621   NdbMgmHandle handle = ndb_mgm_create_handle();
 622   if(handle == 0){
 623     g_logger.critical("Unable to create mgm handle");
 624     return false;
 625   }
 626   BaseString tmp = proc.m_host->m_hostname;
 627   const char * val;
 628   proc.m_options.m_loaded.get("--PortNumber=", &val);
 629   tmp.appfmt(":%s", val);
 630
 631   if (ndb_mgm_set_connectstring(handle,tmp.c_str()))
 632   {
 633     g_logger.critical("Unable to create parse connectstring");
 634     return false;
 635   }
 636
 637   if(ndb_mgm_connect(handle, 30, 1, 0) != -1)
 638   {
 639     proc.m_ndb_mgm_handle = handle;
 640     return true;
 641   }
 642
 643   g_logger.critical("Unable to connect to ndb mgm %s", tmp.c_str());
 644   return false;
 645 }
 646
 647 bool
 648 connect_ndb_mgm(atrt_config& config){
 649   for(size_t i = 0; i<config.m_processes.size(); i++){
 650     atrt_process & proc = *config.m_processes[i];
 651     if((proc.m_type & atrt_process::AP_NDB_MGMD) != 0){
 652       if(!connect_ndb_mgm(proc)){
 653         return false;
 654       }
 655     }
 656   }
 657
 658   return true;
 659 }
 660
 661 static int remap(int i){
 662   if(i == NDB_MGM_NODE_STATUS_NO_CONTACT) return NDB_MGM_NODE_STATUS_UNKNOWN;
 663   if(i == NDB_MGM_NODE_STATUS_UNKNOWN) return NDB_MGM_NODE_STATUS_NO_CONTACT;
 664   return i;
 665 }
 666
 667 bool
 668 wait_ndb(atrt_config& config, int goal){
 669
 670   goal = remap(goal);
 671
 672   size_t cnt = 0;
 673   for (size_t i = 0; i<config.m_clusters.size(); i++)
 674   {
 675     atrt_cluster* cluster = config.m_clusters[i];
 676     /**
 677      * Get mgm handle for cluster
 678      */
 679     NdbMgmHandle handle = 0;
 680     for(size_t j = 0; j<cluster->m_processes.size(); j++){
 681       atrt_process & proc = *cluster->m_processes[j];
 682       if((proc.m_type & atrt_process::AP_NDB_MGMD) != 0){
 683         handle = proc.m_ndb_mgm_handle;
 684         break;
 685       }
 686     }
 687
 688     if(handle == 0){
 689       g_logger.critical("Unable to find mgm handle");
 690       return false;
 691     }
 692
 693     if(goal == NDB_MGM_NODE_STATUS_STARTED){
 694       /**
 695        * 1) wait NOT_STARTED
 696        * 2) send start
 697        * 3) wait STARTED
 698        */
 699       if(!wait_ndb(config, NDB_MGM_NODE_STATUS_NOT_STARTED))
 700         return false;
 701
 702       ndb_mgm_start(handle, 0, 0);
 703     }
 704
 705     struct ndb_mgm_cluster_state * state;
 706
 707     time_t now = time(0);
 708     time_t end = now + 360;
 709     int min = remap(NDB_MGM_NODE_STATUS_NO_CONTACT);
 710     int min2 = goal;
 711
 712     while(now < end){
 713       /**
 714        * 1) retreive current state
 715        */
 716       state = 0;
 717       do {
 718         state = ndb_mgm_get_status(handle);
 719         if(state == 0){
 720           const int err = ndb_mgm_get_latest_error(handle);
 721           g_logger.error("Unable to poll db state: %d %s %s",
 722                          ndb_mgm_get_latest_error(handle),
 723                          ndb_mgm_get_latest_error_msg(handle),
 724                          ndb_mgm_get_latest_error_desc(handle));
 725           if(err == NDB_MGM_SERVER_NOT_CONNECTED && connect_ndb_mgm(config)){
 726             g_logger.error("Reconnected...");
 727             continue;
 728           }
 729           return false;
 730         }
 731       } while(state == 0);
 732       NdbAutoPtr<void> tmp(state);
 733
 734       min2 = goal;
 735       for(int j = 0; j<state->no_of_nodes; j++){
 736         if(state->node_states[j].node_type == NDB_MGM_NODE_TYPE_NDB){
 737           const int s = remap(state->node_states[j].node_status);
 738           min2 = (min2 < s ? min2 : s );
 739
 740           if(s < remap(NDB_MGM_NODE_STATUS_NO_CONTACT) ||
 741              s > NDB_MGM_NODE_STATUS_STARTED){
 742             g_logger.critical("Strange DB status during start: %d %d",
 743                               j, min2);
 744             return false;
 745           }
 746
 747           if(min2 < min){
 748             g_logger.critical("wait ndb failed node: %d %d %d %d",
 749                               state->node_states[j].node_id, min, min2, goal);
 750           }
 751         }
 752       }
 753
 754       if(min2 < min){
 755         g_logger.critical("wait ndb failed %d %d %d", min, min2, goal);
 756         return false;
 757       }
 758
 759       if(min2 == goal){
 760         cnt++;
 761         goto next;
 762       }
 763
 764       min = min2;
 765       now = time(0);
 766     }
 767
 768     g_logger.critical("wait ndb timed out %d %d %d", min, min2, goal);
 769     break;
 770
 771 next:
 772     ;
 773   }
 774
 775   return cnt == config.m_clusters.size();
 776 }
 777
 778 bool
 779 start_process(atrt_process & proc){
 780   if(proc.m_proc.m_id != -1){
 781     g_logger.critical("starting already started process: %d", proc.m_index);
 782     return false;
 783   }
 784
 785   BaseString tmp = g_setup_progname;
 786   tmp.appfmt(" %s %s/ %s",
 787              proc.m_host->m_hostname.c_str(),
 788              proc.m_proc.m_cwd.c_str(),
 789              proc.m_proc.m_cwd.c_str());
 790
 791   g_logger.debug("system(%s)", tmp.c_str());
 792   const int r1 = system(tmp.c_str());
 793   if(r1 != 0){
 794     g_logger.critical("Failed to setup process");
 795     return false;
 796   }
 797
 798   {
 799     Properties reply;
 800     if(proc.m_host->m_cpcd->define_process(proc.m_proc, reply) != 0){
 801       BaseString msg;
 802       reply.get("errormessage", msg);
 803       g_logger.error("Unable to define process: %s", msg.c_str());
 804       return false;
 805     }
 806   }
 807   {
 808     Properties reply;
 809     if(proc.m_host->m_cpcd->start_process(proc.m_proc.m_id, reply) != 0){
 810       BaseString msg;
 811       reply.get("errormessage", msg);
 812       g_logger.error("Unable to start process: %s", msg.c_str());
 813       return false;
 814     }
 815   }
 816   return true;
 817 }
 818
 819 bool
 820 start_processes(atrt_config& config, int types){
 821   for(size_t i = 0; i<config.m_processes.size(); i++){
 822     atrt_process & proc = *config.m_processes[i];
 823     if((types & proc.m_type) != 0 && proc.m_proc.m_path != ""){
 824       if(!start_process(proc)){
 825         return false;
 826       }
 827     }
 828   }
 829   return true;
 830 }
 831
 832 bool
 833 stop_process(atrt_process & proc){
 834   if(proc.m_proc.m_id == -1){
 835     return true;
 836   }
 837
 838   {
 839     Properties reply;
 840     if(proc.m_host->m_cpcd->stop_process(proc.m_proc.m_id, reply) != 0){
 841       Uint32 status;
 842       reply.get("status", &status);
 843       if(status != 4){
 844         BaseString msg;
 845         reply.get("errormessage", msg);
 846         g_logger.error("Unable to stop process: %s(%d)", msg.c_str(), status);
 847         return false;
 848       }
 849     }
 850   }
 851   {
 852     Properties reply;
 853     if(proc.m_host->m_cpcd->undefine_process(proc.m_proc.m_id, reply) != 0){
 854       BaseString msg;
 855       reply.get("errormessage", msg);
 856       g_logger.error("Unable to undefine process: %s", msg.c_str());
 857       return false;
 858     }
 859     proc.m_proc.m_id = -1;
 860   }
 861   return true;
 862 }
 863
 864 bool
 865 stop_processes(atrt_config& config, int types){
 866   for(size_t i = 0; i<config.m_processes.size(); i++){
 867     atrt_process & proc = *config.m_processes[i];
 868     if((types & proc.m_type) != 0){
 869       if(!stop_process(proc)){
 870         return false;
 871       }
 872     }
 873   }
 874   return true;
 875 }
 876
 877 bool
 878 update_status(atrt_config& config, int){
 879
 880   Vector<Vector<SimpleCpcClient::Process> > m_procs;
 881
 882   Vector<SimpleCpcClient::Process> dummy;
 883   m_procs.fill(config.m_hosts.size(), dummy);
 884   for(size_t i = 0; i<config.m_hosts.size(); i++){
 885     Properties p;
 886     config.m_hosts[i]->m_cpcd->list_processes(m_procs[i], p);
 887   }
 888
 889   for(size_t i = 0; i<config.m_processes.size(); i++){
 890     atrt_process & proc = *config.m_processes[i];
 891     if(proc.m_proc.m_id != -1){
 892       Vector<SimpleCpcClient::Process> &h_procs= m_procs[proc.m_host->m_index];
 893       bool found = false;
 894       for(size_t j = 0; j<h_procs.size(); j++){
 895         if(proc.m_proc.m_id == h_procs[j].m_id){
 896           found = true;
 897           proc.m_proc.m_status = h_procs[j].m_status;
 898           break;
 899         }
 900       }
 901       if(!found){
 902         g_logger.error("update_status: not found");
 903         g_logger.error("id: %d host: %s cmd: %s",
 904                        proc.m_proc.m_id,
 905                        proc.m_host->m_hostname.c_str(),
 906                        proc.m_proc.m_path.c_str());
 907         for(size_t j = 0; j<h_procs.size(); j++){
 908           g_logger.error("found: %d %s", h_procs[j].m_id,
 909                          h_procs[j].m_path.c_str());
 910         }
 911         return false;
 912       }
 913     }
 914   }
 915   return true;
 916 }
 917
 918 int
 919 is_running(atrt_config& config, int types){
 920   int found = 0, running = 0;
 921   for(size_t i = 0; i<config.m_processes.size(); i++){
 922     atrt_process & proc = *config.m_processes[i];
 923     if((types & proc.m_type) != 0){
 924       found++;
 925       if(proc.m_proc.m_status == "running")
 926         running++;
 927     }
 928   }
 929
 930   if(found == running)
 931     return 2;
 932   if(running == 0)
 933     return 0;
 934   return 1;
 935 }
 936
 937
 938 int
 939 insert(const char * pair, Properties & p){
 940   BaseString tmp(pair);
 941
 942   tmp.trim(" \t\n\r");
 943
 944   Vector<BaseString> split;
 945   tmp.split(split, ":=", 2);
 946
 947   if(split.size() != 2)
 948     return -1;
 949
 950   p.put(split[0].trim().c_str(), split[1].trim().c_str());
 951
 952   return 0;
 953 }
 954
 955 bool
 956 read_test_case(FILE * file, atrt_testcase& tc, int& line){
 957
 958   Properties p;
 959   int elements = 0;
 960   char buf[1024];
 961   while(!feof(file)){
 962     if(!fgets(buf, 1024, file))
 963       break;
 964
 965     line++;
 966     BaseString tmp = buf;
 967
 968     if(tmp.length() > 0 && tmp.c_str()[0] == '#')
 969       continue;
 970
 971     if(insert(tmp.c_str(), p) != 0)
 972       break;
 973
 974     elements++;
 975   }
 976
 977   if(elements == 0){
 978     if(file == stdin){
 979       BaseString tmp(buf);
 980       tmp.trim(" \t\n\r");
 981       Vector<BaseString> split;
 982       tmp.split(split, " ", 2);
 983       tc.m_command = split[0];
 984       if(split.size() == 2)
 985         tc.m_args = split[1];
 986       else
 987         tc.m_args = "";
 988       tc.m_max_time = 60000;
 989       return true;
 990     }
 991     return false;
 992   }
 993
 994   if(!p.get("cmd", tc.m_command)){
 995     g_logger.critical("Invalid test file: cmd is missing near line: %d", line);
 996     return false;
 997   }
 998
 999   if(!p.get("args", tc.m_args))
1000     tc.m_args = "";
1001
1002   const char * mt = 0;
1003   if(!p.get("max-time", &mt))
1004     tc.m_max_time = 60000;
1005   else
1006     tc.m_max_time = atoi(mt);
1007
1008   if(p.get("type", &mt) && strcmp(mt, "bench") == 0)
1009     tc.m_report= true;
1010   else
1011     tc.m_report= false;
1012
1013   if(p.get("run-all", &mt) && strcmp(mt, "yes") == 0)
1014     tc.m_run_all= true;
1015   else
1016     tc.m_run_all= false;
1017
1018   if (!p.get("name", &mt))
1019   {
1020     tc.m_name.assfmt("%s %s",
1021                      tc.m_command.c_str(),
1022                      tc.m_args.c_str());
1023   }
1024   else
1025   {
1026     tc.m_name.assign(mt);
1027   }
1028
1029   return true;
1030 }
1031
1032 bool
1033 setup_test_case(atrt_config& config, const atrt_testcase& tc){
1034   g_logger.debug("system(%s)", g_clear_progname);
1035   const int r1 = system(g_clear_progname);
1036   if(r1 != 0){
1037     g_logger.critical("Failed to clear result");
1038     return false;
1039   }
1040
1041   size_t i = 0;
1042   for(; i<config.m_processes.size(); i++)
1043   {
1044     atrt_process & proc = *config.m_processes[i];
1045     if(proc.m_type == atrt_process::AP_NDB_API || proc.m_type == atrt_process::AP_CLIENT){
1046       proc.m_proc.m_path = "";
1047       if (tc.m_command.c_str()[0] != '/')
1048       {
1049         proc.m_proc.m_path.appfmt("%s/bin/", g_prefix);
1050       }
1051       proc.m_proc.m_path.append(tc.m_command.c_str());
1052       proc.m_proc.m_args.assign(tc.m_args);
1053       if(!tc.m_run_all)
1054         break;
1055     }
1056   }
1057   for(i++; i<config.m_processes.size(); i++){
1058     atrt_process & proc = *config.m_processes[i];
1059     if(proc.m_type == atrt_process::AP_NDB_API || proc.m_type == atrt_process::AP_CLIENT){
1060       proc.m_proc.m_path.assign("");
1061       proc.m_proc.m_args.assign("");
1062     }
1063   }
1064   return true;
1065 }
1066
1067 bool
1068 gather_result(atrt_config& config, int * result){
1069   BaseString tmp = g_gather_progname;
1070
1071   for(size_t i = 0; i<config.m_hosts.size(); i++)
1072   {
1073     tmp.appfmt(" %s:%s/*",
1074                config.m_hosts[i]->m_hostname.c_str(),
1075                config.m_hosts[i]->m_basedir.c_str());
1076   }
1077
1078   g_logger.debug("system(%s)", tmp.c_str());
1079   const int r1 = system(tmp.c_str());
1080   if(r1 != 0)
1081   {
1082     g_logger.critical("Failed to gather result!");
1083     return false;
1084   }
1085
1086   g_logger.debug("system(%s)", g_analyze_progname);
1087   const int r2 = system(g_analyze_progname);
1088
1089   if(r2 == -1 || r2 == (127 << 8))
1090   {
1091     g_logger.critical("Failed to analyze results");
1092     return false;
1093   }
1094
1095   * result = r2 ;
1096   return true;
1097 }
1098
1099 bool
1100 setup_hosts(atrt_config& config){
1101   g_logger.debug("system(%s)", g_clear_progname);
1102   const int r1 = system(g_clear_progname);
1103   if(r1 != 0){
1104     g_logger.critical("Failed to clear result");
1105     return false;
1106   }
1107
1108   for(size_t i = 0; i<config.m_hosts.size(); i++){
1109     BaseString tmp = g_setup_progname;
1110     tmp.appfmt(" %s %s/ %s/",
1111                config.m_hosts[i]->m_hostname.c_str(),
1112                g_basedir,
1113                config.m_hosts[i]->m_basedir.c_str());
1114
1115     g_logger.debug("system(%s)", tmp.c_str());
1116     const int r1 = system(tmp.c_str());
1117     if(r1 != 0){
1118       g_logger.critical("Failed to setup %s",
1119                         config.m_hosts[i]->m_hostname.c_str());
1120       return false;
1121     }
1122   }
1123   return true;
1124 }
1125
1126 bool
1127 deploy(atrt_config & config)
1128 {
1129   for (size_t i = 0; i<config.m_hosts.size(); i++)
1130   {
1131     BaseString tmp = g_setup_progname;
1132     tmp.appfmt(" %s %s/ %s",
1133                config.m_hosts[i]->m_hostname.c_str(),
1134                g_prefix,
1135                g_prefix);
1136
1137     g_logger.info("rsyncing %s to %s", g_prefix,
1138                   config.m_hosts[i]->m_hostname.c_str());
1139     g_logger.debug("system(%s)", tmp.c_str());
1140     const int r1 = system(tmp.c_str());
1141     if(r1 != 0)
1142     {
1143       g_logger.critical("Failed to rsync %s to %s",
1144                         g_prefix,
1145                         config.m_hosts[i]->m_hostname.c_str());
1146       return false;
1147     }
1148   }
1149
1150   return true;
1151 }
1152
1153 bool
1154 sshx(atrt_config & config, unsigned mask)
1155 {
1156   for (size_t i = 0; i<config.m_processes.size(); i++)
1157   {
1158     atrt_process & proc = *config.m_processes[i];
1159
1160     BaseString tmp;
1161     const char * type = 0;
1162     switch(proc.m_type){
1163     case atrt_process::AP_NDB_MGMD:
1164       type = (mask & proc.m_type) ? "ndb_mgmd" : 0;
1165       break;
1166     case atrt_process::AP_NDBD:
1167       type = (mask & proc.m_type) ? "ndbd" : 0;
1168       break;
1169     case atrt_process::AP_MYSQLD:
1170       type = (mask & proc.m_type) ? "mysqld" : 0;
1171       break;
1172     case atrt_process::AP_NDB_API:
1173       type = (mask & proc.m_type) ? "ndbapi" : 0;
1174       break;
1175     case atrt_process::AP_CLIENT:
1176       type = (mask & proc.m_type) ? "client" : 0;
1177       break;
1178     default:
1179       type = "<unknown>";
1180     }
1181
1182     if (type == 0)
1183       continue;
1184
1185     tmp.appfmt("xterm -fg black -title \"%s(%s) on %s\""
1186                " -e 'ssh -t -X %s sh %s/ssh-login.sh' &",
1187                type,
1188                proc.m_cluster->m_name.c_str(),
1189                proc.m_host->m_hostname.c_str(),
1190                proc.m_host->m_hostname.c_str(),
1191                proc.m_proc.m_cwd.c_str());
1192
1193     g_logger.debug("system(%s)", tmp.c_str());
1194     const int r1 = system(tmp.c_str());
1195     if(r1 != 0)
1196     {
1197       g_logger.critical("Failed sshx (%s)",
1198                         tmp.c_str());
1199       return false;
1200     }
1201     NdbSleep_MilliSleep(300); // To prevent xlock problem
1202   }
1203
1204   return true;
1205 }
1206
1207 bool
1208 start(atrt_config & config, unsigned proc_mask)
1209 {
1210   if (proc_mask & atrt_process::AP_NDB_MGMD)
1211     if(!start_processes(g_config, atrt_process::AP_NDB_MGMD))
1212       return false;
1213
1214   if (proc_mask & atrt_process::AP_NDBD)
1215   {
1216     if(!connect_ndb_mgm(g_config)){
1217       return false;
1218     }
1219
1220     if(!start_processes(g_config, atrt_process::AP_NDBD))
1221       return false;
1222
1223     if(!wait_ndb(g_config, NDB_MGM_NODE_STATUS_NOT_STARTED))
1224       return false;
1225
1226     for(Uint32 i = 0; i<3; i++)
1227       if(wait_ndb(g_config, NDB_MGM_NODE_STATUS_STARTED))
1228         goto started;
1229     return false;
1230   }
1231
1232 started:
1233   if(!start_processes(g_config, p_servers & proc_mask))
1234     return false;
1235
1236   return true;
1237 }
1238
1239 void
1240 require(bool x)
1241 {
1242   if (!x)
1243     abort();
1244 }
1245
1246 template class Vector<Vector<SimpleCpcClient::Process> >;
1247 template class Vector<atrt_host*>;
1248 template class Vector<atrt_cluster*>;
1249 template class Vector<atrt_process*>;