release/src/router/mysql/storage/ndb/test/ndbapi/testNodeRestart.cpp

   1 /* Copyright (c) 2003-2008 MySQL AB
   2
   3    This program is free software; you can redistribute it and/or modify
   4    it under the terms of the GNU General Public License as published by
   5    the Free Software Foundation; version 2 of the License.
   6
   7    This program is distributed in the hope that it will be useful,
   8    but WITHOUT ANY WARRANTY; without even the implied warranty of
   9    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10    GNU General Public License for more details.
  11
  12    You should have received a copy of the GNU General Public License
  13    along with this program; if not, write to the Free Software
  14    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA */
  15
  16 #include <NDBT.hpp>
  17 #include <NDBT_Test.hpp>
  18 #include <HugoTransactions.hpp>
  19 #include <UtilTransactions.hpp>
  20 #include <NdbRestarter.hpp>
  21 #include <NdbRestarts.hpp>
  22 #include <Vector.hpp>
  23 #include <signaldata/DumpStateOrd.hpp>
  24 #include <Bitmask.hpp>
  25 #include <RefConvert.hpp>
  26 #include <NdbEnv.h>
  27
  28 int runLoadTable(NDBT_Context* ctx, NDBT_Step* step){
  29
  30   int records = ctx->getNumRecords();
  31   HugoTransactions hugoTrans(*ctx->getTab());
  32   if (hugoTrans.loadTable(GETNDB(step), records) != 0){
  33     return NDBT_FAILED;
  34   }
  35   return NDBT_OK;
  36 }
  37
  38 int runFillTable(NDBT_Context* ctx, NDBT_Step* step){
  39
  40   HugoTransactions hugoTrans(*ctx->getTab());
  41   if (hugoTrans.fillTable(GETNDB(step)) != 0){
  42     return NDBT_FAILED;
  43   }
  44   return NDBT_OK;
  45 }
  46
  47 int runInsertUntilStopped(NDBT_Context* ctx, NDBT_Step* step){
  48   int result = NDBT_OK;
  49   int records = ctx->getNumRecords();
  50   int i = 0;
  51   HugoTransactions hugoTrans(*ctx->getTab());
  52   while (ctx->isTestStopped() == false) {
  53     g_info << i << ": ";
  54     if (hugoTrans.loadTable(GETNDB(step), records) != 0){
  55       return NDBT_FAILED;
  56     }
  57     i++;
  58   }
  59   return result;
  60 }
  61
  62 int runClearTable(NDBT_Context* ctx, NDBT_Step* step){
  63   int records = ctx->getNumRecords();
  64
  65   UtilTransactions utilTrans(*ctx->getTab());
  66   if (utilTrans.clearTable(GETNDB(step),  records) != 0){
  67     return NDBT_FAILED;
  68   }
  69   return NDBT_OK;
  70 }
  71
  72 int runClearTableUntilStopped(NDBT_Context* ctx, NDBT_Step* step){
  73   int records = ctx->getNumRecords();
  74   int i = 0;
  75
  76   UtilTransactions utilTrans(*ctx->getTab());
  77   while (ctx->isTestStopped() == false) {
  78     g_info << i << ": ";
  79     if (utilTrans.clearTable(GETNDB(step),  records) != 0){
  80       return NDBT_FAILED;
  81     }
  82     i++;
  83   }
  84   return NDBT_OK;
  85 }
  86
  87 int runScanReadUntilStopped(NDBT_Context* ctx, NDBT_Step* step){
  88   int result = NDBT_OK;
  89   int records = ctx->getNumRecords();
  90   int i = 0;
  91   HugoTransactions hugoTrans(*ctx->getTab());
  92   while (ctx->isTestStopped() == false) {
  93     g_info << i << ": ";
  94     if (hugoTrans.scanReadRecords(GETNDB(step), records) != 0){
  95       return NDBT_FAILED;
  96     }
  97     i++;
  98   }
  99   return result;
 100 }
 101
 102 int runPkReadUntilStopped(NDBT_Context* ctx, NDBT_Step* step){
 103   int result = NDBT_OK;
 104   int records = ctx->getNumRecords();
 105   NdbOperation::LockMode lm =
 106     (NdbOperation::LockMode)ctx->getProperty("ReadLockMode",
 107                                              (Uint32)NdbOperation::LM_Read);
 108   int i = 0;
 109   HugoTransactions hugoTrans(*ctx->getTab());
 110   while (ctx->isTestStopped() == false) {
 111     g_info << i << ": ";
 112     int rows = (rand()%records)+1;
 113     int batch = (rand()%rows)+1;
 114     if (hugoTrans.pkReadRecords(GETNDB(step), rows, batch, lm) != 0){
 115       return NDBT_FAILED;
 116     }
 117     i++;
 118   }
 119   return result;
 120 }
 121
 122 int runPkUpdateUntilStopped(NDBT_Context* ctx, NDBT_Step* step){
 123   int result = NDBT_OK;
 124   int records = ctx->getNumRecords();
 125   int multiop = ctx->getProperty("MULTI_OP", 1);
 126   Ndb* pNdb = GETNDB(step);
 127   int i = 0;
 128
 129   HugoOperations hugoOps(*ctx->getTab());
 130   while (ctx->isTestStopped() == false)
 131   {
 132     g_info << i << ": ";
 133     int batch = (rand()%records)+1;
 134     int row = rand() % records;
 135
 136     if (batch > 25)
 137       batch = 25;
 138
 139     if(row + batch > records)
 140       batch = records - row;
 141
 142     if(hugoOps.startTransaction(pNdb) != 0)
 143       goto err;
 144
 145     if(hugoOps.pkUpdateRecord(pNdb, row, batch, rand()) != 0)
 146       goto err;
 147
 148     for (int j = 1; j<multiop; j++)
 149     {
 150       if(hugoOps.execute_NoCommit(pNdb) != 0)
 151         goto err;
 152
 153       if(hugoOps.pkUpdateRecord(pNdb, row, batch, rand()) != 0)
 154         goto err;
 155     }
 156
 157     if(hugoOps.execute_Commit(pNdb) != 0)
 158       goto err;
 159
 160     hugoOps.closeTransaction(pNdb);
 161
 162     continue;
 163
 164 err:
 165     NdbConnection* pCon = hugoOps.getTransaction();
 166     if(pCon == 0)
 167       continue;
 168     NdbError error = pCon->getNdbError();
 169     hugoOps.closeTransaction(pNdb);
 170     if (error.status == NdbError::TemporaryError){
 171       NdbSleep_MilliSleep(50);
 172       continue;
 173     }
 174     return NDBT_FAILED;
 175
 176     i++;
 177   }
 178   return result;
 179 }
 180
 181 int runPkReadPkUpdateUntilStopped(NDBT_Context* ctx, NDBT_Step* step){
 182   int result = NDBT_OK;
 183   int records = ctx->getNumRecords();
 184   Ndb* pNdb = GETNDB(step);
 185   int i = 0;
 186   HugoOperations hugoOps(*ctx->getTab());
 187   while (ctx->isTestStopped() == false) {
 188     g_info << i++ << ": ";
 189     int rows = (rand()%records)+1;
 190     int batch = (rand()%rows)+1;
 191     int row = (records - rows) ? rand() % (records - rows) : 0;
 192
 193     int j,k;
 194     for(j = 0; j<rows; j += batch)
 195     {
 196       k = batch;
 197       if(j+k > rows)
 198         k = rows - j;
 199
 200       if(hugoOps.startTransaction(pNdb) != 0)
 201         goto err;
 202
 203       if(hugoOps.pkReadRecord(pNdb, row+j, k, NdbOperation::LM_Exclusive) != 0)
 204         goto err;
 205
 206       if(hugoOps.execute_NoCommit(pNdb) != 0)
 207         goto err;
 208
 209       if(hugoOps.pkUpdateRecord(pNdb, row+j, k, rand()) != 0)
 210         goto err;
 211
 212       if(hugoOps.execute_Commit(pNdb) != 0)
 213         goto err;
 214
 215       if(hugoOps.closeTransaction(pNdb) != 0)
 216         return NDBT_FAILED;
 217     }
 218
 219     continue;
 220 err:
 221     NdbConnection* pCon = hugoOps.getTransaction();
 222     if(pCon == 0)
 223       continue;
 224     NdbError error = pCon->getNdbError();
 225     hugoOps.closeTransaction(pNdb);
 226     if (error.status == NdbError::TemporaryError){
 227       NdbSleep_MilliSleep(50);
 228       continue;
 229     }
 230     return NDBT_FAILED;
 231   }
 232   return NDBT_OK;
 233 }
 234
 235 int runScanUpdateUntilStopped(NDBT_Context* ctx, NDBT_Step* step){
 236   int result = NDBT_OK;
 237   int records = ctx->getNumRecords();
 238   int parallelism = ctx->getProperty("Parallelism", 1);
 239   int abort = ctx->getProperty("AbortProb", (Uint32)0);
 240   int i = 0;
 241   HugoTransactions hugoTrans(*ctx->getTab());
 242   while (ctx->isTestStopped() == false) {
 243     g_info << i << ": ";
 244     if (hugoTrans.scanUpdateRecords(GETNDB(step), records, abort,
 245                                     parallelism) == NDBT_FAILED){
 246       return NDBT_FAILED;
 247     }
 248     i++;
 249   }
 250   return result;
 251 }
 252
 253 int runScanReadVerify(NDBT_Context* ctx, NDBT_Step* step){
 254   int records = ctx->getNumRecords();
 255   HugoTransactions hugoTrans(*ctx->getTab());
 256
 257   if (hugoTrans.scanReadRecords(GETNDB(step), records, 0, 64) != 0){
 258     return NDBT_FAILED;
 259   }
 260   return NDBT_OK;
 261 }
 262
 263 int runRestarter(NDBT_Context* ctx, NDBT_Step* step){
 264   int result = NDBT_OK;
 265   int loops = ctx->getNumLoops();
 266   int sync_threads = ctx->getProperty("SyncThreads", (unsigned)0);
 267   NdbRestarter restarter;
 268   int i = 0;
 269   int lastId = 0;
 270
 271   if (restarter.getNumDbNodes() < 2){
 272     ctx->stopTest();
 273     return NDBT_OK;
 274   }
 275
 276   if(restarter.waitClusterStarted() != 0){
 277     g_err << "Cluster failed to start" << endl;
 278     return NDBT_FAILED;
 279   }
 280
 281   loops *= restarter.getNumDbNodes();
 282   while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped()){
 283
 284     int id = lastId % restarter.getNumDbNodes();
 285     int nodeId = restarter.getDbNodeId(id);
 286     ndbout << "Restart node " << nodeId << endl;
 287     if(restarter.restartOneDbNode(nodeId, false, true, true) != 0){
 288       g_err << "Failed to restartNextDbNode" << endl;
 289       result = NDBT_FAILED;
 290       break;
 291     }
 292
 293     if (restarter.waitNodesNoStart(&nodeId, 1))
 294     {
 295       g_err << "Failed to waitNodesNoStart" << endl;
 296       result = NDBT_FAILED;
 297       break;
 298     }
 299
 300     if (restarter.startNodes(&nodeId, 1))
 301     {
 302       g_err << "Failed to start node" << endl;
 303       result = NDBT_FAILED;
 304       break;
 305     }
 306
 307     if(restarter.waitClusterStarted() != 0){
 308       g_err << "Cluster failed to start" << endl;
 309       result = NDBT_FAILED;
 310       break;
 311     }
 312
 313     ctx->sync_up_and_wait("PauseThreads", sync_threads);
 314
 315     lastId++;
 316     i++;
 317   }
 318
 319   ctx->stopTest();
 320
 321   return result;
 322 }
 323
 324 int runCheckAllNodesStarted(NDBT_Context* ctx, NDBT_Step* step){
 325   NdbRestarter restarter;
 326
 327   if(restarter.waitClusterStarted(1) != 0){
 328     g_err << "All nodes was not started " << endl;
 329     return NDBT_FAILED;
 330   }
 331
 332   return NDBT_OK;
 333 }
 334
 335
 336
 337 int runRestarts(NDBT_Context* ctx, NDBT_Step* step){
 338   int result = NDBT_OK;
 339   int loops = ctx->getNumLoops();
 340   NDBT_TestCase* pCase = ctx->getCase();
 341   NdbRestarts restarts;
 342   int i = 0;
 343   int timeout = 240;
 344
 345   while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped()){
 346
 347     if(restarts.executeRestart(pCase->getName(), timeout) != 0){
 348       g_err << "Failed to executeRestart(" <<pCase->getName() <<")" << endl;
 349       result = NDBT_FAILED;
 350       break;
 351     }
 352     i++;
 353   }
 354   ctx->stopTest();
 355   return result;
 356 }
 357
 358 int runDirtyRead(NDBT_Context* ctx, NDBT_Step* step){
 359   int result = NDBT_OK;
 360   int loops = ctx->getNumLoops();
 361   int records = ctx->getNumRecords();
 362   NdbRestarter restarter;
 363   HugoOperations hugoOps(*ctx->getTab());
 364   Ndb* pNdb = GETNDB(step);
 365
 366   int i = 0;
 367   while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped()){
 368     g_info << i << ": ";
 369
 370     int id = i % restarter.getNumDbNodes();
 371     int nodeId = restarter.getDbNodeId(id);
 372     ndbout << "Restart node " << nodeId << endl;
 373     restarter.insertErrorInNode(nodeId, 5041);
 374     restarter.insertErrorInAllNodes(8048 + (i & 1));
 375
 376     for(int j = 0; j<records; j++){
 377       if(hugoOps.startTransaction(pNdb) != 0)
 378         return NDBT_FAILED;
 379
 380       if(hugoOps.pkReadRecord(pNdb, j, 1, NdbOperation::LM_CommittedRead) != 0)
 381         goto err;
 382
 383       int res;
 384       if((res = hugoOps.execute_Commit(pNdb)) == 4119)
 385         goto done;
 386
 387       if(res != 0)
 388         goto err;
 389
 390       if(hugoOps.closeTransaction(pNdb) != 0)
 391         return NDBT_FAILED;
 392     }
 393 done:
 394     if(hugoOps.closeTransaction(pNdb) != 0)
 395       return NDBT_FAILED;
 396
 397     i++;
 398     restarter.waitClusterStarted(60) ;
 399   }
 400   return result;
 401 err:
 402   hugoOps.closeTransaction(pNdb);
 403   return NDBT_FAILED;
 404 }
 405
 406 int runLateCommit(NDBT_Context* ctx, NDBT_Step* step){
 407   int result = NDBT_OK;
 408   int loops = ctx->getNumLoops();
 409   int records = ctx->getNumRecords();
 410   NdbRestarter restarter;
 411   HugoOperations hugoOps(*ctx->getTab());
 412   Ndb* pNdb = GETNDB(step);
 413
 414   int i = 0;
 415   while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped()){
 416     g_info << i << ": ";
 417
 418     if(hugoOps.startTransaction(pNdb) != 0)
 419       return NDBT_FAILED;
 420
 421     if(hugoOps.pkUpdateRecord(pNdb, 1, 128) != 0)
 422       return NDBT_FAILED;
 423
 424     if(hugoOps.execute_NoCommit(pNdb) != 0)
 425       return NDBT_FAILED;
 426
 427     Uint32 transNode= hugoOps.getTransaction()->getConnectedNodeId();
 428     int id = i % restarter.getNumDbNodes();
 429     int nodeId;
 430     while((nodeId = restarter.getDbNodeId(id)) == transNode)
 431       id = (id + 1) % restarter.getNumDbNodes();
 432
 433     ndbout << "Restart node " << nodeId << endl;
 434
 435     restarter.restartOneDbNode(nodeId,
 436                              /** initial */ false,
 437                              /** nostart */ true,
 438                              /** abort   */ true);
 439
 440     restarter.waitNodesNoStart(&nodeId, 1);
 441
 442     int res;
 443     if(i & 1)
 444       res= hugoOps.execute_Commit(pNdb);
 445     else
 446       res= hugoOps.execute_Rollback(pNdb);
 447
 448     ndbout_c("res= %d", res);
 449
 450     hugoOps.closeTransaction(pNdb);
 451
 452     restarter.startNodes(&nodeId, 1);
 453     restarter.waitNodesStarted(&nodeId, 1);
 454
 455     if(i & 1)
 456     {
 457       if(res != 286)
 458         return NDBT_FAILED;
 459     }
 460     else
 461     {
 462       if(res != 0)
 463         return NDBT_FAILED;
 464     }
 465     i++;
 466   }
 467
 468   return NDBT_OK;
 469 }
 470
 471 int runBug15587(NDBT_Context* ctx, NDBT_Step* step){
 472   int result = NDBT_OK;
 473   int loops = ctx->getNumLoops();
 474   int records = ctx->getNumRecords();
 475   NdbRestarter restarter;
 476
 477   Uint32 tableId = ctx->getTab()->getTableId();
 478   int dump[2] = { DumpStateOrd::LqhErrorInsert5042, 0 };
 479   dump[1] = tableId;
 480
 481   int nodeId = restarter.getDbNodeId(1);
 482
 483   ndbout << "Restart node " << nodeId << endl;
 484
 485   if (restarter.restartOneDbNode(nodeId,
 486                                  /** initial */ false,
 487                                  /** nostart */ true,
 488                                  /** abort   */ true))
 489     return NDBT_FAILED;
 490
 491   if (restarter.waitNodesNoStart(&nodeId, 1))
 492     return NDBT_FAILED;
 493
 494   int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
 495
 496   if (restarter.dumpStateOneNode(nodeId, val2, 2))
 497     return NDBT_FAILED;
 498
 499   if (restarter.dumpStateOneNode(nodeId, dump, 2))
 500     return NDBT_FAILED;
 501
 502   if (restarter.startNodes(&nodeId, 1))
 503     return NDBT_FAILED;
 504
 505   restarter.waitNodesStartPhase(&nodeId, 1, 3);
 506
 507   if (restarter.waitNodesNoStart(&nodeId, 1))
 508     return NDBT_FAILED;
 509
 510   if (restarter.dumpStateOneNode(nodeId, val2, 1))
 511     return NDBT_FAILED;
 512
 513   if (restarter.startNodes(&nodeId, 1))
 514     return NDBT_FAILED;
 515
 516   if (restarter.waitNodesStarted(&nodeId, 1))
 517     return NDBT_FAILED;
 518
 519   ctx->stopTest();
 520   return NDBT_OK;
 521 }
 522
 523 int runBug15632(NDBT_Context* ctx, NDBT_Step* step){
 524   int result = NDBT_OK;
 525   int loops = ctx->getNumLoops();
 526   int records = ctx->getNumRecords();
 527   NdbRestarter restarter;
 528
 529   int nodeId = restarter.getDbNodeId(1);
 530
 531   ndbout << "Restart node " << nodeId << endl;
 532
 533   if (restarter.restartOneDbNode(nodeId,
 534                                  /** initial */ false,
 535                                  /** nostart */ true,
 536                                  /** abort   */ true))
 537     return NDBT_FAILED;
 538
 539   if (restarter.waitNodesNoStart(&nodeId, 1))
 540     return NDBT_FAILED;
 541
 542   if (restarter.insertErrorInNode(nodeId, 7165))
 543     return NDBT_FAILED;
 544
 545   if (restarter.startNodes(&nodeId, 1))
 546     return NDBT_FAILED;
 547
 548   if (restarter.waitNodesStarted(&nodeId, 1))
 549     return NDBT_FAILED;
 550
 551   if (restarter.restartOneDbNode(nodeId,
 552                                  /** initial */ false,
 553                                  /** nostart */ true,
 554                                  /** abort   */ true))
 555     return NDBT_FAILED;
 556
 557   if (restarter.waitNodesNoStart(&nodeId, 1))
 558     return NDBT_FAILED;
 559
 560   if (restarter.insertErrorInNode(nodeId, 7171))
 561     return NDBT_FAILED;
 562
 563   if (restarter.startNodes(&nodeId, 1))
 564     return NDBT_FAILED;
 565
 566   if (restarter.waitNodesStarted(&nodeId, 1))
 567     return NDBT_FAILED;
 568
 569   ctx->stopTest();
 570   return NDBT_OK;
 571 }
 572
 573 int runBug15685(NDBT_Context* ctx, NDBT_Step* step){
 574
 575   Ndb* pNdb = GETNDB(step);
 576   HugoOperations hugoOps(*ctx->getTab());
 577   NdbRestarter restarter;
 578
 579   HugoTransactions hugoTrans(*ctx->getTab());
 580   if (hugoTrans.loadTable(GETNDB(step), 10) != 0){
 581     return NDBT_FAILED;
 582   }
 583
 584   if(hugoOps.startTransaction(pNdb) != 0)
 585     goto err;
 586
 587   if(hugoOps.pkUpdateRecord(pNdb, 0, 1, rand()) != 0)
 588     goto err;
 589
 590   if(hugoOps.execute_NoCommit(pNdb) != 0)
 591     goto err;
 592
 593   if (restarter.insertErrorInAllNodes(5100))
 594     return NDBT_FAILED;
 595
 596   hugoOps.execute_Rollback(pNdb);
 597
 598   if (restarter.waitClusterStarted() != 0)
 599     goto err;
 600
 601   if (restarter.insertErrorInAllNodes(0))
 602     return NDBT_FAILED;
 603
 604   ctx->stopTest();
 605   return NDBT_OK;
 606
 607 err:
 608   ctx->stopTest();
 609   return NDBT_FAILED;
 610 }
 611
 612 int
 613 runBug16772(NDBT_Context* ctx, NDBT_Step* step){
 614
 615   NdbRestarter restarter;
 616   if (restarter.getNumDbNodes() < 2)
 617   {
 618     ctx->stopTest();
 619     return NDBT_OK;
 620   }
 621
 622   int aliveNodeId = restarter.getRandomNotMasterNodeId(rand());
 623   int deadNodeId = aliveNodeId;
 624   while (deadNodeId == aliveNodeId)
 625     deadNodeId = restarter.getDbNodeId(rand() % restarter.getNumDbNodes());
 626
 627   if (restarter.insertErrorInNode(aliveNodeId, 930))
 628     return NDBT_FAILED;
 629
 630   if (restarter.restartOneDbNode(deadNodeId,
 631                                  /** initial */ false,
 632                                  /** nostart */ true,
 633                                  /** abort   */ true))
 634     return NDBT_FAILED;
 635
 636   if (restarter.waitNodesNoStart(&deadNodeId, 1))
 637     return NDBT_FAILED;
 638
 639   if (restarter.startNodes(&deadNodeId, 1))
 640     return NDBT_FAILED;
 641
 642   // It should now be hanging since we throw away NDB_FAILCONF
 643   int ret = restarter.waitNodesStartPhase(&deadNodeId, 1, 3, 10);
 644   // So this should fail...i.e it should not reach startphase 3
 645
 646   // Now send a NDB_FAILCONF for deadNo
 647   int dump[] = { 7020, 323, 252, 0 };
 648   dump[3] = deadNodeId;
 649   if (restarter.dumpStateOneNode(aliveNodeId, dump, 4))
 650     return NDBT_FAILED;
 651
 652   if (restarter.waitNodesStarted(&deadNodeId, 1))
 653     return NDBT_FAILED;
 654
 655   return ret ? NDBT_OK : NDBT_FAILED;
 656 }
 657
 658 int
 659 runBug18414(NDBT_Context* ctx, NDBT_Step* step){
 660
 661   NdbRestarter restarter;
 662   if (restarter.getNumDbNodes() < 2)
 663   {
 664     ctx->stopTest();
 665     return NDBT_OK;
 666   }
 667
 668   Ndb* pNdb = GETNDB(step);
 669   HugoOperations hugoOps(*ctx->getTab());
 670   HugoTransactions hugoTrans(*ctx->getTab());
 671   int loop = 0;
 672   do
 673   {
 674     if(hugoOps.startTransaction(pNdb) != 0)
 675       goto err;
 676
 677     if(hugoOps.pkUpdateRecord(pNdb, 0, 128, rand()) != 0)
 678       goto err;
 679
 680     if(hugoOps.execute_NoCommit(pNdb) != 0)
 681       goto err;
 682
 683     int node1 = hugoOps.getTransaction()->getConnectedNodeId();
 684     int node2 = restarter.getRandomNodeSameNodeGroup(node1, rand());
 685
 686     if (node1 == -1 || node2 == -1)
 687       break;
 688
 689     if (loop & 1)
 690     {
 691       if (restarter.insertErrorInNode(node1, 8050))
 692         goto err;
 693     }
 694
 695     int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
 696     if (restarter.dumpStateOneNode(node2, val2, 2))
 697       goto err;
 698
 699     if (restarter.insertErrorInNode(node2, 5003))
 700       goto err;
 701
 702     int res= hugoOps.execute_Rollback(pNdb);
 703
 704     if (restarter.waitNodesNoStart(&node2, 1) != 0)
 705       goto err;
 706
 707     if (restarter.insertErrorInAllNodes(0))
 708       goto err;
 709
 710     if (restarter.startNodes(&node2, 1) != 0)
 711       goto err;
 712
 713     if (restarter.waitClusterStarted() != 0)
 714       goto err;
 715
 716     if (hugoTrans.scanUpdateRecords(pNdb, 128) != 0)
 717       goto err;
 718
 719     hugoOps.closeTransaction(pNdb);
 720
 721   } while(++loop < 5);
 722
 723   return NDBT_OK;
 724
 725 err:
 726   hugoOps.closeTransaction(pNdb);
 727   return NDBT_FAILED;
 728 }
 729
 730 int
 731 runBug18612(NDBT_Context* ctx, NDBT_Step* step){
 732
 733   // Assume two replicas
 734   NdbRestarter restarter;
 735   if (restarter.getNumDbNodes() < 2)
 736   {
 737     ctx->stopTest();
 738     return NDBT_OK;
 739   }
 740
 741   Uint32 cnt = restarter.getNumDbNodes();
 742
 743   for(int loop = 0; loop < ctx->getNumLoops(); loop++)
 744   {
 745     int partition0[256];
 746     int partition1[256];
 747     bzero(partition0, sizeof(partition0));
 748     bzero(partition1, sizeof(partition1));
 749     Bitmask<4> nodesmask;
 750
 751     Uint32 node1 = restarter.getDbNodeId(rand()%cnt);
 752     for (Uint32 i = 0; i<cnt/2; i++)
 753     {
 754       do {
 755         int tmp = restarter.getRandomNodeOtherNodeGroup(node1, rand());
 756         if (tmp == -1)
 757         {
 758           ctx->stopTest();
 759           return NDBT_OK;
 760         }
 761         node1 = tmp;
 762       } while(nodesmask.get(node1));
 763
 764       partition0[i] = node1;
 765       partition1[i] = restarter.getRandomNodeSameNodeGroup(node1, rand());
 766
 767       ndbout_c("nodes %d %d", node1, partition1[i]);
 768
 769       assert(!nodesmask.get(node1));
 770       assert(!nodesmask.get(partition1[i]));
 771       nodesmask.set(node1);
 772       nodesmask.set(partition1[i]);
 773     }
 774
 775     ndbout_c("done");
 776
 777     int dump[255];
 778     dump[0] = DumpStateOrd::NdbcntrStopNodes;
 779     memcpy(dump + 1, partition0, sizeof(int)*cnt/2);
 780
 781     Uint32 master = restarter.getMasterNodeId();
 782
 783     if (restarter.dumpStateOneNode(master, dump, 1+cnt/2))
 784       return NDBT_FAILED;
 785
 786     if (restarter.waitNodesNoStart(partition0, cnt/2))
 787       return NDBT_FAILED;
 788
 789     int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
 790
 791     if (restarter.dumpStateAllNodes(val2, 2))
 792       return NDBT_FAILED;
 793
 794     if (restarter.insertErrorInAllNodes(932))
 795       return NDBT_FAILED;
 796
 797     dump[0] = 9000;
 798     memcpy(dump + 1, partition0, sizeof(int)*cnt/2);
 799     for (Uint32 i = 0; i<cnt/2; i++)
 800       if (restarter.dumpStateOneNode(partition1[i], dump, 1+cnt/2))
 801         return NDBT_FAILED;
 802
 803     dump[0] = 9000;
 804     memcpy(dump + 1, partition1, sizeof(int)*cnt/2);
 805     for (Uint32 i = 0; i<cnt/2; i++)
 806       if (restarter.dumpStateOneNode(partition0[i], dump, 1+cnt/2))
 807         return NDBT_FAILED;
 808
 809     if (restarter.startNodes(partition0, cnt/2))
 810       return NDBT_FAILED;
 811
 812     if (restarter.waitNodesStartPhase(partition0, cnt/2, 2))
 813       return NDBT_FAILED;
 814
 815     dump[0] = 9001;
 816     for (Uint32 i = 0; i<cnt/2; i++)
 817       if (restarter.dumpStateAllNodes(dump, 2))
 818         return NDBT_FAILED;
 819
 820     if (restarter.waitNodesNoStart(partition0, cnt/2))
 821       return NDBT_FAILED;
 822
 823     for (Uint32 i = 0; i<cnt/2; i++)
 824       if (restarter.restartOneDbNode(partition0[i], true, true, true))
 825         return NDBT_FAILED;
 826
 827     if (restarter.waitNodesNoStart(partition0, cnt/2))
 828       return NDBT_FAILED;
 829
 830     if (restarter.startAll())
 831       return NDBT_FAILED;
 832
 833     if (restarter.waitClusterStarted())
 834       return NDBT_FAILED;
 835   }
 836   return NDBT_OK;
 837 }
 838
 839 int
 840 runBug18612SR(NDBT_Context* ctx, NDBT_Step* step){
 841
 842   // Assume two replicas
 843   NdbRestarter restarter;
 844   if (restarter.getNumDbNodes() < 2)
 845   {
 846     ctx->stopTest();
 847     return NDBT_OK;
 848   }
 849
 850   Uint32 cnt = restarter.getNumDbNodes();
 851
 852   for(int loop = 0; loop < ctx->getNumLoops(); loop++)
 853   {
 854     int partition0[256];
 855     int partition1[256];
 856     bzero(partition0, sizeof(partition0));
 857     bzero(partition1, sizeof(partition1));
 858     Bitmask<4> nodesmask;
 859
 860     Uint32 node1 = restarter.getDbNodeId(rand()%cnt);
 861     for (Uint32 i = 0; i<cnt/2; i++)
 862     {
 863       do {
 864         int tmp = restarter.getRandomNodeOtherNodeGroup(node1, rand());
 865         if (tmp == -1)
 866           break;
 867         node1 = tmp;
 868       } while(nodesmask.get(node1));
 869
 870       partition0[i] = node1;
 871       partition1[i] = restarter.getRandomNodeSameNodeGroup(node1, rand());
 872
 873       ndbout_c("nodes %d %d", node1, partition1[i]);
 874
 875       assert(!nodesmask.get(node1));
 876       assert(!nodesmask.get(partition1[i]));
 877       nodesmask.set(node1);
 878       nodesmask.set(partition1[i]);
 879     }
 880
 881     ndbout_c("done");
 882
 883     if (restarter.restartAll(false, true, false))
 884       return NDBT_FAILED;
 885
 886     int dump[255];
 887     dump[0] = 9000;
 888     memcpy(dump + 1, partition0, sizeof(int)*cnt/2);
 889     for (Uint32 i = 0; i<cnt/2; i++)
 890       if (restarter.dumpStateOneNode(partition1[i], dump, 1+cnt/2))
 891         return NDBT_FAILED;
 892
 893     dump[0] = 9000;
 894     memcpy(dump + 1, partition1, sizeof(int)*cnt/2);
 895     for (Uint32 i = 0; i<cnt/2; i++)
 896       if (restarter.dumpStateOneNode(partition0[i], dump, 1+cnt/2))
 897         return NDBT_FAILED;
 898
 899     int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
 900
 901     if (restarter.dumpStateAllNodes(val2, 2))
 902       return NDBT_FAILED;
 903
 904     if (restarter.insertErrorInAllNodes(932))
 905       return NDBT_FAILED;
 906
 907     if (restarter.startAll())
 908       return NDBT_FAILED;
 909
 910     if (restarter.waitClusterStartPhase(2))
 911       return NDBT_FAILED;
 912
 913     dump[0] = 9001;
 914     for (Uint32 i = 0; i<cnt/2; i++)
 915       if (restarter.dumpStateAllNodes(dump, 2))
 916         return NDBT_FAILED;
 917
 918     if (restarter.waitClusterNoStart(30))
 919       if (restarter.waitNodesNoStart(partition0, cnt/2, 10))
 920         if (restarter.waitNodesNoStart(partition1, cnt/2, 10))
 921           return NDBT_FAILED;
 922
 923     if (restarter.startAll())
 924       return NDBT_FAILED;
 925
 926     if (restarter.waitClusterStarted())
 927       return NDBT_FAILED;
 928   }
 929   return NDBT_OK;
 930 }
 931
 932 int runBug20185(NDBT_Context* ctx, NDBT_Step* step){
 933   int result = NDBT_OK;
 934   int loops = ctx->getNumLoops();
 935   int records = ctx->getNumRecords();
 936   NdbRestarter restarter;
 937   HugoOperations hugoOps(*ctx->getTab());
 938   Ndb* pNdb = GETNDB(step);
 939
 940   const int masterNode = restarter.getMasterNodeId();
 941
 942   int dump[] = { 7090, 20 } ;
 943   if (restarter.dumpStateAllNodes(dump, 2))
 944     return NDBT_FAILED;
 945
 946   NdbSleep_MilliSleep(3000);
 947   Vector<int> nodes;
 948   for (Uint32 i = 0; i<restarter.getNumDbNodes(); i++)
 949     nodes.push_back(restarter.getDbNodeId(i));
 950
 951 retry:
 952   if(hugoOps.startTransaction(pNdb) != 0)
 953     return NDBT_FAILED;
 954
 955   if(hugoOps.pkUpdateRecord(pNdb, 1, 1) != 0)
 956     return NDBT_FAILED;
 957
 958   if (hugoOps.execute_NoCommit(pNdb) != 0)
 959     return NDBT_FAILED;
 960
 961   const int node = hugoOps.getTransaction()->getConnectedNodeId();
 962   if (node != masterNode)
 963   {
 964     hugoOps.closeTransaction(pNdb);
 965     goto retry;
 966   }
 967
 968   int nodeId;
 969   do {
 970     nodeId = restarter.getDbNodeId(rand() % restarter.getNumDbNodes());
 971   } while (nodeId == node);
 972
 973   ndbout_c("7031 to %d", nodeId);
 974   if (restarter.insertErrorInNode(nodeId, 7031))
 975     return NDBT_FAILED;
 976
 977   for (Uint32 i = 0; i<nodes.size(); i++)
 978   {
 979     if (nodes[i] != nodeId)
 980       if (restarter.insertErrorInNode(nodes[i], 7030))
 981         return NDBT_FAILED;
 982   }
 983
 984   NdbSleep_MilliSleep(500);
 985
 986   if (hugoOps.execute_Commit(pNdb) == 0)
 987     return NDBT_FAILED;
 988
 989   NdbSleep_MilliSleep(3000);
 990
 991   restarter.waitClusterStarted();
 992
 993   if (restarter.dumpStateAllNodes(dump, 1))
 994     return NDBT_FAILED;
 995
 996   return NDBT_OK;
 997 }
 998
 999 int runBug24717(NDBT_Context* ctx, NDBT_Step* step){
1000   int result = NDBT_OK;
1001   int loops = ctx->getNumLoops();
1002   int records = ctx->getNumRecords();
1003   NdbRestarter restarter;
1004   Ndb* pNdb = GETNDB(step);
1005
1006   HugoTransactions hugoTrans(*ctx->getTab());
1007
1008   int dump[] = { 9002, 0 } ;
1009   Uint32 ownNode = refToNode(pNdb->getReference());
1010   dump[1] = ownNode;
1011
1012   for (; loops; loops --)
1013   {
1014     int nodeId = restarter.getRandomNotMasterNodeId(rand());
1015     restarter.restartOneDbNode(nodeId, false, true, true);
1016     restarter.waitNodesNoStart(&nodeId, 1);
1017
1018     if (restarter.dumpStateOneNode(nodeId, dump, 2))
1019       return NDBT_FAILED;
1020
1021     restarter.startNodes(&nodeId, 1);
1022
1023     do {
1024       for (Uint32 i = 0; i < 100; i++)
1025       {
1026         hugoTrans.pkReadRecords(pNdb, 100, 1, NdbOperation::LM_CommittedRead);
1027       }
1028     } while (restarter.waitClusterStarted(5) != 0);
1029   }
1030
1031   return NDBT_OK;
1032 }
1033
1034 int
1035 runBug29364(NDBT_Context* ctx, NDBT_Step* step){
1036   int result = NDBT_OK;
1037   int loops = ctx->getNumLoops();
1038   int records = ctx->getNumRecords();
1039   NdbRestarter restarter;
1040   Ndb* pNdb = GETNDB(step);
1041
1042   HugoTransactions hugoTrans(*ctx->getTab());
1043
1044   if (restarter.getNumDbNodes() < 4)
1045     return NDBT_OK;
1046
1047   int dump0[] = { 9000, 0 } ;
1048   int dump1[] = { 9001, 0 } ;
1049   Uint32 ownNode = refToNode(pNdb->getReference());
1050   dump0[1] = ownNode;
1051
1052   for (; loops; loops --)
1053   {
1054     int node0 = restarter.getDbNodeId(rand() % restarter.getNumDbNodes());
1055     int node1 = restarter.getRandomNodeOtherNodeGroup(node0, rand());
1056
1057     restarter.restartOneDbNode(node0, false, true, true);
1058     restarter.waitNodesNoStart(&node0, 1);
1059     restarter.startNodes(&node0, 1);
1060     restarter.waitClusterStarted();
1061
1062     restarter.restartOneDbNode(node1, false, true, true);
1063     restarter.waitNodesNoStart(&node1, 1);
1064     if (restarter.dumpStateOneNode(node1, dump0, 2))
1065       return NDBT_FAILED;
1066
1067     restarter.startNodes(&node1, 1);
1068
1069     do {
1070
1071       for (Uint32 i = 0; i < 100; i++)
1072       {
1073         hugoTrans.pkReadRecords(pNdb, 100, 1, NdbOperation::LM_CommittedRead);
1074       }
1075     } while (restarter.waitClusterStarted(5) != 0);
1076
1077     if (restarter.dumpStateOneNode(node1, dump1, 1))
1078       return NDBT_FAILED;
1079   }
1080
1081   return NDBT_OK;
1082 }
1083
1084 int runBug25364(NDBT_Context* ctx, NDBT_Step* step){
1085   int result = NDBT_OK;
1086   NdbRestarter restarter;
1087   Ndb* pNdb = GETNDB(step);
1088   int loops = ctx->getNumLoops();
1089
1090   if (restarter.getNumDbNodes() < 4)
1091     return NDBT_OK;
1092
1093   int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1094
1095   for (; loops; loops --)
1096   {
1097     int master = restarter.getMasterNodeId();
1098     int victim = restarter.getRandomNodeOtherNodeGroup(master, rand());
1099     int second = restarter.getRandomNodeSameNodeGroup(victim, rand());
1100
1101     int dump[] = { 935, victim } ;
1102     if (restarter.dumpStateOneNode(master, dump, 2))
1103       return NDBT_FAILED;
1104
1105     if (restarter.dumpStateOneNode(master, val2, 2))
1106       return NDBT_FAILED;
1107
1108     if (restarter.restartOneDbNode(second, false, true, true))
1109       return NDBT_FAILED;
1110
1111     int nodes[2] = { master, second };
1112     if (restarter.waitNodesNoStart(nodes, 2))
1113       return NDBT_FAILED;
1114
1115     restarter.startNodes(nodes, 2);
1116
1117     if (restarter.waitNodesStarted(nodes, 2))
1118       return NDBT_FAILED;
1119   }
1120
1121   return NDBT_OK;
1122 }
1123
1124 int
1125 runBug21271(NDBT_Context* ctx, NDBT_Step* step){
1126   int result = NDBT_OK;
1127   int loops = ctx->getNumLoops();
1128   int records = ctx->getNumRecords();
1129   NdbRestarter restarter;
1130   HugoOperations hugoOps(*ctx->getTab());
1131   Ndb* pNdb = GETNDB(step);
1132
1133   const int masterNode = restarter.getMasterNodeId();
1134   const int nodeId = restarter.getRandomNodeSameNodeGroup(masterNode, rand());
1135
1136   int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1137   if (restarter.dumpStateOneNode(nodeId, val2, 2))
1138     return NDBT_FAILED;
1139
1140   Uint32 tableId = ctx->getTab()->getTableId();
1141   int dump[] = { DumpStateOrd::LqhErrorInsert5042, 0, 5044 };
1142   dump[1] = tableId;
1143
1144   if (restarter.dumpStateOneNode(nodeId, dump, 3))
1145     return NDBT_FAILED;
1146
1147   restarter.waitNodesNoStart(&nodeId, 1);
1148   ctx->stopTest();
1149
1150   restarter.startNodes(&nodeId, 1);
1151
1152   if (restarter.waitClusterStarted() != 0)
1153     return NDBT_FAILED;
1154
1155   return NDBT_OK;
1156   return NDBT_OK;
1157 }
1158
1159 int
1160 runBug24543(NDBT_Context* ctx, NDBT_Step* step){
1161   NdbRestarter restarter;
1162
1163   int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1164   if (restarter.dumpStateAllNodes(val2, 2))
1165     return NDBT_FAILED;
1166
1167   int nodes[2];
1168   nodes[0] = restarter.getMasterNodeId();
1169   restarter.insertErrorInNode(nodes[0], 934);
1170
1171   nodes[1] = restarter.getRandomNodeOtherNodeGroup(nodes[0], rand());
1172   if (nodes[1] == -1)
1173   {
1174     nodes[1] = restarter.getRandomNodeSameNodeGroup(nodes[0], rand());
1175   }
1176
1177   restarter.restartOneDbNode(nodes[1], false, true, true);
1178   if (restarter.waitNodesNoStart(nodes, 2))
1179     return NDBT_FAILED;
1180
1181   restarter.startNodes(nodes, 2);
1182   if (restarter.waitNodesStarted(nodes, 2))
1183   {
1184     return NDBT_FAILED;
1185   }
1186   return NDBT_OK;
1187 }
1188
1189 int runBug25468(NDBT_Context* ctx, NDBT_Step* step){
1190
1191   int result = NDBT_OK;
1192   int loops = ctx->getNumLoops();
1193   int records = ctx->getNumRecords();
1194   NdbRestarter restarter;
1195
1196   for (int i = 0; i<loops; i++)
1197   {
1198     int master = restarter.getMasterNodeId();
1199     int node1, node2;
1200     switch(i % 5){
1201     case 0:
1202       node1 = master;
1203       node2 = restarter.getRandomNodeSameNodeGroup(master, rand());
1204       break;
1205     case 1:
1206       node1 = restarter.getRandomNodeSameNodeGroup(master, rand());
1207       node2 = master;
1208       break;
1209     case 2:
1210     case 3:
1211     case 4:
1212       node1 = restarter.getRandomNodeOtherNodeGroup(master, rand());
1213       if (node1 == -1)
1214         node1 = master;
1215       node2 = restarter.getRandomNodeSameNodeGroup(node1, rand());
1216       break;
1217     }
1218
1219     ndbout_c("node1: %d node2: %d master: %d", node1, node2, master);
1220
1221     int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1222
1223     if (restarter.dumpStateOneNode(node2, val2, 2))
1224       return NDBT_FAILED;
1225
1226     if (restarter.insertErrorInNode(node1, 7178))
1227       return NDBT_FAILED;
1228
1229     int val1 = 7099;
1230     if (restarter.dumpStateOneNode(master, &val1, 1))
1231       return NDBT_FAILED;
1232
1233     if (restarter.waitNodesNoStart(&node2, 1))
1234       return NDBT_FAILED;
1235
1236     if (restarter.startAll())
1237       return NDBT_FAILED;
1238
1239     if (restarter.waitClusterStarted())
1240       return NDBT_FAILED;
1241   }
1242
1243   return NDBT_OK;
1244 }
1245
1246 int runBug25554(NDBT_Context* ctx, NDBT_Step* step){
1247
1248   int result = NDBT_OK;
1249   int loops = ctx->getNumLoops();
1250   int records = ctx->getNumRecords();
1251   NdbRestarter restarter;
1252
1253   if (restarter.getNumDbNodes() < 4)
1254     return NDBT_OK;
1255
1256   for (int i = 0; i<loops; i++)
1257   {
1258     int master = restarter.getMasterNodeId();
1259     int node1 = restarter.getRandomNodeOtherNodeGroup(master, rand());
1260     restarter.restartOneDbNode(node1, false, true, true);
1261
1262     int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1263
1264     if (restarter.dumpStateOneNode(master, val2, 2))
1265       return NDBT_FAILED;
1266
1267     if (restarter.insertErrorInNode(master, 7141))
1268       return NDBT_FAILED;
1269
1270     if (restarter.waitNodesNoStart(&node1, 1))
1271       return NDBT_FAILED;
1272
1273     if (restarter.dumpStateOneNode(node1, val2, 2))
1274       return NDBT_FAILED;
1275
1276     if (restarter.insertErrorInNode(node1, 932))
1277       return NDBT_FAILED;
1278
1279     if (restarter.startNodes(&node1, 1))
1280       return NDBT_FAILED;
1281
1282     int nodes[] = { master, node1 };
1283     if (restarter.waitNodesNoStart(nodes, 2))
1284       return NDBT_FAILED;
1285
1286     if (restarter.startNodes(nodes, 2))
1287       return NDBT_FAILED;
1288
1289     if (restarter.waitClusterStarted())
1290       return NDBT_FAILED;
1291   }
1292
1293   return NDBT_OK;
1294 }
1295
1296 int runBug25984(NDBT_Context* ctx, NDBT_Step* step){
1297
1298   int result = NDBT_OK;
1299   int loops = ctx->getNumLoops();
1300   int records = ctx->getNumRecords();
1301   NdbRestarter restarter;
1302
1303   if (restarter.getNumDbNodes() < 2)
1304     return NDBT_OK;
1305
1306   if (restarter.restartAll(true, true, true))
1307     return NDBT_FAILED;
1308
1309   if (restarter.waitClusterNoStart())
1310     return NDBT_FAILED;
1311
1312   if (restarter.startAll())
1313     return NDBT_FAILED;
1314
1315   if (restarter.waitClusterStarted())
1316     return NDBT_FAILED;
1317
1318   int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1319   int master = restarter.getMasterNodeId();
1320   int victim = restarter.getRandomNodeOtherNodeGroup(master, rand());
1321   if (victim == -1)
1322     victim = restarter.getRandomNodeSameNodeGroup(master, rand());
1323
1324   restarter.restartOneDbNode(victim, false, true, true);
1325
1326   for (Uint32 i = 0; i<6; i++)
1327   {
1328     ndbout_c("Loop: %d", i);
1329     if (restarter.waitNodesNoStart(&victim, 1))
1330       return NDBT_FAILED;
1331
1332     if (restarter.dumpStateOneNode(victim, val2, 2))
1333       return NDBT_FAILED;
1334
1335     if (restarter.insertErrorInNode(victim, 7016))
1336       return NDBT_FAILED;
1337
1338     if (restarter.startNodes(&victim, 1))
1339       return NDBT_FAILED;
1340
1341     NdbSleep_SecSleep(3);
1342   }
1343
1344   if (restarter.waitNodesNoStart(&victim, 1))
1345     return NDBT_FAILED;
1346
1347   if (restarter.dumpStateOneNode(victim, val2, 2))
1348     return NDBT_FAILED;
1349
1350   if (restarter.insertErrorInNode(victim, 7170))
1351     return NDBT_FAILED;
1352
1353   if (restarter.startNodes(&victim, 1))
1354     return NDBT_FAILED;
1355
1356   if (restarter.waitNodesNoStart(&victim, 1))
1357     return NDBT_FAILED;
1358
1359   if (restarter.restartAll(false, true, true))
1360     return NDBT_FAILED;
1361
1362   if (restarter.insertErrorInAllNodes(932))
1363     return NDBT_FAILED;
1364
1365   if (restarter.insertErrorInNode(master, 7170))
1366     return NDBT_FAILED;
1367
1368   if (restarter.dumpStateAllNodes(val2, 2))
1369     return NDBT_FAILED;
1370
1371   restarter.startNodes(&master, 1);
1372   NdbSleep_MilliSleep(3000);
1373   restarter.startAll();
1374
1375   if (restarter.waitClusterNoStart())
1376     return NDBT_FAILED;
1377
1378   if (restarter.restartOneDbNode(victim, true, true, true))
1379     return NDBT_FAILED;
1380
1381   if (restarter.startAll())
1382     return NDBT_FAILED;
1383
1384   if (restarter.waitClusterStarted())
1385     return NDBT_FAILED;
1386
1387   return NDBT_OK;
1388 }
1389
1390 int
1391 runBug26457(NDBT_Context* ctx, NDBT_Step* step)
1392 {
1393   NdbRestarter res;
1394   if (res.getNumDbNodes() < 4)
1395     return NDBT_OK;
1396
1397   int loops = ctx->getNumLoops();
1398   while (loops --)
1399   {
1400 retry:
1401     int master = res.getMasterNodeId();
1402     int next = res.getNextMasterNodeId(master);
1403
1404     ndbout_c("master: %d next: %d", master, next);
1405
1406     if (res.getNodeGroup(master) == res.getNodeGroup(next))
1407     {
1408       res.restartOneDbNode(next, false, false, true);
1409       if (res.waitClusterStarted())
1410         return NDBT_FAILED;
1411       goto retry;
1412     }
1413
1414     int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 2 };
1415
1416     if (res.dumpStateOneNode(next, val2, 2))
1417       return NDBT_FAILED;
1418
1419     if (res.insertErrorInNode(next, 7180))
1420       return NDBT_FAILED;
1421
1422     res.restartOneDbNode(master, false, false, true);
1423     if (res.waitClusterStarted())
1424       return NDBT_FAILED;
1425   }
1426
1427   return NDBT_OK;
1428 }
1429
1430 int
1431 runBug26481(NDBT_Context* ctx, NDBT_Step* step)
1432 {
1433
1434   int result = NDBT_OK;
1435   int loops = ctx->getNumLoops();
1436   int records = ctx->getNumRecords();
1437   NdbRestarter res;
1438
1439   int node = res.getRandomNotMasterNodeId(rand());
1440   ndbout_c("node: %d", node);
1441   if (res.restartOneDbNode(node, true, true, true))
1442     return NDBT_FAILED;
1443
1444   if (res.waitNodesNoStart(&node, 1))
1445     return NDBT_FAILED;
1446
1447   int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1448   if (res.dumpStateOneNode(node, val2, 2))
1449     return NDBT_FAILED;
1450
1451   if (res.insertErrorInNode(node, 7018))
1452     return NDBT_FAILED;
1453
1454   if (res.startNodes(&node, 1))
1455     return NDBT_FAILED;
1456
1457   res.waitNodesStartPhase(&node, 1, 3);
1458
1459   if (res.waitNodesNoStart(&node, 1))
1460     return NDBT_FAILED;
1461
1462   res.startNodes(&node, 1);
1463
1464   if (res.waitClusterStarted())
1465     return NDBT_FAILED;
1466
1467   return NDBT_OK;
1468 }
1469
1470 int
1471 runBug26450(NDBT_Context* ctx, NDBT_Step* step)
1472 {
1473   Uint32 i;
1474   int result = NDBT_OK;
1475   int loops = ctx->getNumLoops();
1476   int records = ctx->getNumRecords();
1477   NdbRestarter res;
1478   Ndb* pNdb = GETNDB(step);
1479
1480   int node = res.getRandomNotMasterNodeId(rand());
1481   Vector<int> nodes;
1482   for (unsigned i = 0; i<res.getNumDbNodes(); i++)
1483   {
1484     if (res.getDbNodeId(i) != node)
1485       nodes.push_back(res.getDbNodeId(i));
1486   }
1487
1488   if (res.restartAll())
1489     return NDBT_FAILED;
1490
1491   if (res.waitClusterStarted())
1492     return NDBT_FAILED;
1493
1494   ndbout_c("node: %d", node);
1495   if (res.restartOneDbNode(node, false, true, true))
1496     return NDBT_FAILED;
1497
1498   if (res.waitNodesNoStart(&node, 1))
1499     return NDBT_FAILED;
1500
1501   if (runClearTable(ctx, step))
1502     return NDBT_FAILED;
1503
1504   for (i = 0; i < 2; i++)
1505   {
1506     if (res.restartAll(false, true, i > 0))
1507       return NDBT_FAILED;
1508
1509     if (res.waitClusterNoStart())
1510       return NDBT_FAILED;
1511
1512     if (res.startNodes(nodes.getBase(), nodes.size()))
1513       return NDBT_FAILED;
1514
1515     if (res.waitNodesStarted(nodes.getBase(), nodes.size()))
1516       return NDBT_FAILED;
1517   }
1518
1519   if (res.startNodes(&node, 1))
1520     return NDBT_FAILED;
1521
1522   if (res.waitNodesStarted(&node, 1))
1523     return NDBT_FAILED;
1524
1525   HugoTransactions trans (* ctx->getTab());
1526   if (trans.selectCount(pNdb) != 0)
1527     return NDBT_FAILED;
1528
1529   return NDBT_OK;
1530 }
1531
1532 int
1533 runBug27003(NDBT_Context* ctx, NDBT_Step* step)
1534 {
1535   int result = NDBT_OK;
1536   int loops = ctx->getNumLoops();
1537   int records = ctx->getNumRecords();
1538   NdbRestarter res;
1539
1540   static const int errnos[] = { 4025, 4026, 4027, 4028, 0 };
1541
1542   int node = res.getRandomNotMasterNodeId(rand());
1543   ndbout_c("node: %d", node);
1544   if (res.restartOneDbNode(node, true, true, true))
1545     return NDBT_FAILED;
1546
1547   Uint32 pos = 0;
1548   for (Uint32 i = 0; i<loops; i++)
1549   {
1550     while (errnos[pos] != 0)
1551     {
1552       ndbout_c("Tesing err: %d", errnos[pos]);
1553
1554       if (res.waitNodesNoStart(&node, 1))
1555         return NDBT_FAILED;
1556
1557       if (res.insertErrorInNode(node, 1000))
1558         return NDBT_FAILED;
1559
1560       if (res.insertErrorInNode(node, errnos[pos]))
1561         return NDBT_FAILED;
1562
1563       int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 3 };
1564       if (res.dumpStateOneNode(node, val2, 2))
1565         return NDBT_FAILED;
1566
1567       res.startNodes(&node, 1);
1568       NdbSleep_SecSleep(3);
1569       pos++;
1570     }
1571     pos = 0;
1572   }
1573
1574   if (res.waitNodesNoStart(&node, 1))
1575     return NDBT_FAILED;
1576
1577   res.startNodes(&node, 1);
1578   if (res.waitClusterStarted())
1579     return NDBT_FAILED;
1580
1581   return NDBT_OK;
1582 }
1583
1584
1585 int
1586 runBug27283(NDBT_Context* ctx, NDBT_Step* step)
1587 {
1588   int result = NDBT_OK;
1589   int loops = ctx->getNumLoops();
1590   int records = ctx->getNumRecords();
1591   NdbRestarter res;
1592
1593   if (res.getNumDbNodes() < 2)
1594   {
1595     return NDBT_OK;
1596   }
1597
1598   static const int errnos[] = { 7181, 7182, 0 };
1599
1600   Uint32 pos = 0;
1601   for (Uint32 i = 0; i<loops; i++)
1602   {
1603     while (errnos[pos] != 0)
1604     {
1605       int master = res.getMasterNodeId();
1606       int next = res.getNextMasterNodeId(master);
1607       int next2 = res.getNextMasterNodeId(next);
1608
1609       int node = (i & 1) ? next : next2;
1610       ndbout_c("Tesing err: %d", errnos[pos]);
1611       if (res.insertErrorInNode(next, errnos[pos]))
1612         return NDBT_FAILED;
1613
1614       NdbSleep_SecSleep(3);
1615
1616       if (res.waitClusterStarted())
1617         return NDBT_FAILED;
1618
1619       pos++;
1620     }
1621     pos = 0;
1622   }
1623
1624   return NDBT_OK;
1625 }
1626
1627 int
1628 runBug27466(NDBT_Context* ctx, NDBT_Step* step)
1629 {
1630   int result = NDBT_OK;
1631   int loops = ctx->getNumLoops();
1632   int records = ctx->getNumRecords();
1633   NdbRestarter res;
1634
1635   if (res.getNumDbNodes() < 2)
1636   {
1637     return NDBT_OK;
1638   }
1639
1640   Uint32 pos = 0;
1641   for (Uint32 i = 0; i<loops; i++)
1642   {
1643     int node1 = res.getDbNodeId(rand() % res.getNumDbNodes());
1644     int node2 = node1;
1645     while (node1 == node2)
1646     {
1647       node2 = res.getDbNodeId(rand() % res.getNumDbNodes());
1648     }
1649
1650     ndbout_c("nodes %u %u", node1, node2);
1651
1652     if (res.restartOneDbNode(node1, false, true, true))
1653       return NDBT_FAILED;
1654
1655     if (res.waitNodesNoStart(&node1, 1))
1656       return NDBT_FAILED;
1657
1658     int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1659     if (res.dumpStateOneNode(node1, val2, 2))
1660       return NDBT_FAILED;
1661
1662     if (res.insertErrorInNode(node2, 8039))
1663       return NDBT_FAILED;
1664
1665     res.startNodes(&node1, 1);
1666     NdbSleep_SecSleep(3);
1667     if (res.waitNodesNoStart(&node1, 1))
1668       return NDBT_FAILED;
1669     NdbSleep_SecSleep(5); // Wait for delayed INCL_NODECONF to arrive
1670
1671     res.startNodes(&node1, 1);
1672     if (res.waitClusterStarted())
1673       return NDBT_FAILED;
1674   }
1675
1676   return NDBT_OK;
1677 }
1678
1679 int
1680 runBug28023(NDBT_Context* ctx, NDBT_Step* step)
1681 {
1682   int result = NDBT_OK;
1683   int loops = ctx->getNumLoops();
1684   int records = ctx->getNumRecords();
1685   Ndb* pNdb = GETNDB(step);
1686   NdbRestarter res;
1687
1688   if (res.getNumDbNodes() < 2)
1689   {
1690     return NDBT_OK;
1691   }
1692
1693
1694   HugoTransactions hugoTrans(*ctx->getTab());
1695   if (hugoTrans.loadTable(pNdb, records) != 0){
1696     return NDBT_FAILED;
1697   }
1698
1699   if (hugoTrans.clearTable(pNdb, records) != 0)
1700   {
1701     return NDBT_FAILED;
1702   }
1703
1704   for (Uint32 i = 0; i<loops; i++)
1705   {
1706     int node1 = res.getDbNodeId(rand() % res.getNumDbNodes());
1707
1708     if (res.restartOneDbNode2(node1,
1709                               NdbRestarter::NRRF_ABORT |
1710                               NdbRestarter::NRRF_NOSTART))
1711       return NDBT_FAILED;
1712
1713     if (res.waitNodesNoStart(&node1, 1))
1714       return NDBT_FAILED;
1715
1716     if (hugoTrans.loadTable(pNdb, records) != 0){
1717       return NDBT_FAILED;
1718     }
1719
1720     if (hugoTrans.clearTable(pNdb, records) != 0)
1721     {
1722       return NDBT_FAILED;
1723     }
1724
1725     res.startNodes(&node1, 1);
1726     if (res.waitClusterStarted())
1727       return NDBT_FAILED;
1728
1729     if (hugoTrans.loadTable(pNdb, records) != 0){
1730       return NDBT_FAILED;
1731     }
1732
1733     if (hugoTrans.scanUpdateRecords(pNdb, records) != 0)
1734       return NDBT_FAILED;
1735
1736     if (hugoTrans.clearTable(pNdb, records) != 0)
1737     {
1738       return NDBT_FAILED;
1739     }
1740   }
1741
1742   return NDBT_OK;
1743 }
1744
1745
1746 int
1747 runBug28717(NDBT_Context* ctx, NDBT_Step* step)
1748 {
1749   int result = NDBT_OK;
1750   int loops = ctx->getNumLoops();
1751   int records = ctx->getNumRecords();
1752   Ndb* pNdb = GETNDB(step);
1753   NdbRestarter res;
1754
1755   if (res.getNumDbNodes() < 4)
1756   {
1757     return NDBT_OK;
1758   }
1759
1760   int master = res.getMasterNodeId();
1761   int node0 = res.getRandomNodeOtherNodeGroup(master, rand());
1762   int node1 = res.getRandomNodeSameNodeGroup(node0, rand());
1763
1764   ndbout_c("master: %d node0: %d node1: %d", master, node0, node1);
1765
1766   if (res.restartOneDbNode(node0, false, true, true))
1767   {
1768     return NDBT_FAILED;
1769   }
1770
1771   {
1772     int filter[] = { 15, NDB_MGM_EVENT_CATEGORY_CHECKPOINT, 0 };
1773     NdbLogEventHandle handle =
1774       ndb_mgm_create_logevent_handle(res.handle, filter);
1775
1776
1777     int dump[] = { DumpStateOrd::DihStartLcpImmediately };
1778     struct ndb_logevent event;
1779
1780     for (Uint32 i = 0; i<3; i++)
1781     {
1782       res.dumpStateOneNode(master, dump, 1);
1783       while(ndb_logevent_get_next(handle, &event, 0) >= 0 &&
1784             event.type != NDB_LE_LocalCheckpointStarted);
1785       while(ndb_logevent_get_next(handle, &event, 0) >= 0 &&
1786             event.type != NDB_LE_LocalCheckpointCompleted);
1787     }
1788   }
1789
1790   if (res.waitNodesNoStart(&node0, 1))
1791     return NDBT_FAILED;
1792
1793   int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1794
1795   if (res.dumpStateOneNode(node0, val2, 2))
1796     return NDBT_FAILED;
1797
1798   if (res.insertErrorInNode(node0, 5010))
1799     return NDBT_FAILED;
1800
1801   if (res.insertErrorInNode(node1, 1001))
1802     return NDBT_FAILED;
1803
1804   if (res.startNodes(&node0, 1))
1805     return NDBT_FAILED;
1806
1807   NdbSleep_SecSleep(3);
1808
1809   if (res.insertErrorInNode(node1, 0))
1810     return NDBT_FAILED;
1811
1812   if (res.waitNodesNoStart(&node0, 1))
1813     return NDBT_FAILED;
1814
1815   if (res.startNodes(&node0, 1))
1816     return NDBT_FAILED;
1817
1818   if (res.waitClusterStarted())
1819     return NDBT_FAILED;
1820
1821   return NDBT_OK;
1822 }
1823
1824 int
1825 runBug31525(NDBT_Context* ctx, NDBT_Step* step)
1826 {
1827   int result = NDBT_OK;
1828   int loops = ctx->getNumLoops();
1829   int records = ctx->getNumRecords();
1830   Ndb* pNdb = GETNDB(step);
1831   NdbRestarter res;
1832
1833   if (res.getNumDbNodes() < 2)
1834   {
1835     return NDBT_OK;
1836   }
1837
1838   int nodes[2];
1839   nodes[0] = res.getMasterNodeId();
1840   nodes[1] = res.getNextMasterNodeId(nodes[0]);
1841
1842   while (res.getNodeGroup(nodes[0]) != res.getNodeGroup(nodes[1]))
1843   {
1844     ndbout_c("Restarting %u as it not in same node group as %u",
1845              nodes[1], nodes[0]);
1846     if (res.restartOneDbNode(nodes[1], false, true, true))
1847       return NDBT_FAILED;
1848
1849     if (res.waitNodesNoStart(nodes+1, 1))
1850       return NDBT_FAILED;
1851
1852     if (res.startNodes(nodes+1, 1))
1853       return NDBT_FAILED;
1854
1855     if (res.waitClusterStarted())
1856       return NDBT_FAILED;
1857
1858     nodes[1] = res.getNextMasterNodeId(nodes[0]);
1859   }
1860
1861   ndbout_c("nodes[0]: %u nodes[1]: %u", nodes[0], nodes[1]);
1862
1863   int val = DumpStateOrd::DihMinTimeBetweenLCP;
1864   if (res.dumpStateAllNodes(&val, 1))
1865     return NDBT_FAILED;
1866
1867   int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1868   if (res.dumpStateAllNodes(val2, 2))
1869     return NDBT_FAILED;
1870
1871   if (res.insertErrorInAllNodes(932))
1872     return NDBT_FAILED;
1873
1874   if (res.insertErrorInNode(nodes[1], 7192))
1875     return NDBT_FAILED;
1876
1877   if (res.insertErrorInNode(nodes[0], 7191))
1878     return NDBT_FAILED;
1879
1880   if (res.waitClusterNoStart())
1881     return NDBT_FAILED;
1882
1883   if (res.startAll())
1884     return NDBT_FAILED;
1885
1886   if (res.waitClusterStarted())
1887     return NDBT_FAILED;
1888
1889   if (res.restartOneDbNode(nodes[1], false, false, true))
1890     return NDBT_FAILED;
1891
1892   if (res.waitClusterStarted())
1893     return NDBT_FAILED;
1894
1895   return NDBT_OK;
1896 }
1897
1898 int
1899 runBug32160(NDBT_Context* ctx, NDBT_Step* step)
1900 {
1901   int result = NDBT_OK;
1902   int loops = ctx->getNumLoops();
1903   int records = ctx->getNumRecords();
1904   Ndb* pNdb = GETNDB(step);
1905   NdbRestarter res;
1906
1907   if (res.getNumDbNodes() < 2)
1908   {
1909     return NDBT_OK;
1910   }
1911
1912   int master = res.getMasterNodeId();
1913   int next = res.getNextMasterNodeId(master);
1914
1915   if (res.insertErrorInNode(next, 7194))
1916   {
1917     return NDBT_FAILED;
1918   }
1919
1920   int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1921   if (res.dumpStateOneNode(master, val2, 2))
1922     return NDBT_FAILED;
1923
1924   if (res.insertErrorInNode(master, 7193))
1925     return NDBT_FAILED;
1926
1927   int val3[] = { 7099 };
1928   if (res.dumpStateOneNode(master, val3, 1))
1929     return NDBT_FAILED;
1930
1931   if (res.waitNodesNoStart(&master, 1))
1932     return NDBT_FAILED;
1933
1934   if (res.startNodes(&master, 1))
1935     return NDBT_FAILED;
1936
1937   if (res.waitClusterStarted())
1938     return NDBT_FAILED;
1939
1940   return NDBT_OK;
1941 }
1942
1943 int
1944 runBug34216(NDBT_Context* ctx, NDBT_Step* step)
1945 {
1946   int result = NDBT_OK;
1947   int loops = ctx->getNumLoops();
1948   NdbRestarter restarter;
1949   int i = 0;
1950   int lastId = 0;
1951   HugoOperations hugoOps(*ctx->getTab());
1952   int records = ctx->getNumRecords();
1953   Ndb* pNdb = GETNDB(step);
1954
1955   if (restarter.getNumDbNodes() < 2)
1956   {
1957     ctx->stopTest();
1958     return NDBT_OK;
1959   }
1960
1961   if(restarter.waitClusterStarted() != 0){
1962     g_err << "Cluster failed to start" << endl;
1963     return NDBT_FAILED;
1964   }
1965
1966   char buf[100];
1967   const char * off = NdbEnv_GetEnv("NDB_ERR_OFFSET", buf, sizeof(buf));
1968   int offset = off ? atoi(off) : 0;
1969
1970   while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped())
1971   {
1972     int id = lastId % restarter.getNumDbNodes();
1973     int nodeId = restarter.getDbNodeId(id);
1974     int err = 5048 + ((i+offset) % 2);
1975
1976     int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
1977
1978     if(hugoOps.startTransaction(pNdb) != 0)
1979       goto err;
1980
1981     nodeId = hugoOps.getTransaction()->getConnectedNodeId();
1982     ndbout << "Restart node " << nodeId << " " << err <<endl;
1983
1984     if (restarter.dumpStateOneNode(nodeId, val2, 2))
1985       return NDBT_FAILED;
1986
1987     if(restarter.insertErrorInNode(nodeId, err) != 0){
1988       g_err << "Failed to restartNextDbNode" << endl;
1989       result = NDBT_FAILED;
1990       break;
1991     }
1992
1993     if (restarter.insertErrorInNode(nodeId, 8057) != 0)
1994     {
1995       g_err << "Failed to insert error 8057" << endl;
1996       result = NDBT_FAILED;
1997       break;
1998     }
1999
2000     int rows = 10;
2001     int batch = 1;
2002     int row = (records - rows) ? rand() % (records - rows) : 0;
2003
2004     if(hugoOps.pkUpdateRecord(pNdb, row, batch, rand()) != 0)
2005       goto err;
2006
2007     for (int l = 1; l<5; l++)
2008     {
2009       if (hugoOps.execute_NoCommit(pNdb) != 0)
2010         goto err;
2011
2012       if(hugoOps.pkUpdateRecord(pNdb, row, batch, rand()) != 0)
2013         goto err;
2014     }
2015
2016     hugoOps.execute_Commit(pNdb);
2017     hugoOps.closeTransaction(pNdb);
2018
2019     if (restarter.waitNodesNoStart(&nodeId, 1))
2020     {
2021       g_err << "Failed to waitNodesNoStart" << endl;
2022       result = NDBT_FAILED;
2023       break;
2024     }
2025
2026     if (restarter.startNodes(&nodeId, 1))
2027     {
2028       g_err << "Failed to startNodes" << endl;
2029       result = NDBT_FAILED;
2030       break;
2031     }
2032
2033     if(restarter.waitClusterStarted() != 0){
2034       g_err << "Cluster failed to start" << endl;
2035       result = NDBT_FAILED;
2036       break;
2037     }
2038
2039     lastId++;
2040     i++;
2041   }
2042
2043   ctx->stopTest();
2044
2045   return result;
2046 err:
2047   return NDBT_FAILED;
2048 }
2049
2050
2051 int
2052 runNF_commit(NDBT_Context* ctx, NDBT_Step* step)
2053 {
2054   int result = NDBT_OK;
2055   int loops = ctx->getNumLoops();
2056   NdbRestarter restarter;
2057   if (restarter.getNumDbNodes() < 2)
2058   {
2059     ctx->stopTest();
2060     return NDBT_OK;
2061   }
2062
2063   if(restarter.waitClusterStarted() != 0){
2064     g_err << "Cluster failed to start" << endl;
2065     return NDBT_FAILED;
2066   }
2067
2068   int i = 0;
2069   while(i<loops && result != NDBT_FAILED && !ctx->isTestStopped())
2070   {
2071     int nodeId = restarter.getDbNodeId(rand() % restarter.getNumDbNodes());
2072     int err = 5048;
2073
2074     ndbout << "Restart node " << nodeId << " " << err <<endl;
2075
2076     int val2[] = { DumpStateOrd::CmvmiSetRestartOnErrorInsert, 1 };
2077     if (restarter.dumpStateOneNode(nodeId, val2, 2))
2078       return NDBT_FAILED;
2079
2080     if(restarter.insertErrorInNode(nodeId, err) != 0){
2081       g_err << "Failed to restartNextDbNode" << endl;
2082       result = NDBT_FAILED;
2083       break;
2084     }
2085
2086     if (restarter.waitNodesNoStart(&nodeId, 1))
2087     {
2088       g_err << "Failed to waitNodesNoStart" << endl;
2089       result = NDBT_FAILED;
2090       break;
2091     }
2092
2093     if (restarter.startNodes(&nodeId, 1))
2094     {
2095       g_err << "Failed to startNodes" << endl;
2096       result = NDBT_FAILED;
2097       break;
2098     }
2099
2100     if(restarter.waitClusterStarted() != 0){
2101       g_err << "Cluster failed to start" << endl;
2102       result = NDBT_FAILED;
2103       break;
2104     }
2105
2106     i++;
2107   }
2108
2109   ctx->stopTest();
2110
2111   return result;
2112 }
2113
2114
2115 NDBT_TESTSUITE(testNodeRestart);
2116 TESTCASE("NoLoad",
2117          "Test that one node at a time can be stopped and then restarted "\
2118          "when there are no load on the system. Do this loop number of times"){
2119   INITIALIZER(runCheckAllNodesStarted);
2120   INITIALIZER(runLoadTable);
2121   STEP(runRestarter);
2122   FINALIZER(runClearTable);
2123 }
2124 TESTCASE("PkRead",
2125          "Test that one node at a time can be stopped and then restarted "\
2126          "perform pk read while restarting. Do this loop number of times"){
2127   TC_PROPERTY("ReadLockMode", NdbOperation::LM_Read);
2128   INITIALIZER(runCheckAllNodesStarted);
2129   INITIALIZER(runLoadTable);
2130   STEP(runRestarter);
2131   STEP(runPkReadUntilStopped);
2132   FINALIZER(runClearTable);
2133 }
2134 TESTCASE("PkReadCommitted",
2135          "Test that one node at a time can be stopped and then restarted "\
2136          "perform pk read while restarting. Do this loop number of times"){
2137   TC_PROPERTY("ReadLockMode", NdbOperation::LM_CommittedRead);
2138   INITIALIZER(runCheckAllNodesStarted);
2139   INITIALIZER(runLoadTable);
2140   STEP(runRestarter);
2141   STEP(runPkReadUntilStopped);
2142   FINALIZER(runClearTable);
2143 }
2144 TESTCASE("MixedPkRead",
2145          "Test that one node at a time can be stopped and then restarted "\
2146          "perform pk read while restarting. Do this loop number of times"){
2147   TC_PROPERTY("ReadLockMode", -1);
2148   INITIALIZER(runCheckAllNodesStarted);
2149   INITIALIZER(runLoadTable);
2150   STEP(runRestarter);
2151   STEP(runPkReadUntilStopped);
2152   FINALIZER(runClearTable);
2153 }
2154 TESTCASE("PkReadPkUpdate",
2155          "Test that one node at a time can be stopped and then restarted "\
2156          "perform pk read and pk update while restarting. Do this loop number of times"){
2157   TC_PROPERTY("ReadLockMode", NdbOperation::LM_Read);
2158   INITIALIZER(runCheckAllNodesStarted);
2159   INITIALIZER(runLoadTable);
2160   STEP(runRestarter);
2161   STEP(runPkReadUntilStopped);
2162   STEP(runPkUpdateUntilStopped);
2163   STEP(runPkReadPkUpdateUntilStopped);
2164   STEP(runPkReadUntilStopped);
2165   STEP(runPkUpdateUntilStopped);
2166   STEP(runPkReadPkUpdateUntilStopped);
2167   FINALIZER(runClearTable);
2168 }
2169 TESTCASE("MixedPkReadPkUpdate",
2170          "Test that one node at a time can be stopped and then restarted "\
2171          "perform pk read and pk update while restarting. Do this loop number of times"){
2172   TC_PROPERTY("ReadLockMode", -1);
2173   INITIALIZER(runCheckAllNodesStarted);
2174   INITIALIZER(runLoadTable);
2175   STEP(runRestarter);
2176   STEP(runPkReadUntilStopped);
2177   STEP(runPkUpdateUntilStopped);
2178   STEP(runPkReadPkUpdateUntilStopped);
2179   STEP(runPkReadUntilStopped);
2180   STEP(runPkUpdateUntilStopped);
2181   STEP(runPkReadPkUpdateUntilStopped);
2182   FINALIZER(runClearTable);
2183 }
2184 TESTCASE("ReadUpdateScan",
2185          "Test that one node at a time can be stopped and then restarted "\
2186          "perform pk read, pk update and scan reads while restarting. Do this loop number of times"){
2187   INITIALIZER(runCheckAllNodesStarted);
2188   INITIALIZER(runLoadTable);
2189   STEP(runRestarter);
2190   STEP(runPkReadUntilStopped);
2191   STEP(runPkUpdateUntilStopped);
2192   STEP(runPkReadPkUpdateUntilStopped);
2193   STEP(runScanReadUntilStopped);
2194   STEP(runScanUpdateUntilStopped);
2195   FINALIZER(runClearTable);
2196 }
2197 TESTCASE("MixedReadUpdateScan",
2198          "Test that one node at a time can be stopped and then restarted "\
2199          "perform pk read, pk update and scan reads while restarting. Do this loop number of times"){
2200   TC_PROPERTY("ReadLockMode", -1);
2201   INITIALIZER(runCheckAllNodesStarted);
2202   INITIALIZER(runLoadTable);
2203   STEP(runRestarter);
2204   STEP(runPkReadUntilStopped);
2205   STEP(runPkUpdateUntilStopped);
2206   STEP(runPkReadPkUpdateUntilStopped);
2207   STEP(runScanReadUntilStopped);
2208   STEP(runScanUpdateUntilStopped);
2209   FINALIZER(runClearTable);
2210 }
2211 TESTCASE("Terror",
2212          "Test that one node at a time can be stopped and then restarted "\
2213          "perform all kind of transactions while restarting. Do this loop number of times"){
2214   INITIALIZER(runCheckAllNodesStarted);
2215   INITIALIZER(runLoadTable);
2216   STEP(runRestarter);
2217   STEP(runPkReadUntilStopped);
2218   STEP(runPkUpdateUntilStopped);
2219   STEP(runScanReadUntilStopped);
2220   STEP(runScanUpdateUntilStopped);
2221   FINALIZER(runClearTable);
2222 }
2223 TESTCASE("FullDb",
2224          "Test that one node at a time can be stopped and then restarted "\
2225          "when db is full. Do this loop number of times"){
2226   INITIALIZER(runCheckAllNodesStarted);
2227   INITIALIZER(runFillTable);
2228   STEP(runRestarter);
2229   FINALIZER(runClearTable);
2230 }
2231 TESTCASE("RestartRandomNode",
2232          "Test that we can execute the restart RestartRandomNode loop\n"\
2233          "number of times"){
2234   INITIALIZER(runCheckAllNodesStarted);
2235   INITIALIZER(runLoadTable);
2236   STEP(runRestarts);
2237   FINALIZER(runScanReadVerify);
2238   FINALIZER(runClearTable);
2239 }
2240 TESTCASE("RestartRandomNodeError",
2241          "Test that we can execute the restart RestartRandomNodeError loop\n"\
2242          "number of times"){
2243   INITIALIZER(runCheckAllNodesStarted);
2244   INITIALIZER(runLoadTable);
2245   STEP(runRestarts);
2246   FINALIZER(runScanReadVerify);
2247   FINALIZER(runClearTable);
2248 }
2249 TESTCASE("RestartRandomNodeInitial",
2250          "Test that we can execute the restart RestartRandomNodeInitial loop\n"\
2251          "number of times"){
2252   INITIALIZER(runCheckAllNodesStarted);
2253   INITIALIZER(runLoadTable);
2254   STEP(runRestarts);
2255   FINALIZER(runScanReadVerify);
2256   FINALIZER(runClearTable);
2257 }
2258 TESTCASE("RestartNFDuringNR",
2259          "Test that we can execute the restart RestartNFDuringNR loop\n"\
2260          "number of times"){
2261   INITIALIZER(runCheckAllNodesStarted);
2262   INITIALIZER(runLoadTable);
2263   STEP(runRestarts);
2264   STEP(runPkUpdateUntilStopped);
2265   STEP(runScanUpdateUntilStopped);
2266   FINALIZER(runScanReadVerify);
2267   FINALIZER(runClearTable);
2268 }
2269 TESTCASE("RestartMasterNodeError",
2270          "Test that we can execute the restart RestartMasterNodeError loop\n"\
2271          "number of times"){
2272   INITIALIZER(runCheckAllNodesStarted);
2273   INITIALIZER(runLoadTable);
2274   STEP(runRestarts);
2275   FINALIZER(runScanReadVerify);
2276   FINALIZER(runClearTable);
2277 }
2278
2279 TESTCASE("TwoNodeFailure",
2280          "Test that we can execute the restart TwoNodeFailure\n"\
2281          "(which is a multiple node failure restart) loop\n"\
2282          "number of times"){
2283   INITIALIZER(runCheckAllNodesStarted);
2284   INITIALIZER(runLoadTable);
2285   STEP(runRestarts);
2286   FINALIZER(runScanReadVerify);
2287   FINALIZER(runClearTable);
2288 }
2289 TESTCASE("TwoMasterNodeFailure",
2290          "Test that we can execute the restart TwoMasterNodeFailure\n"\
2291          "(which is a multiple node failure restart) loop\n"\
2292          "number of times"){
2293   INITIALIZER(runCheckAllNodesStarted);
2294   INITIALIZER(runLoadTable);
2295   STEP(runRestarts);
2296   FINALIZER(runScanReadVerify);
2297   FINALIZER(runClearTable);
2298 }
2299 TESTCASE("FiftyPercentFail",
2300          "Test that we can execute the restart FiftyPercentFail\n"\
2301          "(which is a multiple node failure restart) loop\n"\
2302          "number of times"){
2303   INITIALIZER(runCheckAllNodesStarted);
2304   INITIALIZER(runLoadTable);
2305   STEP(runRestarts);
2306   FINALIZER(runScanReadVerify);
2307   FINALIZER(runClearTable);
2308 }
2309 TESTCASE("RestartAllNodes",
2310          "Test that we can execute the restart RestartAllNodes\n"\
2311          "(which is a system  restart) loop\n"\
2312          "number of times"){
2313   INITIALIZER(runCheckAllNodesStarted);
2314   INITIALIZER(runLoadTable);
2315   STEP(runRestarts);
2316   FINALIZER(runScanReadVerify);
2317   FINALIZER(runClearTable);
2318 }
2319 TESTCASE("RestartAllNodesAbort",
2320          "Test that we can execute the restart RestartAllNodesAbort\n"\
2321          "(which is a system  restart) loop\n"\
2322          "number of times"){
2323   INITIALIZER(runCheckAllNodesStarted);
2324   INITIALIZER(runLoadTable);
2325   STEP(runRestarts);
2326   FINALIZER(runScanReadVerify);
2327   FINALIZER(runClearTable);
2328 }
2329 TESTCASE("RestartAllNodesError9999",
2330          "Test that we can execute the restart RestartAllNodesError9999\n"\
2331          "(which is a system  restart) loop\n"\
2332          "number of times"){
2333   INITIALIZER(runCheckAllNodesStarted);
2334   INITIALIZER(runLoadTable);
2335   STEP(runRestarts);
2336   FINALIZER(runScanReadVerify);
2337   FINALIZER(runClearTable);
2338 }
2339 TESTCASE("FiftyPercentStopAndWait",
2340          "Test that we can execute the restart FiftyPercentStopAndWait\n"\
2341          "(which is a system  restart) loop\n"\
2342          "number of times"){
2343   INITIALIZER(runCheckAllNodesStarted);
2344   INITIALIZER(runLoadTable);
2345   STEP(runRestarts);
2346   FINALIZER(runScanReadVerify);
2347   FINALIZER(runClearTable);
2348 }
2349 TESTCASE("RestartNodeDuringLCP",
2350          "Test that we can execute the restart RestartRandomNode loop\n"\
2351          "number of times"){
2352   INITIALIZER(runCheckAllNodesStarted);
2353   INITIALIZER(runLoadTable);
2354   STEP(runRestarts);
2355   STEP(runPkUpdateUntilStopped);
2356   STEP(runScanUpdateUntilStopped);
2357   FINALIZER(runScanReadVerify);
2358   FINALIZER(runClearTable);
2359 }
2360 TESTCASE("StopOnError",
2361          "Test StopOnError. A node that has StopOnError set to false "\
2362          "should restart automatically when an error occurs"){
2363   INITIALIZER(runCheckAllNodesStarted);
2364   INITIALIZER(runLoadTable);
2365   STEP(runRestarts);
2366   FINALIZER(runScanReadVerify);
2367   FINALIZER(runClearTable);
2368 }
2369 TESTCASE("CommittedRead",
2370          "Test committed read"){
2371   INITIALIZER(runLoadTable);
2372   STEP(runDirtyRead);
2373   FINALIZER(runClearTable);
2374 }
2375 TESTCASE("LateCommit",
2376          "Test commit after node failure"){
2377   INITIALIZER(runLoadTable);
2378   STEP(runLateCommit);
2379   FINALIZER(runClearTable);
2380 }
2381 TESTCASE("Bug15587",
2382          "Test bug with NF during NR"){
2383   INITIALIZER(runLoadTable);
2384   STEP(runScanUpdateUntilStopped);
2385   STEP(runBug15587);
2386   FINALIZER(runClearTable);
2387 }
2388 TESTCASE("Bug15632",
2389          "Test bug with NF during NR"){
2390   INITIALIZER(runLoadTable);
2391   STEP(runBug15632);
2392   FINALIZER(runClearTable);
2393 }
2394 TESTCASE("Bug15685",
2395          "Test bug with NF during abort"){
2396   STEP(runBug15685);
2397   FINALIZER(runClearTable);
2398 }
2399 TESTCASE("Bug16772",
2400          "Test bug with restarting before NF handling is complete"){
2401   STEP(runBug16772);
2402 }
2403 TESTCASE("Bug18414",
2404          "Test bug with NF during NR"){
2405   INITIALIZER(runLoadTable);
2406   STEP(runBug18414);
2407   FINALIZER(runClearTable);
2408 }
2409 TESTCASE("Bug18612",
2410          "Test bug with partitioned clusters"){
2411   INITIALIZER(runLoadTable);
2412   STEP(runBug18612);
2413   FINALIZER(runClearTable);
2414 }
2415 TESTCASE("Bug18612SR",
2416          "Test bug with partitioned clusters"){
2417   INITIALIZER(runLoadTable);
2418   STEP(runBug18612SR);
2419   FINALIZER(runClearTable);
2420 }
2421 TESTCASE("Bug20185",
2422          ""){
2423   INITIALIZER(runLoadTable);
2424   STEP(runBug20185);
2425   FINALIZER(runClearTable);
2426 }
2427 TESTCASE("Bug24543", "")
2428 {
2429   INITIALIZER(runBug24543);
2430 }
2431 TESTCASE("Bug21271",
2432          ""){
2433   INITIALIZER(runLoadTable);
2434   STEP(runBug21271);
2435   STEP(runPkUpdateUntilStopped);
2436   FINALIZER(runClearTable);
2437 }
2438 TESTCASE("Bug31525", ""){
2439   INITIALIZER(runBug31525);
2440 }
2441 TESTCASE("Bug24717", ""){
2442   INITIALIZER(runBug24717);
2443 }
2444 TESTCASE("Bug25364", ""){
2445   INITIALIZER(runBug25364);
2446 }
2447 TESTCASE("Bug25468", ""){
2448   INITIALIZER(runBug25468);
2449 }
2450 TESTCASE("Bug25554", ""){
2451   INITIALIZER(runBug25554);
2452 }
2453 TESTCASE("Bug25984", ""){
2454   INITIALIZER(runBug25984);
2455 }
2456 TESTCASE("Bug26457", ""){
2457   INITIALIZER(runBug26457);
2458 }
2459 TESTCASE("Bug26481", ""){
2460   INITIALIZER(runBug26481);
2461 }
2462 TESTCASE("Bug26450", ""){
2463   INITIALIZER(runLoadTable);
2464   INITIALIZER(runBug26450);
2465 }
2466 TESTCASE("Bug27003", ""){
2467   INITIALIZER(runBug27003);
2468 }
2469 TESTCASE("Bug27283", ""){
2470   INITIALIZER(runBug27283);
2471 }
2472 TESTCASE("Bug27466", ""){
2473   INITIALIZER(runBug27466);
2474 }
2475 TESTCASE("Bug28023", ""){
2476   INITIALIZER(runBug28023);
2477 }
2478 TESTCASE("Bug28717", ""){
2479   INITIALIZER(runBug28717);
2480 }
2481 TESTCASE("Bug29364", ""){
2482   INITIALIZER(runBug29364);
2483 }
2484 TESTCASE("Bug32160", ""){
2485   INITIALIZER(runBug32160);
2486 }
2487 TESTCASE("Bug34216", ""){
2488   INITIALIZER(runCheckAllNodesStarted);
2489   INITIALIZER(runLoadTable);
2490   STEP(runBug34216);
2491   FINALIZER(runClearTable);
2492 }
2493 TESTCASE("mixedmultiop", ""){
2494   TC_PROPERTY("MULTI_OP", 5);
2495   INITIALIZER(runCheckAllNodesStarted);
2496   INITIALIZER(runLoadTable);
2497   STEP(runNF_commit);
2498   STEP(runPkUpdateUntilStopped);
2499   STEP(runPkUpdateUntilStopped);
2500   FINALIZER(runClearTable);
2501 }
2502 NDBT_TESTSUITE_END(testNodeRestart);
2503
2504 int main(int argc, const char** argv){
2505   ndb_init();
2506 #if 0
2507   // It might be interesting to have longer defaults for num
2508   // loops in this test
2509   // Just performing 100 node restarts would not be enough?
2510   // We can have initialisers in the NDBT_Testcase class like
2511   // this...
2512   testNodeRestart.setDefaultLoops(1000);
2513 #endif
2514   return testNodeRestart.execute(argc, argv);
2515 }
2516