ctdb/server/ctdb_server.c

   1 /*
   2    ctdb main protocol code
   3
   4    Copyright (C) Andrew Tridgell  2006
   5
   6    This program is free software; you can redistribute it and/or modify
   7    it under the terms of the GNU General Public License as published by
   8    the Free Software Foundation; either version 3 of the License, or
   9    (at your option) any later version.
  10
  11    This program is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14    GNU General Public License for more details.
  15
  16    You should have received a copy of the GNU General Public License
  17    along with this program; if not, see <http://www.gnu.org/licenses/>.
  18 */
  19
  20 #include "includes.h"
  21 #include "tdb.h"
  22 #include "lib/util/dlinklist.h"
  23 #include "system/network.h"
  24 #include "system/filesys.h"
  25 #include "../include/ctdb_private.h"
  26
  27 /*
  28   choose the transport we will use
  29 */
  30 int ctdb_set_transport(struct ctdb_context *ctdb, const char *transport)
  31 {
  32         ctdb->transport = talloc_strdup(ctdb, transport);
  33         CTDB_NO_MEMORY(ctdb, ctdb->transport);
  34
  35         return 0;
  36 }
  37
  38 /*
  39   Check whether an ip is a valid node ip
  40   Returns the node id for this ip address or -1
  41 */
  42 int ctdb_ip_to_nodeid(struct ctdb_context *ctdb, const char *nodeip)
  43 {
  44         int nodeid;
  45
  46         for (nodeid=0;nodeid<ctdb->num_nodes;nodeid++) {
  47                 if (ctdb->nodes[nodeid]->flags & NODE_FLAGS_DELETED) {
  48                         continue;
  49                 }
  50                 if (!strcmp(ctdb->nodes[nodeid]->address.address, nodeip)) {
  51                         return nodeid;
  52                 }
  53         }
  54
  55         return -1;
  56 }
  57
  58 /*
  59   choose the recovery lock file
  60 */
  61 int ctdb_set_recovery_lock_file(struct ctdb_context *ctdb, const char *file)
  62 {
  63         if (ctdb->recovery_lock_file != NULL) {
  64                 talloc_free(ctdb->recovery_lock_file);
  65                 ctdb->recovery_lock_file = NULL;
  66         }
  67
  68         if (file == NULL) {
  69                 DEBUG(DEBUG_ALERT,("Recovery lock file set to \"\". Disabling recovery lock checking\n"));
  70                 ctdb->tunable.verify_recovery_lock = 0;
  71                 return 0;
  72         }
  73
  74         ctdb->recovery_lock_file = talloc_strdup(ctdb, file);
  75         CTDB_NO_MEMORY(ctdb, ctdb->recovery_lock_file);
  76
  77         return 0;
  78 }
  79
  80 /*
  81   add a node to the list of nodes
  82 */
  83 static int ctdb_add_node(struct ctdb_context *ctdb, char *nstr)
  84 {
  85         struct ctdb_node *node, **nodep;
  86
  87         nodep = talloc_realloc(ctdb, ctdb->nodes, struct ctdb_node *, ctdb->num_nodes+1);
  88         CTDB_NO_MEMORY(ctdb, nodep);
  89
  90         ctdb->nodes = nodep;
  91         nodep = &ctdb->nodes[ctdb->num_nodes];
  92         (*nodep) = talloc_zero(ctdb->nodes, struct ctdb_node);
  93         CTDB_NO_MEMORY(ctdb, *nodep);
  94         node = *nodep;
  95
  96         if (ctdb_parse_address(ctdb, node, nstr, &node->address) != 0) {
  97                 return -1;
  98         }
  99         node->ctdb = ctdb;
 100         node->name = talloc_asprintf(node, "%s:%u",
 101                                      node->address.address,
 102                                      node->address.port);
 103         /* this assumes that the nodes are kept in sorted order, and no gaps */
 104         node->pnn = ctdb->num_nodes;
 105
 106         /* nodes start out disconnected and unhealthy */
 107         node->flags = (NODE_FLAGS_DISCONNECTED | NODE_FLAGS_UNHEALTHY);
 108
 109         if (ctdb->address.address &&
 110             ctdb_same_address(&ctdb->address, &node->address)) {
 111                 /* for automatic binding to interfaces, see tcp_connect.c */
 112                 ctdb->pnn = node->pnn;
 113         }
 114
 115         ctdb->num_nodes++;
 116         node->dead_count = 0;
 117
 118         return 0;
 119 }
 120
 121 /*
 122   add an entry for a "deleted" node to the list of nodes.
 123   a "deleted" node is a node that is commented out from the nodes file.
 124   this is used to prevent that subsequent nodes in the nodes list
 125   change their pnn value if a node is "delete" by commenting it out and then
 126   using "ctdb reloadnodes" at runtime.
 127 */
 128 static int ctdb_add_deleted_node(struct ctdb_context *ctdb)
 129 {
 130         struct ctdb_node *node, **nodep;
 131
 132         nodep = talloc_realloc(ctdb, ctdb->nodes, struct ctdb_node *, ctdb->num_nodes+1);
 133         CTDB_NO_MEMORY(ctdb, nodep);
 134
 135         ctdb->nodes = nodep;
 136         nodep = &ctdb->nodes[ctdb->num_nodes];
 137         (*nodep) = talloc_zero(ctdb->nodes, struct ctdb_node);
 138         CTDB_NO_MEMORY(ctdb, *nodep);
 139         node = *nodep;
 140
 141         if (ctdb_parse_address(ctdb, node, "0.0.0.0", &node->address) != 0) {
 142                 DEBUG(DEBUG_ERR,("Failed to setup deleted node %d\n", ctdb->num_nodes));
 143                 return -1;
 144         }
 145         node->ctdb = ctdb;
 146         node->name = talloc_strdup(node, "0.0.0.0:0");
 147
 148         /* this assumes that the nodes are kept in sorted order, and no gaps */
 149         node->pnn = ctdb->num_nodes;
 150
 151         /* this node is permanently deleted/disconnected */
 152         node->flags = NODE_FLAGS_DELETED|NODE_FLAGS_DISCONNECTED;
 153
 154         ctdb->num_nodes++;
 155         node->dead_count = 0;
 156
 157         return 0;
 158 }
 159
 160
 161 /*
 162   setup the node list from a file
 163 */
 164 static int ctdb_set_nlist(struct ctdb_context *ctdb, const char *nlist)
 165 {
 166         char **lines;
 167         int nlines;
 168         int i, j, num_present;
 169
 170         talloc_free(ctdb->nodes);
 171         ctdb->nodes     = NULL;
 172         ctdb->num_nodes = 0;
 173
 174         lines = file_lines_load(nlist, &nlines, 0, ctdb);
 175         if (lines == NULL) {
 176                 ctdb_set_error(ctdb, "Failed to load nlist '%s'\n", nlist);
 177                 return -1;
 178         }
 179         while (nlines > 0 && strcmp(lines[nlines-1], "") == 0) {
 180                 nlines--;
 181         }
 182
 183         num_present = 0;
 184         for (i=0; i < nlines; i++) {
 185                 char *node;
 186                 size_t len;
 187
 188                 node = lines[i];
 189                 /* strip leading spaces */
 190                 while((*node == ' ') || (*node == '\t')) {
 191                         node++;
 192                 }
 193                 if (*node == '#') {
 194                         if (ctdb_add_deleted_node(ctdb) != 0) {
 195                                 talloc_free(lines);
 196                                 return -1;
 197                         }
 198                         continue;
 199                 }
 200
 201                 /* strip trailing spaces */
 202
 203                 len = strlen(node);
 204
 205                 while ((len > 1) &&
 206                        ((node[len-1] == ' ') || (node[len-1] == '\t')))
 207                 {
 208                         node[len-1] = '\0';
 209                         len--;
 210                 }
 211
 212                 if (len == 0) {
 213                         continue;
 214                 }
 215                 if (ctdb_add_node(ctdb, node) != 0) {
 216                         talloc_free(lines);
 217                         return -1;
 218                 }
 219                 num_present++;
 220         }
 221
 222         /* initialize the vnn mapping table now that we have the nodes list,
 223            skipping any deleted nodes
 224         */
 225         ctdb->vnn_map = talloc(ctdb, struct ctdb_vnn_map);
 226         CTDB_NO_MEMORY(ctdb, ctdb->vnn_map);
 227
 228         ctdb->vnn_map->generation = INVALID_GENERATION;
 229         ctdb->vnn_map->size = num_present;
 230         ctdb->vnn_map->map = talloc_array(ctdb->vnn_map, uint32_t, ctdb->vnn_map->size);
 231         CTDB_NO_MEMORY(ctdb, ctdb->vnn_map->map);
 232
 233         for(i=0, j=0; i < ctdb->vnn_map->size; i++) {
 234                 if (ctdb->nodes[i]->flags & NODE_FLAGS_DELETED) {
 235                         continue;
 236                 }
 237                 ctdb->vnn_map->map[j] = i;
 238                 j++;
 239         }
 240
 241         talloc_free(lines);
 242         return 0;
 243 }
 244
 245 void ctdb_load_nodes_file(struct ctdb_context *ctdb)
 246 {
 247         int ret;
 248
 249         ret = ctdb_set_nlist(ctdb, ctdb->nodes_file);
 250         if (ret == -1) {
 251                 DEBUG(DEBUG_ALERT,("ctdb_set_nlist failed - %s\n", ctdb_errstr(ctdb)));
 252                 exit(1);
 253         }
 254 }
 255
 256 /*
 257   setup the local node address
 258 */
 259 int ctdb_set_address(struct ctdb_context *ctdb, const char *address)
 260 {
 261         if (ctdb_parse_address(ctdb, ctdb, address, &ctdb->address) != 0) {
 262                 return -1;
 263         }
 264
 265         ctdb->name = talloc_asprintf(ctdb, "%s:%u",
 266                                      ctdb->address.address,
 267                                      ctdb->address.port);
 268         return 0;
 269 }
 270
 271
 272 /*
 273   return the number of active nodes
 274 */
 275 uint32_t ctdb_get_num_active_nodes(struct ctdb_context *ctdb)
 276 {
 277         int i;
 278         uint32_t count=0;
 279         for (i=0; i < ctdb->num_nodes; i++) {
 280                 if (!(ctdb->nodes[i]->flags & NODE_FLAGS_INACTIVE)) {
 281                         count++;
 282                 }
 283         }
 284         return count;
 285 }
 286
 287
 288 /*
 289   called when we need to process a packet. This can be a requeued packet
 290   after a lockwait, or a real packet from another node
 291 */
 292 void ctdb_input_pkt(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
 293 {
 294         TALLOC_CTX *tmp_ctx;
 295
 296         /* place the packet as a child of the tmp_ctx. We then use
 297            talloc_free() below to free it. If any of the calls want
 298            to keep it, then they will steal it somewhere else, and the
 299            talloc_free() will only free the tmp_ctx */
 300         tmp_ctx = talloc_new(ctdb);
 301         talloc_steal(tmp_ctx, hdr);
 302
 303         DEBUG(DEBUG_DEBUG,(__location__ " ctdb request %u of type %u length %u from "
 304                  "node %u to %u\n", hdr->reqid, hdr->operation, hdr->length,
 305                  hdr->srcnode, hdr->destnode));
 306
 307         switch (hdr->operation) {
 308         case CTDB_REQ_CALL:
 309         case CTDB_REPLY_CALL:
 310         case CTDB_REQ_DMASTER:
 311         case CTDB_REPLY_DMASTER:
 312                 /* we dont allow these calls when banned */
 313                 if (ctdb->nodes[ctdb->pnn]->flags & NODE_FLAGS_BANNED) {
 314                         DEBUG(DEBUG_DEBUG,(__location__ " ctdb operation %u"
 315                                 " request %u"
 316                                 " length %u from node %u to %u while node"
 317                                 " is banned\n",
 318                                  hdr->operation, hdr->reqid,
 319                                  hdr->length,
 320                                  hdr->srcnode, hdr->destnode));
 321                         goto done;
 322                 }
 323
 324                 /* for ctdb_call inter-node operations verify that the
 325                    remote node that sent us the call is running in the
 326                    same generation instance as this node
 327                 */
 328                 if (ctdb->vnn_map->generation != hdr->generation) {
 329                         DEBUG(DEBUG_DEBUG,(__location__ " ctdb operation %u"
 330                                 " request %u"
 331                                 " length %u from node %u to %u had an"
 332                                 " invalid generation id:%u while our"
 333                                 " generation id is:%u\n",
 334                                  hdr->operation, hdr->reqid,
 335                                  hdr->length,
 336                                  hdr->srcnode, hdr->destnode,
 337                                  hdr->generation, ctdb->vnn_map->generation));
 338                         goto done;
 339                 }
 340         }
 341
 342         switch (hdr->operation) {
 343         case CTDB_REQ_CALL:
 344                 CTDB_INCREMENT_STAT(ctdb, node.req_call);
 345                 ctdb_request_call(ctdb, hdr);
 346                 break;
 347
 348         case CTDB_REPLY_CALL:
 349                 CTDB_INCREMENT_STAT(ctdb, node.reply_call);
 350                 ctdb_reply_call(ctdb, hdr);
 351                 break;
 352
 353         case CTDB_REPLY_ERROR:
 354                 CTDB_INCREMENT_STAT(ctdb, node.reply_error);
 355                 ctdb_reply_error(ctdb, hdr);
 356                 break;
 357
 358         case CTDB_REQ_DMASTER:
 359                 CTDB_INCREMENT_STAT(ctdb, node.req_dmaster);
 360                 ctdb_request_dmaster(ctdb, hdr);
 361                 break;
 362
 363         case CTDB_REPLY_DMASTER:
 364                 CTDB_INCREMENT_STAT(ctdb, node.reply_dmaster);
 365                 ctdb_reply_dmaster(ctdb, hdr);
 366                 break;
 367
 368         case CTDB_REQ_MESSAGE:
 369                 CTDB_INCREMENT_STAT(ctdb, node.req_message);
 370                 ctdb_request_message(ctdb, hdr);
 371                 break;
 372
 373         case CTDB_REQ_CONTROL:
 374                 CTDB_INCREMENT_STAT(ctdb, node.req_control);
 375                 ctdb_request_control(ctdb, hdr);
 376                 break;
 377
 378         case CTDB_REPLY_CONTROL:
 379                 CTDB_INCREMENT_STAT(ctdb, node.reply_control);
 380                 ctdb_reply_control(ctdb, hdr);
 381                 break;
 382
 383         case CTDB_REQ_KEEPALIVE:
 384                 CTDB_INCREMENT_STAT(ctdb, keepalive_packets_recv);
 385                 break;
 386
 387         default:
 388                 DEBUG(DEBUG_CRIT,("%s: Packet with unknown operation %u\n",
 389                          __location__, hdr->operation));
 390                 break;
 391         }
 392
 393 done:
 394         talloc_free(tmp_ctx);
 395 }
 396
 397
 398 /*
 399   called by the transport layer when a node is dead
 400 */
 401 void ctdb_node_dead(struct ctdb_node *node)
 402 {
 403         if (node->flags & NODE_FLAGS_DISCONNECTED) {
 404                 DEBUG(DEBUG_INFO,("%s: node %s is already marked disconnected: %u connected\n",
 405                          node->ctdb->name, node->name,
 406                          node->ctdb->num_connected));
 407                 return;
 408         }
 409         node->ctdb->num_connected--;
 410         node->flags |= NODE_FLAGS_DISCONNECTED | NODE_FLAGS_UNHEALTHY;
 411         node->rx_cnt = 0;
 412         node->dead_count = 0;
 413
 414         DEBUG(DEBUG_NOTICE,("%s: node %s is dead: %u connected\n",
 415                  node->ctdb->name, node->name, node->ctdb->num_connected));
 416         ctdb_daemon_cancel_controls(node->ctdb, node);
 417
 418         if (node->ctdb->methods == NULL) {
 419                 DEBUG(DEBUG_ERR,(__location__ " Can not restart transport while shutting down daemon.\n"));
 420                 return;
 421         }
 422
 423         node->ctdb->methods->restart(node);
 424 }
 425
 426 /*
 427   called by the transport layer when a node is connected
 428 */
 429 void ctdb_node_connected(struct ctdb_node *node)
 430 {
 431         if (!(node->flags & NODE_FLAGS_DISCONNECTED)) {
 432                 DEBUG(DEBUG_INFO,("%s: node %s is already marked connected: %u connected\n",
 433                          node->ctdb->name, node->name,
 434                          node->ctdb->num_connected));
 435                 return;
 436         }
 437         node->ctdb->num_connected++;
 438         node->dead_count = 0;
 439         node->flags &= ~NODE_FLAGS_DISCONNECTED;
 440         node->flags |= NODE_FLAGS_UNHEALTHY;
 441         DEBUG(DEBUG_NOTICE,
 442               ("%s: connected to %s - %u connected\n",
 443                node->ctdb->name, node->name, node->ctdb->num_connected));
 444 }
 445
 446 struct queue_next {
 447         struct ctdb_context *ctdb;
 448         struct ctdb_req_header *hdr;
 449 };
 450
 451
 452 /*
 453   triggered when a deferred packet is due
 454  */
 455 static void queue_next_trigger(struct event_context *ev, struct timed_event *te,
 456                                struct timeval t, void *private_data)
 457 {
 458         struct queue_next *q = talloc_get_type(private_data, struct queue_next);
 459         ctdb_input_pkt(q->ctdb, q->hdr);
 460         talloc_free(q);
 461 }
 462
 463 /*
 464   defer a packet, so it is processed on the next event loop
 465   this is used for sending packets to ourselves
 466  */
 467 static void ctdb_defer_packet(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
 468 {
 469         struct queue_next *q;
 470         q = talloc(ctdb, struct queue_next);
 471         if (q == NULL) {
 472                 DEBUG(DEBUG_ERR,(__location__ " Failed to allocate deferred packet\n"));
 473                 return;
 474         }
 475         q->ctdb = ctdb;
 476         q->hdr = talloc_memdup(ctdb, hdr, hdr->length);
 477         if (q->hdr == NULL) {
 478                 DEBUG(DEBUG_ERR,("Error copying deferred packet to self\n"));
 479                 return;
 480         }
 481 #if 0
 482         /* use this to put packets directly into our recv function */
 483         ctdb_input_pkt(q->ctdb, q->hdr);
 484 #else
 485         event_add_timed(ctdb->ev, q, timeval_zero(), queue_next_trigger, q);
 486 #endif
 487 }
 488
 489
 490 /*
 491   broadcast a packet to all nodes
 492 */
 493 static void ctdb_broadcast_packet_all(struct ctdb_context *ctdb,
 494                                       struct ctdb_req_header *hdr)
 495 {
 496         int i;
 497         for (i=0; i < ctdb->num_nodes; i++) {
 498                 if (ctdb->nodes[i]->flags & NODE_FLAGS_DELETED) {
 499                         continue;
 500                 }
 501                 hdr->destnode = ctdb->nodes[i]->pnn;
 502                 ctdb_queue_packet(ctdb, hdr);
 503         }
 504 }
 505
 506 /*
 507   broadcast a packet to all nodes in the current vnnmap
 508 */
 509 static void ctdb_broadcast_packet_vnnmap(struct ctdb_context *ctdb,
 510                                          struct ctdb_req_header *hdr)
 511 {
 512         int i;
 513         for (i=0;i<ctdb->vnn_map->size;i++) {
 514                 hdr->destnode = ctdb->vnn_map->map[i];
 515                 ctdb_queue_packet(ctdb, hdr);
 516         }
 517 }
 518
 519 /*
 520   broadcast a packet to all connected nodes
 521 */
 522 static void ctdb_broadcast_packet_connected(struct ctdb_context *ctdb,
 523                                             struct ctdb_req_header *hdr)
 524 {
 525         int i;
 526         for (i=0; i < ctdb->num_nodes; i++) {
 527                 if (ctdb->nodes[i]->flags & NODE_FLAGS_DELETED) {
 528                         continue;
 529                 }
 530                 if (!(ctdb->nodes[i]->flags & NODE_FLAGS_DISCONNECTED)) {
 531                         hdr->destnode = ctdb->nodes[i]->pnn;
 532                         ctdb_queue_packet(ctdb, hdr);
 533                 }
 534         }
 535 }
 536
 537 /*
 538   queue a packet or die
 539 */
 540 void ctdb_queue_packet(struct ctdb_context *ctdb, struct ctdb_req_header *hdr)
 541 {
 542         struct ctdb_node *node;
 543
 544         switch (hdr->destnode) {
 545         case CTDB_BROADCAST_ALL:
 546                 ctdb_broadcast_packet_all(ctdb, hdr);
 547                 return;
 548         case CTDB_BROADCAST_VNNMAP:
 549                 ctdb_broadcast_packet_vnnmap(ctdb, hdr);
 550                 return;
 551         case CTDB_BROADCAST_CONNECTED:
 552                 ctdb_broadcast_packet_connected(ctdb, hdr);
 553                 return;
 554         }
 555
 556         CTDB_INCREMENT_STAT(ctdb, node_packets_sent);
 557
 558         if (!ctdb_validate_pnn(ctdb, hdr->destnode)) {
 559                 DEBUG(DEBUG_CRIT,(__location__ " cant send to node %u that does not exist\n",
 560                          hdr->destnode));
 561                 return;
 562         }
 563
 564         node = ctdb->nodes[hdr->destnode];
 565
 566         if (node->flags & NODE_FLAGS_DELETED) {
 567                 DEBUG(DEBUG_ERR, (__location__ " Can not queue packet to DELETED node %d\n", hdr->destnode));
 568                 return;
 569         }
 570
 571         if (node->pnn == ctdb->pnn) {
 572                 ctdb_defer_packet(ctdb, hdr);
 573                 return;
 574         }
 575
 576         if (ctdb->methods == NULL) {
 577                 DEBUG(DEBUG_ALERT, (__location__ " Can not queue packet. "
 578                                     "Transport is DOWN\n"));
 579                 return;
 580         }
 581
 582         node->tx_cnt++;
 583         if (ctdb->methods->queue_pkt(node, (uint8_t *)hdr, hdr->length) != 0) {
 584                 ctdb_fatal(ctdb, "Unable to queue packet\n");
 585         }
 586 }
 587
 588
 589
 590
 591 /*
 592   a valgrind hack to allow us to get opcode specific backtraces
 593   very ugly, and relies on no compiler optimisation!
 594 */
 595 void ctdb_queue_packet_opcode(struct ctdb_context *ctdb, struct ctdb_req_header *hdr, unsigned opcode)
 596 {
 597         switch (opcode) {
 598 #define DO_OP(x) case x: ctdb_queue_packet(ctdb, hdr); break
 599                 DO_OP(1);
 600                 DO_OP(2);
 601                 DO_OP(3);
 602                 DO_OP(4);
 603                 DO_OP(5);
 604                 DO_OP(6);
 605                 DO_OP(7);
 606                 DO_OP(8);
 607                 DO_OP(9);
 608                 DO_OP(10);
 609                 DO_OP(11);
 610                 DO_OP(12);
 611                 DO_OP(13);
 612                 DO_OP(14);
 613                 DO_OP(15);
 614                 DO_OP(16);
 615                 DO_OP(17);
 616                 DO_OP(18);
 617                 DO_OP(19);
 618                 DO_OP(20);
 619                 DO_OP(21);
 620                 DO_OP(22);
 621                 DO_OP(23);
 622                 DO_OP(24);
 623                 DO_OP(25);
 624                 DO_OP(26);
 625                 DO_OP(27);
 626                 DO_OP(28);
 627                 DO_OP(29);
 628                 DO_OP(30);
 629                 DO_OP(31);
 630                 DO_OP(32);
 631                 DO_OP(33);
 632                 DO_OP(34);
 633                 DO_OP(35);
 634                 DO_OP(36);
 635                 DO_OP(37);
 636                 DO_OP(38);
 637                 DO_OP(39);
 638                 DO_OP(40);
 639                 DO_OP(41);
 640                 DO_OP(42);
 641                 DO_OP(43);
 642                 DO_OP(44);
 643                 DO_OP(45);
 644                 DO_OP(46);
 645                 DO_OP(47);
 646                 DO_OP(48);
 647                 DO_OP(49);
 648                 DO_OP(50);
 649                 DO_OP(51);
 650                 DO_OP(52);
 651                 DO_OP(53);
 652                 DO_OP(54);
 653                 DO_OP(55);
 654                 DO_OP(56);
 655                 DO_OP(57);
 656                 DO_OP(58);
 657                 DO_OP(59);
 658                 DO_OP(60);
 659                 DO_OP(61);
 660                 DO_OP(62);
 661                 DO_OP(63);
 662                 DO_OP(64);
 663                 DO_OP(65);
 664                 DO_OP(66);
 665                 DO_OP(67);
 666                 DO_OP(68);
 667                 DO_OP(69);
 668                 DO_OP(70);
 669                 DO_OP(71);
 670                 DO_OP(72);
 671                 DO_OP(73);
 672                 DO_OP(74);
 673                 DO_OP(75);
 674                 DO_OP(76);
 675                 DO_OP(77);
 676                 DO_OP(78);
 677                 DO_OP(79);
 678                 DO_OP(80);
 679                 DO_OP(81);
 680                 DO_OP(82);
 681                 DO_OP(83);
 682                 DO_OP(84);
 683                 DO_OP(85);
 684                 DO_OP(86);
 685                 DO_OP(87);
 686                 DO_OP(88);
 687                 DO_OP(89);
 688                 DO_OP(90);
 689                 DO_OP(91);
 690                 DO_OP(92);
 691                 DO_OP(93);
 692                 DO_OP(94);
 693                 DO_OP(95);
 694                 DO_OP(96);
 695                 DO_OP(97);
 696                 DO_OP(98);
 697                 DO_OP(99);
 698                 DO_OP(100);
 699         default:
 700                 ctdb_queue_packet(ctdb, hdr);
 701                 break;
 702         }
 703 }