2 ctdb main protocol code
4 Copyright (C) Andrew Tridgell 2006
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 3 of the License, or
9 (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, see <http://www.gnu.org/licenses/>.
22 #include "lib/util/dlinklist.h"
23 #include "system/network.h"
24 #include "system/filesys.h"
25 #include "../include/ctdb_private.h"
28 choose the transport we will use
30 int ctdb_set_transport(struct ctdb_context
*ctdb
, const char *transport
)
32 ctdb
->transport
= talloc_strdup(ctdb
, transport
);
33 CTDB_NO_MEMORY(ctdb
, ctdb
->transport
);
39 Check whether an ip is a valid node ip
40 Returns the node id for this ip address or -1
42 int ctdb_ip_to_nodeid(struct ctdb_context
*ctdb
, const char *nodeip
)
46 for (nodeid
=0;nodeid
<ctdb
->num_nodes
;nodeid
++) {
47 if (ctdb
->nodes
[nodeid
]->flags
& NODE_FLAGS_DELETED
) {
50 if (!strcmp(ctdb
->nodes
[nodeid
]->address
.address
, nodeip
)) {
59 choose the recovery lock file
61 int ctdb_set_recovery_lock_file(struct ctdb_context
*ctdb
, const char *file
)
63 if (ctdb
->recovery_lock_file
!= NULL
) {
64 talloc_free(ctdb
->recovery_lock_file
);
65 ctdb
->recovery_lock_file
= NULL
;
69 DEBUG(DEBUG_ALERT
,("Recovery lock file set to \"\". Disabling recovery lock checking\n"));
70 ctdb
->tunable
.verify_recovery_lock
= 0;
74 ctdb
->recovery_lock_file
= talloc_strdup(ctdb
, file
);
75 CTDB_NO_MEMORY(ctdb
, ctdb
->recovery_lock_file
);
81 add a node to the list of nodes
83 static int ctdb_add_node(struct ctdb_context
*ctdb
, char *nstr
)
85 struct ctdb_node
*node
, **nodep
;
87 nodep
= talloc_realloc(ctdb
, ctdb
->nodes
, struct ctdb_node
*, ctdb
->num_nodes
+1);
88 CTDB_NO_MEMORY(ctdb
, nodep
);
91 nodep
= &ctdb
->nodes
[ctdb
->num_nodes
];
92 (*nodep
) = talloc_zero(ctdb
->nodes
, struct ctdb_node
);
93 CTDB_NO_MEMORY(ctdb
, *nodep
);
96 if (ctdb_parse_address(ctdb
, node
, nstr
, &node
->address
) != 0) {
100 node
->name
= talloc_asprintf(node
, "%s:%u",
101 node
->address
.address
,
103 /* this assumes that the nodes are kept in sorted order, and no gaps */
104 node
->pnn
= ctdb
->num_nodes
;
106 /* nodes start out disconnected and unhealthy */
107 node
->flags
= (NODE_FLAGS_DISCONNECTED
| NODE_FLAGS_UNHEALTHY
);
109 if (ctdb
->address
.address
&&
110 ctdb_same_address(&ctdb
->address
, &node
->address
)) {
111 /* for automatic binding to interfaces, see tcp_connect.c */
112 ctdb
->pnn
= node
->pnn
;
116 node
->dead_count
= 0;
122 add an entry for a "deleted" node to the list of nodes.
123 a "deleted" node is a node that is commented out from the nodes file.
124 this is used to prevent that subsequent nodes in the nodes list
125 change their pnn value if a node is "delete" by commenting it out and then
126 using "ctdb reloadnodes" at runtime.
128 static int ctdb_add_deleted_node(struct ctdb_context
*ctdb
)
130 struct ctdb_node
*node
, **nodep
;
132 nodep
= talloc_realloc(ctdb
, ctdb
->nodes
, struct ctdb_node
*, ctdb
->num_nodes
+1);
133 CTDB_NO_MEMORY(ctdb
, nodep
);
136 nodep
= &ctdb
->nodes
[ctdb
->num_nodes
];
137 (*nodep
) = talloc_zero(ctdb
->nodes
, struct ctdb_node
);
138 CTDB_NO_MEMORY(ctdb
, *nodep
);
141 if (ctdb_parse_address(ctdb
, node
, "0.0.0.0", &node
->address
) != 0) {
142 DEBUG(DEBUG_ERR
,("Failed to setup deleted node %d\n", ctdb
->num_nodes
));
146 node
->name
= talloc_strdup(node
, "0.0.0.0:0");
148 /* this assumes that the nodes are kept in sorted order, and no gaps */
149 node
->pnn
= ctdb
->num_nodes
;
151 /* this node is permanently deleted/disconnected */
152 node
->flags
= NODE_FLAGS_DELETED
|NODE_FLAGS_DISCONNECTED
;
155 node
->dead_count
= 0;
162 setup the node list from a file
164 static int ctdb_set_nlist(struct ctdb_context
*ctdb
, const char *nlist
)
168 int i
, j
, num_present
;
170 talloc_free(ctdb
->nodes
);
174 lines
= file_lines_load(nlist
, &nlines
, 0, ctdb
);
176 ctdb_set_error(ctdb
, "Failed to load nlist '%s'\n", nlist
);
179 while (nlines
> 0 && strcmp(lines
[nlines
-1], "") == 0) {
184 for (i
=0; i
< nlines
; i
++) {
189 /* strip leading spaces */
190 while((*node
== ' ') || (*node
== '\t')) {
194 if (ctdb_add_deleted_node(ctdb
) != 0) {
201 /* strip trailing spaces */
206 ((node
[len
-1] == ' ') || (node
[len
-1] == '\t')))
215 if (ctdb_add_node(ctdb
, node
) != 0) {
222 /* initialize the vnn mapping table now that we have the nodes list,
223 skipping any deleted nodes
225 ctdb
->vnn_map
= talloc(ctdb
, struct ctdb_vnn_map
);
226 CTDB_NO_MEMORY(ctdb
, ctdb
->vnn_map
);
228 ctdb
->vnn_map
->generation
= INVALID_GENERATION
;
229 ctdb
->vnn_map
->size
= num_present
;
230 ctdb
->vnn_map
->map
= talloc_array(ctdb
->vnn_map
, uint32_t, ctdb
->vnn_map
->size
);
231 CTDB_NO_MEMORY(ctdb
, ctdb
->vnn_map
->map
);
233 for(i
=0, j
=0; i
< ctdb
->vnn_map
->size
; i
++) {
234 if (ctdb
->nodes
[i
]->flags
& NODE_FLAGS_DELETED
) {
237 ctdb
->vnn_map
->map
[j
] = i
;
245 void ctdb_load_nodes_file(struct ctdb_context
*ctdb
)
249 ret
= ctdb_set_nlist(ctdb
, ctdb
->nodes_file
);
251 DEBUG(DEBUG_ALERT
,("ctdb_set_nlist failed - %s\n", ctdb_errstr(ctdb
)));
257 setup the local node address
259 int ctdb_set_address(struct ctdb_context
*ctdb
, const char *address
)
261 if (ctdb_parse_address(ctdb
, ctdb
, address
, &ctdb
->address
) != 0) {
265 ctdb
->name
= talloc_asprintf(ctdb
, "%s:%u",
266 ctdb
->address
.address
,
273 return the number of active nodes
275 uint32_t ctdb_get_num_active_nodes(struct ctdb_context
*ctdb
)
279 for (i
=0; i
< ctdb
->num_nodes
; i
++) {
280 if (!(ctdb
->nodes
[i
]->flags
& NODE_FLAGS_INACTIVE
)) {
289 called when we need to process a packet. This can be a requeued packet
290 after a lockwait, or a real packet from another node
292 void ctdb_input_pkt(struct ctdb_context
*ctdb
, struct ctdb_req_header
*hdr
)
296 /* place the packet as a child of the tmp_ctx. We then use
297 talloc_free() below to free it. If any of the calls want
298 to keep it, then they will steal it somewhere else, and the
299 talloc_free() will only free the tmp_ctx */
300 tmp_ctx
= talloc_new(ctdb
);
301 talloc_steal(tmp_ctx
, hdr
);
303 DEBUG(DEBUG_DEBUG
,(__location__
" ctdb request %u of type %u length %u from "
304 "node %u to %u\n", hdr
->reqid
, hdr
->operation
, hdr
->length
,
305 hdr
->srcnode
, hdr
->destnode
));
307 switch (hdr
->operation
) {
309 case CTDB_REPLY_CALL
:
310 case CTDB_REQ_DMASTER
:
311 case CTDB_REPLY_DMASTER
:
312 /* we dont allow these calls when banned */
313 if (ctdb
->nodes
[ctdb
->pnn
]->flags
& NODE_FLAGS_BANNED
) {
314 DEBUG(DEBUG_DEBUG
,(__location__
" ctdb operation %u"
316 " length %u from node %u to %u while node"
318 hdr
->operation
, hdr
->reqid
,
320 hdr
->srcnode
, hdr
->destnode
));
324 /* for ctdb_call inter-node operations verify that the
325 remote node that sent us the call is running in the
326 same generation instance as this node
328 if (ctdb
->vnn_map
->generation
!= hdr
->generation
) {
329 DEBUG(DEBUG_DEBUG
,(__location__
" ctdb operation %u"
331 " length %u from node %u to %u had an"
332 " invalid generation id:%u while our"
333 " generation id is:%u\n",
334 hdr
->operation
, hdr
->reqid
,
336 hdr
->srcnode
, hdr
->destnode
,
337 hdr
->generation
, ctdb
->vnn_map
->generation
));
342 switch (hdr
->operation
) {
344 CTDB_INCREMENT_STAT(ctdb
, node
.req_call
);
345 ctdb_request_call(ctdb
, hdr
);
348 case CTDB_REPLY_CALL
:
349 CTDB_INCREMENT_STAT(ctdb
, node
.reply_call
);
350 ctdb_reply_call(ctdb
, hdr
);
353 case CTDB_REPLY_ERROR
:
354 CTDB_INCREMENT_STAT(ctdb
, node
.reply_error
);
355 ctdb_reply_error(ctdb
, hdr
);
358 case CTDB_REQ_DMASTER
:
359 CTDB_INCREMENT_STAT(ctdb
, node
.req_dmaster
);
360 ctdb_request_dmaster(ctdb
, hdr
);
363 case CTDB_REPLY_DMASTER
:
364 CTDB_INCREMENT_STAT(ctdb
, node
.reply_dmaster
);
365 ctdb_reply_dmaster(ctdb
, hdr
);
368 case CTDB_REQ_MESSAGE
:
369 CTDB_INCREMENT_STAT(ctdb
, node
.req_message
);
370 ctdb_request_message(ctdb
, hdr
);
373 case CTDB_REQ_CONTROL
:
374 CTDB_INCREMENT_STAT(ctdb
, node
.req_control
);
375 ctdb_request_control(ctdb
, hdr
);
378 case CTDB_REPLY_CONTROL
:
379 CTDB_INCREMENT_STAT(ctdb
, node
.reply_control
);
380 ctdb_reply_control(ctdb
, hdr
);
383 case CTDB_REQ_KEEPALIVE
:
384 CTDB_INCREMENT_STAT(ctdb
, keepalive_packets_recv
);
388 DEBUG(DEBUG_CRIT
,("%s: Packet with unknown operation %u\n",
389 __location__
, hdr
->operation
));
394 talloc_free(tmp_ctx
);
399 called by the transport layer when a node is dead
401 void ctdb_node_dead(struct ctdb_node
*node
)
403 if (node
->flags
& NODE_FLAGS_DISCONNECTED
) {
404 DEBUG(DEBUG_INFO
,("%s: node %s is already marked disconnected: %u connected\n",
405 node
->ctdb
->name
, node
->name
,
406 node
->ctdb
->num_connected
));
409 node
->ctdb
->num_connected
--;
410 node
->flags
|= NODE_FLAGS_DISCONNECTED
| NODE_FLAGS_UNHEALTHY
;
412 node
->dead_count
= 0;
414 DEBUG(DEBUG_NOTICE
,("%s: node %s is dead: %u connected\n",
415 node
->ctdb
->name
, node
->name
, node
->ctdb
->num_connected
));
416 ctdb_daemon_cancel_controls(node
->ctdb
, node
);
418 if (node
->ctdb
->methods
== NULL
) {
419 DEBUG(DEBUG_ERR
,(__location__
" Can not restart transport while shutting down daemon.\n"));
423 node
->ctdb
->methods
->restart(node
);
427 called by the transport layer when a node is connected
429 void ctdb_node_connected(struct ctdb_node
*node
)
431 if (!(node
->flags
& NODE_FLAGS_DISCONNECTED
)) {
432 DEBUG(DEBUG_INFO
,("%s: node %s is already marked connected: %u connected\n",
433 node
->ctdb
->name
, node
->name
,
434 node
->ctdb
->num_connected
));
437 node
->ctdb
->num_connected
++;
438 node
->dead_count
= 0;
439 node
->flags
&= ~NODE_FLAGS_DISCONNECTED
;
440 node
->flags
|= NODE_FLAGS_UNHEALTHY
;
442 ("%s: connected to %s - %u connected\n",
443 node
->ctdb
->name
, node
->name
, node
->ctdb
->num_connected
));
447 struct ctdb_context
*ctdb
;
448 struct ctdb_req_header
*hdr
;
453 triggered when a deferred packet is due
455 static void queue_next_trigger(struct event_context
*ev
, struct timed_event
*te
,
456 struct timeval t
, void *private_data
)
458 struct queue_next
*q
= talloc_get_type(private_data
, struct queue_next
);
459 ctdb_input_pkt(q
->ctdb
, q
->hdr
);
464 defer a packet, so it is processed on the next event loop
465 this is used for sending packets to ourselves
467 static void ctdb_defer_packet(struct ctdb_context
*ctdb
, struct ctdb_req_header
*hdr
)
469 struct queue_next
*q
;
470 q
= talloc(ctdb
, struct queue_next
);
472 DEBUG(DEBUG_ERR
,(__location__
" Failed to allocate deferred packet\n"));
476 q
->hdr
= talloc_memdup(ctdb
, hdr
, hdr
->length
);
477 if (q
->hdr
== NULL
) {
478 DEBUG(DEBUG_ERR
,("Error copying deferred packet to self\n"));
482 /* use this to put packets directly into our recv function */
483 ctdb_input_pkt(q
->ctdb
, q
->hdr
);
485 event_add_timed(ctdb
->ev
, q
, timeval_zero(), queue_next_trigger
, q
);
491 broadcast a packet to all nodes
493 static void ctdb_broadcast_packet_all(struct ctdb_context
*ctdb
,
494 struct ctdb_req_header
*hdr
)
497 for (i
=0; i
< ctdb
->num_nodes
; i
++) {
498 if (ctdb
->nodes
[i
]->flags
& NODE_FLAGS_DELETED
) {
501 hdr
->destnode
= ctdb
->nodes
[i
]->pnn
;
502 ctdb_queue_packet(ctdb
, hdr
);
507 broadcast a packet to all nodes in the current vnnmap
509 static void ctdb_broadcast_packet_vnnmap(struct ctdb_context
*ctdb
,
510 struct ctdb_req_header
*hdr
)
513 for (i
=0;i
<ctdb
->vnn_map
->size
;i
++) {
514 hdr
->destnode
= ctdb
->vnn_map
->map
[i
];
515 ctdb_queue_packet(ctdb
, hdr
);
520 broadcast a packet to all connected nodes
522 static void ctdb_broadcast_packet_connected(struct ctdb_context
*ctdb
,
523 struct ctdb_req_header
*hdr
)
526 for (i
=0; i
< ctdb
->num_nodes
; i
++) {
527 if (ctdb
->nodes
[i
]->flags
& NODE_FLAGS_DELETED
) {
530 if (!(ctdb
->nodes
[i
]->flags
& NODE_FLAGS_DISCONNECTED
)) {
531 hdr
->destnode
= ctdb
->nodes
[i
]->pnn
;
532 ctdb_queue_packet(ctdb
, hdr
);
538 queue a packet or die
540 void ctdb_queue_packet(struct ctdb_context
*ctdb
, struct ctdb_req_header
*hdr
)
542 struct ctdb_node
*node
;
544 switch (hdr
->destnode
) {
545 case CTDB_BROADCAST_ALL
:
546 ctdb_broadcast_packet_all(ctdb
, hdr
);
548 case CTDB_BROADCAST_VNNMAP
:
549 ctdb_broadcast_packet_vnnmap(ctdb
, hdr
);
551 case CTDB_BROADCAST_CONNECTED
:
552 ctdb_broadcast_packet_connected(ctdb
, hdr
);
556 CTDB_INCREMENT_STAT(ctdb
, node_packets_sent
);
558 if (!ctdb_validate_pnn(ctdb
, hdr
->destnode
)) {
559 DEBUG(DEBUG_CRIT
,(__location__
" cant send to node %u that does not exist\n",
564 node
= ctdb
->nodes
[hdr
->destnode
];
566 if (node
->flags
& NODE_FLAGS_DELETED
) {
567 DEBUG(DEBUG_ERR
, (__location__
" Can not queue packet to DELETED node %d\n", hdr
->destnode
));
571 if (node
->pnn
== ctdb
->pnn
) {
572 ctdb_defer_packet(ctdb
, hdr
);
576 if (ctdb
->methods
== NULL
) {
577 DEBUG(DEBUG_ALERT
, (__location__
" Can not queue packet. "
578 "Transport is DOWN\n"));
583 if (ctdb
->methods
->queue_pkt(node
, (uint8_t *)hdr
, hdr
->length
) != 0) {
584 ctdb_fatal(ctdb
, "Unable to queue packet\n");
592 a valgrind hack to allow us to get opcode specific backtraces
593 very ugly, and relies on no compiler optimisation!
595 void ctdb_queue_packet_opcode(struct ctdb_context
*ctdb
, struct ctdb_req_header
*hdr
, unsigned opcode
)
598 #define DO_OP(x) case x: ctdb_queue_packet(ctdb, hdr); break
700 ctdb_queue_packet(ctdb
, hdr
);