4 Copyright (C) Andrew Tridgell 2006
5 Copyright (C) Ronnie Sahlberg 2008
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3 of the License, or
10 (at your option) any later version.
12 This program is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with this program; if not, see <http://www.gnu.org/licenses/>.
22 #include "system/network.h"
23 #include "system/filesys.h"
28 #include "lib/util/debug.h"
29 #include "lib/util/time.h"
31 #include "ctdb_private.h"
33 #include "common/system.h"
34 #include "common/common.h"
35 #include "common/logging.h"
40 stop any connecting (established or pending) to a node
42 void ctdb_tcp_stop_connection(struct ctdb_node
*node
)
44 struct ctdb_tcp_node
*tnode
= talloc_get_type(
45 node
->private_data
, struct ctdb_tcp_node
);
47 ctdb_queue_set_fd(tnode
->out_queue
, -1);
48 talloc_free(tnode
->connect_te
);
49 talloc_free(tnode
->connect_fde
);
50 tnode
->connect_fde
= NULL
;
51 tnode
->connect_te
= NULL
;
52 if (tnode
->fd
!= -1) {
60 called when a complete packet has come in - should not happen on this socket
61 unless the other side closes the connection with RST or FIN
63 void ctdb_tcp_tnode_cb(uint8_t *data
, size_t cnt
, void *private_data
)
65 struct ctdb_node
*node
= talloc_get_type(private_data
, struct ctdb_node
);
66 struct ctdb_tcp_node
*tnode
= talloc_get_type(
67 node
->private_data
, struct ctdb_tcp_node
);
70 node
->ctdb
->upcalls
->node_dead(node
);
73 ctdb_tcp_stop_connection(node
);
74 tnode
->connect_te
= tevent_add_timer(node
->ctdb
->ev
, tnode
,
75 timeval_current_ofs(3, 0),
76 ctdb_tcp_node_connect
, node
);
80 called when socket becomes writeable on connect
82 static void ctdb_node_connect_write(struct tevent_context
*ev
,
83 struct tevent_fd
*fde
,
84 uint16_t flags
, void *private_data
)
86 struct ctdb_node
*node
= talloc_get_type(private_data
,
88 struct ctdb_tcp_node
*tnode
= talloc_get_type(node
->private_data
,
89 struct ctdb_tcp_node
);
90 struct ctdb_context
*ctdb
= node
->ctdb
;
92 socklen_t len
= sizeof(error
);
95 talloc_free(tnode
->connect_te
);
96 tnode
->connect_te
= NULL
;
98 if (getsockopt(tnode
->fd
, SOL_SOCKET
, SO_ERROR
, &error
, &len
) != 0 ||
100 ctdb_tcp_stop_connection(node
);
101 tnode
->connect_te
= tevent_add_timer(ctdb
->ev
, tnode
,
102 timeval_current_ofs(1, 0),
103 ctdb_tcp_node_connect
, node
);
107 talloc_free(tnode
->connect_fde
);
108 tnode
->connect_fde
= NULL
;
110 if (setsockopt(tnode
->fd
,IPPROTO_TCP
,TCP_NODELAY
,(char *)&one
,sizeof(one
)) == -1) {
111 DEBUG(DEBUG_WARNING
, ("Failed to set TCP_NODELAY on fd - %s\n",
114 if (setsockopt(tnode
->fd
,SOL_SOCKET
,SO_KEEPALIVE
,(char *)&one
,sizeof(one
)) == -1) {
115 DEBUG(DEBUG_WARNING
, ("Failed to set KEEPALIVE on fd - %s\n",
119 ctdb_queue_set_fd(tnode
->out_queue
, tnode
->fd
);
121 /* the queue subsystem now owns this fd */
127 called when we should try and establish a tcp connection to a node
129 void ctdb_tcp_node_connect(struct tevent_context
*ev
, struct tevent_timer
*te
,
130 struct timeval t
, void *private_data
)
132 struct ctdb_node
*node
= talloc_get_type(private_data
,
134 struct ctdb_tcp_node
*tnode
= talloc_get_type(node
->private_data
,
135 struct ctdb_tcp_node
);
136 struct ctdb_context
*ctdb
= node
->ctdb
;
137 ctdb_sock_addr sock_in
;
140 ctdb_sock_addr sock_out
;
142 ctdb_tcp_stop_connection(node
);
144 sock_out
= node
->address
;
146 tnode
->fd
= socket(sock_out
.sa
.sa_family
, SOCK_STREAM
, IPPROTO_TCP
);
147 if (tnode
->fd
== -1) {
148 DEBUG(DEBUG_ERR
, (__location__
"Failed to create socket\n"));
151 set_nonblocking(tnode
->fd
);
152 set_close_on_exec(tnode
->fd
);
154 DEBUG(DEBUG_DEBUG
, (__location__
" Created TCP SOCKET FD:%d\n", tnode
->fd
));
156 /* Bind our side of the socketpair to the same address we use to listen
157 * on incoming CTDB traffic.
158 * We must specify this address to make sure that the address we expose to
159 * the remote side is actually routable in case CTDB traffic will run on
160 * a dedicated non-routeable network.
162 sock_in
= *ctdb
->address
;
164 /* AIX libs check to see if the socket address and length
165 arguments are consistent with each other on calls like
166 connect(). Can not get by with just sizeof(sock_in),
167 need sizeof(sock_in.ip).
169 switch (sock_in
.sa
.sa_family
) {
171 sock_in
.ip
.sin_port
= 0 /* Any port */;
172 sockin_size
= sizeof(sock_in
.ip
);
173 sockout_size
= sizeof(sock_out
.ip
);
176 sock_in
.ip6
.sin6_port
= 0 /* Any port */;
177 sockin_size
= sizeof(sock_in
.ip6
);
178 sockout_size
= sizeof(sock_out
.ip6
);
181 DEBUG(DEBUG_ERR
, (__location__
" unknown family %u\n",
182 sock_in
.sa
.sa_family
));
187 if (bind(tnode
->fd
, (struct sockaddr
*)&sock_in
, sockin_size
) == -1) {
188 DEBUG(DEBUG_ERR
, (__location__
"Failed to bind socket %s(%d)\n",
189 strerror(errno
), errno
));
194 if (connect(tnode
->fd
, (struct sockaddr
*)&sock_out
, sockout_size
) != 0 &&
195 errno
!= EINPROGRESS
) {
196 ctdb_tcp_stop_connection(node
);
197 tnode
->connect_te
= tevent_add_timer(ctdb
->ev
, tnode
,
198 timeval_current_ofs(1, 0),
199 ctdb_tcp_node_connect
, node
);
203 /* non-blocking connect - wait for write event */
204 tnode
->connect_fde
= tevent_add_fd(node
->ctdb
->ev
, tnode
, tnode
->fd
,
205 TEVENT_FD_WRITE
|TEVENT_FD_READ
,
206 ctdb_node_connect_write
, node
);
208 /* don't give it long to connect - retry in one second. This ensures
209 that we find a node is up quickly (tcp normally backs off a syn reply
210 delay by quite a lot) */
211 tnode
->connect_te
= tevent_add_timer(ctdb
->ev
, tnode
,
212 timeval_current_ofs(1, 0),
213 ctdb_tcp_node_connect
, node
);
217 called when we get contacted by another node
218 currently makes no attempt to check if the connection is really from a ctdb
221 static void ctdb_listen_event(struct tevent_context
*ev
, struct tevent_fd
*fde
,
222 uint16_t flags
, void *private_data
)
224 struct ctdb_context
*ctdb
= talloc_get_type(private_data
, struct ctdb_context
);
225 struct ctdb_tcp
*ctcp
= talloc_get_type(ctdb
->private_data
, struct ctdb_tcp
);
229 struct ctdb_incoming
*in
;
232 memset(&addr
, 0, sizeof(addr
));
234 fd
= accept(ctcp
->listen_fd
, (struct sockaddr
*)&addr
, &len
);
235 if (fd
== -1) return;
237 nodeid
= ctdb_ip_to_nodeid(ctdb
, &addr
);
240 DEBUG(DEBUG_ERR
, ("Refused connection from unknown node %s\n", ctdb_addr_to_str(&addr
)));
245 in
= talloc_zero(ctcp
, struct ctdb_incoming
);
249 set_nonblocking(in
->fd
);
250 set_close_on_exec(in
->fd
);
252 DEBUG(DEBUG_DEBUG
, (__location__
" Created SOCKET FD:%d to incoming ctdb connection\n", fd
));
254 if (setsockopt(in
->fd
,SOL_SOCKET
,SO_KEEPALIVE
,(char *)&one
,sizeof(one
)) == -1) {
255 DEBUG(DEBUG_WARNING
, ("Failed to set KEEPALIVE on fd - %s\n",
259 in
->queue
= ctdb_queue_setup(ctdb
, in
, in
->fd
, CTDB_TCP_ALIGNMENT
,
260 ctdb_tcp_read_cb
, in
, "ctdbd-%s", ctdb_addr_to_str(&addr
));
265 automatically find which address to listen on
267 static int ctdb_tcp_listen_automatic(struct ctdb_context
*ctdb
)
269 struct ctdb_tcp
*ctcp
= talloc_get_type(ctdb
->private_data
,
273 const char *lock_path
= CTDB_RUNDIR
"/.socket_lock";
277 struct tevent_fd
*fde
;
279 /* If there are no nodes, then it won't be possible to find
280 * the first one. Log a failure and short circuit the whole
283 if (ctdb
->num_nodes
== 0) {
284 DEBUG(DEBUG_CRIT
,("No nodes available to attempt bind to - is the nodes file empty?\n"));
288 /* in order to ensure that we don't get two nodes with the
289 same adddress, we must make the bind() and listen() calls
290 atomic. The SO_REUSEADDR setsockopt only prevents double
291 binds if the first socket is in LISTEN state */
292 lock_fd
= open(lock_path
, O_RDWR
|O_CREAT
, 0666);
294 DEBUG(DEBUG_CRIT
,("Unable to open %s\n", lock_path
));
298 lock
.l_type
= F_WRLCK
;
299 lock
.l_whence
= SEEK_SET
;
304 if (fcntl(lock_fd
, F_SETLKW
, &lock
) != 0) {
305 DEBUG(DEBUG_CRIT
,("Unable to lock %s\n", lock_path
));
310 for (i
=0; i
< ctdb
->num_nodes
; i
++) {
311 if (ctdb
->nodes
[i
]->flags
& NODE_FLAGS_DELETED
) {
314 sock
= ctdb
->nodes
[i
]->address
;
316 switch (sock
.sa
.sa_family
) {
318 sock_size
= sizeof(sock
.ip
);
321 sock_size
= sizeof(sock
.ip6
);
324 DEBUG(DEBUG_ERR
, (__location__
" unknown family %u\n",
329 ctcp
->listen_fd
= socket(sock
.sa
.sa_family
, SOCK_STREAM
, IPPROTO_TCP
);
330 if (ctcp
->listen_fd
== -1) {
331 ctdb_set_error(ctdb
, "socket failed\n");
335 set_close_on_exec(ctcp
->listen_fd
);
337 if (setsockopt(ctcp
->listen_fd
,SOL_SOCKET
,SO_REUSEADDR
,
338 (char *)&one
,sizeof(one
)) == -1) {
339 DEBUG(DEBUG_WARNING
, ("Failed to set REUSEADDR on fd - %s\n",
343 if (bind(ctcp
->listen_fd
, (struct sockaddr
* )&sock
, sock_size
) == 0) {
347 if (errno
== EADDRNOTAVAIL
) {
348 DEBUG(DEBUG_DEBUG
,(__location__
" Failed to bind() to socket. %s(%d)\n",
349 strerror(errno
), errno
));
351 DEBUG(DEBUG_ERR
,(__location__
" Failed to bind() to socket. %s(%d)\n",
352 strerror(errno
), errno
));
355 close(ctcp
->listen_fd
);
356 ctcp
->listen_fd
= -1;
359 if (i
== ctdb
->num_nodes
) {
360 DEBUG(DEBUG_CRIT
,("Unable to bind to any of the node addresses - giving up\n"));
363 ctdb
->address
= talloc_memdup(ctdb
,
364 &ctdb
->nodes
[i
]->address
,
365 sizeof(ctdb_sock_addr
));
366 if (ctdb
->address
== NULL
) {
367 ctdb_set_error(ctdb
, "Out of memory at %s:%d",
372 ctdb
->name
= talloc_asprintf(ctdb
, "%s:%u",
373 ctdb_addr_to_str(ctdb
->address
),
374 ctdb_addr_to_port(ctdb
->address
));
375 if (ctdb
->name
== NULL
) {
376 ctdb_set_error(ctdb
, "Out of memory at %s:%d",
380 DEBUG(DEBUG_INFO
,("ctdb chose network address %s\n", ctdb
->name
));
382 if (listen(ctcp
->listen_fd
, 10) == -1) {
386 fde
= tevent_add_fd(ctdb
->ev
, ctcp
, ctcp
->listen_fd
, TEVENT_FD_READ
,
387 ctdb_listen_event
, ctdb
);
388 tevent_fd_set_auto_close(fde
);
396 if (ctcp
->listen_fd
!= -1) {
397 close(ctcp
->listen_fd
);
398 ctcp
->listen_fd
= -1;
405 listen on our own address
407 int ctdb_tcp_listen(struct ctdb_context
*ctdb
)
409 struct ctdb_tcp
*ctcp
= talloc_get_type(ctdb
->private_data
,
414 struct tevent_fd
*fde
;
416 /* we can either auto-bind to the first available address, or we can
417 use a specified address */
418 if (!ctdb
->address
) {
419 return ctdb_tcp_listen_automatic(ctdb
);
422 sock
= *ctdb
->address
;
424 switch (sock
.sa
.sa_family
) {
426 sock_size
= sizeof(sock
.ip
);
429 sock_size
= sizeof(sock
.ip6
);
432 DEBUG(DEBUG_ERR
, (__location__
" unknown family %u\n",
437 ctcp
->listen_fd
= socket(sock
.sa
.sa_family
, SOCK_STREAM
, IPPROTO_TCP
);
438 if (ctcp
->listen_fd
== -1) {
439 ctdb_set_error(ctdb
, "socket failed\n");
443 set_close_on_exec(ctcp
->listen_fd
);
445 if (setsockopt(ctcp
->listen_fd
,SOL_SOCKET
,SO_REUSEADDR
,(char *)&one
,sizeof(one
)) == -1) {
446 DEBUG(DEBUG_WARNING
, ("Failed to set REUSEADDR on fd - %s\n",
450 if (bind(ctcp
->listen_fd
, (struct sockaddr
* )&sock
, sock_size
) != 0) {
451 DEBUG(DEBUG_ERR
,(__location__
" Failed to bind() to socket. %s(%d)\n", strerror(errno
), errno
));
455 if (listen(ctcp
->listen_fd
, 10) == -1) {
459 fde
= tevent_add_fd(ctdb
->ev
, ctcp
, ctcp
->listen_fd
, TEVENT_FD_READ
,
460 ctdb_listen_event
, ctdb
);
461 tevent_fd_set_auto_close(fde
);
466 if (ctcp
->listen_fd
!= -1) {
467 close(ctcp
->listen_fd
);
469 ctcp
->listen_fd
= -1;