ctdb/ib/ibw_ctdb.c

   1 /*
   2  * Unix SMB/CIFS implementation.
   3  * Join infiniband wrapper and ctdb.
   4  *
   5  * Copyright (C) Sven Oehme <oehmes@de.ibm.com> 2006
   6  *
   7  * Major code contributions by Peter Somogyi <psomogyi@gamax.hu>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 3 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  21  */
  22
  23 #include "replace.h"
  24 #include "system/network.h"
  25
  26 #include <assert.h>
  27 #include <talloc.h>
  28 #include <tevent.h>
  29
  30 #include "lib/util/time.h"
  31 #include "lib/util/debug.h"
  32
  33 #include "ctdb_private.h"
  34
  35 #include "common/common.h"
  36 #include "common/logging.h"
  37
  38 #include "ibwrapper.h"
  39 #include "ibw_ctdb.h"
  40
  41 int ctdb_ibw_get_address(struct ctdb_context *ctdb,
  42         const char *address, struct in_addr *addr)
  43 {
  44         if (inet_pton(AF_INET, address, addr) <= 0) {
  45                 struct hostent *he = gethostbyname(address);
  46                 if (he == NULL || he->h_length > sizeof(*addr)) {
  47                         ctdb_set_error(ctdb, "invalid network address '%s'\n",
  48                                        address);
  49                         return -1;
  50                 }
  51                 memcpy(addr, he->h_addr, he->h_length);
  52         }
  53         return 0;
  54 }
  55
  56 int ctdb_ibw_node_connect(struct ctdb_node *node)
  57 {
  58         struct ctdb_ibw_node *cn = talloc_get_type(node->transport_data,
  59                                                    struct ctdb_ibw_node);
  60         int     rc;
  61
  62         assert(cn!=NULL);
  63         assert(cn->conn!=NULL);
  64
  65         rc = ibw_connect(cn->conn, &node->address.ip, node);
  66         if (rc) {
  67                 DEBUG(DEBUG_ERR, ("ctdb_ibw_node_connect/ibw_connect failed - retrying...\n"));
  68                 /* try again once a second */
  69                 tevent_add_timer(node->ctdb->ev, node,
  70                                  timeval_current_ofs(1, 0),
  71                                  ctdb_ibw_node_connect_event, node);
  72         }
  73
  74         /* continues at ibw_ctdb.c/IBWC_CONNECTED in good case */
  75         return 0;
  76 }
  77
  78 void ctdb_ibw_node_connect_event(struct tevent_context *ev,
  79                                  struct tevent_timer *te,
  80                                  struct timeval t, void *private_data)
  81 {
  82         struct ctdb_node *node = talloc_get_type(private_data, struct ctdb_node);
  83
  84         ctdb_ibw_node_connect(node);
  85 }
  86
  87 int ctdb_ibw_connstate_handler(struct ibw_ctx *ctx, struct ibw_conn *conn)
  88 {
  89         if (ctx!=NULL) {
  90                 /* ctx->state changed */
  91                 switch(ctx->state) {
  92                 case IBWS_INIT: /* ctx start - after ibw_init */
  93                         break;
  94                 case IBWS_READY: /* after ibw_bind & ibw_listen */
  95                         break;
  96                 case IBWS_CONNECT_REQUEST: /* after [IBWS_READY + incoming request] */
  97                                 /* => [(ibw_accept)IBWS_READY | (ibw_disconnect)STOPPED | ERROR] */
  98                         if (ibw_accept(ctx, conn, NULL)) {
  99                                 DEBUG(DEBUG_ERR, ("connstate_handler/ibw_accept failed\n"));
 100                                 return -1;
 101                         } /* else continue in IBWC_CONNECTED */
 102                         break;
 103                 case IBWS_STOPPED: /* normal stop <= ibw_disconnect+(IBWS_READY | IBWS_CONNECT_REQUEST) */
 104                         /* TODO: have a CTDB upcall for which CTDB should wait in a (final) loop */
 105                         break;
 106                 case IBWS_ERROR: /* abnormal state; ibw_stop must be called after this */
 107                         break;
 108                 default:
 109                         assert(0);
 110                         break;
 111                 }
 112         }
 113
 114         if (conn!=NULL) {
 115                 /* conn->state changed */
 116                 switch(conn->state) {
 117                 case IBWC_INIT: /* conn start - internal state */
 118                         break;
 119                 case IBWC_CONNECTED: { /* after ibw_accept or ibw_connect */
 120                         struct ctdb_node *node = talloc_get_type(conn->conn_userdata, struct ctdb_node);
 121                         if (node!=NULL) { /* after ibw_connect */
 122                                 struct ctdb_ibw_node *cn = talloc_get_type(
 123                                         node->transport_data,
 124                                         struct ctdb_ibw_node);
 125
 126                                 node->ctdb->upcalls->node_connected(node);
 127                                 ctdb_flush_cn_queue(cn);
 128                         } else { /* after ibw_accept */
 129                                 /* NOP in CTDB case */
 130                         }
 131                 } break;
 132                 case IBWC_DISCONNECTED: { /* after ibw_disconnect */
 133                         struct ctdb_node *node = talloc_get_type(conn->conn_userdata, struct ctdb_node);
 134                         if (node!=NULL)
 135                                 node->ctdb->upcalls->node_dead(node);
 136                         talloc_free(conn);
 137                         /* normal + intended disconnect => not reconnecting in this layer */
 138                 } break;
 139                 case IBWC_ERROR: {
 140                         struct ctdb_node *node = talloc_get_type(conn->conn_userdata, struct ctdb_node);
 141                         if (node!=NULL) {
 142                                 struct ctdb_ibw_node *cn = talloc_get_type(
 143                                         node->transport_data,
 144                                         struct ctdb_ibw_node);
 145                                 struct ibw_ctx *ictx = cn->conn->ctx;
 146
 147                                 DEBUG(DEBUG_DEBUG, ("IBWC_ERROR, reconnecting...\n"));
 148                                 talloc_free(cn->conn); /* internal queue content is destroyed */
 149                                 cn->conn = (void *)ibw_conn_new(ictx, node);
 150                                 tevent_add_timer(node->ctdb->ev, node,
 151                                                  timeval_current_ofs(1, 0),
 152                                                  ctdb_ibw_node_connect_event, node);
 153                         }
 154                 } break;
 155                 default:
 156                         assert(0);
 157                         break;
 158                 }
 159         }
 160
 161         return 0;
 162 }
 163
 164 int ctdb_ibw_receive_handler(struct ibw_conn *conn, void *buf, int n)
 165 {
 166         struct ctdb_context *ctdb = talloc_get_type(conn->ctx->ctx_userdata, struct ctdb_context);
 167         void    *buf2; /* future TODO: a solution for removal of this */
 168
 169         assert(ctdb!=NULL);
 170         assert(buf!=NULL);
 171         assert(conn!=NULL);
 172         assert(conn->state==IBWC_CONNECTED);
 173
 174         /* so far "buf" is an ib-registered memory area
 175          * and being reused for next receive
 176          * noticed that HL requires talloc-ed memory to be stolen */
 177         buf2 = talloc_zero_size(conn, n);
 178         CTDB_NO_MEMORY(ctdb, buf2);
 179
 180         memcpy(buf2, buf, n);
 181
 182         ctdb->upcalls->recv_pkt(ctdb, (uint8_t *)buf2, (uint32_t)n);
 183
 184         return 0;
 185 }