ctdb/ib/ibw_ctdb_init.c

   1 /*
   2  * Unix SMB/CIFS implementation.
   3  * Join infiniband wrapper and ctdb.
   4  *
   5  * Copyright (C) Sven Oehme <oehmes@de.ibm.com> 2006
   6  *
   7  * Major code contributions by Peter Somogyi <psomogyi@gamax.hu>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 3 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  21  */
  22
  23 #include "includes.h"
  24 #include <system/network.h>
  25 #include <assert.h>
  26 #include "ctdb_private.h"
  27 #include "ibwrapper.h"
  28 #include "ibw_ctdb.h"
  29 #include "lib/util/dlinklist.h"
  30
  31 static int ctdb_ibw_listen(struct ctdb_context *ctdb, int backlog)
  32 {
  33         struct ibw_ctx *ictx = talloc_get_type(ctdb->private_data, struct ibw_ctx);
  34         struct sockaddr_in my_addr;
  35
  36         assert(ictx!=NULL);
  37         memset(&my_addr, 0, sizeof(struct sockaddr_in));
  38         my_addr.sin_port = htons(ctdb->address.port);
  39         my_addr.sin_family = PF_INET;
  40         if (ctdb_ibw_get_address(ctdb, ctdb->address.address, &my_addr.sin_addr))
  41                 return -1;
  42
  43         if (ibw_bind(ictx, &my_addr)) {
  44                 DEBUG(DEBUG_CRIT, ("ctdb_ibw_listen: ibw_bind failed\n"));
  45                 return -1;
  46         }
  47
  48         if (ibw_listen(ictx, backlog)) {
  49                 DEBUG(DEBUG_CRIT, ("ctdb_ibw_listen: ibw_listen failed\n"));
  50                 return -1;
  51         }
  52
  53         return 0;
  54 }
  55
  56 /*
  57  * initialise ibw portion of a ctdb node
  58  */
  59 static int ctdb_ibw_add_node(struct ctdb_node *node)
  60 {
  61         struct ibw_ctx *ictx = talloc_get_type(node->ctdb->private_data, struct ibw_ctx);
  62         struct ctdb_ibw_node *cn = talloc_zero(node, struct ctdb_ibw_node);
  63
  64         assert(cn!=NULL);
  65         cn->conn = ibw_conn_new(ictx, node);
  66         node->private_data = (void *)cn;
  67
  68         return (cn->conn!=NULL ? 0 : -1);
  69 }
  70
  71 /*
  72  * initialise infiniband
  73  */
  74 static int ctdb_ibw_initialise(struct ctdb_context *ctdb)
  75 {
  76         int i, ret;
  77
  78         ret = ctdb_ibw_init(ctdb);
  79         if (ret != 0) {
  80                 return ret;
  81         }
  82
  83         for (i=0; i<ctdb->num_nodes; i++) {
  84                 if (ctdb_ibw_add_node(ctdb->nodes[i]) != 0) {
  85                         DEBUG(DEBUG_CRIT, ("methods->add_node failed at %d\n", i));
  86                         return -1;
  87                 }
  88         }
  89
  90         /* listen on our own address */
  91         if (ctdb_ibw_listen(ctdb, 10)) /* TODO: backlog as param */
  92                 return -1;
  93
  94         return 0;
  95 }
  96
  97
  98 /*
  99  * Start infiniband
 100  */
 101 static int ctdb_ibw_start(struct ctdb_context *ctdb)
 102 {
 103         int i;
 104
 105         /* everything async here */
 106         for (i=0;i<ctdb->num_nodes;i++) {
 107                 struct ctdb_node *node = ctdb->nodes[i];
 108                 if (!ctdb_same_address(&ctdb->address, &node->address)) {
 109                         ctdb_ibw_node_connect(node);
 110                 }
 111         }
 112
 113         return 0;
 114 }
 115
 116 static int ctdb_ibw_send_pkt(struct ibw_conn *conn, uint8_t *data, uint32_t length)
 117 {
 118         void    *buf, *key;
 119
 120         if (ibw_alloc_send_buf(conn, &buf, &key, length)) {
 121                 DEBUG(DEBUG_ERR, ("queue_pkt/ibw_alloc_send_buf failed\n"));
 122                 return -1;
 123         }
 124
 125         memcpy(buf, data, length);
 126         return ibw_send(conn, buf, key, length);
 127 }
 128
 129 int ctdb_flush_cn_queue(struct ctdb_ibw_node *cn)
 130 {
 131         struct ctdb_ibw_msg *p;
 132         int     rc = 0;
 133
 134         while(cn->queue) {
 135                 p = cn->queue;
 136                 rc = ctdb_ibw_send_pkt(cn->conn, p->data, p->length);
 137                 if (rc)
 138                         return -1; /* will be retried later when conn is up */
 139
 140                 DLIST_REMOVE(cn->queue, p);
 141                 cn->qcnt--;
 142                 talloc_free(p); /* it will talloc_free p->data as well */
 143         }
 144         assert(cn->qcnt==0);
 145         /* cn->queue_last = NULL is not needed - see DLIST_ADD_AFTER */
 146
 147         return rc;
 148 }
 149
 150 static int ctdb_ibw_queue_pkt(struct ctdb_node *node, uint8_t *data, uint32_t length)
 151 {
 152         struct ctdb_ibw_node *cn = talloc_get_type(node->private_data, struct ctdb_ibw_node);
 153         int     rc;
 154
 155         assert(length>=sizeof(uint32_t));
 156         assert(cn!=NULL);
 157
 158         if (cn->conn==NULL) {
 159                 DEBUG(DEBUG_ERR, ("ctdb_ibw_queue_pkt: conn is NULL\n"));
 160                 return -1;
 161         }
 162
 163         if (cn->conn->state==IBWC_CONNECTED) {
 164                 rc = ctdb_ibw_send_pkt(cn->conn, data, length);
 165         } else {
 166                 struct ctdb_ibw_msg *p = talloc_zero(cn, struct ctdb_ibw_msg);
 167                 CTDB_NO_MEMORY(node->ctdb, p);
 168
 169                 p->data = talloc_memdup(p, data, length);
 170                 CTDB_NO_MEMORY(node->ctdb, p->data);
 171
 172                 p->length = length;
 173
 174                 DLIST_ADD_AFTER(cn->queue, p, cn->queue_last);
 175                 cn->queue_last = p;
 176                 cn->qcnt++;
 177
 178                 rc = 0;
 179         }
 180
 181         return rc;
 182 }
 183
 184 static void ctdb_ibw_restart(struct ctdb_node *node)
 185 {
 186         /* TODO: implement this method for IB */
 187         DEBUG(DEBUG_ALERT,("WARNING: method restart is not yet implemented for IB\n"));
 188 }
 189
 190 /*
 191  * transport packet allocator - allows transport to control memory for packets
 192  */
 193 static void *ctdb_ibw_allocate_pkt(TALLOC_CTX *mem_ctx, size_t size)
 194 {
 195         /* TODO: use ibw_alloc_send_buf instead... */
 196         return talloc_size(mem_ctx, size);
 197 }
 198
 199 #ifdef __NOTDEF__
 200
 201 static int ctdb_ibw_stop(struct ctdb_context *cctx)
 202 {
 203         struct ibw_ctx *ictx = talloc_get_type(cctx->private_data, struct ibw_ctx);
 204
 205         assert(ictx!=NULL);
 206         return ibw_stop(ictx);
 207 }
 208
 209 #endif /* __NOTDEF__ */
 210
 211 static const struct ctdb_methods ctdb_ibw_methods = {
 212         .initialise= ctdb_ibw_initialise,
 213         .start     = ctdb_ibw_start,
 214         .queue_pkt = ctdb_ibw_queue_pkt,
 215         .add_node = ctdb_ibw_add_node,
 216         .allocate_pkt = ctdb_ibw_allocate_pkt,
 217         .restart      = ctdb_ibw_restart,
 218
 219 //      .stop = ctdb_ibw_stop
 220 };
 221
 222 /*
 223  * initialise ibw portion of ctdb
 224  */
 225 int ctdb_ibw_init(struct ctdb_context *ctdb)
 226 {
 227         struct ibw_ctx *ictx;
 228
 229         DEBUG(DEBUG_DEBUG, ("ctdb_ibw_init invoked...\n"));
 230         ictx = ibw_init(
 231                 NULL, //struct ibw_initattr *attr, /* TODO */
 232                 0, //int nattr, /* TODO */
 233                 ctdb,
 234                 ctdb_ibw_connstate_handler,
 235                 ctdb_ibw_receive_handler,
 236                 ctdb->ev);
 237
 238         if (ictx==NULL) {
 239                 DEBUG(DEBUG_CRIT, ("ctdb_ibw_init: ibw_init failed\n"));
 240                 return -1;
 241         }
 242
 243         ctdb->methods = &ctdb_ibw_methods;
 244         ctdb->private_data = ictx;
 245
 246         DEBUG(DEBUG_DEBUG, ("ctdb_ibw_init succeeded.\n"));
 247         return 0;
 248 }