ctdb/ib/ibw_ctdb_init.c

   1 /*
   2  * Unix SMB/CIFS implementation.
   3  * Join infiniband wrapper and ctdb.
   4  *
   5  * Copyright (C) Sven Oehme <oehmes@de.ibm.com> 2006
   6  *
   7  * Major code contributions by Peter Somogyi <psomogyi@gamax.hu>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 3 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  21  */
  22
  23 #include "replace.h"
  24 #include "system/network.h"
  25
  26 #include <assert.h>
  27 #include <talloc.h>
  28 #include <tevent.h>
  29
  30 #include "lib/util/dlinklist.h"
  31 #include "lib/util/debug.h"
  32
  33 #include "ctdb_private.h"
  34
  35 #include "common/common.h"
  36 #include "common/logging.h"
  37
  38 #include "ibwrapper.h"
  39 #include "ibw_ctdb.h"
  40
  41 static int ctdb_ibw_listen(struct ctdb_context *ctdb, int backlog)
  42 {
  43         struct ibw_ctx *ictx = talloc_get_type(ctdb->transport_data,
  44                                                struct ibw_ctx);
  45
  46         assert(ictx!=NULL);
  47
  48         if (ibw_bind(ictx, &ctdb->address->ip)) {
  49                 DEBUG(DEBUG_CRIT, ("ctdb_ibw_listen: ibw_bind failed\n"));
  50                 return -1;
  51         }
  52
  53         if (ibw_listen(ictx, backlog)) {
  54                 DEBUG(DEBUG_CRIT, ("ctdb_ibw_listen: ibw_listen failed\n"));
  55                 return -1;
  56         }
  57
  58         return 0;
  59 }
  60
  61 /*
  62  * initialise ibw portion of a ctdb node
  63  */
  64 static int ctdb_ibw_add_node(struct ctdb_node *node)
  65 {
  66         struct ibw_ctx *ictx = talloc_get_type(node->ctdb->transport_data,
  67                                                struct ibw_ctx);
  68         struct ctdb_ibw_node *cn = talloc_zero(node, struct ctdb_ibw_node);
  69
  70         assert(cn!=NULL);
  71         cn->conn = ibw_conn_new(ictx, node);
  72         node->transport_data = (void *)cn;
  73
  74         return (cn->conn!=NULL ? 0 : -1);
  75 }
  76
  77 /*
  78  * initialise infiniband
  79  */
  80 static int ctdb_ibw_initialise(struct ctdb_context *ctdb)
  81 {
  82         int i, ret;
  83
  84         ret = ctdb_ibw_init(ctdb);
  85         if (ret != 0) {
  86                 return ret;
  87         }
  88
  89         for (i=0; i<ctdb->num_nodes; i++) {
  90                 if (ctdb_ibw_add_node(ctdb->nodes[i]) != 0) {
  91                         DEBUG(DEBUG_CRIT, ("methods->add_node failed at %d\n", i));
  92                         return -1;
  93                 }
  94         }
  95
  96         /* listen on our own address */
  97         if (ctdb_ibw_listen(ctdb, 10)) /* TODO: backlog as param */
  98                 return -1;
  99
 100         return 0;
 101 }
 102
 103
 104 /*
 105  * Start infiniband
 106  */
 107 static int ctdb_ibw_start(struct ctdb_context *ctdb)
 108 {
 109         int i;
 110
 111         /* everything async here */
 112         for (i=0;i<ctdb->num_nodes;i++) {
 113                 struct ctdb_node *node = ctdb->nodes[i];
 114                 if (!ctdb_same_address(ctdb->address, &node->address)) {
 115                         ctdb_ibw_node_connect(node);
 116                 }
 117         }
 118
 119         return 0;
 120 }
 121
 122 static int ctdb_ibw_send_pkt(struct ibw_conn *conn, uint8_t *data, uint32_t length)
 123 {
 124         void    *buf, *key;
 125
 126         if (ibw_alloc_send_buf(conn, &buf, &key, length)) {
 127                 DEBUG(DEBUG_ERR, ("queue_pkt/ibw_alloc_send_buf failed\n"));
 128                 return -1;
 129         }
 130
 131         memcpy(buf, data, length);
 132         return ibw_send(conn, buf, key, length);
 133 }
 134
 135 int ctdb_flush_cn_queue(struct ctdb_ibw_node *cn)
 136 {
 137         struct ctdb_ibw_msg *p;
 138         int     rc = 0;
 139
 140         while(cn->queue) {
 141                 p = cn->queue;
 142                 rc = ctdb_ibw_send_pkt(cn->conn, p->data, p->length);
 143                 if (rc)
 144                         return -1; /* will be retried later when conn is up */
 145
 146                 DLIST_REMOVE(cn->queue, p);
 147                 cn->qcnt--;
 148                 talloc_free(p); /* it will talloc_free p->data as well */
 149         }
 150         assert(cn->qcnt==0);
 151         /* cn->queue_last = NULL is not needed - see DLIST_ADD_AFTER */
 152
 153         return rc;
 154 }
 155
 156 static int ctdb_ibw_queue_pkt(struct ctdb_node *node, uint8_t *data, uint32_t length)
 157 {
 158         struct ctdb_ibw_node *cn = talloc_get_type(node->transport_data,
 159                                                    struct ctdb_ibw_node);
 160         int     rc;
 161
 162         assert(length>=sizeof(uint32_t));
 163         assert(cn!=NULL);
 164
 165         if (cn->conn==NULL) {
 166                 DEBUG(DEBUG_ERR, ("ctdb_ibw_queue_pkt: conn is NULL\n"));
 167                 return -1;
 168         }
 169
 170         if (cn->conn->state==IBWC_CONNECTED) {
 171                 rc = ctdb_ibw_send_pkt(cn->conn, data, length);
 172         } else {
 173                 struct ctdb_ibw_msg *p = talloc_zero(cn, struct ctdb_ibw_msg);
 174                 CTDB_NO_MEMORY(node->ctdb, p);
 175
 176                 p->data = talloc_memdup(p, data, length);
 177                 CTDB_NO_MEMORY(node->ctdb, p->data);
 178
 179                 p->length = length;
 180
 181                 DLIST_ADD_AFTER(cn->queue, p, cn->queue_last);
 182                 cn->queue_last = p;
 183                 cn->qcnt++;
 184
 185                 rc = 0;
 186         }
 187
 188         return rc;
 189 }
 190
 191 static void ctdb_ibw_restart(struct ctdb_node *node)
 192 {
 193         /* TODO: implement this method for IB */
 194         DEBUG(DEBUG_ALERT,("WARNING: method restart is not yet implemented for IB\n"));
 195 }
 196
 197 /*
 198  * transport packet allocator - allows transport to control memory for packets
 199  */
 200 static void *ctdb_ibw_allocate_pkt(TALLOC_CTX *mem_ctx, size_t size)
 201 {
 202         /* TODO: use ibw_alloc_send_buf instead... */
 203         return talloc_size(mem_ctx, size);
 204 }
 205
 206 #ifdef __NOTDEF__
 207
 208 static int ctdb_ibw_stop(struct ctdb_context *cctx)
 209 {
 210         struct ibw_ctx *ictx = talloc_get_type(cctx->private_data, struct ibw_ctx);
 211
 212         assert(ictx!=NULL);
 213         return ibw_stop(ictx);
 214 }
 215
 216 #endif /* __NOTDEF__ */
 217
 218 static const struct ctdb_methods ctdb_ibw_methods = {
 219         .initialise= ctdb_ibw_initialise,
 220         .start     = ctdb_ibw_start,
 221         .queue_pkt = ctdb_ibw_queue_pkt,
 222         .add_node = ctdb_ibw_add_node,
 223         .allocate_pkt = ctdb_ibw_allocate_pkt,
 224         .restart      = ctdb_ibw_restart,
 225
 226 //      .stop = ctdb_ibw_stop
 227 };
 228
 229 /*
 230  * initialise ibw portion of ctdb
 231  */
 232 int ctdb_ibw_init(struct ctdb_context *ctdb)
 233 {
 234         struct ibw_ctx *ictx;
 235
 236         DEBUG(DEBUG_DEBUG, ("ctdb_ibw_init invoked...\n"));
 237         ictx = ibw_init(
 238                 NULL, //struct ibw_initattr *attr, /* TODO */
 239                 0, //int nattr, /* TODO */
 240                 ctdb,
 241                 ctdb_ibw_connstate_handler,
 242                 ctdb_ibw_receive_handler,
 243                 ctdb->ev);
 244
 245         if (ictx==NULL) {
 246                 DEBUG(DEBUG_CRIT, ("ctdb_ibw_init: ibw_init failed\n"));
 247                 return -1;
 248         }
 249
 250         ctdb->methods = &ctdb_ibw_methods;
 251         ctdb->transport_data = ictx;
 252
 253         DEBUG(DEBUG_DEBUG, ("ctdb_ibw_init succeeded.\n"));
 254         return 0;
 255 }