ctdb/ib/ibwrapper.h

   1 /*
   2  * Unix SMB/CIFS implementation.
   3  * Wrap Infiniband calls.
   4  *
   5  * Copyright (C) Sven Oehme <oehmes@de.ibm.com> 2006
   6  *
   7  * Major code contributions by Peter Somogyi <psomogyi@gamax.hu>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 3 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  21  */
  22
  23 /* Server communication state */
  24 enum ibw_state_ctx {
  25         IBWS_INIT = 0, /* ctx start - after ibw_init */
  26         IBWS_READY, /* after ibw_bind & ibw_listen */
  27         IBWS_CONNECT_REQUEST, /* after [IBWS_READY + incoming request] */
  28                 /* => [(ibw_accept)IBWS_READY | (ibw_disconnect)STOPPED | ERROR] */
  29         IBWS_STOPPED, /* normal stop <= ibw_disconnect+(IBWS_READY | IBWS_CONNECT_REQUEST) */
  30         IBWS_ERROR /* abnormal state; ibw_stop must be called after this */
  31 };
  32
  33 /* Connection state */
  34 struct ibw_ctx {
  35         void *ctx_userdata; /* see ibw_init */
  36
  37         enum ibw_state_ctx state;
  38         void *internal;
  39
  40         struct ibw_conn *conn_list; /* 1st elem of double linked list */
  41 };
  42
  43 enum ibw_state_conn {
  44         IBWC_INIT = 0, /* conn start - internal state */
  45         IBWC_CONNECTED, /* after ibw_accept or ibw_connect */
  46         IBWC_DISCONNECTED, /* after ibw_disconnect */
  47         IBWC_ERROR
  48 };
  49
  50 struct ibw_conn {
  51         struct ibw_ctx *ctx;
  52         enum ibw_state_conn state;
  53
  54         void *conn_userdata; /* see ibw_connect and ibw_accept */
  55         void *internal;
  56
  57         struct ibw_conn *prev, *next;
  58 };
  59
  60 /*
  61  * (name, value) pair for array param of ibw_init
  62  */
  63 struct ibw_initattr {
  64         const char *name;
  65         const char *value;
  66 };
  67
  68 /*
  69  * Callback function definition which should inform you about
  70  * connection state change
  71  * This callback is invoked whenever server or client connection changes.
  72  * Both <conn> and <ctx> can be NULL if their state didn't change.
  73  * Return nonzero on error.
  74  */
  75 typedef int (*ibw_connstate_fn_t)(struct ibw_ctx *ctx, struct ibw_conn *conn);
  76
  77 /*
  78  * Callback function definition which should process incoming packets
  79  * This callback is invoked whenever any message arrives.
  80  * Return nonzero on error.
  81  *
  82  * Important: you mustn't store buf pointer for later use.
  83  * Process its contents before returning.
  84  */
  85 typedef int (*ibw_receive_fn_t)(struct ibw_conn *conn, void *buf, int n);
  86
  87 /*
  88  * settings: array of (name, value) pairs
  89  * where name is one of:
  90  *      max_send_wr [default is 256]
  91  *      max_recv_wr [default is 1024]
  92  * <...>
  93  *
  94  * Must be called _ONCE_ for each node.
  95  *
  96  * max_msg_size is the maximum size of a message
  97  * (max_send_wr + max_recv_wr) * max_msg_size bytes allocated per connection
  98  *
  99  * returns non-NULL on success
 100  *
 101  * talloc_free must be called for the result in IBWS_STOPPED;
 102  *    it will close resources by destructor
 103  *    connections(ibw_conn *) must have been closed prior talloc_free
 104  */
 105 struct ibw_ctx *ibw_init(struct ibw_initattr *attr, int nattr,
 106         void *ctx_userdata,
 107         ibw_connstate_fn_t ibw_connstate,
 108         ibw_receive_fn_t ibw_receive,
 109         struct event_context *ectx);
 110
 111 /*
 112  * Must be called in states of (IBWS_ERROR, IBWS_READY, IBWS_CONNECT_REQUEST)
 113  *
 114  * It will send out disconnect requests and free up ibw_conn structures.
 115  * The ctx->state will transit to IBWS_STOPPED after every conn are disconnected.
 116  * During that time, you mustn't send/recv/disconnect any more.
 117  * Only after ctx->state=IBWS_STOPPED you can talloc_free the ctx.
 118  */
 119 int ibw_stop(struct ibw_ctx *ctx);
 120
 121 /*************** connection initiation - like stream sockets *****/
 122
 123 /*
 124  * works like socket bind
 125  * needs a normal internet address here
 126  *
 127  * return 0 on success
 128  */
 129 int ibw_bind(struct ibw_ctx *ctx, struct sockaddr_in *my_addr);
 130
 131 /*
 132  * works like socket listen
 133  * non-blocking
 134  * enables accepting incoming connections (after IBWS_READY)
 135  * (it doesn't touch ctx->state by itself)
 136  *
 137  * returns 0 on success
 138  */
 139 int ibw_listen(struct ibw_ctx *ctx, int backlog);
 140
 141 /*
 142  * works like socket accept
 143  * initializes a connection to a client
 144  * must be called when state=IBWS_CONNECT_REQUEST
 145  *
 146  * returns 0 on success
 147  *
 148  * You have +1 waiting here: you will get ibw_conn (having the
 149  * same <conn_userdata> member) structure in ibw_connstate_fn_t.
 150  *
 151  * Important: you won't get remote IP address (only internal conn info)
 152  */
 153 int ibw_accept(struct ibw_ctx *ctx, struct ibw_conn *conn, void *conn_userdata);
 154
 155 /*
 156  * Create a new connection structure
 157  * available for queueing ibw_send
 158  *
 159  * <parent> is needed to be notified by talloc destruct action.
 160  */
 161 struct ibw_conn *ibw_conn_new(struct ibw_ctx *ctx, TALLOC_CTX *mem_ctx);
 162
 163 /*
 164  * Needs a normal internet address here
 165  * can be called within IBWS_READY|IBWS_CONNECT_REQUEST
 166  *
 167  * returns non-NULL on success
 168  *
 169  * You have +1 waiting here: you will get ibw_conn (having the
 170  * same <conn_userdata> member) structure in ibw_connstate_fn_t.
 171  */
 172 int ibw_connect(struct ibw_conn *conn, struct sockaddr_in *serv_addr, void *conn_userdata);
 173
 174 /*
 175  * Sends out a disconnect request.
 176  * You should process fds after calling this function
 177  * and then process it with ibw_process_event normally
 178  * until you get conn->state = IBWC_DISCONNECTED
 179  *
 180  * You mustn't talloc_free <conn> yet right after this,
 181  * first wait for IBWC_DISCONNECTED.
 182  */
 183 int ibw_disconnect(struct ibw_conn *conn);
 184
 185 /************ Infiniband specific event loop wrapping ******************/
 186
 187 /*
 188  * You have to use this buf to fill in before send.
 189  * It's just to avoid memcpy.in ibw_send.
 190  * Use the same (buf, key) pair with ibw_send.
 191  * Don't use more space than maxsize (see ibw_init).
 192  *
 193  * Returns 0 on success.
 194  */
 195 int ibw_alloc_send_buf(struct ibw_conn *conn, void **buf, void **key, uint32_t len);
 196
 197 /*
 198  * Send the message in one
 199  * Can be invoked any times (should fit into buffers) and at any time
 200  * (in conn->state=IBWC_CONNECTED)
 201  * n must be less or equal than max_msg_size (see ibw_init)
 202  *
 203  * You mustn't use (buf, key) any more for sending.
 204  */
 205 int ibw_send(struct ibw_conn *conn, void *buf, void *key, uint32_t len);
 206
 207 /*
 208  * Call this after ibw_alloc_send_buf
 209  * when you won't call ibw_send for (buf, key)
 210  * You mustn't use (buf, key) any more.
 211  */
 212 int ibw_cancel_send_buf(struct ibw_conn *conn, void *buf, void *key);
 213
 214 /*
 215  * Retrieves the last error
 216  * result: always non-zero, mustn't be freed (static)
 217  */
 218 const char *ibw_getLastError(void);