?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
virtio: don't enable notifications during polling
[qemu/kevin.git] / hw / rdma / rdma_backend.c
blobc346407cd3645cc0741a5f8e0dd820a0c1a4b81a
1 /*
2 * QEMU paravirtual RDMA - Generic RDMA backend
3 *
4 * Copyright (C) 2018 Oracle
5 * Copyright (C) 2018 Red Hat Inc
6 *
7 * Authors:
8 * Yuval Shaia <yuval.shaia@oracle.com>
9 * Marcel Apfelbaum <marcel@redhat.com>
10 *
11 * This work is licensed under the terms of the GNU GPL, version 2 or later.
12 * See the COPYING file in the top-level directory.
13 *
14 */
15
16 #include "qemu/osdep.h"
17 #include "qapi/qapi-events-rdma.h"
18
19 #include <infiniband/verbs.h>
20
21 #include "contrib/rdmacm-mux/rdmacm-mux.h"
22 #include "trace.h"
23 #include "rdma_utils.h"
24 #include "rdma_rm.h"
25 #include "rdma_backend.h"
26
27 #define THR_NAME_LEN 16
28 #define THR_POLL_TO 5000
29
30 #define MAD_HDR_SIZE sizeof(struct ibv_grh)
31
32 typedef struct BackendCtx {
33 void *up_ctx;
34 struct ibv_sge sge; /* Used to save MAD recv buffer */
35 RdmaBackendQP *backend_qp; /* To maintain recv buffers */
36 RdmaBackendSRQ *backend_srq;
37 } BackendCtx;
38
39 struct backend_umad {
40 struct ib_user_mad hdr;
41 char mad[RDMA_MAX_PRIVATE_DATA];
42 };
43
44 static void (*comp_handler)(void *ctx, struct ibv_wc *wc);
45
46 static void dummy_comp_handler(void *ctx, struct ibv_wc *wc)
47 {
48 rdma_error_report("No completion handler is registered");
49 }
50
51 static inline void complete_work(enum ibv_wc_status status, uint32_t vendor_err,
52 void *ctx)
53 {
54 struct ibv_wc wc = {};
55
56 wc.status = status;
57 wc.vendor_err = vendor_err;
58
59 comp_handler(ctx, &wc);
60 }
61
62 static void free_cqe_ctx(gpointer data, gpointer user_data)
63 {
64 BackendCtx *bctx;
65 RdmaDeviceResources *rdma_dev_res = user_data;
66 unsigned long cqe_ctx_id = GPOINTER_TO_INT(data);
67
68 bctx = rdma_rm_get_cqe_ctx(rdma_dev_res, cqe_ctx_id);
69 if (bctx) {
70 rdma_rm_dealloc_cqe_ctx(rdma_dev_res, cqe_ctx_id);
71 atomic_dec(&rdma_dev_res->stats.missing_cqe);
72 }
73 g_free(bctx);
74 }
75
76 static void clean_recv_mads(RdmaBackendDev *backend_dev)
77 {
78 unsigned long cqe_ctx_id;
79
80 do {
81 cqe_ctx_id = rdma_protected_qlist_pop_int64(&backend_dev->
82 recv_mads_list);
83 if (cqe_ctx_id != -ENOENT) {
84 atomic_inc(&backend_dev->rdma_dev_res->stats.missing_cqe);
85 free_cqe_ctx(GINT_TO_POINTER(cqe_ctx_id),
86 backend_dev->rdma_dev_res);
87 }
88 } while (cqe_ctx_id != -ENOENT);
89 }
90
91 static int rdma_poll_cq(RdmaDeviceResources *rdma_dev_res, struct ibv_cq *ibcq)
92 {
93 int i, ne, total_ne = 0;
94 BackendCtx *bctx;
95 struct ibv_wc wc[2];
96 RdmaProtectedGSList *cqe_ctx_list;
97
98 qemu_mutex_lock(&rdma_dev_res->lock);
99 do {
100 ne = ibv_poll_cq(ibcq, ARRAY_SIZE(wc), wc);
101
102 trace_rdma_poll_cq(ne, ibcq);
103
104 for (i = 0; i < ne; i++) {
105 bctx = rdma_rm_get_cqe_ctx(rdma_dev_res, wc[i].wr_id);
106 if (unlikely(!bctx)) {
107 rdma_error_report("No matching ctx for req %"PRId64,
108 wc[i].wr_id);
109 continue;
110 }
111
112 comp_handler(bctx->up_ctx, &wc[i]);
113
114 if (bctx->backend_qp) {
115 cqe_ctx_list = &bctx->backend_qp->cqe_ctx_list;
116 } else {
117 cqe_ctx_list = &bctx->backend_srq->cqe_ctx_list;
118 }
119
120 rdma_protected_gslist_remove_int32(cqe_ctx_list, wc[i].wr_id);
121 rdma_rm_dealloc_cqe_ctx(rdma_dev_res, wc[i].wr_id);
122 g_free(bctx);
123 }
124 total_ne += ne;
125 } while (ne > 0);
126 atomic_sub(&rdma_dev_res->stats.missing_cqe, total_ne);
127 qemu_mutex_unlock(&rdma_dev_res->lock);
128
129 if (ne < 0) {
130 rdma_error_report("ibv_poll_cq fail, rc=%d, errno=%d", ne, errno);
131 }
132
133 rdma_dev_res->stats.completions += total_ne;
134
135 return total_ne;
136 }
137
138 static void *comp_handler_thread(void *arg)
139 {
140 RdmaBackendDev *backend_dev = (RdmaBackendDev *)arg;
141 int rc;
142 struct ibv_cq *ev_cq;
143 void *ev_ctx;
144 int flags;
145 GPollFD pfds[1];
146
147 /* Change to non-blocking mode */
148 flags = fcntl(backend_dev->channel->fd, F_GETFL);
149 rc = fcntl(backend_dev->channel->fd, F_SETFL, flags | O_NONBLOCK);
150 if (rc < 0) {
151 rdma_error_report("Failed to change backend channel FD to non-blocking");
152 return NULL;
153 }
154
155 pfds[0].fd = backend_dev->channel->fd;
156 pfds[0].events = G_IO_IN | G_IO_HUP | G_IO_ERR;
157
158 backend_dev->comp_thread.is_running = true;
159
160 while (backend_dev->comp_thread.run) {
161 do {
162 rc = qemu_poll_ns(pfds, 1, THR_POLL_TO * (int64_t)SCALE_MS);
163 if (!rc) {
164 backend_dev->rdma_dev_res->stats.poll_cq_ppoll_to++;
165 }
166 } while (!rc && backend_dev->comp_thread.run);
167
168 if (backend_dev->comp_thread.run) {
169 rc = ibv_get_cq_event(backend_dev->channel, &ev_cq, &ev_ctx);
170 if (unlikely(rc)) {
171 rdma_error_report("ibv_get_cq_event fail, rc=%d, errno=%d", rc,
172 errno);
173 continue;
174 }
175
176 rc = ibv_req_notify_cq(ev_cq, 0);
177 if (unlikely(rc)) {
178 rdma_error_report("ibv_req_notify_cq fail, rc=%d, errno=%d", rc,
179 errno);
180 }
181
182 backend_dev->rdma_dev_res->stats.poll_cq_from_bk++;
183 rdma_poll_cq(backend_dev->rdma_dev_res, ev_cq);
184
185 ibv_ack_cq_events(ev_cq, 1);
186 }
187 }
188
189 backend_dev->comp_thread.is_running = false;
190
191 qemu_thread_exit(0);
192
193 return NULL;
194 }
195
196 static inline void disable_rdmacm_mux_async(RdmaBackendDev *backend_dev)
197 {
198 atomic_set(&backend_dev->rdmacm_mux.can_receive, 0);
199 }
200
201 static inline void enable_rdmacm_mux_async(RdmaBackendDev *backend_dev)
202 {
203 atomic_set(&backend_dev->rdmacm_mux.can_receive, sizeof(RdmaCmMuxMsg));
204 }
205
206 static inline int rdmacm_mux_can_process_async(RdmaBackendDev *backend_dev)
207 {
208 return atomic_read(&backend_dev->rdmacm_mux.can_receive);
209 }
210
211 static int rdmacm_mux_check_op_status(CharBackend *mad_chr_be)
212 {
213 RdmaCmMuxMsg msg = {};
214 int ret;
215
216 ret = qemu_chr_fe_read_all(mad_chr_be, (uint8_t *)&msg, sizeof(msg));
217 if (ret != sizeof(msg)) {
218 rdma_error_report("Got invalid message from mux: size %d, expecting %d",
219 ret, (int)sizeof(msg));
220 return -EIO;
221 }
222
223 trace_rdmacm_mux_check_op_status(msg.hdr.msg_type, msg.hdr.op_code,
224 msg.hdr.err_code);
225
226 if (msg.hdr.msg_type != RDMACM_MUX_MSG_TYPE_RESP) {
227 rdma_error_report("Got invalid message type %d", msg.hdr.msg_type);
228 return -EIO;
229 }
230
231 if (msg.hdr.err_code != RDMACM_MUX_ERR_CODE_OK) {
232 rdma_error_report("Operation failed in mux, error code %d",
233 msg.hdr.err_code);
234 return -EIO;
235 }
236
237 return 0;
238 }
239
240 static int rdmacm_mux_send(RdmaBackendDev *backend_dev, RdmaCmMuxMsg *msg)
241 {
242 int rc = 0;
243
244 msg->hdr.msg_type = RDMACM_MUX_MSG_TYPE_REQ;
245 trace_rdmacm_mux("send", msg->hdr.msg_type, msg->hdr.op_code);
246 disable_rdmacm_mux_async(backend_dev);
247 rc = qemu_chr_fe_write(backend_dev->rdmacm_mux.chr_be,
248 (const uint8_t *)msg, sizeof(*msg));
249 if (rc != sizeof(*msg)) {
250 enable_rdmacm_mux_async(backend_dev);
251 rdma_error_report("Failed to send request to rdmacm_mux (rc=%d)", rc);
252 return -EIO;
253 }
254
255 rc = rdmacm_mux_check_op_status(backend_dev->rdmacm_mux.chr_be);
256 if (rc) {
257 rdma_error_report("Failed to execute rdmacm_mux request %d (rc=%d)",
258 msg->hdr.op_code, rc);
259 }
260
261 enable_rdmacm_mux_async(backend_dev);
262
263 return 0;
264 }
265
266 static void stop_backend_thread(RdmaBackendThread *thread)
267 {
268 thread->run = false;
269 while (thread->is_running) {
270 sleep(THR_POLL_TO / SCALE_US / 2);
271 }
272 }
273
274 static void start_comp_thread(RdmaBackendDev *backend_dev)
275 {
276 char thread_name[THR_NAME_LEN] = {};
277
278 stop_backend_thread(&backend_dev->comp_thread);
279
280 snprintf(thread_name, sizeof(thread_name), "rdma_comp_%s",
281 ibv_get_device_name(backend_dev->ib_dev));
282 backend_dev->comp_thread.run = true;
283 qemu_thread_create(&backend_dev->comp_thread.thread, thread_name,
284 comp_handler_thread, backend_dev, QEMU_THREAD_DETACHED);
285 }
286
287 void rdma_backend_register_comp_handler(void (*handler)(void *ctx,
288 struct ibv_wc *wc))
289 {
290 comp_handler = handler;
291 }
292
293 void rdma_backend_unregister_comp_handler(void)
294 {
295 rdma_backend_register_comp_handler(dummy_comp_handler);
296 }
297
298 int rdma_backend_query_port(RdmaBackendDev *backend_dev,
299 struct ibv_port_attr *port_attr)
300 {
301 int rc;
302
303 rc = ibv_query_port(backend_dev->context, backend_dev->port_num, port_attr);
304 if (rc) {
305 rdma_error_report("ibv_query_port fail, rc=%d, errno=%d", rc, errno);
306 return -EIO;
307 }
308
309 return 0;
310 }
311
312 void rdma_backend_poll_cq(RdmaDeviceResources *rdma_dev_res, RdmaBackendCQ *cq)
313 {
314 int polled;
315
316 rdma_dev_res->stats.poll_cq_from_guest++;
317 polled = rdma_poll_cq(rdma_dev_res, cq->ibcq);
318 if (!polled) {
319 rdma_dev_res->stats.poll_cq_from_guest_empty++;
320 }
321 }
322
323 static GHashTable *ah_hash;
324
325 static struct ibv_ah *create_ah(RdmaBackendDev *backend_dev, struct ibv_pd *pd,
326 uint8_t sgid_idx, union ibv_gid *dgid)
327 {
328 GBytes *ah_key = g_bytes_new(dgid, sizeof(*dgid));
329 struct ibv_ah *ah = g_hash_table_lookup(ah_hash, ah_key);
330
331 if (ah) {
332 trace_rdma_create_ah_cache_hit(be64_to_cpu(dgid->global.subnet_prefix),
333 be64_to_cpu(dgid->global.interface_id));
334 g_bytes_unref(ah_key);
335 } else {
336 struct ibv_ah_attr ah_attr = {
337 .is_global = 1,
338 .port_num = backend_dev->port_num,
339 .grh.hop_limit = 1,
340 };
341
342 ah_attr.grh.dgid = *dgid;
343 ah_attr.grh.sgid_index = sgid_idx;
344
345 ah = ibv_create_ah(pd, &ah_attr);
346 if (ah) {
347 g_hash_table_insert(ah_hash, ah_key, ah);
348 } else {
349 g_bytes_unref(ah_key);
350 rdma_error_report("Failed to create AH for gid <0x%" PRIx64", 0x%"PRIx64">",
351 be64_to_cpu(dgid->global.subnet_prefix),
352 be64_to_cpu(dgid->global.interface_id));
353 }
354
355 trace_rdma_create_ah_cache_miss(be64_to_cpu(dgid->global.subnet_prefix),
356 be64_to_cpu(dgid->global.interface_id));
357 }
358
359 return ah;
360 }
361
362 static void destroy_ah_hash_key(gpointer data)
363 {
364 g_bytes_unref(data);
365 }
366
367 static void destroy_ah_hast_data(gpointer data)
368 {
369 struct ibv_ah *ah = data;
370
371 ibv_destroy_ah(ah);
372 }
373
374 static void ah_cache_init(void)
375 {
376 ah_hash = g_hash_table_new_full(g_bytes_hash, g_bytes_equal,
377 destroy_ah_hash_key, destroy_ah_hast_data);
378 }
379
380 static int build_host_sge_array(RdmaDeviceResources *rdma_dev_res,
381 struct ibv_sge *dsge, struct ibv_sge *ssge,
382 uint8_t num_sge, uint64_t *total_length)
383 {
384 RdmaRmMR *mr;
385 int ssge_idx;
386
387 for (ssge_idx = 0; ssge_idx < num_sge; ssge_idx++) {
388 mr = rdma_rm_get_mr(rdma_dev_res, ssge[ssge_idx].lkey);
389 if (unlikely(!mr)) {
390 rdma_error_report("Invalid lkey 0x%x", ssge[ssge_idx].lkey);
391 return VENDOR_ERR_INVLKEY | ssge[ssge_idx].lkey;
392 }
393
394 #ifdef LEGACY_RDMA_REG_MR
395 dsge->addr = (uintptr_t)mr->virt + ssge[ssge_idx].addr - mr->start;
396 #else
397 dsge->addr = ssge[ssge_idx].addr;
398 #endif
399 dsge->length = ssge[ssge_idx].length;
400 dsge->lkey = rdma_backend_mr_lkey(&mr->backend_mr);
401
402 *total_length += dsge->length;
403
404 dsge++;
405 }
406
407 return 0;
408 }
409
410 static void trace_mad_message(const char *title, char *buf, int len)
411 {
412 int i;
413 char *b = g_malloc0(len * 3 + 1);
414 char b1[4];
415
416 for (i = 0; i < len; i++) {
417 sprintf(b1, "%.2X ", buf[i] & 0x000000FF);
418 strcat(b, b1);
419 }
420
421 trace_rdma_mad_message(title, len, b);
422
423 g_free(b);
424 }
425
426 static int mad_send(RdmaBackendDev *backend_dev, uint8_t sgid_idx,
427 union ibv_gid *sgid, struct ibv_sge *sge, uint32_t num_sge)
428 {
429 RdmaCmMuxMsg msg = {};
430 char *hdr, *data;
431 int ret;
432
433 if (num_sge != 2) {
434 return -EINVAL;
435 }
436
437 msg.hdr.op_code = RDMACM_MUX_OP_CODE_MAD;
438 memcpy(msg.hdr.sgid.raw, sgid->raw, sizeof(msg.hdr.sgid));
439
440 msg.umad_len = sge[0].length + sge[1].length;
441
442 if (msg.umad_len > sizeof(msg.umad.mad)) {
443 return -ENOMEM;
444 }
445
446 msg.umad.hdr.addr.qpn = htobe32(1);
447 msg.umad.hdr.addr.grh_present = 1;
448 msg.umad.hdr.addr.gid_index = sgid_idx;
449 memcpy(msg.umad.hdr.addr.gid, sgid->raw, sizeof(msg.umad.hdr.addr.gid));
450 msg.umad.hdr.addr.hop_limit = 0xFF;
451
452 hdr = rdma_pci_dma_map(backend_dev->dev, sge[0].addr, sge[0].length);
453 if (!hdr) {
454 return -ENOMEM;
455 }
456 data = rdma_pci_dma_map(backend_dev->dev, sge[1].addr, sge[1].length);
457 if (!data) {
458 rdma_pci_dma_unmap(backend_dev->dev, hdr, sge[0].length);
459 return -ENOMEM;
460 }
461
462 memcpy(&msg.umad.mad[0], hdr, sge[0].length);
463 memcpy(&msg.umad.mad[sge[0].length], data, sge[1].length);
464
465 rdma_pci_dma_unmap(backend_dev->dev, data, sge[1].length);
466 rdma_pci_dma_unmap(backend_dev->dev, hdr, sge[0].length);
467
468 trace_mad_message("send", msg.umad.mad, msg.umad_len);
469
470 ret = rdmacm_mux_send(backend_dev, &msg);
471 if (ret) {
472 rdma_error_report("Failed to send MAD to rdma_umadmux (%d)", ret);
473 return -EIO;
474 }
475
476 return 0;
477 }
478
479 void rdma_backend_post_send(RdmaBackendDev *backend_dev,
480 RdmaBackendQP *qp, uint8_t qp_type,
481 struct ibv_sge *sge, uint32_t num_sge,
482 uint8_t sgid_idx, union ibv_gid *sgid,
483 union ibv_gid *dgid, uint32_t dqpn, uint32_t dqkey,
484 void *ctx)
485 {
486 BackendCtx *bctx;
487 struct ibv_sge new_sge[MAX_SGE];
488 uint32_t bctx_id;
489 int rc;
490 struct ibv_send_wr wr = {}, *bad_wr;
491
492 if (!qp->ibqp) { /* This field is not initialized for QP0 and QP1 */
493 if (qp_type == IBV_QPT_SMI) {
494 rdma_error_report("Got QP0 request");
495 complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_QP0, ctx);
496 } else if (qp_type == IBV_QPT_GSI) {
497 rc = mad_send(backend_dev, sgid_idx, sgid, sge, num_sge);
498 if (rc) {
499 complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_MAD_SEND, ctx);
500 backend_dev->rdma_dev_res->stats.mad_tx_err++;
501 } else {
502 complete_work(IBV_WC_SUCCESS, 0, ctx);
503 backend_dev->rdma_dev_res->stats.mad_tx++;
504 }
505 }
506 return;
507 }
508
509 bctx = g_malloc0(sizeof(*bctx));
510 bctx->up_ctx = ctx;
511 bctx->backend_qp = qp;
512
513 rc = rdma_rm_alloc_cqe_ctx(backend_dev->rdma_dev_res, &bctx_id, bctx);
514 if (unlikely(rc)) {
515 complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_NOMEM, ctx);
516 goto err_free_bctx;
517 }
518
519 rdma_protected_gslist_append_int32(&qp->cqe_ctx_list, bctx_id);
520
521 rc = build_host_sge_array(backend_dev->rdma_dev_res, new_sge, sge, num_sge,
522 &backend_dev->rdma_dev_res->stats.tx_len);
523 if (rc) {
524 complete_work(IBV_WC_GENERAL_ERR, rc, ctx);
525 goto err_dealloc_cqe_ctx;
526 }
527
528 if (qp_type == IBV_QPT_UD) {
529 wr.wr.ud.ah = create_ah(backend_dev, qp->ibpd, sgid_idx, dgid);
530 if (!wr.wr.ud.ah) {
531 complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx);
532 goto err_dealloc_cqe_ctx;
533 }
534 wr.wr.ud.remote_qpn = dqpn;
535 wr.wr.ud.remote_qkey = dqkey;
536 }
537
538 wr.num_sge = num_sge;
539 wr.opcode = IBV_WR_SEND;
540 wr.send_flags = IBV_SEND_SIGNALED;
541 wr.sg_list = new_sge;
542 wr.wr_id = bctx_id;
543
544 rc = ibv_post_send(qp->ibqp, &wr, &bad_wr);
545 if (rc) {
546 rdma_error_report("ibv_post_send fail, qpn=0x%x, rc=%d, errno=%d",
547 qp->ibqp->qp_num, rc, errno);
548 complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx);
549 goto err_dealloc_cqe_ctx;
550 }
551
552 atomic_inc(&backend_dev->rdma_dev_res->stats.missing_cqe);
553 backend_dev->rdma_dev_res->stats.tx++;
554
555 return;
556
557 err_dealloc_cqe_ctx:
558 backend_dev->rdma_dev_res->stats.tx_err++;
559 rdma_rm_dealloc_cqe_ctx(backend_dev->rdma_dev_res, bctx_id);
560
561 err_free_bctx:
562 g_free(bctx);
563 }
564
565 static unsigned int save_mad_recv_buffer(RdmaBackendDev *backend_dev,
566 struct ibv_sge *sge, uint32_t num_sge,
567 void *ctx)
568 {
569 BackendCtx *bctx;
570 int rc;
571 uint32_t bctx_id;
572
573 if (num_sge != 1) {
574 rdma_error_report("Invalid num_sge (%d), expecting 1", num_sge);
575 return VENDOR_ERR_INV_NUM_SGE;
576 }
577
578 if (sge[0].length < RDMA_MAX_PRIVATE_DATA + sizeof(struct ibv_grh)) {
579 rdma_error_report("Too small buffer for MAD");
580 return VENDOR_ERR_INV_MAD_BUFF;
581 }
582
583 bctx = g_malloc0(sizeof(*bctx));
584
585 rc = rdma_rm_alloc_cqe_ctx(backend_dev->rdma_dev_res, &bctx_id, bctx);
586 if (unlikely(rc)) {
587 g_free(bctx);
588 return VENDOR_ERR_NOMEM;
589 }
590
591 bctx->up_ctx = ctx;
592 bctx->sge = *sge;
593
594 rdma_protected_qlist_append_int64(&backend_dev->recv_mads_list, bctx_id);
595
596 return 0;
597 }
598
599 void rdma_backend_post_recv(RdmaBackendDev *backend_dev,
600 RdmaBackendQP *qp, uint8_t qp_type,
601 struct ibv_sge *sge, uint32_t num_sge, void *ctx)
602 {
603 BackendCtx *bctx;
604 struct ibv_sge new_sge[MAX_SGE];
605 uint32_t bctx_id;
606 int rc;
607 struct ibv_recv_wr wr = {}, *bad_wr;
608
609 if (!qp->ibqp) { /* This field does not get initialized for QP0 and QP1 */
610 if (qp_type == IBV_QPT_SMI) {
611 rdma_error_report("Got QP0 request");
612 complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_QP0, ctx);
613 }
614 if (qp_type == IBV_QPT_GSI) {
615 rc = save_mad_recv_buffer(backend_dev, sge, num_sge, ctx);
616 if (rc) {
617 complete_work(IBV_WC_GENERAL_ERR, rc, ctx);
618 backend_dev->rdma_dev_res->stats.mad_rx_bufs_err++;
619 } else {
620 backend_dev->rdma_dev_res->stats.mad_rx_bufs++;
621 }
622 }
623 return;
624 }
625
626 bctx = g_malloc0(sizeof(*bctx));
627 bctx->up_ctx = ctx;
628 bctx->backend_qp = qp;
629
630 rc = rdma_rm_alloc_cqe_ctx(backend_dev->rdma_dev_res, &bctx_id, bctx);
631 if (unlikely(rc)) {
632 complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_NOMEM, ctx);
633 goto err_free_bctx;
634 }
635
636 rdma_protected_gslist_append_int32(&qp->cqe_ctx_list, bctx_id);
637
638 rc = build_host_sge_array(backend_dev->rdma_dev_res, new_sge, sge, num_sge,
639 &backend_dev->rdma_dev_res->stats.rx_bufs_len);
640 if (rc) {
641 complete_work(IBV_WC_GENERAL_ERR, rc, ctx);
642 goto err_dealloc_cqe_ctx;
643 }
644
645 wr.num_sge = num_sge;
646 wr.sg_list = new_sge;
647 wr.wr_id = bctx_id;
648 rc = ibv_post_recv(qp->ibqp, &wr, &bad_wr);
649 if (rc) {
650 rdma_error_report("ibv_post_recv fail, qpn=0x%x, rc=%d, errno=%d",
651 qp->ibqp->qp_num, rc, errno);
652 complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx);
653 goto err_dealloc_cqe_ctx;
654 }
655
656 atomic_inc(&backend_dev->rdma_dev_res->stats.missing_cqe);
657 backend_dev->rdma_dev_res->stats.rx_bufs++;
658
659 return;
660
661 err_dealloc_cqe_ctx:
662 backend_dev->rdma_dev_res->stats.rx_bufs_err++;
663 rdma_rm_dealloc_cqe_ctx(backend_dev->rdma_dev_res, bctx_id);
664
665 err_free_bctx:
666 g_free(bctx);
667 }
668
669 void rdma_backend_post_srq_recv(RdmaBackendDev *backend_dev,
670 RdmaBackendSRQ *srq, struct ibv_sge *sge,
671 uint32_t num_sge, void *ctx)
672 {
673 BackendCtx *bctx;
674 struct ibv_sge new_sge[MAX_SGE];
675 uint32_t bctx_id;
676 int rc;
677 struct ibv_recv_wr wr = {}, *bad_wr;
678
679 bctx = g_malloc0(sizeof(*bctx));
680 bctx->up_ctx = ctx;
681 bctx->backend_srq = srq;
682
683 rc = rdma_rm_alloc_cqe_ctx(backend_dev->rdma_dev_res, &bctx_id, bctx);
684 if (unlikely(rc)) {
685 complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_NOMEM, ctx);
686 goto err_free_bctx;
687 }
688
689 rdma_protected_gslist_append_int32(&srq->cqe_ctx_list, bctx_id);
690
691 rc = build_host_sge_array(backend_dev->rdma_dev_res, new_sge, sge, num_sge,
692 &backend_dev->rdma_dev_res->stats.rx_bufs_len);
693 if (rc) {
694 complete_work(IBV_WC_GENERAL_ERR, rc, ctx);
695 goto err_dealloc_cqe_ctx;
696 }
697
698 wr.num_sge = num_sge;
699 wr.sg_list = new_sge;
700 wr.wr_id = bctx_id;
701 rc = ibv_post_srq_recv(srq->ibsrq, &wr, &bad_wr);
702 if (rc) {
703 rdma_error_report("ibv_post_srq_recv fail, srqn=0x%x, rc=%d, errno=%d",
704 srq->ibsrq->handle, rc, errno);
705 complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx);
706 goto err_dealloc_cqe_ctx;
707 }
708
709 atomic_inc(&backend_dev->rdma_dev_res->stats.missing_cqe);
710 backend_dev->rdma_dev_res->stats.rx_bufs++;
711 backend_dev->rdma_dev_res->stats.rx_srq++;
712
713 return;
714
715 err_dealloc_cqe_ctx:
716 backend_dev->rdma_dev_res->stats.rx_bufs_err++;
717 rdma_rm_dealloc_cqe_ctx(backend_dev->rdma_dev_res, bctx_id);
718
719 err_free_bctx:
720 g_free(bctx);
721 }
722
723 int rdma_backend_create_pd(RdmaBackendDev *backend_dev, RdmaBackendPD *pd)
724 {
725 pd->ibpd = ibv_alloc_pd(backend_dev->context);
726
727 if (!pd->ibpd) {
728 rdma_error_report("ibv_alloc_pd fail, errno=%d", errno);
729 return -EIO;
730 }
731
732 return 0;
733 }
734
735 void rdma_backend_destroy_pd(RdmaBackendPD *pd)
736 {
737 if (pd->ibpd) {
738 ibv_dealloc_pd(pd->ibpd);
739 }
740 }
741
742 #ifdef LEGACY_RDMA_REG_MR
743 int rdma_backend_create_mr(RdmaBackendMR *mr, RdmaBackendPD *pd, void *addr,
744 size_t length, int access)
745 #else
746 int rdma_backend_create_mr(RdmaBackendMR *mr, RdmaBackendPD *pd, void *addr,
747 size_t length, uint64_t guest_start, int access)
748 #endif
749 {
750 #ifdef LEGACY_RDMA_REG_MR
751 mr->ibmr = ibv_reg_mr(pd->ibpd, addr, length, access);
752 #else
753 mr->ibmr = ibv_reg_mr_iova(pd->ibpd, addr, length, guest_start, access);
754 #endif
755 if (!mr->ibmr) {
756 rdma_error_report("ibv_reg_mr fail, errno=%d", errno);
757 return -EIO;
758 }
759
760 mr->ibpd = pd->ibpd;
761
762 return 0;
763 }
764
765 void rdma_backend_destroy_mr(RdmaBackendMR *mr)
766 {
767 if (mr->ibmr) {
768 ibv_dereg_mr(mr->ibmr);
769 }
770 }
771
772 int rdma_backend_create_cq(RdmaBackendDev *backend_dev, RdmaBackendCQ *cq,
773 int cqe)
774 {
775 int rc;
776
777 cq->ibcq = ibv_create_cq(backend_dev->context, cqe + 1, NULL,
778 backend_dev->channel, 0);
779 if (!cq->ibcq) {
780 rdma_error_report("ibv_create_cq fail, errno=%d", errno);
781 return -EIO;
782 }
783
784 rc = ibv_req_notify_cq(cq->ibcq, 0);
785 if (rc) {
786 rdma_warn_report("ibv_req_notify_cq fail, rc=%d, errno=%d", rc, errno);
787 }
788
789 cq->backend_dev = backend_dev;
790
791 return 0;
792 }
793
794 void rdma_backend_destroy_cq(RdmaBackendCQ *cq)
795 {
796 if (cq->ibcq) {
797 ibv_destroy_cq(cq->ibcq);
798 }
799 }
800
801 int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t qp_type,
802 RdmaBackendPD *pd, RdmaBackendCQ *scq,
803 RdmaBackendCQ *rcq, RdmaBackendSRQ *srq,
804 uint32_t max_send_wr, uint32_t max_recv_wr,
805 uint32_t max_send_sge, uint32_t max_recv_sge)
806 {
807 struct ibv_qp_init_attr attr = {};
808
809 qp->ibqp = 0;
810
811 switch (qp_type) {
812 case IBV_QPT_GSI:
813 return 0;
814
815 case IBV_QPT_RC:
816 /* fall through */
817 case IBV_QPT_UD:
818 /* do nothing */
819 break;
820
821 default:
822 rdma_error_report("Unsupported QP type %d", qp_type);
823 return -EIO;
824 }
825
826 attr.qp_type = qp_type;
827 attr.send_cq = scq->ibcq;
828 attr.recv_cq = rcq->ibcq;
829 attr.cap.max_send_wr = max_send_wr;
830 attr.cap.max_recv_wr = max_recv_wr;
831 attr.cap.max_send_sge = max_send_sge;
832 attr.cap.max_recv_sge = max_recv_sge;
833 if (srq) {
834 attr.srq = srq->ibsrq;
835 }
836
837 qp->ibqp = ibv_create_qp(pd->ibpd, &attr);
838 if (!qp->ibqp) {
839 rdma_error_report("ibv_create_qp fail, errno=%d", errno);
840 return -EIO;
841 }
842
843 rdma_protected_gslist_init(&qp->cqe_ctx_list);
844
845 qp->ibpd = pd->ibpd;
846
847 /* TODO: Query QP to get max_inline_data and save it to be used in send */
848
849 return 0;
850 }
851
852 int rdma_backend_qp_state_init(RdmaBackendDev *backend_dev, RdmaBackendQP *qp,
853 uint8_t qp_type, uint32_t qkey)
854 {
855 struct ibv_qp_attr attr = {};
856 int rc, attr_mask;
857
858 attr_mask = IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT;
859 attr.qp_state = IBV_QPS_INIT;
860 attr.pkey_index = 0;
861 attr.port_num = backend_dev->port_num;
862
863 switch (qp_type) {
864 case IBV_QPT_RC:
865 attr_mask |= IBV_QP_ACCESS_FLAGS;
866 trace_rdma_backend_rc_qp_state_init(qp->ibqp->qp_num);
867 break;
868
869 case IBV_QPT_UD:
870 attr.qkey = qkey;
871 attr_mask |= IBV_QP_QKEY;
872 trace_rdma_backend_ud_qp_state_init(qp->ibqp->qp_num, qkey);
873 break;
874
875 default:
876 rdma_error_report("Unsupported QP type %d", qp_type);
877 return -EIO;
878 }
879
880 rc = ibv_modify_qp(qp->ibqp, &attr, attr_mask);
881 if (rc) {
882 rdma_error_report("ibv_modify_qp fail, rc=%d, errno=%d", rc, errno);
883 return -EIO;
884 }
885
886 return 0;
887 }
888
889 int rdma_backend_qp_state_rtr(RdmaBackendDev *backend_dev, RdmaBackendQP *qp,
890 uint8_t qp_type, uint8_t sgid_idx,
891 union ibv_gid *dgid, uint32_t dqpn,
892 uint32_t rq_psn, uint32_t qkey, bool use_qkey)
893 {
894 struct ibv_qp_attr attr = {};
895 union ibv_gid ibv_gid = {
896 .global.interface_id = dgid->global.interface_id,
897 .global.subnet_prefix = dgid->global.subnet_prefix
898 };
899 int rc, attr_mask;
900
901 attr.qp_state = IBV_QPS_RTR;
902 attr_mask = IBV_QP_STATE;
903
904 qp->sgid_idx = sgid_idx;
905
906 switch (qp_type) {
907 case IBV_QPT_RC:
908 attr.path_mtu = IBV_MTU_1024;
909 attr.dest_qp_num = dqpn;
910 attr.max_dest_rd_atomic = 1;
911 attr.min_rnr_timer = 12;
912 attr.ah_attr.port_num = backend_dev->port_num;
913 attr.ah_attr.is_global = 1;
914 attr.ah_attr.grh.hop_limit = 1;
915 attr.ah_attr.grh.dgid = ibv_gid;
916 attr.ah_attr.grh.sgid_index = qp->sgid_idx;
917 attr.rq_psn = rq_psn;
918
919 attr_mask |= IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN |
920 IBV_QP_RQ_PSN | IBV_QP_MAX_DEST_RD_ATOMIC |
921 IBV_QP_MIN_RNR_TIMER;
922
923 trace_rdma_backend_rc_qp_state_rtr(qp->ibqp->qp_num,
924 be64_to_cpu(ibv_gid.global.
925 subnet_prefix),
926 be64_to_cpu(ibv_gid.global.
927 interface_id),
928 qp->sgid_idx, dqpn, rq_psn);
929 break;
930
931 case IBV_QPT_UD:
932 if (use_qkey) {
933 attr.qkey = qkey;
934 attr_mask |= IBV_QP_QKEY;
935 }
936 trace_rdma_backend_ud_qp_state_rtr(qp->ibqp->qp_num, use_qkey ? qkey :
937 0);
938 break;
939 }
940
941 rc = ibv_modify_qp(qp->ibqp, &attr, attr_mask);
942 if (rc) {
943 rdma_error_report("ibv_modify_qp fail, rc=%d, errno=%d", rc, errno);
944 return -EIO;
945 }
946
947 return 0;
948 }
949
950 int rdma_backend_qp_state_rts(RdmaBackendQP *qp, uint8_t qp_type,
951 uint32_t sq_psn, uint32_t qkey, bool use_qkey)
952 {
953 struct ibv_qp_attr attr = {};
954 int rc, attr_mask;
955
956 attr.qp_state = IBV_QPS_RTS;
957 attr.sq_psn = sq_psn;
958 attr_mask = IBV_QP_STATE | IBV_QP_SQ_PSN;
959
960 switch (qp_type) {
961 case IBV_QPT_RC:
962 attr.timeout = 14;
963 attr.retry_cnt = 7;
964 attr.rnr_retry = 7;
965 attr.max_rd_atomic = 1;
966
967 attr_mask |= IBV_QP_TIMEOUT | IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY |
968 IBV_QP_MAX_QP_RD_ATOMIC;
969 trace_rdma_backend_rc_qp_state_rts(qp->ibqp->qp_num, sq_psn);
970 break;
971
972 case IBV_QPT_UD:
973 if (use_qkey) {
974 attr.qkey = qkey;
975 attr_mask |= IBV_QP_QKEY;
976 }
977 trace_rdma_backend_ud_qp_state_rts(qp->ibqp->qp_num, sq_psn,
978 use_qkey ? qkey : 0);
979 break;
980 }
981
982 rc = ibv_modify_qp(qp->ibqp, &attr, attr_mask);
983 if (rc) {
984 rdma_error_report("ibv_modify_qp fail, rc=%d, errno=%d", rc, errno);
985 return -EIO;
986 }
987
988 return 0;
989 }
990
991 int rdma_backend_query_qp(RdmaBackendQP *qp, struct ibv_qp_attr *attr,
992 int attr_mask, struct ibv_qp_init_attr *init_attr)
993 {
994 if (!qp->ibqp) {
995 attr->qp_state = IBV_QPS_RTS;
996 return 0;
997 }
998
999 return ibv_query_qp(qp->ibqp, attr, attr_mask, init_attr);
1000 }
1001
1002 void rdma_backend_destroy_qp(RdmaBackendQP *qp, RdmaDeviceResources *dev_res)
1003 {
1004 if (qp->ibqp) {
1005 ibv_destroy_qp(qp->ibqp);
1006 }
1007 g_slist_foreach(qp->cqe_ctx_list.list, free_cqe_ctx, dev_res);
1008 rdma_protected_gslist_destroy(&qp->cqe_ctx_list);
1009 }
1010
1011 int rdma_backend_create_srq(RdmaBackendSRQ *srq, RdmaBackendPD *pd,
1012 uint32_t max_wr, uint32_t max_sge,
1013 uint32_t srq_limit)
1014 {
1015 struct ibv_srq_init_attr srq_init_attr = {};
1016
1017 srq_init_attr.attr.max_wr = max_wr;
1018 srq_init_attr.attr.max_sge = max_sge;
1019 srq_init_attr.attr.srq_limit = srq_limit;
1020
1021 srq->ibsrq = ibv_create_srq(pd->ibpd, &srq_init_attr);
1022 if (!srq->ibsrq) {
1023 rdma_error_report("ibv_create_srq failed, errno=%d", errno);
1024 return -EIO;
1025 }
1026
1027 rdma_protected_gslist_init(&srq->cqe_ctx_list);
1028
1029 return 0;
1030 }
1031
1032 int rdma_backend_query_srq(RdmaBackendSRQ *srq, struct ibv_srq_attr *srq_attr)
1033 {
1034 if (!srq->ibsrq) {
1035 return -EINVAL;
1036 }
1037
1038 return ibv_query_srq(srq->ibsrq, srq_attr);
1039 }
1040
1041 int rdma_backend_modify_srq(RdmaBackendSRQ *srq, struct ibv_srq_attr *srq_attr,
1042 int srq_attr_mask)
1043 {
1044 if (!srq->ibsrq) {
1045 return -EINVAL;
1046 }
1047
1048 return ibv_modify_srq(srq->ibsrq, srq_attr, srq_attr_mask);
1049 }
1050
1051 void rdma_backend_destroy_srq(RdmaBackendSRQ *srq, RdmaDeviceResources *dev_res)
1052 {
1053 if (srq->ibsrq) {
1054 ibv_destroy_srq(srq->ibsrq);
1055 }
1056 g_slist_foreach(srq->cqe_ctx_list.list, free_cqe_ctx, dev_res);
1057 rdma_protected_gslist_destroy(&srq->cqe_ctx_list);
1058 }
1059
1060 #define CHK_ATTR(req, dev, member, fmt) ({ \
1061 trace_rdma_check_dev_attr(#member, dev.member, req->member); \
1062 if (req->member > dev.member) { \
1063 rdma_warn_report("%s = "fmt" is higher than host device capability "fmt, \
1064 #member, req->member, dev.member); \
1065 req->member = dev.member; \
1066 } \
1067 })
1068
1069 static int init_device_caps(RdmaBackendDev *backend_dev,
1070 struct ibv_device_attr *dev_attr)
1071 {
1072 struct ibv_device_attr bk_dev_attr;
1073 int rc;
1074
1075 rc = ibv_query_device(backend_dev->context, &bk_dev_attr);
1076 if (rc) {
1077 rdma_error_report("ibv_query_device fail, rc=%d, errno=%d", rc, errno);
1078 return -EIO;
1079 }
1080
1081 dev_attr->max_sge = MAX_SGE;
1082 dev_attr->max_srq_sge = MAX_SGE;
1083
1084 CHK_ATTR(dev_attr, bk_dev_attr, max_mr_size, "%" PRId64);
1085 CHK_ATTR(dev_attr, bk_dev_attr, max_qp, "%d");
1086 CHK_ATTR(dev_attr, bk_dev_attr, max_sge, "%d");
1087 CHK_ATTR(dev_attr, bk_dev_attr, max_cq, "%d");
1088 CHK_ATTR(dev_attr, bk_dev_attr, max_mr, "%d");
1089 CHK_ATTR(dev_attr, bk_dev_attr, max_pd, "%d");
1090 CHK_ATTR(dev_attr, bk_dev_attr, max_qp_rd_atom, "%d");
1091 CHK_ATTR(dev_attr, bk_dev_attr, max_qp_init_rd_atom, "%d");
1092 CHK_ATTR(dev_attr, bk_dev_attr, max_ah, "%d");
1093 CHK_ATTR(dev_attr, bk_dev_attr, max_srq, "%d");
1094
1095 return 0;
1096 }
1097
1098 static inline void build_mad_hdr(struct ibv_grh *grh, union ibv_gid *sgid,
1099 union ibv_gid *my_gid, int paylen)
1100 {
1101 grh->paylen = htons(paylen);
1102 grh->sgid = *sgid;
1103 grh->dgid = *my_gid;
1104 }
1105
1106 static void process_incoming_mad_req(RdmaBackendDev *backend_dev,
1107 RdmaCmMuxMsg *msg)
1108 {
1109 unsigned long cqe_ctx_id;
1110 BackendCtx *bctx;
1111 char *mad;
1112
1113 trace_mad_message("recv", msg->umad.mad, msg->umad_len);
1114
1115 cqe_ctx_id = rdma_protected_qlist_pop_int64(&backend_dev->recv_mads_list);
1116 if (cqe_ctx_id == -ENOENT) {
1117 rdma_warn_report("No more free MADs buffers, waiting for a while");
1118 sleep(THR_POLL_TO);
1119 return;
1120 }
1121
1122 bctx = rdma_rm_get_cqe_ctx(backend_dev->rdma_dev_res, cqe_ctx_id);
1123 if (unlikely(!bctx)) {
1124 rdma_error_report("No matching ctx for req %ld", cqe_ctx_id);
1125 backend_dev->rdma_dev_res->stats.mad_rx_err++;
1126 return;
1127 }
1128
1129 mad = rdma_pci_dma_map(backend_dev->dev, bctx->sge.addr,
1130 bctx->sge.length);
1131 if (!mad || bctx->sge.length < msg->umad_len + MAD_HDR_SIZE) {
1132 backend_dev->rdma_dev_res->stats.mad_rx_err++;
1133 complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_INV_MAD_BUFF,
1134 bctx->up_ctx);
1135 } else {
1136 struct ibv_wc wc = {};
1137 memset(mad, 0, bctx->sge.length);
1138 build_mad_hdr((struct ibv_grh *)mad,
1139 (union ibv_gid *)&msg->umad.hdr.addr.gid, &msg->hdr.sgid,
1140 msg->umad_len);
1141 memcpy(&mad[MAD_HDR_SIZE], msg->umad.mad, msg->umad_len);
1142 rdma_pci_dma_unmap(backend_dev->dev, mad, bctx->sge.length);
1143
1144 wc.byte_len = msg->umad_len;
1145 wc.status = IBV_WC_SUCCESS;
1146 wc.wc_flags = IBV_WC_GRH;
1147 backend_dev->rdma_dev_res->stats.mad_rx++;
1148 comp_handler(bctx->up_ctx, &wc);
1149 }
1150
1151 g_free(bctx);
1152 rdma_rm_dealloc_cqe_ctx(backend_dev->rdma_dev_res, cqe_ctx_id);
1153 }
1154
1155 static inline int rdmacm_mux_can_receive(void *opaque)
1156 {
1157 RdmaBackendDev *backend_dev = (RdmaBackendDev *)opaque;
1158
1159 return rdmacm_mux_can_process_async(backend_dev);
1160 }
1161
1162 static void rdmacm_mux_read(void *opaque, const uint8_t *buf, int size)
1163 {
1164 RdmaBackendDev *backend_dev = (RdmaBackendDev *)opaque;
1165 RdmaCmMuxMsg *msg = (RdmaCmMuxMsg *)buf;
1166
1167 trace_rdmacm_mux("read", msg->hdr.msg_type, msg->hdr.op_code);
1168
1169 if (msg->hdr.msg_type != RDMACM_MUX_MSG_TYPE_REQ &&
1170 msg->hdr.op_code != RDMACM_MUX_OP_CODE_MAD) {
1171 rdma_error_report("Error: Not a MAD request, skipping");
1172 return;
1173 }
1174 process_incoming_mad_req(backend_dev, msg);
1175 }
1176
1177 static int mad_init(RdmaBackendDev *backend_dev, CharBackend *mad_chr_be)
1178 {
1179 int ret;
1180
1181 backend_dev->rdmacm_mux.chr_be = mad_chr_be;
1182
1183 ret = qemu_chr_fe_backend_connected(backend_dev->rdmacm_mux.chr_be);
1184 if (!ret) {
1185 rdma_error_report("Missing chardev for MAD multiplexer");
1186 return -EIO;
1187 }
1188
1189 rdma_protected_qlist_init(&backend_dev->recv_mads_list);
1190
1191 enable_rdmacm_mux_async(backend_dev);
1192
1193 qemu_chr_fe_set_handlers(backend_dev->rdmacm_mux.chr_be,
1194 rdmacm_mux_can_receive, rdmacm_mux_read, NULL,
1195 NULL, backend_dev, NULL, true);
1196
1197 return 0;
1198 }
1199
1200 static void mad_stop(RdmaBackendDev *backend_dev)
1201 {
1202 clean_recv_mads(backend_dev);
1203 }
1204
1205 static void mad_fini(RdmaBackendDev *backend_dev)
1206 {
1207 disable_rdmacm_mux_async(backend_dev);
1208 qemu_chr_fe_disconnect(backend_dev->rdmacm_mux.chr_be);
1209 rdma_protected_qlist_destroy(&backend_dev->recv_mads_list);
1210 }
1211
1212 int rdma_backend_get_gid_index(RdmaBackendDev *backend_dev,
1213 union ibv_gid *gid)
1214 {
1215 union ibv_gid sgid;
1216 int ret;
1217 int i = 0;
1218
1219 do {
1220 ret = ibv_query_gid(backend_dev->context, backend_dev->port_num, i,
1221 &sgid);
1222 i++;
1223 } while (!ret && (memcmp(&sgid, gid, sizeof(*gid))));
1224
1225 trace_rdma_backend_get_gid_index(be64_to_cpu(gid->global.subnet_prefix),
1226 be64_to_cpu(gid->global.interface_id),
1227 i - 1);
1228
1229 return ret ? ret : i - 1;
1230 }
1231
1232 int rdma_backend_add_gid(RdmaBackendDev *backend_dev, const char *ifname,
1233 union ibv_gid *gid)
1234 {
1235 RdmaCmMuxMsg msg = {};
1236 int ret;
1237
1238 trace_rdma_backend_gid_change("add", be64_to_cpu(gid->global.subnet_prefix),
1239 be64_to_cpu(gid->global.interface_id));
1240
1241 msg.hdr.op_code = RDMACM_MUX_OP_CODE_REG;
1242 memcpy(msg.hdr.sgid.raw, gid->raw, sizeof(msg.hdr.sgid));
1243
1244 ret = rdmacm_mux_send(backend_dev, &msg);
1245 if (ret) {
1246 rdma_error_report("Failed to register GID to rdma_umadmux (%d)", ret);
1247 return -EIO;
1248 }
1249
1250 qapi_event_send_rdma_gid_status_changed(ifname, true,
1251 gid->global.subnet_prefix,
1252 gid->global.interface_id);
1253
1254 return ret;
1255 }
1256
1257 int rdma_backend_del_gid(RdmaBackendDev *backend_dev, const char *ifname,
1258 union ibv_gid *gid)
1259 {
1260 RdmaCmMuxMsg msg = {};
1261 int ret;
1262
1263 trace_rdma_backend_gid_change("del", be64_to_cpu(gid->global.subnet_prefix),
1264 be64_to_cpu(gid->global.interface_id));
1265
1266 msg.hdr.op_code = RDMACM_MUX_OP_CODE_UNREG;
1267 memcpy(msg.hdr.sgid.raw, gid->raw, sizeof(msg.hdr.sgid));
1268
1269 ret = rdmacm_mux_send(backend_dev, &msg);
1270 if (ret) {
1271 rdma_error_report("Failed to unregister GID from rdma_umadmux (%d)",
1272 ret);
1273 return -EIO;
1274 }
1275
1276 qapi_event_send_rdma_gid_status_changed(ifname, false,
1277 gid->global.subnet_prefix,
1278 gid->global.interface_id);
1279
1280 return 0;
1281 }
1282
1283 int rdma_backend_init(RdmaBackendDev *backend_dev, PCIDevice *pdev,
1284 RdmaDeviceResources *rdma_dev_res,
1285 const char *backend_device_name, uint8_t port_num,
1286 struct ibv_device_attr *dev_attr, CharBackend *mad_chr_be)
1287 {
1288 int i;
1289 int ret = 0;
1290 int num_ibv_devices;
1291 struct ibv_device **dev_list;
1292
1293 memset(backend_dev, 0, sizeof(*backend_dev));
1294
1295 backend_dev->dev = pdev;
1296 backend_dev->port_num = port_num;
1297 backend_dev->rdma_dev_res = rdma_dev_res;
1298
1299 rdma_backend_register_comp_handler(dummy_comp_handler);
1300
1301 dev_list = ibv_get_device_list(&num_ibv_devices);
1302 if (!dev_list) {
1303 rdma_error_report("Failed to get IB devices list");
1304 return -EIO;
1305 }
1306
1307 if (num_ibv_devices == 0) {
1308 rdma_error_report("No IB devices were found");
1309 ret = -ENXIO;
1310 goto out_free_dev_list;
1311 }
1312
1313 if (backend_device_name) {
1314 for (i = 0; dev_list[i]; ++i) {
1315 if (!strcmp(ibv_get_device_name(dev_list[i]),
1316 backend_device_name)) {
1317 break;
1318 }
1319 }
1320
1321 backend_dev->ib_dev = dev_list[i];
1322 if (!backend_dev->ib_dev) {
1323 rdma_error_report("Failed to find IB device %s",
1324 backend_device_name);
1325 ret = -EIO;
1326 goto out_free_dev_list;
1327 }
1328 } else {
1329 backend_dev->ib_dev = *dev_list;
1330 }
1331
1332 rdma_info_report("uverb device %s", backend_dev->ib_dev->dev_name);
1333
1334 backend_dev->context = ibv_open_device(backend_dev->ib_dev);
1335 if (!backend_dev->context) {
1336 rdma_error_report("Failed to open IB device %s",
1337 ibv_get_device_name(backend_dev->ib_dev));
1338 ret = -EIO;
1339 goto out;
1340 }
1341
1342 backend_dev->channel = ibv_create_comp_channel(backend_dev->context);
1343 if (!backend_dev->channel) {
1344 rdma_error_report("Failed to create IB communication channel");
1345 ret = -EIO;
1346 goto out_close_device;
1347 }
1348
1349 ret = init_device_caps(backend_dev, dev_attr);
1350 if (ret) {
1351 rdma_error_report("Failed to initialize device capabilities");
1352 ret = -EIO;
1353 goto out_destroy_comm_channel;
1354 }
1355
1356
1357 ret = mad_init(backend_dev, mad_chr_be);
1358 if (ret) {
1359 rdma_error_report("Failed to initialize mad");
1360 ret = -EIO;
1361 goto out_destroy_comm_channel;
1362 }
1363
1364 backend_dev->comp_thread.run = false;
1365 backend_dev->comp_thread.is_running = false;
1366
1367 ah_cache_init();
1368
1369 goto out_free_dev_list;
1370
1371 out_destroy_comm_channel:
1372 ibv_destroy_comp_channel(backend_dev->channel);
1373
1374 out_close_device:
1375 ibv_close_device(backend_dev->context);
1376
1377 out_free_dev_list:
1378 ibv_free_device_list(dev_list);
1379
1380 out:
1381 return ret;
1382 }
1383
1384
1385 void rdma_backend_start(RdmaBackendDev *backend_dev)
1386 {
1387 start_comp_thread(backend_dev);
1388 }
1389
1390 void rdma_backend_stop(RdmaBackendDev *backend_dev)
1391 {
1392 mad_stop(backend_dev);
1393 stop_backend_thread(&backend_dev->comp_thread);
1394 }
1395
1396 void rdma_backend_fini(RdmaBackendDev *backend_dev)
1397 {
1398 mad_fini(backend_dev);
1399 g_hash_table_destroy(ah_hash);
1400 ibv_destroy_comp_channel(backend_dev->channel);
1401 ibv_close_device(backend_dev->context);
1402 }
Random patches