2 * QEMU paravirtual RDMA - Resource Manager Implementation
4 * Copyright (C) 2018 Oracle
5 * Copyright (C) 2018 Red Hat Inc
8 * Yuval Shaia <yuval.shaia@oracle.com>
9 * Marcel Apfelbaum <marcel@redhat.com>
11 * This work is licensed under the terms of the GNU GPL, version 2 or later.
12 * See the COPYING file in the top-level directory.
16 #include "qemu/osdep.h"
17 #include "qapi/error.h"
20 #include "rdma_utils.h"
21 #include "rdma_backend.h"
24 /* Page directory and page tables */
25 #define PG_DIR_SZ { TARGET_PAGE_SIZE / sizeof(__u64) }
26 #define PG_TBL_SZ { TARGET_PAGE_SIZE / sizeof(__u64) }
28 static inline void res_tbl_init(const char *name
, RdmaRmResTbl
*tbl
,
29 uint32_t tbl_sz
, uint32_t res_sz
)
31 tbl
->tbl
= g_malloc(tbl_sz
* res_sz
);
33 strncpy(tbl
->name
, name
, MAX_RM_TBL_NAME
);
34 tbl
->name
[MAX_RM_TBL_NAME
- 1] = 0;
36 tbl
->bitmap
= bitmap_new(tbl_sz
);
39 qemu_mutex_init(&tbl
->lock
);
42 static inline void res_tbl_free(RdmaRmResTbl
*tbl
)
44 qemu_mutex_destroy(&tbl
->lock
);
49 static inline void *res_tbl_get(RdmaRmResTbl
*tbl
, uint32_t handle
)
51 pr_dbg("%s, handle=%d\n", tbl
->name
, handle
);
53 if ((handle
< tbl
->tbl_sz
) && (test_bit(handle
, tbl
->bitmap
))) {
54 return tbl
->tbl
+ handle
* tbl
->res_sz
;
56 pr_dbg("Invalid handle %d\n", handle
);
61 static inline void *res_tbl_alloc(RdmaRmResTbl
*tbl
, uint32_t *handle
)
63 qemu_mutex_lock(&tbl
->lock
);
65 *handle
= find_first_zero_bit(tbl
->bitmap
, tbl
->tbl_sz
);
66 if (*handle
> tbl
->tbl_sz
) {
67 pr_dbg("Failed to alloc, bitmap is full\n");
68 qemu_mutex_unlock(&tbl
->lock
);
72 set_bit(*handle
, tbl
->bitmap
);
74 qemu_mutex_unlock(&tbl
->lock
);
76 memset(tbl
->tbl
+ *handle
* tbl
->res_sz
, 0, tbl
->res_sz
);
78 pr_dbg("%s, handle=%d\n", tbl
->name
, *handle
);
80 return tbl
->tbl
+ *handle
* tbl
->res_sz
;
83 static inline void res_tbl_dealloc(RdmaRmResTbl
*tbl
, uint32_t handle
)
85 pr_dbg("%s, handle=%d\n", tbl
->name
, handle
);
87 qemu_mutex_lock(&tbl
->lock
);
89 if (handle
< tbl
->tbl_sz
) {
90 clear_bit(handle
, tbl
->bitmap
);
93 qemu_mutex_unlock(&tbl
->lock
);
96 int rdma_rm_alloc_pd(RdmaDeviceResources
*dev_res
, RdmaBackendDev
*backend_dev
,
97 uint32_t *pd_handle
, uint32_t ctx_handle
)
102 pd
= res_tbl_alloc(&dev_res
->pd_tbl
, pd_handle
);
107 ret
= rdma_backend_create_pd(backend_dev
, &pd
->backend_pd
);
110 goto out_tbl_dealloc
;
113 pd
->ctx_handle
= ctx_handle
;
118 res_tbl_dealloc(&dev_res
->pd_tbl
, *pd_handle
);
124 RdmaRmPD
*rdma_rm_get_pd(RdmaDeviceResources
*dev_res
, uint32_t pd_handle
)
126 return res_tbl_get(&dev_res
->pd_tbl
, pd_handle
);
129 void rdma_rm_dealloc_pd(RdmaDeviceResources
*dev_res
, uint32_t pd_handle
)
131 RdmaRmPD
*pd
= rdma_rm_get_pd(dev_res
, pd_handle
);
134 rdma_backend_destroy_pd(&pd
->backend_pd
);
135 res_tbl_dealloc(&dev_res
->pd_tbl
, pd_handle
);
139 int rdma_rm_alloc_mr(RdmaDeviceResources
*dev_res
, uint32_t pd_handle
,
140 uint64_t guest_start
, size_t guest_length
, void *host_virt
,
141 int access_flags
, uint32_t *mr_handle
, uint32_t *lkey
,
148 pd
= rdma_rm_get_pd(dev_res
, pd_handle
);
150 pr_dbg("Invalid PD\n");
154 mr
= res_tbl_alloc(&dev_res
->mr_tbl
, mr_handle
);
156 pr_dbg("Failed to allocate obj in table\n");
159 pr_dbg("mr_handle=%d\n", *mr_handle
);
161 pr_dbg("host_virt=0x%p\n", host_virt
);
162 pr_dbg("guest_start=0x%" PRIx64
"\n", guest_start
);
163 pr_dbg("length=%zu\n", guest_length
);
166 mr
->virt
= host_virt
;
167 mr
->start
= guest_start
;
168 mr
->length
= guest_length
;
169 mr
->virt
+= (mr
->start
& (TARGET_PAGE_SIZE
- 1));
171 ret
= rdma_backend_create_mr(&mr
->backend_mr
, &pd
->backend_pd
, mr
->virt
,
172 mr
->length
, access_flags
);
174 pr_dbg("Fail in rdma_backend_create_mr, err=%d\n", ret
);
180 /* We keep mr_handle in lkey so send and recv get get mr ptr */
184 mr
->pd_handle
= pd_handle
;
189 res_tbl_dealloc(&dev_res
->mr_tbl
, *mr_handle
);
194 RdmaRmMR
*rdma_rm_get_mr(RdmaDeviceResources
*dev_res
, uint32_t mr_handle
)
196 return res_tbl_get(&dev_res
->mr_tbl
, mr_handle
);
199 void rdma_rm_dealloc_mr(RdmaDeviceResources
*dev_res
, uint32_t mr_handle
)
201 RdmaRmMR
*mr
= rdma_rm_get_mr(dev_res
, mr_handle
);
204 rdma_backend_destroy_mr(&mr
->backend_mr
);
205 pr_dbg("start=0x%" PRIx64
"\n", mr
->start
);
207 mr
->virt
-= (mr
->start
& (TARGET_PAGE_SIZE
- 1));
208 munmap(mr
->virt
, mr
->length
);
210 res_tbl_dealloc(&dev_res
->mr_tbl
, mr_handle
);
214 int rdma_rm_alloc_uc(RdmaDeviceResources
*dev_res
, uint32_t pfn
,
219 /* TODO: Need to make sure pfn is between bar start address and
220 * bsd+RDMA_BAR2_UAR_SIZE
221 if (pfn > RDMA_BAR2_UAR_SIZE) {
222 pr_err("pfn out of range (%d > %d)\n", pfn, RDMA_BAR2_UAR_SIZE);
227 uc
= res_tbl_alloc(&dev_res
->uc_tbl
, uc_handle
);
235 RdmaRmUC
*rdma_rm_get_uc(RdmaDeviceResources
*dev_res
, uint32_t uc_handle
)
237 return res_tbl_get(&dev_res
->uc_tbl
, uc_handle
);
240 void rdma_rm_dealloc_uc(RdmaDeviceResources
*dev_res
, uint32_t uc_handle
)
242 RdmaRmUC
*uc
= rdma_rm_get_uc(dev_res
, uc_handle
);
245 res_tbl_dealloc(&dev_res
->uc_tbl
, uc_handle
);
249 RdmaRmCQ
*rdma_rm_get_cq(RdmaDeviceResources
*dev_res
, uint32_t cq_handle
)
251 return res_tbl_get(&dev_res
->cq_tbl
, cq_handle
);
254 int rdma_rm_alloc_cq(RdmaDeviceResources
*dev_res
, RdmaBackendDev
*backend_dev
,
255 uint32_t cqe
, uint32_t *cq_handle
, void *opaque
)
260 cq
= res_tbl_alloc(&dev_res
->cq_tbl
, cq_handle
);
266 cq
->notify
= CNT_CLEAR
;
268 rc
= rdma_backend_create_cq(backend_dev
, &cq
->backend_cq
, cqe
);
277 rdma_rm_dealloc_cq(dev_res
, *cq_handle
);
282 void rdma_rm_req_notify_cq(RdmaDeviceResources
*dev_res
, uint32_t cq_handle
,
287 pr_dbg("cq_handle=%d, notify=0x%x\n", cq_handle
, notify
);
289 cq
= rdma_rm_get_cq(dev_res
, cq_handle
);
294 if (cq
->notify
!= CNT_SET
) {
295 cq
->notify
= notify
? CNT_ARM
: CNT_CLEAR
;
298 pr_dbg("notify=%d\n", cq
->notify
);
301 void rdma_rm_dealloc_cq(RdmaDeviceResources
*dev_res
, uint32_t cq_handle
)
305 cq
= rdma_rm_get_cq(dev_res
, cq_handle
);
310 rdma_backend_destroy_cq(&cq
->backend_cq
);
312 res_tbl_dealloc(&dev_res
->cq_tbl
, cq_handle
);
315 RdmaRmQP
*rdma_rm_get_qp(RdmaDeviceResources
*dev_res
, uint32_t qpn
)
317 GBytes
*key
= g_bytes_new(&qpn
, sizeof(qpn
));
319 RdmaRmQP
*qp
= g_hash_table_lookup(dev_res
->qp_hash
, key
);
326 int rdma_rm_alloc_qp(RdmaDeviceResources
*dev_res
, uint32_t pd_handle
,
327 uint8_t qp_type
, uint32_t max_send_wr
,
328 uint32_t max_send_sge
, uint32_t send_cq_handle
,
329 uint32_t max_recv_wr
, uint32_t max_recv_sge
,
330 uint32_t recv_cq_handle
, void *opaque
, uint32_t *qpn
)
338 pr_dbg("qp_type=%d\n", qp_type
);
340 pd
= rdma_rm_get_pd(dev_res
, pd_handle
);
342 pr_err("Invalid pd handle (%d)\n", pd_handle
);
346 scq
= rdma_rm_get_cq(dev_res
, send_cq_handle
);
347 rcq
= rdma_rm_get_cq(dev_res
, recv_cq_handle
);
350 pr_err("Invalid send_cqn or recv_cqn (%d, %d)\n",
351 send_cq_handle
, recv_cq_handle
);
355 if (qp_type
== IBV_QPT_GSI
) {
356 scq
->notify
= CNT_SET
;
357 rcq
->notify
= CNT_SET
;
360 qp
= res_tbl_alloc(&dev_res
->qp_tbl
, &rm_qpn
);
364 pr_dbg("rm_qpn=%d\n", rm_qpn
);
367 qp
->qp_state
= IBV_QPS_RESET
;
368 qp
->qp_type
= qp_type
;
369 qp
->send_cq_handle
= send_cq_handle
;
370 qp
->recv_cq_handle
= recv_cq_handle
;
373 rc
= rdma_backend_create_qp(&qp
->backend_qp
, qp_type
, &pd
->backend_pd
,
374 &scq
->backend_cq
, &rcq
->backend_cq
, max_send_wr
,
375 max_recv_wr
, max_send_sge
, max_recv_sge
);
381 *qpn
= rdma_backend_qpn(&qp
->backend_qp
);
382 pr_dbg("rm_qpn=%d, backend_qpn=0x%x\n", rm_qpn
, *qpn
);
383 g_hash_table_insert(dev_res
->qp_hash
, g_bytes_new(qpn
, sizeof(*qpn
)), qp
);
388 res_tbl_dealloc(&dev_res
->qp_tbl
, qp
->qpn
);
393 int rdma_rm_modify_qp(RdmaDeviceResources
*dev_res
, RdmaBackendDev
*backend_dev
,
394 uint32_t qp_handle
, uint32_t attr_mask
, uint8_t sgid_idx
,
395 union ibv_gid
*dgid
, uint32_t dqpn
,
396 enum ibv_qp_state qp_state
, uint32_t qkey
,
397 uint32_t rq_psn
, uint32_t sq_psn
)
402 pr_dbg("qpn=0x%x\n", qp_handle
);
403 pr_dbg("qkey=0x%x\n", qkey
);
405 qp
= rdma_rm_get_qp(dev_res
, qp_handle
);
410 pr_dbg("qp_type=%d\n", qp
->qp_type
);
411 pr_dbg("attr_mask=0x%x\n", attr_mask
);
413 if (qp
->qp_type
== IBV_QPT_SMI
) {
414 pr_dbg("QP0 unsupported\n");
416 } else if (qp
->qp_type
== IBV_QPT_GSI
) {
421 if (attr_mask
& IBV_QP_STATE
) {
422 qp
->qp_state
= qp_state
;
423 pr_dbg("qp_state=%d\n", qp
->qp_state
);
425 if (qp
->qp_state
== IBV_QPS_INIT
) {
426 ret
= rdma_backend_qp_state_init(backend_dev
, &qp
->backend_qp
,
433 if (qp
->qp_state
== IBV_QPS_RTR
) {
434 /* Get backend gid index */
435 pr_dbg("Guest sgid_idx=%d\n", sgid_idx
);
436 sgid_idx
= rdma_rm_get_backend_gid_index(dev_res
, backend_dev
,
438 if (sgid_idx
<= 0) { /* TODO check also less than bk.max_sgid */
439 pr_dbg("Fail to get bk sgid_idx for sgid_idx %d\n", sgid_idx
);
443 ret
= rdma_backend_qp_state_rtr(backend_dev
, &qp
->backend_qp
,
444 qp
->qp_type
, sgid_idx
, dgid
, dqpn
,
446 attr_mask
& IBV_QP_QKEY
);
452 if (qp
->qp_state
== IBV_QPS_RTS
) {
453 ret
= rdma_backend_qp_state_rts(&qp
->backend_qp
, qp
->qp_type
,
455 attr_mask
& IBV_QP_QKEY
);
465 int rdma_rm_query_qp(RdmaDeviceResources
*dev_res
, RdmaBackendDev
*backend_dev
,
466 uint32_t qp_handle
, struct ibv_qp_attr
*attr
,
467 int attr_mask
, struct ibv_qp_init_attr
*init_attr
)
471 pr_dbg("qpn=0x%x\n", qp_handle
);
473 qp
= rdma_rm_get_qp(dev_res
, qp_handle
);
478 pr_dbg("qp_type=%d\n", qp
->qp_type
);
480 return rdma_backend_query_qp(&qp
->backend_qp
, attr
, attr_mask
, init_attr
);
483 void rdma_rm_dealloc_qp(RdmaDeviceResources
*dev_res
, uint32_t qp_handle
)
488 key
= g_bytes_new(&qp_handle
, sizeof(qp_handle
));
489 qp
= g_hash_table_lookup(dev_res
->qp_hash
, key
);
490 g_hash_table_remove(dev_res
->qp_hash
, key
);
497 rdma_backend_destroy_qp(&qp
->backend_qp
);
499 res_tbl_dealloc(&dev_res
->qp_tbl
, qp
->qpn
);
502 void *rdma_rm_get_cqe_ctx(RdmaDeviceResources
*dev_res
, uint32_t cqe_ctx_id
)
506 cqe_ctx
= res_tbl_get(&dev_res
->cqe_ctx_tbl
, cqe_ctx_id
);
511 pr_dbg("ctx=%p\n", *cqe_ctx
);
516 int rdma_rm_alloc_cqe_ctx(RdmaDeviceResources
*dev_res
, uint32_t *cqe_ctx_id
,
521 cqe_ctx
= res_tbl_alloc(&dev_res
->cqe_ctx_tbl
, cqe_ctx_id
);
526 pr_dbg("ctx=%p\n", ctx
);
532 void rdma_rm_dealloc_cqe_ctx(RdmaDeviceResources
*dev_res
, uint32_t cqe_ctx_id
)
534 res_tbl_dealloc(&dev_res
->cqe_ctx_tbl
, cqe_ctx_id
);
537 int rdma_rm_add_gid(RdmaDeviceResources
*dev_res
, RdmaBackendDev
*backend_dev
,
538 const char *ifname
, union ibv_gid
*gid
, int gid_idx
)
542 rc
= rdma_backend_add_gid(backend_dev
, ifname
, gid
);
544 pr_dbg("Fail to add gid\n");
548 memcpy(&dev_res
->port
.gid_tbl
[gid_idx
].gid
, gid
, sizeof(*gid
));
553 int rdma_rm_del_gid(RdmaDeviceResources
*dev_res
, RdmaBackendDev
*backend_dev
,
554 const char *ifname
, int gid_idx
)
558 if (!dev_res
->port
.gid_tbl
[gid_idx
].gid
.global
.interface_id
) {
562 rc
= rdma_backend_del_gid(backend_dev
, ifname
,
563 &dev_res
->port
.gid_tbl
[gid_idx
].gid
);
565 pr_dbg("Fail to delete gid\n");
569 memset(dev_res
->port
.gid_tbl
[gid_idx
].gid
.raw
, 0,
570 sizeof(dev_res
->port
.gid_tbl
[gid_idx
].gid
));
571 dev_res
->port
.gid_tbl
[gid_idx
].backend_gid_index
= -1;
576 int rdma_rm_get_backend_gid_index(RdmaDeviceResources
*dev_res
,
577 RdmaBackendDev
*backend_dev
, int sgid_idx
)
579 if (unlikely(sgid_idx
< 0 || sgid_idx
> MAX_PORT_GIDS
)) {
580 pr_dbg("Got invalid sgid_idx %d\n", sgid_idx
);
584 if (unlikely(dev_res
->port
.gid_tbl
[sgid_idx
].backend_gid_index
== -1)) {
585 dev_res
->port
.gid_tbl
[sgid_idx
].backend_gid_index
=
586 rdma_backend_get_gid_index(backend_dev
,
587 &dev_res
->port
.gid_tbl
[sgid_idx
].gid
);
590 pr_dbg("backend_gid_index=%d\n",
591 dev_res
->port
.gid_tbl
[sgid_idx
].backend_gid_index
);
593 return dev_res
->port
.gid_tbl
[sgid_idx
].backend_gid_index
;
596 static void destroy_qp_hash_key(gpointer data
)
601 static void init_ports(RdmaDeviceResources
*dev_res
)
605 memset(&dev_res
->port
, 0, sizeof(dev_res
->port
));
607 dev_res
->port
.state
= IBV_PORT_DOWN
;
608 for (i
= 0; i
< MAX_PORT_GIDS
; i
++) {
609 dev_res
->port
.gid_tbl
[i
].backend_gid_index
= -1;
613 static void fini_ports(RdmaDeviceResources
*dev_res
,
614 RdmaBackendDev
*backend_dev
, const char *ifname
)
618 dev_res
->port
.state
= IBV_PORT_DOWN
;
619 for (i
= 0; i
< MAX_PORT_GIDS
; i
++) {
620 rdma_rm_del_gid(dev_res
, backend_dev
, ifname
, i
);
624 int rdma_rm_init(RdmaDeviceResources
*dev_res
, struct ibv_device_attr
*dev_attr
,
627 dev_res
->qp_hash
= g_hash_table_new_full(g_bytes_hash
, g_bytes_equal
,
628 destroy_qp_hash_key
, NULL
);
629 if (!dev_res
->qp_hash
) {
633 res_tbl_init("PD", &dev_res
->pd_tbl
, dev_attr
->max_pd
, sizeof(RdmaRmPD
));
634 res_tbl_init("CQ", &dev_res
->cq_tbl
, dev_attr
->max_cq
, sizeof(RdmaRmCQ
));
635 res_tbl_init("MR", &dev_res
->mr_tbl
, dev_attr
->max_mr
, sizeof(RdmaRmMR
));
636 res_tbl_init("QP", &dev_res
->qp_tbl
, dev_attr
->max_qp
, sizeof(RdmaRmQP
));
637 res_tbl_init("CQE_CTX", &dev_res
->cqe_ctx_tbl
, dev_attr
->max_qp
*
638 dev_attr
->max_qp_wr
, sizeof(void *));
639 res_tbl_init("UC", &dev_res
->uc_tbl
, MAX_UCS
, sizeof(RdmaRmUC
));
646 void rdma_rm_fini(RdmaDeviceResources
*dev_res
, RdmaBackendDev
*backend_dev
,
649 fini_ports(dev_res
, backend_dev
, ifname
);
651 res_tbl_free(&dev_res
->uc_tbl
);
652 res_tbl_free(&dev_res
->cqe_ctx_tbl
);
653 res_tbl_free(&dev_res
->qp_tbl
);
654 res_tbl_free(&dev_res
->mr_tbl
);
655 res_tbl_free(&dev_res
->cq_tbl
);
656 res_tbl_free(&dev_res
->pd_tbl
);
658 g_hash_table_destroy(dev_res
->qp_hash
);