2 * QEMU paravirtual RDMA - Resource Manager Implementation
4 * Copyright (C) 2018 Oracle
5 * Copyright (C) 2018 Red Hat Inc
8 * Yuval Shaia <yuval.shaia@oracle.com>
9 * Marcel Apfelbaum <marcel@redhat.com>
11 * This work is licensed under the terms of the GNU GPL, version 2 or later.
12 * See the COPYING file in the top-level directory.
16 #include "qemu/osdep.h"
17 #include "qapi/error.h"
20 #include "rdma_utils.h"
21 #include "rdma_backend.h"
24 /* Page directory and page tables */
25 #define PG_DIR_SZ { TARGET_PAGE_SIZE / sizeof(__u64) }
26 #define PG_TBL_SZ { TARGET_PAGE_SIZE / sizeof(__u64) }
28 static inline void res_tbl_init(const char *name
, RdmaRmResTbl
*tbl
,
29 uint32_t tbl_sz
, uint32_t res_sz
)
31 tbl
->tbl
= g_malloc(tbl_sz
* res_sz
);
33 strncpy(tbl
->name
, name
, MAX_RM_TBL_NAME
);
34 tbl
->name
[MAX_RM_TBL_NAME
- 1] = 0;
36 tbl
->bitmap
= bitmap_new(tbl_sz
);
39 qemu_mutex_init(&tbl
->lock
);
42 static inline void res_tbl_free(RdmaRmResTbl
*tbl
)
44 qemu_mutex_destroy(&tbl
->lock
);
46 bitmap_zero_extend(tbl
->bitmap
, tbl
->tbl_sz
, 0);
49 static inline void *res_tbl_get(RdmaRmResTbl
*tbl
, uint32_t handle
)
51 pr_dbg("%s, handle=%d\n", tbl
->name
, handle
);
53 if ((handle
< tbl
->tbl_sz
) && (test_bit(handle
, tbl
->bitmap
))) {
54 return tbl
->tbl
+ handle
* tbl
->res_sz
;
56 pr_dbg("Invalid handle %d\n", handle
);
61 static inline void *res_tbl_alloc(RdmaRmResTbl
*tbl
, uint32_t *handle
)
63 qemu_mutex_lock(&tbl
->lock
);
65 *handle
= find_first_zero_bit(tbl
->bitmap
, tbl
->tbl_sz
);
66 if (*handle
> tbl
->tbl_sz
) {
67 pr_dbg("Failed to alloc, bitmap is full\n");
68 qemu_mutex_unlock(&tbl
->lock
);
72 set_bit(*handle
, tbl
->bitmap
);
74 qemu_mutex_unlock(&tbl
->lock
);
76 memset(tbl
->tbl
+ *handle
* tbl
->res_sz
, 0, tbl
->res_sz
);
78 pr_dbg("%s, handle=%d\n", tbl
->name
, *handle
);
80 return tbl
->tbl
+ *handle
* tbl
->res_sz
;
83 static inline void res_tbl_dealloc(RdmaRmResTbl
*tbl
, uint32_t handle
)
85 pr_dbg("%s, handle=%d\n", tbl
->name
, handle
);
87 qemu_mutex_lock(&tbl
->lock
);
89 if (handle
< tbl
->tbl_sz
) {
90 clear_bit(handle
, tbl
->bitmap
);
93 qemu_mutex_unlock(&tbl
->lock
);
96 int rdma_rm_alloc_pd(RdmaDeviceResources
*dev_res
, RdmaBackendDev
*backend_dev
,
97 uint32_t *pd_handle
, uint32_t ctx_handle
)
102 pd
= res_tbl_alloc(&dev_res
->pd_tbl
, pd_handle
);
107 ret
= rdma_backend_create_pd(backend_dev
, &pd
->backend_pd
);
110 goto out_tbl_dealloc
;
113 pd
->ctx_handle
= ctx_handle
;
118 res_tbl_dealloc(&dev_res
->pd_tbl
, *pd_handle
);
124 RdmaRmPD
*rdma_rm_get_pd(RdmaDeviceResources
*dev_res
, uint32_t pd_handle
)
126 return res_tbl_get(&dev_res
->pd_tbl
, pd_handle
);
129 void rdma_rm_dealloc_pd(RdmaDeviceResources
*dev_res
, uint32_t pd_handle
)
131 RdmaRmPD
*pd
= rdma_rm_get_pd(dev_res
, pd_handle
);
134 rdma_backend_destroy_pd(&pd
->backend_pd
);
135 res_tbl_dealloc(&dev_res
->pd_tbl
, pd_handle
);
139 int rdma_rm_alloc_mr(RdmaDeviceResources
*dev_res
, uint32_t pd_handle
,
140 uint64_t guest_start
, size_t guest_length
, void *host_virt
,
141 int access_flags
, uint32_t *mr_handle
, uint32_t *lkey
,
150 pd
= rdma_rm_get_pd(dev_res
, pd_handle
);
152 pr_dbg("Invalid PD\n");
156 mr
= res_tbl_alloc(&dev_res
->mr_tbl
, mr_handle
);
158 pr_dbg("Failed to allocate obj in table\n");
163 /* TODO: This is my guess but not so sure that this needs to be
165 length
= TARGET_PAGE_SIZE
;
166 addr
= g_malloc(length
);
168 mr
->user_mr
.host_virt
= host_virt
;
169 pr_dbg("host_virt=0x%p\n", mr
->user_mr
.host_virt
);
170 mr
->user_mr
.length
= guest_length
;
171 pr_dbg("length=%zu\n", guest_length
);
172 mr
->user_mr
.guest_start
= guest_start
;
173 pr_dbg("guest_start=0x%" PRIx64
"\n", mr
->user_mr
.guest_start
);
175 length
= mr
->user_mr
.length
;
176 addr
= mr
->user_mr
.host_virt
;
179 ret
= rdma_backend_create_mr(&mr
->backend_mr
, &pd
->backend_pd
, addr
, length
,
182 pr_dbg("Fail in rdma_backend_create_mr, err=%d\n", ret
);
188 *lkey
= mr
->lkey
= rdma_backend_mr_lkey(&mr
->backend_mr
);
189 *rkey
= mr
->rkey
= rdma_backend_mr_rkey(&mr
->backend_mr
);
191 /* We keep mr_handle in lkey so send and recv get get mr ptr */
196 mr
->pd_handle
= pd_handle
;
201 res_tbl_dealloc(&dev_res
->mr_tbl
, *mr_handle
);
206 RdmaRmMR
*rdma_rm_get_mr(RdmaDeviceResources
*dev_res
, uint32_t mr_handle
)
208 return res_tbl_get(&dev_res
->mr_tbl
, mr_handle
);
211 void rdma_rm_dealloc_mr(RdmaDeviceResources
*dev_res
, uint32_t mr_handle
)
213 RdmaRmMR
*mr
= rdma_rm_get_mr(dev_res
, mr_handle
);
216 rdma_backend_destroy_mr(&mr
->backend_mr
);
217 munmap(mr
->user_mr
.host_virt
, mr
->user_mr
.length
);
218 res_tbl_dealloc(&dev_res
->mr_tbl
, mr_handle
);
222 int rdma_rm_alloc_uc(RdmaDeviceResources
*dev_res
, uint32_t pfn
,
227 /* TODO: Need to make sure pfn is between bar start address and
228 * bsd+RDMA_BAR2_UAR_SIZE
229 if (pfn > RDMA_BAR2_UAR_SIZE) {
230 pr_err("pfn out of range (%d > %d)\n", pfn, RDMA_BAR2_UAR_SIZE);
235 uc
= res_tbl_alloc(&dev_res
->uc_tbl
, uc_handle
);
243 RdmaRmUC
*rdma_rm_get_uc(RdmaDeviceResources
*dev_res
, uint32_t uc_handle
)
245 return res_tbl_get(&dev_res
->uc_tbl
, uc_handle
);
248 void rdma_rm_dealloc_uc(RdmaDeviceResources
*dev_res
, uint32_t uc_handle
)
250 RdmaRmUC
*uc
= rdma_rm_get_uc(dev_res
, uc_handle
);
253 res_tbl_dealloc(&dev_res
->uc_tbl
, uc_handle
);
257 RdmaRmCQ
*rdma_rm_get_cq(RdmaDeviceResources
*dev_res
, uint32_t cq_handle
)
259 return res_tbl_get(&dev_res
->cq_tbl
, cq_handle
);
262 int rdma_rm_alloc_cq(RdmaDeviceResources
*dev_res
, RdmaBackendDev
*backend_dev
,
263 uint32_t cqe
, uint32_t *cq_handle
, void *opaque
)
268 cq
= res_tbl_alloc(&dev_res
->cq_tbl
, cq_handle
);
276 rc
= rdma_backend_create_cq(backend_dev
, &cq
->backend_cq
, cqe
);
285 rdma_rm_dealloc_cq(dev_res
, *cq_handle
);
290 void rdma_rm_req_notify_cq(RdmaDeviceResources
*dev_res
, uint32_t cq_handle
,
295 pr_dbg("cq_handle=%d, notify=0x%x\n", cq_handle
, notify
);
297 cq
= rdma_rm_get_cq(dev_res
, cq_handle
);
303 pr_dbg("notify=%d\n", cq
->notify
);
306 void rdma_rm_dealloc_cq(RdmaDeviceResources
*dev_res
, uint32_t cq_handle
)
310 cq
= rdma_rm_get_cq(dev_res
, cq_handle
);
315 rdma_backend_destroy_cq(&cq
->backend_cq
);
317 res_tbl_dealloc(&dev_res
->cq_tbl
, cq_handle
);
320 RdmaRmQP
*rdma_rm_get_qp(RdmaDeviceResources
*dev_res
, uint32_t qpn
)
322 GBytes
*key
= g_bytes_new(&qpn
, sizeof(qpn
));
324 RdmaRmQP
*qp
= g_hash_table_lookup(dev_res
->qp_hash
, key
);
331 int rdma_rm_alloc_qp(RdmaDeviceResources
*dev_res
, uint32_t pd_handle
,
332 uint8_t qp_type
, uint32_t max_send_wr
,
333 uint32_t max_send_sge
, uint32_t send_cq_handle
,
334 uint32_t max_recv_wr
, uint32_t max_recv_sge
,
335 uint32_t recv_cq_handle
, void *opaque
, uint32_t *qpn
)
343 pr_dbg("qp_type=%d\n", qp_type
);
345 pd
= rdma_rm_get_pd(dev_res
, pd_handle
);
347 pr_err("Invalid pd handle (%d)\n", pd_handle
);
351 scq
= rdma_rm_get_cq(dev_res
, send_cq_handle
);
352 rcq
= rdma_rm_get_cq(dev_res
, recv_cq_handle
);
355 pr_err("Invalid send_cqn or recv_cqn (%d, %d)\n",
356 send_cq_handle
, recv_cq_handle
);
360 qp
= res_tbl_alloc(&dev_res
->qp_tbl
, &rm_qpn
);
364 pr_dbg("rm_qpn=%d\n", rm_qpn
);
367 qp
->qp_state
= IBV_QPS_RESET
;
368 qp
->qp_type
= qp_type
;
369 qp
->send_cq_handle
= send_cq_handle
;
370 qp
->recv_cq_handle
= recv_cq_handle
;
373 rc
= rdma_backend_create_qp(&qp
->backend_qp
, qp_type
, &pd
->backend_pd
,
374 &scq
->backend_cq
, &rcq
->backend_cq
, max_send_wr
,
375 max_recv_wr
, max_send_sge
, max_recv_sge
);
381 *qpn
= rdma_backend_qpn(&qp
->backend_qp
);
382 pr_dbg("rm_qpn=%d, backend_qpn=0x%x\n", rm_qpn
, *qpn
);
383 g_hash_table_insert(dev_res
->qp_hash
, g_bytes_new(qpn
, sizeof(*qpn
)), qp
);
388 res_tbl_dealloc(&dev_res
->qp_tbl
, qp
->qpn
);
393 int rdma_rm_modify_qp(RdmaDeviceResources
*dev_res
, RdmaBackendDev
*backend_dev
,
394 uint32_t qp_handle
, uint32_t attr_mask
,
395 union ibv_gid
*dgid
, uint32_t dqpn
,
396 enum ibv_qp_state qp_state
, uint32_t qkey
,
397 uint32_t rq_psn
, uint32_t sq_psn
)
402 pr_dbg("qpn=%d\n", qp_handle
);
404 qp
= rdma_rm_get_qp(dev_res
, qp_handle
);
409 pr_dbg("qp_type=%d\n", qp
->qp_type
);
410 pr_dbg("attr_mask=0x%x\n", attr_mask
);
412 if (qp
->qp_type
== IBV_QPT_SMI
) {
413 pr_dbg("QP0 unsupported\n");
415 } else if (qp
->qp_type
== IBV_QPT_GSI
) {
420 if (attr_mask
& IBV_QP_STATE
) {
421 qp
->qp_state
= qp_state
;
422 pr_dbg("qp_state=%d\n", qp
->qp_state
);
424 if (qp
->qp_state
== IBV_QPS_INIT
) {
425 ret
= rdma_backend_qp_state_init(backend_dev
, &qp
->backend_qp
,
432 if (qp
->qp_state
== IBV_QPS_RTR
) {
433 ret
= rdma_backend_qp_state_rtr(backend_dev
, &qp
->backend_qp
,
434 qp
->qp_type
, dgid
, dqpn
, rq_psn
,
435 qkey
, attr_mask
& IBV_QP_QKEY
);
441 if (qp
->qp_state
== IBV_QPS_RTS
) {
442 ret
= rdma_backend_qp_state_rts(&qp
->backend_qp
, qp
->qp_type
,
444 attr_mask
& IBV_QP_QKEY
);
454 int rdma_rm_query_qp(RdmaDeviceResources
*dev_res
, RdmaBackendDev
*backend_dev
,
455 uint32_t qp_handle
, struct ibv_qp_attr
*attr
,
456 int attr_mask
, struct ibv_qp_init_attr
*init_attr
)
460 pr_dbg("qpn=%d\n", qp_handle
);
462 qp
= rdma_rm_get_qp(dev_res
, qp_handle
);
467 pr_dbg("qp_type=%d\n", qp
->qp_type
);
469 return rdma_backend_query_qp(&qp
->backend_qp
, attr
, attr_mask
, init_attr
);
472 void rdma_rm_dealloc_qp(RdmaDeviceResources
*dev_res
, uint32_t qp_handle
)
477 key
= g_bytes_new(&qp_handle
, sizeof(qp_handle
));
478 qp
= g_hash_table_lookup(dev_res
->qp_hash
, key
);
479 g_hash_table_remove(dev_res
->qp_hash
, key
);
486 rdma_backend_destroy_qp(&qp
->backend_qp
);
488 res_tbl_dealloc(&dev_res
->qp_tbl
, qp
->qpn
);
491 void *rdma_rm_get_cqe_ctx(RdmaDeviceResources
*dev_res
, uint32_t cqe_ctx_id
)
495 cqe_ctx
= res_tbl_get(&dev_res
->cqe_ctx_tbl
, cqe_ctx_id
);
500 pr_dbg("ctx=%p\n", *cqe_ctx
);
505 int rdma_rm_alloc_cqe_ctx(RdmaDeviceResources
*dev_res
, uint32_t *cqe_ctx_id
,
510 cqe_ctx
= res_tbl_alloc(&dev_res
->cqe_ctx_tbl
, cqe_ctx_id
);
515 pr_dbg("ctx=%p\n", ctx
);
521 void rdma_rm_dealloc_cqe_ctx(RdmaDeviceResources
*dev_res
, uint32_t cqe_ctx_id
)
523 res_tbl_dealloc(&dev_res
->cqe_ctx_tbl
, cqe_ctx_id
);
526 static void destroy_qp_hash_key(gpointer data
)
531 int rdma_rm_init(RdmaDeviceResources
*dev_res
, struct ibv_device_attr
*dev_attr
,
534 dev_res
->qp_hash
= g_hash_table_new_full(g_bytes_hash
, g_bytes_equal
,
535 destroy_qp_hash_key
, NULL
);
536 if (!dev_res
->qp_hash
) {
540 res_tbl_init("PD", &dev_res
->pd_tbl
, dev_attr
->max_pd
, sizeof(RdmaRmPD
));
541 res_tbl_init("CQ", &dev_res
->cq_tbl
, dev_attr
->max_cq
, sizeof(RdmaRmCQ
));
542 res_tbl_init("MR", &dev_res
->mr_tbl
, dev_attr
->max_mr
, sizeof(RdmaRmMR
));
543 res_tbl_init("QP", &dev_res
->qp_tbl
, dev_attr
->max_qp
, sizeof(RdmaRmQP
));
544 res_tbl_init("CQE_CTX", &dev_res
->cqe_ctx_tbl
, dev_attr
->max_qp
*
545 dev_attr
->max_qp_wr
, sizeof(void *));
546 res_tbl_init("UC", &dev_res
->uc_tbl
, MAX_UCS
, sizeof(RdmaRmUC
));
551 void rdma_rm_fini(RdmaDeviceResources
*dev_res
)
553 res_tbl_free(&dev_res
->uc_tbl
);
554 res_tbl_free(&dev_res
->cqe_ctx_tbl
);
555 res_tbl_free(&dev_res
->qp_tbl
);
556 res_tbl_free(&dev_res
->cq_tbl
);
557 res_tbl_free(&dev_res
->mr_tbl
);
558 res_tbl_free(&dev_res
->pd_tbl
);
559 g_hash_table_destroy(dev_res
->qp_hash
);