hw/rdma/rdma_backend.c

   1 /*
   2  * QEMU paravirtual RDMA - Generic RDMA backend
   3  *
   4  * Copyright (C) 2018 Oracle
   5  * Copyright (C) 2018 Red Hat Inc
   6  *
   7  * Authors:
   8  *     Yuval Shaia <yuval.shaia@oracle.com>
   9  *     Marcel Apfelbaum <marcel@redhat.com>
  10  *
  11  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  12  * See the COPYING file in the top-level directory.
  13  *
  14  */
  15
  16 #include <qemu/osdep.h>
  17 #include <qemu/error-report.h>
  18 #include <qapi/error.h>
  19
  20 #include <infiniband/verbs.h>
  21
  22 #include "trace.h"
  23 #include "rdma_utils.h"
  24 #include "rdma_rm.h"
  25 #include "rdma_backend.h"
  26
  27 /* Vendor Errors */
  28 #define VENDOR_ERR_FAIL_BACKEND     0x201
  29 #define VENDOR_ERR_TOO_MANY_SGES    0x202
  30 #define VENDOR_ERR_NOMEM            0x203
  31 #define VENDOR_ERR_QP0              0x204
  32 #define VENDOR_ERR_NO_SGE           0x205
  33 #define VENDOR_ERR_MAD_SEND         0x206
  34 #define VENDOR_ERR_INVLKEY          0x207
  35 #define VENDOR_ERR_MR_SMALL         0x208
  36
  37 #define THR_NAME_LEN 16
  38
  39 typedef struct BackendCtx {
  40     uint64_t req_id;
  41     void *up_ctx;
  42     bool is_tx_req;
  43 } BackendCtx;
  44
  45 static void (*comp_handler)(int status, unsigned int vendor_err, void *ctx);
  46
  47 static void dummy_comp_handler(int status, unsigned int vendor_err, void *ctx)
  48 {
  49     pr_err("No completion handler is registered\n");
  50 }
  51
  52 static void poll_cq(RdmaDeviceResources *rdma_dev_res, struct ibv_cq *ibcq)
  53 {
  54     int i, ne;
  55     BackendCtx *bctx;
  56     struct ibv_wc wc[2];
  57
  58     pr_dbg("Entering poll_cq loop on cq %p\n", ibcq);
  59     do {
  60         ne = ibv_poll_cq(ibcq, ARRAY_SIZE(wc), wc);
  61
  62         pr_dbg("Got %d completion(s) from cq %p\n", ne, ibcq);
  63
  64         for (i = 0; i < ne; i++) {
  65             pr_dbg("wr_id=0x%lx\n", wc[i].wr_id);
  66             pr_dbg("status=%d\n", wc[i].status);
  67
  68             bctx = rdma_rm_get_cqe_ctx(rdma_dev_res, wc[i].wr_id);
  69             if (unlikely(!bctx)) {
  70                 pr_dbg("Error: Failed to find ctx for req %ld\n", wc[i].wr_id);
  71                 continue;
  72             }
  73             pr_dbg("Processing %s CQE\n", bctx->is_tx_req ? "send" : "recv");
  74
  75             comp_handler(wc[i].status, wc[i].vendor_err, bctx->up_ctx);
  76
  77             rdma_rm_dealloc_cqe_ctx(rdma_dev_res, wc[i].wr_id);
  78             g_free(bctx);
  79         }
  80     } while (ne > 0);
  81
  82     if (ne < 0) {
  83         pr_dbg("Got error %d from ibv_poll_cq\n", ne);
  84     }
  85 }
  86
  87 static void *comp_handler_thread(void *arg)
  88 {
  89     RdmaBackendDev *backend_dev = (RdmaBackendDev *)arg;
  90     int rc;
  91     struct ibv_cq *ev_cq;
  92     void *ev_ctx;
  93
  94     pr_dbg("Starting\n");
  95
  96     while (backend_dev->comp_thread.run) {
  97         pr_dbg("Waiting for completion on channel %p\n", backend_dev->channel);
  98         rc = ibv_get_cq_event(backend_dev->channel, &ev_cq, &ev_ctx);
  99         pr_dbg("ibv_get_cq_event=%d\n", rc);
 100         if (unlikely(rc)) {
 101             pr_dbg("---> ibv_get_cq_event (%d)\n", rc);
 102             continue;
 103         }
 104
 105         rc = ibv_req_notify_cq(ev_cq, 0);
 106         if (unlikely(rc)) {
 107             pr_dbg("Error %d from ibv_req_notify_cq\n", rc);
 108         }
 109
 110         poll_cq(backend_dev->rdma_dev_res, ev_cq);
 111
 112         ibv_ack_cq_events(ev_cq, 1);
 113     }
 114
 115     pr_dbg("Going down\n");
 116
 117     /* TODO: Post cqe for all remaining buffs that were posted */
 118
 119     return NULL;
 120 }
 121
 122 void rdma_backend_register_comp_handler(void (*handler)(int status,
 123                                         unsigned int vendor_err, void *ctx))
 124 {
 125     comp_handler = handler;
 126 }
 127
 128 void rdma_backend_unregister_comp_handler(void)
 129 {
 130     rdma_backend_register_comp_handler(dummy_comp_handler);
 131 }
 132
 133 int rdma_backend_query_port(RdmaBackendDev *backend_dev,
 134                             struct ibv_port_attr *port_attr)
 135 {
 136     int rc;
 137
 138     rc = ibv_query_port(backend_dev->context, backend_dev->port_num, port_attr);
 139     if (rc) {
 140         pr_dbg("Error %d from ibv_query_port\n", rc);
 141         return -EIO;
 142     }
 143
 144     return 0;
 145 }
 146
 147 void rdma_backend_poll_cq(RdmaDeviceResources *rdma_dev_res, RdmaBackendCQ *cq)
 148 {
 149     poll_cq(rdma_dev_res, cq->ibcq);
 150 }
 151
 152 static GHashTable *ah_hash;
 153
 154 static struct ibv_ah *create_ah(RdmaBackendDev *backend_dev, struct ibv_pd *pd,
 155                                 uint8_t sgid_idx, union ibv_gid *dgid)
 156 {
 157     GBytes *ah_key = g_bytes_new(dgid, sizeof(*dgid));
 158     struct ibv_ah *ah = g_hash_table_lookup(ah_hash, ah_key);
 159
 160     if (ah) {
 161         trace_create_ah_cache_hit(be64_to_cpu(dgid->global.subnet_prefix),
 162                                   be64_to_cpu(dgid->global.interface_id));
 163         g_bytes_unref(ah_key);
 164     } else {
 165         struct ibv_ah_attr ah_attr = {
 166             .is_global     = 1,
 167             .port_num      = backend_dev->port_num,
 168             .grh.hop_limit = 1,
 169         };
 170
 171         ah_attr.grh.dgid = *dgid;
 172         ah_attr.grh.sgid_index = sgid_idx;
 173
 174         ah = ibv_create_ah(pd, &ah_attr);
 175         if (ah) {
 176             g_hash_table_insert(ah_hash, ah_key, ah);
 177         } else {
 178             g_bytes_unref(ah_key);
 179             pr_dbg("ibv_create_ah failed for gid <%lx %lx>\n",
 180                     be64_to_cpu(dgid->global.subnet_prefix),
 181                     be64_to_cpu(dgid->global.interface_id));
 182         }
 183
 184         trace_create_ah_cache_miss(be64_to_cpu(dgid->global.subnet_prefix),
 185                                    be64_to_cpu(dgid->global.interface_id));
 186     }
 187
 188     return ah;
 189 }
 190
 191 static void destroy_ah_hash_key(gpointer data)
 192 {
 193     g_bytes_unref(data);
 194 }
 195
 196 static void destroy_ah_hast_data(gpointer data)
 197 {
 198     struct ibv_ah *ah = data;
 199
 200     ibv_destroy_ah(ah);
 201 }
 202
 203 static void ah_cache_init(void)
 204 {
 205     ah_hash = g_hash_table_new_full(g_bytes_hash, g_bytes_equal,
 206                                     destroy_ah_hash_key, destroy_ah_hast_data);
 207 }
 208
 209 static int build_host_sge_array(RdmaDeviceResources *rdma_dev_res,
 210                                 struct ibv_sge *dsge, struct ibv_sge *ssge,
 211                                 uint8_t num_sge)
 212 {
 213     RdmaRmMR *mr;
 214     int ssge_idx;
 215
 216     pr_dbg("num_sge=%d\n", num_sge);
 217
 218     for (ssge_idx = 0; ssge_idx < num_sge; ssge_idx++) {
 219         mr = rdma_rm_get_mr(rdma_dev_res, ssge[ssge_idx].lkey);
 220         if (unlikely(!mr)) {
 221             pr_dbg("Invalid lkey 0x%x\n", ssge[ssge_idx].lkey);
 222             return VENDOR_ERR_INVLKEY | ssge[ssge_idx].lkey;
 223         }
 224
 225         dsge->addr = mr->user_mr.host_virt + ssge[ssge_idx].addr -
 226                      mr->user_mr.guest_start;
 227         dsge->length = ssge[ssge_idx].length;
 228         dsge->lkey = rdma_backend_mr_lkey(&mr->backend_mr);
 229
 230         pr_dbg("ssge->addr=0x%lx\n", (uint64_t)ssge[ssge_idx].addr);
 231         pr_dbg("dsge->addr=0x%lx\n", dsge->addr);
 232         pr_dbg("dsge->length=%d\n", dsge->length);
 233         pr_dbg("dsge->lkey=0x%x\n", dsge->lkey);
 234
 235         dsge++;
 236     }
 237
 238     return 0;
 239 }
 240
 241 void rdma_backend_post_send(RdmaBackendDev *backend_dev,
 242                             RdmaBackendQP *qp, uint8_t qp_type,
 243                             struct ibv_sge *sge, uint32_t num_sge,
 244                             union ibv_gid *dgid, uint32_t dqpn,
 245                             uint32_t dqkey, void *ctx)
 246 {
 247     BackendCtx *bctx;
 248     struct ibv_sge new_sge[MAX_SGE];
 249     uint32_t bctx_id;
 250     int rc;
 251     struct ibv_send_wr wr = {0}, *bad_wr;
 252
 253     if (!qp->ibqp) { /* This field does not get initialized for QP0 and QP1 */
 254         if (qp_type == IBV_QPT_SMI) {
 255             pr_dbg("QP0 unsupported\n");
 256             comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_QP0, ctx);
 257         } else if (qp_type == IBV_QPT_GSI) {
 258             pr_dbg("QP1\n");
 259             comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_MAD_SEND, ctx);
 260         }
 261         pr_dbg("qp->ibqp is NULL for qp_type %d!!!\n", qp_type);
 262         return;
 263     }
 264
 265     pr_dbg("num_sge=%d\n", num_sge);
 266     if (!num_sge) {
 267         pr_dbg("num_sge=0\n");
 268         comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_NO_SGE, ctx);
 269         return;
 270     }
 271
 272     bctx = g_malloc0(sizeof(*bctx));
 273     bctx->up_ctx = ctx;
 274     bctx->is_tx_req = 1;
 275
 276     rc = rdma_rm_alloc_cqe_ctx(backend_dev->rdma_dev_res, &bctx_id, bctx);
 277     if (unlikely(rc)) {
 278         pr_dbg("Failed to allocate cqe_ctx\n");
 279         comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_NOMEM, ctx);
 280         goto out_free_bctx;
 281     }
 282
 283     rc = build_host_sge_array(backend_dev->rdma_dev_res, new_sge, sge, num_sge);
 284     if (rc) {
 285         pr_dbg("Error: Failed to build host SGE array\n");
 286         comp_handler(IBV_WC_GENERAL_ERR, rc, ctx);
 287         goto out_dealloc_cqe_ctx;
 288     }
 289
 290     if (qp_type == IBV_QPT_UD) {
 291         wr.wr.ud.ah = create_ah(backend_dev, qp->ibpd,
 292                                 backend_dev->backend_gid_idx, dgid);
 293         wr.wr.ud.remote_qpn = dqpn;
 294         wr.wr.ud.remote_qkey = dqkey;
 295     }
 296
 297     wr.num_sge = num_sge;
 298     wr.opcode = IBV_WR_SEND;
 299     wr.send_flags = IBV_SEND_SIGNALED;
 300     wr.sg_list = new_sge;
 301     wr.wr_id = bctx_id;
 302
 303     rc = ibv_post_send(qp->ibqp, &wr, &bad_wr);
 304     pr_dbg("ibv_post_send=%d\n", rc);
 305     if (rc) {
 306         pr_dbg("Fail (%d, %d) to post send WQE to qpn %d\n", rc, errno,
 307                 qp->ibqp->qp_num);
 308         comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx);
 309         goto out_dealloc_cqe_ctx;
 310     }
 311
 312     return;
 313
 314 out_dealloc_cqe_ctx:
 315     rdma_rm_dealloc_cqe_ctx(backend_dev->rdma_dev_res, bctx_id);
 316
 317 out_free_bctx:
 318     g_free(bctx);
 319 }
 320
 321 void rdma_backend_post_recv(RdmaBackendDev *backend_dev,
 322                             RdmaDeviceResources *rdma_dev_res,
 323                             RdmaBackendQP *qp, uint8_t qp_type,
 324                             struct ibv_sge *sge, uint32_t num_sge, void *ctx)
 325 {
 326     BackendCtx *bctx;
 327     struct ibv_sge new_sge[MAX_SGE];
 328     uint32_t bctx_id;
 329     int rc;
 330     struct ibv_recv_wr wr = {0}, *bad_wr;
 331
 332     if (!qp->ibqp) { /* This field does not get initialized for QP0 and QP1 */
 333         if (qp_type == IBV_QPT_SMI) {
 334             pr_dbg("QP0 unsupported\n");
 335             comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_QP0, ctx);
 336         }
 337         if (qp_type == IBV_QPT_GSI) {
 338             pr_dbg("QP1\n");
 339             comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_MAD_SEND, ctx);
 340         }
 341         return;
 342     }
 343
 344     pr_dbg("num_sge=%d\n", num_sge);
 345     if (!num_sge) {
 346         pr_dbg("num_sge=0\n");
 347         comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_NO_SGE, ctx);
 348         return;
 349     }
 350
 351     bctx = g_malloc0(sizeof(*bctx));
 352     bctx->up_ctx = ctx;
 353     bctx->is_tx_req = 0;
 354
 355     rc = rdma_rm_alloc_cqe_ctx(rdma_dev_res, &bctx_id, bctx);
 356     if (unlikely(rc)) {
 357         pr_dbg("Failed to allocate cqe_ctx\n");
 358         comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_NOMEM, ctx);
 359         goto out_free_bctx;
 360     }
 361
 362     rc = build_host_sge_array(rdma_dev_res, new_sge, sge, num_sge);
 363     if (rc) {
 364         pr_dbg("Error: Failed to build host SGE array\n");
 365         comp_handler(IBV_WC_GENERAL_ERR, rc, ctx);
 366         goto out_dealloc_cqe_ctx;
 367     }
 368
 369     wr.num_sge = num_sge;
 370     wr.sg_list = new_sge;
 371     wr.wr_id = bctx_id;
 372     rc = ibv_post_recv(qp->ibqp, &wr, &bad_wr);
 373     pr_dbg("ibv_post_recv=%d\n", rc);
 374     if (rc) {
 375         pr_dbg("Fail (%d, %d) to post recv WQE to qpn %d\n", rc, errno,
 376                 qp->ibqp->qp_num);
 377         comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx);
 378         goto out_dealloc_cqe_ctx;
 379     }
 380
 381     return;
 382
 383 out_dealloc_cqe_ctx:
 384     rdma_rm_dealloc_cqe_ctx(rdma_dev_res, bctx_id);
 385
 386 out_free_bctx:
 387     g_free(bctx);
 388 }
 389
 390 int rdma_backend_create_pd(RdmaBackendDev *backend_dev, RdmaBackendPD *pd)
 391 {
 392     pd->ibpd = ibv_alloc_pd(backend_dev->context);
 393
 394     return pd->ibpd ? 0 : -EIO;
 395 }
 396
 397 void rdma_backend_destroy_pd(RdmaBackendPD *pd)
 398 {
 399     if (pd->ibpd) {
 400         ibv_dealloc_pd(pd->ibpd);
 401     }
 402 }
 403
 404 int rdma_backend_create_mr(RdmaBackendMR *mr, RdmaBackendPD *pd, uint64_t addr,
 405                            size_t length, int access)
 406 {
 407     pr_dbg("addr=0x%lx\n", addr);
 408     pr_dbg("len=%ld\n", length);
 409     mr->ibmr = ibv_reg_mr(pd->ibpd, (void *)addr, length, access);
 410     if (mr->ibmr) {
 411         pr_dbg("lkey=0x%x\n", mr->ibmr->lkey);
 412         pr_dbg("rkey=0x%x\n", mr->ibmr->rkey);
 413         mr->ibpd = pd->ibpd;
 414     }
 415
 416     return mr->ibmr ? 0 : -EIO;
 417 }
 418
 419 void rdma_backend_destroy_mr(RdmaBackendMR *mr)
 420 {
 421     if (mr->ibmr) {
 422         ibv_dereg_mr(mr->ibmr);
 423     }
 424 }
 425
 426 int rdma_backend_create_cq(RdmaBackendDev *backend_dev, RdmaBackendCQ *cq,
 427                            int cqe)
 428 {
 429     int rc;
 430
 431     pr_dbg("cqe=%d\n", cqe);
 432
 433     pr_dbg("dev->channel=%p\n", backend_dev->channel);
 434     cq->ibcq = ibv_create_cq(backend_dev->context, cqe + 1, NULL,
 435                              backend_dev->channel, 0);
 436
 437     if (cq->ibcq) {
 438         rc = ibv_req_notify_cq(cq->ibcq, 0);
 439         if (rc) {
 440             pr_dbg("Error %d from ibv_req_notify_cq\n", rc);
 441         }
 442         cq->backend_dev = backend_dev;
 443     }
 444
 445     return cq->ibcq ? 0 : -EIO;
 446 }
 447
 448 void rdma_backend_destroy_cq(RdmaBackendCQ *cq)
 449 {
 450     if (cq->ibcq) {
 451         ibv_destroy_cq(cq->ibcq);
 452     }
 453 }
 454
 455 int rdma_backend_create_qp(RdmaBackendQP *qp, uint8_t qp_type,
 456                            RdmaBackendPD *pd, RdmaBackendCQ *scq,
 457                            RdmaBackendCQ *rcq, uint32_t max_send_wr,
 458                            uint32_t max_recv_wr, uint32_t max_send_sge,
 459                            uint32_t max_recv_sge)
 460 {
 461     struct ibv_qp_init_attr attr = {0};
 462
 463     qp->ibqp = 0;
 464     pr_dbg("qp_type=%d\n", qp_type);
 465
 466     switch (qp_type) {
 467     case IBV_QPT_GSI:
 468         pr_dbg("QP1 unsupported\n");
 469         return 0;
 470
 471     case IBV_QPT_RC:
 472         /* fall through */
 473     case IBV_QPT_UD:
 474         /* do nothing */
 475         break;
 476
 477     default:
 478         pr_dbg("Unsupported QP type %d\n", qp_type);
 479         return -EIO;
 480     }
 481
 482     attr.qp_type = qp_type;
 483     attr.send_cq = scq->ibcq;
 484     attr.recv_cq = rcq->ibcq;
 485     attr.cap.max_send_wr = max_send_wr;
 486     attr.cap.max_recv_wr = max_recv_wr;
 487     attr.cap.max_send_sge = max_send_sge;
 488     attr.cap.max_recv_sge = max_recv_sge;
 489
 490     pr_dbg("max_send_wr=%d\n", max_send_wr);
 491     pr_dbg("max_recv_wr=%d\n", max_recv_wr);
 492     pr_dbg("max_send_sge=%d\n", max_send_sge);
 493     pr_dbg("max_recv_sge=%d\n", max_recv_sge);
 494
 495     qp->ibqp = ibv_create_qp(pd->ibpd, &attr);
 496     if (likely(!qp->ibqp)) {
 497         pr_dbg("Error from ibv_create_qp\n");
 498         return -EIO;
 499     }
 500
 501     qp->ibpd = pd->ibpd;
 502
 503     /* TODO: Query QP to get max_inline_data and save it to be used in send */
 504
 505     pr_dbg("qpn=0x%x\n", qp->ibqp->qp_num);
 506
 507     return 0;
 508 }
 509
 510 int rdma_backend_qp_state_init(RdmaBackendDev *backend_dev, RdmaBackendQP *qp,
 511                                uint8_t qp_type, uint32_t qkey)
 512 {
 513     struct ibv_qp_attr attr = {0};
 514     int rc, attr_mask;
 515
 516     pr_dbg("qpn=0x%x\n", qp->ibqp->qp_num);
 517     pr_dbg("sport_num=%d\n", backend_dev->port_num);
 518
 519     attr_mask = IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT;
 520     attr.qp_state        = IBV_QPS_INIT;
 521     attr.pkey_index      = 0;
 522     attr.port_num        = backend_dev->port_num;
 523
 524     switch (qp_type) {
 525     case IBV_QPT_RC:
 526         attr_mask |= IBV_QP_ACCESS_FLAGS;
 527         break;
 528
 529     case IBV_QPT_UD:
 530         attr.qkey = qkey;
 531         attr_mask |= IBV_QP_QKEY;
 532         break;
 533
 534     default:
 535         pr_dbg("Unsupported QP type %d\n", qp_type);
 536         return -EIO;
 537     }
 538
 539     rc = ibv_modify_qp(qp->ibqp, &attr, attr_mask);
 540     if (rc) {
 541         pr_dbg("Error %d from ibv_modify_qp\n", rc);
 542         return -EIO;
 543     }
 544
 545     return 0;
 546 }
 547
 548 int rdma_backend_qp_state_rtr(RdmaBackendDev *backend_dev, RdmaBackendQP *qp,
 549                               uint8_t qp_type, union ibv_gid *dgid,
 550                               uint32_t dqpn, uint32_t rq_psn, uint32_t qkey,
 551                               bool use_qkey)
 552 {
 553     struct ibv_qp_attr attr = {0};
 554     union ibv_gid ibv_gid = {
 555         .global.interface_id = dgid->global.interface_id,
 556         .global.subnet_prefix = dgid->global.subnet_prefix
 557     };
 558     int rc, attr_mask;
 559
 560     attr.qp_state = IBV_QPS_RTR;
 561     attr_mask = IBV_QP_STATE;
 562
 563     switch (qp_type) {
 564     case IBV_QPT_RC:
 565         pr_dbg("dgid=0x%lx,%lx\n",
 566                be64_to_cpu(ibv_gid.global.subnet_prefix),
 567                be64_to_cpu(ibv_gid.global.interface_id));
 568         pr_dbg("dqpn=0x%x\n", dqpn);
 569         pr_dbg("sgid_idx=%d\n", backend_dev->backend_gid_idx);
 570         pr_dbg("sport_num=%d\n", backend_dev->port_num);
 571         pr_dbg("rq_psn=0x%x\n", rq_psn);
 572
 573         attr.path_mtu               = IBV_MTU_1024;
 574         attr.dest_qp_num            = dqpn;
 575         attr.max_dest_rd_atomic     = 1;
 576         attr.min_rnr_timer          = 12;
 577         attr.ah_attr.port_num       = backend_dev->port_num;
 578         attr.ah_attr.is_global      = 1;
 579         attr.ah_attr.grh.hop_limit  = 1;
 580         attr.ah_attr.grh.dgid       = ibv_gid;
 581         attr.ah_attr.grh.sgid_index = backend_dev->backend_gid_idx;
 582         attr.rq_psn                 = rq_psn;
 583
 584         attr_mask |= IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN |
 585                      IBV_QP_RQ_PSN | IBV_QP_MAX_DEST_RD_ATOMIC |
 586                      IBV_QP_MIN_RNR_TIMER;
 587         break;
 588
 589     case IBV_QPT_UD:
 590         if (use_qkey) {
 591             pr_dbg("qkey=0x%x\n", qkey);
 592             attr.qkey = qkey;
 593             attr_mask |= IBV_QP_QKEY;
 594         }
 595         break;
 596     }
 597
 598     rc = ibv_modify_qp(qp->ibqp, &attr, attr_mask);
 599     if (rc) {
 600         pr_dbg("Error %d from ibv_modify_qp\n", rc);
 601         return -EIO;
 602     }
 603
 604     return 0;
 605 }
 606
 607 int rdma_backend_qp_state_rts(RdmaBackendQP *qp, uint8_t qp_type,
 608                               uint32_t sq_psn, uint32_t qkey, bool use_qkey)
 609 {
 610     struct ibv_qp_attr attr = {0};
 611     int rc, attr_mask;
 612
 613     pr_dbg("qpn=0x%x\n", qp->ibqp->qp_num);
 614     pr_dbg("sq_psn=0x%x\n", sq_psn);
 615
 616     attr.qp_state = IBV_QPS_RTS;
 617     attr.sq_psn = sq_psn;
 618     attr_mask = IBV_QP_STATE | IBV_QP_SQ_PSN;
 619
 620     switch (qp_type) {
 621     case IBV_QPT_RC:
 622         attr.timeout       = 14;
 623         attr.retry_cnt     = 7;
 624         attr.rnr_retry     = 7;
 625         attr.max_rd_atomic = 1;
 626
 627         attr_mask |= IBV_QP_TIMEOUT | IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY |
 628                      IBV_QP_MAX_QP_RD_ATOMIC;
 629         break;
 630
 631     case IBV_QPT_UD:
 632         if (use_qkey) {
 633             pr_dbg("qkey=0x%x\n", qkey);
 634             attr.qkey = qkey;
 635             attr_mask |= IBV_QP_QKEY;
 636         }
 637         break;
 638     }
 639
 640     rc = ibv_modify_qp(qp->ibqp, &attr, attr_mask);
 641     if (rc) {
 642         pr_dbg("Error %d from ibv_modify_qp\n", rc);
 643         return -EIO;
 644     }
 645
 646     return 0;
 647 }
 648
 649 int rdma_backend_query_qp(RdmaBackendQP *qp, struct ibv_qp_attr *attr,
 650                           int attr_mask, struct ibv_qp_init_attr *init_attr)
 651 {
 652     if (!qp->ibqp) {
 653         pr_dbg("QP1\n");
 654         attr->qp_state = IBV_QPS_RTS;
 655         return 0;
 656     }
 657
 658     return ibv_query_qp(qp->ibqp, attr, attr_mask, init_attr);
 659 }
 660
 661 void rdma_backend_destroy_qp(RdmaBackendQP *qp)
 662 {
 663     if (qp->ibqp) {
 664         ibv_destroy_qp(qp->ibqp);
 665     }
 666 }
 667
 668 #define CHK_ATTR(req, dev, member, fmt) ({ \
 669     pr_dbg("%s="fmt","fmt"\n", #member, dev.member, req->member); \
 670     if (req->member > dev.member) { \
 671         warn_report("%s = 0x%lx is higher than host device capability 0x%lx", \
 672                     #member, (uint64_t)req->member, (uint64_t)dev.member); \
 673         req->member = dev.member; \
 674     } \
 675     pr_dbg("%s="fmt"\n", #member, req->member); })
 676
 677 static int init_device_caps(RdmaBackendDev *backend_dev,
 678                             struct ibv_device_attr *dev_attr)
 679 {
 680     if (ibv_query_device(backend_dev->context, &backend_dev->dev_attr)) {
 681         return -EIO;
 682     }
 683
 684     CHK_ATTR(dev_attr, backend_dev->dev_attr, max_mr_size, "%ld");
 685     CHK_ATTR(dev_attr, backend_dev->dev_attr, max_qp, "%d");
 686     CHK_ATTR(dev_attr, backend_dev->dev_attr, max_sge, "%d");
 687     CHK_ATTR(dev_attr, backend_dev->dev_attr, max_qp_wr, "%d");
 688     CHK_ATTR(dev_attr, backend_dev->dev_attr, max_cq, "%d");
 689     CHK_ATTR(dev_attr, backend_dev->dev_attr, max_cqe, "%d");
 690     CHK_ATTR(dev_attr, backend_dev->dev_attr, max_mr, "%d");
 691     CHK_ATTR(dev_attr, backend_dev->dev_attr, max_pd, "%d");
 692     CHK_ATTR(dev_attr, backend_dev->dev_attr, max_qp_rd_atom, "%d");
 693     CHK_ATTR(dev_attr, backend_dev->dev_attr, max_qp_init_rd_atom, "%d");
 694     CHK_ATTR(dev_attr, backend_dev->dev_attr, max_ah, "%d");
 695
 696     return 0;
 697 }
 698
 699 int rdma_backend_init(RdmaBackendDev *backend_dev,
 700                       RdmaDeviceResources *rdma_dev_res,
 701                       const char *backend_device_name, uint8_t port_num,
 702                       uint8_t backend_gid_idx, struct ibv_device_attr *dev_attr,
 703                       Error **errp)
 704 {
 705     int i;
 706     int ret = 0;
 707     int num_ibv_devices;
 708     char thread_name[THR_NAME_LEN] = {0};
 709     struct ibv_device **dev_list;
 710     struct ibv_port_attr port_attr;
 711
 712     backend_dev->backend_gid_idx = backend_gid_idx;
 713     backend_dev->port_num = port_num;
 714     backend_dev->rdma_dev_res = rdma_dev_res;
 715
 716     rdma_backend_register_comp_handler(dummy_comp_handler);
 717
 718     dev_list = ibv_get_device_list(&num_ibv_devices);
 719     if (!dev_list) {
 720         error_setg(errp, "Failed to get IB devices list");
 721         return -EIO;
 722     }
 723
 724     if (num_ibv_devices == 0) {
 725         error_setg(errp, "No IB devices were found");
 726         ret = -ENXIO;
 727         goto out_free_dev_list;
 728     }
 729
 730     if (backend_device_name) {
 731         for (i = 0; dev_list[i]; ++i) {
 732             if (!strcmp(ibv_get_device_name(dev_list[i]),
 733                         backend_device_name)) {
 734                 break;
 735             }
 736         }
 737
 738         backend_dev->ib_dev = dev_list[i];
 739         if (!backend_dev->ib_dev) {
 740             error_setg(errp, "Failed to find IB device %s",
 741                        backend_device_name);
 742             ret = -EIO;
 743             goto out_free_dev_list;
 744         }
 745     } else {
 746         backend_dev->ib_dev = *dev_list;
 747     }
 748
 749     pr_dbg("Using backend device %s, port %d, gid_idx %d\n",
 750            ibv_get_device_name(backend_dev->ib_dev),
 751            backend_dev->port_num, backend_dev->backend_gid_idx);
 752
 753     backend_dev->context = ibv_open_device(backend_dev->ib_dev);
 754     if (!backend_dev->context) {
 755         error_setg(errp, "Failed to open IB device");
 756         ret = -EIO;
 757         goto out;
 758     }
 759
 760     backend_dev->channel = ibv_create_comp_channel(backend_dev->context);
 761     if (!backend_dev->channel) {
 762         error_setg(errp, "Failed to create IB communication channel");
 763         ret = -EIO;
 764         goto out_close_device;
 765     }
 766     pr_dbg("dev->backend_dev.channel=%p\n", backend_dev->channel);
 767
 768     ret = ibv_query_port(backend_dev->context, backend_dev->port_num,
 769                          &port_attr);
 770     if (ret) {
 771         error_setg(errp, "Error %d from ibv_query_port", ret);
 772         ret = -EIO;
 773         goto out_destroy_comm_channel;
 774     }
 775
 776     if (backend_dev->backend_gid_idx > port_attr.gid_tbl_len) {
 777         error_setg(errp, "Invalid backend_gid_idx, should be less than %d",
 778                    port_attr.gid_tbl_len);
 779         goto out_destroy_comm_channel;
 780     }
 781
 782     ret = init_device_caps(backend_dev, dev_attr);
 783     if (ret) {
 784         error_setg(errp, "Failed to initialize device capabilities");
 785         ret = -EIO;
 786         goto out_destroy_comm_channel;
 787     }
 788
 789     ret = ibv_query_gid(backend_dev->context, backend_dev->port_num,
 790                          backend_dev->backend_gid_idx, &backend_dev->gid);
 791     if (ret) {
 792         error_setg(errp, "Failed to query gid %d",
 793                    backend_dev->backend_gid_idx);
 794         ret = -EIO;
 795         goto out_destroy_comm_channel;
 796     }
 797     pr_dbg("subnet_prefix=0x%lx\n",
 798            be64_to_cpu(backend_dev->gid.global.subnet_prefix));
 799     pr_dbg("interface_id=0x%lx\n",
 800            be64_to_cpu(backend_dev->gid.global.interface_id));
 801
 802     snprintf(thread_name, sizeof(thread_name), "rdma_comp_%s",
 803              ibv_get_device_name(backend_dev->ib_dev));
 804     backend_dev->comp_thread.run = true;
 805     qemu_thread_create(&backend_dev->comp_thread.thread, thread_name,
 806                        comp_handler_thread, backend_dev, QEMU_THREAD_DETACHED);
 807
 808     ah_cache_init();
 809
 810     goto out_free_dev_list;
 811
 812 out_destroy_comm_channel:
 813     ibv_destroy_comp_channel(backend_dev->channel);
 814
 815 out_close_device:
 816     ibv_close_device(backend_dev->context);
 817
 818 out_free_dev_list:
 819     ibv_free_device_list(dev_list);
 820
 821 out:
 822     return ret;
 823 }
 824
 825 void rdma_backend_fini(RdmaBackendDev *backend_dev)
 826 {
 827     g_hash_table_destroy(ah_hash);
 828     ibv_destroy_comp_channel(backend_dev->channel);
 829     ibv_close_device(backend_dev->context);
 830 }