net/vhost-vdpa.c

   1 /*
   2  * vhost-vdpa.c
   3  *
   4  * Copyright(c) 2017-2018 Intel Corporation.
   5  * Copyright(c) 2020 Red Hat, Inc.
   6  *
   7  * This work is licensed under the terms of the GNU GPL, version 2 or later.
   8  * See the COPYING file in the top-level directory.
   9  *
  10  */
  11
  12 #include "qemu/osdep.h"
  13 #include "clients.h"
  14 #include "hw/virtio/virtio-net.h"
  15 #include "net/vhost_net.h"
  16 #include "net/vhost-vdpa.h"
  17 #include "hw/virtio/vhost-vdpa.h"
  18 #include "qemu/config-file.h"
  19 #include "qemu/error-report.h"
  20 #include "qemu/log.h"
  21 #include "qemu/memalign.h"
  22 #include "qemu/option.h"
  23 #include "qapi/error.h"
  24 #include <linux/vhost.h>
  25 #include <sys/ioctl.h>
  26 #include <err.h>
  27 #include "standard-headers/linux/virtio_net.h"
  28 #include "monitor/monitor.h"
  29 #include "hw/virtio/vhost.h"
  30
  31 /* Todo:need to add the multiqueue support here */
  32 typedef struct VhostVDPAState {
  33     NetClientState nc;
  34     struct vhost_vdpa vhost_vdpa;
  35     VHostNetState *vhost_net;
  36
  37     /* Control commands shadow buffers */
  38     void *cvq_cmd_out_buffer;
  39     virtio_net_ctrl_ack *status;
  40
  41     bool started;
  42 } VhostVDPAState;
  43
  44 const int vdpa_feature_bits[] = {
  45     VIRTIO_F_NOTIFY_ON_EMPTY,
  46     VIRTIO_RING_F_INDIRECT_DESC,
  47     VIRTIO_RING_F_EVENT_IDX,
  48     VIRTIO_F_ANY_LAYOUT,
  49     VIRTIO_F_VERSION_1,
  50     VIRTIO_NET_F_CSUM,
  51     VIRTIO_NET_F_GUEST_CSUM,
  52     VIRTIO_NET_F_GSO,
  53     VIRTIO_NET_F_GUEST_TSO4,
  54     VIRTIO_NET_F_GUEST_TSO6,
  55     VIRTIO_NET_F_GUEST_ECN,
  56     VIRTIO_NET_F_GUEST_UFO,
  57     VIRTIO_NET_F_HOST_TSO4,
  58     VIRTIO_NET_F_HOST_TSO6,
  59     VIRTIO_NET_F_HOST_ECN,
  60     VIRTIO_NET_F_HOST_UFO,
  61     VIRTIO_NET_F_MRG_RXBUF,
  62     VIRTIO_NET_F_MTU,
  63     VIRTIO_NET_F_CTRL_RX,
  64     VIRTIO_NET_F_CTRL_RX_EXTRA,
  65     VIRTIO_NET_F_CTRL_VLAN,
  66     VIRTIO_NET_F_GUEST_ANNOUNCE,
  67     VIRTIO_NET_F_CTRL_MAC_ADDR,
  68     VIRTIO_NET_F_RSS,
  69     VIRTIO_NET_F_MQ,
  70     VIRTIO_NET_F_CTRL_VQ,
  71     VIRTIO_F_IOMMU_PLATFORM,
  72     VIRTIO_F_RING_PACKED,
  73     VIRTIO_NET_F_RSS,
  74     VIRTIO_NET_F_HASH_REPORT,
  75     VIRTIO_NET_F_GUEST_ANNOUNCE,
  76     VIRTIO_NET_F_STATUS,
  77     VHOST_INVALID_FEATURE_BIT
  78 };
  79
  80 /** Supported device specific feature bits with SVQ */
  81 static const uint64_t vdpa_svq_device_features =
  82     BIT_ULL(VIRTIO_NET_F_CSUM) |
  83     BIT_ULL(VIRTIO_NET_F_GUEST_CSUM) |
  84     BIT_ULL(VIRTIO_NET_F_MTU) |
  85     BIT_ULL(VIRTIO_NET_F_MAC) |
  86     BIT_ULL(VIRTIO_NET_F_GUEST_TSO4) |
  87     BIT_ULL(VIRTIO_NET_F_GUEST_TSO6) |
  88     BIT_ULL(VIRTIO_NET_F_GUEST_ECN) |
  89     BIT_ULL(VIRTIO_NET_F_GUEST_UFO) |
  90     BIT_ULL(VIRTIO_NET_F_HOST_TSO4) |
  91     BIT_ULL(VIRTIO_NET_F_HOST_TSO6) |
  92     BIT_ULL(VIRTIO_NET_F_HOST_ECN) |
  93     BIT_ULL(VIRTIO_NET_F_HOST_UFO) |
  94     BIT_ULL(VIRTIO_NET_F_MRG_RXBUF) |
  95     BIT_ULL(VIRTIO_NET_F_STATUS) |
  96     BIT_ULL(VIRTIO_NET_F_CTRL_VQ) |
  97     BIT_ULL(VIRTIO_NET_F_MQ) |
  98     BIT_ULL(VIRTIO_F_ANY_LAYOUT) |
  99     BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR) |
 100     BIT_ULL(VIRTIO_NET_F_RSC_EXT) |
 101     BIT_ULL(VIRTIO_NET_F_STANDBY);
 102
 103 VHostNetState *vhost_vdpa_get_vhost_net(NetClientState *nc)
 104 {
 105     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
 106     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
 107     return s->vhost_net;
 108 }
 109
 110 static int vhost_vdpa_net_check_device_id(struct vhost_net *net)
 111 {
 112     uint32_t device_id;
 113     int ret;
 114     struct vhost_dev *hdev;
 115
 116     hdev = (struct vhost_dev *)&net->dev;
 117     ret = hdev->vhost_ops->vhost_get_device_id(hdev, &device_id);
 118     if (device_id != VIRTIO_ID_NET) {
 119         return -ENOTSUP;
 120     }
 121     return ret;
 122 }
 123
 124 static int vhost_vdpa_add(NetClientState *ncs, void *be,
 125                           int queue_pair_index, int nvqs)
 126 {
 127     VhostNetOptions options;
 128     struct vhost_net *net = NULL;
 129     VhostVDPAState *s;
 130     int ret;
 131
 132     options.backend_type = VHOST_BACKEND_TYPE_VDPA;
 133     assert(ncs->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
 134     s = DO_UPCAST(VhostVDPAState, nc, ncs);
 135     options.net_backend = ncs;
 136     options.opaque      = be;
 137     options.busyloop_timeout = 0;
 138     options.nvqs = nvqs;
 139
 140     net = vhost_net_init(&options);
 141     if (!net) {
 142         error_report("failed to init vhost_net for queue");
 143         goto err_init;
 144     }
 145     s->vhost_net = net;
 146     ret = vhost_vdpa_net_check_device_id(net);
 147     if (ret) {
 148         goto err_check;
 149     }
 150     return 0;
 151 err_check:
 152     vhost_net_cleanup(net);
 153     g_free(net);
 154 err_init:
 155     return -1;
 156 }
 157
 158 static void vhost_vdpa_cleanup(NetClientState *nc)
 159 {
 160     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
 161     struct vhost_dev *dev = &s->vhost_net->dev;
 162
 163     qemu_vfree(s->cvq_cmd_out_buffer);
 164     qemu_vfree(s->status);
 165     if (dev->vq_index + dev->nvqs == dev->vq_index_end) {
 166         g_clear_pointer(&s->vhost_vdpa.iova_tree, vhost_iova_tree_delete);
 167     }
 168     if (s->vhost_net) {
 169         vhost_net_cleanup(s->vhost_net);
 170         g_free(s->vhost_net);
 171         s->vhost_net = NULL;
 172     }
 173      if (s->vhost_vdpa.device_fd >= 0) {
 174         qemu_close(s->vhost_vdpa.device_fd);
 175         s->vhost_vdpa.device_fd = -1;
 176     }
 177 }
 178
 179 static bool vhost_vdpa_has_vnet_hdr(NetClientState *nc)
 180 {
 181     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
 182
 183     return true;
 184 }
 185
 186 static bool vhost_vdpa_has_ufo(NetClientState *nc)
 187 {
 188     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
 189     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
 190     uint64_t features = 0;
 191     features |= (1ULL << VIRTIO_NET_F_HOST_UFO);
 192     features = vhost_net_get_features(s->vhost_net, features);
 193     return !!(features & (1ULL << VIRTIO_NET_F_HOST_UFO));
 194
 195 }
 196
 197 static bool vhost_vdpa_check_peer_type(NetClientState *nc, ObjectClass *oc,
 198                                        Error **errp)
 199 {
 200     const char *driver = object_class_get_name(oc);
 201
 202     if (!g_str_has_prefix(driver, "virtio-net-")) {
 203         error_setg(errp, "vhost-vdpa requires frontend driver virtio-net-*");
 204         return false;
 205     }
 206
 207     return true;
 208 }
 209
 210 /** Dummy receive in case qemu falls back to userland tap networking */
 211 static ssize_t vhost_vdpa_receive(NetClientState *nc, const uint8_t *buf,
 212                                   size_t size)
 213 {
 214     return 0;
 215 }
 216
 217 static NetClientInfo net_vhost_vdpa_info = {
 218         .type = NET_CLIENT_DRIVER_VHOST_VDPA,
 219         .size = sizeof(VhostVDPAState),
 220         .receive = vhost_vdpa_receive,
 221         .cleanup = vhost_vdpa_cleanup,
 222         .has_vnet_hdr = vhost_vdpa_has_vnet_hdr,
 223         .has_ufo = vhost_vdpa_has_ufo,
 224         .check_peer_type = vhost_vdpa_check_peer_type,
 225 };
 226
 227 static void vhost_vdpa_cvq_unmap_buf(struct vhost_vdpa *v, void *addr)
 228 {
 229     VhostIOVATree *tree = v->iova_tree;
 230     DMAMap needle = {
 231         /*
 232          * No need to specify size or to look for more translations since
 233          * this contiguous chunk was allocated by us.
 234          */
 235         .translated_addr = (hwaddr)(uintptr_t)addr,
 236     };
 237     const DMAMap *map = vhost_iova_tree_find_iova(tree, &needle);
 238     int r;
 239
 240     if (unlikely(!map)) {
 241         error_report("Cannot locate expected map");
 242         return;
 243     }
 244
 245     r = vhost_vdpa_dma_unmap(v, map->iova, map->size + 1);
 246     if (unlikely(r != 0)) {
 247         error_report("Device cannot unmap: %s(%d)", g_strerror(r), r);
 248     }
 249
 250     vhost_iova_tree_remove(tree, *map);
 251 }
 252
 253 static size_t vhost_vdpa_net_cvq_cmd_len(void)
 254 {
 255     /*
 256      * MAC_TABLE_SET is the ctrl command that produces the longer out buffer.
 257      * In buffer is always 1 byte, so it should fit here
 258      */
 259     return sizeof(struct virtio_net_ctrl_hdr) +
 260            2 * sizeof(struct virtio_net_ctrl_mac) +
 261            MAC_TABLE_ENTRIES * ETH_ALEN;
 262 }
 263
 264 static size_t vhost_vdpa_net_cvq_cmd_page_len(void)
 265 {
 266     return ROUND_UP(vhost_vdpa_net_cvq_cmd_len(), qemu_real_host_page_size());
 267 }
 268
 269 /** Map CVQ buffer. */
 270 static int vhost_vdpa_cvq_map_buf(struct vhost_vdpa *v, void *buf, size_t size,
 271                                   bool write)
 272 {
 273     DMAMap map = {};
 274     int r;
 275
 276     map.translated_addr = (hwaddr)(uintptr_t)buf;
 277     map.size = size - 1;
 278     map.perm = write ? IOMMU_RW : IOMMU_RO,
 279     r = vhost_iova_tree_map_alloc(v->iova_tree, &map);
 280     if (unlikely(r != IOVA_OK)) {
 281         error_report("Cannot map injected element");
 282         return r;
 283     }
 284
 285     r = vhost_vdpa_dma_map(v, map.iova, vhost_vdpa_net_cvq_cmd_page_len(), buf,
 286                            !write);
 287     if (unlikely(r < 0)) {
 288         goto dma_map_err;
 289     }
 290
 291     return 0;
 292
 293 dma_map_err:
 294     vhost_iova_tree_remove(v->iova_tree, map);
 295     return r;
 296 }
 297
 298 static int vhost_vdpa_net_cvq_start(NetClientState *nc)
 299 {
 300     VhostVDPAState *s;
 301     int r;
 302
 303     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
 304
 305     s = DO_UPCAST(VhostVDPAState, nc, nc);
 306     if (!s->vhost_vdpa.shadow_vqs_enabled) {
 307         return 0;
 308     }
 309
 310     r = vhost_vdpa_cvq_map_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer,
 311                                vhost_vdpa_net_cvq_cmd_page_len(), false);
 312     if (unlikely(r < 0)) {
 313         return r;
 314     }
 315
 316     r = vhost_vdpa_cvq_map_buf(&s->vhost_vdpa, s->status,
 317                                vhost_vdpa_net_cvq_cmd_page_len(), true);
 318     if (unlikely(r < 0)) {
 319         vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer);
 320     }
 321
 322     return r;
 323 }
 324
 325 static void vhost_vdpa_net_cvq_stop(NetClientState *nc)
 326 {
 327     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
 328
 329     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
 330
 331     if (s->vhost_vdpa.shadow_vqs_enabled) {
 332         vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer);
 333         vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->status);
 334     }
 335 }
 336
 337 static ssize_t vhost_vdpa_net_cvq_add(VhostVDPAState *s, size_t out_len,
 338                                       size_t in_len)
 339 {
 340     /* Buffers for the device */
 341     const struct iovec out = {
 342         .iov_base = s->cvq_cmd_out_buffer,
 343         .iov_len = out_len,
 344     };
 345     const struct iovec in = {
 346         .iov_base = s->status,
 347         .iov_len = sizeof(virtio_net_ctrl_ack),
 348     };
 349     VhostShadowVirtqueue *svq = g_ptr_array_index(s->vhost_vdpa.shadow_vqs, 0);
 350     int r;
 351
 352     r = vhost_svq_add(svq, &out, 1, &in, 1, NULL);
 353     if (unlikely(r != 0)) {
 354         if (unlikely(r == -ENOSPC)) {
 355             qemu_log_mask(LOG_GUEST_ERROR, "%s: No space on device queue\n",
 356                           __func__);
 357         }
 358         return r;
 359     }
 360
 361     /*
 362      * We can poll here since we've had BQL from the time we sent the
 363      * descriptor. Also, we need to take the answer before SVQ pulls by itself,
 364      * when BQL is released
 365      */
 366     return vhost_svq_poll(svq);
 367 }
 368
 369 static ssize_t vhost_vdpa_net_load_cmd(VhostVDPAState *s, uint8_t class,
 370                                        uint8_t cmd, const void *data,
 371                                        size_t data_size)
 372 {
 373     const struct virtio_net_ctrl_hdr ctrl = {
 374         .class = class,
 375         .cmd = cmd,
 376     };
 377
 378     assert(data_size < vhost_vdpa_net_cvq_cmd_page_len() - sizeof(ctrl));
 379
 380     memcpy(s->cvq_cmd_out_buffer, &ctrl, sizeof(ctrl));
 381     memcpy(s->cvq_cmd_out_buffer + sizeof(ctrl), data, data_size);
 382
 383     return vhost_vdpa_net_cvq_add(s, sizeof(ctrl) + data_size,
 384                                   sizeof(virtio_net_ctrl_ack));
 385 }
 386
 387 static int vhost_vdpa_net_load_mac(VhostVDPAState *s, const VirtIONet *n)
 388 {
 389     uint64_t features = n->parent_obj.guest_features;
 390     if (features & BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR)) {
 391         ssize_t dev_written = vhost_vdpa_net_load_cmd(s, VIRTIO_NET_CTRL_MAC,
 392                                                   VIRTIO_NET_CTRL_MAC_ADDR_SET,
 393                                                   n->mac, sizeof(n->mac));
 394         if (unlikely(dev_written < 0)) {
 395             return dev_written;
 396         }
 397
 398         return *s->status != VIRTIO_NET_OK;
 399     }
 400
 401     return 0;
 402 }
 403
 404 static int vhost_vdpa_net_load_mq(VhostVDPAState *s,
 405                                   const VirtIONet *n)
 406 {
 407     struct virtio_net_ctrl_mq mq;
 408     uint64_t features = n->parent_obj.guest_features;
 409     ssize_t dev_written;
 410
 411     if (!(features & BIT_ULL(VIRTIO_NET_F_MQ))) {
 412         return 0;
 413     }
 414
 415     mq.virtqueue_pairs = cpu_to_le16(n->curr_queue_pairs);
 416     dev_written = vhost_vdpa_net_load_cmd(s, VIRTIO_NET_CTRL_MQ,
 417                                           VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, &mq,
 418                                           sizeof(mq));
 419     if (unlikely(dev_written < 0)) {
 420         return dev_written;
 421     }
 422
 423     return *s->status != VIRTIO_NET_OK;
 424 }
 425
 426 static int vhost_vdpa_net_load(NetClientState *nc)
 427 {
 428     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
 429     struct vhost_vdpa *v = &s->vhost_vdpa;
 430     const VirtIONet *n;
 431     int r;
 432
 433     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
 434
 435     if (!v->shadow_vqs_enabled) {
 436         return 0;
 437     }
 438
 439     n = VIRTIO_NET(v->dev->vdev);
 440     r = vhost_vdpa_net_load_mac(s, n);
 441     if (unlikely(r < 0)) {
 442         return r;
 443     }
 444     r = vhost_vdpa_net_load_mq(s, n);
 445     if (unlikely(r)) {
 446         return r;
 447     }
 448
 449     return 0;
 450 }
 451
 452 static NetClientInfo net_vhost_vdpa_cvq_info = {
 453     .type = NET_CLIENT_DRIVER_VHOST_VDPA,
 454     .size = sizeof(VhostVDPAState),
 455     .receive = vhost_vdpa_receive,
 456     .start = vhost_vdpa_net_cvq_start,
 457     .load = vhost_vdpa_net_load,
 458     .stop = vhost_vdpa_net_cvq_stop,
 459     .cleanup = vhost_vdpa_cleanup,
 460     .has_vnet_hdr = vhost_vdpa_has_vnet_hdr,
 461     .has_ufo = vhost_vdpa_has_ufo,
 462     .check_peer_type = vhost_vdpa_check_peer_type,
 463 };
 464
 465 /**
 466  * Do not forward commands not supported by SVQ. Otherwise, the device could
 467  * accept it and qemu would not know how to update the device model.
 468  */
 469 static bool vhost_vdpa_net_cvq_validate_cmd(const void *out_buf, size_t len)
 470 {
 471     struct virtio_net_ctrl_hdr ctrl;
 472
 473     if (unlikely(len < sizeof(ctrl))) {
 474         qemu_log_mask(LOG_GUEST_ERROR,
 475                       "%s: invalid legnth of out buffer %zu\n", __func__, len);
 476         return false;
 477     }
 478
 479     memcpy(&ctrl, out_buf, sizeof(ctrl));
 480     switch (ctrl.class) {
 481     case VIRTIO_NET_CTRL_MAC:
 482         switch (ctrl.cmd) {
 483         case VIRTIO_NET_CTRL_MAC_ADDR_SET:
 484             return true;
 485         default:
 486             qemu_log_mask(LOG_GUEST_ERROR, "%s: invalid mac cmd %u\n",
 487                           __func__, ctrl.cmd);
 488         };
 489         break;
 490     case VIRTIO_NET_CTRL_MQ:
 491         switch (ctrl.cmd) {
 492         case VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET:
 493             return true;
 494         default:
 495             qemu_log_mask(LOG_GUEST_ERROR, "%s: invalid mq cmd %u\n",
 496                           __func__, ctrl.cmd);
 497         };
 498         break;
 499     default:
 500         qemu_log_mask(LOG_GUEST_ERROR, "%s: invalid control class %u\n",
 501                       __func__, ctrl.class);
 502     };
 503
 504     return false;
 505 }
 506
 507 /**
 508  * Validate and copy control virtqueue commands.
 509  *
 510  * Following QEMU guidelines, we offer a copy of the buffers to the device to
 511  * prevent TOCTOU bugs.
 512  */
 513 static int vhost_vdpa_net_handle_ctrl_avail(VhostShadowVirtqueue *svq,
 514                                             VirtQueueElement *elem,
 515                                             void *opaque)
 516 {
 517     VhostVDPAState *s = opaque;
 518     size_t in_len;
 519     virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
 520     /* Out buffer sent to both the vdpa device and the device model */
 521     struct iovec out = {
 522         .iov_base = s->cvq_cmd_out_buffer,
 523     };
 524     /* in buffer used for device model */
 525     const struct iovec in = {
 526         .iov_base = &status,
 527         .iov_len = sizeof(status),
 528     };
 529     ssize_t dev_written = -EINVAL;
 530     bool ok;
 531
 532     out.iov_len = iov_to_buf(elem->out_sg, elem->out_num, 0,
 533                              s->cvq_cmd_out_buffer,
 534                              vhost_vdpa_net_cvq_cmd_len());
 535     ok = vhost_vdpa_net_cvq_validate_cmd(s->cvq_cmd_out_buffer, out.iov_len);
 536     if (unlikely(!ok)) {
 537         goto out;
 538     }
 539
 540     dev_written = vhost_vdpa_net_cvq_add(s, out.iov_len, sizeof(status));
 541     if (unlikely(dev_written < 0)) {
 542         goto out;
 543     }
 544
 545     if (unlikely(dev_written < sizeof(status))) {
 546         error_report("Insufficient written data (%zu)", dev_written);
 547         goto out;
 548     }
 549
 550     if (*s->status != VIRTIO_NET_OK) {
 551         return VIRTIO_NET_ERR;
 552     }
 553
 554     status = VIRTIO_NET_ERR;
 555     virtio_net_handle_ctrl_iov(svq->vdev, &in, 1, &out, 1);
 556     if (status != VIRTIO_NET_OK) {
 557         error_report("Bad CVQ processing in model");
 558     }
 559
 560 out:
 561     in_len = iov_from_buf(elem->in_sg, elem->in_num, 0, &status,
 562                           sizeof(status));
 563     if (unlikely(in_len < sizeof(status))) {
 564         error_report("Bad device CVQ written length");
 565     }
 566     vhost_svq_push_elem(svq, elem, MIN(in_len, sizeof(status)));
 567     g_free(elem);
 568     return dev_written < 0 ? dev_written : 0;
 569 }
 570
 571 static const VhostShadowVirtqueueOps vhost_vdpa_net_svq_ops = {
 572     .avail_handler = vhost_vdpa_net_handle_ctrl_avail,
 573 };
 574
 575 static NetClientState *net_vhost_vdpa_init(NetClientState *peer,
 576                                            const char *device,
 577                                            const char *name,
 578                                            int vdpa_device_fd,
 579                                            int queue_pair_index,
 580                                            int nvqs,
 581                                            bool is_datapath,
 582                                            bool svq,
 583                                            VhostIOVATree *iova_tree)
 584 {
 585     NetClientState *nc = NULL;
 586     VhostVDPAState *s;
 587     int ret = 0;
 588     assert(name);
 589     if (is_datapath) {
 590         nc = qemu_new_net_client(&net_vhost_vdpa_info, peer, device,
 591                                  name);
 592     } else {
 593         nc = qemu_new_net_control_client(&net_vhost_vdpa_cvq_info, peer,
 594                                          device, name);
 595     }
 596     snprintf(nc->info_str, sizeof(nc->info_str), TYPE_VHOST_VDPA);
 597     s = DO_UPCAST(VhostVDPAState, nc, nc);
 598
 599     s->vhost_vdpa.device_fd = vdpa_device_fd;
 600     s->vhost_vdpa.index = queue_pair_index;
 601     s->vhost_vdpa.shadow_vqs_enabled = svq;
 602     s->vhost_vdpa.iova_tree = iova_tree;
 603     if (!is_datapath) {
 604         s->cvq_cmd_out_buffer = qemu_memalign(qemu_real_host_page_size(),
 605                                             vhost_vdpa_net_cvq_cmd_page_len());
 606         memset(s->cvq_cmd_out_buffer, 0, vhost_vdpa_net_cvq_cmd_page_len());
 607         s->status = qemu_memalign(qemu_real_host_page_size(),
 608                                   vhost_vdpa_net_cvq_cmd_page_len());
 609         memset(s->status, 0, vhost_vdpa_net_cvq_cmd_page_len());
 610
 611         s->vhost_vdpa.shadow_vq_ops = &vhost_vdpa_net_svq_ops;
 612         s->vhost_vdpa.shadow_vq_ops_opaque = s;
 613     }
 614     ret = vhost_vdpa_add(nc, (void *)&s->vhost_vdpa, queue_pair_index, nvqs);
 615     if (ret) {
 616         qemu_del_net_client(nc);
 617         return NULL;
 618     }
 619     return nc;
 620 }
 621
 622 static int vhost_vdpa_get_iova_range(int fd,
 623                                      struct vhost_vdpa_iova_range *iova_range)
 624 {
 625     int ret = ioctl(fd, VHOST_VDPA_GET_IOVA_RANGE, iova_range);
 626
 627     return ret < 0 ? -errno : 0;
 628 }
 629
 630 static int vhost_vdpa_get_features(int fd, uint64_t *features, Error **errp)
 631 {
 632     int ret = ioctl(fd, VHOST_GET_FEATURES, features);
 633     if (unlikely(ret < 0)) {
 634         error_setg_errno(errp, errno,
 635                          "Fail to query features from vhost-vDPA device");
 636     }
 637     return ret;
 638 }
 639
 640 static int vhost_vdpa_get_max_queue_pairs(int fd, uint64_t features,
 641                                           int *has_cvq, Error **errp)
 642 {
 643     unsigned long config_size = offsetof(struct vhost_vdpa_config, buf);
 644     g_autofree struct vhost_vdpa_config *config = NULL;
 645     __virtio16 *max_queue_pairs;
 646     int ret;
 647
 648     if (features & (1 << VIRTIO_NET_F_CTRL_VQ)) {
 649         *has_cvq = 1;
 650     } else {
 651         *has_cvq = 0;
 652     }
 653
 654     if (features & (1 << VIRTIO_NET_F_MQ)) {
 655         config = g_malloc0(config_size + sizeof(*max_queue_pairs));
 656         config->off = offsetof(struct virtio_net_config, max_virtqueue_pairs);
 657         config->len = sizeof(*max_queue_pairs);
 658
 659         ret = ioctl(fd, VHOST_VDPA_GET_CONFIG, config);
 660         if (ret) {
 661             error_setg(errp, "Fail to get config from vhost-vDPA device");
 662             return -ret;
 663         }
 664
 665         max_queue_pairs = (__virtio16 *)&config->buf;
 666
 667         return lduw_le_p(max_queue_pairs);
 668     }
 669
 670     return 1;
 671 }
 672
 673 int net_init_vhost_vdpa(const Netdev *netdev, const char *name,
 674                         NetClientState *peer, Error **errp)
 675 {
 676     const NetdevVhostVDPAOptions *opts;
 677     uint64_t features;
 678     int vdpa_device_fd;
 679     g_autofree NetClientState **ncs = NULL;
 680     g_autoptr(VhostIOVATree) iova_tree = NULL;
 681     NetClientState *nc;
 682     int queue_pairs, r, i = 0, has_cvq = 0;
 683
 684     assert(netdev->type == NET_CLIENT_DRIVER_VHOST_VDPA);
 685     opts = &netdev->u.vhost_vdpa;
 686     if (!opts->vhostdev) {
 687         error_setg(errp, "vdpa character device not specified with vhostdev");
 688         return -1;
 689     }
 690
 691     vdpa_device_fd = qemu_open(opts->vhostdev, O_RDWR, errp);
 692     if (vdpa_device_fd == -1) {
 693         return -errno;
 694     }
 695
 696     r = vhost_vdpa_get_features(vdpa_device_fd, &features, errp);
 697     if (unlikely(r < 0)) {
 698         goto err;
 699     }
 700
 701     queue_pairs = vhost_vdpa_get_max_queue_pairs(vdpa_device_fd, features,
 702                                                  &has_cvq, errp);
 703     if (queue_pairs < 0) {
 704         qemu_close(vdpa_device_fd);
 705         return queue_pairs;
 706     }
 707
 708     if (opts->x_svq) {
 709         struct vhost_vdpa_iova_range iova_range;
 710
 711         uint64_t invalid_dev_features =
 712             features & ~vdpa_svq_device_features &
 713             /* Transport are all accepted at this point */
 714             ~MAKE_64BIT_MASK(VIRTIO_TRANSPORT_F_START,
 715                              VIRTIO_TRANSPORT_F_END - VIRTIO_TRANSPORT_F_START);
 716
 717         if (invalid_dev_features) {
 718             error_setg(errp, "vdpa svq does not work with features 0x%" PRIx64,
 719                        invalid_dev_features);
 720             goto err_svq;
 721         }
 722
 723         vhost_vdpa_get_iova_range(vdpa_device_fd, &iova_range);
 724         iova_tree = vhost_iova_tree_new(iova_range.first, iova_range.last);
 725     }
 726
 727     ncs = g_malloc0(sizeof(*ncs) * queue_pairs);
 728
 729     for (i = 0; i < queue_pairs; i++) {
 730         ncs[i] = net_vhost_vdpa_init(peer, TYPE_VHOST_VDPA, name,
 731                                      vdpa_device_fd, i, 2, true, opts->x_svq,
 732                                      iova_tree);
 733         if (!ncs[i])
 734             goto err;
 735     }
 736
 737     if (has_cvq) {
 738         nc = net_vhost_vdpa_init(peer, TYPE_VHOST_VDPA, name,
 739                                  vdpa_device_fd, i, 1, false,
 740                                  opts->x_svq, iova_tree);
 741         if (!nc)
 742             goto err;
 743     }
 744
 745     /* iova_tree ownership belongs to last NetClientState */
 746     g_steal_pointer(&iova_tree);
 747     return 0;
 748
 749 err:
 750     if (i) {
 751         for (i--; i >= 0; i--) {
 752             qemu_del_net_client(ncs[i]);
 753         }
 754     }
 755
 756 err_svq:
 757     qemu_close(vdpa_device_fd);
 758
 759     return -1;
 760 }