net/vhost-vdpa.c

   1 /*
   2  * vhost-vdpa.c
   3  *
   4  * Copyright(c) 2017-2018 Intel Corporation.
   5  * Copyright(c) 2020 Red Hat, Inc.
   6  *
   7  * This work is licensed under the terms of the GNU GPL, version 2 or later.
   8  * See the COPYING file in the top-level directory.
   9  *
  10  */
  11
  12 #include "qemu/osdep.h"
  13 #include "clients.h"
  14 #include "hw/virtio/virtio-net.h"
  15 #include "net/vhost_net.h"
  16 #include "net/vhost-vdpa.h"
  17 #include "hw/virtio/vhost-vdpa.h"
  18 #include "qemu/config-file.h"
  19 #include "qemu/error-report.h"
  20 #include "qemu/log.h"
  21 #include "qemu/memalign.h"
  22 #include "qemu/option.h"
  23 #include "qapi/error.h"
  24 #include <linux/vhost.h>
  25 #include <sys/ioctl.h>
  26 #include <err.h>
  27 #include "standard-headers/linux/virtio_net.h"
  28 #include "monitor/monitor.h"
  29 #include "hw/virtio/vhost.h"
  30
  31 /* Todo:need to add the multiqueue support here */
  32 typedef struct VhostVDPAState {
  33     NetClientState nc;
  34     struct vhost_vdpa vhost_vdpa;
  35     VHostNetState *vhost_net;
  36
  37     /* Control commands shadow buffers */
  38     void *cvq_cmd_out_buffer;
  39     virtio_net_ctrl_ack *status;
  40
  41     bool started;
  42 } VhostVDPAState;
  43
  44 const int vdpa_feature_bits[] = {
  45     VIRTIO_F_NOTIFY_ON_EMPTY,
  46     VIRTIO_RING_F_INDIRECT_DESC,
  47     VIRTIO_RING_F_EVENT_IDX,
  48     VIRTIO_F_ANY_LAYOUT,
  49     VIRTIO_F_VERSION_1,
  50     VIRTIO_NET_F_CSUM,
  51     VIRTIO_NET_F_GUEST_CSUM,
  52     VIRTIO_NET_F_GSO,
  53     VIRTIO_NET_F_GUEST_TSO4,
  54     VIRTIO_NET_F_GUEST_TSO6,
  55     VIRTIO_NET_F_GUEST_ECN,
  56     VIRTIO_NET_F_GUEST_UFO,
  57     VIRTIO_NET_F_HOST_TSO4,
  58     VIRTIO_NET_F_HOST_TSO6,
  59     VIRTIO_NET_F_HOST_ECN,
  60     VIRTIO_NET_F_HOST_UFO,
  61     VIRTIO_NET_F_MRG_RXBUF,
  62     VIRTIO_NET_F_MTU,
  63     VIRTIO_NET_F_CTRL_RX,
  64     VIRTIO_NET_F_CTRL_RX_EXTRA,
  65     VIRTIO_NET_F_CTRL_VLAN,
  66     VIRTIO_NET_F_CTRL_MAC_ADDR,
  67     VIRTIO_NET_F_RSS,
  68     VIRTIO_NET_F_MQ,
  69     VIRTIO_NET_F_CTRL_VQ,
  70     VIRTIO_F_IOMMU_PLATFORM,
  71     VIRTIO_F_RING_PACKED,
  72     VIRTIO_NET_F_RSS,
  73     VIRTIO_NET_F_HASH_REPORT,
  74     VIRTIO_NET_F_GUEST_ANNOUNCE,
  75     VIRTIO_NET_F_STATUS,
  76     VHOST_INVALID_FEATURE_BIT
  77 };
  78
  79 /** Supported device specific feature bits with SVQ */
  80 static const uint64_t vdpa_svq_device_features =
  81     BIT_ULL(VIRTIO_NET_F_CSUM) |
  82     BIT_ULL(VIRTIO_NET_F_GUEST_CSUM) |
  83     BIT_ULL(VIRTIO_NET_F_MTU) |
  84     BIT_ULL(VIRTIO_NET_F_MAC) |
  85     BIT_ULL(VIRTIO_NET_F_GUEST_TSO4) |
  86     BIT_ULL(VIRTIO_NET_F_GUEST_TSO6) |
  87     BIT_ULL(VIRTIO_NET_F_GUEST_ECN) |
  88     BIT_ULL(VIRTIO_NET_F_GUEST_UFO) |
  89     BIT_ULL(VIRTIO_NET_F_HOST_TSO4) |
  90     BIT_ULL(VIRTIO_NET_F_HOST_TSO6) |
  91     BIT_ULL(VIRTIO_NET_F_HOST_ECN) |
  92     BIT_ULL(VIRTIO_NET_F_HOST_UFO) |
  93     BIT_ULL(VIRTIO_NET_F_MRG_RXBUF) |
  94     BIT_ULL(VIRTIO_NET_F_STATUS) |
  95     BIT_ULL(VIRTIO_NET_F_CTRL_VQ) |
  96     BIT_ULL(VIRTIO_NET_F_MQ) |
  97     BIT_ULL(VIRTIO_F_ANY_LAYOUT) |
  98     BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR) |
  99     BIT_ULL(VIRTIO_NET_F_RSC_EXT) |
 100     BIT_ULL(VIRTIO_NET_F_STANDBY);
 101
 102 VHostNetState *vhost_vdpa_get_vhost_net(NetClientState *nc)
 103 {
 104     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
 105     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
 106     return s->vhost_net;
 107 }
 108
 109 static int vhost_vdpa_net_check_device_id(struct vhost_net *net)
 110 {
 111     uint32_t device_id;
 112     int ret;
 113     struct vhost_dev *hdev;
 114
 115     hdev = (struct vhost_dev *)&net->dev;
 116     ret = hdev->vhost_ops->vhost_get_device_id(hdev, &device_id);
 117     if (device_id != VIRTIO_ID_NET) {
 118         return -ENOTSUP;
 119     }
 120     return ret;
 121 }
 122
 123 static int vhost_vdpa_add(NetClientState *ncs, void *be,
 124                           int queue_pair_index, int nvqs)
 125 {
 126     VhostNetOptions options;
 127     struct vhost_net *net = NULL;
 128     VhostVDPAState *s;
 129     int ret;
 130
 131     options.backend_type = VHOST_BACKEND_TYPE_VDPA;
 132     assert(ncs->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
 133     s = DO_UPCAST(VhostVDPAState, nc, ncs);
 134     options.net_backend = ncs;
 135     options.opaque      = be;
 136     options.busyloop_timeout = 0;
 137     options.nvqs = nvqs;
 138
 139     net = vhost_net_init(&options);
 140     if (!net) {
 141         error_report("failed to init vhost_net for queue");
 142         goto err_init;
 143     }
 144     s->vhost_net = net;
 145     ret = vhost_vdpa_net_check_device_id(net);
 146     if (ret) {
 147         goto err_check;
 148     }
 149     return 0;
 150 err_check:
 151     vhost_net_cleanup(net);
 152     g_free(net);
 153 err_init:
 154     return -1;
 155 }
 156
 157 static void vhost_vdpa_cleanup(NetClientState *nc)
 158 {
 159     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
 160     struct vhost_dev *dev = &s->vhost_net->dev;
 161
 162     qemu_vfree(s->cvq_cmd_out_buffer);
 163     qemu_vfree(s->status);
 164     if (dev->vq_index + dev->nvqs == dev->vq_index_end) {
 165         g_clear_pointer(&s->vhost_vdpa.iova_tree, vhost_iova_tree_delete);
 166     }
 167     if (s->vhost_net) {
 168         vhost_net_cleanup(s->vhost_net);
 169         g_free(s->vhost_net);
 170         s->vhost_net = NULL;
 171     }
 172      if (s->vhost_vdpa.device_fd >= 0) {
 173         qemu_close(s->vhost_vdpa.device_fd);
 174         s->vhost_vdpa.device_fd = -1;
 175     }
 176 }
 177
 178 static bool vhost_vdpa_has_vnet_hdr(NetClientState *nc)
 179 {
 180     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
 181
 182     return true;
 183 }
 184
 185 static bool vhost_vdpa_has_ufo(NetClientState *nc)
 186 {
 187     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
 188     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
 189     uint64_t features = 0;
 190     features |= (1ULL << VIRTIO_NET_F_HOST_UFO);
 191     features = vhost_net_get_features(s->vhost_net, features);
 192     return !!(features & (1ULL << VIRTIO_NET_F_HOST_UFO));
 193
 194 }
 195
 196 static bool vhost_vdpa_check_peer_type(NetClientState *nc, ObjectClass *oc,
 197                                        Error **errp)
 198 {
 199     const char *driver = object_class_get_name(oc);
 200
 201     if (!g_str_has_prefix(driver, "virtio-net-")) {
 202         error_setg(errp, "vhost-vdpa requires frontend driver virtio-net-*");
 203         return false;
 204     }
 205
 206     return true;
 207 }
 208
 209 /** Dummy receive in case qemu falls back to userland tap networking */
 210 static ssize_t vhost_vdpa_receive(NetClientState *nc, const uint8_t *buf,
 211                                   size_t size)
 212 {
 213     return 0;
 214 }
 215
 216 static NetClientInfo net_vhost_vdpa_info = {
 217         .type = NET_CLIENT_DRIVER_VHOST_VDPA,
 218         .size = sizeof(VhostVDPAState),
 219         .receive = vhost_vdpa_receive,
 220         .cleanup = vhost_vdpa_cleanup,
 221         .has_vnet_hdr = vhost_vdpa_has_vnet_hdr,
 222         .has_ufo = vhost_vdpa_has_ufo,
 223         .check_peer_type = vhost_vdpa_check_peer_type,
 224 };
 225
 226 static void vhost_vdpa_cvq_unmap_buf(struct vhost_vdpa *v, void *addr)
 227 {
 228     VhostIOVATree *tree = v->iova_tree;
 229     DMAMap needle = {
 230         /*
 231          * No need to specify size or to look for more translations since
 232          * this contiguous chunk was allocated by us.
 233          */
 234         .translated_addr = (hwaddr)(uintptr_t)addr,
 235     };
 236     const DMAMap *map = vhost_iova_tree_find_iova(tree, &needle);
 237     int r;
 238
 239     if (unlikely(!map)) {
 240         error_report("Cannot locate expected map");
 241         return;
 242     }
 243
 244     r = vhost_vdpa_dma_unmap(v, map->iova, map->size + 1);
 245     if (unlikely(r != 0)) {
 246         error_report("Device cannot unmap: %s(%d)", g_strerror(r), r);
 247     }
 248
 249     vhost_iova_tree_remove(tree, *map);
 250 }
 251
 252 static size_t vhost_vdpa_net_cvq_cmd_len(void)
 253 {
 254     /*
 255      * MAC_TABLE_SET is the ctrl command that produces the longer out buffer.
 256      * In buffer is always 1 byte, so it should fit here
 257      */
 258     return sizeof(struct virtio_net_ctrl_hdr) +
 259            2 * sizeof(struct virtio_net_ctrl_mac) +
 260            MAC_TABLE_ENTRIES * ETH_ALEN;
 261 }
 262
 263 static size_t vhost_vdpa_net_cvq_cmd_page_len(void)
 264 {
 265     return ROUND_UP(vhost_vdpa_net_cvq_cmd_len(), qemu_real_host_page_size());
 266 }
 267
 268 /** Map CVQ buffer. */
 269 static int vhost_vdpa_cvq_map_buf(struct vhost_vdpa *v, void *buf, size_t size,
 270                                   bool write)
 271 {
 272     DMAMap map = {};
 273     int r;
 274
 275     map.translated_addr = (hwaddr)(uintptr_t)buf;
 276     map.size = size - 1;
 277     map.perm = write ? IOMMU_RW : IOMMU_RO,
 278     r = vhost_iova_tree_map_alloc(v->iova_tree, &map);
 279     if (unlikely(r != IOVA_OK)) {
 280         error_report("Cannot map injected element");
 281         return r;
 282     }
 283
 284     r = vhost_vdpa_dma_map(v, map.iova, vhost_vdpa_net_cvq_cmd_page_len(), buf,
 285                            !write);
 286     if (unlikely(r < 0)) {
 287         goto dma_map_err;
 288     }
 289
 290     return 0;
 291
 292 dma_map_err:
 293     vhost_iova_tree_remove(v->iova_tree, map);
 294     return r;
 295 }
 296
 297 static int vhost_vdpa_net_cvq_start(NetClientState *nc)
 298 {
 299     VhostVDPAState *s;
 300     int r;
 301
 302     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
 303
 304     s = DO_UPCAST(VhostVDPAState, nc, nc);
 305     if (!s->vhost_vdpa.shadow_vqs_enabled) {
 306         return 0;
 307     }
 308
 309     r = vhost_vdpa_cvq_map_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer,
 310                                vhost_vdpa_net_cvq_cmd_page_len(), false);
 311     if (unlikely(r < 0)) {
 312         return r;
 313     }
 314
 315     r = vhost_vdpa_cvq_map_buf(&s->vhost_vdpa, s->status,
 316                                vhost_vdpa_net_cvq_cmd_page_len(), true);
 317     if (unlikely(r < 0)) {
 318         vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer);
 319     }
 320
 321     return r;
 322 }
 323
 324 static void vhost_vdpa_net_cvq_stop(NetClientState *nc)
 325 {
 326     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
 327
 328     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
 329
 330     if (s->vhost_vdpa.shadow_vqs_enabled) {
 331         vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer);
 332         vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->status);
 333     }
 334 }
 335
 336 static ssize_t vhost_vdpa_net_cvq_add(VhostVDPAState *s, size_t out_len,
 337                                       size_t in_len)
 338 {
 339     /* Buffers for the device */
 340     const struct iovec out = {
 341         .iov_base = s->cvq_cmd_out_buffer,
 342         .iov_len = out_len,
 343     };
 344     const struct iovec in = {
 345         .iov_base = s->status,
 346         .iov_len = sizeof(virtio_net_ctrl_ack),
 347     };
 348     VhostShadowVirtqueue *svq = g_ptr_array_index(s->vhost_vdpa.shadow_vqs, 0);
 349     int r;
 350
 351     r = vhost_svq_add(svq, &out, 1, &in, 1, NULL);
 352     if (unlikely(r != 0)) {
 353         if (unlikely(r == -ENOSPC)) {
 354             qemu_log_mask(LOG_GUEST_ERROR, "%s: No space on device queue\n",
 355                           __func__);
 356         }
 357         return r;
 358     }
 359
 360     /*
 361      * We can poll here since we've had BQL from the time we sent the
 362      * descriptor. Also, we need to take the answer before SVQ pulls by itself,
 363      * when BQL is released
 364      */
 365     return vhost_svq_poll(svq);
 366 }
 367
 368 static ssize_t vhost_vdpa_net_load_cmd(VhostVDPAState *s, uint8_t class,
 369                                        uint8_t cmd, const void *data,
 370                                        size_t data_size)
 371 {
 372     const struct virtio_net_ctrl_hdr ctrl = {
 373         .class = class,
 374         .cmd = cmd,
 375     };
 376
 377     assert(data_size < vhost_vdpa_net_cvq_cmd_page_len() - sizeof(ctrl));
 378
 379     memcpy(s->cvq_cmd_out_buffer, &ctrl, sizeof(ctrl));
 380     memcpy(s->cvq_cmd_out_buffer + sizeof(ctrl), data, data_size);
 381
 382     return vhost_vdpa_net_cvq_add(s, sizeof(ctrl) + data_size,
 383                                   sizeof(virtio_net_ctrl_ack));
 384 }
 385
 386 static int vhost_vdpa_net_load_mac(VhostVDPAState *s, const VirtIONet *n)
 387 {
 388     uint64_t features = n->parent_obj.guest_features;
 389     if (features & BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR)) {
 390         ssize_t dev_written = vhost_vdpa_net_load_cmd(s, VIRTIO_NET_CTRL_MAC,
 391                                                   VIRTIO_NET_CTRL_MAC_ADDR_SET,
 392                                                   n->mac, sizeof(n->mac));
 393         if (unlikely(dev_written < 0)) {
 394             return dev_written;
 395         }
 396
 397         return *s->status != VIRTIO_NET_OK;
 398     }
 399
 400     return 0;
 401 }
 402
 403 static int vhost_vdpa_net_load_mq(VhostVDPAState *s,
 404                                   const VirtIONet *n)
 405 {
 406     struct virtio_net_ctrl_mq mq;
 407     uint64_t features = n->parent_obj.guest_features;
 408     ssize_t dev_written;
 409
 410     if (!(features & BIT_ULL(VIRTIO_NET_F_MQ))) {
 411         return 0;
 412     }
 413
 414     mq.virtqueue_pairs = cpu_to_le16(n->curr_queue_pairs);
 415     dev_written = vhost_vdpa_net_load_cmd(s, VIRTIO_NET_CTRL_MQ,
 416                                           VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, &mq,
 417                                           sizeof(mq));
 418     if (unlikely(dev_written < 0)) {
 419         return dev_written;
 420     }
 421
 422     return *s->status != VIRTIO_NET_OK;
 423 }
 424
 425 static int vhost_vdpa_net_load(NetClientState *nc)
 426 {
 427     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
 428     struct vhost_vdpa *v = &s->vhost_vdpa;
 429     const VirtIONet *n;
 430     int r;
 431
 432     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
 433
 434     if (!v->shadow_vqs_enabled) {
 435         return 0;
 436     }
 437
 438     n = VIRTIO_NET(v->dev->vdev);
 439     r = vhost_vdpa_net_load_mac(s, n);
 440     if (unlikely(r < 0)) {
 441         return r;
 442     }
 443     r = vhost_vdpa_net_load_mq(s, n);
 444     if (unlikely(r)) {
 445         return r;
 446     }
 447
 448     return 0;
 449 }
 450
 451 static NetClientInfo net_vhost_vdpa_cvq_info = {
 452     .type = NET_CLIENT_DRIVER_VHOST_VDPA,
 453     .size = sizeof(VhostVDPAState),
 454     .receive = vhost_vdpa_receive,
 455     .start = vhost_vdpa_net_cvq_start,
 456     .load = vhost_vdpa_net_load,
 457     .stop = vhost_vdpa_net_cvq_stop,
 458     .cleanup = vhost_vdpa_cleanup,
 459     .has_vnet_hdr = vhost_vdpa_has_vnet_hdr,
 460     .has_ufo = vhost_vdpa_has_ufo,
 461     .check_peer_type = vhost_vdpa_check_peer_type,
 462 };
 463
 464 /**
 465  * Validate and copy control virtqueue commands.
 466  *
 467  * Following QEMU guidelines, we offer a copy of the buffers to the device to
 468  * prevent TOCTOU bugs.
 469  */
 470 static int vhost_vdpa_net_handle_ctrl_avail(VhostShadowVirtqueue *svq,
 471                                             VirtQueueElement *elem,
 472                                             void *opaque)
 473 {
 474     VhostVDPAState *s = opaque;
 475     size_t in_len;
 476     virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
 477     /* Out buffer sent to both the vdpa device and the device model */
 478     struct iovec out = {
 479         .iov_base = s->cvq_cmd_out_buffer,
 480     };
 481     /* in buffer used for device model */
 482     const struct iovec in = {
 483         .iov_base = &status,
 484         .iov_len = sizeof(status),
 485     };
 486     ssize_t dev_written = -EINVAL;
 487
 488     out.iov_len = iov_to_buf(elem->out_sg, elem->out_num, 0,
 489                              s->cvq_cmd_out_buffer,
 490                              vhost_vdpa_net_cvq_cmd_len());
 491     dev_written = vhost_vdpa_net_cvq_add(s, out.iov_len, sizeof(status));
 492     if (unlikely(dev_written < 0)) {
 493         goto out;
 494     }
 495
 496     if (unlikely(dev_written < sizeof(status))) {
 497         error_report("Insufficient written data (%zu)", dev_written);
 498         goto out;
 499     }
 500
 501     if (*s->status != VIRTIO_NET_OK) {
 502         return VIRTIO_NET_ERR;
 503     }
 504
 505     status = VIRTIO_NET_ERR;
 506     virtio_net_handle_ctrl_iov(svq->vdev, &in, 1, &out, 1);
 507     if (status != VIRTIO_NET_OK) {
 508         error_report("Bad CVQ processing in model");
 509     }
 510
 511 out:
 512     in_len = iov_from_buf(elem->in_sg, elem->in_num, 0, &status,
 513                           sizeof(status));
 514     if (unlikely(in_len < sizeof(status))) {
 515         error_report("Bad device CVQ written length");
 516     }
 517     vhost_svq_push_elem(svq, elem, MIN(in_len, sizeof(status)));
 518     g_free(elem);
 519     return dev_written < 0 ? dev_written : 0;
 520 }
 521
 522 static const VhostShadowVirtqueueOps vhost_vdpa_net_svq_ops = {
 523     .avail_handler = vhost_vdpa_net_handle_ctrl_avail,
 524 };
 525
 526 static NetClientState *net_vhost_vdpa_init(NetClientState *peer,
 527                                            const char *device,
 528                                            const char *name,
 529                                            int vdpa_device_fd,
 530                                            int queue_pair_index,
 531                                            int nvqs,
 532                                            bool is_datapath,
 533                                            bool svq,
 534                                            VhostIOVATree *iova_tree)
 535 {
 536     NetClientState *nc = NULL;
 537     VhostVDPAState *s;
 538     int ret = 0;
 539     assert(name);
 540     if (is_datapath) {
 541         nc = qemu_new_net_client(&net_vhost_vdpa_info, peer, device,
 542                                  name);
 543     } else {
 544         nc = qemu_new_net_control_client(&net_vhost_vdpa_cvq_info, peer,
 545                                          device, name);
 546     }
 547     qemu_set_info_str(nc, TYPE_VHOST_VDPA);
 548     s = DO_UPCAST(VhostVDPAState, nc, nc);
 549
 550     s->vhost_vdpa.device_fd = vdpa_device_fd;
 551     s->vhost_vdpa.index = queue_pair_index;
 552     s->vhost_vdpa.shadow_vqs_enabled = svq;
 553     s->vhost_vdpa.iova_tree = iova_tree;
 554     if (!is_datapath) {
 555         s->cvq_cmd_out_buffer = qemu_memalign(qemu_real_host_page_size(),
 556                                             vhost_vdpa_net_cvq_cmd_page_len());
 557         memset(s->cvq_cmd_out_buffer, 0, vhost_vdpa_net_cvq_cmd_page_len());
 558         s->status = qemu_memalign(qemu_real_host_page_size(),
 559                                   vhost_vdpa_net_cvq_cmd_page_len());
 560         memset(s->status, 0, vhost_vdpa_net_cvq_cmd_page_len());
 561
 562         s->vhost_vdpa.shadow_vq_ops = &vhost_vdpa_net_svq_ops;
 563         s->vhost_vdpa.shadow_vq_ops_opaque = s;
 564     }
 565     ret = vhost_vdpa_add(nc, (void *)&s->vhost_vdpa, queue_pair_index, nvqs);
 566     if (ret) {
 567         qemu_del_net_client(nc);
 568         return NULL;
 569     }
 570     return nc;
 571 }
 572
 573 static int vhost_vdpa_get_iova_range(int fd,
 574                                      struct vhost_vdpa_iova_range *iova_range)
 575 {
 576     int ret = ioctl(fd, VHOST_VDPA_GET_IOVA_RANGE, iova_range);
 577
 578     return ret < 0 ? -errno : 0;
 579 }
 580
 581 static int vhost_vdpa_get_features(int fd, uint64_t *features, Error **errp)
 582 {
 583     int ret = ioctl(fd, VHOST_GET_FEATURES, features);
 584     if (unlikely(ret < 0)) {
 585         error_setg_errno(errp, errno,
 586                          "Fail to query features from vhost-vDPA device");
 587     }
 588     return ret;
 589 }
 590
 591 static int vhost_vdpa_get_max_queue_pairs(int fd, uint64_t features,
 592                                           int *has_cvq, Error **errp)
 593 {
 594     unsigned long config_size = offsetof(struct vhost_vdpa_config, buf);
 595     g_autofree struct vhost_vdpa_config *config = NULL;
 596     __virtio16 *max_queue_pairs;
 597     int ret;
 598
 599     if (features & (1 << VIRTIO_NET_F_CTRL_VQ)) {
 600         *has_cvq = 1;
 601     } else {
 602         *has_cvq = 0;
 603     }
 604
 605     if (features & (1 << VIRTIO_NET_F_MQ)) {
 606         config = g_malloc0(config_size + sizeof(*max_queue_pairs));
 607         config->off = offsetof(struct virtio_net_config, max_virtqueue_pairs);
 608         config->len = sizeof(*max_queue_pairs);
 609
 610         ret = ioctl(fd, VHOST_VDPA_GET_CONFIG, config);
 611         if (ret) {
 612             error_setg(errp, "Fail to get config from vhost-vDPA device");
 613             return -ret;
 614         }
 615
 616         max_queue_pairs = (__virtio16 *)&config->buf;
 617
 618         return lduw_le_p(max_queue_pairs);
 619     }
 620
 621     return 1;
 622 }
 623
 624 int net_init_vhost_vdpa(const Netdev *netdev, const char *name,
 625                         NetClientState *peer, Error **errp)
 626 {
 627     const NetdevVhostVDPAOptions *opts;
 628     uint64_t features;
 629     int vdpa_device_fd;
 630     g_autofree NetClientState **ncs = NULL;
 631     g_autoptr(VhostIOVATree) iova_tree = NULL;
 632     NetClientState *nc;
 633     int queue_pairs, r, i = 0, has_cvq = 0;
 634
 635     assert(netdev->type == NET_CLIENT_DRIVER_VHOST_VDPA);
 636     opts = &netdev->u.vhost_vdpa;
 637     if (!opts->has_vhostdev && !opts->has_vhostfd) {
 638         error_setg(errp,
 639                    "vhost-vdpa: neither vhostdev= nor vhostfd= was specified");
 640         return -1;
 641     }
 642
 643     if (opts->has_vhostdev && opts->has_vhostfd) {
 644         error_setg(errp,
 645                    "vhost-vdpa: vhostdev= and vhostfd= are mutually exclusive");
 646         return -1;
 647     }
 648
 649     if (opts->has_vhostdev) {
 650         vdpa_device_fd = qemu_open(opts->vhostdev, O_RDWR, errp);
 651         if (vdpa_device_fd == -1) {
 652             return -errno;
 653         }
 654     } else {
 655         /* has_vhostfd */
 656         vdpa_device_fd = monitor_fd_param(monitor_cur(), opts->vhostfd, errp);
 657         if (vdpa_device_fd == -1) {
 658             error_prepend(errp, "vhost-vdpa: unable to parse vhostfd: ");
 659             return -1;
 660         }
 661     }
 662
 663     r = vhost_vdpa_get_features(vdpa_device_fd, &features, errp);
 664     if (unlikely(r < 0)) {
 665         goto err;
 666     }
 667
 668     queue_pairs = vhost_vdpa_get_max_queue_pairs(vdpa_device_fd, features,
 669                                                  &has_cvq, errp);
 670     if (queue_pairs < 0) {
 671         qemu_close(vdpa_device_fd);
 672         return queue_pairs;
 673     }
 674
 675     if (opts->x_svq) {
 676         struct vhost_vdpa_iova_range iova_range;
 677
 678         uint64_t invalid_dev_features =
 679             features & ~vdpa_svq_device_features &
 680             /* Transport are all accepted at this point */
 681             ~MAKE_64BIT_MASK(VIRTIO_TRANSPORT_F_START,
 682                              VIRTIO_TRANSPORT_F_END - VIRTIO_TRANSPORT_F_START);
 683
 684         if (invalid_dev_features) {
 685             error_setg(errp, "vdpa svq does not work with features 0x%" PRIx64,
 686                        invalid_dev_features);
 687             goto err_svq;
 688         }
 689
 690         vhost_vdpa_get_iova_range(vdpa_device_fd, &iova_range);
 691         iova_tree = vhost_iova_tree_new(iova_range.first, iova_range.last);
 692     }
 693
 694     ncs = g_malloc0(sizeof(*ncs) * queue_pairs);
 695
 696     for (i = 0; i < queue_pairs; i++) {
 697         ncs[i] = net_vhost_vdpa_init(peer, TYPE_VHOST_VDPA, name,
 698                                      vdpa_device_fd, i, 2, true, opts->x_svq,
 699                                      iova_tree);
 700         if (!ncs[i])
 701             goto err;
 702     }
 703
 704     if (has_cvq) {
 705         nc = net_vhost_vdpa_init(peer, TYPE_VHOST_VDPA, name,
 706                                  vdpa_device_fd, i, 1, false,
 707                                  opts->x_svq, iova_tree);
 708         if (!nc)
 709             goto err;
 710     }
 711
 712     /* iova_tree ownership belongs to last NetClientState */
 713     g_steal_pointer(&iova_tree);
 714     return 0;
 715
 716 err:
 717     if (i) {
 718         for (i--; i >= 0; i--) {
 719             qemu_del_net_client(ncs[i]);
 720         }
 721     }
 722
 723 err_svq:
 724     qemu_close(vdpa_device_fd);
 725
 726     return -1;
 727 }