net/vhost-vdpa.c

   1 /*
   2  * vhost-vdpa.c
   3  *
   4  * Copyright(c) 2017-2018 Intel Corporation.
   5  * Copyright(c) 2020 Red Hat, Inc.
   6  *
   7  * This work is licensed under the terms of the GNU GPL, version 2 or later.
   8  * See the COPYING file in the top-level directory.
   9  *
  10  */
  11
  12 #include "qemu/osdep.h"
  13 #include "clients.h"
  14 #include "hw/virtio/virtio-net.h"
  15 #include "net/vhost_net.h"
  16 #include "net/vhost-vdpa.h"
  17 #include "hw/virtio/vhost-vdpa.h"
  18 #include "qemu/config-file.h"
  19 #include "qemu/error-report.h"
  20 #include "qemu/log.h"
  21 #include "qemu/memalign.h"
  22 #include "qemu/option.h"
  23 #include "qapi/error.h"
  24 #include <linux/vhost.h>
  25 #include <sys/ioctl.h>
  26 #include <err.h>
  27 #include "standard-headers/linux/virtio_net.h"
  28 #include "monitor/monitor.h"
  29 #include "migration/migration.h"
  30 #include "migration/misc.h"
  31 #include "hw/virtio/vhost.h"
  32
  33 /* Todo:need to add the multiqueue support here */
  34 typedef struct VhostVDPAState {
  35     NetClientState nc;
  36     struct vhost_vdpa vhost_vdpa;
  37     Notifier migration_state;
  38     VHostNetState *vhost_net;
  39
  40     /* Control commands shadow buffers */
  41     void *cvq_cmd_out_buffer;
  42     virtio_net_ctrl_ack *status;
  43
  44     /* The device always have SVQ enabled */
  45     bool always_svq;
  46     bool started;
  47 } VhostVDPAState;
  48
  49 const int vdpa_feature_bits[] = {
  50     VIRTIO_F_NOTIFY_ON_EMPTY,
  51     VIRTIO_RING_F_INDIRECT_DESC,
  52     VIRTIO_RING_F_EVENT_IDX,
  53     VIRTIO_F_ANY_LAYOUT,
  54     VIRTIO_F_VERSION_1,
  55     VIRTIO_NET_F_CSUM,
  56     VIRTIO_NET_F_GUEST_CSUM,
  57     VIRTIO_NET_F_GSO,
  58     VIRTIO_NET_F_GUEST_TSO4,
  59     VIRTIO_NET_F_GUEST_TSO6,
  60     VIRTIO_NET_F_GUEST_ECN,
  61     VIRTIO_NET_F_GUEST_UFO,
  62     VIRTIO_NET_F_HOST_TSO4,
  63     VIRTIO_NET_F_HOST_TSO6,
  64     VIRTIO_NET_F_HOST_ECN,
  65     VIRTIO_NET_F_HOST_UFO,
  66     VIRTIO_NET_F_MRG_RXBUF,
  67     VIRTIO_NET_F_MTU,
  68     VIRTIO_NET_F_CTRL_RX,
  69     VIRTIO_NET_F_CTRL_RX_EXTRA,
  70     VIRTIO_NET_F_CTRL_VLAN,
  71     VIRTIO_NET_F_CTRL_MAC_ADDR,
  72     VIRTIO_NET_F_RSS,
  73     VIRTIO_NET_F_MQ,
  74     VIRTIO_NET_F_CTRL_VQ,
  75     VIRTIO_F_IOMMU_PLATFORM,
  76     VIRTIO_F_RING_PACKED,
  77     VIRTIO_F_RING_RESET,
  78     VIRTIO_NET_F_RSS,
  79     VIRTIO_NET_F_HASH_REPORT,
  80     VIRTIO_NET_F_STATUS,
  81     VHOST_INVALID_FEATURE_BIT
  82 };
  83
  84 /** Supported device specific feature bits with SVQ */
  85 static const uint64_t vdpa_svq_device_features =
  86     BIT_ULL(VIRTIO_NET_F_CSUM) |
  87     BIT_ULL(VIRTIO_NET_F_GUEST_CSUM) |
  88     BIT_ULL(VIRTIO_NET_F_MTU) |
  89     BIT_ULL(VIRTIO_NET_F_MAC) |
  90     BIT_ULL(VIRTIO_NET_F_GUEST_TSO4) |
  91     BIT_ULL(VIRTIO_NET_F_GUEST_TSO6) |
  92     BIT_ULL(VIRTIO_NET_F_GUEST_ECN) |
  93     BIT_ULL(VIRTIO_NET_F_GUEST_UFO) |
  94     BIT_ULL(VIRTIO_NET_F_HOST_TSO4) |
  95     BIT_ULL(VIRTIO_NET_F_HOST_TSO6) |
  96     BIT_ULL(VIRTIO_NET_F_HOST_ECN) |
  97     BIT_ULL(VIRTIO_NET_F_HOST_UFO) |
  98     BIT_ULL(VIRTIO_NET_F_MRG_RXBUF) |
  99     BIT_ULL(VIRTIO_NET_F_STATUS) |
 100     BIT_ULL(VIRTIO_NET_F_CTRL_VQ) |
 101     BIT_ULL(VIRTIO_NET_F_MQ) |
 102     BIT_ULL(VIRTIO_F_ANY_LAYOUT) |
 103     BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR) |
 104     /* VHOST_F_LOG_ALL is exposed by SVQ */
 105     BIT_ULL(VHOST_F_LOG_ALL) |
 106     BIT_ULL(VIRTIO_NET_F_RSC_EXT) |
 107     BIT_ULL(VIRTIO_NET_F_STANDBY) |
 108     BIT_ULL(VIRTIO_NET_F_SPEED_DUPLEX);
 109
 110 #define VHOST_VDPA_NET_CVQ_ASID 1
 111
 112 VHostNetState *vhost_vdpa_get_vhost_net(NetClientState *nc)
 113 {
 114     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
 115     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
 116     return s->vhost_net;
 117 }
 118
 119 static bool vhost_vdpa_net_valid_svq_features(uint64_t features, Error **errp)
 120 {
 121     uint64_t invalid_dev_features =
 122         features & ~vdpa_svq_device_features &
 123         /* Transport are all accepted at this point */
 124         ~MAKE_64BIT_MASK(VIRTIO_TRANSPORT_F_START,
 125                          VIRTIO_TRANSPORT_F_END - VIRTIO_TRANSPORT_F_START);
 126
 127     if (invalid_dev_features) {
 128         error_setg(errp, "vdpa svq does not work with features 0x%" PRIx64,
 129                    invalid_dev_features);
 130         return false;
 131     }
 132
 133     return vhost_svq_valid_features(features, errp);
 134 }
 135
 136 static int vhost_vdpa_net_check_device_id(struct vhost_net *net)
 137 {
 138     uint32_t device_id;
 139     int ret;
 140     struct vhost_dev *hdev;
 141
 142     hdev = (struct vhost_dev *)&net->dev;
 143     ret = hdev->vhost_ops->vhost_get_device_id(hdev, &device_id);
 144     if (device_id != VIRTIO_ID_NET) {
 145         return -ENOTSUP;
 146     }
 147     return ret;
 148 }
 149
 150 static int vhost_vdpa_add(NetClientState *ncs, void *be,
 151                           int queue_pair_index, int nvqs)
 152 {
 153     VhostNetOptions options;
 154     struct vhost_net *net = NULL;
 155     VhostVDPAState *s;
 156     int ret;
 157
 158     options.backend_type = VHOST_BACKEND_TYPE_VDPA;
 159     assert(ncs->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
 160     s = DO_UPCAST(VhostVDPAState, nc, ncs);
 161     options.net_backend = ncs;
 162     options.opaque      = be;
 163     options.busyloop_timeout = 0;
 164     options.nvqs = nvqs;
 165
 166     net = vhost_net_init(&options);
 167     if (!net) {
 168         error_report("failed to init vhost_net for queue");
 169         goto err_init;
 170     }
 171     s->vhost_net = net;
 172     ret = vhost_vdpa_net_check_device_id(net);
 173     if (ret) {
 174         goto err_check;
 175     }
 176     return 0;
 177 err_check:
 178     vhost_net_cleanup(net);
 179     g_free(net);
 180 err_init:
 181     return -1;
 182 }
 183
 184 static void vhost_vdpa_cleanup(NetClientState *nc)
 185 {
 186     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
 187
 188     qemu_vfree(s->cvq_cmd_out_buffer);
 189     qemu_vfree(s->status);
 190     if (s->vhost_net) {
 191         vhost_net_cleanup(s->vhost_net);
 192         g_free(s->vhost_net);
 193         s->vhost_net = NULL;
 194     }
 195      if (s->vhost_vdpa.device_fd >= 0) {
 196         qemu_close(s->vhost_vdpa.device_fd);
 197         s->vhost_vdpa.device_fd = -1;
 198     }
 199 }
 200
 201 static bool vhost_vdpa_has_vnet_hdr(NetClientState *nc)
 202 {
 203     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
 204
 205     return true;
 206 }
 207
 208 static bool vhost_vdpa_has_ufo(NetClientState *nc)
 209 {
 210     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
 211     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
 212     uint64_t features = 0;
 213     features |= (1ULL << VIRTIO_NET_F_HOST_UFO);
 214     features = vhost_net_get_features(s->vhost_net, features);
 215     return !!(features & (1ULL << VIRTIO_NET_F_HOST_UFO));
 216
 217 }
 218
 219 static bool vhost_vdpa_check_peer_type(NetClientState *nc, ObjectClass *oc,
 220                                        Error **errp)
 221 {
 222     const char *driver = object_class_get_name(oc);
 223
 224     if (!g_str_has_prefix(driver, "virtio-net-")) {
 225         error_setg(errp, "vhost-vdpa requires frontend driver virtio-net-*");
 226         return false;
 227     }
 228
 229     return true;
 230 }
 231
 232 /** Dummy receive in case qemu falls back to userland tap networking */
 233 static ssize_t vhost_vdpa_receive(NetClientState *nc, const uint8_t *buf,
 234                                   size_t size)
 235 {
 236     return size;
 237 }
 238
 239 /** From any vdpa net client, get the netclient of the first queue pair */
 240 static VhostVDPAState *vhost_vdpa_net_first_nc_vdpa(VhostVDPAState *s)
 241 {
 242     NICState *nic = qemu_get_nic(s->nc.peer);
 243     NetClientState *nc0 = qemu_get_peer(nic->ncs, 0);
 244
 245     return DO_UPCAST(VhostVDPAState, nc, nc0);
 246 }
 247
 248 static void vhost_vdpa_net_log_global_enable(VhostVDPAState *s, bool enable)
 249 {
 250     struct vhost_vdpa *v = &s->vhost_vdpa;
 251     VirtIONet *n;
 252     VirtIODevice *vdev;
 253     int data_queue_pairs, cvq, r;
 254
 255     /* We are only called on the first data vqs and only if x-svq is not set */
 256     if (s->vhost_vdpa.shadow_vqs_enabled == enable) {
 257         return;
 258     }
 259
 260     vdev = v->dev->vdev;
 261     n = VIRTIO_NET(vdev);
 262     if (!n->vhost_started) {
 263         return;
 264     }
 265
 266     data_queue_pairs = n->multiqueue ? n->max_queue_pairs : 1;
 267     cvq = virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) ?
 268                                   n->max_ncs - n->max_queue_pairs : 0;
 269     /*
 270      * TODO: vhost_net_stop does suspend, get_base and reset. We can be smarter
 271      * in the future and resume the device if read-only operations between
 272      * suspend and reset goes wrong.
 273      */
 274     vhost_net_stop(vdev, n->nic->ncs, data_queue_pairs, cvq);
 275
 276     /* Start will check migration setup_or_active to configure or not SVQ */
 277     r = vhost_net_start(vdev, n->nic->ncs, data_queue_pairs, cvq);
 278     if (unlikely(r < 0)) {
 279         error_report("unable to start vhost net: %s(%d)", g_strerror(-r), -r);
 280     }
 281 }
 282
 283 static void vdpa_net_migration_state_notifier(Notifier *notifier, void *data)
 284 {
 285     MigrationState *migration = data;
 286     VhostVDPAState *s = container_of(notifier, VhostVDPAState,
 287                                      migration_state);
 288
 289     if (migration_in_setup(migration)) {
 290         vhost_vdpa_net_log_global_enable(s, true);
 291     } else if (migration_has_failed(migration)) {
 292         vhost_vdpa_net_log_global_enable(s, false);
 293     }
 294 }
 295
 296 static void vhost_vdpa_net_data_start_first(VhostVDPAState *s)
 297 {
 298     struct vhost_vdpa *v = &s->vhost_vdpa;
 299
 300     add_migration_state_change_notifier(&s->migration_state);
 301     if (v->shadow_vqs_enabled) {
 302         v->iova_tree = vhost_iova_tree_new(v->iova_range.first,
 303                                            v->iova_range.last);
 304     }
 305 }
 306
 307 static int vhost_vdpa_net_data_start(NetClientState *nc)
 308 {
 309     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
 310     struct vhost_vdpa *v = &s->vhost_vdpa;
 311
 312     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
 313
 314     if (s->always_svq ||
 315         migration_is_setup_or_active(migrate_get_current()->state)) {
 316         v->shadow_vqs_enabled = true;
 317         v->shadow_data = true;
 318     } else {
 319         v->shadow_vqs_enabled = false;
 320         v->shadow_data = false;
 321     }
 322
 323     if (v->index == 0) {
 324         vhost_vdpa_net_data_start_first(s);
 325         return 0;
 326     }
 327
 328     if (v->shadow_vqs_enabled) {
 329         VhostVDPAState *s0 = vhost_vdpa_net_first_nc_vdpa(s);
 330         v->iova_tree = s0->vhost_vdpa.iova_tree;
 331     }
 332
 333     return 0;
 334 }
 335
 336 static void vhost_vdpa_net_client_stop(NetClientState *nc)
 337 {
 338     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
 339     struct vhost_dev *dev;
 340
 341     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
 342
 343     if (s->vhost_vdpa.index == 0) {
 344         remove_migration_state_change_notifier(&s->migration_state);
 345     }
 346
 347     dev = s->vhost_vdpa.dev;
 348     if (dev->vq_index + dev->nvqs == dev->vq_index_end) {
 349         g_clear_pointer(&s->vhost_vdpa.iova_tree, vhost_iova_tree_delete);
 350     }
 351 }
 352
 353 static NetClientInfo net_vhost_vdpa_info = {
 354         .type = NET_CLIENT_DRIVER_VHOST_VDPA,
 355         .size = sizeof(VhostVDPAState),
 356         .receive = vhost_vdpa_receive,
 357         .start = vhost_vdpa_net_data_start,
 358         .stop = vhost_vdpa_net_client_stop,
 359         .cleanup = vhost_vdpa_cleanup,
 360         .has_vnet_hdr = vhost_vdpa_has_vnet_hdr,
 361         .has_ufo = vhost_vdpa_has_ufo,
 362         .check_peer_type = vhost_vdpa_check_peer_type,
 363 };
 364
 365 static int64_t vhost_vdpa_get_vring_group(int device_fd, unsigned vq_index)
 366 {
 367     struct vhost_vring_state state = {
 368         .index = vq_index,
 369     };
 370     int r = ioctl(device_fd, VHOST_VDPA_GET_VRING_GROUP, &state);
 371
 372     if (unlikely(r < 0)) {
 373         error_report("Cannot get VQ %u group: %s", vq_index,
 374                      g_strerror(errno));
 375         return r;
 376     }
 377
 378     return state.num;
 379 }
 380
 381 static int vhost_vdpa_set_address_space_id(struct vhost_vdpa *v,
 382                                            unsigned vq_group,
 383                                            unsigned asid_num)
 384 {
 385     struct vhost_vring_state asid = {
 386         .index = vq_group,
 387         .num = asid_num,
 388     };
 389     int r;
 390
 391     r = ioctl(v->device_fd, VHOST_VDPA_SET_GROUP_ASID, &asid);
 392     if (unlikely(r < 0)) {
 393         error_report("Can't set vq group %u asid %u, errno=%d (%s)",
 394                      asid.index, asid.num, errno, g_strerror(errno));
 395     }
 396     return r;
 397 }
 398
 399 static void vhost_vdpa_cvq_unmap_buf(struct vhost_vdpa *v, void *addr)
 400 {
 401     VhostIOVATree *tree = v->iova_tree;
 402     DMAMap needle = {
 403         /*
 404          * No need to specify size or to look for more translations since
 405          * this contiguous chunk was allocated by us.
 406          */
 407         .translated_addr = (hwaddr)(uintptr_t)addr,
 408     };
 409     const DMAMap *map = vhost_iova_tree_find_iova(tree, &needle);
 410     int r;
 411
 412     if (unlikely(!map)) {
 413         error_report("Cannot locate expected map");
 414         return;
 415     }
 416
 417     r = vhost_vdpa_dma_unmap(v, v->address_space_id, map->iova, map->size + 1);
 418     if (unlikely(r != 0)) {
 419         error_report("Device cannot unmap: %s(%d)", g_strerror(r), r);
 420     }
 421
 422     vhost_iova_tree_remove(tree, *map);
 423 }
 424
 425 static size_t vhost_vdpa_net_cvq_cmd_len(void)
 426 {
 427     /*
 428      * MAC_TABLE_SET is the ctrl command that produces the longer out buffer.
 429      * In buffer is always 1 byte, so it should fit here
 430      */
 431     return sizeof(struct virtio_net_ctrl_hdr) +
 432            2 * sizeof(struct virtio_net_ctrl_mac) +
 433            MAC_TABLE_ENTRIES * ETH_ALEN;
 434 }
 435
 436 static size_t vhost_vdpa_net_cvq_cmd_page_len(void)
 437 {
 438     return ROUND_UP(vhost_vdpa_net_cvq_cmd_len(), qemu_real_host_page_size());
 439 }
 440
 441 /** Map CVQ buffer. */
 442 static int vhost_vdpa_cvq_map_buf(struct vhost_vdpa *v, void *buf, size_t size,
 443                                   bool write)
 444 {
 445     DMAMap map = {};
 446     int r;
 447
 448     map.translated_addr = (hwaddr)(uintptr_t)buf;
 449     map.size = size - 1;
 450     map.perm = write ? IOMMU_RW : IOMMU_RO,
 451     r = vhost_iova_tree_map_alloc(v->iova_tree, &map);
 452     if (unlikely(r != IOVA_OK)) {
 453         error_report("Cannot map injected element");
 454         return r;
 455     }
 456
 457     r = vhost_vdpa_dma_map(v, v->address_space_id, map.iova,
 458                            vhost_vdpa_net_cvq_cmd_page_len(), buf, !write);
 459     if (unlikely(r < 0)) {
 460         goto dma_map_err;
 461     }
 462
 463     return 0;
 464
 465 dma_map_err:
 466     vhost_iova_tree_remove(v->iova_tree, map);
 467     return r;
 468 }
 469
 470 static int vhost_vdpa_net_cvq_start(NetClientState *nc)
 471 {
 472     VhostVDPAState *s, *s0;
 473     struct vhost_vdpa *v;
 474     uint64_t backend_features;
 475     int64_t cvq_group;
 476     int cvq_index, r;
 477
 478     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
 479
 480     s = DO_UPCAST(VhostVDPAState, nc, nc);
 481     v = &s->vhost_vdpa;
 482
 483     s0 = vhost_vdpa_net_first_nc_vdpa(s);
 484     v->shadow_data = s0->vhost_vdpa.shadow_vqs_enabled;
 485     v->shadow_vqs_enabled = s->always_svq;
 486     s->vhost_vdpa.address_space_id = VHOST_VDPA_GUEST_PA_ASID;
 487
 488     if (s->vhost_vdpa.shadow_data) {
 489         /* SVQ is already configured for all virtqueues */
 490         goto out;
 491     }
 492
 493     /*
 494      * If we early return in these cases SVQ will not be enabled. The migration
 495      * will be blocked as long as vhost-vdpa backends will not offer _F_LOG.
 496      *
 497      * Calling VHOST_GET_BACKEND_FEATURES as they are not available in v->dev
 498      * yet.
 499      */
 500     r = ioctl(v->device_fd, VHOST_GET_BACKEND_FEATURES, &backend_features);
 501     if (unlikely(r < 0)) {
 502         error_report("Cannot get vdpa backend_features: %s(%d)",
 503             g_strerror(errno), errno);
 504         return -1;
 505     }
 506     if (!(backend_features & BIT_ULL(VHOST_BACKEND_F_IOTLB_ASID)) ||
 507         !vhost_vdpa_net_valid_svq_features(v->dev->features, NULL)) {
 508         return 0;
 509     }
 510
 511     /*
 512      * Check if all the virtqueues of the virtio device are in a different vq
 513      * than the last vq. VQ group of last group passed in cvq_group.
 514      */
 515     cvq_index = v->dev->vq_index_end - 1;
 516     cvq_group = vhost_vdpa_get_vring_group(v->device_fd, cvq_index);
 517     if (unlikely(cvq_group < 0)) {
 518         return cvq_group;
 519     }
 520     for (int i = 0; i < cvq_index; ++i) {
 521         int64_t group = vhost_vdpa_get_vring_group(v->device_fd, i);
 522
 523         if (unlikely(group < 0)) {
 524             return group;
 525         }
 526
 527         if (group == cvq_group) {
 528             return 0;
 529         }
 530     }
 531
 532     r = vhost_vdpa_set_address_space_id(v, cvq_group, VHOST_VDPA_NET_CVQ_ASID);
 533     if (unlikely(r < 0)) {
 534         return r;
 535     }
 536
 537     v->shadow_vqs_enabled = true;
 538     s->vhost_vdpa.address_space_id = VHOST_VDPA_NET_CVQ_ASID;
 539
 540 out:
 541     if (!s->vhost_vdpa.shadow_vqs_enabled) {
 542         return 0;
 543     }
 544
 545     if (s0->vhost_vdpa.iova_tree) {
 546         /*
 547          * SVQ is already configured for all virtqueues.  Reuse IOVA tree for
 548          * simplicity, whether CVQ shares ASID with guest or not, because:
 549          * - Memory listener need access to guest's memory addresses allocated
 550          *   in the IOVA tree.
 551          * - There should be plenty of IOVA address space for both ASID not to
 552          *   worry about collisions between them.  Guest's translations are
 553          *   still validated with virtio virtqueue_pop so there is no risk for
 554          *   the guest to access memory that it shouldn't.
 555          *
 556          * To allocate a iova tree per ASID is doable but it complicates the
 557          * code and it is not worth it for the moment.
 558          */
 559         v->iova_tree = s0->vhost_vdpa.iova_tree;
 560     } else {
 561         v->iova_tree = vhost_iova_tree_new(v->iova_range.first,
 562                                            v->iova_range.last);
 563     }
 564
 565     r = vhost_vdpa_cvq_map_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer,
 566                                vhost_vdpa_net_cvq_cmd_page_len(), false);
 567     if (unlikely(r < 0)) {
 568         return r;
 569     }
 570
 571     r = vhost_vdpa_cvq_map_buf(&s->vhost_vdpa, s->status,
 572                                vhost_vdpa_net_cvq_cmd_page_len(), true);
 573     if (unlikely(r < 0)) {
 574         vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer);
 575     }
 576
 577     return r;
 578 }
 579
 580 static void vhost_vdpa_net_cvq_stop(NetClientState *nc)
 581 {
 582     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
 583
 584     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
 585
 586     if (s->vhost_vdpa.shadow_vqs_enabled) {
 587         vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->cvq_cmd_out_buffer);
 588         vhost_vdpa_cvq_unmap_buf(&s->vhost_vdpa, s->status);
 589     }
 590
 591     vhost_vdpa_net_client_stop(nc);
 592 }
 593
 594 static ssize_t vhost_vdpa_net_cvq_add(VhostVDPAState *s, size_t out_len,
 595                                       size_t in_len)
 596 {
 597     /* Buffers for the device */
 598     const struct iovec out = {
 599         .iov_base = s->cvq_cmd_out_buffer,
 600         .iov_len = out_len,
 601     };
 602     const struct iovec in = {
 603         .iov_base = s->status,
 604         .iov_len = sizeof(virtio_net_ctrl_ack),
 605     };
 606     VhostShadowVirtqueue *svq = g_ptr_array_index(s->vhost_vdpa.shadow_vqs, 0);
 607     int r;
 608
 609     r = vhost_svq_add(svq, &out, 1, &in, 1, NULL);
 610     if (unlikely(r != 0)) {
 611         if (unlikely(r == -ENOSPC)) {
 612             qemu_log_mask(LOG_GUEST_ERROR, "%s: No space on device queue\n",
 613                           __func__);
 614         }
 615         return r;
 616     }
 617
 618     /*
 619      * We can poll here since we've had BQL from the time we sent the
 620      * descriptor. Also, we need to take the answer before SVQ pulls by itself,
 621      * when BQL is released
 622      */
 623     return vhost_svq_poll(svq);
 624 }
 625
 626 static ssize_t vhost_vdpa_net_load_cmd(VhostVDPAState *s, uint8_t class,
 627                                        uint8_t cmd, const void *data,
 628                                        size_t data_size)
 629 {
 630     const struct virtio_net_ctrl_hdr ctrl = {
 631         .class = class,
 632         .cmd = cmd,
 633     };
 634
 635     assert(data_size < vhost_vdpa_net_cvq_cmd_page_len() - sizeof(ctrl));
 636
 637     memcpy(s->cvq_cmd_out_buffer, &ctrl, sizeof(ctrl));
 638     memcpy(s->cvq_cmd_out_buffer + sizeof(ctrl), data, data_size);
 639
 640     return vhost_vdpa_net_cvq_add(s, sizeof(ctrl) + data_size,
 641                                   sizeof(virtio_net_ctrl_ack));
 642 }
 643
 644 static int vhost_vdpa_net_load_mac(VhostVDPAState *s, const VirtIONet *n)
 645 {
 646     uint64_t features = n->parent_obj.guest_features;
 647     if (features & BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR)) {
 648         ssize_t dev_written = vhost_vdpa_net_load_cmd(s, VIRTIO_NET_CTRL_MAC,
 649                                                   VIRTIO_NET_CTRL_MAC_ADDR_SET,
 650                                                   n->mac, sizeof(n->mac));
 651         if (unlikely(dev_written < 0)) {
 652             return dev_written;
 653         }
 654
 655         return *s->status != VIRTIO_NET_OK;
 656     }
 657
 658     return 0;
 659 }
 660
 661 static int vhost_vdpa_net_load_mq(VhostVDPAState *s,
 662                                   const VirtIONet *n)
 663 {
 664     struct virtio_net_ctrl_mq mq;
 665     uint64_t features = n->parent_obj.guest_features;
 666     ssize_t dev_written;
 667
 668     if (!(features & BIT_ULL(VIRTIO_NET_F_MQ))) {
 669         return 0;
 670     }
 671
 672     mq.virtqueue_pairs = cpu_to_le16(n->curr_queue_pairs);
 673     dev_written = vhost_vdpa_net_load_cmd(s, VIRTIO_NET_CTRL_MQ,
 674                                           VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, &mq,
 675                                           sizeof(mq));
 676     if (unlikely(dev_written < 0)) {
 677         return dev_written;
 678     }
 679
 680     return *s->status != VIRTIO_NET_OK;
 681 }
 682
 683 static int vhost_vdpa_net_load(NetClientState *nc)
 684 {
 685     VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc);
 686     struct vhost_vdpa *v = &s->vhost_vdpa;
 687     const VirtIONet *n;
 688     int r;
 689
 690     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
 691
 692     if (!v->shadow_vqs_enabled) {
 693         return 0;
 694     }
 695
 696     n = VIRTIO_NET(v->dev->vdev);
 697     r = vhost_vdpa_net_load_mac(s, n);
 698     if (unlikely(r < 0)) {
 699         return r;
 700     }
 701     r = vhost_vdpa_net_load_mq(s, n);
 702     if (unlikely(r)) {
 703         return r;
 704     }
 705
 706     return 0;
 707 }
 708
 709 static NetClientInfo net_vhost_vdpa_cvq_info = {
 710     .type = NET_CLIENT_DRIVER_VHOST_VDPA,
 711     .size = sizeof(VhostVDPAState),
 712     .receive = vhost_vdpa_receive,
 713     .start = vhost_vdpa_net_cvq_start,
 714     .load = vhost_vdpa_net_load,
 715     .stop = vhost_vdpa_net_cvq_stop,
 716     .cleanup = vhost_vdpa_cleanup,
 717     .has_vnet_hdr = vhost_vdpa_has_vnet_hdr,
 718     .has_ufo = vhost_vdpa_has_ufo,
 719     .check_peer_type = vhost_vdpa_check_peer_type,
 720 };
 721
 722 /**
 723  * Validate and copy control virtqueue commands.
 724  *
 725  * Following QEMU guidelines, we offer a copy of the buffers to the device to
 726  * prevent TOCTOU bugs.
 727  */
 728 static int vhost_vdpa_net_handle_ctrl_avail(VhostShadowVirtqueue *svq,
 729                                             VirtQueueElement *elem,
 730                                             void *opaque)
 731 {
 732     VhostVDPAState *s = opaque;
 733     size_t in_len;
 734     virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
 735     /* Out buffer sent to both the vdpa device and the device model */
 736     struct iovec out = {
 737         .iov_base = s->cvq_cmd_out_buffer,
 738     };
 739     /* in buffer used for device model */
 740     const struct iovec in = {
 741         .iov_base = &status,
 742         .iov_len = sizeof(status),
 743     };
 744     ssize_t dev_written = -EINVAL;
 745
 746     out.iov_len = iov_to_buf(elem->out_sg, elem->out_num, 0,
 747                              s->cvq_cmd_out_buffer,
 748                              vhost_vdpa_net_cvq_cmd_len());
 749     if (*(uint8_t *)s->cvq_cmd_out_buffer == VIRTIO_NET_CTRL_ANNOUNCE) {
 750         /*
 751          * Guest announce capability is emulated by qemu, so don't forward to
 752          * the device.
 753          */
 754         dev_written = sizeof(status);
 755         *s->status = VIRTIO_NET_OK;
 756     } else {
 757         dev_written = vhost_vdpa_net_cvq_add(s, out.iov_len, sizeof(status));
 758         if (unlikely(dev_written < 0)) {
 759             goto out;
 760         }
 761     }
 762
 763     if (unlikely(dev_written < sizeof(status))) {
 764         error_report("Insufficient written data (%zu)", dev_written);
 765         goto out;
 766     }
 767
 768     if (*s->status != VIRTIO_NET_OK) {
 769         return VIRTIO_NET_ERR;
 770     }
 771
 772     status = VIRTIO_NET_ERR;
 773     virtio_net_handle_ctrl_iov(svq->vdev, &in, 1, &out, 1);
 774     if (status != VIRTIO_NET_OK) {
 775         error_report("Bad CVQ processing in model");
 776     }
 777
 778 out:
 779     in_len = iov_from_buf(elem->in_sg, elem->in_num, 0, &status,
 780                           sizeof(status));
 781     if (unlikely(in_len < sizeof(status))) {
 782         error_report("Bad device CVQ written length");
 783     }
 784     vhost_svq_push_elem(svq, elem, MIN(in_len, sizeof(status)));
 785     g_free(elem);
 786     return dev_written < 0 ? dev_written : 0;
 787 }
 788
 789 static const VhostShadowVirtqueueOps vhost_vdpa_net_svq_ops = {
 790     .avail_handler = vhost_vdpa_net_handle_ctrl_avail,
 791 };
 792
 793 static NetClientState *net_vhost_vdpa_init(NetClientState *peer,
 794                                        const char *device,
 795                                        const char *name,
 796                                        int vdpa_device_fd,
 797                                        int queue_pair_index,
 798                                        int nvqs,
 799                                        bool is_datapath,
 800                                        bool svq,
 801                                        struct vhost_vdpa_iova_range iova_range,
 802                                        uint64_t features)
 803 {
 804     NetClientState *nc = NULL;
 805     VhostVDPAState *s;
 806     int ret = 0;
 807     assert(name);
 808     if (is_datapath) {
 809         nc = qemu_new_net_client(&net_vhost_vdpa_info, peer, device,
 810                                  name);
 811     } else {
 812         nc = qemu_new_net_control_client(&net_vhost_vdpa_cvq_info, peer,
 813                                          device, name);
 814     }
 815     qemu_set_info_str(nc, TYPE_VHOST_VDPA);
 816     s = DO_UPCAST(VhostVDPAState, nc, nc);
 817
 818     s->vhost_vdpa.device_fd = vdpa_device_fd;
 819     s->vhost_vdpa.index = queue_pair_index;
 820     s->always_svq = svq;
 821     s->migration_state.notify = vdpa_net_migration_state_notifier;
 822     s->vhost_vdpa.shadow_vqs_enabled = svq;
 823     s->vhost_vdpa.iova_range = iova_range;
 824     s->vhost_vdpa.shadow_data = svq;
 825     if (queue_pair_index == 0) {
 826         vhost_vdpa_net_valid_svq_features(features,
 827                                           &s->vhost_vdpa.migration_blocker);
 828     } else if (!is_datapath) {
 829         s->cvq_cmd_out_buffer = qemu_memalign(qemu_real_host_page_size(),
 830                                             vhost_vdpa_net_cvq_cmd_page_len());
 831         memset(s->cvq_cmd_out_buffer, 0, vhost_vdpa_net_cvq_cmd_page_len());
 832         s->status = qemu_memalign(qemu_real_host_page_size(),
 833                                   vhost_vdpa_net_cvq_cmd_page_len());
 834         memset(s->status, 0, vhost_vdpa_net_cvq_cmd_page_len());
 835
 836         s->vhost_vdpa.shadow_vq_ops = &vhost_vdpa_net_svq_ops;
 837         s->vhost_vdpa.shadow_vq_ops_opaque = s;
 838
 839         /*
 840          * TODO: We cannot migrate devices with CVQ as there is no way to set
 841          * the device state (MAC, MQ, etc) before starting the datapath.
 842          *
 843          * Migration blocker ownership now belongs to s->vhost_vdpa.
 844          */
 845         error_setg(&s->vhost_vdpa.migration_blocker,
 846                    "net vdpa cannot migrate with CVQ feature");
 847     }
 848     ret = vhost_vdpa_add(nc, (void *)&s->vhost_vdpa, queue_pair_index, nvqs);
 849     if (ret) {
 850         qemu_del_net_client(nc);
 851         return NULL;
 852     }
 853     return nc;
 854 }
 855
 856 static int vhost_vdpa_get_features(int fd, uint64_t *features, Error **errp)
 857 {
 858     int ret = ioctl(fd, VHOST_GET_FEATURES, features);
 859     if (unlikely(ret < 0)) {
 860         error_setg_errno(errp, errno,
 861                          "Fail to query features from vhost-vDPA device");
 862     }
 863     return ret;
 864 }
 865
 866 static int vhost_vdpa_get_max_queue_pairs(int fd, uint64_t features,
 867                                           int *has_cvq, Error **errp)
 868 {
 869     unsigned long config_size = offsetof(struct vhost_vdpa_config, buf);
 870     g_autofree struct vhost_vdpa_config *config = NULL;
 871     __virtio16 *max_queue_pairs;
 872     int ret;
 873
 874     if (features & (1 << VIRTIO_NET_F_CTRL_VQ)) {
 875         *has_cvq = 1;
 876     } else {
 877         *has_cvq = 0;
 878     }
 879
 880     if (features & (1 << VIRTIO_NET_F_MQ)) {
 881         config = g_malloc0(config_size + sizeof(*max_queue_pairs));
 882         config->off = offsetof(struct virtio_net_config, max_virtqueue_pairs);
 883         config->len = sizeof(*max_queue_pairs);
 884
 885         ret = ioctl(fd, VHOST_VDPA_GET_CONFIG, config);
 886         if (ret) {
 887             error_setg(errp, "Fail to get config from vhost-vDPA device");
 888             return -ret;
 889         }
 890
 891         max_queue_pairs = (__virtio16 *)&config->buf;
 892
 893         return lduw_le_p(max_queue_pairs);
 894     }
 895
 896     return 1;
 897 }
 898
 899 int net_init_vhost_vdpa(const Netdev *netdev, const char *name,
 900                         NetClientState *peer, Error **errp)
 901 {
 902     const NetdevVhostVDPAOptions *opts;
 903     uint64_t features;
 904     int vdpa_device_fd;
 905     g_autofree NetClientState **ncs = NULL;
 906     struct vhost_vdpa_iova_range iova_range;
 907     NetClientState *nc;
 908     int queue_pairs, r, i = 0, has_cvq = 0;
 909
 910     assert(netdev->type == NET_CLIENT_DRIVER_VHOST_VDPA);
 911     opts = &netdev->u.vhost_vdpa;
 912     if (!opts->vhostdev && !opts->vhostfd) {
 913         error_setg(errp,
 914                    "vhost-vdpa: neither vhostdev= nor vhostfd= was specified");
 915         return -1;
 916     }
 917
 918     if (opts->vhostdev && opts->vhostfd) {
 919         error_setg(errp,
 920                    "vhost-vdpa: vhostdev= and vhostfd= are mutually exclusive");
 921         return -1;
 922     }
 923
 924     if (opts->vhostdev) {
 925         vdpa_device_fd = qemu_open(opts->vhostdev, O_RDWR, errp);
 926         if (vdpa_device_fd == -1) {
 927             return -errno;
 928         }
 929     } else {
 930         /* has_vhostfd */
 931         vdpa_device_fd = monitor_fd_param(monitor_cur(), opts->vhostfd, errp);
 932         if (vdpa_device_fd == -1) {
 933             error_prepend(errp, "vhost-vdpa: unable to parse vhostfd: ");
 934             return -1;
 935         }
 936     }
 937
 938     r = vhost_vdpa_get_features(vdpa_device_fd, &features, errp);
 939     if (unlikely(r < 0)) {
 940         goto err;
 941     }
 942
 943     queue_pairs = vhost_vdpa_get_max_queue_pairs(vdpa_device_fd, features,
 944                                                  &has_cvq, errp);
 945     if (queue_pairs < 0) {
 946         qemu_close(vdpa_device_fd);
 947         return queue_pairs;
 948     }
 949
 950     r = vhost_vdpa_get_iova_range(vdpa_device_fd, &iova_range);
 951     if (unlikely(r < 0)) {
 952         error_setg(errp, "vhost-vdpa: get iova range failed: %s",
 953                    strerror(-r));
 954         goto err;
 955     }
 956
 957     if (opts->x_svq && !vhost_vdpa_net_valid_svq_features(features, errp)) {
 958         goto err;
 959     }
 960
 961     ncs = g_malloc0(sizeof(*ncs) * queue_pairs);
 962
 963     for (i = 0; i < queue_pairs; i++) {
 964         ncs[i] = net_vhost_vdpa_init(peer, TYPE_VHOST_VDPA, name,
 965                                      vdpa_device_fd, i, 2, true, opts->x_svq,
 966                                      iova_range, features);
 967         if (!ncs[i])
 968             goto err;
 969     }
 970
 971     if (has_cvq) {
 972         nc = net_vhost_vdpa_init(peer, TYPE_VHOST_VDPA, name,
 973                                  vdpa_device_fd, i, 1, false,
 974                                  opts->x_svq, iova_range, features);
 975         if (!nc)
 976             goto err;
 977     }
 978
 979     return 0;
 980
 981 err:
 982     if (i) {
 983         for (i--; i >= 0; i--) {
 984             qemu_del_net_client(ncs[i]);
 985         }
 986     }
 987
 988     qemu_close(vdpa_device_fd);
 989
 990     return -1;
 991 }