hw/virtio/vhost-user.c

   1 /*
   2  * vhost-user
   3  *
   4  * Copyright (c) 2013 Virtual Open Systems Sarl.
   5  *
   6  * This work is licensed under the terms of the GNU GPL, version 2 or later.
   7  * See the COPYING file in the top-level directory.
   8  *
   9  */
  10
  11 #include "qemu/osdep.h"
  12 #include "qapi/error.h"
  13 #include "hw/virtio/vhost.h"
  14 #include "hw/virtio/vhost-user.h"
  15 #include "hw/virtio/vhost-backend.h"
  16 #include "hw/virtio/virtio.h"
  17 #include "hw/virtio/virtio-net.h"
  18 #include "chardev/char-fe.h"
  19 #include "sysemu/kvm.h"
  20 #include "qemu/error-report.h"
  21 #include "qemu/main-loop.h"
  22 #include "qemu/sockets.h"
  23 #include "sysemu/cryptodev.h"
  24 #include "migration/migration.h"
  25 #include "migration/postcopy-ram.h"
  26 #include "trace.h"
  27
  28 #include <sys/ioctl.h>
  29 #include <sys/socket.h>
  30 #include <sys/un.h>
  31
  32 #include "standard-headers/linux/vhost_types.h"
  33
  34 #ifdef CONFIG_LINUX
  35 #include <linux/userfaultfd.h>
  36 #endif
  37
  38 #define VHOST_MEMORY_BASELINE_NREGIONS    8
  39 #define VHOST_USER_F_PROTOCOL_FEATURES 30
  40 #define VHOST_USER_SLAVE_MAX_FDS     8
  41
  42 /*
  43  * Set maximum number of RAM slots supported to
  44  * the maximum number supported by the target
  45  * hardware plaform.
  46  */
  47 #if defined(TARGET_X86) || defined(TARGET_X86_64) || \
  48     defined(TARGET_ARM) || defined(TARGET_ARM_64)
  49 #include "hw/acpi/acpi.h"
  50 #define VHOST_USER_MAX_RAM_SLOTS ACPI_MAX_RAM_SLOTS
  51
  52 #elif defined(TARGET_PPC) || defined(TARGET_PPC_64)
  53 #include "hw/ppc/spapr.h"
  54 #define VHOST_USER_MAX_RAM_SLOTS SPAPR_MAX_RAM_SLOTS
  55
  56 #else
  57 #define VHOST_USER_MAX_RAM_SLOTS 512
  58 #endif
  59
  60 /*
  61  * Maximum size of virtio device config space
  62  */
  63 #define VHOST_USER_MAX_CONFIG_SIZE 256
  64
  65 enum VhostUserProtocolFeature {
  66     VHOST_USER_PROTOCOL_F_MQ = 0,
  67     VHOST_USER_PROTOCOL_F_LOG_SHMFD = 1,
  68     VHOST_USER_PROTOCOL_F_RARP = 2,
  69     VHOST_USER_PROTOCOL_F_REPLY_ACK = 3,
  70     VHOST_USER_PROTOCOL_F_NET_MTU = 4,
  71     VHOST_USER_PROTOCOL_F_SLAVE_REQ = 5,
  72     VHOST_USER_PROTOCOL_F_CROSS_ENDIAN = 6,
  73     VHOST_USER_PROTOCOL_F_CRYPTO_SESSION = 7,
  74     VHOST_USER_PROTOCOL_F_PAGEFAULT = 8,
  75     VHOST_USER_PROTOCOL_F_CONFIG = 9,
  76     VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD = 10,
  77     VHOST_USER_PROTOCOL_F_HOST_NOTIFIER = 11,
  78     VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD = 12,
  79     VHOST_USER_PROTOCOL_F_RESET_DEVICE = 13,
  80     /* Feature 14 reserved for VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS. */
  81     VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS = 15,
  82     VHOST_USER_PROTOCOL_F_MAX
  83 };
  84
  85 #define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1)
  86
  87 typedef enum VhostUserRequest {
  88     VHOST_USER_NONE = 0,
  89     VHOST_USER_GET_FEATURES = 1,
  90     VHOST_USER_SET_FEATURES = 2,
  91     VHOST_USER_SET_OWNER = 3,
  92     VHOST_USER_RESET_OWNER = 4,
  93     VHOST_USER_SET_MEM_TABLE = 5,
  94     VHOST_USER_SET_LOG_BASE = 6,
  95     VHOST_USER_SET_LOG_FD = 7,
  96     VHOST_USER_SET_VRING_NUM = 8,
  97     VHOST_USER_SET_VRING_ADDR = 9,
  98     VHOST_USER_SET_VRING_BASE = 10,
  99     VHOST_USER_GET_VRING_BASE = 11,
 100     VHOST_USER_SET_VRING_KICK = 12,
 101     VHOST_USER_SET_VRING_CALL = 13,
 102     VHOST_USER_SET_VRING_ERR = 14,
 103     VHOST_USER_GET_PROTOCOL_FEATURES = 15,
 104     VHOST_USER_SET_PROTOCOL_FEATURES = 16,
 105     VHOST_USER_GET_QUEUE_NUM = 17,
 106     VHOST_USER_SET_VRING_ENABLE = 18,
 107     VHOST_USER_SEND_RARP = 19,
 108     VHOST_USER_NET_SET_MTU = 20,
 109     VHOST_USER_SET_SLAVE_REQ_FD = 21,
 110     VHOST_USER_IOTLB_MSG = 22,
 111     VHOST_USER_SET_VRING_ENDIAN = 23,
 112     VHOST_USER_GET_CONFIG = 24,
 113     VHOST_USER_SET_CONFIG = 25,
 114     VHOST_USER_CREATE_CRYPTO_SESSION = 26,
 115     VHOST_USER_CLOSE_CRYPTO_SESSION = 27,
 116     VHOST_USER_POSTCOPY_ADVISE  = 28,
 117     VHOST_USER_POSTCOPY_LISTEN  = 29,
 118     VHOST_USER_POSTCOPY_END     = 30,
 119     VHOST_USER_GET_INFLIGHT_FD = 31,
 120     VHOST_USER_SET_INFLIGHT_FD = 32,
 121     VHOST_USER_GPU_SET_SOCKET = 33,
 122     VHOST_USER_RESET_DEVICE = 34,
 123     /* Message number 35 reserved for VHOST_USER_VRING_KICK. */
 124     VHOST_USER_GET_MAX_MEM_SLOTS = 36,
 125     VHOST_USER_ADD_MEM_REG = 37,
 126     VHOST_USER_REM_MEM_REG = 38,
 127     VHOST_USER_MAX
 128 } VhostUserRequest;
 129
 130 typedef enum VhostUserSlaveRequest {
 131     VHOST_USER_SLAVE_NONE = 0,
 132     VHOST_USER_SLAVE_IOTLB_MSG = 1,
 133     VHOST_USER_SLAVE_CONFIG_CHANGE_MSG = 2,
 134     VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG = 3,
 135     VHOST_USER_SLAVE_MAX
 136 }  VhostUserSlaveRequest;
 137
 138 typedef struct VhostUserMemoryRegion {
 139     uint64_t guest_phys_addr;
 140     uint64_t memory_size;
 141     uint64_t userspace_addr;
 142     uint64_t mmap_offset;
 143 } VhostUserMemoryRegion;
 144
 145 typedef struct VhostUserMemory {
 146     uint32_t nregions;
 147     uint32_t padding;
 148     VhostUserMemoryRegion regions[VHOST_MEMORY_BASELINE_NREGIONS];
 149 } VhostUserMemory;
 150
 151 typedef struct VhostUserMemRegMsg {
 152     uint32_t padding;
 153     VhostUserMemoryRegion region;
 154 } VhostUserMemRegMsg;
 155
 156 typedef struct VhostUserLog {
 157     uint64_t mmap_size;
 158     uint64_t mmap_offset;
 159 } VhostUserLog;
 160
 161 typedef struct VhostUserConfig {
 162     uint32_t offset;
 163     uint32_t size;
 164     uint32_t flags;
 165     uint8_t region[VHOST_USER_MAX_CONFIG_SIZE];
 166 } VhostUserConfig;
 167
 168 #define VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN    512
 169 #define VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN  64
 170
 171 typedef struct VhostUserCryptoSession {
 172     /* session id for success, -1 on errors */
 173     int64_t session_id;
 174     CryptoDevBackendSymSessionInfo session_setup_data;
 175     uint8_t key[VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN];
 176     uint8_t auth_key[VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN];
 177 } VhostUserCryptoSession;
 178
 179 static VhostUserConfig c __attribute__ ((unused));
 180 #define VHOST_USER_CONFIG_HDR_SIZE (sizeof(c.offset) \
 181                                    + sizeof(c.size) \
 182                                    + sizeof(c.flags))
 183
 184 typedef struct VhostUserVringArea {
 185     uint64_t u64;
 186     uint64_t size;
 187     uint64_t offset;
 188 } VhostUserVringArea;
 189
 190 typedef struct VhostUserInflight {
 191     uint64_t mmap_size;
 192     uint64_t mmap_offset;
 193     uint16_t num_queues;
 194     uint16_t queue_size;
 195 } VhostUserInflight;
 196
 197 typedef struct {
 198     VhostUserRequest request;
 199
 200 #define VHOST_USER_VERSION_MASK     (0x3)
 201 #define VHOST_USER_REPLY_MASK       (0x1<<2)
 202 #define VHOST_USER_NEED_REPLY_MASK  (0x1 << 3)
 203     uint32_t flags;
 204     uint32_t size; /* the following payload size */
 205 } QEMU_PACKED VhostUserHeader;
 206
 207 typedef union {
 208 #define VHOST_USER_VRING_IDX_MASK   (0xff)
 209 #define VHOST_USER_VRING_NOFD_MASK  (0x1<<8)
 210         uint64_t u64;
 211         struct vhost_vring_state state;
 212         struct vhost_vring_addr addr;
 213         VhostUserMemory memory;
 214         VhostUserMemRegMsg mem_reg;
 215         VhostUserLog log;
 216         struct vhost_iotlb_msg iotlb;
 217         VhostUserConfig config;
 218         VhostUserCryptoSession session;
 219         VhostUserVringArea area;
 220         VhostUserInflight inflight;
 221 } VhostUserPayload;
 222
 223 typedef struct VhostUserMsg {
 224     VhostUserHeader hdr;
 225     VhostUserPayload payload;
 226 } QEMU_PACKED VhostUserMsg;
 227
 228 static VhostUserMsg m __attribute__ ((unused));
 229 #define VHOST_USER_HDR_SIZE (sizeof(VhostUserHeader))
 230
 231 #define VHOST_USER_PAYLOAD_SIZE (sizeof(VhostUserPayload))
 232
 233 /* The version of the protocol we support */
 234 #define VHOST_USER_VERSION    (0x1)
 235
 236 struct vhost_user {
 237     struct vhost_dev *dev;
 238     /* Shared between vhost devs of the same virtio device */
 239     VhostUserState *user;
 240     int slave_fd;
 241     NotifierWithReturn postcopy_notifier;
 242     struct PostCopyFD  postcopy_fd;
 243     uint64_t           postcopy_client_bases[VHOST_USER_MAX_RAM_SLOTS];
 244     /* Length of the region_rb and region_rb_offset arrays */
 245     size_t             region_rb_len;
 246     /* RAMBlock associated with a given region */
 247     RAMBlock         **region_rb;
 248     /* The offset from the start of the RAMBlock to the start of the
 249      * vhost region.
 250      */
 251     ram_addr_t        *region_rb_offset;
 252
 253     /* True once we've entered postcopy_listen */
 254     bool               postcopy_listen;
 255
 256     /* Our current regions */
 257     int num_shadow_regions;
 258     struct vhost_memory_region shadow_regions[VHOST_USER_MAX_RAM_SLOTS];
 259 };
 260
 261 struct scrub_regions {
 262     struct vhost_memory_region *region;
 263     int reg_idx;
 264     int fd_idx;
 265 };
 266
 267 static bool ioeventfd_enabled(void)
 268 {
 269     return !kvm_enabled() || kvm_eventfds_enabled();
 270 }
 271
 272 static int vhost_user_read_header(struct vhost_dev *dev, VhostUserMsg *msg)
 273 {
 274     struct vhost_user *u = dev->opaque;
 275     CharBackend *chr = u->user->chr;
 276     uint8_t *p = (uint8_t *) msg;
 277     int r, size = VHOST_USER_HDR_SIZE;
 278
 279     r = qemu_chr_fe_read_all(chr, p, size);
 280     if (r != size) {
 281         error_report("Failed to read msg header. Read %d instead of %d."
 282                      " Original request %d.", r, size, msg->hdr.request);
 283         return -1;
 284     }
 285
 286     /* validate received flags */
 287     if (msg->hdr.flags != (VHOST_USER_REPLY_MASK | VHOST_USER_VERSION)) {
 288         error_report("Failed to read msg header."
 289                 " Flags 0x%x instead of 0x%x.", msg->hdr.flags,
 290                 VHOST_USER_REPLY_MASK | VHOST_USER_VERSION);
 291         return -1;
 292     }
 293
 294     return 0;
 295 }
 296
 297 static int vhost_user_read(struct vhost_dev *dev, VhostUserMsg *msg)
 298 {
 299     struct vhost_user *u = dev->opaque;
 300     CharBackend *chr = u->user->chr;
 301     uint8_t *p = (uint8_t *) msg;
 302     int r, size;
 303
 304     if (vhost_user_read_header(dev, msg) < 0) {
 305         return -1;
 306     }
 307
 308     /* validate message size is sane */
 309     if (msg->hdr.size > VHOST_USER_PAYLOAD_SIZE) {
 310         error_report("Failed to read msg header."
 311                 " Size %d exceeds the maximum %zu.", msg->hdr.size,
 312                 VHOST_USER_PAYLOAD_SIZE);
 313         return -1;
 314     }
 315
 316     if (msg->hdr.size) {
 317         p += VHOST_USER_HDR_SIZE;
 318         size = msg->hdr.size;
 319         r = qemu_chr_fe_read_all(chr, p, size);
 320         if (r != size) {
 321             error_report("Failed to read msg payload."
 322                          " Read %d instead of %d.", r, msg->hdr.size);
 323             return -1;
 324         }
 325     }
 326
 327     return 0;
 328 }
 329
 330 static int process_message_reply(struct vhost_dev *dev,
 331                                  const VhostUserMsg *msg)
 332 {
 333     VhostUserMsg msg_reply;
 334
 335     if ((msg->hdr.flags & VHOST_USER_NEED_REPLY_MASK) == 0) {
 336         return 0;
 337     }
 338
 339     if (vhost_user_read(dev, &msg_reply) < 0) {
 340         return -1;
 341     }
 342
 343     if (msg_reply.hdr.request != msg->hdr.request) {
 344         error_report("Received unexpected msg type."
 345                      "Expected %d received %d",
 346                      msg->hdr.request, msg_reply.hdr.request);
 347         return -1;
 348     }
 349
 350     return msg_reply.payload.u64 ? -1 : 0;
 351 }
 352
 353 static bool vhost_user_one_time_request(VhostUserRequest request)
 354 {
 355     switch (request) {
 356     case VHOST_USER_SET_OWNER:
 357     case VHOST_USER_RESET_OWNER:
 358     case VHOST_USER_SET_MEM_TABLE:
 359     case VHOST_USER_GET_QUEUE_NUM:
 360     case VHOST_USER_NET_SET_MTU:
 361         return true;
 362     default:
 363         return false;
 364     }
 365 }
 366
 367 /* most non-init callers ignore the error */
 368 static int vhost_user_write(struct vhost_dev *dev, VhostUserMsg *msg,
 369                             int *fds, int fd_num)
 370 {
 371     struct vhost_user *u = dev->opaque;
 372     CharBackend *chr = u->user->chr;
 373     int ret, size = VHOST_USER_HDR_SIZE + msg->hdr.size;
 374
 375     /*
 376      * For non-vring specific requests, like VHOST_USER_SET_MEM_TABLE,
 377      * we just need send it once in the first time. For later such
 378      * request, we just ignore it.
 379      */
 380     if (vhost_user_one_time_request(msg->hdr.request) && dev->vq_index != 0) {
 381         msg->hdr.flags &= ~VHOST_USER_NEED_REPLY_MASK;
 382         return 0;
 383     }
 384
 385     if (qemu_chr_fe_set_msgfds(chr, fds, fd_num) < 0) {
 386         error_report("Failed to set msg fds.");
 387         return -1;
 388     }
 389
 390     ret = qemu_chr_fe_write_all(chr, (const uint8_t *) msg, size);
 391     if (ret != size) {
 392         error_report("Failed to write msg."
 393                      " Wrote %d instead of %d.", ret, size);
 394         return -1;
 395     }
 396
 397     return 0;
 398 }
 399
 400 int vhost_user_gpu_set_socket(struct vhost_dev *dev, int fd)
 401 {
 402     VhostUserMsg msg = {
 403         .hdr.request = VHOST_USER_GPU_SET_SOCKET,
 404         .hdr.flags = VHOST_USER_VERSION,
 405     };
 406
 407     return vhost_user_write(dev, &msg, &fd, 1);
 408 }
 409
 410 static int vhost_user_set_log_base(struct vhost_dev *dev, uint64_t base,
 411                                    struct vhost_log *log)
 412 {
 413     int fds[VHOST_USER_MAX_RAM_SLOTS];
 414     size_t fd_num = 0;
 415     bool shmfd = virtio_has_feature(dev->protocol_features,
 416                                     VHOST_USER_PROTOCOL_F_LOG_SHMFD);
 417     VhostUserMsg msg = {
 418         .hdr.request = VHOST_USER_SET_LOG_BASE,
 419         .hdr.flags = VHOST_USER_VERSION,
 420         .payload.log.mmap_size = log->size * sizeof(*(log->log)),
 421         .payload.log.mmap_offset = 0,
 422         .hdr.size = sizeof(msg.payload.log),
 423     };
 424
 425     if (shmfd && log->fd != -1) {
 426         fds[fd_num++] = log->fd;
 427     }
 428
 429     if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
 430         return -1;
 431     }
 432
 433     if (shmfd) {
 434         msg.hdr.size = 0;
 435         if (vhost_user_read(dev, &msg) < 0) {
 436             return -1;
 437         }
 438
 439         if (msg.hdr.request != VHOST_USER_SET_LOG_BASE) {
 440             error_report("Received unexpected msg type. "
 441                          "Expected %d received %d",
 442                          VHOST_USER_SET_LOG_BASE, msg.hdr.request);
 443             return -1;
 444         }
 445     }
 446
 447     return 0;
 448 }
 449
 450 static MemoryRegion *vhost_user_get_mr_data(uint64_t addr, ram_addr_t *offset,
 451                                             int *fd)
 452 {
 453     MemoryRegion *mr;
 454
 455     assert((uintptr_t)addr == addr);
 456     mr = memory_region_from_host((void *)(uintptr_t)addr, offset);
 457     *fd = memory_region_get_fd(mr);
 458
 459     return mr;
 460 }
 461
 462 static void vhost_user_fill_msg_region(VhostUserMemoryRegion *dst,
 463                                        struct vhost_memory_region *src,
 464                                        uint64_t mmap_offset)
 465 {
 466     assert(src != NULL && dst != NULL);
 467     dst->userspace_addr = src->userspace_addr;
 468     dst->memory_size = src->memory_size;
 469     dst->guest_phys_addr = src->guest_phys_addr;
 470     dst->mmap_offset = mmap_offset;
 471 }
 472
 473 static int vhost_user_fill_set_mem_table_msg(struct vhost_user *u,
 474                                              struct vhost_dev *dev,
 475                                              VhostUserMsg *msg,
 476                                              int *fds, size_t *fd_num,
 477                                              bool track_ramblocks)
 478 {
 479     int i, fd;
 480     ram_addr_t offset;
 481     MemoryRegion *mr;
 482     struct vhost_memory_region *reg;
 483     VhostUserMemoryRegion region_buffer;
 484
 485     msg->hdr.request = VHOST_USER_SET_MEM_TABLE;
 486
 487     for (i = 0; i < dev->mem->nregions; ++i) {
 488         reg = dev->mem->regions + i;
 489
 490         mr = vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
 491         if (fd > 0) {
 492             if (track_ramblocks) {
 493                 assert(*fd_num < VHOST_MEMORY_BASELINE_NREGIONS);
 494                 trace_vhost_user_set_mem_table_withfd(*fd_num, mr->name,
 495                                                       reg->memory_size,
 496                                                       reg->guest_phys_addr,
 497                                                       reg->userspace_addr,
 498                                                       offset);
 499                 u->region_rb_offset[i] = offset;
 500                 u->region_rb[i] = mr->ram_block;
 501             } else if (*fd_num == VHOST_MEMORY_BASELINE_NREGIONS) {
 502                 error_report("Failed preparing vhost-user memory table msg");
 503                 return -1;
 504             }
 505             vhost_user_fill_msg_region(&region_buffer, reg, offset);
 506             msg->payload.memory.regions[*fd_num] = region_buffer;
 507             fds[(*fd_num)++] = fd;
 508         } else if (track_ramblocks) {
 509             u->region_rb_offset[i] = 0;
 510             u->region_rb[i] = NULL;
 511         }
 512     }
 513
 514     msg->payload.memory.nregions = *fd_num;
 515
 516     if (!*fd_num) {
 517         error_report("Failed initializing vhost-user memory map, "
 518                      "consider using -object memory-backend-file share=on");
 519         return -1;
 520     }
 521
 522     msg->hdr.size = sizeof(msg->payload.memory.nregions);
 523     msg->hdr.size += sizeof(msg->payload.memory.padding);
 524     msg->hdr.size += *fd_num * sizeof(VhostUserMemoryRegion);
 525
 526     return 1;
 527 }
 528
 529 static inline bool reg_equal(struct vhost_memory_region *shadow_reg,
 530                              struct vhost_memory_region *vdev_reg)
 531 {
 532     return shadow_reg->guest_phys_addr == vdev_reg->guest_phys_addr &&
 533         shadow_reg->userspace_addr == vdev_reg->userspace_addr &&
 534         shadow_reg->memory_size == vdev_reg->memory_size;
 535 }
 536
 537 static void scrub_shadow_regions(struct vhost_dev *dev,
 538                                  struct scrub_regions *add_reg,
 539                                  int *nr_add_reg,
 540                                  struct scrub_regions *rem_reg,
 541                                  int *nr_rem_reg, uint64_t *shadow_pcb,
 542                                  bool track_ramblocks)
 543 {
 544     struct vhost_user *u = dev->opaque;
 545     bool found[VHOST_USER_MAX_RAM_SLOTS] = {};
 546     struct vhost_memory_region *reg, *shadow_reg;
 547     int i, j, fd, add_idx = 0, rm_idx = 0, fd_num = 0;
 548     ram_addr_t offset;
 549     MemoryRegion *mr;
 550     bool matching;
 551
 552     /*
 553      * Find memory regions present in our shadow state which are not in
 554      * the device's current memory state.
 555      *
 556      * Mark regions in both the shadow and device state as "found".
 557      */
 558     for (i = 0; i < u->num_shadow_regions; i++) {
 559         shadow_reg = &u->shadow_regions[i];
 560         matching = false;
 561
 562         for (j = 0; j < dev->mem->nregions; j++) {
 563             reg = &dev->mem->regions[j];
 564
 565             mr = vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
 566
 567             if (reg_equal(shadow_reg, reg)) {
 568                 matching = true;
 569                 found[j] = true;
 570                 if (track_ramblocks) {
 571                     /*
 572                      * Reset postcopy client bases, region_rb, and
 573                      * region_rb_offset in case regions are removed.
 574                      */
 575                     if (fd > 0) {
 576                         u->region_rb_offset[j] = offset;
 577                         u->region_rb[j] = mr->ram_block;
 578                         shadow_pcb[j] = u->postcopy_client_bases[i];
 579                     } else {
 580                         u->region_rb_offset[j] = 0;
 581                         u->region_rb[j] = NULL;
 582                     }
 583                 }
 584                 break;
 585             }
 586         }
 587
 588         /*
 589          * If the region was not found in the current device memory state
 590          * create an entry for it in the removed list.
 591          */
 592         if (!matching) {
 593             rem_reg[rm_idx].region = shadow_reg;
 594             rem_reg[rm_idx++].reg_idx = i;
 595         }
 596     }
 597
 598     /*
 599      * For regions not marked "found", create entries in the added list.
 600      *
 601      * Note their indexes in the device memory state and the indexes of their
 602      * file descriptors.
 603      */
 604     for (i = 0; i < dev->mem->nregions; i++) {
 605         reg = &dev->mem->regions[i];
 606         mr = vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
 607         if (fd > 0) {
 608             ++fd_num;
 609         }
 610
 611         /*
 612          * If the region was in both the shadow and device state we don't
 613          * need to send a VHOST_USER_ADD_MEM_REG message for it.
 614          */
 615         if (found[i]) {
 616             continue;
 617         }
 618
 619         add_reg[add_idx].region = reg;
 620         add_reg[add_idx].reg_idx = i;
 621         add_reg[add_idx++].fd_idx = fd_num;
 622     }
 623     *nr_rem_reg = rm_idx;
 624     *nr_add_reg = add_idx;
 625
 626     return;
 627 }
 628
 629 static int send_remove_regions(struct vhost_dev *dev,
 630                                struct scrub_regions *remove_reg,
 631                                int nr_rem_reg, VhostUserMsg *msg,
 632                                bool reply_supported)
 633 {
 634     struct vhost_user *u = dev->opaque;
 635     struct vhost_memory_region *shadow_reg;
 636     int i, fd, shadow_reg_idx, ret;
 637     ram_addr_t offset;
 638     VhostUserMemoryRegion region_buffer;
 639
 640     /*
 641      * The regions in remove_reg appear in the same order they do in the
 642      * shadow table. Therefore we can minimize memory copies by iterating
 643      * through remove_reg backwards.
 644      */
 645     for (i = nr_rem_reg - 1; i >= 0; i--) {
 646         shadow_reg = remove_reg[i].region;
 647         shadow_reg_idx = remove_reg[i].reg_idx;
 648
 649         vhost_user_get_mr_data(shadow_reg->userspace_addr, &offset, &fd);
 650
 651         if (fd > 0) {
 652             msg->hdr.request = VHOST_USER_REM_MEM_REG;
 653             vhost_user_fill_msg_region(&region_buffer, shadow_reg, 0);
 654             msg->payload.mem_reg.region = region_buffer;
 655
 656             if (vhost_user_write(dev, msg, &fd, 1) < 0) {
 657                 return -1;
 658             }
 659
 660             if (reply_supported) {
 661                 ret = process_message_reply(dev, msg);
 662                 if (ret) {
 663                     return ret;
 664                 }
 665             }
 666         }
 667
 668         /*
 669          * At this point we know the backend has unmapped the region. It is now
 670          * safe to remove it from the shadow table.
 671          */
 672         memmove(&u->shadow_regions[shadow_reg_idx],
 673                 &u->shadow_regions[shadow_reg_idx + 1],
 674                 sizeof(struct vhost_memory_region) *
 675                 (u->num_shadow_regions - shadow_reg_idx));
 676         u->num_shadow_regions--;
 677     }
 678
 679     return 0;
 680 }
 681
 682 static int send_add_regions(struct vhost_dev *dev,
 683                             struct scrub_regions *add_reg, int nr_add_reg,
 684                             VhostUserMsg *msg, uint64_t *shadow_pcb,
 685                             bool reply_supported, bool track_ramblocks)
 686 {
 687     struct vhost_user *u = dev->opaque;
 688     int i, fd, ret, reg_idx, reg_fd_idx;
 689     struct vhost_memory_region *reg;
 690     MemoryRegion *mr;
 691     ram_addr_t offset;
 692     VhostUserMsg msg_reply;
 693     VhostUserMemoryRegion region_buffer;
 694
 695     for (i = 0; i < nr_add_reg; i++) {
 696         reg = add_reg[i].region;
 697         reg_idx = add_reg[i].reg_idx;
 698         reg_fd_idx = add_reg[i].fd_idx;
 699
 700         mr = vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
 701
 702         if (fd > 0) {
 703             if (track_ramblocks) {
 704                 trace_vhost_user_set_mem_table_withfd(reg_fd_idx, mr->name,
 705                                                       reg->memory_size,
 706                                                       reg->guest_phys_addr,
 707                                                       reg->userspace_addr,
 708                                                       offset);
 709                 u->region_rb_offset[reg_idx] = offset;
 710                 u->region_rb[reg_idx] = mr->ram_block;
 711             }
 712             msg->hdr.request = VHOST_USER_ADD_MEM_REG;
 713             vhost_user_fill_msg_region(&region_buffer, reg, offset);
 714             msg->payload.mem_reg.region = region_buffer;
 715
 716             if (vhost_user_write(dev, msg, &fd, 1) < 0) {
 717                 return -1;
 718             }
 719
 720             if (track_ramblocks) {
 721                 uint64_t reply_gpa;
 722
 723                 if (vhost_user_read(dev, &msg_reply) < 0) {
 724                     return -1;
 725                 }
 726
 727                 reply_gpa = msg_reply.payload.mem_reg.region.guest_phys_addr;
 728
 729                 if (msg_reply.hdr.request != VHOST_USER_ADD_MEM_REG) {
 730                     error_report("%s: Received unexpected msg type."
 731                                  "Expected %d received %d", __func__,
 732                                  VHOST_USER_ADD_MEM_REG,
 733                                  msg_reply.hdr.request);
 734                     return -1;
 735                 }
 736
 737                 /*
 738                  * We're using the same structure, just reusing one of the
 739                  * fields, so it should be the same size.
 740                  */
 741                 if (msg_reply.hdr.size != msg->hdr.size) {
 742                     error_report("%s: Unexpected size for postcopy reply "
 743                                  "%d vs %d", __func__, msg_reply.hdr.size,
 744                                  msg->hdr.size);
 745                     return -1;
 746                 }
 747
 748                 /* Get the postcopy client base from the backend's reply. */
 749                 if (reply_gpa == dev->mem->regions[reg_idx].guest_phys_addr) {
 750                     shadow_pcb[reg_idx] =
 751                         msg_reply.payload.mem_reg.region.userspace_addr;
 752                     trace_vhost_user_set_mem_table_postcopy(
 753                         msg_reply.payload.mem_reg.region.userspace_addr,
 754                         msg->payload.mem_reg.region.userspace_addr,
 755                         reg_fd_idx, reg_idx);
 756                 } else {
 757                     error_report("%s: invalid postcopy reply for region. "
 758                                  "Got guest physical address %" PRIX64 ", expected "
 759                                  "%" PRIX64, __func__, reply_gpa,
 760                                  dev->mem->regions[reg_idx].guest_phys_addr);
 761                     return -1;
 762                 }
 763             } else if (reply_supported) {
 764                 ret = process_message_reply(dev, msg);
 765                 if (ret) {
 766                     return ret;
 767                 }
 768             }
 769         } else if (track_ramblocks) {
 770             u->region_rb_offset[reg_idx] = 0;
 771             u->region_rb[reg_idx] = NULL;
 772         }
 773
 774         /*
 775          * At this point, we know the backend has mapped in the new
 776          * region, if the region has a valid file descriptor.
 777          *
 778          * The region should now be added to the shadow table.
 779          */
 780         u->shadow_regions[u->num_shadow_regions].guest_phys_addr =
 781             reg->guest_phys_addr;
 782         u->shadow_regions[u->num_shadow_regions].userspace_addr =
 783             reg->userspace_addr;
 784         u->shadow_regions[u->num_shadow_regions].memory_size =
 785             reg->memory_size;
 786         u->num_shadow_regions++;
 787     }
 788
 789     return 0;
 790 }
 791
 792 static int vhost_user_add_remove_regions(struct vhost_dev *dev,
 793                                          VhostUserMsg *msg,
 794                                          bool reply_supported,
 795                                          bool track_ramblocks)
 796 {
 797     struct vhost_user *u = dev->opaque;
 798     struct scrub_regions add_reg[VHOST_USER_MAX_RAM_SLOTS];
 799     struct scrub_regions rem_reg[VHOST_USER_MAX_RAM_SLOTS];
 800     uint64_t shadow_pcb[VHOST_USER_MAX_RAM_SLOTS] = {};
 801     int nr_add_reg, nr_rem_reg;
 802
 803     msg->hdr.size = sizeof(msg->payload.mem_reg.padding) +
 804         sizeof(VhostUserMemoryRegion);
 805
 806     /* Find the regions which need to be removed or added. */
 807     scrub_shadow_regions(dev, add_reg, &nr_add_reg, rem_reg, &nr_rem_reg,
 808                          shadow_pcb, track_ramblocks);
 809
 810     if (nr_rem_reg && send_remove_regions(dev, rem_reg, nr_rem_reg, msg,
 811                 reply_supported) < 0)
 812     {
 813         goto err;
 814     }
 815
 816     if (nr_add_reg && send_add_regions(dev, add_reg, nr_add_reg, msg,
 817                 shadow_pcb, reply_supported, track_ramblocks) < 0)
 818     {
 819         goto err;
 820     }
 821
 822     if (track_ramblocks) {
 823         memcpy(u->postcopy_client_bases, shadow_pcb,
 824                sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS);
 825         /*
 826          * Now we've registered this with the postcopy code, we ack to the
 827          * client, because now we're in the position to be able to deal with
 828          * any faults it generates.
 829          */
 830         /* TODO: Use this for failure cases as well with a bad value. */
 831         msg->hdr.size = sizeof(msg->payload.u64);
 832         msg->payload.u64 = 0; /* OK */
 833
 834         if (vhost_user_write(dev, msg, NULL, 0) < 0) {
 835             return -1;
 836         }
 837     }
 838
 839     return 0;
 840
 841 err:
 842     if (track_ramblocks) {
 843         memcpy(u->postcopy_client_bases, shadow_pcb,
 844                sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS);
 845     }
 846
 847     return -1;
 848 }
 849
 850 static int vhost_user_set_mem_table_postcopy(struct vhost_dev *dev,
 851                                              struct vhost_memory *mem,
 852                                              bool reply_supported,
 853                                              bool config_mem_slots)
 854 {
 855     struct vhost_user *u = dev->opaque;
 856     int fds[VHOST_MEMORY_BASELINE_NREGIONS];
 857     size_t fd_num = 0;
 858     VhostUserMsg msg_reply;
 859     int region_i, msg_i;
 860
 861     VhostUserMsg msg = {
 862         .hdr.flags = VHOST_USER_VERSION,
 863     };
 864
 865     if (u->region_rb_len < dev->mem->nregions) {
 866         u->region_rb = g_renew(RAMBlock*, u->region_rb, dev->mem->nregions);
 867         u->region_rb_offset = g_renew(ram_addr_t, u->region_rb_offset,
 868                                       dev->mem->nregions);
 869         memset(&(u->region_rb[u->region_rb_len]), '\0',
 870                sizeof(RAMBlock *) * (dev->mem->nregions - u->region_rb_len));
 871         memset(&(u->region_rb_offset[u->region_rb_len]), '\0',
 872                sizeof(ram_addr_t) * (dev->mem->nregions - u->region_rb_len));
 873         u->region_rb_len = dev->mem->nregions;
 874     }
 875
 876     if (config_mem_slots) {
 877         if (vhost_user_add_remove_regions(dev, &msg, reply_supported,
 878                                           true) < 0) {
 879             return -1;
 880         }
 881     } else {
 882         if (vhost_user_fill_set_mem_table_msg(u, dev, &msg, fds, &fd_num,
 883                                               true) < 0) {
 884             return -1;
 885         }
 886
 887         if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
 888             return -1;
 889         }
 890
 891         if (vhost_user_read(dev, &msg_reply) < 0) {
 892             return -1;
 893         }
 894
 895         if (msg_reply.hdr.request != VHOST_USER_SET_MEM_TABLE) {
 896             error_report("%s: Received unexpected msg type."
 897                          "Expected %d received %d", __func__,
 898                          VHOST_USER_SET_MEM_TABLE, msg_reply.hdr.request);
 899             return -1;
 900         }
 901
 902         /*
 903          * We're using the same structure, just reusing one of the
 904          * fields, so it should be the same size.
 905          */
 906         if (msg_reply.hdr.size != msg.hdr.size) {
 907             error_report("%s: Unexpected size for postcopy reply "
 908                          "%d vs %d", __func__, msg_reply.hdr.size,
 909                          msg.hdr.size);
 910             return -1;
 911         }
 912
 913         memset(u->postcopy_client_bases, 0,
 914                sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS);
 915
 916         /*
 917          * They're in the same order as the regions that were sent
 918          * but some of the regions were skipped (above) if they
 919          * didn't have fd's
 920          */
 921         for (msg_i = 0, region_i = 0;
 922              region_i < dev->mem->nregions;
 923              region_i++) {
 924             if (msg_i < fd_num &&
 925                 msg_reply.payload.memory.regions[msg_i].guest_phys_addr ==
 926                 dev->mem->regions[region_i].guest_phys_addr) {
 927                 u->postcopy_client_bases[region_i] =
 928                     msg_reply.payload.memory.regions[msg_i].userspace_addr;
 929                 trace_vhost_user_set_mem_table_postcopy(
 930                     msg_reply.payload.memory.regions[msg_i].userspace_addr,
 931                     msg.payload.memory.regions[msg_i].userspace_addr,
 932                     msg_i, region_i);
 933                 msg_i++;
 934             }
 935         }
 936         if (msg_i != fd_num) {
 937             error_report("%s: postcopy reply not fully consumed "
 938                          "%d vs %zd",
 939                          __func__, msg_i, fd_num);
 940             return -1;
 941         }
 942
 943         /*
 944          * Now we've registered this with the postcopy code, we ack to the
 945          * client, because now we're in the position to be able to deal
 946          * with any faults it generates.
 947          */
 948         /* TODO: Use this for failure cases as well with a bad value. */
 949         msg.hdr.size = sizeof(msg.payload.u64);
 950         msg.payload.u64 = 0; /* OK */
 951         if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 952             return -1;
 953         }
 954     }
 955
 956     return 0;
 957 }
 958
 959 static int vhost_user_set_mem_table(struct vhost_dev *dev,
 960                                     struct vhost_memory *mem)
 961 {
 962     struct vhost_user *u = dev->opaque;
 963     int fds[VHOST_MEMORY_BASELINE_NREGIONS];
 964     size_t fd_num = 0;
 965     bool do_postcopy = u->postcopy_listen && u->postcopy_fd.handler;
 966     bool reply_supported = virtio_has_feature(dev->protocol_features,
 967                                               VHOST_USER_PROTOCOL_F_REPLY_ACK);
 968     bool config_mem_slots =
 969         virtio_has_feature(dev->protocol_features,
 970                            VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS);
 971
 972     if (do_postcopy) {
 973         /*
 974          * Postcopy has enough differences that it's best done in it's own
 975          * version
 976          */
 977         return vhost_user_set_mem_table_postcopy(dev, mem, reply_supported,
 978                                                  config_mem_slots);
 979     }
 980
 981     VhostUserMsg msg = {
 982         .hdr.flags = VHOST_USER_VERSION,
 983     };
 984
 985     if (reply_supported) {
 986         msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
 987     }
 988
 989     if (config_mem_slots) {
 990         if (vhost_user_add_remove_regions(dev, &msg, reply_supported,
 991                                           false) < 0) {
 992             return -1;
 993         }
 994     } else {
 995         if (vhost_user_fill_set_mem_table_msg(u, dev, &msg, fds, &fd_num,
 996                                               false) < 0) {
 997             return -1;
 998         }
 999         if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
1000             return -1;
1001         }
1002
1003         if (reply_supported) {
1004             return process_message_reply(dev, &msg);
1005         }
1006     }
1007
1008     return 0;
1009 }
1010
1011 static int vhost_user_set_vring_addr(struct vhost_dev *dev,
1012                                      struct vhost_vring_addr *addr)
1013 {
1014     VhostUserMsg msg = {
1015         .hdr.request = VHOST_USER_SET_VRING_ADDR,
1016         .hdr.flags = VHOST_USER_VERSION,
1017         .payload.addr = *addr,
1018         .hdr.size = sizeof(msg.payload.addr),
1019     };
1020
1021     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1022         return -1;
1023     }
1024
1025     return 0;
1026 }
1027
1028 static int vhost_user_set_vring_endian(struct vhost_dev *dev,
1029                                        struct vhost_vring_state *ring)
1030 {
1031     bool cross_endian = virtio_has_feature(dev->protocol_features,
1032                                            VHOST_USER_PROTOCOL_F_CROSS_ENDIAN);
1033     VhostUserMsg msg = {
1034         .hdr.request = VHOST_USER_SET_VRING_ENDIAN,
1035         .hdr.flags = VHOST_USER_VERSION,
1036         .payload.state = *ring,
1037         .hdr.size = sizeof(msg.payload.state),
1038     };
1039
1040     if (!cross_endian) {
1041         error_report("vhost-user trying to send unhandled ioctl");
1042         return -1;
1043     }
1044
1045     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1046         return -1;
1047     }
1048
1049     return 0;
1050 }
1051
1052 static int vhost_set_vring(struct vhost_dev *dev,
1053                            unsigned long int request,
1054                            struct vhost_vring_state *ring)
1055 {
1056     VhostUserMsg msg = {
1057         .hdr.request = request,
1058         .hdr.flags = VHOST_USER_VERSION,
1059         .payload.state = *ring,
1060         .hdr.size = sizeof(msg.payload.state),
1061     };
1062
1063     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1064         return -1;
1065     }
1066
1067     return 0;
1068 }
1069
1070 static int vhost_user_set_vring_num(struct vhost_dev *dev,
1071                                     struct vhost_vring_state *ring)
1072 {
1073     return vhost_set_vring(dev, VHOST_USER_SET_VRING_NUM, ring);
1074 }
1075
1076 static void vhost_user_host_notifier_restore(struct vhost_dev *dev,
1077                                              int queue_idx)
1078 {
1079     struct vhost_user *u = dev->opaque;
1080     VhostUserHostNotifier *n = &u->user->notifier[queue_idx];
1081     VirtIODevice *vdev = dev->vdev;
1082
1083     if (n->addr && !n->set) {
1084         virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true);
1085         n->set = true;
1086     }
1087 }
1088
1089 static void vhost_user_host_notifier_remove(struct vhost_dev *dev,
1090                                             int queue_idx)
1091 {
1092     struct vhost_user *u = dev->opaque;
1093     VhostUserHostNotifier *n = &u->user->notifier[queue_idx];
1094     VirtIODevice *vdev = dev->vdev;
1095
1096     if (n->addr && n->set) {
1097         virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, false);
1098         n->set = false;
1099     }
1100 }
1101
1102 static int vhost_user_set_vring_base(struct vhost_dev *dev,
1103                                      struct vhost_vring_state *ring)
1104 {
1105     vhost_user_host_notifier_restore(dev, ring->index);
1106
1107     return vhost_set_vring(dev, VHOST_USER_SET_VRING_BASE, ring);
1108 }
1109
1110 static int vhost_user_set_vring_enable(struct vhost_dev *dev, int enable)
1111 {
1112     int i;
1113
1114     if (!virtio_has_feature(dev->features, VHOST_USER_F_PROTOCOL_FEATURES)) {
1115         return -1;
1116     }
1117
1118     for (i = 0; i < dev->nvqs; ++i) {
1119         struct vhost_vring_state state = {
1120             .index = dev->vq_index + i,
1121             .num   = enable,
1122         };
1123
1124         vhost_set_vring(dev, VHOST_USER_SET_VRING_ENABLE, &state);
1125     }
1126
1127     return 0;
1128 }
1129
1130 static int vhost_user_get_vring_base(struct vhost_dev *dev,
1131                                      struct vhost_vring_state *ring)
1132 {
1133     VhostUserMsg msg = {
1134         .hdr.request = VHOST_USER_GET_VRING_BASE,
1135         .hdr.flags = VHOST_USER_VERSION,
1136         .payload.state = *ring,
1137         .hdr.size = sizeof(msg.payload.state),
1138     };
1139
1140     vhost_user_host_notifier_remove(dev, ring->index);
1141
1142     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1143         return -1;
1144     }
1145
1146     if (vhost_user_read(dev, &msg) < 0) {
1147         return -1;
1148     }
1149
1150     if (msg.hdr.request != VHOST_USER_GET_VRING_BASE) {
1151         error_report("Received unexpected msg type. Expected %d received %d",
1152                      VHOST_USER_GET_VRING_BASE, msg.hdr.request);
1153         return -1;
1154     }
1155
1156     if (msg.hdr.size != sizeof(msg.payload.state)) {
1157         error_report("Received bad msg size.");
1158         return -1;
1159     }
1160
1161     *ring = msg.payload.state;
1162
1163     return 0;
1164 }
1165
1166 static int vhost_set_vring_file(struct vhost_dev *dev,
1167                                 VhostUserRequest request,
1168                                 struct vhost_vring_file *file)
1169 {
1170     int fds[VHOST_USER_MAX_RAM_SLOTS];
1171     size_t fd_num = 0;
1172     VhostUserMsg msg = {
1173         .hdr.request = request,
1174         .hdr.flags = VHOST_USER_VERSION,
1175         .payload.u64 = file->index & VHOST_USER_VRING_IDX_MASK,
1176         .hdr.size = sizeof(msg.payload.u64),
1177     };
1178
1179     if (ioeventfd_enabled() && file->fd > 0) {
1180         fds[fd_num++] = file->fd;
1181     } else {
1182         msg.payload.u64 |= VHOST_USER_VRING_NOFD_MASK;
1183     }
1184
1185     if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
1186         return -1;
1187     }
1188
1189     return 0;
1190 }
1191
1192 static int vhost_user_set_vring_kick(struct vhost_dev *dev,
1193                                      struct vhost_vring_file *file)
1194 {
1195     return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_KICK, file);
1196 }
1197
1198 static int vhost_user_set_vring_call(struct vhost_dev *dev,
1199                                      struct vhost_vring_file *file)
1200 {
1201     return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_CALL, file);
1202 }
1203
1204 static int vhost_user_set_u64(struct vhost_dev *dev, int request, uint64_t u64)
1205 {
1206     VhostUserMsg msg = {
1207         .hdr.request = request,
1208         .hdr.flags = VHOST_USER_VERSION,
1209         .payload.u64 = u64,
1210         .hdr.size = sizeof(msg.payload.u64),
1211     };
1212
1213     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1214         return -1;
1215     }
1216
1217     return 0;
1218 }
1219
1220 static int vhost_user_set_features(struct vhost_dev *dev,
1221                                    uint64_t features)
1222 {
1223     return vhost_user_set_u64(dev, VHOST_USER_SET_FEATURES, features);
1224 }
1225
1226 static int vhost_user_set_protocol_features(struct vhost_dev *dev,
1227                                             uint64_t features)
1228 {
1229     return vhost_user_set_u64(dev, VHOST_USER_SET_PROTOCOL_FEATURES, features);
1230 }
1231
1232 static int vhost_user_get_u64(struct vhost_dev *dev, int request, uint64_t *u64)
1233 {
1234     VhostUserMsg msg = {
1235         .hdr.request = request,
1236         .hdr.flags = VHOST_USER_VERSION,
1237     };
1238
1239     if (vhost_user_one_time_request(request) && dev->vq_index != 0) {
1240         return 0;
1241     }
1242
1243     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1244         return -1;
1245     }
1246
1247     if (vhost_user_read(dev, &msg) < 0) {
1248         return -1;
1249     }
1250
1251     if (msg.hdr.request != request) {
1252         error_report("Received unexpected msg type. Expected %d received %d",
1253                      request, msg.hdr.request);
1254         return -1;
1255     }
1256
1257     if (msg.hdr.size != sizeof(msg.payload.u64)) {
1258         error_report("Received bad msg size.");
1259         return -1;
1260     }
1261
1262     *u64 = msg.payload.u64;
1263
1264     return 0;
1265 }
1266
1267 static int vhost_user_get_features(struct vhost_dev *dev, uint64_t *features)
1268 {
1269     return vhost_user_get_u64(dev, VHOST_USER_GET_FEATURES, features);
1270 }
1271
1272 static int vhost_user_set_owner(struct vhost_dev *dev)
1273 {
1274     VhostUserMsg msg = {
1275         .hdr.request = VHOST_USER_SET_OWNER,
1276         .hdr.flags = VHOST_USER_VERSION,
1277     };
1278
1279     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1280         return -1;
1281     }
1282
1283     return 0;
1284 }
1285
1286 static int vhost_user_get_max_memslots(struct vhost_dev *dev,
1287                                        uint64_t *max_memslots)
1288 {
1289     uint64_t backend_max_memslots;
1290     int err;
1291
1292     err = vhost_user_get_u64(dev, VHOST_USER_GET_MAX_MEM_SLOTS,
1293                              &backend_max_memslots);
1294     if (err < 0) {
1295         return err;
1296     }
1297
1298     *max_memslots = backend_max_memslots;
1299
1300     return 0;
1301 }
1302
1303 static int vhost_user_reset_device(struct vhost_dev *dev)
1304 {
1305     VhostUserMsg msg = {
1306         .hdr.flags = VHOST_USER_VERSION,
1307     };
1308
1309     msg.hdr.request = virtio_has_feature(dev->protocol_features,
1310                                          VHOST_USER_PROTOCOL_F_RESET_DEVICE)
1311         ? VHOST_USER_RESET_DEVICE
1312         : VHOST_USER_RESET_OWNER;
1313
1314     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1315         return -1;
1316     }
1317
1318     return 0;
1319 }
1320
1321 static int vhost_user_slave_handle_config_change(struct vhost_dev *dev)
1322 {
1323     int ret = -1;
1324
1325     if (!dev->config_ops) {
1326         return -1;
1327     }
1328
1329     if (dev->config_ops->vhost_dev_config_notifier) {
1330         ret = dev->config_ops->vhost_dev_config_notifier(dev);
1331     }
1332
1333     return ret;
1334 }
1335
1336 static int vhost_user_slave_handle_vring_host_notifier(struct vhost_dev *dev,
1337                                                        VhostUserVringArea *area,
1338                                                        int fd)
1339 {
1340     int queue_idx = area->u64 & VHOST_USER_VRING_IDX_MASK;
1341     size_t page_size = qemu_real_host_page_size;
1342     struct vhost_user *u = dev->opaque;
1343     VhostUserState *user = u->user;
1344     VirtIODevice *vdev = dev->vdev;
1345     VhostUserHostNotifier *n;
1346     void *addr;
1347     char *name;
1348
1349     if (!virtio_has_feature(dev->protocol_features,
1350                             VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) ||
1351         vdev == NULL || queue_idx >= virtio_get_num_queues(vdev)) {
1352         return -1;
1353     }
1354
1355     n = &user->notifier[queue_idx];
1356
1357     if (n->addr) {
1358         virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, false);
1359         object_unparent(OBJECT(&n->mr));
1360         munmap(n->addr, page_size);
1361         n->addr = NULL;
1362     }
1363
1364     if (area->u64 & VHOST_USER_VRING_NOFD_MASK) {
1365         return 0;
1366     }
1367
1368     /* Sanity check. */
1369     if (area->size != page_size) {
1370         return -1;
1371     }
1372
1373     addr = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED,
1374                 fd, area->offset);
1375     if (addr == MAP_FAILED) {
1376         return -1;
1377     }
1378
1379     name = g_strdup_printf("vhost-user/host-notifier@%p mmaps[%d]",
1380                            user, queue_idx);
1381     memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name,
1382                                       page_size, addr);
1383     g_free(name);
1384
1385     if (virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true)) {
1386         munmap(addr, page_size);
1387         return -1;
1388     }
1389
1390     n->addr = addr;
1391     n->set = true;
1392
1393     return 0;
1394 }
1395
1396 static void slave_read(void *opaque)
1397 {
1398     struct vhost_dev *dev = opaque;
1399     struct vhost_user *u = dev->opaque;
1400     VhostUserHeader hdr = { 0, };
1401     VhostUserPayload payload = { 0, };
1402     int size, ret = 0;
1403     struct iovec iov;
1404     struct msghdr msgh;
1405     int fd[VHOST_USER_SLAVE_MAX_FDS];
1406     char control[CMSG_SPACE(sizeof(fd))];
1407     struct cmsghdr *cmsg;
1408     int i, fdsize = 0;
1409
1410     memset(&msgh, 0, sizeof(msgh));
1411     msgh.msg_iov = &iov;
1412     msgh.msg_iovlen = 1;
1413     msgh.msg_control = control;
1414     msgh.msg_controllen = sizeof(control);
1415
1416     memset(fd, -1, sizeof(fd));
1417
1418     /* Read header */
1419     iov.iov_base = &hdr;
1420     iov.iov_len = VHOST_USER_HDR_SIZE;
1421
1422     do {
1423         size = recvmsg(u->slave_fd, &msgh, 0);
1424     } while (size < 0 && (errno == EINTR || errno == EAGAIN));
1425
1426     if (size != VHOST_USER_HDR_SIZE) {
1427         error_report("Failed to read from slave.");
1428         goto err;
1429     }
1430
1431     if (msgh.msg_flags & MSG_CTRUNC) {
1432         error_report("Truncated message.");
1433         goto err;
1434     }
1435
1436     for (cmsg = CMSG_FIRSTHDR(&msgh); cmsg != NULL;
1437          cmsg = CMSG_NXTHDR(&msgh, cmsg)) {
1438             if (cmsg->cmsg_level == SOL_SOCKET &&
1439                 cmsg->cmsg_type == SCM_RIGHTS) {
1440                     fdsize = cmsg->cmsg_len - CMSG_LEN(0);
1441                     memcpy(fd, CMSG_DATA(cmsg), fdsize);
1442                     break;
1443             }
1444     }
1445
1446     if (hdr.size > VHOST_USER_PAYLOAD_SIZE) {
1447         error_report("Failed to read msg header."
1448                 " Size %d exceeds the maximum %zu.", hdr.size,
1449                 VHOST_USER_PAYLOAD_SIZE);
1450         goto err;
1451     }
1452
1453     /* Read payload */
1454     do {
1455         size = read(u->slave_fd, &payload, hdr.size);
1456     } while (size < 0 && (errno == EINTR || errno == EAGAIN));
1457
1458     if (size != hdr.size) {
1459         error_report("Failed to read payload from slave.");
1460         goto err;
1461     }
1462
1463     switch (hdr.request) {
1464     case VHOST_USER_SLAVE_IOTLB_MSG:
1465         ret = vhost_backend_handle_iotlb_msg(dev, &payload.iotlb);
1466         break;
1467     case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG :
1468         ret = vhost_user_slave_handle_config_change(dev);
1469         break;
1470     case VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG:
1471         ret = vhost_user_slave_handle_vring_host_notifier(dev, &payload.area,
1472                                                           fd[0]);
1473         break;
1474     default:
1475         error_report("Received unexpected msg type: %d.", hdr.request);
1476         ret = -EINVAL;
1477     }
1478
1479     /* Close the remaining file descriptors. */
1480     for (i = 0; i < fdsize; i++) {
1481         if (fd[i] != -1) {
1482             close(fd[i]);
1483         }
1484     }
1485
1486     /*
1487      * REPLY_ACK feature handling. Other reply types has to be managed
1488      * directly in their request handlers.
1489      */
1490     if (hdr.flags & VHOST_USER_NEED_REPLY_MASK) {
1491         struct iovec iovec[2];
1492
1493
1494         hdr.flags &= ~VHOST_USER_NEED_REPLY_MASK;
1495         hdr.flags |= VHOST_USER_REPLY_MASK;
1496
1497         payload.u64 = !!ret;
1498         hdr.size = sizeof(payload.u64);
1499
1500         iovec[0].iov_base = &hdr;
1501         iovec[0].iov_len = VHOST_USER_HDR_SIZE;
1502         iovec[1].iov_base = &payload;
1503         iovec[1].iov_len = hdr.size;
1504
1505         do {
1506             size = writev(u->slave_fd, iovec, ARRAY_SIZE(iovec));
1507         } while (size < 0 && (errno == EINTR || errno == EAGAIN));
1508
1509         if (size != VHOST_USER_HDR_SIZE + hdr.size) {
1510             error_report("Failed to send msg reply to slave.");
1511             goto err;
1512         }
1513     }
1514
1515     return;
1516
1517 err:
1518     qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL);
1519     close(u->slave_fd);
1520     u->slave_fd = -1;
1521     for (i = 0; i < fdsize; i++) {
1522         if (fd[i] != -1) {
1523             close(fd[i]);
1524         }
1525     }
1526     return;
1527 }
1528
1529 static int vhost_setup_slave_channel(struct vhost_dev *dev)
1530 {
1531     VhostUserMsg msg = {
1532         .hdr.request = VHOST_USER_SET_SLAVE_REQ_FD,
1533         .hdr.flags = VHOST_USER_VERSION,
1534     };
1535     struct vhost_user *u = dev->opaque;
1536     int sv[2], ret = 0;
1537     bool reply_supported = virtio_has_feature(dev->protocol_features,
1538                                               VHOST_USER_PROTOCOL_F_REPLY_ACK);
1539
1540     if (!virtio_has_feature(dev->protocol_features,
1541                             VHOST_USER_PROTOCOL_F_SLAVE_REQ)) {
1542         return 0;
1543     }
1544
1545     if (socketpair(PF_UNIX, SOCK_STREAM, 0, sv) == -1) {
1546         error_report("socketpair() failed");
1547         return -1;
1548     }
1549
1550     u->slave_fd = sv[0];
1551     qemu_set_fd_handler(u->slave_fd, slave_read, NULL, dev);
1552
1553     if (reply_supported) {
1554         msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
1555     }
1556
1557     ret = vhost_user_write(dev, &msg, &sv[1], 1);
1558     if (ret) {
1559         goto out;
1560     }
1561
1562     if (reply_supported) {
1563         ret = process_message_reply(dev, &msg);
1564     }
1565
1566 out:
1567     close(sv[1]);
1568     if (ret) {
1569         qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL);
1570         close(u->slave_fd);
1571         u->slave_fd = -1;
1572     }
1573
1574     return ret;
1575 }
1576
1577 #ifdef CONFIG_LINUX
1578 /*
1579  * Called back from the postcopy fault thread when a fault is received on our
1580  * ufd.
1581  * TODO: This is Linux specific
1582  */
1583 static int vhost_user_postcopy_fault_handler(struct PostCopyFD *pcfd,
1584                                              void *ufd)
1585 {
1586     struct vhost_dev *dev = pcfd->data;
1587     struct vhost_user *u = dev->opaque;
1588     struct uffd_msg *msg = ufd;
1589     uint64_t faultaddr = msg->arg.pagefault.address;
1590     RAMBlock *rb = NULL;
1591     uint64_t rb_offset;
1592     int i;
1593
1594     trace_vhost_user_postcopy_fault_handler(pcfd->idstr, faultaddr,
1595                                             dev->mem->nregions);
1596     for (i = 0; i < MIN(dev->mem->nregions, u->region_rb_len); i++) {
1597         trace_vhost_user_postcopy_fault_handler_loop(i,
1598                 u->postcopy_client_bases[i], dev->mem->regions[i].memory_size);
1599         if (faultaddr >= u->postcopy_client_bases[i]) {
1600             /* Ofset of the fault address in the vhost region */
1601             uint64_t region_offset = faultaddr - u->postcopy_client_bases[i];
1602             if (region_offset < dev->mem->regions[i].memory_size) {
1603                 rb_offset = region_offset + u->region_rb_offset[i];
1604                 trace_vhost_user_postcopy_fault_handler_found(i,
1605                         region_offset, rb_offset);
1606                 rb = u->region_rb[i];
1607                 return postcopy_request_shared_page(pcfd, rb, faultaddr,
1608                                                     rb_offset);
1609             }
1610         }
1611     }
1612     error_report("%s: Failed to find region for fault %" PRIx64,
1613                  __func__, faultaddr);
1614     return -1;
1615 }
1616
1617 static int vhost_user_postcopy_waker(struct PostCopyFD *pcfd, RAMBlock *rb,
1618                                      uint64_t offset)
1619 {
1620     struct vhost_dev *dev = pcfd->data;
1621     struct vhost_user *u = dev->opaque;
1622     int i;
1623
1624     trace_vhost_user_postcopy_waker(qemu_ram_get_idstr(rb), offset);
1625
1626     if (!u) {
1627         return 0;
1628     }
1629     /* Translate the offset into an address in the clients address space */
1630     for (i = 0; i < MIN(dev->mem->nregions, u->region_rb_len); i++) {
1631         if (u->region_rb[i] == rb &&
1632             offset >= u->region_rb_offset[i] &&
1633             offset < (u->region_rb_offset[i] +
1634                       dev->mem->regions[i].memory_size)) {
1635             uint64_t client_addr = (offset - u->region_rb_offset[i]) +
1636                                    u->postcopy_client_bases[i];
1637             trace_vhost_user_postcopy_waker_found(client_addr);
1638             return postcopy_wake_shared(pcfd, client_addr, rb);
1639         }
1640     }
1641
1642     trace_vhost_user_postcopy_waker_nomatch(qemu_ram_get_idstr(rb), offset);
1643     return 0;
1644 }
1645 #endif
1646
1647 /*
1648  * Called at the start of an inbound postcopy on reception of the
1649  * 'advise' command.
1650  */
1651 static int vhost_user_postcopy_advise(struct vhost_dev *dev, Error **errp)
1652 {
1653 #ifdef CONFIG_LINUX
1654     struct vhost_user *u = dev->opaque;
1655     CharBackend *chr = u->user->chr;
1656     int ufd;
1657     VhostUserMsg msg = {
1658         .hdr.request = VHOST_USER_POSTCOPY_ADVISE,
1659         .hdr.flags = VHOST_USER_VERSION,
1660     };
1661
1662     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1663         error_setg(errp, "Failed to send postcopy_advise to vhost");
1664         return -1;
1665     }
1666
1667     if (vhost_user_read(dev, &msg) < 0) {
1668         error_setg(errp, "Failed to get postcopy_advise reply from vhost");
1669         return -1;
1670     }
1671
1672     if (msg.hdr.request != VHOST_USER_POSTCOPY_ADVISE) {
1673         error_setg(errp, "Unexpected msg type. Expected %d received %d",
1674                      VHOST_USER_POSTCOPY_ADVISE, msg.hdr.request);
1675         return -1;
1676     }
1677
1678     if (msg.hdr.size) {
1679         error_setg(errp, "Received bad msg size.");
1680         return -1;
1681     }
1682     ufd = qemu_chr_fe_get_msgfd(chr);
1683     if (ufd < 0) {
1684         error_setg(errp, "%s: Failed to get ufd", __func__);
1685         return -1;
1686     }
1687     qemu_set_nonblock(ufd);
1688
1689     /* register ufd with userfault thread */
1690     u->postcopy_fd.fd = ufd;
1691     u->postcopy_fd.data = dev;
1692     u->postcopy_fd.handler = vhost_user_postcopy_fault_handler;
1693     u->postcopy_fd.waker = vhost_user_postcopy_waker;
1694     u->postcopy_fd.idstr = "vhost-user"; /* Need to find unique name */
1695     postcopy_register_shared_ufd(&u->postcopy_fd);
1696     return 0;
1697 #else
1698     error_setg(errp, "Postcopy not supported on non-Linux systems");
1699     return -1;
1700 #endif
1701 }
1702
1703 /*
1704  * Called at the switch to postcopy on reception of the 'listen' command.
1705  */
1706 static int vhost_user_postcopy_listen(struct vhost_dev *dev, Error **errp)
1707 {
1708     struct vhost_user *u = dev->opaque;
1709     int ret;
1710     VhostUserMsg msg = {
1711         .hdr.request = VHOST_USER_POSTCOPY_LISTEN,
1712         .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
1713     };
1714     u->postcopy_listen = true;
1715     trace_vhost_user_postcopy_listen();
1716     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1717         error_setg(errp, "Failed to send postcopy_listen to vhost");
1718         return -1;
1719     }
1720
1721     ret = process_message_reply(dev, &msg);
1722     if (ret) {
1723         error_setg(errp, "Failed to receive reply to postcopy_listen");
1724         return ret;
1725     }
1726
1727     return 0;
1728 }
1729
1730 /*
1731  * Called at the end of postcopy
1732  */
1733 static int vhost_user_postcopy_end(struct vhost_dev *dev, Error **errp)
1734 {
1735     VhostUserMsg msg = {
1736         .hdr.request = VHOST_USER_POSTCOPY_END,
1737         .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
1738     };
1739     int ret;
1740     struct vhost_user *u = dev->opaque;
1741
1742     trace_vhost_user_postcopy_end_entry();
1743     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1744         error_setg(errp, "Failed to send postcopy_end to vhost");
1745         return -1;
1746     }
1747
1748     ret = process_message_reply(dev, &msg);
1749     if (ret) {
1750         error_setg(errp, "Failed to receive reply to postcopy_end");
1751         return ret;
1752     }
1753     postcopy_unregister_shared_ufd(&u->postcopy_fd);
1754     close(u->postcopy_fd.fd);
1755     u->postcopy_fd.handler = NULL;
1756
1757     trace_vhost_user_postcopy_end_exit();
1758
1759     return 0;
1760 }
1761
1762 static int vhost_user_postcopy_notifier(NotifierWithReturn *notifier,
1763                                         void *opaque)
1764 {
1765     struct PostcopyNotifyData *pnd = opaque;
1766     struct vhost_user *u = container_of(notifier, struct vhost_user,
1767                                          postcopy_notifier);
1768     struct vhost_dev *dev = u->dev;
1769
1770     switch (pnd->reason) {
1771     case POSTCOPY_NOTIFY_PROBE:
1772         if (!virtio_has_feature(dev->protocol_features,
1773                                 VHOST_USER_PROTOCOL_F_PAGEFAULT)) {
1774             /* TODO: Get the device name into this error somehow */
1775             error_setg(pnd->errp,
1776                        "vhost-user backend not capable of postcopy");
1777             return -ENOENT;
1778         }
1779         break;
1780
1781     case POSTCOPY_NOTIFY_INBOUND_ADVISE:
1782         return vhost_user_postcopy_advise(dev, pnd->errp);
1783
1784     case POSTCOPY_NOTIFY_INBOUND_LISTEN:
1785         return vhost_user_postcopy_listen(dev, pnd->errp);
1786
1787     case POSTCOPY_NOTIFY_INBOUND_END:
1788         return vhost_user_postcopy_end(dev, pnd->errp);
1789
1790     default:
1791         /* We ignore notifications we don't know */
1792         break;
1793     }
1794
1795     return 0;
1796 }
1797
1798 static int vhost_user_backend_init(struct vhost_dev *dev, void *opaque)
1799 {
1800     uint64_t features, protocol_features, ram_slots;
1801     struct vhost_user *u;
1802     int err;
1803
1804     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
1805
1806     u = g_new0(struct vhost_user, 1);
1807     u->user = opaque;
1808     u->slave_fd = -1;
1809     u->dev = dev;
1810     dev->opaque = u;
1811
1812     err = vhost_user_get_features(dev, &features);
1813     if (err < 0) {
1814         return err;
1815     }
1816
1817     if (virtio_has_feature(features, VHOST_USER_F_PROTOCOL_FEATURES)) {
1818         dev->backend_features |= 1ULL << VHOST_USER_F_PROTOCOL_FEATURES;
1819
1820         err = vhost_user_get_u64(dev, VHOST_USER_GET_PROTOCOL_FEATURES,
1821                                  &protocol_features);
1822         if (err < 0) {
1823             return err;
1824         }
1825
1826         dev->protocol_features =
1827             protocol_features & VHOST_USER_PROTOCOL_FEATURE_MASK;
1828
1829         if (!dev->config_ops || !dev->config_ops->vhost_dev_config_notifier) {
1830             /* Don't acknowledge CONFIG feature if device doesn't support it */
1831             dev->protocol_features &= ~(1ULL << VHOST_USER_PROTOCOL_F_CONFIG);
1832         } else if (!(protocol_features &
1833                     (1ULL << VHOST_USER_PROTOCOL_F_CONFIG))) {
1834             error_report("Device expects VHOST_USER_PROTOCOL_F_CONFIG "
1835                     "but backend does not support it.");
1836             return -1;
1837         }
1838
1839         err = vhost_user_set_protocol_features(dev, dev->protocol_features);
1840         if (err < 0) {
1841             return err;
1842         }
1843
1844         /* query the max queues we support if backend supports Multiple Queue */
1845         if (dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_MQ)) {
1846             err = vhost_user_get_u64(dev, VHOST_USER_GET_QUEUE_NUM,
1847                                      &dev->max_queues);
1848             if (err < 0) {
1849                 return err;
1850             }
1851         }
1852
1853         if (virtio_has_feature(features, VIRTIO_F_IOMMU_PLATFORM) &&
1854                 !(virtio_has_feature(dev->protocol_features,
1855                     VHOST_USER_PROTOCOL_F_SLAVE_REQ) &&
1856                  virtio_has_feature(dev->protocol_features,
1857                     VHOST_USER_PROTOCOL_F_REPLY_ACK))) {
1858             error_report("IOMMU support requires reply-ack and "
1859                          "slave-req protocol features.");
1860             return -1;
1861         }
1862
1863         /* get max memory regions if backend supports configurable RAM slots */
1864         if (!virtio_has_feature(dev->protocol_features,
1865                                 VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS)) {
1866             u->user->memory_slots = VHOST_MEMORY_BASELINE_NREGIONS;
1867         } else {
1868             err = vhost_user_get_max_memslots(dev, &ram_slots);
1869             if (err < 0) {
1870                 return err;
1871             }
1872
1873             if (ram_slots < u->user->memory_slots) {
1874                 error_report("The backend specified a max ram slots limit "
1875                              "of %" PRIu64", when the prior validated limit was %d. "
1876                              "This limit should never decrease.", ram_slots,
1877                              u->user->memory_slots);
1878                 return -1;
1879             }
1880
1881             u->user->memory_slots = MIN(ram_slots, VHOST_USER_MAX_RAM_SLOTS);
1882         }
1883     }
1884
1885     if (dev->migration_blocker == NULL &&
1886         !virtio_has_feature(dev->protocol_features,
1887                             VHOST_USER_PROTOCOL_F_LOG_SHMFD)) {
1888         error_setg(&dev->migration_blocker,
1889                    "Migration disabled: vhost-user backend lacks "
1890                    "VHOST_USER_PROTOCOL_F_LOG_SHMFD feature.");
1891     }
1892
1893     if (dev->vq_index == 0) {
1894         err = vhost_setup_slave_channel(dev);
1895         if (err < 0) {
1896             return err;
1897         }
1898     }
1899
1900     u->postcopy_notifier.notify = vhost_user_postcopy_notifier;
1901     postcopy_add_notifier(&u->postcopy_notifier);
1902
1903     return 0;
1904 }
1905
1906 static int vhost_user_backend_cleanup(struct vhost_dev *dev)
1907 {
1908     struct vhost_user *u;
1909
1910     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
1911
1912     u = dev->opaque;
1913     if (u->postcopy_notifier.notify) {
1914         postcopy_remove_notifier(&u->postcopy_notifier);
1915         u->postcopy_notifier.notify = NULL;
1916     }
1917     u->postcopy_listen = false;
1918     if (u->postcopy_fd.handler) {
1919         postcopy_unregister_shared_ufd(&u->postcopy_fd);
1920         close(u->postcopy_fd.fd);
1921         u->postcopy_fd.handler = NULL;
1922     }
1923     if (u->slave_fd >= 0) {
1924         qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL);
1925         close(u->slave_fd);
1926         u->slave_fd = -1;
1927     }
1928     g_free(u->region_rb);
1929     u->region_rb = NULL;
1930     g_free(u->region_rb_offset);
1931     u->region_rb_offset = NULL;
1932     u->region_rb_len = 0;
1933     g_free(u);
1934     dev->opaque = 0;
1935
1936     return 0;
1937 }
1938
1939 static int vhost_user_get_vq_index(struct vhost_dev *dev, int idx)
1940 {
1941     assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
1942
1943     return idx;
1944 }
1945
1946 static int vhost_user_memslots_limit(struct vhost_dev *dev)
1947 {
1948     struct vhost_user *u = dev->opaque;
1949
1950     return u->user->memory_slots;
1951 }
1952
1953 static bool vhost_user_requires_shm_log(struct vhost_dev *dev)
1954 {
1955     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
1956
1957     return virtio_has_feature(dev->protocol_features,
1958                               VHOST_USER_PROTOCOL_F_LOG_SHMFD);
1959 }
1960
1961 static int vhost_user_migration_done(struct vhost_dev *dev, char* mac_addr)
1962 {
1963     VhostUserMsg msg = { };
1964
1965     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
1966
1967     /* If guest supports GUEST_ANNOUNCE do nothing */
1968     if (virtio_has_feature(dev->acked_features, VIRTIO_NET_F_GUEST_ANNOUNCE)) {
1969         return 0;
1970     }
1971
1972     /* if backend supports VHOST_USER_PROTOCOL_F_RARP ask it to send the RARP */
1973     if (virtio_has_feature(dev->protocol_features,
1974                            VHOST_USER_PROTOCOL_F_RARP)) {
1975         msg.hdr.request = VHOST_USER_SEND_RARP;
1976         msg.hdr.flags = VHOST_USER_VERSION;
1977         memcpy((char *)&msg.payload.u64, mac_addr, 6);
1978         msg.hdr.size = sizeof(msg.payload.u64);
1979
1980         return vhost_user_write(dev, &msg, NULL, 0);
1981     }
1982     return -1;
1983 }
1984
1985 static bool vhost_user_can_merge(struct vhost_dev *dev,
1986                                  uint64_t start1, uint64_t size1,
1987                                  uint64_t start2, uint64_t size2)
1988 {
1989     ram_addr_t offset;
1990     int mfd, rfd;
1991
1992     (void)vhost_user_get_mr_data(start1, &offset, &mfd);
1993     (void)vhost_user_get_mr_data(start2, &offset, &rfd);
1994
1995     return mfd == rfd;
1996 }
1997
1998 static int vhost_user_net_set_mtu(struct vhost_dev *dev, uint16_t mtu)
1999 {
2000     VhostUserMsg msg;
2001     bool reply_supported = virtio_has_feature(dev->protocol_features,
2002                                               VHOST_USER_PROTOCOL_F_REPLY_ACK);
2003
2004     if (!(dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_NET_MTU))) {
2005         return 0;
2006     }
2007
2008     msg.hdr.request = VHOST_USER_NET_SET_MTU;
2009     msg.payload.u64 = mtu;
2010     msg.hdr.size = sizeof(msg.payload.u64);
2011     msg.hdr.flags = VHOST_USER_VERSION;
2012     if (reply_supported) {
2013         msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
2014     }
2015
2016     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
2017         return -1;
2018     }
2019
2020     /* If reply_ack supported, slave has to ack specified MTU is valid */
2021     if (reply_supported) {
2022         return process_message_reply(dev, &msg);
2023     }
2024
2025     return 0;
2026 }
2027
2028 static int vhost_user_send_device_iotlb_msg(struct vhost_dev *dev,
2029                                             struct vhost_iotlb_msg *imsg)
2030 {
2031     VhostUserMsg msg = {
2032         .hdr.request = VHOST_USER_IOTLB_MSG,
2033         .hdr.size = sizeof(msg.payload.iotlb),
2034         .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
2035         .payload.iotlb = *imsg,
2036     };
2037
2038     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
2039         return -EFAULT;
2040     }
2041
2042     return process_message_reply(dev, &msg);
2043 }
2044
2045
2046 static void vhost_user_set_iotlb_callback(struct vhost_dev *dev, int enabled)
2047 {
2048     /* No-op as the receive channel is not dedicated to IOTLB messages. */
2049 }
2050
2051 static int vhost_user_get_config(struct vhost_dev *dev, uint8_t *config,
2052                                  uint32_t config_len)
2053 {
2054     VhostUserMsg msg = {
2055         .hdr.request = VHOST_USER_GET_CONFIG,
2056         .hdr.flags = VHOST_USER_VERSION,
2057         .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + config_len,
2058     };
2059
2060     if (!virtio_has_feature(dev->protocol_features,
2061                 VHOST_USER_PROTOCOL_F_CONFIG)) {
2062         return -1;
2063     }
2064
2065     if (config_len > VHOST_USER_MAX_CONFIG_SIZE) {
2066         return -1;
2067     }
2068
2069     msg.payload.config.offset = 0;
2070     msg.payload.config.size = config_len;
2071     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
2072         return -1;
2073     }
2074
2075     if (vhost_user_read(dev, &msg) < 0) {
2076         return -1;
2077     }
2078
2079     if (msg.hdr.request != VHOST_USER_GET_CONFIG) {
2080         error_report("Received unexpected msg type. Expected %d received %d",
2081                      VHOST_USER_GET_CONFIG, msg.hdr.request);
2082         return -1;
2083     }
2084
2085     if (msg.hdr.size != VHOST_USER_CONFIG_HDR_SIZE + config_len) {
2086         error_report("Received bad msg size.");
2087         return -1;
2088     }
2089
2090     memcpy(config, msg.payload.config.region, config_len);
2091
2092     return 0;
2093 }
2094
2095 static int vhost_user_set_config(struct vhost_dev *dev, const uint8_t *data,
2096                                  uint32_t offset, uint32_t size, uint32_t flags)
2097 {
2098     uint8_t *p;
2099     bool reply_supported = virtio_has_feature(dev->protocol_features,
2100                                               VHOST_USER_PROTOCOL_F_REPLY_ACK);
2101
2102     VhostUserMsg msg = {
2103         .hdr.request = VHOST_USER_SET_CONFIG,
2104         .hdr.flags = VHOST_USER_VERSION,
2105         .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + size,
2106     };
2107
2108     if (!virtio_has_feature(dev->protocol_features,
2109                 VHOST_USER_PROTOCOL_F_CONFIG)) {
2110         return -1;
2111     }
2112
2113     if (reply_supported) {
2114         msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
2115     }
2116
2117     if (size > VHOST_USER_MAX_CONFIG_SIZE) {
2118         return -1;
2119     }
2120
2121     msg.payload.config.offset = offset,
2122     msg.payload.config.size = size,
2123     msg.payload.config.flags = flags,
2124     p = msg.payload.config.region;
2125     memcpy(p, data, size);
2126
2127     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
2128         return -1;
2129     }
2130
2131     if (reply_supported) {
2132         return process_message_reply(dev, &msg);
2133     }
2134
2135     return 0;
2136 }
2137
2138 static int vhost_user_crypto_create_session(struct vhost_dev *dev,
2139                                             void *session_info,
2140                                             uint64_t *session_id)
2141 {
2142     bool crypto_session = virtio_has_feature(dev->protocol_features,
2143                                        VHOST_USER_PROTOCOL_F_CRYPTO_SESSION);
2144     CryptoDevBackendSymSessionInfo *sess_info = session_info;
2145     VhostUserMsg msg = {
2146         .hdr.request = VHOST_USER_CREATE_CRYPTO_SESSION,
2147         .hdr.flags = VHOST_USER_VERSION,
2148         .hdr.size = sizeof(msg.payload.session),
2149     };
2150
2151     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
2152
2153     if (!crypto_session) {
2154         error_report("vhost-user trying to send unhandled ioctl");
2155         return -1;
2156     }
2157
2158     memcpy(&msg.payload.session.session_setup_data, sess_info,
2159               sizeof(CryptoDevBackendSymSessionInfo));
2160     if (sess_info->key_len) {
2161         memcpy(&msg.payload.session.key, sess_info->cipher_key,
2162                sess_info->key_len);
2163     }
2164     if (sess_info->auth_key_len > 0) {
2165         memcpy(&msg.payload.session.auth_key, sess_info->auth_key,
2166                sess_info->auth_key_len);
2167     }
2168     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
2169         error_report("vhost_user_write() return -1, create session failed");
2170         return -1;
2171     }
2172
2173     if (vhost_user_read(dev, &msg) < 0) {
2174         error_report("vhost_user_read() return -1, create session failed");
2175         return -1;
2176     }
2177
2178     if (msg.hdr.request != VHOST_USER_CREATE_CRYPTO_SESSION) {
2179         error_report("Received unexpected msg type. Expected %d received %d",
2180                      VHOST_USER_CREATE_CRYPTO_SESSION, msg.hdr.request);
2181         return -1;
2182     }
2183
2184     if (msg.hdr.size != sizeof(msg.payload.session)) {
2185         error_report("Received bad msg size.");
2186         return -1;
2187     }
2188
2189     if (msg.payload.session.session_id < 0) {
2190         error_report("Bad session id: %" PRId64 "",
2191                               msg.payload.session.session_id);
2192         return -1;
2193     }
2194     *session_id = msg.payload.session.session_id;
2195
2196     return 0;
2197 }
2198
2199 static int
2200 vhost_user_crypto_close_session(struct vhost_dev *dev, uint64_t session_id)
2201 {
2202     bool crypto_session = virtio_has_feature(dev->protocol_features,
2203                                        VHOST_USER_PROTOCOL_F_CRYPTO_SESSION);
2204     VhostUserMsg msg = {
2205         .hdr.request = VHOST_USER_CLOSE_CRYPTO_SESSION,
2206         .hdr.flags = VHOST_USER_VERSION,
2207         .hdr.size = sizeof(msg.payload.u64),
2208     };
2209     msg.payload.u64 = session_id;
2210
2211     if (!crypto_session) {
2212         error_report("vhost-user trying to send unhandled ioctl");
2213         return -1;
2214     }
2215
2216     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
2217         error_report("vhost_user_write() return -1, close session failed");
2218         return -1;
2219     }
2220
2221     return 0;
2222 }
2223
2224 static bool vhost_user_mem_section_filter(struct vhost_dev *dev,
2225                                           MemoryRegionSection *section)
2226 {
2227     bool result;
2228
2229     result = memory_region_get_fd(section->mr) >= 0;
2230
2231     return result;
2232 }
2233
2234 static int vhost_user_get_inflight_fd(struct vhost_dev *dev,
2235                                       uint16_t queue_size,
2236                                       struct vhost_inflight *inflight)
2237 {
2238     void *addr;
2239     int fd;
2240     struct vhost_user *u = dev->opaque;
2241     CharBackend *chr = u->user->chr;
2242     VhostUserMsg msg = {
2243         .hdr.request = VHOST_USER_GET_INFLIGHT_FD,
2244         .hdr.flags = VHOST_USER_VERSION,
2245         .payload.inflight.num_queues = dev->nvqs,
2246         .payload.inflight.queue_size = queue_size,
2247         .hdr.size = sizeof(msg.payload.inflight),
2248     };
2249
2250     if (!virtio_has_feature(dev->protocol_features,
2251                             VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
2252         return 0;
2253     }
2254
2255     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
2256         return -1;
2257     }
2258
2259     if (vhost_user_read(dev, &msg) < 0) {
2260         return -1;
2261     }
2262
2263     if (msg.hdr.request != VHOST_USER_GET_INFLIGHT_FD) {
2264         error_report("Received unexpected msg type. "
2265                      "Expected %d received %d",
2266                      VHOST_USER_GET_INFLIGHT_FD, msg.hdr.request);
2267         return -1;
2268     }
2269
2270     if (msg.hdr.size != sizeof(msg.payload.inflight)) {
2271         error_report("Received bad msg size.");
2272         return -1;
2273     }
2274
2275     if (!msg.payload.inflight.mmap_size) {
2276         return 0;
2277     }
2278
2279     fd = qemu_chr_fe_get_msgfd(chr);
2280     if (fd < 0) {
2281         error_report("Failed to get mem fd");
2282         return -1;
2283     }
2284
2285     addr = mmap(0, msg.payload.inflight.mmap_size, PROT_READ | PROT_WRITE,
2286                 MAP_SHARED, fd, msg.payload.inflight.mmap_offset);
2287
2288     if (addr == MAP_FAILED) {
2289         error_report("Failed to mmap mem fd");
2290         close(fd);
2291         return -1;
2292     }
2293
2294     inflight->addr = addr;
2295     inflight->fd = fd;
2296     inflight->size = msg.payload.inflight.mmap_size;
2297     inflight->offset = msg.payload.inflight.mmap_offset;
2298     inflight->queue_size = queue_size;
2299
2300     return 0;
2301 }
2302
2303 static int vhost_user_set_inflight_fd(struct vhost_dev *dev,
2304                                       struct vhost_inflight *inflight)
2305 {
2306     VhostUserMsg msg = {
2307         .hdr.request = VHOST_USER_SET_INFLIGHT_FD,
2308         .hdr.flags = VHOST_USER_VERSION,
2309         .payload.inflight.mmap_size = inflight->size,
2310         .payload.inflight.mmap_offset = inflight->offset,
2311         .payload.inflight.num_queues = dev->nvqs,
2312         .payload.inflight.queue_size = inflight->queue_size,
2313         .hdr.size = sizeof(msg.payload.inflight),
2314     };
2315
2316     if (!virtio_has_feature(dev->protocol_features,
2317                             VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
2318         return 0;
2319     }
2320
2321     if (vhost_user_write(dev, &msg, &inflight->fd, 1) < 0) {
2322         return -1;
2323     }
2324
2325     return 0;
2326 }
2327
2328 bool vhost_user_init(VhostUserState *user, CharBackend *chr, Error **errp)
2329 {
2330     if (user->chr) {
2331         error_setg(errp, "Cannot initialize vhost-user state");
2332         return false;
2333     }
2334     user->chr = chr;
2335     user->memory_slots = 0;
2336     return true;
2337 }
2338
2339 void vhost_user_cleanup(VhostUserState *user)
2340 {
2341     int i;
2342
2343     if (!user->chr) {
2344         return;
2345     }
2346
2347     for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2348         if (user->notifier[i].addr) {
2349             object_unparent(OBJECT(&user->notifier[i].mr));
2350             munmap(user->notifier[i].addr, qemu_real_host_page_size);
2351             user->notifier[i].addr = NULL;
2352         }
2353     }
2354     user->chr = NULL;
2355 }
2356
2357 const VhostOps user_ops = {
2358         .backend_type = VHOST_BACKEND_TYPE_USER,
2359         .vhost_backend_init = vhost_user_backend_init,
2360         .vhost_backend_cleanup = vhost_user_backend_cleanup,
2361         .vhost_backend_memslots_limit = vhost_user_memslots_limit,
2362         .vhost_set_log_base = vhost_user_set_log_base,
2363         .vhost_set_mem_table = vhost_user_set_mem_table,
2364         .vhost_set_vring_addr = vhost_user_set_vring_addr,
2365         .vhost_set_vring_endian = vhost_user_set_vring_endian,
2366         .vhost_set_vring_num = vhost_user_set_vring_num,
2367         .vhost_set_vring_base = vhost_user_set_vring_base,
2368         .vhost_get_vring_base = vhost_user_get_vring_base,
2369         .vhost_set_vring_kick = vhost_user_set_vring_kick,
2370         .vhost_set_vring_call = vhost_user_set_vring_call,
2371         .vhost_set_features = vhost_user_set_features,
2372         .vhost_get_features = vhost_user_get_features,
2373         .vhost_set_owner = vhost_user_set_owner,
2374         .vhost_reset_device = vhost_user_reset_device,
2375         .vhost_get_vq_index = vhost_user_get_vq_index,
2376         .vhost_set_vring_enable = vhost_user_set_vring_enable,
2377         .vhost_requires_shm_log = vhost_user_requires_shm_log,
2378         .vhost_migration_done = vhost_user_migration_done,
2379         .vhost_backend_can_merge = vhost_user_can_merge,
2380         .vhost_net_set_mtu = vhost_user_net_set_mtu,
2381         .vhost_set_iotlb_callback = vhost_user_set_iotlb_callback,
2382         .vhost_send_device_iotlb_msg = vhost_user_send_device_iotlb_msg,
2383         .vhost_get_config = vhost_user_get_config,
2384         .vhost_set_config = vhost_user_set_config,
2385         .vhost_crypto_create_session = vhost_user_crypto_create_session,
2386         .vhost_crypto_close_session = vhost_user_crypto_close_session,
2387         .vhost_backend_mem_section_filter = vhost_user_mem_section_filter,
2388         .vhost_get_inflight_fd = vhost_user_get_inflight_fd,
2389         .vhost_set_inflight_fd = vhost_user_set_inflight_fd,
2390 };