hw/virtio/vhost-user.c

   1 /*
   2  * vhost-user
   3  *
   4  * Copyright (c) 2013 Virtual Open Systems Sarl.
   5  *
   6  * This work is licensed under the terms of the GNU GPL, version 2 or later.
   7  * See the COPYING file in the top-level directory.
   8  *
   9  */
  10
  11 #include "qemu/osdep.h"
  12 #include "qapi/error.h"
  13 #include "hw/virtio/vhost.h"
  14 #include "hw/virtio/vhost-backend.h"
  15 #include "hw/virtio/virtio-net.h"
  16 #include "chardev/char-fe.h"
  17 #include "sysemu/kvm.h"
  18 #include "qemu/error-report.h"
  19 #include "qemu/sockets.h"
  20 #include "sysemu/cryptodev.h"
  21 #include "migration/migration.h"
  22 #include "migration/postcopy-ram.h"
  23 #include "trace.h"
  24
  25 #include <sys/ioctl.h>
  26 #include <sys/socket.h>
  27 #include <sys/un.h>
  28 #include <linux/vhost.h>
  29 #include <linux/userfaultfd.h>
  30
  31 #define VHOST_MEMORY_MAX_NREGIONS    8
  32 #define VHOST_USER_F_PROTOCOL_FEATURES 30
  33
  34 /*
  35  * Maximum size of virtio device config space
  36  */
  37 #define VHOST_USER_MAX_CONFIG_SIZE 256
  38
  39 enum VhostUserProtocolFeature {
  40     VHOST_USER_PROTOCOL_F_MQ = 0,
  41     VHOST_USER_PROTOCOL_F_LOG_SHMFD = 1,
  42     VHOST_USER_PROTOCOL_F_RARP = 2,
  43     VHOST_USER_PROTOCOL_F_REPLY_ACK = 3,
  44     VHOST_USER_PROTOCOL_F_NET_MTU = 4,
  45     VHOST_USER_PROTOCOL_F_SLAVE_REQ = 5,
  46     VHOST_USER_PROTOCOL_F_CROSS_ENDIAN = 6,
  47     VHOST_USER_PROTOCOL_F_CRYPTO_SESSION = 7,
  48     VHOST_USER_PROTOCOL_F_PAGEFAULT = 8,
  49     VHOST_USER_PROTOCOL_F_MAX
  50 };
  51
  52 #define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1)
  53
  54 typedef enum VhostUserRequest {
  55     VHOST_USER_NONE = 0,
  56     VHOST_USER_GET_FEATURES = 1,
  57     VHOST_USER_SET_FEATURES = 2,
  58     VHOST_USER_SET_OWNER = 3,
  59     VHOST_USER_RESET_OWNER = 4,
  60     VHOST_USER_SET_MEM_TABLE = 5,
  61     VHOST_USER_SET_LOG_BASE = 6,
  62     VHOST_USER_SET_LOG_FD = 7,
  63     VHOST_USER_SET_VRING_NUM = 8,
  64     VHOST_USER_SET_VRING_ADDR = 9,
  65     VHOST_USER_SET_VRING_BASE = 10,
  66     VHOST_USER_GET_VRING_BASE = 11,
  67     VHOST_USER_SET_VRING_KICK = 12,
  68     VHOST_USER_SET_VRING_CALL = 13,
  69     VHOST_USER_SET_VRING_ERR = 14,
  70     VHOST_USER_GET_PROTOCOL_FEATURES = 15,
  71     VHOST_USER_SET_PROTOCOL_FEATURES = 16,
  72     VHOST_USER_GET_QUEUE_NUM = 17,
  73     VHOST_USER_SET_VRING_ENABLE = 18,
  74     VHOST_USER_SEND_RARP = 19,
  75     VHOST_USER_NET_SET_MTU = 20,
  76     VHOST_USER_SET_SLAVE_REQ_FD = 21,
  77     VHOST_USER_IOTLB_MSG = 22,
  78     VHOST_USER_SET_VRING_ENDIAN = 23,
  79     VHOST_USER_GET_CONFIG = 24,
  80     VHOST_USER_SET_CONFIG = 25,
  81     VHOST_USER_CREATE_CRYPTO_SESSION = 26,
  82     VHOST_USER_CLOSE_CRYPTO_SESSION = 27,
  83     VHOST_USER_POSTCOPY_ADVISE  = 28,
  84     VHOST_USER_POSTCOPY_LISTEN  = 29,
  85     VHOST_USER_POSTCOPY_END     = 30,
  86     VHOST_USER_MAX
  87 } VhostUserRequest;
  88
  89 typedef enum VhostUserSlaveRequest {
  90     VHOST_USER_SLAVE_NONE = 0,
  91     VHOST_USER_SLAVE_IOTLB_MSG = 1,
  92     VHOST_USER_SLAVE_CONFIG_CHANGE_MSG = 2,
  93     VHOST_USER_SLAVE_MAX
  94 }  VhostUserSlaveRequest;
  95
  96 typedef struct VhostUserMemoryRegion {
  97     uint64_t guest_phys_addr;
  98     uint64_t memory_size;
  99     uint64_t userspace_addr;
 100     uint64_t mmap_offset;
 101 } VhostUserMemoryRegion;
 102
 103 typedef struct VhostUserMemory {
 104     uint32_t nregions;
 105     uint32_t padding;
 106     VhostUserMemoryRegion regions[VHOST_MEMORY_MAX_NREGIONS];
 107 } VhostUserMemory;
 108
 109 typedef struct VhostUserLog {
 110     uint64_t mmap_size;
 111     uint64_t mmap_offset;
 112 } VhostUserLog;
 113
 114 typedef struct VhostUserConfig {
 115     uint32_t offset;
 116     uint32_t size;
 117     uint32_t flags;
 118     uint8_t region[VHOST_USER_MAX_CONFIG_SIZE];
 119 } VhostUserConfig;
 120
 121 #define VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN    512
 122 #define VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN  64
 123
 124 typedef struct VhostUserCryptoSession {
 125     /* session id for success, -1 on errors */
 126     int64_t session_id;
 127     CryptoDevBackendSymSessionInfo session_setup_data;
 128     uint8_t key[VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN];
 129     uint8_t auth_key[VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN];
 130 } VhostUserCryptoSession;
 131
 132 static VhostUserConfig c __attribute__ ((unused));
 133 #define VHOST_USER_CONFIG_HDR_SIZE (sizeof(c.offset) \
 134                                    + sizeof(c.size) \
 135                                    + sizeof(c.flags))
 136
 137 typedef struct {
 138     VhostUserRequest request;
 139
 140 #define VHOST_USER_VERSION_MASK     (0x3)
 141 #define VHOST_USER_REPLY_MASK       (0x1<<2)
 142 #define VHOST_USER_NEED_REPLY_MASK  (0x1 << 3)
 143     uint32_t flags;
 144     uint32_t size; /* the following payload size */
 145 } QEMU_PACKED VhostUserHeader;
 146
 147 typedef union {
 148 #define VHOST_USER_VRING_IDX_MASK   (0xff)
 149 #define VHOST_USER_VRING_NOFD_MASK  (0x1<<8)
 150         uint64_t u64;
 151         struct vhost_vring_state state;
 152         struct vhost_vring_addr addr;
 153         VhostUserMemory memory;
 154         VhostUserLog log;
 155         struct vhost_iotlb_msg iotlb;
 156         VhostUserConfig config;
 157         VhostUserCryptoSession session;
 158 } VhostUserPayload;
 159
 160 typedef struct VhostUserMsg {
 161     VhostUserHeader hdr;
 162     VhostUserPayload payload;
 163 } QEMU_PACKED VhostUserMsg;
 164
 165 static VhostUserMsg m __attribute__ ((unused));
 166 #define VHOST_USER_HDR_SIZE (sizeof(VhostUserHeader))
 167
 168 #define VHOST_USER_PAYLOAD_SIZE (sizeof(VhostUserPayload))
 169
 170 /* The version of the protocol we support */
 171 #define VHOST_USER_VERSION    (0x1)
 172
 173 struct vhost_user {
 174     struct vhost_dev *dev;
 175     CharBackend *chr;
 176     int slave_fd;
 177     NotifierWithReturn postcopy_notifier;
 178     struct PostCopyFD  postcopy_fd;
 179     uint64_t           postcopy_client_bases[VHOST_MEMORY_MAX_NREGIONS];
 180     /* Length of the region_rb and region_rb_offset arrays */
 181     size_t             region_rb_len;
 182     /* RAMBlock associated with a given region */
 183     RAMBlock         **region_rb;
 184     /* The offset from the start of the RAMBlock to the start of the
 185      * vhost region.
 186      */
 187     ram_addr_t        *region_rb_offset;
 188
 189     /* True once we've entered postcopy_listen */
 190     bool               postcopy_listen;
 191 };
 192
 193 static bool ioeventfd_enabled(void)
 194 {
 195     return kvm_enabled() && kvm_eventfds_enabled();
 196 }
 197
 198 static int vhost_user_read(struct vhost_dev *dev, VhostUserMsg *msg)
 199 {
 200     struct vhost_user *u = dev->opaque;
 201     CharBackend *chr = u->chr;
 202     uint8_t *p = (uint8_t *) msg;
 203     int r, size = VHOST_USER_HDR_SIZE;
 204
 205     r = qemu_chr_fe_read_all(chr, p, size);
 206     if (r != size) {
 207         error_report("Failed to read msg header. Read %d instead of %d."
 208                      " Original request %d.", r, size, msg->hdr.request);
 209         goto fail;
 210     }
 211
 212     /* validate received flags */
 213     if (msg->hdr.flags != (VHOST_USER_REPLY_MASK | VHOST_USER_VERSION)) {
 214         error_report("Failed to read msg header."
 215                 " Flags 0x%x instead of 0x%x.", msg->hdr.flags,
 216                 VHOST_USER_REPLY_MASK | VHOST_USER_VERSION);
 217         goto fail;
 218     }
 219
 220     /* validate message size is sane */
 221     if (msg->hdr.size > VHOST_USER_PAYLOAD_SIZE) {
 222         error_report("Failed to read msg header."
 223                 " Size %d exceeds the maximum %zu.", msg->hdr.size,
 224                 VHOST_USER_PAYLOAD_SIZE);
 225         goto fail;
 226     }
 227
 228     if (msg->hdr.size) {
 229         p += VHOST_USER_HDR_SIZE;
 230         size = msg->hdr.size;
 231         r = qemu_chr_fe_read_all(chr, p, size);
 232         if (r != size) {
 233             error_report("Failed to read msg payload."
 234                          " Read %d instead of %d.", r, msg->hdr.size);
 235             goto fail;
 236         }
 237     }
 238
 239     return 0;
 240
 241 fail:
 242     return -1;
 243 }
 244
 245 static int process_message_reply(struct vhost_dev *dev,
 246                                  const VhostUserMsg *msg)
 247 {
 248     VhostUserMsg msg_reply;
 249
 250     if ((msg->hdr.flags & VHOST_USER_NEED_REPLY_MASK) == 0) {
 251         return 0;
 252     }
 253
 254     if (vhost_user_read(dev, &msg_reply) < 0) {
 255         return -1;
 256     }
 257
 258     if (msg_reply.hdr.request != msg->hdr.request) {
 259         error_report("Received unexpected msg type."
 260                      "Expected %d received %d",
 261                      msg->hdr.request, msg_reply.hdr.request);
 262         return -1;
 263     }
 264
 265     return msg_reply.payload.u64 ? -1 : 0;
 266 }
 267
 268 static bool vhost_user_one_time_request(VhostUserRequest request)
 269 {
 270     switch (request) {
 271     case VHOST_USER_SET_OWNER:
 272     case VHOST_USER_RESET_OWNER:
 273     case VHOST_USER_SET_MEM_TABLE:
 274     case VHOST_USER_GET_QUEUE_NUM:
 275     case VHOST_USER_NET_SET_MTU:
 276         return true;
 277     default:
 278         return false;
 279     }
 280 }
 281
 282 /* most non-init callers ignore the error */
 283 static int vhost_user_write(struct vhost_dev *dev, VhostUserMsg *msg,
 284                             int *fds, int fd_num)
 285 {
 286     struct vhost_user *u = dev->opaque;
 287     CharBackend *chr = u->chr;
 288     int ret, size = VHOST_USER_HDR_SIZE + msg->hdr.size;
 289
 290     /*
 291      * For non-vring specific requests, like VHOST_USER_SET_MEM_TABLE,
 292      * we just need send it once in the first time. For later such
 293      * request, we just ignore it.
 294      */
 295     if (vhost_user_one_time_request(msg->hdr.request) && dev->vq_index != 0) {
 296         msg->hdr.flags &= ~VHOST_USER_NEED_REPLY_MASK;
 297         return 0;
 298     }
 299
 300     if (qemu_chr_fe_set_msgfds(chr, fds, fd_num) < 0) {
 301         error_report("Failed to set msg fds.");
 302         return -1;
 303     }
 304
 305     ret = qemu_chr_fe_write_all(chr, (const uint8_t *) msg, size);
 306     if (ret != size) {
 307         error_report("Failed to write msg."
 308                      " Wrote %d instead of %d.", ret, size);
 309         return -1;
 310     }
 311
 312     return 0;
 313 }
 314
 315 static int vhost_user_set_log_base(struct vhost_dev *dev, uint64_t base,
 316                                    struct vhost_log *log)
 317 {
 318     int fds[VHOST_MEMORY_MAX_NREGIONS];
 319     size_t fd_num = 0;
 320     bool shmfd = virtio_has_feature(dev->protocol_features,
 321                                     VHOST_USER_PROTOCOL_F_LOG_SHMFD);
 322     VhostUserMsg msg = {
 323         .hdr.request = VHOST_USER_SET_LOG_BASE,
 324         .hdr.flags = VHOST_USER_VERSION,
 325         .payload.log.mmap_size = log->size * sizeof(*(log->log)),
 326         .payload.log.mmap_offset = 0,
 327         .hdr.size = sizeof(msg.payload.log),
 328     };
 329
 330     if (shmfd && log->fd != -1) {
 331         fds[fd_num++] = log->fd;
 332     }
 333
 334     if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
 335         return -1;
 336     }
 337
 338     if (shmfd) {
 339         msg.hdr.size = 0;
 340         if (vhost_user_read(dev, &msg) < 0) {
 341             return -1;
 342         }
 343
 344         if (msg.hdr.request != VHOST_USER_SET_LOG_BASE) {
 345             error_report("Received unexpected msg type. "
 346                          "Expected %d received %d",
 347                          VHOST_USER_SET_LOG_BASE, msg.hdr.request);
 348             return -1;
 349         }
 350     }
 351
 352     return 0;
 353 }
 354
 355 static int vhost_user_set_mem_table_postcopy(struct vhost_dev *dev,
 356                                              struct vhost_memory *mem)
 357 {
 358     struct vhost_user *u = dev->opaque;
 359     int fds[VHOST_MEMORY_MAX_NREGIONS];
 360     int i, fd;
 361     size_t fd_num = 0;
 362     bool reply_supported = virtio_has_feature(dev->protocol_features,
 363                                               VHOST_USER_PROTOCOL_F_REPLY_ACK);
 364     VhostUserMsg msg_reply;
 365     int region_i, msg_i;
 366
 367     VhostUserMsg msg = {
 368         .hdr.request = VHOST_USER_SET_MEM_TABLE,
 369         .hdr.flags = VHOST_USER_VERSION,
 370     };
 371
 372     if (reply_supported) {
 373         msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
 374     }
 375
 376     if (u->region_rb_len < dev->mem->nregions) {
 377         u->region_rb = g_renew(RAMBlock*, u->region_rb, dev->mem->nregions);
 378         u->region_rb_offset = g_renew(ram_addr_t, u->region_rb_offset,
 379                                       dev->mem->nregions);
 380         memset(&(u->region_rb[u->region_rb_len]), '\0',
 381                sizeof(RAMBlock *) * (dev->mem->nregions - u->region_rb_len));
 382         memset(&(u->region_rb_offset[u->region_rb_len]), '\0',
 383                sizeof(ram_addr_t) * (dev->mem->nregions - u->region_rb_len));
 384         u->region_rb_len = dev->mem->nregions;
 385     }
 386
 387     for (i = 0; i < dev->mem->nregions; ++i) {
 388         struct vhost_memory_region *reg = dev->mem->regions + i;
 389         ram_addr_t offset;
 390         MemoryRegion *mr;
 391
 392         assert((uintptr_t)reg->userspace_addr == reg->userspace_addr);
 393         mr = memory_region_from_host((void *)(uintptr_t)reg->userspace_addr,
 394                                      &offset);
 395         fd = memory_region_get_fd(mr);
 396         if (fd > 0) {
 397             trace_vhost_user_set_mem_table_withfd(fd_num, mr->name,
 398                                                   reg->memory_size,
 399                                                   reg->guest_phys_addr,
 400                                                   reg->userspace_addr, offset);
 401             u->region_rb_offset[i] = offset;
 402             u->region_rb[i] = mr->ram_block;
 403             msg.payload.memory.regions[fd_num].userspace_addr =
 404                 reg->userspace_addr;
 405             msg.payload.memory.regions[fd_num].memory_size  = reg->memory_size;
 406             msg.payload.memory.regions[fd_num].guest_phys_addr =
 407                 reg->guest_phys_addr;
 408             msg.payload.memory.regions[fd_num].mmap_offset = offset;
 409             assert(fd_num < VHOST_MEMORY_MAX_NREGIONS);
 410             fds[fd_num++] = fd;
 411         } else {
 412             u->region_rb_offset[i] = 0;
 413             u->region_rb[i] = NULL;
 414         }
 415     }
 416
 417     msg.payload.memory.nregions = fd_num;
 418
 419     if (!fd_num) {
 420         error_report("Failed initializing vhost-user memory map, "
 421                      "consider using -object memory-backend-file share=on");
 422         return -1;
 423     }
 424
 425     msg.hdr.size = sizeof(msg.payload.memory.nregions);
 426     msg.hdr.size += sizeof(msg.payload.memory.padding);
 427     msg.hdr.size += fd_num * sizeof(VhostUserMemoryRegion);
 428
 429     if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
 430         return -1;
 431     }
 432
 433     if (vhost_user_read(dev, &msg_reply) < 0) {
 434         return -1;
 435     }
 436
 437     if (msg_reply.hdr.request != VHOST_USER_SET_MEM_TABLE) {
 438         error_report("%s: Received unexpected msg type."
 439                      "Expected %d received %d", __func__,
 440                      VHOST_USER_SET_MEM_TABLE, msg_reply.hdr.request);
 441         return -1;
 442     }
 443     /* We're using the same structure, just reusing one of the
 444      * fields, so it should be the same size.
 445      */
 446     if (msg_reply.hdr.size != msg.hdr.size) {
 447         error_report("%s: Unexpected size for postcopy reply "
 448                      "%d vs %d", __func__, msg_reply.hdr.size, msg.hdr.size);
 449         return -1;
 450     }
 451
 452     memset(u->postcopy_client_bases, 0,
 453            sizeof(uint64_t) * VHOST_MEMORY_MAX_NREGIONS);
 454
 455     /* They're in the same order as the regions that were sent
 456      * but some of the regions were skipped (above) if they
 457      * didn't have fd's
 458     */
 459     for (msg_i = 0, region_i = 0;
 460          region_i < dev->mem->nregions;
 461         region_i++) {
 462         if (msg_i < fd_num &&
 463             msg_reply.payload.memory.regions[msg_i].guest_phys_addr ==
 464             dev->mem->regions[region_i].guest_phys_addr) {
 465             u->postcopy_client_bases[region_i] =
 466                 msg_reply.payload.memory.regions[msg_i].userspace_addr;
 467             trace_vhost_user_set_mem_table_postcopy(
 468                 msg_reply.payload.memory.regions[msg_i].userspace_addr,
 469                 msg.payload.memory.regions[msg_i].userspace_addr,
 470                 msg_i, region_i);
 471             msg_i++;
 472         }
 473     }
 474     if (msg_i != fd_num) {
 475         error_report("%s: postcopy reply not fully consumed "
 476                      "%d vs %zd",
 477                      __func__, msg_i, fd_num);
 478         return -1;
 479     }
 480     /* Now we've registered this with the postcopy code, we ack to the client,
 481      * because now we're in the position to be able to deal with any faults
 482      * it generates.
 483      */
 484     /* TODO: Use this for failure cases as well with a bad value */
 485     msg.hdr.size = sizeof(msg.payload.u64);
 486     msg.payload.u64 = 0; /* OK */
 487     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 488         return -1;
 489     }
 490
 491     if (reply_supported) {
 492         return process_message_reply(dev, &msg);
 493     }
 494
 495     return 0;
 496 }
 497
 498 static int vhost_user_set_mem_table(struct vhost_dev *dev,
 499                                     struct vhost_memory *mem)
 500 {
 501     struct vhost_user *u = dev->opaque;
 502     int fds[VHOST_MEMORY_MAX_NREGIONS];
 503     int i, fd;
 504     size_t fd_num = 0;
 505     bool do_postcopy = u->postcopy_listen && u->postcopy_fd.handler;
 506     bool reply_supported = virtio_has_feature(dev->protocol_features,
 507                                           VHOST_USER_PROTOCOL_F_REPLY_ACK) &&
 508                                           !do_postcopy;
 509
 510     if (do_postcopy) {
 511         /* Postcopy has enough differences that it's best done in it's own
 512          * version
 513          */
 514         return vhost_user_set_mem_table_postcopy(dev, mem);
 515     }
 516
 517     VhostUserMsg msg = {
 518         .hdr.request = VHOST_USER_SET_MEM_TABLE,
 519         .hdr.flags = VHOST_USER_VERSION,
 520     };
 521
 522     if (reply_supported) {
 523         msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
 524     }
 525
 526     for (i = 0; i < dev->mem->nregions; ++i) {
 527         struct vhost_memory_region *reg = dev->mem->regions + i;
 528         ram_addr_t offset;
 529         MemoryRegion *mr;
 530
 531         assert((uintptr_t)reg->userspace_addr == reg->userspace_addr);
 532         mr = memory_region_from_host((void *)(uintptr_t)reg->userspace_addr,
 533                                      &offset);
 534         fd = memory_region_get_fd(mr);
 535         if (fd > 0) {
 536             if (fd_num == VHOST_MEMORY_MAX_NREGIONS) {
 537                 error_report("Failed preparing vhost-user memory table msg");
 538                 return -1;
 539             }
 540             msg.payload.memory.regions[fd_num].userspace_addr =
 541                 reg->userspace_addr;
 542             msg.payload.memory.regions[fd_num].memory_size  = reg->memory_size;
 543             msg.payload.memory.regions[fd_num].guest_phys_addr =
 544                 reg->guest_phys_addr;
 545             msg.payload.memory.regions[fd_num].mmap_offset = offset;
 546             fds[fd_num++] = fd;
 547         }
 548     }
 549
 550     msg.payload.memory.nregions = fd_num;
 551
 552     if (!fd_num) {
 553         error_report("Failed initializing vhost-user memory map, "
 554                      "consider using -object memory-backend-file share=on");
 555         return -1;
 556     }
 557
 558     msg.hdr.size = sizeof(msg.payload.memory.nregions);
 559     msg.hdr.size += sizeof(msg.payload.memory.padding);
 560     msg.hdr.size += fd_num * sizeof(VhostUserMemoryRegion);
 561
 562     if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
 563         return -1;
 564     }
 565
 566     if (reply_supported) {
 567         return process_message_reply(dev, &msg);
 568     }
 569
 570     return 0;
 571 }
 572
 573 static int vhost_user_set_vring_addr(struct vhost_dev *dev,
 574                                      struct vhost_vring_addr *addr)
 575 {
 576     VhostUserMsg msg = {
 577         .hdr.request = VHOST_USER_SET_VRING_ADDR,
 578         .hdr.flags = VHOST_USER_VERSION,
 579         .payload.addr = *addr,
 580         .hdr.size = sizeof(msg.payload.addr),
 581     };
 582
 583     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 584         return -1;
 585     }
 586
 587     return 0;
 588 }
 589
 590 static int vhost_user_set_vring_endian(struct vhost_dev *dev,
 591                                        struct vhost_vring_state *ring)
 592 {
 593     bool cross_endian = virtio_has_feature(dev->protocol_features,
 594                                            VHOST_USER_PROTOCOL_F_CROSS_ENDIAN);
 595     VhostUserMsg msg = {
 596         .hdr.request = VHOST_USER_SET_VRING_ENDIAN,
 597         .hdr.flags = VHOST_USER_VERSION,
 598         .payload.state = *ring,
 599         .hdr.size = sizeof(msg.payload.state),
 600     };
 601
 602     if (!cross_endian) {
 603         error_report("vhost-user trying to send unhandled ioctl");
 604         return -1;
 605     }
 606
 607     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 608         return -1;
 609     }
 610
 611     return 0;
 612 }
 613
 614 static int vhost_set_vring(struct vhost_dev *dev,
 615                            unsigned long int request,
 616                            struct vhost_vring_state *ring)
 617 {
 618     VhostUserMsg msg = {
 619         .hdr.request = request,
 620         .hdr.flags = VHOST_USER_VERSION,
 621         .payload.state = *ring,
 622         .hdr.size = sizeof(msg.payload.state),
 623     };
 624
 625     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 626         return -1;
 627     }
 628
 629     return 0;
 630 }
 631
 632 static int vhost_user_set_vring_num(struct vhost_dev *dev,
 633                                     struct vhost_vring_state *ring)
 634 {
 635     return vhost_set_vring(dev, VHOST_USER_SET_VRING_NUM, ring);
 636 }
 637
 638 static int vhost_user_set_vring_base(struct vhost_dev *dev,
 639                                      struct vhost_vring_state *ring)
 640 {
 641     return vhost_set_vring(dev, VHOST_USER_SET_VRING_BASE, ring);
 642 }
 643
 644 static int vhost_user_set_vring_enable(struct vhost_dev *dev, int enable)
 645 {
 646     int i;
 647
 648     if (!virtio_has_feature(dev->features, VHOST_USER_F_PROTOCOL_FEATURES)) {
 649         return -1;
 650     }
 651
 652     for (i = 0; i < dev->nvqs; ++i) {
 653         struct vhost_vring_state state = {
 654             .index = dev->vq_index + i,
 655             .num   = enable,
 656         };
 657
 658         vhost_set_vring(dev, VHOST_USER_SET_VRING_ENABLE, &state);
 659     }
 660
 661     return 0;
 662 }
 663
 664 static int vhost_user_get_vring_base(struct vhost_dev *dev,
 665                                      struct vhost_vring_state *ring)
 666 {
 667     VhostUserMsg msg = {
 668         .hdr.request = VHOST_USER_GET_VRING_BASE,
 669         .hdr.flags = VHOST_USER_VERSION,
 670         .payload.state = *ring,
 671         .hdr.size = sizeof(msg.payload.state),
 672     };
 673
 674     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 675         return -1;
 676     }
 677
 678     if (vhost_user_read(dev, &msg) < 0) {
 679         return -1;
 680     }
 681
 682     if (msg.hdr.request != VHOST_USER_GET_VRING_BASE) {
 683         error_report("Received unexpected msg type. Expected %d received %d",
 684                      VHOST_USER_GET_VRING_BASE, msg.hdr.request);
 685         return -1;
 686     }
 687
 688     if (msg.hdr.size != sizeof(msg.payload.state)) {
 689         error_report("Received bad msg size.");
 690         return -1;
 691     }
 692
 693     *ring = msg.payload.state;
 694
 695     return 0;
 696 }
 697
 698 static int vhost_set_vring_file(struct vhost_dev *dev,
 699                                 VhostUserRequest request,
 700                                 struct vhost_vring_file *file)
 701 {
 702     int fds[VHOST_MEMORY_MAX_NREGIONS];
 703     size_t fd_num = 0;
 704     VhostUserMsg msg = {
 705         .hdr.request = request,
 706         .hdr.flags = VHOST_USER_VERSION,
 707         .payload.u64 = file->index & VHOST_USER_VRING_IDX_MASK,
 708         .hdr.size = sizeof(msg.payload.u64),
 709     };
 710
 711     if (ioeventfd_enabled() && file->fd > 0) {
 712         fds[fd_num++] = file->fd;
 713     } else {
 714         msg.payload.u64 |= VHOST_USER_VRING_NOFD_MASK;
 715     }
 716
 717     if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
 718         return -1;
 719     }
 720
 721     return 0;
 722 }
 723
 724 static int vhost_user_set_vring_kick(struct vhost_dev *dev,
 725                                      struct vhost_vring_file *file)
 726 {
 727     return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_KICK, file);
 728 }
 729
 730 static int vhost_user_set_vring_call(struct vhost_dev *dev,
 731                                      struct vhost_vring_file *file)
 732 {
 733     return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_CALL, file);
 734 }
 735
 736 static int vhost_user_set_u64(struct vhost_dev *dev, int request, uint64_t u64)
 737 {
 738     VhostUserMsg msg = {
 739         .hdr.request = request,
 740         .hdr.flags = VHOST_USER_VERSION,
 741         .payload.u64 = u64,
 742         .hdr.size = sizeof(msg.payload.u64),
 743     };
 744
 745     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 746         return -1;
 747     }
 748
 749     return 0;
 750 }
 751
 752 static int vhost_user_set_features(struct vhost_dev *dev,
 753                                    uint64_t features)
 754 {
 755     return vhost_user_set_u64(dev, VHOST_USER_SET_FEATURES, features);
 756 }
 757
 758 static int vhost_user_set_protocol_features(struct vhost_dev *dev,
 759                                             uint64_t features)
 760 {
 761     return vhost_user_set_u64(dev, VHOST_USER_SET_PROTOCOL_FEATURES, features);
 762 }
 763
 764 static int vhost_user_get_u64(struct vhost_dev *dev, int request, uint64_t *u64)
 765 {
 766     VhostUserMsg msg = {
 767         .hdr.request = request,
 768         .hdr.flags = VHOST_USER_VERSION,
 769     };
 770
 771     if (vhost_user_one_time_request(request) && dev->vq_index != 0) {
 772         return 0;
 773     }
 774
 775     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 776         return -1;
 777     }
 778
 779     if (vhost_user_read(dev, &msg) < 0) {
 780         return -1;
 781     }
 782
 783     if (msg.hdr.request != request) {
 784         error_report("Received unexpected msg type. Expected %d received %d",
 785                      request, msg.hdr.request);
 786         return -1;
 787     }
 788
 789     if (msg.hdr.size != sizeof(msg.payload.u64)) {
 790         error_report("Received bad msg size.");
 791         return -1;
 792     }
 793
 794     *u64 = msg.payload.u64;
 795
 796     return 0;
 797 }
 798
 799 static int vhost_user_get_features(struct vhost_dev *dev, uint64_t *features)
 800 {
 801     return vhost_user_get_u64(dev, VHOST_USER_GET_FEATURES, features);
 802 }
 803
 804 static int vhost_user_set_owner(struct vhost_dev *dev)
 805 {
 806     VhostUserMsg msg = {
 807         .hdr.request = VHOST_USER_SET_OWNER,
 808         .hdr.flags = VHOST_USER_VERSION,
 809     };
 810
 811     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 812         return -1;
 813     }
 814
 815     return 0;
 816 }
 817
 818 static int vhost_user_reset_device(struct vhost_dev *dev)
 819 {
 820     VhostUserMsg msg = {
 821         .hdr.request = VHOST_USER_RESET_OWNER,
 822         .hdr.flags = VHOST_USER_VERSION,
 823     };
 824
 825     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 826         return -1;
 827     }
 828
 829     return 0;
 830 }
 831
 832 static int vhost_user_slave_handle_config_change(struct vhost_dev *dev)
 833 {
 834     int ret = -1;
 835
 836     if (!dev->config_ops) {
 837         return -1;
 838     }
 839
 840     if (dev->config_ops->vhost_dev_config_notifier) {
 841         ret = dev->config_ops->vhost_dev_config_notifier(dev);
 842     }
 843
 844     return ret;
 845 }
 846
 847 static void slave_read(void *opaque)
 848 {
 849     struct vhost_dev *dev = opaque;
 850     struct vhost_user *u = dev->opaque;
 851     VhostUserHeader hdr = { 0, };
 852     VhostUserPayload payload = { 0, };
 853     int size, ret = 0;
 854
 855     /* Read header */
 856     size = read(u->slave_fd, &hdr, VHOST_USER_HDR_SIZE);
 857     if (size != VHOST_USER_HDR_SIZE) {
 858         error_report("Failed to read from slave.");
 859         goto err;
 860     }
 861
 862     if (hdr.size > VHOST_USER_PAYLOAD_SIZE) {
 863         error_report("Failed to read msg header."
 864                 " Size %d exceeds the maximum %zu.", hdr.size,
 865                 VHOST_USER_PAYLOAD_SIZE);
 866         goto err;
 867     }
 868
 869     /* Read payload */
 870     size = read(u->slave_fd, &payload, hdr.size);
 871     if (size != hdr.size) {
 872         error_report("Failed to read payload from slave.");
 873         goto err;
 874     }
 875
 876     switch (hdr.request) {
 877     case VHOST_USER_SLAVE_IOTLB_MSG:
 878         ret = vhost_backend_handle_iotlb_msg(dev, &payload.iotlb);
 879         break;
 880     case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG :
 881         ret = vhost_user_slave_handle_config_change(dev);
 882         break;
 883     default:
 884         error_report("Received unexpected msg type.");
 885         ret = -EINVAL;
 886     }
 887
 888     /*
 889      * REPLY_ACK feature handling. Other reply types has to be managed
 890      * directly in their request handlers.
 891      */
 892     if (hdr.flags & VHOST_USER_NEED_REPLY_MASK) {
 893         struct iovec iovec[2];
 894
 895
 896         hdr.flags &= ~VHOST_USER_NEED_REPLY_MASK;
 897         hdr.flags |= VHOST_USER_REPLY_MASK;
 898
 899         payload.u64 = !!ret;
 900         hdr.size = sizeof(payload.u64);
 901
 902         iovec[0].iov_base = &hdr;
 903         iovec[0].iov_len = VHOST_USER_HDR_SIZE;
 904         iovec[1].iov_base = &payload;
 905         iovec[1].iov_len = hdr.size;
 906
 907         size = writev(u->slave_fd, iovec, ARRAY_SIZE(iovec));
 908         if (size != VHOST_USER_HDR_SIZE + hdr.size) {
 909             error_report("Failed to send msg reply to slave.");
 910             goto err;
 911         }
 912     }
 913
 914     return;
 915
 916 err:
 917     qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL);
 918     close(u->slave_fd);
 919     u->slave_fd = -1;
 920     return;
 921 }
 922
 923 static int vhost_setup_slave_channel(struct vhost_dev *dev)
 924 {
 925     VhostUserMsg msg = {
 926         .hdr.request = VHOST_USER_SET_SLAVE_REQ_FD,
 927         .hdr.flags = VHOST_USER_VERSION,
 928     };
 929     struct vhost_user *u = dev->opaque;
 930     int sv[2], ret = 0;
 931     bool reply_supported = virtio_has_feature(dev->protocol_features,
 932                                               VHOST_USER_PROTOCOL_F_REPLY_ACK);
 933
 934     if (!virtio_has_feature(dev->protocol_features,
 935                             VHOST_USER_PROTOCOL_F_SLAVE_REQ)) {
 936         return 0;
 937     }
 938
 939     if (socketpair(PF_UNIX, SOCK_STREAM, 0, sv) == -1) {
 940         error_report("socketpair() failed");
 941         return -1;
 942     }
 943
 944     u->slave_fd = sv[0];
 945     qemu_set_fd_handler(u->slave_fd, slave_read, NULL, dev);
 946
 947     if (reply_supported) {
 948         msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
 949     }
 950
 951     ret = vhost_user_write(dev, &msg, &sv[1], 1);
 952     if (ret) {
 953         goto out;
 954     }
 955
 956     if (reply_supported) {
 957         ret = process_message_reply(dev, &msg);
 958     }
 959
 960 out:
 961     close(sv[1]);
 962     if (ret) {
 963         qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL);
 964         close(u->slave_fd);
 965         u->slave_fd = -1;
 966     }
 967
 968     return ret;
 969 }
 970
 971 /*
 972  * Called back from the postcopy fault thread when a fault is received on our
 973  * ufd.
 974  * TODO: This is Linux specific
 975  */
 976 static int vhost_user_postcopy_fault_handler(struct PostCopyFD *pcfd,
 977                                              void *ufd)
 978 {
 979     struct vhost_dev *dev = pcfd->data;
 980     struct vhost_user *u = dev->opaque;
 981     struct uffd_msg *msg = ufd;
 982     uint64_t faultaddr = msg->arg.pagefault.address;
 983     RAMBlock *rb = NULL;
 984     uint64_t rb_offset;
 985     int i;
 986
 987     trace_vhost_user_postcopy_fault_handler(pcfd->idstr, faultaddr,
 988                                             dev->mem->nregions);
 989     for (i = 0; i < MIN(dev->mem->nregions, u->region_rb_len); i++) {
 990         trace_vhost_user_postcopy_fault_handler_loop(i,
 991                 u->postcopy_client_bases[i], dev->mem->regions[i].memory_size);
 992         if (faultaddr >= u->postcopy_client_bases[i]) {
 993             /* Ofset of the fault address in the vhost region */
 994             uint64_t region_offset = faultaddr - u->postcopy_client_bases[i];
 995             if (region_offset < dev->mem->regions[i].memory_size) {
 996                 rb_offset = region_offset + u->region_rb_offset[i];
 997                 trace_vhost_user_postcopy_fault_handler_found(i,
 998                         region_offset, rb_offset);
 999                 rb = u->region_rb[i];
1000                 return postcopy_request_shared_page(pcfd, rb, faultaddr,
1001                                                     rb_offset);
1002             }
1003         }
1004     }
1005     error_report("%s: Failed to find region for fault %" PRIx64,
1006                  __func__, faultaddr);
1007     return -1;
1008 }
1009
1010 static int vhost_user_postcopy_waker(struct PostCopyFD *pcfd, RAMBlock *rb,
1011                                      uint64_t offset)
1012 {
1013     struct vhost_dev *dev = pcfd->data;
1014     struct vhost_user *u = dev->opaque;
1015     int i;
1016
1017     trace_vhost_user_postcopy_waker(qemu_ram_get_idstr(rb), offset);
1018
1019     if (!u) {
1020         return 0;
1021     }
1022     /* Translate the offset into an address in the clients address space */
1023     for (i = 0; i < MIN(dev->mem->nregions, u->region_rb_len); i++) {
1024         if (u->region_rb[i] == rb &&
1025             offset >= u->region_rb_offset[i] &&
1026             offset < (u->region_rb_offset[i] +
1027                       dev->mem->regions[i].memory_size)) {
1028             uint64_t client_addr = (offset - u->region_rb_offset[i]) +
1029                                    u->postcopy_client_bases[i];
1030             trace_vhost_user_postcopy_waker_found(client_addr);
1031             return postcopy_wake_shared(pcfd, client_addr, rb);
1032         }
1033     }
1034
1035     trace_vhost_user_postcopy_waker_nomatch(qemu_ram_get_idstr(rb), offset);
1036     return 0;
1037 }
1038
1039 /*
1040  * Called at the start of an inbound postcopy on reception of the
1041  * 'advise' command.
1042  */
1043 static int vhost_user_postcopy_advise(struct vhost_dev *dev, Error **errp)
1044 {
1045     struct vhost_user *u = dev->opaque;
1046     CharBackend *chr = u->chr;
1047     int ufd;
1048     VhostUserMsg msg = {
1049         .hdr.request = VHOST_USER_POSTCOPY_ADVISE,
1050         .hdr.flags = VHOST_USER_VERSION,
1051     };
1052
1053     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1054         error_setg(errp, "Failed to send postcopy_advise to vhost");
1055         return -1;
1056     }
1057
1058     if (vhost_user_read(dev, &msg) < 0) {
1059         error_setg(errp, "Failed to get postcopy_advise reply from vhost");
1060         return -1;
1061     }
1062
1063     if (msg.hdr.request != VHOST_USER_POSTCOPY_ADVISE) {
1064         error_setg(errp, "Unexpected msg type. Expected %d received %d",
1065                      VHOST_USER_POSTCOPY_ADVISE, msg.hdr.request);
1066         return -1;
1067     }
1068
1069     if (msg.hdr.size) {
1070         error_setg(errp, "Received bad msg size.");
1071         return -1;
1072     }
1073     ufd = qemu_chr_fe_get_msgfd(chr);
1074     if (ufd < 0) {
1075         error_setg(errp, "%s: Failed to get ufd", __func__);
1076         return -1;
1077     }
1078     fcntl(ufd, F_SETFL, O_NONBLOCK);
1079
1080     /* register ufd with userfault thread */
1081     u->postcopy_fd.fd = ufd;
1082     u->postcopy_fd.data = dev;
1083     u->postcopy_fd.handler = vhost_user_postcopy_fault_handler;
1084     u->postcopy_fd.waker = vhost_user_postcopy_waker;
1085     u->postcopy_fd.idstr = "vhost-user"; /* Need to find unique name */
1086     postcopy_register_shared_ufd(&u->postcopy_fd);
1087     return 0;
1088 }
1089
1090 /*
1091  * Called at the switch to postcopy on reception of the 'listen' command.
1092  */
1093 static int vhost_user_postcopy_listen(struct vhost_dev *dev, Error **errp)
1094 {
1095     struct vhost_user *u = dev->opaque;
1096     int ret;
1097     VhostUserMsg msg = {
1098         .hdr.request = VHOST_USER_POSTCOPY_LISTEN,
1099         .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
1100     };
1101     u->postcopy_listen = true;
1102     trace_vhost_user_postcopy_listen();
1103     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1104         error_setg(errp, "Failed to send postcopy_listen to vhost");
1105         return -1;
1106     }
1107
1108     ret = process_message_reply(dev, &msg);
1109     if (ret) {
1110         error_setg(errp, "Failed to receive reply to postcopy_listen");
1111         return ret;
1112     }
1113
1114     return 0;
1115 }
1116
1117 /*
1118  * Called at the end of postcopy
1119  */
1120 static int vhost_user_postcopy_end(struct vhost_dev *dev, Error **errp)
1121 {
1122     VhostUserMsg msg = {
1123         .hdr.request = VHOST_USER_POSTCOPY_END,
1124         .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
1125     };
1126     int ret;
1127     struct vhost_user *u = dev->opaque;
1128
1129     trace_vhost_user_postcopy_end_entry();
1130     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1131         error_setg(errp, "Failed to send postcopy_end to vhost");
1132         return -1;
1133     }
1134
1135     ret = process_message_reply(dev, &msg);
1136     if (ret) {
1137         error_setg(errp, "Failed to receive reply to postcopy_end");
1138         return ret;
1139     }
1140     postcopy_unregister_shared_ufd(&u->postcopy_fd);
1141     u->postcopy_fd.handler = NULL;
1142
1143     trace_vhost_user_postcopy_end_exit();
1144
1145     return 0;
1146 }
1147
1148 static int vhost_user_postcopy_notifier(NotifierWithReturn *notifier,
1149                                         void *opaque)
1150 {
1151     struct PostcopyNotifyData *pnd = opaque;
1152     struct vhost_user *u = container_of(notifier, struct vhost_user,
1153                                          postcopy_notifier);
1154     struct vhost_dev *dev = u->dev;
1155
1156     switch (pnd->reason) {
1157     case POSTCOPY_NOTIFY_PROBE:
1158         if (!virtio_has_feature(dev->protocol_features,
1159                                 VHOST_USER_PROTOCOL_F_PAGEFAULT)) {
1160             /* TODO: Get the device name into this error somehow */
1161             error_setg(pnd->errp,
1162                        "vhost-user backend not capable of postcopy");
1163             return -ENOENT;
1164         }
1165         break;
1166
1167     case POSTCOPY_NOTIFY_INBOUND_ADVISE:
1168         return vhost_user_postcopy_advise(dev, pnd->errp);
1169
1170     case POSTCOPY_NOTIFY_INBOUND_LISTEN:
1171         return vhost_user_postcopy_listen(dev, pnd->errp);
1172
1173     case POSTCOPY_NOTIFY_INBOUND_END:
1174         return vhost_user_postcopy_end(dev, pnd->errp);
1175
1176     default:
1177         /* We ignore notifications we don't know */
1178         break;
1179     }
1180
1181     return 0;
1182 }
1183
1184 static int vhost_user_init(struct vhost_dev *dev, void *opaque)
1185 {
1186     uint64_t features, protocol_features;
1187     struct vhost_user *u;
1188     int err;
1189
1190     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
1191
1192     u = g_new0(struct vhost_user, 1);
1193     u->chr = opaque;
1194     u->slave_fd = -1;
1195     u->dev = dev;
1196     dev->opaque = u;
1197
1198     err = vhost_user_get_features(dev, &features);
1199     if (err < 0) {
1200         return err;
1201     }
1202
1203     if (virtio_has_feature(features, VHOST_USER_F_PROTOCOL_FEATURES)) {
1204         dev->backend_features |= 1ULL << VHOST_USER_F_PROTOCOL_FEATURES;
1205
1206         err = vhost_user_get_u64(dev, VHOST_USER_GET_PROTOCOL_FEATURES,
1207                                  &protocol_features);
1208         if (err < 0) {
1209             return err;
1210         }
1211
1212         dev->protocol_features =
1213             protocol_features & VHOST_USER_PROTOCOL_FEATURE_MASK;
1214         err = vhost_user_set_protocol_features(dev, dev->protocol_features);
1215         if (err < 0) {
1216             return err;
1217         }
1218
1219         /* query the max queues we support if backend supports Multiple Queue */
1220         if (dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_MQ)) {
1221             err = vhost_user_get_u64(dev, VHOST_USER_GET_QUEUE_NUM,
1222                                      &dev->max_queues);
1223             if (err < 0) {
1224                 return err;
1225             }
1226         }
1227
1228         if (virtio_has_feature(features, VIRTIO_F_IOMMU_PLATFORM) &&
1229                 !(virtio_has_feature(dev->protocol_features,
1230                     VHOST_USER_PROTOCOL_F_SLAVE_REQ) &&
1231                  virtio_has_feature(dev->protocol_features,
1232                     VHOST_USER_PROTOCOL_F_REPLY_ACK))) {
1233             error_report("IOMMU support requires reply-ack and "
1234                          "slave-req protocol features.");
1235             return -1;
1236         }
1237     }
1238
1239     if (dev->migration_blocker == NULL &&
1240         !virtio_has_feature(dev->protocol_features,
1241                             VHOST_USER_PROTOCOL_F_LOG_SHMFD)) {
1242         error_setg(&dev->migration_blocker,
1243                    "Migration disabled: vhost-user backend lacks "
1244                    "VHOST_USER_PROTOCOL_F_LOG_SHMFD feature.");
1245     }
1246
1247     err = vhost_setup_slave_channel(dev);
1248     if (err < 0) {
1249         return err;
1250     }
1251
1252     u->postcopy_notifier.notify = vhost_user_postcopy_notifier;
1253     postcopy_add_notifier(&u->postcopy_notifier);
1254
1255     return 0;
1256 }
1257
1258 static int vhost_user_cleanup(struct vhost_dev *dev)
1259 {
1260     struct vhost_user *u;
1261
1262     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
1263
1264     u = dev->opaque;
1265     if (u->postcopy_notifier.notify) {
1266         postcopy_remove_notifier(&u->postcopy_notifier);
1267         u->postcopy_notifier.notify = NULL;
1268     }
1269     if (u->slave_fd >= 0) {
1270         qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL);
1271         close(u->slave_fd);
1272         u->slave_fd = -1;
1273     }
1274     g_free(u->region_rb);
1275     u->region_rb = NULL;
1276     g_free(u->region_rb_offset);
1277     u->region_rb_offset = NULL;
1278     u->region_rb_len = 0;
1279     g_free(u);
1280     dev->opaque = 0;
1281
1282     return 0;
1283 }
1284
1285 static int vhost_user_get_vq_index(struct vhost_dev *dev, int idx)
1286 {
1287     assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
1288
1289     return idx;
1290 }
1291
1292 static int vhost_user_memslots_limit(struct vhost_dev *dev)
1293 {
1294     return VHOST_MEMORY_MAX_NREGIONS;
1295 }
1296
1297 static bool vhost_user_requires_shm_log(struct vhost_dev *dev)
1298 {
1299     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
1300
1301     return virtio_has_feature(dev->protocol_features,
1302                               VHOST_USER_PROTOCOL_F_LOG_SHMFD);
1303 }
1304
1305 static int vhost_user_migration_done(struct vhost_dev *dev, char* mac_addr)
1306 {
1307     VhostUserMsg msg = { 0 };
1308
1309     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
1310
1311     /* If guest supports GUEST_ANNOUNCE do nothing */
1312     if (virtio_has_feature(dev->acked_features, VIRTIO_NET_F_GUEST_ANNOUNCE)) {
1313         return 0;
1314     }
1315
1316     /* if backend supports VHOST_USER_PROTOCOL_F_RARP ask it to send the RARP */
1317     if (virtio_has_feature(dev->protocol_features,
1318                            VHOST_USER_PROTOCOL_F_RARP)) {
1319         msg.hdr.request = VHOST_USER_SEND_RARP;
1320         msg.hdr.flags = VHOST_USER_VERSION;
1321         memcpy((char *)&msg.payload.u64, mac_addr, 6);
1322         msg.hdr.size = sizeof(msg.payload.u64);
1323
1324         return vhost_user_write(dev, &msg, NULL, 0);
1325     }
1326     return -1;
1327 }
1328
1329 static bool vhost_user_can_merge(struct vhost_dev *dev,
1330                                  uint64_t start1, uint64_t size1,
1331                                  uint64_t start2, uint64_t size2)
1332 {
1333     ram_addr_t offset;
1334     int mfd, rfd;
1335     MemoryRegion *mr;
1336
1337     mr = memory_region_from_host((void *)(uintptr_t)start1, &offset);
1338     mfd = memory_region_get_fd(mr);
1339
1340     mr = memory_region_from_host((void *)(uintptr_t)start2, &offset);
1341     rfd = memory_region_get_fd(mr);
1342
1343     return mfd == rfd;
1344 }
1345
1346 static int vhost_user_net_set_mtu(struct vhost_dev *dev, uint16_t mtu)
1347 {
1348     VhostUserMsg msg;
1349     bool reply_supported = virtio_has_feature(dev->protocol_features,
1350                                               VHOST_USER_PROTOCOL_F_REPLY_ACK);
1351
1352     if (!(dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_NET_MTU))) {
1353         return 0;
1354     }
1355
1356     msg.hdr.request = VHOST_USER_NET_SET_MTU;
1357     msg.payload.u64 = mtu;
1358     msg.hdr.size = sizeof(msg.payload.u64);
1359     msg.hdr.flags = VHOST_USER_VERSION;
1360     if (reply_supported) {
1361         msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
1362     }
1363
1364     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1365         return -1;
1366     }
1367
1368     /* If reply_ack supported, slave has to ack specified MTU is valid */
1369     if (reply_supported) {
1370         return process_message_reply(dev, &msg);
1371     }
1372
1373     return 0;
1374 }
1375
1376 static int vhost_user_send_device_iotlb_msg(struct vhost_dev *dev,
1377                                             struct vhost_iotlb_msg *imsg)
1378 {
1379     VhostUserMsg msg = {
1380         .hdr.request = VHOST_USER_IOTLB_MSG,
1381         .hdr.size = sizeof(msg.payload.iotlb),
1382         .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
1383         .payload.iotlb = *imsg,
1384     };
1385
1386     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1387         return -EFAULT;
1388     }
1389
1390     return process_message_reply(dev, &msg);
1391 }
1392
1393
1394 static void vhost_user_set_iotlb_callback(struct vhost_dev *dev, int enabled)
1395 {
1396     /* No-op as the receive channel is not dedicated to IOTLB messages. */
1397 }
1398
1399 static int vhost_user_get_config(struct vhost_dev *dev, uint8_t *config,
1400                                  uint32_t config_len)
1401 {
1402     VhostUserMsg msg = {
1403         .hdr.request = VHOST_USER_GET_CONFIG,
1404         .hdr.flags = VHOST_USER_VERSION,
1405         .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + config_len,
1406     };
1407
1408     if (config_len > VHOST_USER_MAX_CONFIG_SIZE) {
1409         return -1;
1410     }
1411
1412     msg.payload.config.offset = 0;
1413     msg.payload.config.size = config_len;
1414     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1415         return -1;
1416     }
1417
1418     if (vhost_user_read(dev, &msg) < 0) {
1419         return -1;
1420     }
1421
1422     if (msg.hdr.request != VHOST_USER_GET_CONFIG) {
1423         error_report("Received unexpected msg type. Expected %d received %d",
1424                      VHOST_USER_GET_CONFIG, msg.hdr.request);
1425         return -1;
1426     }
1427
1428     if (msg.hdr.size != VHOST_USER_CONFIG_HDR_SIZE + config_len) {
1429         error_report("Received bad msg size.");
1430         return -1;
1431     }
1432
1433     memcpy(config, msg.payload.config.region, config_len);
1434
1435     return 0;
1436 }
1437
1438 static int vhost_user_set_config(struct vhost_dev *dev, const uint8_t *data,
1439                                  uint32_t offset, uint32_t size, uint32_t flags)
1440 {
1441     uint8_t *p;
1442     bool reply_supported = virtio_has_feature(dev->protocol_features,
1443                                               VHOST_USER_PROTOCOL_F_REPLY_ACK);
1444
1445     VhostUserMsg msg = {
1446         .hdr.request = VHOST_USER_SET_CONFIG,
1447         .hdr.flags = VHOST_USER_VERSION,
1448         .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + size,
1449     };
1450
1451     if (reply_supported) {
1452         msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
1453     }
1454
1455     if (size > VHOST_USER_MAX_CONFIG_SIZE) {
1456         return -1;
1457     }
1458
1459     msg.payload.config.offset = offset,
1460     msg.payload.config.size = size,
1461     msg.payload.config.flags = flags,
1462     p = msg.payload.config.region;
1463     memcpy(p, data, size);
1464
1465     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1466         return -1;
1467     }
1468
1469     if (reply_supported) {
1470         return process_message_reply(dev, &msg);
1471     }
1472
1473     return 0;
1474 }
1475
1476 static int vhost_user_crypto_create_session(struct vhost_dev *dev,
1477                                             void *session_info,
1478                                             uint64_t *session_id)
1479 {
1480     bool crypto_session = virtio_has_feature(dev->protocol_features,
1481                                        VHOST_USER_PROTOCOL_F_CRYPTO_SESSION);
1482     CryptoDevBackendSymSessionInfo *sess_info = session_info;
1483     VhostUserMsg msg = {
1484         .hdr.request = VHOST_USER_CREATE_CRYPTO_SESSION,
1485         .hdr.flags = VHOST_USER_VERSION,
1486         .hdr.size = sizeof(msg.payload.session),
1487     };
1488
1489     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
1490
1491     if (!crypto_session) {
1492         error_report("vhost-user trying to send unhandled ioctl");
1493         return -1;
1494     }
1495
1496     memcpy(&msg.payload.session.session_setup_data, sess_info,
1497               sizeof(CryptoDevBackendSymSessionInfo));
1498     if (sess_info->key_len) {
1499         memcpy(&msg.payload.session.key, sess_info->cipher_key,
1500                sess_info->key_len);
1501     }
1502     if (sess_info->auth_key_len > 0) {
1503         memcpy(&msg.payload.session.auth_key, sess_info->auth_key,
1504                sess_info->auth_key_len);
1505     }
1506     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1507         error_report("vhost_user_write() return -1, create session failed");
1508         return -1;
1509     }
1510
1511     if (vhost_user_read(dev, &msg) < 0) {
1512         error_report("vhost_user_read() return -1, create session failed");
1513         return -1;
1514     }
1515
1516     if (msg.hdr.request != VHOST_USER_CREATE_CRYPTO_SESSION) {
1517         error_report("Received unexpected msg type. Expected %d received %d",
1518                      VHOST_USER_CREATE_CRYPTO_SESSION, msg.hdr.request);
1519         return -1;
1520     }
1521
1522     if (msg.hdr.size != sizeof(msg.payload.session)) {
1523         error_report("Received bad msg size.");
1524         return -1;
1525     }
1526
1527     if (msg.payload.session.session_id < 0) {
1528         error_report("Bad session id: %" PRId64 "",
1529                               msg.payload.session.session_id);
1530         return -1;
1531     }
1532     *session_id = msg.payload.session.session_id;
1533
1534     return 0;
1535 }
1536
1537 static int
1538 vhost_user_crypto_close_session(struct vhost_dev *dev, uint64_t session_id)
1539 {
1540     bool crypto_session = virtio_has_feature(dev->protocol_features,
1541                                        VHOST_USER_PROTOCOL_F_CRYPTO_SESSION);
1542     VhostUserMsg msg = {
1543         .hdr.request = VHOST_USER_CLOSE_CRYPTO_SESSION,
1544         .hdr.flags = VHOST_USER_VERSION,
1545         .hdr.size = sizeof(msg.payload.u64),
1546     };
1547     msg.payload.u64 = session_id;
1548
1549     if (!crypto_session) {
1550         error_report("vhost-user trying to send unhandled ioctl");
1551         return -1;
1552     }
1553
1554     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1555         error_report("vhost_user_write() return -1, close session failed");
1556         return -1;
1557     }
1558
1559     return 0;
1560 }
1561
1562 const VhostOps user_ops = {
1563         .backend_type = VHOST_BACKEND_TYPE_USER,
1564         .vhost_backend_init = vhost_user_init,
1565         .vhost_backend_cleanup = vhost_user_cleanup,
1566         .vhost_backend_memslots_limit = vhost_user_memslots_limit,
1567         .vhost_set_log_base = vhost_user_set_log_base,
1568         .vhost_set_mem_table = vhost_user_set_mem_table,
1569         .vhost_set_vring_addr = vhost_user_set_vring_addr,
1570         .vhost_set_vring_endian = vhost_user_set_vring_endian,
1571         .vhost_set_vring_num = vhost_user_set_vring_num,
1572         .vhost_set_vring_base = vhost_user_set_vring_base,
1573         .vhost_get_vring_base = vhost_user_get_vring_base,
1574         .vhost_set_vring_kick = vhost_user_set_vring_kick,
1575         .vhost_set_vring_call = vhost_user_set_vring_call,
1576         .vhost_set_features = vhost_user_set_features,
1577         .vhost_get_features = vhost_user_get_features,
1578         .vhost_set_owner = vhost_user_set_owner,
1579         .vhost_reset_device = vhost_user_reset_device,
1580         .vhost_get_vq_index = vhost_user_get_vq_index,
1581         .vhost_set_vring_enable = vhost_user_set_vring_enable,
1582         .vhost_requires_shm_log = vhost_user_requires_shm_log,
1583         .vhost_migration_done = vhost_user_migration_done,
1584         .vhost_backend_can_merge = vhost_user_can_merge,
1585         .vhost_net_set_mtu = vhost_user_net_set_mtu,
1586         .vhost_set_iotlb_callback = vhost_user_set_iotlb_callback,
1587         .vhost_send_device_iotlb_msg = vhost_user_send_device_iotlb_msg,
1588         .vhost_get_config = vhost_user_get_config,
1589         .vhost_set_config = vhost_user_set_config,
1590         .vhost_crypto_create_session = vhost_user_crypto_create_session,
1591         .vhost_crypto_close_session = vhost_user_crypto_close_session,
1592 };