hw/virtio/vhost-user.c

   1 /*
   2  * vhost-user
   3  *
   4  * Copyright (c) 2013 Virtual Open Systems Sarl.
   5  *
   6  * This work is licensed under the terms of the GNU GPL, version 2 or later.
   7  * See the COPYING file in the top-level directory.
   8  *
   9  */
  10
  11 #include "qemu/osdep.h"
  12 #include "qapi/error.h"
  13 #include "hw/virtio/vhost.h"
  14 #include "hw/virtio/vhost-backend.h"
  15 #include "hw/virtio/virtio-net.h"
  16 #include "chardev/char-fe.h"
  17 #include "sysemu/kvm.h"
  18 #include "qemu/error-report.h"
  19 #include "qemu/sockets.h"
  20 #include "sysemu/cryptodev.h"
  21 #include "migration/migration.h"
  22 #include "migration/postcopy-ram.h"
  23 #include "trace.h"
  24
  25 #include <sys/ioctl.h>
  26 #include <sys/socket.h>
  27 #include <sys/un.h>
  28 #include <linux/vhost.h>
  29 #include <linux/userfaultfd.h>
  30
  31 #define VHOST_MEMORY_MAX_NREGIONS    8
  32 #define VHOST_USER_F_PROTOCOL_FEATURES 30
  33 #define VHOST_USER_SLAVE_MAX_FDS     8
  34
  35 /*
  36  * Maximum size of virtio device config space
  37  */
  38 #define VHOST_USER_MAX_CONFIG_SIZE 256
  39
  40 enum VhostUserProtocolFeature {
  41     VHOST_USER_PROTOCOL_F_MQ = 0,
  42     VHOST_USER_PROTOCOL_F_LOG_SHMFD = 1,
  43     VHOST_USER_PROTOCOL_F_RARP = 2,
  44     VHOST_USER_PROTOCOL_F_REPLY_ACK = 3,
  45     VHOST_USER_PROTOCOL_F_NET_MTU = 4,
  46     VHOST_USER_PROTOCOL_F_SLAVE_REQ = 5,
  47     VHOST_USER_PROTOCOL_F_CROSS_ENDIAN = 6,
  48     VHOST_USER_PROTOCOL_F_CRYPTO_SESSION = 7,
  49     VHOST_USER_PROTOCOL_F_PAGEFAULT = 8,
  50     VHOST_USER_PROTOCOL_F_CONFIG = 9,
  51     VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD = 10,
  52     VHOST_USER_PROTOCOL_F_MAX
  53 };
  54
  55 #define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1)
  56
  57 typedef enum VhostUserRequest {
  58     VHOST_USER_NONE = 0,
  59     VHOST_USER_GET_FEATURES = 1,
  60     VHOST_USER_SET_FEATURES = 2,
  61     VHOST_USER_SET_OWNER = 3,
  62     VHOST_USER_RESET_OWNER = 4,
  63     VHOST_USER_SET_MEM_TABLE = 5,
  64     VHOST_USER_SET_LOG_BASE = 6,
  65     VHOST_USER_SET_LOG_FD = 7,
  66     VHOST_USER_SET_VRING_NUM = 8,
  67     VHOST_USER_SET_VRING_ADDR = 9,
  68     VHOST_USER_SET_VRING_BASE = 10,
  69     VHOST_USER_GET_VRING_BASE = 11,
  70     VHOST_USER_SET_VRING_KICK = 12,
  71     VHOST_USER_SET_VRING_CALL = 13,
  72     VHOST_USER_SET_VRING_ERR = 14,
  73     VHOST_USER_GET_PROTOCOL_FEATURES = 15,
  74     VHOST_USER_SET_PROTOCOL_FEATURES = 16,
  75     VHOST_USER_GET_QUEUE_NUM = 17,
  76     VHOST_USER_SET_VRING_ENABLE = 18,
  77     VHOST_USER_SEND_RARP = 19,
  78     VHOST_USER_NET_SET_MTU = 20,
  79     VHOST_USER_SET_SLAVE_REQ_FD = 21,
  80     VHOST_USER_IOTLB_MSG = 22,
  81     VHOST_USER_SET_VRING_ENDIAN = 23,
  82     VHOST_USER_GET_CONFIG = 24,
  83     VHOST_USER_SET_CONFIG = 25,
  84     VHOST_USER_CREATE_CRYPTO_SESSION = 26,
  85     VHOST_USER_CLOSE_CRYPTO_SESSION = 27,
  86     VHOST_USER_POSTCOPY_ADVISE  = 28,
  87     VHOST_USER_POSTCOPY_LISTEN  = 29,
  88     VHOST_USER_POSTCOPY_END     = 30,
  89     VHOST_USER_MAX
  90 } VhostUserRequest;
  91
  92 typedef enum VhostUserSlaveRequest {
  93     VHOST_USER_SLAVE_NONE = 0,
  94     VHOST_USER_SLAVE_IOTLB_MSG = 1,
  95     VHOST_USER_SLAVE_CONFIG_CHANGE_MSG = 2,
  96     VHOST_USER_SLAVE_MAX
  97 }  VhostUserSlaveRequest;
  98
  99 typedef struct VhostUserMemoryRegion {
 100     uint64_t guest_phys_addr;
 101     uint64_t memory_size;
 102     uint64_t userspace_addr;
 103     uint64_t mmap_offset;
 104 } VhostUserMemoryRegion;
 105
 106 typedef struct VhostUserMemory {
 107     uint32_t nregions;
 108     uint32_t padding;
 109     VhostUserMemoryRegion regions[VHOST_MEMORY_MAX_NREGIONS];
 110 } VhostUserMemory;
 111
 112 typedef struct VhostUserLog {
 113     uint64_t mmap_size;
 114     uint64_t mmap_offset;
 115 } VhostUserLog;
 116
 117 typedef struct VhostUserConfig {
 118     uint32_t offset;
 119     uint32_t size;
 120     uint32_t flags;
 121     uint8_t region[VHOST_USER_MAX_CONFIG_SIZE];
 122 } VhostUserConfig;
 123
 124 #define VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN    512
 125 #define VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN  64
 126
 127 typedef struct VhostUserCryptoSession {
 128     /* session id for success, -1 on errors */
 129     int64_t session_id;
 130     CryptoDevBackendSymSessionInfo session_setup_data;
 131     uint8_t key[VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN];
 132     uint8_t auth_key[VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN];
 133 } VhostUserCryptoSession;
 134
 135 static VhostUserConfig c __attribute__ ((unused));
 136 #define VHOST_USER_CONFIG_HDR_SIZE (sizeof(c.offset) \
 137                                    + sizeof(c.size) \
 138                                    + sizeof(c.flags))
 139
 140 typedef struct {
 141     VhostUserRequest request;
 142
 143 #define VHOST_USER_VERSION_MASK     (0x3)
 144 #define VHOST_USER_REPLY_MASK       (0x1<<2)
 145 #define VHOST_USER_NEED_REPLY_MASK  (0x1 << 3)
 146     uint32_t flags;
 147     uint32_t size; /* the following payload size */
 148 } QEMU_PACKED VhostUserHeader;
 149
 150 typedef union {
 151 #define VHOST_USER_VRING_IDX_MASK   (0xff)
 152 #define VHOST_USER_VRING_NOFD_MASK  (0x1<<8)
 153         uint64_t u64;
 154         struct vhost_vring_state state;
 155         struct vhost_vring_addr addr;
 156         VhostUserMemory memory;
 157         VhostUserLog log;
 158         struct vhost_iotlb_msg iotlb;
 159         VhostUserConfig config;
 160         VhostUserCryptoSession session;
 161 } VhostUserPayload;
 162
 163 typedef struct VhostUserMsg {
 164     VhostUserHeader hdr;
 165     VhostUserPayload payload;
 166 } QEMU_PACKED VhostUserMsg;
 167
 168 static VhostUserMsg m __attribute__ ((unused));
 169 #define VHOST_USER_HDR_SIZE (sizeof(VhostUserHeader))
 170
 171 #define VHOST_USER_PAYLOAD_SIZE (sizeof(VhostUserPayload))
 172
 173 /* The version of the protocol we support */
 174 #define VHOST_USER_VERSION    (0x1)
 175
 176 struct vhost_user {
 177     struct vhost_dev *dev;
 178     CharBackend *chr;
 179     int slave_fd;
 180     NotifierWithReturn postcopy_notifier;
 181     struct PostCopyFD  postcopy_fd;
 182     uint64_t           postcopy_client_bases[VHOST_MEMORY_MAX_NREGIONS];
 183     /* Length of the region_rb and region_rb_offset arrays */
 184     size_t             region_rb_len;
 185     /* RAMBlock associated with a given region */
 186     RAMBlock         **region_rb;
 187     /* The offset from the start of the RAMBlock to the start of the
 188      * vhost region.
 189      */
 190     ram_addr_t        *region_rb_offset;
 191
 192     /* True once we've entered postcopy_listen */
 193     bool               postcopy_listen;
 194 };
 195
 196 static bool ioeventfd_enabled(void)
 197 {
 198     return kvm_enabled() && kvm_eventfds_enabled();
 199 }
 200
 201 static int vhost_user_read(struct vhost_dev *dev, VhostUserMsg *msg)
 202 {
 203     struct vhost_user *u = dev->opaque;
 204     CharBackend *chr = u->chr;
 205     uint8_t *p = (uint8_t *) msg;
 206     int r, size = VHOST_USER_HDR_SIZE;
 207
 208     r = qemu_chr_fe_read_all(chr, p, size);
 209     if (r != size) {
 210         error_report("Failed to read msg header. Read %d instead of %d."
 211                      " Original request %d.", r, size, msg->hdr.request);
 212         goto fail;
 213     }
 214
 215     /* validate received flags */
 216     if (msg->hdr.flags != (VHOST_USER_REPLY_MASK | VHOST_USER_VERSION)) {
 217         error_report("Failed to read msg header."
 218                 " Flags 0x%x instead of 0x%x.", msg->hdr.flags,
 219                 VHOST_USER_REPLY_MASK | VHOST_USER_VERSION);
 220         goto fail;
 221     }
 222
 223     /* validate message size is sane */
 224     if (msg->hdr.size > VHOST_USER_PAYLOAD_SIZE) {
 225         error_report("Failed to read msg header."
 226                 " Size %d exceeds the maximum %zu.", msg->hdr.size,
 227                 VHOST_USER_PAYLOAD_SIZE);
 228         goto fail;
 229     }
 230
 231     if (msg->hdr.size) {
 232         p += VHOST_USER_HDR_SIZE;
 233         size = msg->hdr.size;
 234         r = qemu_chr_fe_read_all(chr, p, size);
 235         if (r != size) {
 236             error_report("Failed to read msg payload."
 237                          " Read %d instead of %d.", r, msg->hdr.size);
 238             goto fail;
 239         }
 240     }
 241
 242     return 0;
 243
 244 fail:
 245     return -1;
 246 }
 247
 248 static int process_message_reply(struct vhost_dev *dev,
 249                                  const VhostUserMsg *msg)
 250 {
 251     VhostUserMsg msg_reply;
 252
 253     if ((msg->hdr.flags & VHOST_USER_NEED_REPLY_MASK) == 0) {
 254         return 0;
 255     }
 256
 257     if (vhost_user_read(dev, &msg_reply) < 0) {
 258         return -1;
 259     }
 260
 261     if (msg_reply.hdr.request != msg->hdr.request) {
 262         error_report("Received unexpected msg type."
 263                      "Expected %d received %d",
 264                      msg->hdr.request, msg_reply.hdr.request);
 265         return -1;
 266     }
 267
 268     return msg_reply.payload.u64 ? -1 : 0;
 269 }
 270
 271 static bool vhost_user_one_time_request(VhostUserRequest request)
 272 {
 273     switch (request) {
 274     case VHOST_USER_SET_OWNER:
 275     case VHOST_USER_RESET_OWNER:
 276     case VHOST_USER_SET_MEM_TABLE:
 277     case VHOST_USER_GET_QUEUE_NUM:
 278     case VHOST_USER_NET_SET_MTU:
 279         return true;
 280     default:
 281         return false;
 282     }
 283 }
 284
 285 /* most non-init callers ignore the error */
 286 static int vhost_user_write(struct vhost_dev *dev, VhostUserMsg *msg,
 287                             int *fds, int fd_num)
 288 {
 289     struct vhost_user *u = dev->opaque;
 290     CharBackend *chr = u->chr;
 291     int ret, size = VHOST_USER_HDR_SIZE + msg->hdr.size;
 292
 293     /*
 294      * For non-vring specific requests, like VHOST_USER_SET_MEM_TABLE,
 295      * we just need send it once in the first time. For later such
 296      * request, we just ignore it.
 297      */
 298     if (vhost_user_one_time_request(msg->hdr.request) && dev->vq_index != 0) {
 299         msg->hdr.flags &= ~VHOST_USER_NEED_REPLY_MASK;
 300         return 0;
 301     }
 302
 303     if (qemu_chr_fe_set_msgfds(chr, fds, fd_num) < 0) {
 304         error_report("Failed to set msg fds.");
 305         return -1;
 306     }
 307
 308     ret = qemu_chr_fe_write_all(chr, (const uint8_t *) msg, size);
 309     if (ret != size) {
 310         error_report("Failed to write msg."
 311                      " Wrote %d instead of %d.", ret, size);
 312         return -1;
 313     }
 314
 315     return 0;
 316 }
 317
 318 static int vhost_user_set_log_base(struct vhost_dev *dev, uint64_t base,
 319                                    struct vhost_log *log)
 320 {
 321     int fds[VHOST_MEMORY_MAX_NREGIONS];
 322     size_t fd_num = 0;
 323     bool shmfd = virtio_has_feature(dev->protocol_features,
 324                                     VHOST_USER_PROTOCOL_F_LOG_SHMFD);
 325     VhostUserMsg msg = {
 326         .hdr.request = VHOST_USER_SET_LOG_BASE,
 327         .hdr.flags = VHOST_USER_VERSION,
 328         .payload.log.mmap_size = log->size * sizeof(*(log->log)),
 329         .payload.log.mmap_offset = 0,
 330         .hdr.size = sizeof(msg.payload.log),
 331     };
 332
 333     if (shmfd && log->fd != -1) {
 334         fds[fd_num++] = log->fd;
 335     }
 336
 337     if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
 338         return -1;
 339     }
 340
 341     if (shmfd) {
 342         msg.hdr.size = 0;
 343         if (vhost_user_read(dev, &msg) < 0) {
 344             return -1;
 345         }
 346
 347         if (msg.hdr.request != VHOST_USER_SET_LOG_BASE) {
 348             error_report("Received unexpected msg type. "
 349                          "Expected %d received %d",
 350                          VHOST_USER_SET_LOG_BASE, msg.hdr.request);
 351             return -1;
 352         }
 353     }
 354
 355     return 0;
 356 }
 357
 358 static int vhost_user_set_mem_table_postcopy(struct vhost_dev *dev,
 359                                              struct vhost_memory *mem)
 360 {
 361     struct vhost_user *u = dev->opaque;
 362     int fds[VHOST_MEMORY_MAX_NREGIONS];
 363     int i, fd;
 364     size_t fd_num = 0;
 365     bool reply_supported = virtio_has_feature(dev->protocol_features,
 366                                               VHOST_USER_PROTOCOL_F_REPLY_ACK);
 367     VhostUserMsg msg_reply;
 368     int region_i, msg_i;
 369
 370     VhostUserMsg msg = {
 371         .hdr.request = VHOST_USER_SET_MEM_TABLE,
 372         .hdr.flags = VHOST_USER_VERSION,
 373     };
 374
 375     if (reply_supported) {
 376         msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
 377     }
 378
 379     if (u->region_rb_len < dev->mem->nregions) {
 380         u->region_rb = g_renew(RAMBlock*, u->region_rb, dev->mem->nregions);
 381         u->region_rb_offset = g_renew(ram_addr_t, u->region_rb_offset,
 382                                       dev->mem->nregions);
 383         memset(&(u->region_rb[u->region_rb_len]), '\0',
 384                sizeof(RAMBlock *) * (dev->mem->nregions - u->region_rb_len));
 385         memset(&(u->region_rb_offset[u->region_rb_len]), '\0',
 386                sizeof(ram_addr_t) * (dev->mem->nregions - u->region_rb_len));
 387         u->region_rb_len = dev->mem->nregions;
 388     }
 389
 390     for (i = 0; i < dev->mem->nregions; ++i) {
 391         struct vhost_memory_region *reg = dev->mem->regions + i;
 392         ram_addr_t offset;
 393         MemoryRegion *mr;
 394
 395         assert((uintptr_t)reg->userspace_addr == reg->userspace_addr);
 396         mr = memory_region_from_host((void *)(uintptr_t)reg->userspace_addr,
 397                                      &offset);
 398         fd = memory_region_get_fd(mr);
 399         if (fd > 0) {
 400             trace_vhost_user_set_mem_table_withfd(fd_num, mr->name,
 401                                                   reg->memory_size,
 402                                                   reg->guest_phys_addr,
 403                                                   reg->userspace_addr, offset);
 404             u->region_rb_offset[i] = offset;
 405             u->region_rb[i] = mr->ram_block;
 406             msg.payload.memory.regions[fd_num].userspace_addr =
 407                 reg->userspace_addr;
 408             msg.payload.memory.regions[fd_num].memory_size  = reg->memory_size;
 409             msg.payload.memory.regions[fd_num].guest_phys_addr =
 410                 reg->guest_phys_addr;
 411             msg.payload.memory.regions[fd_num].mmap_offset = offset;
 412             assert(fd_num < VHOST_MEMORY_MAX_NREGIONS);
 413             fds[fd_num++] = fd;
 414         } else {
 415             u->region_rb_offset[i] = 0;
 416             u->region_rb[i] = NULL;
 417         }
 418     }
 419
 420     msg.payload.memory.nregions = fd_num;
 421
 422     if (!fd_num) {
 423         error_report("Failed initializing vhost-user memory map, "
 424                      "consider using -object memory-backend-file share=on");
 425         return -1;
 426     }
 427
 428     msg.hdr.size = sizeof(msg.payload.memory.nregions);
 429     msg.hdr.size += sizeof(msg.payload.memory.padding);
 430     msg.hdr.size += fd_num * sizeof(VhostUserMemoryRegion);
 431
 432     if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
 433         return -1;
 434     }
 435
 436     if (vhost_user_read(dev, &msg_reply) < 0) {
 437         return -1;
 438     }
 439
 440     if (msg_reply.hdr.request != VHOST_USER_SET_MEM_TABLE) {
 441         error_report("%s: Received unexpected msg type."
 442                      "Expected %d received %d", __func__,
 443                      VHOST_USER_SET_MEM_TABLE, msg_reply.hdr.request);
 444         return -1;
 445     }
 446     /* We're using the same structure, just reusing one of the
 447      * fields, so it should be the same size.
 448      */
 449     if (msg_reply.hdr.size != msg.hdr.size) {
 450         error_report("%s: Unexpected size for postcopy reply "
 451                      "%d vs %d", __func__, msg_reply.hdr.size, msg.hdr.size);
 452         return -1;
 453     }
 454
 455     memset(u->postcopy_client_bases, 0,
 456            sizeof(uint64_t) * VHOST_MEMORY_MAX_NREGIONS);
 457
 458     /* They're in the same order as the regions that were sent
 459      * but some of the regions were skipped (above) if they
 460      * didn't have fd's
 461     */
 462     for (msg_i = 0, region_i = 0;
 463          region_i < dev->mem->nregions;
 464         region_i++) {
 465         if (msg_i < fd_num &&
 466             msg_reply.payload.memory.regions[msg_i].guest_phys_addr ==
 467             dev->mem->regions[region_i].guest_phys_addr) {
 468             u->postcopy_client_bases[region_i] =
 469                 msg_reply.payload.memory.regions[msg_i].userspace_addr;
 470             trace_vhost_user_set_mem_table_postcopy(
 471                 msg_reply.payload.memory.regions[msg_i].userspace_addr,
 472                 msg.payload.memory.regions[msg_i].userspace_addr,
 473                 msg_i, region_i);
 474             msg_i++;
 475         }
 476     }
 477     if (msg_i != fd_num) {
 478         error_report("%s: postcopy reply not fully consumed "
 479                      "%d vs %zd",
 480                      __func__, msg_i, fd_num);
 481         return -1;
 482     }
 483     /* Now we've registered this with the postcopy code, we ack to the client,
 484      * because now we're in the position to be able to deal with any faults
 485      * it generates.
 486      */
 487     /* TODO: Use this for failure cases as well with a bad value */
 488     msg.hdr.size = sizeof(msg.payload.u64);
 489     msg.payload.u64 = 0; /* OK */
 490     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 491         return -1;
 492     }
 493
 494     if (reply_supported) {
 495         return process_message_reply(dev, &msg);
 496     }
 497
 498     return 0;
 499 }
 500
 501 static int vhost_user_set_mem_table(struct vhost_dev *dev,
 502                                     struct vhost_memory *mem)
 503 {
 504     struct vhost_user *u = dev->opaque;
 505     int fds[VHOST_MEMORY_MAX_NREGIONS];
 506     int i, fd;
 507     size_t fd_num = 0;
 508     bool do_postcopy = u->postcopy_listen && u->postcopy_fd.handler;
 509     bool reply_supported = virtio_has_feature(dev->protocol_features,
 510                                           VHOST_USER_PROTOCOL_F_REPLY_ACK) &&
 511                                           !do_postcopy;
 512
 513     if (do_postcopy) {
 514         /* Postcopy has enough differences that it's best done in it's own
 515          * version
 516          */
 517         return vhost_user_set_mem_table_postcopy(dev, mem);
 518     }
 519
 520     VhostUserMsg msg = {
 521         .hdr.request = VHOST_USER_SET_MEM_TABLE,
 522         .hdr.flags = VHOST_USER_VERSION,
 523     };
 524
 525     if (reply_supported) {
 526         msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
 527     }
 528
 529     for (i = 0; i < dev->mem->nregions; ++i) {
 530         struct vhost_memory_region *reg = dev->mem->regions + i;
 531         ram_addr_t offset;
 532         MemoryRegion *mr;
 533
 534         assert((uintptr_t)reg->userspace_addr == reg->userspace_addr);
 535         mr = memory_region_from_host((void *)(uintptr_t)reg->userspace_addr,
 536                                      &offset);
 537         fd = memory_region_get_fd(mr);
 538         if (fd > 0) {
 539             if (fd_num == VHOST_MEMORY_MAX_NREGIONS) {
 540                 error_report("Failed preparing vhost-user memory table msg");
 541                 return -1;
 542             }
 543             msg.payload.memory.regions[fd_num].userspace_addr =
 544                 reg->userspace_addr;
 545             msg.payload.memory.regions[fd_num].memory_size  = reg->memory_size;
 546             msg.payload.memory.regions[fd_num].guest_phys_addr =
 547                 reg->guest_phys_addr;
 548             msg.payload.memory.regions[fd_num].mmap_offset = offset;
 549             fds[fd_num++] = fd;
 550         }
 551     }
 552
 553     msg.payload.memory.nregions = fd_num;
 554
 555     if (!fd_num) {
 556         error_report("Failed initializing vhost-user memory map, "
 557                      "consider using -object memory-backend-file share=on");
 558         return -1;
 559     }
 560
 561     msg.hdr.size = sizeof(msg.payload.memory.nregions);
 562     msg.hdr.size += sizeof(msg.payload.memory.padding);
 563     msg.hdr.size += fd_num * sizeof(VhostUserMemoryRegion);
 564
 565     if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
 566         return -1;
 567     }
 568
 569     if (reply_supported) {
 570         return process_message_reply(dev, &msg);
 571     }
 572
 573     return 0;
 574 }
 575
 576 static int vhost_user_set_vring_addr(struct vhost_dev *dev,
 577                                      struct vhost_vring_addr *addr)
 578 {
 579     VhostUserMsg msg = {
 580         .hdr.request = VHOST_USER_SET_VRING_ADDR,
 581         .hdr.flags = VHOST_USER_VERSION,
 582         .payload.addr = *addr,
 583         .hdr.size = sizeof(msg.payload.addr),
 584     };
 585
 586     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 587         return -1;
 588     }
 589
 590     return 0;
 591 }
 592
 593 static int vhost_user_set_vring_endian(struct vhost_dev *dev,
 594                                        struct vhost_vring_state *ring)
 595 {
 596     bool cross_endian = virtio_has_feature(dev->protocol_features,
 597                                            VHOST_USER_PROTOCOL_F_CROSS_ENDIAN);
 598     VhostUserMsg msg = {
 599         .hdr.request = VHOST_USER_SET_VRING_ENDIAN,
 600         .hdr.flags = VHOST_USER_VERSION,
 601         .payload.state = *ring,
 602         .hdr.size = sizeof(msg.payload.state),
 603     };
 604
 605     if (!cross_endian) {
 606         error_report("vhost-user trying to send unhandled ioctl");
 607         return -1;
 608     }
 609
 610     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 611         return -1;
 612     }
 613
 614     return 0;
 615 }
 616
 617 static int vhost_set_vring(struct vhost_dev *dev,
 618                            unsigned long int request,
 619                            struct vhost_vring_state *ring)
 620 {
 621     VhostUserMsg msg = {
 622         .hdr.request = request,
 623         .hdr.flags = VHOST_USER_VERSION,
 624         .payload.state = *ring,
 625         .hdr.size = sizeof(msg.payload.state),
 626     };
 627
 628     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 629         return -1;
 630     }
 631
 632     return 0;
 633 }
 634
 635 static int vhost_user_set_vring_num(struct vhost_dev *dev,
 636                                     struct vhost_vring_state *ring)
 637 {
 638     return vhost_set_vring(dev, VHOST_USER_SET_VRING_NUM, ring);
 639 }
 640
 641 static int vhost_user_set_vring_base(struct vhost_dev *dev,
 642                                      struct vhost_vring_state *ring)
 643 {
 644     return vhost_set_vring(dev, VHOST_USER_SET_VRING_BASE, ring);
 645 }
 646
 647 static int vhost_user_set_vring_enable(struct vhost_dev *dev, int enable)
 648 {
 649     int i;
 650
 651     if (!virtio_has_feature(dev->features, VHOST_USER_F_PROTOCOL_FEATURES)) {
 652         return -1;
 653     }
 654
 655     for (i = 0; i < dev->nvqs; ++i) {
 656         struct vhost_vring_state state = {
 657             .index = dev->vq_index + i,
 658             .num   = enable,
 659         };
 660
 661         vhost_set_vring(dev, VHOST_USER_SET_VRING_ENABLE, &state);
 662     }
 663
 664     return 0;
 665 }
 666
 667 static int vhost_user_get_vring_base(struct vhost_dev *dev,
 668                                      struct vhost_vring_state *ring)
 669 {
 670     VhostUserMsg msg = {
 671         .hdr.request = VHOST_USER_GET_VRING_BASE,
 672         .hdr.flags = VHOST_USER_VERSION,
 673         .payload.state = *ring,
 674         .hdr.size = sizeof(msg.payload.state),
 675     };
 676
 677     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 678         return -1;
 679     }
 680
 681     if (vhost_user_read(dev, &msg) < 0) {
 682         return -1;
 683     }
 684
 685     if (msg.hdr.request != VHOST_USER_GET_VRING_BASE) {
 686         error_report("Received unexpected msg type. Expected %d received %d",
 687                      VHOST_USER_GET_VRING_BASE, msg.hdr.request);
 688         return -1;
 689     }
 690
 691     if (msg.hdr.size != sizeof(msg.payload.state)) {
 692         error_report("Received bad msg size.");
 693         return -1;
 694     }
 695
 696     *ring = msg.payload.state;
 697
 698     return 0;
 699 }
 700
 701 static int vhost_set_vring_file(struct vhost_dev *dev,
 702                                 VhostUserRequest request,
 703                                 struct vhost_vring_file *file)
 704 {
 705     int fds[VHOST_MEMORY_MAX_NREGIONS];
 706     size_t fd_num = 0;
 707     VhostUserMsg msg = {
 708         .hdr.request = request,
 709         .hdr.flags = VHOST_USER_VERSION,
 710         .payload.u64 = file->index & VHOST_USER_VRING_IDX_MASK,
 711         .hdr.size = sizeof(msg.payload.u64),
 712     };
 713
 714     if (ioeventfd_enabled() && file->fd > 0) {
 715         fds[fd_num++] = file->fd;
 716     } else {
 717         msg.payload.u64 |= VHOST_USER_VRING_NOFD_MASK;
 718     }
 719
 720     if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
 721         return -1;
 722     }
 723
 724     return 0;
 725 }
 726
 727 static int vhost_user_set_vring_kick(struct vhost_dev *dev,
 728                                      struct vhost_vring_file *file)
 729 {
 730     return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_KICK, file);
 731 }
 732
 733 static int vhost_user_set_vring_call(struct vhost_dev *dev,
 734                                      struct vhost_vring_file *file)
 735 {
 736     return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_CALL, file);
 737 }
 738
 739 static int vhost_user_set_u64(struct vhost_dev *dev, int request, uint64_t u64)
 740 {
 741     VhostUserMsg msg = {
 742         .hdr.request = request,
 743         .hdr.flags = VHOST_USER_VERSION,
 744         .payload.u64 = u64,
 745         .hdr.size = sizeof(msg.payload.u64),
 746     };
 747
 748     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 749         return -1;
 750     }
 751
 752     return 0;
 753 }
 754
 755 static int vhost_user_set_features(struct vhost_dev *dev,
 756                                    uint64_t features)
 757 {
 758     return vhost_user_set_u64(dev, VHOST_USER_SET_FEATURES, features);
 759 }
 760
 761 static int vhost_user_set_protocol_features(struct vhost_dev *dev,
 762                                             uint64_t features)
 763 {
 764     return vhost_user_set_u64(dev, VHOST_USER_SET_PROTOCOL_FEATURES, features);
 765 }
 766
 767 static int vhost_user_get_u64(struct vhost_dev *dev, int request, uint64_t *u64)
 768 {
 769     VhostUserMsg msg = {
 770         .hdr.request = request,
 771         .hdr.flags = VHOST_USER_VERSION,
 772     };
 773
 774     if (vhost_user_one_time_request(request) && dev->vq_index != 0) {
 775         return 0;
 776     }
 777
 778     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 779         return -1;
 780     }
 781
 782     if (vhost_user_read(dev, &msg) < 0) {
 783         return -1;
 784     }
 785
 786     if (msg.hdr.request != request) {
 787         error_report("Received unexpected msg type. Expected %d received %d",
 788                      request, msg.hdr.request);
 789         return -1;
 790     }
 791
 792     if (msg.hdr.size != sizeof(msg.payload.u64)) {
 793         error_report("Received bad msg size.");
 794         return -1;
 795     }
 796
 797     *u64 = msg.payload.u64;
 798
 799     return 0;
 800 }
 801
 802 static int vhost_user_get_features(struct vhost_dev *dev, uint64_t *features)
 803 {
 804     return vhost_user_get_u64(dev, VHOST_USER_GET_FEATURES, features);
 805 }
 806
 807 static int vhost_user_set_owner(struct vhost_dev *dev)
 808 {
 809     VhostUserMsg msg = {
 810         .hdr.request = VHOST_USER_SET_OWNER,
 811         .hdr.flags = VHOST_USER_VERSION,
 812     };
 813
 814     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 815         return -1;
 816     }
 817
 818     return 0;
 819 }
 820
 821 static int vhost_user_reset_device(struct vhost_dev *dev)
 822 {
 823     VhostUserMsg msg = {
 824         .hdr.request = VHOST_USER_RESET_OWNER,
 825         .hdr.flags = VHOST_USER_VERSION,
 826     };
 827
 828     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 829         return -1;
 830     }
 831
 832     return 0;
 833 }
 834
 835 static int vhost_user_slave_handle_config_change(struct vhost_dev *dev)
 836 {
 837     int ret = -1;
 838
 839     if (!dev->config_ops) {
 840         return -1;
 841     }
 842
 843     if (dev->config_ops->vhost_dev_config_notifier) {
 844         ret = dev->config_ops->vhost_dev_config_notifier(dev);
 845     }
 846
 847     return ret;
 848 }
 849
 850 static void slave_read(void *opaque)
 851 {
 852     struct vhost_dev *dev = opaque;
 853     struct vhost_user *u = dev->opaque;
 854     VhostUserHeader hdr = { 0, };
 855     VhostUserPayload payload = { 0, };
 856     int size, ret = 0;
 857     struct iovec iov;
 858     struct msghdr msgh;
 859     int fd[VHOST_USER_SLAVE_MAX_FDS];
 860     char control[CMSG_SPACE(sizeof(fd))];
 861     struct cmsghdr *cmsg;
 862     int i, fdsize = 0;
 863
 864     memset(&msgh, 0, sizeof(msgh));
 865     msgh.msg_iov = &iov;
 866     msgh.msg_iovlen = 1;
 867     msgh.msg_control = control;
 868     msgh.msg_controllen = sizeof(control);
 869
 870     memset(fd, -1, sizeof(fd));
 871
 872     /* Read header */
 873     iov.iov_base = &hdr;
 874     iov.iov_len = VHOST_USER_HDR_SIZE;
 875
 876     size = recvmsg(u->slave_fd, &msgh, 0);
 877     if (size != VHOST_USER_HDR_SIZE) {
 878         error_report("Failed to read from slave.");
 879         goto err;
 880     }
 881
 882     if (msgh.msg_flags & MSG_CTRUNC) {
 883         error_report("Truncated message.");
 884         goto err;
 885     }
 886
 887     for (cmsg = CMSG_FIRSTHDR(&msgh); cmsg != NULL;
 888          cmsg = CMSG_NXTHDR(&msgh, cmsg)) {
 889             if (cmsg->cmsg_level == SOL_SOCKET &&
 890                 cmsg->cmsg_type == SCM_RIGHTS) {
 891                     fdsize = cmsg->cmsg_len - CMSG_LEN(0);
 892                     memcpy(fd, CMSG_DATA(cmsg), fdsize);
 893                     break;
 894             }
 895     }
 896
 897     if (hdr.size > VHOST_USER_PAYLOAD_SIZE) {
 898         error_report("Failed to read msg header."
 899                 " Size %d exceeds the maximum %zu.", hdr.size,
 900                 VHOST_USER_PAYLOAD_SIZE);
 901         goto err;
 902     }
 903
 904     /* Read payload */
 905     size = read(u->slave_fd, &payload, hdr.size);
 906     if (size != hdr.size) {
 907         error_report("Failed to read payload from slave.");
 908         goto err;
 909     }
 910
 911     switch (hdr.request) {
 912     case VHOST_USER_SLAVE_IOTLB_MSG:
 913         ret = vhost_backend_handle_iotlb_msg(dev, &payload.iotlb);
 914         break;
 915     case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG :
 916         ret = vhost_user_slave_handle_config_change(dev);
 917         break;
 918     default:
 919         error_report("Received unexpected msg type.");
 920         ret = -EINVAL;
 921     }
 922
 923     /* Close the remaining file descriptors. */
 924     for (i = 0; i < fdsize; i++) {
 925         if (fd[i] != -1) {
 926             close(fd[i]);
 927         }
 928     }
 929
 930     /*
 931      * REPLY_ACK feature handling. Other reply types has to be managed
 932      * directly in their request handlers.
 933      */
 934     if (hdr.flags & VHOST_USER_NEED_REPLY_MASK) {
 935         struct iovec iovec[2];
 936
 937
 938         hdr.flags &= ~VHOST_USER_NEED_REPLY_MASK;
 939         hdr.flags |= VHOST_USER_REPLY_MASK;
 940
 941         payload.u64 = !!ret;
 942         hdr.size = sizeof(payload.u64);
 943
 944         iovec[0].iov_base = &hdr;
 945         iovec[0].iov_len = VHOST_USER_HDR_SIZE;
 946         iovec[1].iov_base = &payload;
 947         iovec[1].iov_len = hdr.size;
 948
 949         size = writev(u->slave_fd, iovec, ARRAY_SIZE(iovec));
 950         if (size != VHOST_USER_HDR_SIZE + hdr.size) {
 951             error_report("Failed to send msg reply to slave.");
 952             goto err;
 953         }
 954     }
 955
 956     return;
 957
 958 err:
 959     qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL);
 960     close(u->slave_fd);
 961     u->slave_fd = -1;
 962     for (i = 0; i < fdsize; i++) {
 963         if (fd[i] != -1) {
 964             close(fd[i]);
 965         }
 966     }
 967     return;
 968 }
 969
 970 static int vhost_setup_slave_channel(struct vhost_dev *dev)
 971 {
 972     VhostUserMsg msg = {
 973         .hdr.request = VHOST_USER_SET_SLAVE_REQ_FD,
 974         .hdr.flags = VHOST_USER_VERSION,
 975     };
 976     struct vhost_user *u = dev->opaque;
 977     int sv[2], ret = 0;
 978     bool reply_supported = virtio_has_feature(dev->protocol_features,
 979                                               VHOST_USER_PROTOCOL_F_REPLY_ACK);
 980
 981     if (!virtio_has_feature(dev->protocol_features,
 982                             VHOST_USER_PROTOCOL_F_SLAVE_REQ)) {
 983         return 0;
 984     }
 985
 986     if (socketpair(PF_UNIX, SOCK_STREAM, 0, sv) == -1) {
 987         error_report("socketpair() failed");
 988         return -1;
 989     }
 990
 991     u->slave_fd = sv[0];
 992     qemu_set_fd_handler(u->slave_fd, slave_read, NULL, dev);
 993
 994     if (reply_supported) {
 995         msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
 996     }
 997
 998     ret = vhost_user_write(dev, &msg, &sv[1], 1);
 999     if (ret) {
1000         goto out;
1001     }
1002
1003     if (reply_supported) {
1004         ret = process_message_reply(dev, &msg);
1005     }
1006
1007 out:
1008     close(sv[1]);
1009     if (ret) {
1010         qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL);
1011         close(u->slave_fd);
1012         u->slave_fd = -1;
1013     }
1014
1015     return ret;
1016 }
1017
1018 /*
1019  * Called back from the postcopy fault thread when a fault is received on our
1020  * ufd.
1021  * TODO: This is Linux specific
1022  */
1023 static int vhost_user_postcopy_fault_handler(struct PostCopyFD *pcfd,
1024                                              void *ufd)
1025 {
1026     struct vhost_dev *dev = pcfd->data;
1027     struct vhost_user *u = dev->opaque;
1028     struct uffd_msg *msg = ufd;
1029     uint64_t faultaddr = msg->arg.pagefault.address;
1030     RAMBlock *rb = NULL;
1031     uint64_t rb_offset;
1032     int i;
1033
1034     trace_vhost_user_postcopy_fault_handler(pcfd->idstr, faultaddr,
1035                                             dev->mem->nregions);
1036     for (i = 0; i < MIN(dev->mem->nregions, u->region_rb_len); i++) {
1037         trace_vhost_user_postcopy_fault_handler_loop(i,
1038                 u->postcopy_client_bases[i], dev->mem->regions[i].memory_size);
1039         if (faultaddr >= u->postcopy_client_bases[i]) {
1040             /* Ofset of the fault address in the vhost region */
1041             uint64_t region_offset = faultaddr - u->postcopy_client_bases[i];
1042             if (region_offset < dev->mem->regions[i].memory_size) {
1043                 rb_offset = region_offset + u->region_rb_offset[i];
1044                 trace_vhost_user_postcopy_fault_handler_found(i,
1045                         region_offset, rb_offset);
1046                 rb = u->region_rb[i];
1047                 return postcopy_request_shared_page(pcfd, rb, faultaddr,
1048                                                     rb_offset);
1049             }
1050         }
1051     }
1052     error_report("%s: Failed to find region for fault %" PRIx64,
1053                  __func__, faultaddr);
1054     return -1;
1055 }
1056
1057 static int vhost_user_postcopy_waker(struct PostCopyFD *pcfd, RAMBlock *rb,
1058                                      uint64_t offset)
1059 {
1060     struct vhost_dev *dev = pcfd->data;
1061     struct vhost_user *u = dev->opaque;
1062     int i;
1063
1064     trace_vhost_user_postcopy_waker(qemu_ram_get_idstr(rb), offset);
1065
1066     if (!u) {
1067         return 0;
1068     }
1069     /* Translate the offset into an address in the clients address space */
1070     for (i = 0; i < MIN(dev->mem->nregions, u->region_rb_len); i++) {
1071         if (u->region_rb[i] == rb &&
1072             offset >= u->region_rb_offset[i] &&
1073             offset < (u->region_rb_offset[i] +
1074                       dev->mem->regions[i].memory_size)) {
1075             uint64_t client_addr = (offset - u->region_rb_offset[i]) +
1076                                    u->postcopy_client_bases[i];
1077             trace_vhost_user_postcopy_waker_found(client_addr);
1078             return postcopy_wake_shared(pcfd, client_addr, rb);
1079         }
1080     }
1081
1082     trace_vhost_user_postcopy_waker_nomatch(qemu_ram_get_idstr(rb), offset);
1083     return 0;
1084 }
1085
1086 /*
1087  * Called at the start of an inbound postcopy on reception of the
1088  * 'advise' command.
1089  */
1090 static int vhost_user_postcopy_advise(struct vhost_dev *dev, Error **errp)
1091 {
1092     struct vhost_user *u = dev->opaque;
1093     CharBackend *chr = u->chr;
1094     int ufd;
1095     VhostUserMsg msg = {
1096         .hdr.request = VHOST_USER_POSTCOPY_ADVISE,
1097         .hdr.flags = VHOST_USER_VERSION,
1098     };
1099
1100     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1101         error_setg(errp, "Failed to send postcopy_advise to vhost");
1102         return -1;
1103     }
1104
1105     if (vhost_user_read(dev, &msg) < 0) {
1106         error_setg(errp, "Failed to get postcopy_advise reply from vhost");
1107         return -1;
1108     }
1109
1110     if (msg.hdr.request != VHOST_USER_POSTCOPY_ADVISE) {
1111         error_setg(errp, "Unexpected msg type. Expected %d received %d",
1112                      VHOST_USER_POSTCOPY_ADVISE, msg.hdr.request);
1113         return -1;
1114     }
1115
1116     if (msg.hdr.size) {
1117         error_setg(errp, "Received bad msg size.");
1118         return -1;
1119     }
1120     ufd = qemu_chr_fe_get_msgfd(chr);
1121     if (ufd < 0) {
1122         error_setg(errp, "%s: Failed to get ufd", __func__);
1123         return -1;
1124     }
1125     qemu_set_nonblock(ufd);
1126
1127     /* register ufd with userfault thread */
1128     u->postcopy_fd.fd = ufd;
1129     u->postcopy_fd.data = dev;
1130     u->postcopy_fd.handler = vhost_user_postcopy_fault_handler;
1131     u->postcopy_fd.waker = vhost_user_postcopy_waker;
1132     u->postcopy_fd.idstr = "vhost-user"; /* Need to find unique name */
1133     postcopy_register_shared_ufd(&u->postcopy_fd);
1134     return 0;
1135 }
1136
1137 /*
1138  * Called at the switch to postcopy on reception of the 'listen' command.
1139  */
1140 static int vhost_user_postcopy_listen(struct vhost_dev *dev, Error **errp)
1141 {
1142     struct vhost_user *u = dev->opaque;
1143     int ret;
1144     VhostUserMsg msg = {
1145         .hdr.request = VHOST_USER_POSTCOPY_LISTEN,
1146         .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
1147     };
1148     u->postcopy_listen = true;
1149     trace_vhost_user_postcopy_listen();
1150     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1151         error_setg(errp, "Failed to send postcopy_listen to vhost");
1152         return -1;
1153     }
1154
1155     ret = process_message_reply(dev, &msg);
1156     if (ret) {
1157         error_setg(errp, "Failed to receive reply to postcopy_listen");
1158         return ret;
1159     }
1160
1161     return 0;
1162 }
1163
1164 /*
1165  * Called at the end of postcopy
1166  */
1167 static int vhost_user_postcopy_end(struct vhost_dev *dev, Error **errp)
1168 {
1169     VhostUserMsg msg = {
1170         .hdr.request = VHOST_USER_POSTCOPY_END,
1171         .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
1172     };
1173     int ret;
1174     struct vhost_user *u = dev->opaque;
1175
1176     trace_vhost_user_postcopy_end_entry();
1177     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1178         error_setg(errp, "Failed to send postcopy_end to vhost");
1179         return -1;
1180     }
1181
1182     ret = process_message_reply(dev, &msg);
1183     if (ret) {
1184         error_setg(errp, "Failed to receive reply to postcopy_end");
1185         return ret;
1186     }
1187     postcopy_unregister_shared_ufd(&u->postcopy_fd);
1188     u->postcopy_fd.handler = NULL;
1189
1190     trace_vhost_user_postcopy_end_exit();
1191
1192     return 0;
1193 }
1194
1195 static int vhost_user_postcopy_notifier(NotifierWithReturn *notifier,
1196                                         void *opaque)
1197 {
1198     struct PostcopyNotifyData *pnd = opaque;
1199     struct vhost_user *u = container_of(notifier, struct vhost_user,
1200                                          postcopy_notifier);
1201     struct vhost_dev *dev = u->dev;
1202
1203     switch (pnd->reason) {
1204     case POSTCOPY_NOTIFY_PROBE:
1205         if (!virtio_has_feature(dev->protocol_features,
1206                                 VHOST_USER_PROTOCOL_F_PAGEFAULT)) {
1207             /* TODO: Get the device name into this error somehow */
1208             error_setg(pnd->errp,
1209                        "vhost-user backend not capable of postcopy");
1210             return -ENOENT;
1211         }
1212         break;
1213
1214     case POSTCOPY_NOTIFY_INBOUND_ADVISE:
1215         return vhost_user_postcopy_advise(dev, pnd->errp);
1216
1217     case POSTCOPY_NOTIFY_INBOUND_LISTEN:
1218         return vhost_user_postcopy_listen(dev, pnd->errp);
1219
1220     case POSTCOPY_NOTIFY_INBOUND_END:
1221         return vhost_user_postcopy_end(dev, pnd->errp);
1222
1223     default:
1224         /* We ignore notifications we don't know */
1225         break;
1226     }
1227
1228     return 0;
1229 }
1230
1231 static int vhost_user_init(struct vhost_dev *dev, void *opaque)
1232 {
1233     uint64_t features, protocol_features;
1234     struct vhost_user *u;
1235     int err;
1236
1237     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
1238
1239     u = g_new0(struct vhost_user, 1);
1240     u->chr = opaque;
1241     u->slave_fd = -1;
1242     u->dev = dev;
1243     dev->opaque = u;
1244
1245     err = vhost_user_get_features(dev, &features);
1246     if (err < 0) {
1247         return err;
1248     }
1249
1250     if (virtio_has_feature(features, VHOST_USER_F_PROTOCOL_FEATURES)) {
1251         dev->backend_features |= 1ULL << VHOST_USER_F_PROTOCOL_FEATURES;
1252
1253         err = vhost_user_get_u64(dev, VHOST_USER_GET_PROTOCOL_FEATURES,
1254                                  &protocol_features);
1255         if (err < 0) {
1256             return err;
1257         }
1258
1259         dev->protocol_features =
1260             protocol_features & VHOST_USER_PROTOCOL_FEATURE_MASK;
1261
1262         if (!dev->config_ops || !dev->config_ops->vhost_dev_config_notifier) {
1263             /* Don't acknowledge CONFIG feature if device doesn't support it */
1264             dev->protocol_features &= ~(1ULL << VHOST_USER_PROTOCOL_F_CONFIG);
1265         } else if (!(protocol_features &
1266                     (1ULL << VHOST_USER_PROTOCOL_F_CONFIG))) {
1267             error_report("Device expects VHOST_USER_PROTOCOL_F_CONFIG "
1268                     "but backend does not support it.");
1269             return -1;
1270         }
1271
1272         err = vhost_user_set_protocol_features(dev, dev->protocol_features);
1273         if (err < 0) {
1274             return err;
1275         }
1276
1277         /* query the max queues we support if backend supports Multiple Queue */
1278         if (dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_MQ)) {
1279             err = vhost_user_get_u64(dev, VHOST_USER_GET_QUEUE_NUM,
1280                                      &dev->max_queues);
1281             if (err < 0) {
1282                 return err;
1283             }
1284         }
1285
1286         if (virtio_has_feature(features, VIRTIO_F_IOMMU_PLATFORM) &&
1287                 !(virtio_has_feature(dev->protocol_features,
1288                     VHOST_USER_PROTOCOL_F_SLAVE_REQ) &&
1289                  virtio_has_feature(dev->protocol_features,
1290                     VHOST_USER_PROTOCOL_F_REPLY_ACK))) {
1291             error_report("IOMMU support requires reply-ack and "
1292                          "slave-req protocol features.");
1293             return -1;
1294         }
1295     }
1296
1297     if (dev->migration_blocker == NULL &&
1298         !virtio_has_feature(dev->protocol_features,
1299                             VHOST_USER_PROTOCOL_F_LOG_SHMFD)) {
1300         error_setg(&dev->migration_blocker,
1301                    "Migration disabled: vhost-user backend lacks "
1302                    "VHOST_USER_PROTOCOL_F_LOG_SHMFD feature.");
1303     }
1304
1305     err = vhost_setup_slave_channel(dev);
1306     if (err < 0) {
1307         return err;
1308     }
1309
1310     u->postcopy_notifier.notify = vhost_user_postcopy_notifier;
1311     postcopy_add_notifier(&u->postcopy_notifier);
1312
1313     return 0;
1314 }
1315
1316 static int vhost_user_cleanup(struct vhost_dev *dev)
1317 {
1318     struct vhost_user *u;
1319
1320     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
1321
1322     u = dev->opaque;
1323     if (u->postcopy_notifier.notify) {
1324         postcopy_remove_notifier(&u->postcopy_notifier);
1325         u->postcopy_notifier.notify = NULL;
1326     }
1327     if (u->slave_fd >= 0) {
1328         qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL);
1329         close(u->slave_fd);
1330         u->slave_fd = -1;
1331     }
1332     g_free(u->region_rb);
1333     u->region_rb = NULL;
1334     g_free(u->region_rb_offset);
1335     u->region_rb_offset = NULL;
1336     u->region_rb_len = 0;
1337     g_free(u);
1338     dev->opaque = 0;
1339
1340     return 0;
1341 }
1342
1343 static int vhost_user_get_vq_index(struct vhost_dev *dev, int idx)
1344 {
1345     assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
1346
1347     return idx;
1348 }
1349
1350 static int vhost_user_memslots_limit(struct vhost_dev *dev)
1351 {
1352     return VHOST_MEMORY_MAX_NREGIONS;
1353 }
1354
1355 static bool vhost_user_requires_shm_log(struct vhost_dev *dev)
1356 {
1357     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
1358
1359     return virtio_has_feature(dev->protocol_features,
1360                               VHOST_USER_PROTOCOL_F_LOG_SHMFD);
1361 }
1362
1363 static int vhost_user_migration_done(struct vhost_dev *dev, char* mac_addr)
1364 {
1365     VhostUserMsg msg = { };
1366
1367     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
1368
1369     /* If guest supports GUEST_ANNOUNCE do nothing */
1370     if (virtio_has_feature(dev->acked_features, VIRTIO_NET_F_GUEST_ANNOUNCE)) {
1371         return 0;
1372     }
1373
1374     /* if backend supports VHOST_USER_PROTOCOL_F_RARP ask it to send the RARP */
1375     if (virtio_has_feature(dev->protocol_features,
1376                            VHOST_USER_PROTOCOL_F_RARP)) {
1377         msg.hdr.request = VHOST_USER_SEND_RARP;
1378         msg.hdr.flags = VHOST_USER_VERSION;
1379         memcpy((char *)&msg.payload.u64, mac_addr, 6);
1380         msg.hdr.size = sizeof(msg.payload.u64);
1381
1382         return vhost_user_write(dev, &msg, NULL, 0);
1383     }
1384     return -1;
1385 }
1386
1387 static bool vhost_user_can_merge(struct vhost_dev *dev,
1388                                  uint64_t start1, uint64_t size1,
1389                                  uint64_t start2, uint64_t size2)
1390 {
1391     ram_addr_t offset;
1392     int mfd, rfd;
1393     MemoryRegion *mr;
1394
1395     mr = memory_region_from_host((void *)(uintptr_t)start1, &offset);
1396     mfd = memory_region_get_fd(mr);
1397
1398     mr = memory_region_from_host((void *)(uintptr_t)start2, &offset);
1399     rfd = memory_region_get_fd(mr);
1400
1401     return mfd == rfd;
1402 }
1403
1404 static int vhost_user_net_set_mtu(struct vhost_dev *dev, uint16_t mtu)
1405 {
1406     VhostUserMsg msg;
1407     bool reply_supported = virtio_has_feature(dev->protocol_features,
1408                                               VHOST_USER_PROTOCOL_F_REPLY_ACK);
1409
1410     if (!(dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_NET_MTU))) {
1411         return 0;
1412     }
1413
1414     msg.hdr.request = VHOST_USER_NET_SET_MTU;
1415     msg.payload.u64 = mtu;
1416     msg.hdr.size = sizeof(msg.payload.u64);
1417     msg.hdr.flags = VHOST_USER_VERSION;
1418     if (reply_supported) {
1419         msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
1420     }
1421
1422     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1423         return -1;
1424     }
1425
1426     /* If reply_ack supported, slave has to ack specified MTU is valid */
1427     if (reply_supported) {
1428         return process_message_reply(dev, &msg);
1429     }
1430
1431     return 0;
1432 }
1433
1434 static int vhost_user_send_device_iotlb_msg(struct vhost_dev *dev,
1435                                             struct vhost_iotlb_msg *imsg)
1436 {
1437     VhostUserMsg msg = {
1438         .hdr.request = VHOST_USER_IOTLB_MSG,
1439         .hdr.size = sizeof(msg.payload.iotlb),
1440         .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
1441         .payload.iotlb = *imsg,
1442     };
1443
1444     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1445         return -EFAULT;
1446     }
1447
1448     return process_message_reply(dev, &msg);
1449 }
1450
1451
1452 static void vhost_user_set_iotlb_callback(struct vhost_dev *dev, int enabled)
1453 {
1454     /* No-op as the receive channel is not dedicated to IOTLB messages. */
1455 }
1456
1457 static int vhost_user_get_config(struct vhost_dev *dev, uint8_t *config,
1458                                  uint32_t config_len)
1459 {
1460     VhostUserMsg msg = {
1461         .hdr.request = VHOST_USER_GET_CONFIG,
1462         .hdr.flags = VHOST_USER_VERSION,
1463         .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + config_len,
1464     };
1465
1466     if (!virtio_has_feature(dev->protocol_features,
1467                 VHOST_USER_PROTOCOL_F_CONFIG)) {
1468         return -1;
1469     }
1470
1471     if (config_len > VHOST_USER_MAX_CONFIG_SIZE) {
1472         return -1;
1473     }
1474
1475     msg.payload.config.offset = 0;
1476     msg.payload.config.size = config_len;
1477     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1478         return -1;
1479     }
1480
1481     if (vhost_user_read(dev, &msg) < 0) {
1482         return -1;
1483     }
1484
1485     if (msg.hdr.request != VHOST_USER_GET_CONFIG) {
1486         error_report("Received unexpected msg type. Expected %d received %d",
1487                      VHOST_USER_GET_CONFIG, msg.hdr.request);
1488         return -1;
1489     }
1490
1491     if (msg.hdr.size != VHOST_USER_CONFIG_HDR_SIZE + config_len) {
1492         error_report("Received bad msg size.");
1493         return -1;
1494     }
1495
1496     memcpy(config, msg.payload.config.region, config_len);
1497
1498     return 0;
1499 }
1500
1501 static int vhost_user_set_config(struct vhost_dev *dev, const uint8_t *data,
1502                                  uint32_t offset, uint32_t size, uint32_t flags)
1503 {
1504     uint8_t *p;
1505     bool reply_supported = virtio_has_feature(dev->protocol_features,
1506                                               VHOST_USER_PROTOCOL_F_REPLY_ACK);
1507
1508     VhostUserMsg msg = {
1509         .hdr.request = VHOST_USER_SET_CONFIG,
1510         .hdr.flags = VHOST_USER_VERSION,
1511         .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + size,
1512     };
1513
1514     if (!virtio_has_feature(dev->protocol_features,
1515                 VHOST_USER_PROTOCOL_F_CONFIG)) {
1516         return -1;
1517     }
1518
1519     if (reply_supported) {
1520         msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
1521     }
1522
1523     if (size > VHOST_USER_MAX_CONFIG_SIZE) {
1524         return -1;
1525     }
1526
1527     msg.payload.config.offset = offset,
1528     msg.payload.config.size = size,
1529     msg.payload.config.flags = flags,
1530     p = msg.payload.config.region;
1531     memcpy(p, data, size);
1532
1533     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1534         return -1;
1535     }
1536
1537     if (reply_supported) {
1538         return process_message_reply(dev, &msg);
1539     }
1540
1541     return 0;
1542 }
1543
1544 static int vhost_user_crypto_create_session(struct vhost_dev *dev,
1545                                             void *session_info,
1546                                             uint64_t *session_id)
1547 {
1548     bool crypto_session = virtio_has_feature(dev->protocol_features,
1549                                        VHOST_USER_PROTOCOL_F_CRYPTO_SESSION);
1550     CryptoDevBackendSymSessionInfo *sess_info = session_info;
1551     VhostUserMsg msg = {
1552         .hdr.request = VHOST_USER_CREATE_CRYPTO_SESSION,
1553         .hdr.flags = VHOST_USER_VERSION,
1554         .hdr.size = sizeof(msg.payload.session),
1555     };
1556
1557     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
1558
1559     if (!crypto_session) {
1560         error_report("vhost-user trying to send unhandled ioctl");
1561         return -1;
1562     }
1563
1564     memcpy(&msg.payload.session.session_setup_data, sess_info,
1565               sizeof(CryptoDevBackendSymSessionInfo));
1566     if (sess_info->key_len) {
1567         memcpy(&msg.payload.session.key, sess_info->cipher_key,
1568                sess_info->key_len);
1569     }
1570     if (sess_info->auth_key_len > 0) {
1571         memcpy(&msg.payload.session.auth_key, sess_info->auth_key,
1572                sess_info->auth_key_len);
1573     }
1574     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1575         error_report("vhost_user_write() return -1, create session failed");
1576         return -1;
1577     }
1578
1579     if (vhost_user_read(dev, &msg) < 0) {
1580         error_report("vhost_user_read() return -1, create session failed");
1581         return -1;
1582     }
1583
1584     if (msg.hdr.request != VHOST_USER_CREATE_CRYPTO_SESSION) {
1585         error_report("Received unexpected msg type. Expected %d received %d",
1586                      VHOST_USER_CREATE_CRYPTO_SESSION, msg.hdr.request);
1587         return -1;
1588     }
1589
1590     if (msg.hdr.size != sizeof(msg.payload.session)) {
1591         error_report("Received bad msg size.");
1592         return -1;
1593     }
1594
1595     if (msg.payload.session.session_id < 0) {
1596         error_report("Bad session id: %" PRId64 "",
1597                               msg.payload.session.session_id);
1598         return -1;
1599     }
1600     *session_id = msg.payload.session.session_id;
1601
1602     return 0;
1603 }
1604
1605 static int
1606 vhost_user_crypto_close_session(struct vhost_dev *dev, uint64_t session_id)
1607 {
1608     bool crypto_session = virtio_has_feature(dev->protocol_features,
1609                                        VHOST_USER_PROTOCOL_F_CRYPTO_SESSION);
1610     VhostUserMsg msg = {
1611         .hdr.request = VHOST_USER_CLOSE_CRYPTO_SESSION,
1612         .hdr.flags = VHOST_USER_VERSION,
1613         .hdr.size = sizeof(msg.payload.u64),
1614     };
1615     msg.payload.u64 = session_id;
1616
1617     if (!crypto_session) {
1618         error_report("vhost-user trying to send unhandled ioctl");
1619         return -1;
1620     }
1621
1622     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1623         error_report("vhost_user_write() return -1, close session failed");
1624         return -1;
1625     }
1626
1627     return 0;
1628 }
1629
1630 static bool vhost_user_mem_section_filter(struct vhost_dev *dev,
1631                                           MemoryRegionSection *section)
1632 {
1633     bool result;
1634
1635     result = memory_region_get_fd(section->mr) >= 0;
1636
1637     return result;
1638 }
1639
1640 const VhostOps user_ops = {
1641         .backend_type = VHOST_BACKEND_TYPE_USER,
1642         .vhost_backend_init = vhost_user_init,
1643         .vhost_backend_cleanup = vhost_user_cleanup,
1644         .vhost_backend_memslots_limit = vhost_user_memslots_limit,
1645         .vhost_set_log_base = vhost_user_set_log_base,
1646         .vhost_set_mem_table = vhost_user_set_mem_table,
1647         .vhost_set_vring_addr = vhost_user_set_vring_addr,
1648         .vhost_set_vring_endian = vhost_user_set_vring_endian,
1649         .vhost_set_vring_num = vhost_user_set_vring_num,
1650         .vhost_set_vring_base = vhost_user_set_vring_base,
1651         .vhost_get_vring_base = vhost_user_get_vring_base,
1652         .vhost_set_vring_kick = vhost_user_set_vring_kick,
1653         .vhost_set_vring_call = vhost_user_set_vring_call,
1654         .vhost_set_features = vhost_user_set_features,
1655         .vhost_get_features = vhost_user_get_features,
1656         .vhost_set_owner = vhost_user_set_owner,
1657         .vhost_reset_device = vhost_user_reset_device,
1658         .vhost_get_vq_index = vhost_user_get_vq_index,
1659         .vhost_set_vring_enable = vhost_user_set_vring_enable,
1660         .vhost_requires_shm_log = vhost_user_requires_shm_log,
1661         .vhost_migration_done = vhost_user_migration_done,
1662         .vhost_backend_can_merge = vhost_user_can_merge,
1663         .vhost_net_set_mtu = vhost_user_net_set_mtu,
1664         .vhost_set_iotlb_callback = vhost_user_set_iotlb_callback,
1665         .vhost_send_device_iotlb_msg = vhost_user_send_device_iotlb_msg,
1666         .vhost_get_config = vhost_user_get_config,
1667         .vhost_set_config = vhost_user_set_config,
1668         .vhost_crypto_create_session = vhost_user_crypto_create_session,
1669         .vhost_crypto_close_session = vhost_user_crypto_close_session,
1670         .vhost_backend_mem_section_filter = vhost_user_mem_section_filter,
1671 };