hw/virtio/vhost-user.c

   1 /*
   2  * vhost-user
   3  *
   4  * Copyright (c) 2013 Virtual Open Systems Sarl.
   5  *
   6  * This work is licensed under the terms of the GNU GPL, version 2 or later.
   7  * See the COPYING file in the top-level directory.
   8  *
   9  */
  10
  11 #include "qemu/osdep.h"
  12 #include "qapi/error.h"
  13 #include "hw/virtio/vhost.h"
  14 #include "hw/virtio/vhost-backend.h"
  15 #include "hw/virtio/virtio-net.h"
  16 #include "chardev/char-fe.h"
  17 #include "sysemu/kvm.h"
  18 #include "qemu/error-report.h"
  19 #include "qemu/sockets.h"
  20 #include "sysemu/cryptodev.h"
  21
  22 #include <sys/ioctl.h>
  23 #include <sys/socket.h>
  24 #include <sys/un.h>
  25 #include <linux/vhost.h>
  26
  27 #define VHOST_MEMORY_MAX_NREGIONS    8
  28 #define VHOST_USER_F_PROTOCOL_FEATURES 30
  29
  30 /*
  31  * Maximum size of virtio device config space
  32  */
  33 #define VHOST_USER_MAX_CONFIG_SIZE 256
  34
  35 enum VhostUserProtocolFeature {
  36     VHOST_USER_PROTOCOL_F_MQ = 0,
  37     VHOST_USER_PROTOCOL_F_LOG_SHMFD = 1,
  38     VHOST_USER_PROTOCOL_F_RARP = 2,
  39     VHOST_USER_PROTOCOL_F_REPLY_ACK = 3,
  40     VHOST_USER_PROTOCOL_F_NET_MTU = 4,
  41     VHOST_USER_PROTOCOL_F_SLAVE_REQ = 5,
  42     VHOST_USER_PROTOCOL_F_CROSS_ENDIAN = 6,
  43     VHOST_USER_PROTOCOL_F_CRYPTO_SESSION = 7,
  44
  45     VHOST_USER_PROTOCOL_F_MAX
  46 };
  47
  48 #define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1)
  49
  50 typedef enum VhostUserRequest {
  51     VHOST_USER_NONE = 0,
  52     VHOST_USER_GET_FEATURES = 1,
  53     VHOST_USER_SET_FEATURES = 2,
  54     VHOST_USER_SET_OWNER = 3,
  55     VHOST_USER_RESET_OWNER = 4,
  56     VHOST_USER_SET_MEM_TABLE = 5,
  57     VHOST_USER_SET_LOG_BASE = 6,
  58     VHOST_USER_SET_LOG_FD = 7,
  59     VHOST_USER_SET_VRING_NUM = 8,
  60     VHOST_USER_SET_VRING_ADDR = 9,
  61     VHOST_USER_SET_VRING_BASE = 10,
  62     VHOST_USER_GET_VRING_BASE = 11,
  63     VHOST_USER_SET_VRING_KICK = 12,
  64     VHOST_USER_SET_VRING_CALL = 13,
  65     VHOST_USER_SET_VRING_ERR = 14,
  66     VHOST_USER_GET_PROTOCOL_FEATURES = 15,
  67     VHOST_USER_SET_PROTOCOL_FEATURES = 16,
  68     VHOST_USER_GET_QUEUE_NUM = 17,
  69     VHOST_USER_SET_VRING_ENABLE = 18,
  70     VHOST_USER_SEND_RARP = 19,
  71     VHOST_USER_NET_SET_MTU = 20,
  72     VHOST_USER_SET_SLAVE_REQ_FD = 21,
  73     VHOST_USER_IOTLB_MSG = 22,
  74     VHOST_USER_SET_VRING_ENDIAN = 23,
  75     VHOST_USER_GET_CONFIG = 24,
  76     VHOST_USER_SET_CONFIG = 25,
  77     VHOST_USER_CREATE_CRYPTO_SESSION = 26,
  78     VHOST_USER_CLOSE_CRYPTO_SESSION = 27,
  79     VHOST_USER_MAX
  80 } VhostUserRequest;
  81
  82 typedef enum VhostUserSlaveRequest {
  83     VHOST_USER_SLAVE_NONE = 0,
  84     VHOST_USER_SLAVE_IOTLB_MSG = 1,
  85     VHOST_USER_SLAVE_CONFIG_CHANGE_MSG = 2,
  86     VHOST_USER_SLAVE_MAX
  87 }  VhostUserSlaveRequest;
  88
  89 typedef struct VhostUserMemoryRegion {
  90     uint64_t guest_phys_addr;
  91     uint64_t memory_size;
  92     uint64_t userspace_addr;
  93     uint64_t mmap_offset;
  94 } VhostUserMemoryRegion;
  95
  96 typedef struct VhostUserMemory {
  97     uint32_t nregions;
  98     uint32_t padding;
  99     VhostUserMemoryRegion regions[VHOST_MEMORY_MAX_NREGIONS];
 100 } VhostUserMemory;
 101
 102 typedef struct VhostUserLog {
 103     uint64_t mmap_size;
 104     uint64_t mmap_offset;
 105 } VhostUserLog;
 106
 107 typedef struct VhostUserConfig {
 108     uint32_t offset;
 109     uint32_t size;
 110     uint32_t flags;
 111     uint8_t region[VHOST_USER_MAX_CONFIG_SIZE];
 112 } VhostUserConfig;
 113
 114 #define VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN    512
 115 #define VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN  64
 116
 117 typedef struct VhostUserCryptoSession {
 118     /* session id for success, -1 on errors */
 119     int64_t session_id;
 120     CryptoDevBackendSymSessionInfo session_setup_data;
 121     uint8_t key[VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN];
 122     uint8_t auth_key[VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN];
 123 } VhostUserCryptoSession;
 124
 125 static VhostUserConfig c __attribute__ ((unused));
 126 #define VHOST_USER_CONFIG_HDR_SIZE (sizeof(c.offset) \
 127                                    + sizeof(c.size) \
 128                                    + sizeof(c.flags))
 129
 130 typedef struct {
 131     VhostUserRequest request;
 132
 133 #define VHOST_USER_VERSION_MASK     (0x3)
 134 #define VHOST_USER_REPLY_MASK       (0x1<<2)
 135 #define VHOST_USER_NEED_REPLY_MASK  (0x1 << 3)
 136     uint32_t flags;
 137     uint32_t size; /* the following payload size */
 138 } QEMU_PACKED VhostUserHeader;
 139
 140 typedef union {
 141 #define VHOST_USER_VRING_IDX_MASK   (0xff)
 142 #define VHOST_USER_VRING_NOFD_MASK  (0x1<<8)
 143         uint64_t u64;
 144         struct vhost_vring_state state;
 145         struct vhost_vring_addr addr;
 146         VhostUserMemory memory;
 147         VhostUserLog log;
 148         struct vhost_iotlb_msg iotlb;
 149         VhostUserConfig config;
 150         VhostUserCryptoSession session;
 151 } VhostUserPayload;
 152
 153 typedef struct VhostUserMsg {
 154     VhostUserHeader hdr;
 155     VhostUserPayload payload;
 156 } QEMU_PACKED VhostUserMsg;
 157
 158 static VhostUserMsg m __attribute__ ((unused));
 159 #define VHOST_USER_HDR_SIZE (sizeof(VhostUserHeader))
 160
 161 #define VHOST_USER_PAYLOAD_SIZE (sizeof(VhostUserPayload))
 162
 163 /* The version of the protocol we support */
 164 #define VHOST_USER_VERSION    (0x1)
 165
 166 struct vhost_user {
 167     CharBackend *chr;
 168     int slave_fd;
 169 };
 170
 171 static bool ioeventfd_enabled(void)
 172 {
 173     return kvm_enabled() && kvm_eventfds_enabled();
 174 }
 175
 176 static int vhost_user_read(struct vhost_dev *dev, VhostUserMsg *msg)
 177 {
 178     struct vhost_user *u = dev->opaque;
 179     CharBackend *chr = u->chr;
 180     uint8_t *p = (uint8_t *) msg;
 181     int r, size = VHOST_USER_HDR_SIZE;
 182
 183     r = qemu_chr_fe_read_all(chr, p, size);
 184     if (r != size) {
 185         error_report("Failed to read msg header. Read %d instead of %d."
 186                      " Original request %d.", r, size, msg->hdr.request);
 187         goto fail;
 188     }
 189
 190     /* validate received flags */
 191     if (msg->hdr.flags != (VHOST_USER_REPLY_MASK | VHOST_USER_VERSION)) {
 192         error_report("Failed to read msg header."
 193                 " Flags 0x%x instead of 0x%x.", msg->hdr.flags,
 194                 VHOST_USER_REPLY_MASK | VHOST_USER_VERSION);
 195         goto fail;
 196     }
 197
 198     /* validate message size is sane */
 199     if (msg->hdr.size > VHOST_USER_PAYLOAD_SIZE) {
 200         error_report("Failed to read msg header."
 201                 " Size %d exceeds the maximum %zu.", msg->hdr.size,
 202                 VHOST_USER_PAYLOAD_SIZE);
 203         goto fail;
 204     }
 205
 206     if (msg->hdr.size) {
 207         p += VHOST_USER_HDR_SIZE;
 208         size = msg->hdr.size;
 209         r = qemu_chr_fe_read_all(chr, p, size);
 210         if (r != size) {
 211             error_report("Failed to read msg payload."
 212                          " Read %d instead of %d.", r, msg->hdr.size);
 213             goto fail;
 214         }
 215     }
 216
 217     return 0;
 218
 219 fail:
 220     return -1;
 221 }
 222
 223 static int process_message_reply(struct vhost_dev *dev,
 224                                  const VhostUserMsg *msg)
 225 {
 226     VhostUserMsg msg_reply;
 227
 228     if ((msg->hdr.flags & VHOST_USER_NEED_REPLY_MASK) == 0) {
 229         return 0;
 230     }
 231
 232     if (vhost_user_read(dev, &msg_reply) < 0) {
 233         return -1;
 234     }
 235
 236     if (msg_reply.hdr.request != msg->hdr.request) {
 237         error_report("Received unexpected msg type."
 238                      "Expected %d received %d",
 239                      msg->hdr.request, msg_reply.hdr.request);
 240         return -1;
 241     }
 242
 243     return msg_reply.payload.u64 ? -1 : 0;
 244 }
 245
 246 static bool vhost_user_one_time_request(VhostUserRequest request)
 247 {
 248     switch (request) {
 249     case VHOST_USER_SET_OWNER:
 250     case VHOST_USER_RESET_OWNER:
 251     case VHOST_USER_SET_MEM_TABLE:
 252     case VHOST_USER_GET_QUEUE_NUM:
 253     case VHOST_USER_NET_SET_MTU:
 254         return true;
 255     default:
 256         return false;
 257     }
 258 }
 259
 260 /* most non-init callers ignore the error */
 261 static int vhost_user_write(struct vhost_dev *dev, VhostUserMsg *msg,
 262                             int *fds, int fd_num)
 263 {
 264     struct vhost_user *u = dev->opaque;
 265     CharBackend *chr = u->chr;
 266     int ret, size = VHOST_USER_HDR_SIZE + msg->hdr.size;
 267
 268     /*
 269      * For non-vring specific requests, like VHOST_USER_SET_MEM_TABLE,
 270      * we just need send it once in the first time. For later such
 271      * request, we just ignore it.
 272      */
 273     if (vhost_user_one_time_request(msg->hdr.request) && dev->vq_index != 0) {
 274         msg->hdr.flags &= ~VHOST_USER_NEED_REPLY_MASK;
 275         return 0;
 276     }
 277
 278     if (qemu_chr_fe_set_msgfds(chr, fds, fd_num) < 0) {
 279         error_report("Failed to set msg fds.");
 280         return -1;
 281     }
 282
 283     ret = qemu_chr_fe_write_all(chr, (const uint8_t *) msg, size);
 284     if (ret != size) {
 285         error_report("Failed to write msg."
 286                      " Wrote %d instead of %d.", ret, size);
 287         return -1;
 288     }
 289
 290     return 0;
 291 }
 292
 293 static int vhost_user_set_log_base(struct vhost_dev *dev, uint64_t base,
 294                                    struct vhost_log *log)
 295 {
 296     int fds[VHOST_MEMORY_MAX_NREGIONS];
 297     size_t fd_num = 0;
 298     bool shmfd = virtio_has_feature(dev->protocol_features,
 299                                     VHOST_USER_PROTOCOL_F_LOG_SHMFD);
 300     VhostUserMsg msg = {
 301         .hdr.request = VHOST_USER_SET_LOG_BASE,
 302         .hdr.flags = VHOST_USER_VERSION,
 303         .payload.log.mmap_size = log->size * sizeof(*(log->log)),
 304         .payload.log.mmap_offset = 0,
 305         .hdr.size = sizeof(msg.payload.log),
 306     };
 307
 308     if (shmfd && log->fd != -1) {
 309         fds[fd_num++] = log->fd;
 310     }
 311
 312     if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
 313         return -1;
 314     }
 315
 316     if (shmfd) {
 317         msg.hdr.size = 0;
 318         if (vhost_user_read(dev, &msg) < 0) {
 319             return -1;
 320         }
 321
 322         if (msg.hdr.request != VHOST_USER_SET_LOG_BASE) {
 323             error_report("Received unexpected msg type. "
 324                          "Expected %d received %d",
 325                          VHOST_USER_SET_LOG_BASE, msg.hdr.request);
 326             return -1;
 327         }
 328     }
 329
 330     return 0;
 331 }
 332
 333 static int vhost_user_set_mem_table(struct vhost_dev *dev,
 334                                     struct vhost_memory *mem)
 335 {
 336     int fds[VHOST_MEMORY_MAX_NREGIONS];
 337     int i, fd;
 338     size_t fd_num = 0;
 339     bool reply_supported = virtio_has_feature(dev->protocol_features,
 340                                               VHOST_USER_PROTOCOL_F_REPLY_ACK);
 341
 342     VhostUserMsg msg = {
 343         .hdr.request = VHOST_USER_SET_MEM_TABLE,
 344         .hdr.flags = VHOST_USER_VERSION,
 345     };
 346
 347     if (reply_supported) {
 348         msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
 349     }
 350
 351     for (i = 0; i < dev->mem->nregions; ++i) {
 352         struct vhost_memory_region *reg = dev->mem->regions + i;
 353         ram_addr_t offset;
 354         MemoryRegion *mr;
 355
 356         assert((uintptr_t)reg->userspace_addr == reg->userspace_addr);
 357         mr = memory_region_from_host((void *)(uintptr_t)reg->userspace_addr,
 358                                      &offset);
 359         fd = memory_region_get_fd(mr);
 360         if (fd > 0) {
 361             if (fd_num == VHOST_MEMORY_MAX_NREGIONS) {
 362                 error_report("Failed preparing vhost-user memory table msg");
 363                 return -1;
 364             }
 365             msg.payload.memory.regions[fd_num].userspace_addr = reg->userspace_addr;
 366             msg.payload.memory.regions[fd_num].memory_size  = reg->memory_size;
 367             msg.payload.memory.regions[fd_num].guest_phys_addr = reg->guest_phys_addr;
 368             msg.payload.memory.regions[fd_num].mmap_offset = offset;
 369             fds[fd_num++] = fd;
 370         }
 371     }
 372
 373     msg.payload.memory.nregions = fd_num;
 374
 375     if (!fd_num) {
 376         error_report("Failed initializing vhost-user memory map, "
 377                      "consider using -object memory-backend-file share=on");
 378         return -1;
 379     }
 380
 381     msg.hdr.size = sizeof(msg.payload.memory.nregions);
 382     msg.hdr.size += sizeof(msg.payload.memory.padding);
 383     msg.hdr.size += fd_num * sizeof(VhostUserMemoryRegion);
 384
 385     if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
 386         return -1;
 387     }
 388
 389     if (reply_supported) {
 390         return process_message_reply(dev, &msg);
 391     }
 392
 393     return 0;
 394 }
 395
 396 static int vhost_user_set_vring_addr(struct vhost_dev *dev,
 397                                      struct vhost_vring_addr *addr)
 398 {
 399     VhostUserMsg msg = {
 400         .hdr.request = VHOST_USER_SET_VRING_ADDR,
 401         .hdr.flags = VHOST_USER_VERSION,
 402         .payload.addr = *addr,
 403         .hdr.size = sizeof(msg.payload.addr),
 404     };
 405
 406     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 407         return -1;
 408     }
 409
 410     return 0;
 411 }
 412
 413 static int vhost_user_set_vring_endian(struct vhost_dev *dev,
 414                                        struct vhost_vring_state *ring)
 415 {
 416     bool cross_endian = virtio_has_feature(dev->protocol_features,
 417                                            VHOST_USER_PROTOCOL_F_CROSS_ENDIAN);
 418     VhostUserMsg msg = {
 419         .hdr.request = VHOST_USER_SET_VRING_ENDIAN,
 420         .hdr.flags = VHOST_USER_VERSION,
 421         .payload.state = *ring,
 422         .hdr.size = sizeof(msg.payload.state),
 423     };
 424
 425     if (!cross_endian) {
 426         error_report("vhost-user trying to send unhandled ioctl");
 427         return -1;
 428     }
 429
 430     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 431         return -1;
 432     }
 433
 434     return 0;
 435 }
 436
 437 static int vhost_set_vring(struct vhost_dev *dev,
 438                            unsigned long int request,
 439                            struct vhost_vring_state *ring)
 440 {
 441     VhostUserMsg msg = {
 442         .hdr.request = request,
 443         .hdr.flags = VHOST_USER_VERSION,
 444         .payload.state = *ring,
 445         .hdr.size = sizeof(msg.payload.state),
 446     };
 447
 448     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 449         return -1;
 450     }
 451
 452     return 0;
 453 }
 454
 455 static int vhost_user_set_vring_num(struct vhost_dev *dev,
 456                                     struct vhost_vring_state *ring)
 457 {
 458     return vhost_set_vring(dev, VHOST_USER_SET_VRING_NUM, ring);
 459 }
 460
 461 static int vhost_user_set_vring_base(struct vhost_dev *dev,
 462                                      struct vhost_vring_state *ring)
 463 {
 464     return vhost_set_vring(dev, VHOST_USER_SET_VRING_BASE, ring);
 465 }
 466
 467 static int vhost_user_set_vring_enable(struct vhost_dev *dev, int enable)
 468 {
 469     int i;
 470
 471     if (!virtio_has_feature(dev->features, VHOST_USER_F_PROTOCOL_FEATURES)) {
 472         return -1;
 473     }
 474
 475     for (i = 0; i < dev->nvqs; ++i) {
 476         struct vhost_vring_state state = {
 477             .index = dev->vq_index + i,
 478             .num   = enable,
 479         };
 480
 481         vhost_set_vring(dev, VHOST_USER_SET_VRING_ENABLE, &state);
 482     }
 483
 484     return 0;
 485 }
 486
 487 static int vhost_user_get_vring_base(struct vhost_dev *dev,
 488                                      struct vhost_vring_state *ring)
 489 {
 490     VhostUserMsg msg = {
 491         .hdr.request = VHOST_USER_GET_VRING_BASE,
 492         .hdr.flags = VHOST_USER_VERSION,
 493         .payload.state = *ring,
 494         .hdr.size = sizeof(msg.payload.state),
 495     };
 496
 497     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 498         return -1;
 499     }
 500
 501     if (vhost_user_read(dev, &msg) < 0) {
 502         return -1;
 503     }
 504
 505     if (msg.hdr.request != VHOST_USER_GET_VRING_BASE) {
 506         error_report("Received unexpected msg type. Expected %d received %d",
 507                      VHOST_USER_GET_VRING_BASE, msg.hdr.request);
 508         return -1;
 509     }
 510
 511     if (msg.hdr.size != sizeof(msg.payload.state)) {
 512         error_report("Received bad msg size.");
 513         return -1;
 514     }
 515
 516     *ring = msg.payload.state;
 517
 518     return 0;
 519 }
 520
 521 static int vhost_set_vring_file(struct vhost_dev *dev,
 522                                 VhostUserRequest request,
 523                                 struct vhost_vring_file *file)
 524 {
 525     int fds[VHOST_MEMORY_MAX_NREGIONS];
 526     size_t fd_num = 0;
 527     VhostUserMsg msg = {
 528         .hdr.request = request,
 529         .hdr.flags = VHOST_USER_VERSION,
 530         .payload.u64 = file->index & VHOST_USER_VRING_IDX_MASK,
 531         .hdr.size = sizeof(msg.payload.u64),
 532     };
 533
 534     if (ioeventfd_enabled() && file->fd > 0) {
 535         fds[fd_num++] = file->fd;
 536     } else {
 537         msg.payload.u64 |= VHOST_USER_VRING_NOFD_MASK;
 538     }
 539
 540     if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
 541         return -1;
 542     }
 543
 544     return 0;
 545 }
 546
 547 static int vhost_user_set_vring_kick(struct vhost_dev *dev,
 548                                      struct vhost_vring_file *file)
 549 {
 550     return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_KICK, file);
 551 }
 552
 553 static int vhost_user_set_vring_call(struct vhost_dev *dev,
 554                                      struct vhost_vring_file *file)
 555 {
 556     return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_CALL, file);
 557 }
 558
 559 static int vhost_user_set_u64(struct vhost_dev *dev, int request, uint64_t u64)
 560 {
 561     VhostUserMsg msg = {
 562         .hdr.request = request,
 563         .hdr.flags = VHOST_USER_VERSION,
 564         .payload.u64 = u64,
 565         .hdr.size = sizeof(msg.payload.u64),
 566     };
 567
 568     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 569         return -1;
 570     }
 571
 572     return 0;
 573 }
 574
 575 static int vhost_user_set_features(struct vhost_dev *dev,
 576                                    uint64_t features)
 577 {
 578     return vhost_user_set_u64(dev, VHOST_USER_SET_FEATURES, features);
 579 }
 580
 581 static int vhost_user_set_protocol_features(struct vhost_dev *dev,
 582                                             uint64_t features)
 583 {
 584     return vhost_user_set_u64(dev, VHOST_USER_SET_PROTOCOL_FEATURES, features);
 585 }
 586
 587 static int vhost_user_get_u64(struct vhost_dev *dev, int request, uint64_t *u64)
 588 {
 589     VhostUserMsg msg = {
 590         .hdr.request = request,
 591         .hdr.flags = VHOST_USER_VERSION,
 592     };
 593
 594     if (vhost_user_one_time_request(request) && dev->vq_index != 0) {
 595         return 0;
 596     }
 597
 598     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 599         return -1;
 600     }
 601
 602     if (vhost_user_read(dev, &msg) < 0) {
 603         return -1;
 604     }
 605
 606     if (msg.hdr.request != request) {
 607         error_report("Received unexpected msg type. Expected %d received %d",
 608                      request, msg.hdr.request);
 609         return -1;
 610     }
 611
 612     if (msg.hdr.size != sizeof(msg.payload.u64)) {
 613         error_report("Received bad msg size.");
 614         return -1;
 615     }
 616
 617     *u64 = msg.payload.u64;
 618
 619     return 0;
 620 }
 621
 622 static int vhost_user_get_features(struct vhost_dev *dev, uint64_t *features)
 623 {
 624     return vhost_user_get_u64(dev, VHOST_USER_GET_FEATURES, features);
 625 }
 626
 627 static int vhost_user_set_owner(struct vhost_dev *dev)
 628 {
 629     VhostUserMsg msg = {
 630         .hdr.request = VHOST_USER_SET_OWNER,
 631         .hdr.flags = VHOST_USER_VERSION,
 632     };
 633
 634     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 635         return -1;
 636     }
 637
 638     return 0;
 639 }
 640
 641 static int vhost_user_reset_device(struct vhost_dev *dev)
 642 {
 643     VhostUserMsg msg = {
 644         .hdr.request = VHOST_USER_RESET_OWNER,
 645         .hdr.flags = VHOST_USER_VERSION,
 646     };
 647
 648     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 649         return -1;
 650     }
 651
 652     return 0;
 653 }
 654
 655 static int vhost_user_slave_handle_config_change(struct vhost_dev *dev)
 656 {
 657     int ret = -1;
 658
 659     if (!dev->config_ops) {
 660         return -1;
 661     }
 662
 663     if (dev->config_ops->vhost_dev_config_notifier) {
 664         ret = dev->config_ops->vhost_dev_config_notifier(dev);
 665     }
 666
 667     return ret;
 668 }
 669
 670 static void slave_read(void *opaque)
 671 {
 672     struct vhost_dev *dev = opaque;
 673     struct vhost_user *u = dev->opaque;
 674     VhostUserHeader hdr = { 0, };
 675     VhostUserPayload payload = { 0, };
 676     int size, ret = 0;
 677
 678     /* Read header */
 679     size = read(u->slave_fd, &hdr, VHOST_USER_HDR_SIZE);
 680     if (size != VHOST_USER_HDR_SIZE) {
 681         error_report("Failed to read from slave.");
 682         goto err;
 683     }
 684
 685     if (hdr.size > VHOST_USER_PAYLOAD_SIZE) {
 686         error_report("Failed to read msg header."
 687                 " Size %d exceeds the maximum %zu.", hdr.size,
 688                 VHOST_USER_PAYLOAD_SIZE);
 689         goto err;
 690     }
 691
 692     /* Read payload */
 693     size = read(u->slave_fd, &payload, hdr.size);
 694     if (size != hdr.size) {
 695         error_report("Failed to read payload from slave.");
 696         goto err;
 697     }
 698
 699     switch (hdr.request) {
 700     case VHOST_USER_SLAVE_IOTLB_MSG:
 701         ret = vhost_backend_handle_iotlb_msg(dev, &payload.iotlb);
 702         break;
 703     case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG :
 704         ret = vhost_user_slave_handle_config_change(dev);
 705         break;
 706     default:
 707         error_report("Received unexpected msg type.");
 708         ret = -EINVAL;
 709     }
 710
 711     /*
 712      * REPLY_ACK feature handling. Other reply types has to be managed
 713      * directly in their request handlers.
 714      */
 715     if (hdr.flags & VHOST_USER_NEED_REPLY_MASK) {
 716         struct iovec iovec[2];
 717
 718
 719         hdr.flags &= ~VHOST_USER_NEED_REPLY_MASK;
 720         hdr.flags |= VHOST_USER_REPLY_MASK;
 721
 722         payload.u64 = !!ret;
 723         hdr.size = sizeof(payload.u64);
 724
 725         iovec[0].iov_base = &hdr;
 726         iovec[0].iov_len = VHOST_USER_HDR_SIZE;
 727         iovec[1].iov_base = &payload;
 728         iovec[1].iov_len = hdr.size;
 729
 730         size = writev(u->slave_fd, iovec, ARRAY_SIZE(iovec));
 731         if (size != VHOST_USER_HDR_SIZE + hdr.size) {
 732             error_report("Failed to send msg reply to slave.");
 733             goto err;
 734         }
 735     }
 736
 737     return;
 738
 739 err:
 740     qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL);
 741     close(u->slave_fd);
 742     u->slave_fd = -1;
 743     return;
 744 }
 745
 746 static int vhost_setup_slave_channel(struct vhost_dev *dev)
 747 {
 748     VhostUserMsg msg = {
 749         .hdr.request = VHOST_USER_SET_SLAVE_REQ_FD,
 750         .hdr.flags = VHOST_USER_VERSION,
 751     };
 752     struct vhost_user *u = dev->opaque;
 753     int sv[2], ret = 0;
 754     bool reply_supported = virtio_has_feature(dev->protocol_features,
 755                                               VHOST_USER_PROTOCOL_F_REPLY_ACK);
 756
 757     if (!virtio_has_feature(dev->protocol_features,
 758                             VHOST_USER_PROTOCOL_F_SLAVE_REQ)) {
 759         return 0;
 760     }
 761
 762     if (socketpair(PF_UNIX, SOCK_STREAM, 0, sv) == -1) {
 763         error_report("socketpair() failed");
 764         return -1;
 765     }
 766
 767     u->slave_fd = sv[0];
 768     qemu_set_fd_handler(u->slave_fd, slave_read, NULL, dev);
 769
 770     if (reply_supported) {
 771         msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
 772     }
 773
 774     ret = vhost_user_write(dev, &msg, &sv[1], 1);
 775     if (ret) {
 776         goto out;
 777     }
 778
 779     if (reply_supported) {
 780         ret = process_message_reply(dev, &msg);
 781     }
 782
 783 out:
 784     close(sv[1]);
 785     if (ret) {
 786         qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL);
 787         close(u->slave_fd);
 788         u->slave_fd = -1;
 789     }
 790
 791     return ret;
 792 }
 793
 794 static int vhost_user_init(struct vhost_dev *dev, void *opaque)
 795 {
 796     uint64_t features, protocol_features;
 797     struct vhost_user *u;
 798     int err;
 799
 800     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
 801
 802     u = g_new0(struct vhost_user, 1);
 803     u->chr = opaque;
 804     u->slave_fd = -1;
 805     dev->opaque = u;
 806
 807     err = vhost_user_get_features(dev, &features);
 808     if (err < 0) {
 809         return err;
 810     }
 811
 812     if (virtio_has_feature(features, VHOST_USER_F_PROTOCOL_FEATURES)) {
 813         dev->backend_features |= 1ULL << VHOST_USER_F_PROTOCOL_FEATURES;
 814
 815         err = vhost_user_get_u64(dev, VHOST_USER_GET_PROTOCOL_FEATURES,
 816                                  &protocol_features);
 817         if (err < 0) {
 818             return err;
 819         }
 820
 821         dev->protocol_features =
 822             protocol_features & VHOST_USER_PROTOCOL_FEATURE_MASK;
 823         err = vhost_user_set_protocol_features(dev, dev->protocol_features);
 824         if (err < 0) {
 825             return err;
 826         }
 827
 828         /* query the max queues we support if backend supports Multiple Queue */
 829         if (dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_MQ)) {
 830             err = vhost_user_get_u64(dev, VHOST_USER_GET_QUEUE_NUM,
 831                                      &dev->max_queues);
 832             if (err < 0) {
 833                 return err;
 834             }
 835         }
 836
 837         if (virtio_has_feature(features, VIRTIO_F_IOMMU_PLATFORM) &&
 838                 !(virtio_has_feature(dev->protocol_features,
 839                     VHOST_USER_PROTOCOL_F_SLAVE_REQ) &&
 840                  virtio_has_feature(dev->protocol_features,
 841                     VHOST_USER_PROTOCOL_F_REPLY_ACK))) {
 842             error_report("IOMMU support requires reply-ack and "
 843                          "slave-req protocol features.");
 844             return -1;
 845         }
 846     }
 847
 848     if (dev->migration_blocker == NULL &&
 849         !virtio_has_feature(dev->protocol_features,
 850                             VHOST_USER_PROTOCOL_F_LOG_SHMFD)) {
 851         error_setg(&dev->migration_blocker,
 852                    "Migration disabled: vhost-user backend lacks "
 853                    "VHOST_USER_PROTOCOL_F_LOG_SHMFD feature.");
 854     }
 855
 856     err = vhost_setup_slave_channel(dev);
 857     if (err < 0) {
 858         return err;
 859     }
 860
 861     return 0;
 862 }
 863
 864 static int vhost_user_cleanup(struct vhost_dev *dev)
 865 {
 866     struct vhost_user *u;
 867
 868     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
 869
 870     u = dev->opaque;
 871     if (u->slave_fd >= 0) {
 872         qemu_set_fd_handler(u->slave_fd, NULL, NULL, NULL);
 873         close(u->slave_fd);
 874         u->slave_fd = -1;
 875     }
 876     g_free(u);
 877     dev->opaque = 0;
 878
 879     return 0;
 880 }
 881
 882 static int vhost_user_get_vq_index(struct vhost_dev *dev, int idx)
 883 {
 884     assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
 885
 886     return idx;
 887 }
 888
 889 static int vhost_user_memslots_limit(struct vhost_dev *dev)
 890 {
 891     return VHOST_MEMORY_MAX_NREGIONS;
 892 }
 893
 894 static bool vhost_user_requires_shm_log(struct vhost_dev *dev)
 895 {
 896     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
 897
 898     return virtio_has_feature(dev->protocol_features,
 899                               VHOST_USER_PROTOCOL_F_LOG_SHMFD);
 900 }
 901
 902 static int vhost_user_migration_done(struct vhost_dev *dev, char* mac_addr)
 903 {
 904     VhostUserMsg msg = { 0 };
 905
 906     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
 907
 908     /* If guest supports GUEST_ANNOUNCE do nothing */
 909     if (virtio_has_feature(dev->acked_features, VIRTIO_NET_F_GUEST_ANNOUNCE)) {
 910         return 0;
 911     }
 912
 913     /* if backend supports VHOST_USER_PROTOCOL_F_RARP ask it to send the RARP */
 914     if (virtio_has_feature(dev->protocol_features,
 915                            VHOST_USER_PROTOCOL_F_RARP)) {
 916         msg.hdr.request = VHOST_USER_SEND_RARP;
 917         msg.hdr.flags = VHOST_USER_VERSION;
 918         memcpy((char *)&msg.payload.u64, mac_addr, 6);
 919         msg.hdr.size = sizeof(msg.payload.u64);
 920
 921         return vhost_user_write(dev, &msg, NULL, 0);
 922     }
 923     return -1;
 924 }
 925
 926 static bool vhost_user_can_merge(struct vhost_dev *dev,
 927                                  uint64_t start1, uint64_t size1,
 928                                  uint64_t start2, uint64_t size2)
 929 {
 930     ram_addr_t offset;
 931     int mfd, rfd;
 932     MemoryRegion *mr;
 933
 934     mr = memory_region_from_host((void *)(uintptr_t)start1, &offset);
 935     mfd = memory_region_get_fd(mr);
 936
 937     mr = memory_region_from_host((void *)(uintptr_t)start2, &offset);
 938     rfd = memory_region_get_fd(mr);
 939
 940     return mfd == rfd;
 941 }
 942
 943 static int vhost_user_net_set_mtu(struct vhost_dev *dev, uint16_t mtu)
 944 {
 945     VhostUserMsg msg;
 946     bool reply_supported = virtio_has_feature(dev->protocol_features,
 947                                               VHOST_USER_PROTOCOL_F_REPLY_ACK);
 948
 949     if (!(dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_NET_MTU))) {
 950         return 0;
 951     }
 952
 953     msg.hdr.request = VHOST_USER_NET_SET_MTU;
 954     msg.payload.u64 = mtu;
 955     msg.hdr.size = sizeof(msg.payload.u64);
 956     msg.hdr.flags = VHOST_USER_VERSION;
 957     if (reply_supported) {
 958         msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
 959     }
 960
 961     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 962         return -1;
 963     }
 964
 965     /* If reply_ack supported, slave has to ack specified MTU is valid */
 966     if (reply_supported) {
 967         return process_message_reply(dev, &msg);
 968     }
 969
 970     return 0;
 971 }
 972
 973 static int vhost_user_send_device_iotlb_msg(struct vhost_dev *dev,
 974                                             struct vhost_iotlb_msg *imsg)
 975 {
 976     VhostUserMsg msg = {
 977         .hdr.request = VHOST_USER_IOTLB_MSG,
 978         .hdr.size = sizeof(msg.payload.iotlb),
 979         .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
 980         .payload.iotlb = *imsg,
 981     };
 982
 983     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
 984         return -EFAULT;
 985     }
 986
 987     return process_message_reply(dev, &msg);
 988 }
 989
 990
 991 static void vhost_user_set_iotlb_callback(struct vhost_dev *dev, int enabled)
 992 {
 993     /* No-op as the receive channel is not dedicated to IOTLB messages. */
 994 }
 995
 996 static int vhost_user_get_config(struct vhost_dev *dev, uint8_t *config,
 997                                  uint32_t config_len)
 998 {
 999     VhostUserMsg msg = {
1000         .hdr.request = VHOST_USER_GET_CONFIG,
1001         .hdr.flags = VHOST_USER_VERSION,
1002         .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + config_len,
1003     };
1004
1005     if (config_len > VHOST_USER_MAX_CONFIG_SIZE) {
1006         return -1;
1007     }
1008
1009     msg.payload.config.offset = 0;
1010     msg.payload.config.size = config_len;
1011     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1012         return -1;
1013     }
1014
1015     if (vhost_user_read(dev, &msg) < 0) {
1016         return -1;
1017     }
1018
1019     if (msg.hdr.request != VHOST_USER_GET_CONFIG) {
1020         error_report("Received unexpected msg type. Expected %d received %d",
1021                      VHOST_USER_GET_CONFIG, msg.hdr.request);
1022         return -1;
1023     }
1024
1025     if (msg.hdr.size != VHOST_USER_CONFIG_HDR_SIZE + config_len) {
1026         error_report("Received bad msg size.");
1027         return -1;
1028     }
1029
1030     memcpy(config, msg.payload.config.region, config_len);
1031
1032     return 0;
1033 }
1034
1035 static int vhost_user_set_config(struct vhost_dev *dev, const uint8_t *data,
1036                                  uint32_t offset, uint32_t size, uint32_t flags)
1037 {
1038     uint8_t *p;
1039     bool reply_supported = virtio_has_feature(dev->protocol_features,
1040                                               VHOST_USER_PROTOCOL_F_REPLY_ACK);
1041
1042     VhostUserMsg msg = {
1043         .hdr.request = VHOST_USER_SET_CONFIG,
1044         .hdr.flags = VHOST_USER_VERSION,
1045         .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + size,
1046     };
1047
1048     if (reply_supported) {
1049         msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
1050     }
1051
1052     if (size > VHOST_USER_MAX_CONFIG_SIZE) {
1053         return -1;
1054     }
1055
1056     msg.payload.config.offset = offset,
1057     msg.payload.config.size = size,
1058     msg.payload.config.flags = flags,
1059     p = msg.payload.config.region;
1060     memcpy(p, data, size);
1061
1062     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1063         return -1;
1064     }
1065
1066     if (reply_supported) {
1067         return process_message_reply(dev, &msg);
1068     }
1069
1070     return 0;
1071 }
1072
1073 static int vhost_user_crypto_create_session(struct vhost_dev *dev,
1074                                             void *session_info,
1075                                             uint64_t *session_id)
1076 {
1077     bool crypto_session = virtio_has_feature(dev->protocol_features,
1078                                        VHOST_USER_PROTOCOL_F_CRYPTO_SESSION);
1079     CryptoDevBackendSymSessionInfo *sess_info = session_info;
1080     VhostUserMsg msg = {
1081         .hdr.request = VHOST_USER_CREATE_CRYPTO_SESSION,
1082         .hdr.flags = VHOST_USER_VERSION,
1083         .hdr.size = sizeof(msg.payload.session),
1084     };
1085
1086     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
1087
1088     if (!crypto_session) {
1089         error_report("vhost-user trying to send unhandled ioctl");
1090         return -1;
1091     }
1092
1093     memcpy(&msg.payload.session.session_setup_data, sess_info,
1094               sizeof(CryptoDevBackendSymSessionInfo));
1095     if (sess_info->key_len) {
1096         memcpy(&msg.payload.session.key, sess_info->cipher_key,
1097                sess_info->key_len);
1098     }
1099     if (sess_info->auth_key_len > 0) {
1100         memcpy(&msg.payload.session.auth_key, sess_info->auth_key,
1101                sess_info->auth_key_len);
1102     }
1103     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1104         error_report("vhost_user_write() return -1, create session failed");
1105         return -1;
1106     }
1107
1108     if (vhost_user_read(dev, &msg) < 0) {
1109         error_report("vhost_user_read() return -1, create session failed");
1110         return -1;
1111     }
1112
1113     if (msg.hdr.request != VHOST_USER_CREATE_CRYPTO_SESSION) {
1114         error_report("Received unexpected msg type. Expected %d received %d",
1115                      VHOST_USER_CREATE_CRYPTO_SESSION, msg.hdr.request);
1116         return -1;
1117     }
1118
1119     if (msg.hdr.size != sizeof(msg.payload.session)) {
1120         error_report("Received bad msg size.");
1121         return -1;
1122     }
1123
1124     if (msg.payload.session.session_id < 0) {
1125         error_report("Bad session id: %" PRId64 "",
1126                               msg.payload.session.session_id);
1127         return -1;
1128     }
1129     *session_id = msg.payload.session.session_id;
1130
1131     return 0;
1132 }
1133
1134 static int
1135 vhost_user_crypto_close_session(struct vhost_dev *dev, uint64_t session_id)
1136 {
1137     bool crypto_session = virtio_has_feature(dev->protocol_features,
1138                                        VHOST_USER_PROTOCOL_F_CRYPTO_SESSION);
1139     VhostUserMsg msg = {
1140         .hdr.request = VHOST_USER_CLOSE_CRYPTO_SESSION,
1141         .hdr.flags = VHOST_USER_VERSION,
1142         .hdr.size = sizeof(msg.payload.u64),
1143     };
1144     msg.payload.u64 = session_id;
1145
1146     if (!crypto_session) {
1147         error_report("vhost-user trying to send unhandled ioctl");
1148         return -1;
1149     }
1150
1151     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1152         error_report("vhost_user_write() return -1, close session failed");
1153         return -1;
1154     }
1155
1156     return 0;
1157 }
1158
1159 const VhostOps user_ops = {
1160         .backend_type = VHOST_BACKEND_TYPE_USER,
1161         .vhost_backend_init = vhost_user_init,
1162         .vhost_backend_cleanup = vhost_user_cleanup,
1163         .vhost_backend_memslots_limit = vhost_user_memslots_limit,
1164         .vhost_set_log_base = vhost_user_set_log_base,
1165         .vhost_set_mem_table = vhost_user_set_mem_table,
1166         .vhost_set_vring_addr = vhost_user_set_vring_addr,
1167         .vhost_set_vring_endian = vhost_user_set_vring_endian,
1168         .vhost_set_vring_num = vhost_user_set_vring_num,
1169         .vhost_set_vring_base = vhost_user_set_vring_base,
1170         .vhost_get_vring_base = vhost_user_get_vring_base,
1171         .vhost_set_vring_kick = vhost_user_set_vring_kick,
1172         .vhost_set_vring_call = vhost_user_set_vring_call,
1173         .vhost_set_features = vhost_user_set_features,
1174         .vhost_get_features = vhost_user_get_features,
1175         .vhost_set_owner = vhost_user_set_owner,
1176         .vhost_reset_device = vhost_user_reset_device,
1177         .vhost_get_vq_index = vhost_user_get_vq_index,
1178         .vhost_set_vring_enable = vhost_user_set_vring_enable,
1179         .vhost_requires_shm_log = vhost_user_requires_shm_log,
1180         .vhost_migration_done = vhost_user_migration_done,
1181         .vhost_backend_can_merge = vhost_user_can_merge,
1182         .vhost_net_set_mtu = vhost_user_net_set_mtu,
1183         .vhost_set_iotlb_callback = vhost_user_set_iotlb_callback,
1184         .vhost_send_device_iotlb_msg = vhost_user_send_device_iotlb_msg,
1185         .vhost_get_config = vhost_user_get_config,
1186         .vhost_set_config = vhost_user_set_config,
1187         .vhost_crypto_create_session = vhost_user_crypto_create_session,
1188         .vhost_crypto_close_session = vhost_user_crypto_close_session,
1189 };