?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
hw/display/next-fb: Fix comment typo
[qemu/ar7.git] / hw / virtio / vhost-user.c
blobabe23d4ebee87b795b4b6aa621b5da30b48fbfcb
1 /*
2 * vhost-user
3 *
4 * Copyright (c) 2013 Virtual Open Systems Sarl.
5 *
6 * This work is licensed under the terms of the GNU GPL, version 2 or later.
7 * See the COPYING file in the top-level directory.
8 *
9 */
10
11 #include "qemu/osdep.h"
12 #include "qapi/error.h"
13 #include "hw/virtio/vhost.h"
14 #include "hw/virtio/vhost-user.h"
15 #include "hw/virtio/vhost-backend.h"
16 #include "hw/virtio/virtio.h"
17 #include "hw/virtio/virtio-net.h"
18 #include "chardev/char-fe.h"
19 #include "io/channel-socket.h"
20 #include "sysemu/kvm.h"
21 #include "qemu/error-report.h"
22 #include "qemu/main-loop.h"
23 #include "qemu/sockets.h"
24 #include "sysemu/cryptodev.h"
25 #include "migration/migration.h"
26 #include "migration/postcopy-ram.h"
27 #include "trace.h"
28 #include "exec/ramblock.h"
29
30 #include <sys/ioctl.h>
31 #include <sys/socket.h>
32 #include <sys/un.h>
33
34 #include "standard-headers/linux/vhost_types.h"
35
36 #ifdef CONFIG_LINUX
37 #include <linux/userfaultfd.h>
38 #endif
39
40 #define VHOST_MEMORY_BASELINE_NREGIONS 8
41 #define VHOST_USER_F_PROTOCOL_FEATURES 30
42 #define VHOST_USER_SLAVE_MAX_FDS 8
43
44 /*
45 * Set maximum number of RAM slots supported to
46 * the maximum number supported by the target
47 * hardware plaform.
48 */
49 #if defined(TARGET_X86) || defined(TARGET_X86_64) || \
50 defined(TARGET_ARM) || defined(TARGET_ARM_64)
51 #include "hw/acpi/acpi.h"
52 #define VHOST_USER_MAX_RAM_SLOTS ACPI_MAX_RAM_SLOTS
53
54 #elif defined(TARGET_PPC) || defined(TARGET_PPC64)
55 #include "hw/ppc/spapr.h"
56 #define VHOST_USER_MAX_RAM_SLOTS SPAPR_MAX_RAM_SLOTS
57
58 #else
59 #define VHOST_USER_MAX_RAM_SLOTS 512
60 #endif
61
62 /*
63 * Maximum size of virtio device config space
64 */
65 #define VHOST_USER_MAX_CONFIG_SIZE 256
66
67 enum VhostUserProtocolFeature {
68 VHOST_USER_PROTOCOL_F_MQ = 0,
69 VHOST_USER_PROTOCOL_F_LOG_SHMFD = 1,
70 VHOST_USER_PROTOCOL_F_RARP = 2,
71 VHOST_USER_PROTOCOL_F_REPLY_ACK = 3,
72 VHOST_USER_PROTOCOL_F_NET_MTU = 4,
73 VHOST_USER_PROTOCOL_F_SLAVE_REQ = 5,
74 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN = 6,
75 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION = 7,
76 VHOST_USER_PROTOCOL_F_PAGEFAULT = 8,
77 VHOST_USER_PROTOCOL_F_CONFIG = 9,
78 VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD = 10,
79 VHOST_USER_PROTOCOL_F_HOST_NOTIFIER = 11,
80 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD = 12,
81 VHOST_USER_PROTOCOL_F_RESET_DEVICE = 13,
82 /* Feature 14 reserved for VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS. */
83 VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS = 15,
84 VHOST_USER_PROTOCOL_F_STATUS = 16,
85 VHOST_USER_PROTOCOL_F_MAX
86 };
87
88 #define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1)
89
90 typedef enum VhostUserRequest {
91 VHOST_USER_NONE = 0,
92 VHOST_USER_GET_FEATURES = 1,
93 VHOST_USER_SET_FEATURES = 2,
94 VHOST_USER_SET_OWNER = 3,
95 VHOST_USER_RESET_OWNER = 4,
96 VHOST_USER_SET_MEM_TABLE = 5,
97 VHOST_USER_SET_LOG_BASE = 6,
98 VHOST_USER_SET_LOG_FD = 7,
99 VHOST_USER_SET_VRING_NUM = 8,
100 VHOST_USER_SET_VRING_ADDR = 9,
101 VHOST_USER_SET_VRING_BASE = 10,
102 VHOST_USER_GET_VRING_BASE = 11,
103 VHOST_USER_SET_VRING_KICK = 12,
104 VHOST_USER_SET_VRING_CALL = 13,
105 VHOST_USER_SET_VRING_ERR = 14,
106 VHOST_USER_GET_PROTOCOL_FEATURES = 15,
107 VHOST_USER_SET_PROTOCOL_FEATURES = 16,
108 VHOST_USER_GET_QUEUE_NUM = 17,
109 VHOST_USER_SET_VRING_ENABLE = 18,
110 VHOST_USER_SEND_RARP = 19,
111 VHOST_USER_NET_SET_MTU = 20,
112 VHOST_USER_SET_SLAVE_REQ_FD = 21,
113 VHOST_USER_IOTLB_MSG = 22,
114 VHOST_USER_SET_VRING_ENDIAN = 23,
115 VHOST_USER_GET_CONFIG = 24,
116 VHOST_USER_SET_CONFIG = 25,
117 VHOST_USER_CREATE_CRYPTO_SESSION = 26,
118 VHOST_USER_CLOSE_CRYPTO_SESSION = 27,
119 VHOST_USER_POSTCOPY_ADVISE = 28,
120 VHOST_USER_POSTCOPY_LISTEN = 29,
121 VHOST_USER_POSTCOPY_END = 30,
122 VHOST_USER_GET_INFLIGHT_FD = 31,
123 VHOST_USER_SET_INFLIGHT_FD = 32,
124 VHOST_USER_GPU_SET_SOCKET = 33,
125 VHOST_USER_RESET_DEVICE = 34,
126 /* Message number 35 reserved for VHOST_USER_VRING_KICK. */
127 VHOST_USER_GET_MAX_MEM_SLOTS = 36,
128 VHOST_USER_ADD_MEM_REG = 37,
129 VHOST_USER_REM_MEM_REG = 38,
130 VHOST_USER_SET_STATUS = 39,
131 VHOST_USER_GET_STATUS = 40,
132 VHOST_USER_MAX
133 } VhostUserRequest;
134
135 typedef enum VhostUserSlaveRequest {
136 VHOST_USER_SLAVE_NONE = 0,
137 VHOST_USER_SLAVE_IOTLB_MSG = 1,
138 VHOST_USER_SLAVE_CONFIG_CHANGE_MSG = 2,
139 VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG = 3,
140 VHOST_USER_SLAVE_MAX
141 } VhostUserSlaveRequest;
142
143 typedef struct VhostUserMemoryRegion {
144 uint64_t guest_phys_addr;
145 uint64_t memory_size;
146 uint64_t userspace_addr;
147 uint64_t mmap_offset;
148 } VhostUserMemoryRegion;
149
150 typedef struct VhostUserMemory {
151 uint32_t nregions;
152 uint32_t padding;
153 VhostUserMemoryRegion regions[VHOST_MEMORY_BASELINE_NREGIONS];
154 } VhostUserMemory;
155
156 typedef struct VhostUserMemRegMsg {
157 uint64_t padding;
158 VhostUserMemoryRegion region;
159 } VhostUserMemRegMsg;
160
161 typedef struct VhostUserLog {
162 uint64_t mmap_size;
163 uint64_t mmap_offset;
164 } VhostUserLog;
165
166 typedef struct VhostUserConfig {
167 uint32_t offset;
168 uint32_t size;
169 uint32_t flags;
170 uint8_t region[VHOST_USER_MAX_CONFIG_SIZE];
171 } VhostUserConfig;
172
173 #define VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN 512
174 #define VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN 64
175
176 typedef struct VhostUserCryptoSession {
177 /* session id for success, -1 on errors */
178 int64_t session_id;
179 CryptoDevBackendSymSessionInfo session_setup_data;
180 uint8_t key[VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN];
181 uint8_t auth_key[VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN];
182 } VhostUserCryptoSession;
183
184 static VhostUserConfig c __attribute__ ((unused));
185 #define VHOST_USER_CONFIG_HDR_SIZE (sizeof(c.offset) \
186 + sizeof(c.size) \
187 + sizeof(c.flags))
188
189 typedef struct VhostUserVringArea {
190 uint64_t u64;
191 uint64_t size;
192 uint64_t offset;
193 } VhostUserVringArea;
194
195 typedef struct VhostUserInflight {
196 uint64_t mmap_size;
197 uint64_t mmap_offset;
198 uint16_t num_queues;
199 uint16_t queue_size;
200 } VhostUserInflight;
201
202 typedef struct {
203 VhostUserRequest request;
204
205 #define VHOST_USER_VERSION_MASK (0x3)
206 #define VHOST_USER_REPLY_MASK (0x1 << 2)
207 #define VHOST_USER_NEED_REPLY_MASK (0x1 << 3)
208 uint32_t flags;
209 uint32_t size; /* the following payload size */
210 } QEMU_PACKED VhostUserHeader;
211
212 typedef union {
213 #define VHOST_USER_VRING_IDX_MASK (0xff)
214 #define VHOST_USER_VRING_NOFD_MASK (0x1 << 8)
215 uint64_t u64;
216 struct vhost_vring_state state;
217 struct vhost_vring_addr addr;
218 VhostUserMemory memory;
219 VhostUserMemRegMsg mem_reg;
220 VhostUserLog log;
221 struct vhost_iotlb_msg iotlb;
222 VhostUserConfig config;
223 VhostUserCryptoSession session;
224 VhostUserVringArea area;
225 VhostUserInflight inflight;
226 } VhostUserPayload;
227
228 typedef struct VhostUserMsg {
229 VhostUserHeader hdr;
230 VhostUserPayload payload;
231 } QEMU_PACKED VhostUserMsg;
232
233 static VhostUserMsg m __attribute__ ((unused));
234 #define VHOST_USER_HDR_SIZE (sizeof(VhostUserHeader))
235
236 #define VHOST_USER_PAYLOAD_SIZE (sizeof(VhostUserPayload))
237
238 /* The version of the protocol we support */
239 #define VHOST_USER_VERSION (0x1)
240
241 struct vhost_user {
242 struct vhost_dev *dev;
243 /* Shared between vhost devs of the same virtio device */
244 VhostUserState *user;
245 QIOChannel *slave_ioc;
246 GSource *slave_src;
247 NotifierWithReturn postcopy_notifier;
248 struct PostCopyFD postcopy_fd;
249 uint64_t postcopy_client_bases[VHOST_USER_MAX_RAM_SLOTS];
250 /* Length of the region_rb and region_rb_offset arrays */
251 size_t region_rb_len;
252 /* RAMBlock associated with a given region */
253 RAMBlock **region_rb;
254 /*
255 * The offset from the start of the RAMBlock to the start of the
256 * vhost region.
257 */
258 ram_addr_t *region_rb_offset;
259
260 /* True once we've entered postcopy_listen */
261 bool postcopy_listen;
262
263 /* Our current regions */
264 int num_shadow_regions;
265 struct vhost_memory_region shadow_regions[VHOST_USER_MAX_RAM_SLOTS];
266 };
267
268 struct scrub_regions {
269 struct vhost_memory_region *region;
270 int reg_idx;
271 int fd_idx;
272 };
273
274 static bool ioeventfd_enabled(void)
275 {
276 return !kvm_enabled() || kvm_eventfds_enabled();
277 }
278
279 static int vhost_user_read_header(struct vhost_dev *dev, VhostUserMsg *msg)
280 {
281 struct vhost_user *u = dev->opaque;
282 CharBackend *chr = u->user->chr;
283 uint8_t *p = (uint8_t *) msg;
284 int r, size = VHOST_USER_HDR_SIZE;
285
286 r = qemu_chr_fe_read_all(chr, p, size);
287 if (r != size) {
288 int saved_errno = errno;
289 error_report("Failed to read msg header. Read %d instead of %d."
290 " Original request %d.", r, size, msg->hdr.request);
291 return r < 0 ? -saved_errno : -EIO;
292 }
293
294 /* validate received flags */
295 if (msg->hdr.flags != (VHOST_USER_REPLY_MASK | VHOST_USER_VERSION)) {
296 error_report("Failed to read msg header."
297 " Flags 0x%x instead of 0x%x.", msg->hdr.flags,
298 VHOST_USER_REPLY_MASK | VHOST_USER_VERSION);
299 return -EPROTO;
300 }
301
302 trace_vhost_user_read(msg->hdr.request, msg->hdr.flags);
303
304 return 0;
305 }
306
307 struct vhost_user_read_cb_data {
308 struct vhost_dev *dev;
309 VhostUserMsg *msg;
310 GMainLoop *loop;
311 int ret;
312 };
313
314 static gboolean vhost_user_read_cb(void *do_not_use, GIOCondition condition,
315 gpointer opaque)
316 {
317 struct vhost_user_read_cb_data *data = opaque;
318 struct vhost_dev *dev = data->dev;
319 VhostUserMsg *msg = data->msg;
320 struct vhost_user *u = dev->opaque;
321 CharBackend *chr = u->user->chr;
322 uint8_t *p = (uint8_t *) msg;
323 int r, size;
324
325 r = vhost_user_read_header(dev, msg);
326 if (r < 0) {
327 data->ret = r;
328 goto end;
329 }
330
331 /* validate message size is sane */
332 if (msg->hdr.size > VHOST_USER_PAYLOAD_SIZE) {
333 error_report("Failed to read msg header."
334 " Size %d exceeds the maximum %zu.", msg->hdr.size,
335 VHOST_USER_PAYLOAD_SIZE);
336 data->ret = -EPROTO;
337 goto end;
338 }
339
340 if (msg->hdr.size) {
341 p += VHOST_USER_HDR_SIZE;
342 size = msg->hdr.size;
343 r = qemu_chr_fe_read_all(chr, p, size);
344 if (r != size) {
345 int saved_errno = errno;
346 error_report("Failed to read msg payload."
347 " Read %d instead of %d.", r, msg->hdr.size);
348 data->ret = r < 0 ? -saved_errno : -EIO;
349 goto end;
350 }
351 }
352
353 end:
354 g_main_loop_quit(data->loop);
355 return G_SOURCE_REMOVE;
356 }
357
358 static gboolean slave_read(QIOChannel *ioc, GIOCondition condition,
359 gpointer opaque);
360
361 /*
362 * This updates the read handler to use a new event loop context.
363 * Event sources are removed from the previous context : this ensures
364 * that events detected in the previous context are purged. They will
365 * be re-detected and processed in the new context.
366 */
367 static void slave_update_read_handler(struct vhost_dev *dev,
368 GMainContext *ctxt)
369 {
370 struct vhost_user *u = dev->opaque;
371
372 if (!u->slave_ioc) {
373 return;
374 }
375
376 if (u->slave_src) {
377 g_source_destroy(u->slave_src);
378 g_source_unref(u->slave_src);
379 }
380
381 u->slave_src = qio_channel_add_watch_source(u->slave_ioc,
382 G_IO_IN | G_IO_HUP,
383 slave_read, dev, NULL,
384 ctxt);
385 }
386
387 static int vhost_user_read(struct vhost_dev *dev, VhostUserMsg *msg)
388 {
389 struct vhost_user *u = dev->opaque;
390 CharBackend *chr = u->user->chr;
391 GMainContext *prev_ctxt = chr->chr->gcontext;
392 GMainContext *ctxt = g_main_context_new();
393 GMainLoop *loop = g_main_loop_new(ctxt, FALSE);
394 struct vhost_user_read_cb_data data = {
395 .dev = dev,
396 .loop = loop,
397 .msg = msg,
398 .ret = 0
399 };
400
401 /*
402 * We want to be able to monitor the slave channel fd while waiting
403 * for chr I/O. This requires an event loop, but we can't nest the
404 * one to which chr is currently attached : its fd handlers might not
405 * be prepared for re-entrancy. So we create a new one and switch chr
406 * to use it.
407 */
408 slave_update_read_handler(dev, ctxt);
409 qemu_chr_be_update_read_handlers(chr->chr, ctxt);
410 qemu_chr_fe_add_watch(chr, G_IO_IN | G_IO_HUP, vhost_user_read_cb, &data);
411
412 g_main_loop_run(loop);
413
414 /*
415 * Restore the previous event loop context. This also destroys/recreates
416 * event sources : this guarantees that all pending events in the original
417 * context that have been processed by the nested loop are purged.
418 */
419 qemu_chr_be_update_read_handlers(chr->chr, prev_ctxt);
420 slave_update_read_handler(dev, NULL);
421
422 g_main_loop_unref(loop);
423 g_main_context_unref(ctxt);
424
425 return data.ret;
426 }
427
428 static int process_message_reply(struct vhost_dev *dev,
429 const VhostUserMsg *msg)
430 {
431 int ret;
432 VhostUserMsg msg_reply;
433
434 if ((msg->hdr.flags & VHOST_USER_NEED_REPLY_MASK) == 0) {
435 return 0;
436 }
437
438 ret = vhost_user_read(dev, &msg_reply);
439 if (ret < 0) {
440 return ret;
441 }
442
443 if (msg_reply.hdr.request != msg->hdr.request) {
444 error_report("Received unexpected msg type. "
445 "Expected %d received %d",
446 msg->hdr.request, msg_reply.hdr.request);
447 return -EPROTO;
448 }
449
450 return msg_reply.payload.u64 ? -EIO : 0;
451 }
452
453 static bool vhost_user_one_time_request(VhostUserRequest request)
454 {
455 switch (request) {
456 case VHOST_USER_SET_OWNER:
457 case VHOST_USER_RESET_OWNER:
458 case VHOST_USER_SET_MEM_TABLE:
459 case VHOST_USER_GET_QUEUE_NUM:
460 case VHOST_USER_NET_SET_MTU:
461 return true;
462 default:
463 return false;
464 }
465 }
466
467 /* most non-init callers ignore the error */
468 static int vhost_user_write(struct vhost_dev *dev, VhostUserMsg *msg,
469 int *fds, int fd_num)
470 {
471 struct vhost_user *u = dev->opaque;
472 CharBackend *chr = u->user->chr;
473 int ret, size = VHOST_USER_HDR_SIZE + msg->hdr.size;
474
475 /*
476 * For non-vring specific requests, like VHOST_USER_SET_MEM_TABLE,
477 * we just need send it once in the first time. For later such
478 * request, we just ignore it.
479 */
480 if (vhost_user_one_time_request(msg->hdr.request) && dev->vq_index != 0) {
481 msg->hdr.flags &= ~VHOST_USER_NEED_REPLY_MASK;
482 return 0;
483 }
484
485 if (qemu_chr_fe_set_msgfds(chr, fds, fd_num) < 0) {
486 error_report("Failed to set msg fds.");
487 return -EINVAL;
488 }
489
490 ret = qemu_chr_fe_write_all(chr, (const uint8_t *) msg, size);
491 if (ret != size) {
492 int saved_errno = errno;
493 error_report("Failed to write msg."
494 " Wrote %d instead of %d.", ret, size);
495 return ret < 0 ? -saved_errno : -EIO;
496 }
497
498 trace_vhost_user_write(msg->hdr.request, msg->hdr.flags);
499
500 return 0;
501 }
502
503 int vhost_user_gpu_set_socket(struct vhost_dev *dev, int fd)
504 {
505 VhostUserMsg msg = {
506 .hdr.request = VHOST_USER_GPU_SET_SOCKET,
507 .hdr.flags = VHOST_USER_VERSION,
508 };
509
510 return vhost_user_write(dev, &msg, &fd, 1);
511 }
512
513 static int vhost_user_set_log_base(struct vhost_dev *dev, uint64_t base,
514 struct vhost_log *log)
515 {
516 int fds[VHOST_USER_MAX_RAM_SLOTS];
517 size_t fd_num = 0;
518 bool shmfd = virtio_has_feature(dev->protocol_features,
519 VHOST_USER_PROTOCOL_F_LOG_SHMFD);
520 int ret;
521 VhostUserMsg msg = {
522 .hdr.request = VHOST_USER_SET_LOG_BASE,
523 .hdr.flags = VHOST_USER_VERSION,
524 .payload.log.mmap_size = log->size * sizeof(*(log->log)),
525 .payload.log.mmap_offset = 0,
526 .hdr.size = sizeof(msg.payload.log),
527 };
528
529 if (shmfd && log->fd != -1) {
530 fds[fd_num++] = log->fd;
531 }
532
533 ret = vhost_user_write(dev, &msg, fds, fd_num);
534 if (ret < 0) {
535 return ret;
536 }
537
538 if (shmfd) {
539 msg.hdr.size = 0;
540 ret = vhost_user_read(dev, &msg);
541 if (ret < 0) {
542 return ret;
543 }
544
545 if (msg.hdr.request != VHOST_USER_SET_LOG_BASE) {
546 error_report("Received unexpected msg type. "
547 "Expected %d received %d",
548 VHOST_USER_SET_LOG_BASE, msg.hdr.request);
549 return -EPROTO;
550 }
551 }
552
553 return 0;
554 }
555
556 static MemoryRegion *vhost_user_get_mr_data(uint64_t addr, ram_addr_t *offset,
557 int *fd)
558 {
559 MemoryRegion *mr;
560
561 assert((uintptr_t)addr == addr);
562 mr = memory_region_from_host((void *)(uintptr_t)addr, offset);
563 *fd = memory_region_get_fd(mr);
564
565 return mr;
566 }
567
568 static void vhost_user_fill_msg_region(VhostUserMemoryRegion *dst,
569 struct vhost_memory_region *src,
570 uint64_t mmap_offset)
571 {
572 assert(src != NULL && dst != NULL);
573 dst->userspace_addr = src->userspace_addr;
574 dst->memory_size = src->memory_size;
575 dst->guest_phys_addr = src->guest_phys_addr;
576 dst->mmap_offset = mmap_offset;
577 }
578
579 static int vhost_user_fill_set_mem_table_msg(struct vhost_user *u,
580 struct vhost_dev *dev,
581 VhostUserMsg *msg,
582 int *fds, size_t *fd_num,
583 bool track_ramblocks)
584 {
585 int i, fd;
586 ram_addr_t offset;
587 MemoryRegion *mr;
588 struct vhost_memory_region *reg;
589 VhostUserMemoryRegion region_buffer;
590
591 msg->hdr.request = VHOST_USER_SET_MEM_TABLE;
592
593 for (i = 0; i < dev->mem->nregions; ++i) {
594 reg = dev->mem->regions + i;
595
596 mr = vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
597 if (fd > 0) {
598 if (track_ramblocks) {
599 assert(*fd_num < VHOST_MEMORY_BASELINE_NREGIONS);
600 trace_vhost_user_set_mem_table_withfd(*fd_num, mr->name,
601 reg->memory_size,
602 reg->guest_phys_addr,
603 reg->userspace_addr,
604 offset);
605 u->region_rb_offset[i] = offset;
606 u->region_rb[i] = mr->ram_block;
607 } else if (*fd_num == VHOST_MEMORY_BASELINE_NREGIONS) {
608 error_report("Failed preparing vhost-user memory table msg");
609 return -ENOBUFS;
610 }
611 vhost_user_fill_msg_region(&region_buffer, reg, offset);
612 msg->payload.memory.regions[*fd_num] = region_buffer;
613 fds[(*fd_num)++] = fd;
614 } else if (track_ramblocks) {
615 u->region_rb_offset[i] = 0;
616 u->region_rb[i] = NULL;
617 }
618 }
619
620 msg->payload.memory.nregions = *fd_num;
621
622 if (!*fd_num) {
623 error_report("Failed initializing vhost-user memory map, "
624 "consider using -object memory-backend-file share=on");
625 return -EINVAL;
626 }
627
628 msg->hdr.size = sizeof(msg->payload.memory.nregions);
629 msg->hdr.size += sizeof(msg->payload.memory.padding);
630 msg->hdr.size += *fd_num * sizeof(VhostUserMemoryRegion);
631
632 return 0;
633 }
634
635 static inline bool reg_equal(struct vhost_memory_region *shadow_reg,
636 struct vhost_memory_region *vdev_reg)
637 {
638 return shadow_reg->guest_phys_addr == vdev_reg->guest_phys_addr &&
639 shadow_reg->userspace_addr == vdev_reg->userspace_addr &&
640 shadow_reg->memory_size == vdev_reg->memory_size;
641 }
642
643 static void scrub_shadow_regions(struct vhost_dev *dev,
644 struct scrub_regions *add_reg,
645 int *nr_add_reg,
646 struct scrub_regions *rem_reg,
647 int *nr_rem_reg, uint64_t *shadow_pcb,
648 bool track_ramblocks)
649 {
650 struct vhost_user *u = dev->opaque;
651 bool found[VHOST_USER_MAX_RAM_SLOTS] = {};
652 struct vhost_memory_region *reg, *shadow_reg;
653 int i, j, fd, add_idx = 0, rm_idx = 0, fd_num = 0;
654 ram_addr_t offset;
655 MemoryRegion *mr;
656 bool matching;
657
658 /*
659 * Find memory regions present in our shadow state which are not in
660 * the device's current memory state.
661 *
662 * Mark regions in both the shadow and device state as "found".
663 */
664 for (i = 0; i < u->num_shadow_regions; i++) {
665 shadow_reg = &u->shadow_regions[i];
666 matching = false;
667
668 for (j = 0; j < dev->mem->nregions; j++) {
669 reg = &dev->mem->regions[j];
670
671 mr = vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
672
673 if (reg_equal(shadow_reg, reg)) {
674 matching = true;
675 found[j] = true;
676 if (track_ramblocks) {
677 /*
678 * Reset postcopy client bases, region_rb, and
679 * region_rb_offset in case regions are removed.
680 */
681 if (fd > 0) {
682 u->region_rb_offset[j] = offset;
683 u->region_rb[j] = mr->ram_block;
684 shadow_pcb[j] = u->postcopy_client_bases[i];
685 } else {
686 u->region_rb_offset[j] = 0;
687 u->region_rb[j] = NULL;
688 }
689 }
690 break;
691 }
692 }
693
694 /*
695 * If the region was not found in the current device memory state
696 * create an entry for it in the removed list.
697 */
698 if (!matching) {
699 rem_reg[rm_idx].region = shadow_reg;
700 rem_reg[rm_idx++].reg_idx = i;
701 }
702 }
703
704 /*
705 * For regions not marked "found", create entries in the added list.
706 *
707 * Note their indexes in the device memory state and the indexes of their
708 * file descriptors.
709 */
710 for (i = 0; i < dev->mem->nregions; i++) {
711 reg = &dev->mem->regions[i];
712 vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
713 if (fd > 0) {
714 ++fd_num;
715 }
716
717 /*
718 * If the region was in both the shadow and device state we don't
719 * need to send a VHOST_USER_ADD_MEM_REG message for it.
720 */
721 if (found[i]) {
722 continue;
723 }
724
725 add_reg[add_idx].region = reg;
726 add_reg[add_idx].reg_idx = i;
727 add_reg[add_idx++].fd_idx = fd_num;
728 }
729 *nr_rem_reg = rm_idx;
730 *nr_add_reg = add_idx;
731
732 return;
733 }
734
735 static int send_remove_regions(struct vhost_dev *dev,
736 struct scrub_regions *remove_reg,
737 int nr_rem_reg, VhostUserMsg *msg,
738 bool reply_supported)
739 {
740 struct vhost_user *u = dev->opaque;
741 struct vhost_memory_region *shadow_reg;
742 int i, fd, shadow_reg_idx, ret;
743 ram_addr_t offset;
744 VhostUserMemoryRegion region_buffer;
745
746 /*
747 * The regions in remove_reg appear in the same order they do in the
748 * shadow table. Therefore we can minimize memory copies by iterating
749 * through remove_reg backwards.
750 */
751 for (i = nr_rem_reg - 1; i >= 0; i--) {
752 shadow_reg = remove_reg[i].region;
753 shadow_reg_idx = remove_reg[i].reg_idx;
754
755 vhost_user_get_mr_data(shadow_reg->userspace_addr, &offset, &fd);
756
757 if (fd > 0) {
758 msg->hdr.request = VHOST_USER_REM_MEM_REG;
759 vhost_user_fill_msg_region(&region_buffer, shadow_reg, 0);
760 msg->payload.mem_reg.region = region_buffer;
761
762 ret = vhost_user_write(dev, msg, NULL, 0);
763 if (ret < 0) {
764 return ret;
765 }
766
767 if (reply_supported) {
768 ret = process_message_reply(dev, msg);
769 if (ret) {
770 return ret;
771 }
772 }
773 }
774
775 /*
776 * At this point we know the backend has unmapped the region. It is now
777 * safe to remove it from the shadow table.
778 */
779 memmove(&u->shadow_regions[shadow_reg_idx],
780 &u->shadow_regions[shadow_reg_idx + 1],
781 sizeof(struct vhost_memory_region) *
782 (u->num_shadow_regions - shadow_reg_idx - 1));
783 u->num_shadow_regions--;
784 }
785
786 return 0;
787 }
788
789 static int send_add_regions(struct vhost_dev *dev,
790 struct scrub_regions *add_reg, int nr_add_reg,
791 VhostUserMsg *msg, uint64_t *shadow_pcb,
792 bool reply_supported, bool track_ramblocks)
793 {
794 struct vhost_user *u = dev->opaque;
795 int i, fd, ret, reg_idx, reg_fd_idx;
796 struct vhost_memory_region *reg;
797 MemoryRegion *mr;
798 ram_addr_t offset;
799 VhostUserMsg msg_reply;
800 VhostUserMemoryRegion region_buffer;
801
802 for (i = 0; i < nr_add_reg; i++) {
803 reg = add_reg[i].region;
804 reg_idx = add_reg[i].reg_idx;
805 reg_fd_idx = add_reg[i].fd_idx;
806
807 mr = vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
808
809 if (fd > 0) {
810 if (track_ramblocks) {
811 trace_vhost_user_set_mem_table_withfd(reg_fd_idx, mr->name,
812 reg->memory_size,
813 reg->guest_phys_addr,
814 reg->userspace_addr,
815 offset);
816 u->region_rb_offset[reg_idx] = offset;
817 u->region_rb[reg_idx] = mr->ram_block;
818 }
819 msg->hdr.request = VHOST_USER_ADD_MEM_REG;
820 vhost_user_fill_msg_region(&region_buffer, reg, offset);
821 msg->payload.mem_reg.region = region_buffer;
822
823 ret = vhost_user_write(dev, msg, &fd, 1);
824 if (ret < 0) {
825 return ret;
826 }
827
828 if (track_ramblocks) {
829 uint64_t reply_gpa;
830
831 ret = vhost_user_read(dev, &msg_reply);
832 if (ret < 0) {
833 return ret;
834 }
835
836 reply_gpa = msg_reply.payload.mem_reg.region.guest_phys_addr;
837
838 if (msg_reply.hdr.request != VHOST_USER_ADD_MEM_REG) {
839 error_report("%s: Received unexpected msg type."
840 "Expected %d received %d", __func__,
841 VHOST_USER_ADD_MEM_REG,
842 msg_reply.hdr.request);
843 return -EPROTO;
844 }
845
846 /*
847 * We're using the same structure, just reusing one of the
848 * fields, so it should be the same size.
849 */
850 if (msg_reply.hdr.size != msg->hdr.size) {
851 error_report("%s: Unexpected size for postcopy reply "
852 "%d vs %d", __func__, msg_reply.hdr.size,
853 msg->hdr.size);
854 return -EPROTO;
855 }
856
857 /* Get the postcopy client base from the backend's reply. */
858 if (reply_gpa == dev->mem->regions[reg_idx].guest_phys_addr) {
859 shadow_pcb[reg_idx] =
860 msg_reply.payload.mem_reg.region.userspace_addr;
861 trace_vhost_user_set_mem_table_postcopy(
862 msg_reply.payload.mem_reg.region.userspace_addr,
863 msg->payload.mem_reg.region.userspace_addr,
864 reg_fd_idx, reg_idx);
865 } else {
866 error_report("%s: invalid postcopy reply for region. "
867 "Got guest physical address %" PRIX64 ", expected "
868 "%" PRIX64, __func__, reply_gpa,
869 dev->mem->regions[reg_idx].guest_phys_addr);
870 return -EPROTO;
871 }
872 } else if (reply_supported) {
873 ret = process_message_reply(dev, msg);
874 if (ret) {
875 return ret;
876 }
877 }
878 } else if (track_ramblocks) {
879 u->region_rb_offset[reg_idx] = 0;
880 u->region_rb[reg_idx] = NULL;
881 }
882
883 /*
884 * At this point, we know the backend has mapped in the new
885 * region, if the region has a valid file descriptor.
886 *
887 * The region should now be added to the shadow table.
888 */
889 u->shadow_regions[u->num_shadow_regions].guest_phys_addr =
890 reg->guest_phys_addr;
891 u->shadow_regions[u->num_shadow_regions].userspace_addr =
892 reg->userspace_addr;
893 u->shadow_regions[u->num_shadow_regions].memory_size =
894 reg->memory_size;
895 u->num_shadow_regions++;
896 }
897
898 return 0;
899 }
900
901 static int vhost_user_add_remove_regions(struct vhost_dev *dev,
902 VhostUserMsg *msg,
903 bool reply_supported,
904 bool track_ramblocks)
905 {
906 struct vhost_user *u = dev->opaque;
907 struct scrub_regions add_reg[VHOST_USER_MAX_RAM_SLOTS];
908 struct scrub_regions rem_reg[VHOST_USER_MAX_RAM_SLOTS];
909 uint64_t shadow_pcb[VHOST_USER_MAX_RAM_SLOTS] = {};
910 int nr_add_reg, nr_rem_reg;
911 int ret;
912
913 msg->hdr.size = sizeof(msg->payload.mem_reg);
914
915 /* Find the regions which need to be removed or added. */
916 scrub_shadow_regions(dev, add_reg, &nr_add_reg, rem_reg, &nr_rem_reg,
917 shadow_pcb, track_ramblocks);
918
919 if (nr_rem_reg) {
920 ret = send_remove_regions(dev, rem_reg, nr_rem_reg, msg,
921 reply_supported);
922 if (ret < 0) {
923 goto err;
924 }
925 }
926
927 if (nr_add_reg) {
928 ret = send_add_regions(dev, add_reg, nr_add_reg, msg, shadow_pcb,
929 reply_supported, track_ramblocks);
930 if (ret < 0) {
931 goto err;
932 }
933 }
934
935 if (track_ramblocks) {
936 memcpy(u->postcopy_client_bases, shadow_pcb,
937 sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS);
938 /*
939 * Now we've registered this with the postcopy code, we ack to the
940 * client, because now we're in the position to be able to deal with
941 * any faults it generates.
942 */
943 /* TODO: Use this for failure cases as well with a bad value. */
944 msg->hdr.size = sizeof(msg->payload.u64);
945 msg->payload.u64 = 0; /* OK */
946
947 ret = vhost_user_write(dev, msg, NULL, 0);
948 if (ret < 0) {
949 return ret;
950 }
951 }
952
953 return 0;
954
955 err:
956 if (track_ramblocks) {
957 memcpy(u->postcopy_client_bases, shadow_pcb,
958 sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS);
959 }
960
961 return ret;
962 }
963
964 static int vhost_user_set_mem_table_postcopy(struct vhost_dev *dev,
965 struct vhost_memory *mem,
966 bool reply_supported,
967 bool config_mem_slots)
968 {
969 struct vhost_user *u = dev->opaque;
970 int fds[VHOST_MEMORY_BASELINE_NREGIONS];
971 size_t fd_num = 0;
972 VhostUserMsg msg_reply;
973 int region_i, msg_i;
974 int ret;
975
976 VhostUserMsg msg = {
977 .hdr.flags = VHOST_USER_VERSION,
978 };
979
980 if (u->region_rb_len < dev->mem->nregions) {
981 u->region_rb = g_renew(RAMBlock*, u->region_rb, dev->mem->nregions);
982 u->region_rb_offset = g_renew(ram_addr_t, u->region_rb_offset,
983 dev->mem->nregions);
984 memset(&(u->region_rb[u->region_rb_len]), '\0',
985 sizeof(RAMBlock *) * (dev->mem->nregions - u->region_rb_len));
986 memset(&(u->region_rb_offset[u->region_rb_len]), '\0',
987 sizeof(ram_addr_t) * (dev->mem->nregions - u->region_rb_len));
988 u->region_rb_len = dev->mem->nregions;
989 }
990
991 if (config_mem_slots) {
992 ret = vhost_user_add_remove_regions(dev, &msg, reply_supported, true);
993 if (ret < 0) {
994 return ret;
995 }
996 } else {
997 ret = vhost_user_fill_set_mem_table_msg(u, dev, &msg, fds, &fd_num,
998 true);
999 if (ret < 0) {
1000 return ret;
1001 }
1002
1003 ret = vhost_user_write(dev, &msg, fds, fd_num);
1004 if (ret < 0) {
1005 return ret;
1006 }
1007
1008 ret = vhost_user_read(dev, &msg_reply);
1009 if (ret < 0) {
1010 return ret;
1011 }
1012
1013 if (msg_reply.hdr.request != VHOST_USER_SET_MEM_TABLE) {
1014 error_report("%s: Received unexpected msg type."
1015 "Expected %d received %d", __func__,
1016 VHOST_USER_SET_MEM_TABLE, msg_reply.hdr.request);
1017 return -EPROTO;
1018 }
1019
1020 /*
1021 * We're using the same structure, just reusing one of the
1022 * fields, so it should be the same size.
1023 */
1024 if (msg_reply.hdr.size != msg.hdr.size) {
1025 error_report("%s: Unexpected size for postcopy reply "
1026 "%d vs %d", __func__, msg_reply.hdr.size,
1027 msg.hdr.size);
1028 return -EPROTO;
1029 }
1030
1031 memset(u->postcopy_client_bases, 0,
1032 sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS);
1033
1034 /*
1035 * They're in the same order as the regions that were sent
1036 * but some of the regions were skipped (above) if they
1037 * didn't have fd's
1038 */
1039 for (msg_i = 0, region_i = 0;
1040 region_i < dev->mem->nregions;
1041 region_i++) {
1042 if (msg_i < fd_num &&
1043 msg_reply.payload.memory.regions[msg_i].guest_phys_addr ==
1044 dev->mem->regions[region_i].guest_phys_addr) {
1045 u->postcopy_client_bases[region_i] =
1046 msg_reply.payload.memory.regions[msg_i].userspace_addr;
1047 trace_vhost_user_set_mem_table_postcopy(
1048 msg_reply.payload.memory.regions[msg_i].userspace_addr,
1049 msg.payload.memory.regions[msg_i].userspace_addr,
1050 msg_i, region_i);
1051 msg_i++;
1052 }
1053 }
1054 if (msg_i != fd_num) {
1055 error_report("%s: postcopy reply not fully consumed "
1056 "%d vs %zd",
1057 __func__, msg_i, fd_num);
1058 return -EIO;
1059 }
1060
1061 /*
1062 * Now we've registered this with the postcopy code, we ack to the
1063 * client, because now we're in the position to be able to deal
1064 * with any faults it generates.
1065 */
1066 /* TODO: Use this for failure cases as well with a bad value. */
1067 msg.hdr.size = sizeof(msg.payload.u64);
1068 msg.payload.u64 = 0; /* OK */
1069 ret = vhost_user_write(dev, &msg, NULL, 0);
1070 if (ret < 0) {
1071 return ret;
1072 }
1073 }
1074
1075 return 0;
1076 }
1077
1078 static int vhost_user_set_mem_table(struct vhost_dev *dev,
1079 struct vhost_memory *mem)
1080 {
1081 struct vhost_user *u = dev->opaque;
1082 int fds[VHOST_MEMORY_BASELINE_NREGIONS];
1083 size_t fd_num = 0;
1084 bool do_postcopy = u->postcopy_listen && u->postcopy_fd.handler;
1085 bool reply_supported = virtio_has_feature(dev->protocol_features,
1086 VHOST_USER_PROTOCOL_F_REPLY_ACK);
1087 bool config_mem_slots =
1088 virtio_has_feature(dev->protocol_features,
1089 VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS);
1090 int ret;
1091
1092 if (do_postcopy) {
1093 /*
1094 * Postcopy has enough differences that it's best done in it's own
1095 * version
1096 */
1097 return vhost_user_set_mem_table_postcopy(dev, mem, reply_supported,
1098 config_mem_slots);
1099 }
1100
1101 VhostUserMsg msg = {
1102 .hdr.flags = VHOST_USER_VERSION,
1103 };
1104
1105 if (reply_supported) {
1106 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
1107 }
1108
1109 if (config_mem_slots) {
1110 ret = vhost_user_add_remove_regions(dev, &msg, reply_supported, false);
1111 if (ret < 0) {
1112 return ret;
1113 }
1114 } else {
1115 ret = vhost_user_fill_set_mem_table_msg(u, dev, &msg, fds, &fd_num,
1116 false);
1117 if (ret < 0) {
1118 return ret;
1119 }
1120
1121 ret = vhost_user_write(dev, &msg, fds, fd_num);
1122 if (ret < 0) {
1123 return ret;
1124 }
1125
1126 if (reply_supported) {
1127 return process_message_reply(dev, &msg);
1128 }
1129 }
1130
1131 return 0;
1132 }
1133
1134 static int vhost_user_set_vring_endian(struct vhost_dev *dev,
1135 struct vhost_vring_state *ring)
1136 {
1137 bool cross_endian = virtio_has_feature(dev->protocol_features,
1138 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN);
1139 VhostUserMsg msg = {
1140 .hdr.request = VHOST_USER_SET_VRING_ENDIAN,
1141 .hdr.flags = VHOST_USER_VERSION,
1142 .payload.state = *ring,
1143 .hdr.size = sizeof(msg.payload.state),
1144 };
1145
1146 if (!cross_endian) {
1147 error_report("vhost-user trying to send unhandled ioctl");
1148 return -ENOTSUP;
1149 }
1150
1151 return vhost_user_write(dev, &msg, NULL, 0);
1152 }
1153
1154 static int vhost_set_vring(struct vhost_dev *dev,
1155 unsigned long int request,
1156 struct vhost_vring_state *ring)
1157 {
1158 VhostUserMsg msg = {
1159 .hdr.request = request,
1160 .hdr.flags = VHOST_USER_VERSION,
1161 .payload.state = *ring,
1162 .hdr.size = sizeof(msg.payload.state),
1163 };
1164
1165 return vhost_user_write(dev, &msg, NULL, 0);
1166 }
1167
1168 static int vhost_user_set_vring_num(struct vhost_dev *dev,
1169 struct vhost_vring_state *ring)
1170 {
1171 return vhost_set_vring(dev, VHOST_USER_SET_VRING_NUM, ring);
1172 }
1173
1174 static void vhost_user_host_notifier_free(VhostUserHostNotifier *n)
1175 {
1176 assert(n && n->unmap_addr);
1177 munmap(n->unmap_addr, qemu_real_host_page_size());
1178 n->unmap_addr = NULL;
1179 }
1180
1181 /*
1182 * clean-up function for notifier, will finally free the structure
1183 * under rcu.
1184 */
1185 static void vhost_user_host_notifier_remove(VhostUserHostNotifier *n,
1186 VirtIODevice *vdev)
1187 {
1188 if (n->addr) {
1189 if (vdev) {
1190 virtio_queue_set_host_notifier_mr(vdev, n->idx, &n->mr, false);
1191 }
1192 assert(!n->unmap_addr);
1193 n->unmap_addr = n->addr;
1194 n->addr = NULL;
1195 call_rcu(n, vhost_user_host_notifier_free, rcu);
1196 }
1197 }
1198
1199 static int vhost_user_set_vring_base(struct vhost_dev *dev,
1200 struct vhost_vring_state *ring)
1201 {
1202 return vhost_set_vring(dev, VHOST_USER_SET_VRING_BASE, ring);
1203 }
1204
1205 static int vhost_user_set_vring_enable(struct vhost_dev *dev, int enable)
1206 {
1207 int i;
1208
1209 if (!virtio_has_feature(dev->features, VHOST_USER_F_PROTOCOL_FEATURES)) {
1210 return -EINVAL;
1211 }
1212
1213 for (i = 0; i < dev->nvqs; ++i) {
1214 int ret;
1215 struct vhost_vring_state state = {
1216 .index = dev->vq_index + i,
1217 .num = enable,
1218 };
1219
1220 ret = vhost_set_vring(dev, VHOST_USER_SET_VRING_ENABLE, &state);
1221 if (ret < 0) {
1222 /*
1223 * Restoring the previous state is likely infeasible, as well as
1224 * proceeding regardless the error, so just bail out and hope for
1225 * the device-level recovery.
1226 */
1227 return ret;
1228 }
1229 }
1230
1231 return 0;
1232 }
1233
1234 static VhostUserHostNotifier *fetch_notifier(VhostUserState *u,
1235 int idx)
1236 {
1237 if (idx >= u->notifiers->len) {
1238 return NULL;
1239 }
1240 return g_ptr_array_index(u->notifiers, idx);
1241 }
1242
1243 static int vhost_user_get_vring_base(struct vhost_dev *dev,
1244 struct vhost_vring_state *ring)
1245 {
1246 int ret;
1247 VhostUserMsg msg = {
1248 .hdr.request = VHOST_USER_GET_VRING_BASE,
1249 .hdr.flags = VHOST_USER_VERSION,
1250 .payload.state = *ring,
1251 .hdr.size = sizeof(msg.payload.state),
1252 };
1253 struct vhost_user *u = dev->opaque;
1254
1255 VhostUserHostNotifier *n = fetch_notifier(u->user, ring->index);
1256 if (n) {
1257 vhost_user_host_notifier_remove(n, dev->vdev);
1258 }
1259
1260 ret = vhost_user_write(dev, &msg, NULL, 0);
1261 if (ret < 0) {
1262 return ret;
1263 }
1264
1265 ret = vhost_user_read(dev, &msg);
1266 if (ret < 0) {
1267 return ret;
1268 }
1269
1270 if (msg.hdr.request != VHOST_USER_GET_VRING_BASE) {
1271 error_report("Received unexpected msg type. Expected %d received %d",
1272 VHOST_USER_GET_VRING_BASE, msg.hdr.request);
1273 return -EPROTO;
1274 }
1275
1276 if (msg.hdr.size != sizeof(msg.payload.state)) {
1277 error_report("Received bad msg size.");
1278 return -EPROTO;
1279 }
1280
1281 *ring = msg.payload.state;
1282
1283 return 0;
1284 }
1285
1286 static int vhost_set_vring_file(struct vhost_dev *dev,
1287 VhostUserRequest request,
1288 struct vhost_vring_file *file)
1289 {
1290 int fds[VHOST_USER_MAX_RAM_SLOTS];
1291 size_t fd_num = 0;
1292 VhostUserMsg msg = {
1293 .hdr.request = request,
1294 .hdr.flags = VHOST_USER_VERSION,
1295 .payload.u64 = file->index & VHOST_USER_VRING_IDX_MASK,
1296 .hdr.size = sizeof(msg.payload.u64),
1297 };
1298
1299 if (ioeventfd_enabled() && file->fd > 0) {
1300 fds[fd_num++] = file->fd;
1301 } else {
1302 msg.payload.u64 |= VHOST_USER_VRING_NOFD_MASK;
1303 }
1304
1305 return vhost_user_write(dev, &msg, fds, fd_num);
1306 }
1307
1308 static int vhost_user_set_vring_kick(struct vhost_dev *dev,
1309 struct vhost_vring_file *file)
1310 {
1311 return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_KICK, file);
1312 }
1313
1314 static int vhost_user_set_vring_call(struct vhost_dev *dev,
1315 struct vhost_vring_file *file)
1316 {
1317 return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_CALL, file);
1318 }
1319
1320 static int vhost_user_set_vring_err(struct vhost_dev *dev,
1321 struct vhost_vring_file *file)
1322 {
1323 return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_ERR, file);
1324 }
1325
1326 static int vhost_user_get_u64(struct vhost_dev *dev, int request, uint64_t *u64)
1327 {
1328 int ret;
1329 VhostUserMsg msg = {
1330 .hdr.request = request,
1331 .hdr.flags = VHOST_USER_VERSION,
1332 };
1333
1334 if (vhost_user_one_time_request(request) && dev->vq_index != 0) {
1335 return 0;
1336 }
1337
1338 ret = vhost_user_write(dev, &msg, NULL, 0);
1339 if (ret < 0) {
1340 return ret;
1341 }
1342
1343 ret = vhost_user_read(dev, &msg);
1344 if (ret < 0) {
1345 return ret;
1346 }
1347
1348 if (msg.hdr.request != request) {
1349 error_report("Received unexpected msg type. Expected %d received %d",
1350 request, msg.hdr.request);
1351 return -EPROTO;
1352 }
1353
1354 if (msg.hdr.size != sizeof(msg.payload.u64)) {
1355 error_report("Received bad msg size.");
1356 return -EPROTO;
1357 }
1358
1359 *u64 = msg.payload.u64;
1360
1361 return 0;
1362 }
1363
1364 static int vhost_user_get_features(struct vhost_dev *dev, uint64_t *features)
1365 {
1366 if (vhost_user_get_u64(dev, VHOST_USER_GET_FEATURES, features) < 0) {
1367 return -EPROTO;
1368 }
1369
1370 return 0;
1371 }
1372
1373 static int enforce_reply(struct vhost_dev *dev,
1374 const VhostUserMsg *msg)
1375 {
1376 uint64_t dummy;
1377
1378 if (msg->hdr.flags & VHOST_USER_NEED_REPLY_MASK) {
1379 return process_message_reply(dev, msg);
1380 }
1381
1382 /*
1383 * We need to wait for a reply but the backend does not
1384 * support replies for the command we just sent.
1385 * Send VHOST_USER_GET_FEATURES which makes all backends
1386 * send a reply.
1387 */
1388 return vhost_user_get_features(dev, &dummy);
1389 }
1390
1391 static int vhost_user_set_vring_addr(struct vhost_dev *dev,
1392 struct vhost_vring_addr *addr)
1393 {
1394 int ret;
1395 VhostUserMsg msg = {
1396 .hdr.request = VHOST_USER_SET_VRING_ADDR,
1397 .hdr.flags = VHOST_USER_VERSION,
1398 .payload.addr = *addr,
1399 .hdr.size = sizeof(msg.payload.addr),
1400 };
1401
1402 bool reply_supported = virtio_has_feature(dev->protocol_features,
1403 VHOST_USER_PROTOCOL_F_REPLY_ACK);
1404
1405 /*
1406 * wait for a reply if logging is enabled to make sure
1407 * backend is actually logging changes
1408 */
1409 bool wait_for_reply = addr->flags & (1 << VHOST_VRING_F_LOG);
1410
1411 if (reply_supported && wait_for_reply) {
1412 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
1413 }
1414
1415 ret = vhost_user_write(dev, &msg, NULL, 0);
1416 if (ret < 0) {
1417 return ret;
1418 }
1419
1420 if (wait_for_reply) {
1421 return enforce_reply(dev, &msg);
1422 }
1423
1424 return 0;
1425 }
1426
1427 static int vhost_user_set_u64(struct vhost_dev *dev, int request, uint64_t u64,
1428 bool wait_for_reply)
1429 {
1430 VhostUserMsg msg = {
1431 .hdr.request = request,
1432 .hdr.flags = VHOST_USER_VERSION,
1433 .payload.u64 = u64,
1434 .hdr.size = sizeof(msg.payload.u64),
1435 };
1436 int ret;
1437
1438 if (wait_for_reply) {
1439 bool reply_supported = virtio_has_feature(dev->protocol_features,
1440 VHOST_USER_PROTOCOL_F_REPLY_ACK);
1441 if (reply_supported) {
1442 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
1443 }
1444 }
1445
1446 ret = vhost_user_write(dev, &msg, NULL, 0);
1447 if (ret < 0) {
1448 return ret;
1449 }
1450
1451 if (wait_for_reply) {
1452 return enforce_reply(dev, &msg);
1453 }
1454
1455 return 0;
1456 }
1457
1458 static int vhost_user_set_status(struct vhost_dev *dev, uint8_t status)
1459 {
1460 return vhost_user_set_u64(dev, VHOST_USER_SET_STATUS, status, false);
1461 }
1462
1463 static int vhost_user_get_status(struct vhost_dev *dev, uint8_t *status)
1464 {
1465 uint64_t value;
1466 int ret;
1467
1468 ret = vhost_user_get_u64(dev, VHOST_USER_GET_STATUS, &value);
1469 if (ret < 0) {
1470 return ret;
1471 }
1472 *status = value;
1473
1474 return 0;
1475 }
1476
1477 static int vhost_user_add_status(struct vhost_dev *dev, uint8_t status)
1478 {
1479 uint8_t s;
1480 int ret;
1481
1482 ret = vhost_user_get_status(dev, &s);
1483 if (ret < 0) {
1484 return ret;
1485 }
1486
1487 if ((s & status) == status) {
1488 return 0;
1489 }
1490 s |= status;
1491
1492 return vhost_user_set_status(dev, s);
1493 }
1494
1495 static int vhost_user_set_features(struct vhost_dev *dev,
1496 uint64_t features)
1497 {
1498 /*
1499 * wait for a reply if logging is enabled to make sure
1500 * backend is actually logging changes
1501 */
1502 bool log_enabled = features & (0x1ULL << VHOST_F_LOG_ALL);
1503 int ret;
1504
1505 /*
1506 * We need to include any extra backend only feature bits that
1507 * might be needed by our device. Currently this includes the
1508 * VHOST_USER_F_PROTOCOL_FEATURES bit for enabling protocol
1509 * features.
1510 */
1511 ret = vhost_user_set_u64(dev, VHOST_USER_SET_FEATURES,
1512 features | dev->backend_features,
1513 log_enabled);
1514
1515 if (virtio_has_feature(dev->protocol_features,
1516 VHOST_USER_PROTOCOL_F_STATUS)) {
1517 if (!ret) {
1518 return vhost_user_add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK);
1519 }
1520 }
1521
1522 return ret;
1523 }
1524
1525 static int vhost_user_set_protocol_features(struct vhost_dev *dev,
1526 uint64_t features)
1527 {
1528 return vhost_user_set_u64(dev, VHOST_USER_SET_PROTOCOL_FEATURES, features,
1529 false);
1530 }
1531
1532 static int vhost_user_set_owner(struct vhost_dev *dev)
1533 {
1534 VhostUserMsg msg = {
1535 .hdr.request = VHOST_USER_SET_OWNER,
1536 .hdr.flags = VHOST_USER_VERSION,
1537 };
1538
1539 return vhost_user_write(dev, &msg, NULL, 0);
1540 }
1541
1542 static int vhost_user_get_max_memslots(struct vhost_dev *dev,
1543 uint64_t *max_memslots)
1544 {
1545 uint64_t backend_max_memslots;
1546 int err;
1547
1548 err = vhost_user_get_u64(dev, VHOST_USER_GET_MAX_MEM_SLOTS,
1549 &backend_max_memslots);
1550 if (err < 0) {
1551 return err;
1552 }
1553
1554 *max_memslots = backend_max_memslots;
1555
1556 return 0;
1557 }
1558
1559 static int vhost_user_reset_device(struct vhost_dev *dev)
1560 {
1561 VhostUserMsg msg = {
1562 .hdr.flags = VHOST_USER_VERSION,
1563 };
1564
1565 msg.hdr.request = virtio_has_feature(dev->protocol_features,
1566 VHOST_USER_PROTOCOL_F_RESET_DEVICE)
1567 ? VHOST_USER_RESET_DEVICE
1568 : VHOST_USER_RESET_OWNER;
1569
1570 return vhost_user_write(dev, &msg, NULL, 0);
1571 }
1572
1573 static int vhost_user_slave_handle_config_change(struct vhost_dev *dev)
1574 {
1575 if (!dev->config_ops || !dev->config_ops->vhost_dev_config_notifier) {
1576 return -ENOSYS;
1577 }
1578
1579 return dev->config_ops->vhost_dev_config_notifier(dev);
1580 }
1581
1582 /*
1583 * Fetch or create the notifier for a given idx. Newly created
1584 * notifiers are added to the pointer array that tracks them.
1585 */
1586 static VhostUserHostNotifier *fetch_or_create_notifier(VhostUserState *u,
1587 int idx)
1588 {
1589 VhostUserHostNotifier *n = NULL;
1590 if (idx >= u->notifiers->len) {
1591 g_ptr_array_set_size(u->notifiers, idx + 1);
1592 }
1593
1594 n = g_ptr_array_index(u->notifiers, idx);
1595 if (!n) {
1596 /*
1597 * In case notification arrive out-of-order,
1598 * make room for current index.
1599 */
1600 g_ptr_array_remove_index(u->notifiers, idx);
1601 n = g_new0(VhostUserHostNotifier, 1);
1602 n->idx = idx;
1603 g_ptr_array_insert(u->notifiers, idx, n);
1604 trace_vhost_user_create_notifier(idx, n);
1605 }
1606
1607 return n;
1608 }
1609
1610 static int vhost_user_slave_handle_vring_host_notifier(struct vhost_dev *dev,
1611 VhostUserVringArea *area,
1612 int fd)
1613 {
1614 int queue_idx = area->u64 & VHOST_USER_VRING_IDX_MASK;
1615 size_t page_size = qemu_real_host_page_size();
1616 struct vhost_user *u = dev->opaque;
1617 VhostUserState *user = u->user;
1618 VirtIODevice *vdev = dev->vdev;
1619 VhostUserHostNotifier *n;
1620 void *addr;
1621 char *name;
1622
1623 if (!virtio_has_feature(dev->protocol_features,
1624 VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) ||
1625 vdev == NULL || queue_idx >= virtio_get_num_queues(vdev)) {
1626 return -EINVAL;
1627 }
1628
1629 /*
1630 * Fetch notifier and invalidate any old data before setting up
1631 * new mapped address.
1632 */
1633 n = fetch_or_create_notifier(user, queue_idx);
1634 vhost_user_host_notifier_remove(n, vdev);
1635
1636 if (area->u64 & VHOST_USER_VRING_NOFD_MASK) {
1637 return 0;
1638 }
1639
1640 /* Sanity check. */
1641 if (area->size != page_size) {
1642 return -EINVAL;
1643 }
1644
1645 addr = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED,
1646 fd, area->offset);
1647 if (addr == MAP_FAILED) {
1648 return -EFAULT;
1649 }
1650
1651 name = g_strdup_printf("vhost-user/host-notifier@%p mmaps[%d]",
1652 user, queue_idx);
1653 if (!n->mr.ram) { /* Don't init again after suspend. */
1654 memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name,
1655 page_size, addr);
1656 } else {
1657 n->mr.ram_block->host = addr;
1658 }
1659 g_free(name);
1660
1661 if (virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true)) {
1662 object_unparent(OBJECT(&n->mr));
1663 munmap(addr, page_size);
1664 return -ENXIO;
1665 }
1666
1667 n->addr = addr;
1668
1669 return 0;
1670 }
1671
1672 static void close_slave_channel(struct vhost_user *u)
1673 {
1674 g_source_destroy(u->slave_src);
1675 g_source_unref(u->slave_src);
1676 u->slave_src = NULL;
1677 object_unref(OBJECT(u->slave_ioc));
1678 u->slave_ioc = NULL;
1679 }
1680
1681 static gboolean slave_read(QIOChannel *ioc, GIOCondition condition,
1682 gpointer opaque)
1683 {
1684 struct vhost_dev *dev = opaque;
1685 struct vhost_user *u = dev->opaque;
1686 VhostUserHeader hdr = { 0, };
1687 VhostUserPayload payload = { 0, };
1688 Error *local_err = NULL;
1689 gboolean rc = G_SOURCE_CONTINUE;
1690 int ret = 0;
1691 struct iovec iov;
1692 g_autofree int *fd = NULL;
1693 size_t fdsize = 0;
1694 int i;
1695
1696 /* Read header */
1697 iov.iov_base = &hdr;
1698 iov.iov_len = VHOST_USER_HDR_SIZE;
1699
1700 if (qio_channel_readv_full_all(ioc, &iov, 1, &fd, &fdsize, &local_err)) {
1701 error_report_err(local_err);
1702 goto err;
1703 }
1704
1705 if (hdr.size > VHOST_USER_PAYLOAD_SIZE) {
1706 error_report("Failed to read msg header."
1707 " Size %d exceeds the maximum %zu.", hdr.size,
1708 VHOST_USER_PAYLOAD_SIZE);
1709 goto err;
1710 }
1711
1712 /* Read payload */
1713 if (qio_channel_read_all(ioc, (char *) &payload, hdr.size, &local_err)) {
1714 error_report_err(local_err);
1715 goto err;
1716 }
1717
1718 switch (hdr.request) {
1719 case VHOST_USER_SLAVE_IOTLB_MSG:
1720 ret = vhost_backend_handle_iotlb_msg(dev, &payload.iotlb);
1721 break;
1722 case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG :
1723 ret = vhost_user_slave_handle_config_change(dev);
1724 break;
1725 case VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG:
1726 ret = vhost_user_slave_handle_vring_host_notifier(dev, &payload.area,
1727 fd ? fd[0] : -1);
1728 break;
1729 default:
1730 error_report("Received unexpected msg type: %d.", hdr.request);
1731 ret = -EINVAL;
1732 }
1733
1734 /*
1735 * REPLY_ACK feature handling. Other reply types has to be managed
1736 * directly in their request handlers.
1737 */
1738 if (hdr.flags & VHOST_USER_NEED_REPLY_MASK) {
1739 struct iovec iovec[2];
1740
1741
1742 hdr.flags &= ~VHOST_USER_NEED_REPLY_MASK;
1743 hdr.flags |= VHOST_USER_REPLY_MASK;
1744
1745 payload.u64 = !!ret;
1746 hdr.size = sizeof(payload.u64);
1747
1748 iovec[0].iov_base = &hdr;
1749 iovec[0].iov_len = VHOST_USER_HDR_SIZE;
1750 iovec[1].iov_base = &payload;
1751 iovec[1].iov_len = hdr.size;
1752
1753 if (qio_channel_writev_all(ioc, iovec, ARRAY_SIZE(iovec), &local_err)) {
1754 error_report_err(local_err);
1755 goto err;
1756 }
1757 }
1758
1759 goto fdcleanup;
1760
1761 err:
1762 close_slave_channel(u);
1763 rc = G_SOURCE_REMOVE;
1764
1765 fdcleanup:
1766 if (fd) {
1767 for (i = 0; i < fdsize; i++) {
1768 close(fd[i]);
1769 }
1770 }
1771 return rc;
1772 }
1773
1774 static int vhost_setup_slave_channel(struct vhost_dev *dev)
1775 {
1776 VhostUserMsg msg = {
1777 .hdr.request = VHOST_USER_SET_SLAVE_REQ_FD,
1778 .hdr.flags = VHOST_USER_VERSION,
1779 };
1780 struct vhost_user *u = dev->opaque;
1781 int sv[2], ret = 0;
1782 bool reply_supported = virtio_has_feature(dev->protocol_features,
1783 VHOST_USER_PROTOCOL_F_REPLY_ACK);
1784 Error *local_err = NULL;
1785 QIOChannel *ioc;
1786
1787 if (!virtio_has_feature(dev->protocol_features,
1788 VHOST_USER_PROTOCOL_F_SLAVE_REQ)) {
1789 return 0;
1790 }
1791
1792 if (qemu_socketpair(PF_UNIX, SOCK_STREAM, 0, sv) == -1) {
1793 int saved_errno = errno;
1794 error_report("socketpair() failed");
1795 return -saved_errno;
1796 }
1797
1798 ioc = QIO_CHANNEL(qio_channel_socket_new_fd(sv[0], &local_err));
1799 if (!ioc) {
1800 error_report_err(local_err);
1801 return -ECONNREFUSED;
1802 }
1803 u->slave_ioc = ioc;
1804 slave_update_read_handler(dev, NULL);
1805
1806 if (reply_supported) {
1807 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
1808 }
1809
1810 ret = vhost_user_write(dev, &msg, &sv[1], 1);
1811 if (ret) {
1812 goto out;
1813 }
1814
1815 if (reply_supported) {
1816 ret = process_message_reply(dev, &msg);
1817 }
1818
1819 out:
1820 close(sv[1]);
1821 if (ret) {
1822 close_slave_channel(u);
1823 }
1824
1825 return ret;
1826 }
1827
1828 #ifdef CONFIG_LINUX
1829 /*
1830 * Called back from the postcopy fault thread when a fault is received on our
1831 * ufd.
1832 * TODO: This is Linux specific
1833 */
1834 static int vhost_user_postcopy_fault_handler(struct PostCopyFD *pcfd,
1835 void *ufd)
1836 {
1837 struct vhost_dev *dev = pcfd->data;
1838 struct vhost_user *u = dev->opaque;
1839 struct uffd_msg *msg = ufd;
1840 uint64_t faultaddr = msg->arg.pagefault.address;
1841 RAMBlock *rb = NULL;
1842 uint64_t rb_offset;
1843 int i;
1844
1845 trace_vhost_user_postcopy_fault_handler(pcfd->idstr, faultaddr,
1846 dev->mem->nregions);
1847 for (i = 0; i < MIN(dev->mem->nregions, u->region_rb_len); i++) {
1848 trace_vhost_user_postcopy_fault_handler_loop(i,
1849 u->postcopy_client_bases[i], dev->mem->regions[i].memory_size);
1850 if (faultaddr >= u->postcopy_client_bases[i]) {
1851 /* Ofset of the fault address in the vhost region */
1852 uint64_t region_offset = faultaddr - u->postcopy_client_bases[i];
1853 if (region_offset < dev->mem->regions[i].memory_size) {
1854 rb_offset = region_offset + u->region_rb_offset[i];
1855 trace_vhost_user_postcopy_fault_handler_found(i,
1856 region_offset, rb_offset);
1857 rb = u->region_rb[i];
1858 return postcopy_request_shared_page(pcfd, rb, faultaddr,
1859 rb_offset);
1860 }
1861 }
1862 }
1863 error_report("%s: Failed to find region for fault %" PRIx64,
1864 __func__, faultaddr);
1865 return -1;
1866 }
1867
1868 static int vhost_user_postcopy_waker(struct PostCopyFD *pcfd, RAMBlock *rb,
1869 uint64_t offset)
1870 {
1871 struct vhost_dev *dev = pcfd->data;
1872 struct vhost_user *u = dev->opaque;
1873 int i;
1874
1875 trace_vhost_user_postcopy_waker(qemu_ram_get_idstr(rb), offset);
1876
1877 if (!u) {
1878 return 0;
1879 }
1880 /* Translate the offset into an address in the clients address space */
1881 for (i = 0; i < MIN(dev->mem->nregions, u->region_rb_len); i++) {
1882 if (u->region_rb[i] == rb &&
1883 offset >= u->region_rb_offset[i] &&
1884 offset < (u->region_rb_offset[i] +
1885 dev->mem->regions[i].memory_size)) {
1886 uint64_t client_addr = (offset - u->region_rb_offset[i]) +
1887 u->postcopy_client_bases[i];
1888 trace_vhost_user_postcopy_waker_found(client_addr);
1889 return postcopy_wake_shared(pcfd, client_addr, rb);
1890 }
1891 }
1892
1893 trace_vhost_user_postcopy_waker_nomatch(qemu_ram_get_idstr(rb), offset);
1894 return 0;
1895 }
1896 #endif
1897
1898 /*
1899 * Called at the start of an inbound postcopy on reception of the
1900 * 'advise' command.
1901 */
1902 static int vhost_user_postcopy_advise(struct vhost_dev *dev, Error **errp)
1903 {
1904 #ifdef CONFIG_LINUX
1905 struct vhost_user *u = dev->opaque;
1906 CharBackend *chr = u->user->chr;
1907 int ufd;
1908 int ret;
1909 VhostUserMsg msg = {
1910 .hdr.request = VHOST_USER_POSTCOPY_ADVISE,
1911 .hdr.flags = VHOST_USER_VERSION,
1912 };
1913
1914 ret = vhost_user_write(dev, &msg, NULL, 0);
1915 if (ret < 0) {
1916 error_setg(errp, "Failed to send postcopy_advise to vhost");
1917 return ret;
1918 }
1919
1920 ret = vhost_user_read(dev, &msg);
1921 if (ret < 0) {
1922 error_setg(errp, "Failed to get postcopy_advise reply from vhost");
1923 return ret;
1924 }
1925
1926 if (msg.hdr.request != VHOST_USER_POSTCOPY_ADVISE) {
1927 error_setg(errp, "Unexpected msg type. Expected %d received %d",
1928 VHOST_USER_POSTCOPY_ADVISE, msg.hdr.request);
1929 return -EPROTO;
1930 }
1931
1932 if (msg.hdr.size) {
1933 error_setg(errp, "Received bad msg size.");
1934 return -EPROTO;
1935 }
1936 ufd = qemu_chr_fe_get_msgfd(chr);
1937 if (ufd < 0) {
1938 error_setg(errp, "%s: Failed to get ufd", __func__);
1939 return -EIO;
1940 }
1941 qemu_socket_set_nonblock(ufd);
1942
1943 /* register ufd with userfault thread */
1944 u->postcopy_fd.fd = ufd;
1945 u->postcopy_fd.data = dev;
1946 u->postcopy_fd.handler = vhost_user_postcopy_fault_handler;
1947 u->postcopy_fd.waker = vhost_user_postcopy_waker;
1948 u->postcopy_fd.idstr = "vhost-user"; /* Need to find unique name */
1949 postcopy_register_shared_ufd(&u->postcopy_fd);
1950 return 0;
1951 #else
1952 error_setg(errp, "Postcopy not supported on non-Linux systems");
1953 return -ENOSYS;
1954 #endif
1955 }
1956
1957 /*
1958 * Called at the switch to postcopy on reception of the 'listen' command.
1959 */
1960 static int vhost_user_postcopy_listen(struct vhost_dev *dev, Error **errp)
1961 {
1962 struct vhost_user *u = dev->opaque;
1963 int ret;
1964 VhostUserMsg msg = {
1965 .hdr.request = VHOST_USER_POSTCOPY_LISTEN,
1966 .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
1967 };
1968 u->postcopy_listen = true;
1969
1970 trace_vhost_user_postcopy_listen();
1971
1972 ret = vhost_user_write(dev, &msg, NULL, 0);
1973 if (ret < 0) {
1974 error_setg(errp, "Failed to send postcopy_listen to vhost");
1975 return ret;
1976 }
1977
1978 ret = process_message_reply(dev, &msg);
1979 if (ret) {
1980 error_setg(errp, "Failed to receive reply to postcopy_listen");
1981 return ret;
1982 }
1983
1984 return 0;
1985 }
1986
1987 /*
1988 * Called at the end of postcopy
1989 */
1990 static int vhost_user_postcopy_end(struct vhost_dev *dev, Error **errp)
1991 {
1992 VhostUserMsg msg = {
1993 .hdr.request = VHOST_USER_POSTCOPY_END,
1994 .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
1995 };
1996 int ret;
1997 struct vhost_user *u = dev->opaque;
1998
1999 trace_vhost_user_postcopy_end_entry();
2000
2001 ret = vhost_user_write(dev, &msg, NULL, 0);
2002 if (ret < 0) {
2003 error_setg(errp, "Failed to send postcopy_end to vhost");
2004 return ret;
2005 }
2006
2007 ret = process_message_reply(dev, &msg);
2008 if (ret) {
2009 error_setg(errp, "Failed to receive reply to postcopy_end");
2010 return ret;
2011 }
2012 postcopy_unregister_shared_ufd(&u->postcopy_fd);
2013 close(u->postcopy_fd.fd);
2014 u->postcopy_fd.handler = NULL;
2015
2016 trace_vhost_user_postcopy_end_exit();
2017
2018 return 0;
2019 }
2020
2021 static int vhost_user_postcopy_notifier(NotifierWithReturn *notifier,
2022 void *opaque)
2023 {
2024 struct PostcopyNotifyData *pnd = opaque;
2025 struct vhost_user *u = container_of(notifier, struct vhost_user,
2026 postcopy_notifier);
2027 struct vhost_dev *dev = u->dev;
2028
2029 switch (pnd->reason) {
2030 case POSTCOPY_NOTIFY_PROBE:
2031 if (!virtio_has_feature(dev->protocol_features,
2032 VHOST_USER_PROTOCOL_F_PAGEFAULT)) {
2033 /* TODO: Get the device name into this error somehow */
2034 error_setg(pnd->errp,
2035 "vhost-user backend not capable of postcopy");
2036 return -ENOENT;
2037 }
2038 break;
2039
2040 case POSTCOPY_NOTIFY_INBOUND_ADVISE:
2041 return vhost_user_postcopy_advise(dev, pnd->errp);
2042
2043 case POSTCOPY_NOTIFY_INBOUND_LISTEN:
2044 return vhost_user_postcopy_listen(dev, pnd->errp);
2045
2046 case POSTCOPY_NOTIFY_INBOUND_END:
2047 return vhost_user_postcopy_end(dev, pnd->errp);
2048
2049 default:
2050 /* We ignore notifications we don't know */
2051 break;
2052 }
2053
2054 return 0;
2055 }
2056
2057 static int vhost_user_backend_init(struct vhost_dev *dev, void *opaque,
2058 Error **errp)
2059 {
2060 uint64_t features, ram_slots;
2061 struct vhost_user *u;
2062 VhostUserState *vus = (VhostUserState *) opaque;
2063 int err;
2064
2065 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
2066
2067 u = g_new0(struct vhost_user, 1);
2068 u->user = vus;
2069 u->dev = dev;
2070 dev->opaque = u;
2071
2072 err = vhost_user_get_features(dev, &features);
2073 if (err < 0) {
2074 error_setg_errno(errp, -err, "vhost_backend_init failed");
2075 return err;
2076 }
2077
2078 if (virtio_has_feature(features, VHOST_USER_F_PROTOCOL_FEATURES)) {
2079 bool supports_f_config = vus->supports_config ||
2080 (dev->config_ops && dev->config_ops->vhost_dev_config_notifier);
2081 uint64_t protocol_features;
2082
2083 dev->backend_features |= 1ULL << VHOST_USER_F_PROTOCOL_FEATURES;
2084
2085 err = vhost_user_get_u64(dev, VHOST_USER_GET_PROTOCOL_FEATURES,
2086 &protocol_features);
2087 if (err < 0) {
2088 error_setg_errno(errp, EPROTO, "vhost_backend_init failed");
2089 return -EPROTO;
2090 }
2091
2092 /*
2093 * We will use all the protocol features we support - although
2094 * we suppress F_CONFIG if we know QEMUs internal code can not support
2095 * it.
2096 */
2097 protocol_features &= VHOST_USER_PROTOCOL_FEATURE_MASK;
2098
2099 if (supports_f_config) {
2100 if (!virtio_has_feature(protocol_features,
2101 VHOST_USER_PROTOCOL_F_CONFIG)) {
2102 error_setg(errp, "vhost-user device expecting "
2103 "VHOST_USER_PROTOCOL_F_CONFIG but the vhost-user backend does "
2104 "not support it.");
2105 return -EPROTO;
2106 }
2107 } else {
2108 if (virtio_has_feature(protocol_features,
2109 VHOST_USER_PROTOCOL_F_CONFIG)) {
2110 warn_reportf_err(*errp, "vhost-user backend supports "
2111 "VHOST_USER_PROTOCOL_F_CONFIG but QEMU does not.");
2112 protocol_features &= ~(1ULL << VHOST_USER_PROTOCOL_F_CONFIG);
2113 }
2114 }
2115
2116 /* final set of protocol features */
2117 dev->protocol_features = protocol_features;
2118 err = vhost_user_set_protocol_features(dev, dev->protocol_features);
2119 if (err < 0) {
2120 error_setg_errno(errp, EPROTO, "vhost_backend_init failed");
2121 return -EPROTO;
2122 }
2123
2124 /* query the max queues we support if backend supports Multiple Queue */
2125 if (dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_MQ)) {
2126 err = vhost_user_get_u64(dev, VHOST_USER_GET_QUEUE_NUM,
2127 &dev->max_queues);
2128 if (err < 0) {
2129 error_setg_errno(errp, EPROTO, "vhost_backend_init failed");
2130 return -EPROTO;
2131 }
2132 } else {
2133 dev->max_queues = 1;
2134 }
2135
2136 if (dev->num_queues && dev->max_queues < dev->num_queues) {
2137 error_setg(errp, "The maximum number of queues supported by the "
2138 "backend is %" PRIu64, dev->max_queues);
2139 return -EINVAL;
2140 }
2141
2142 if (virtio_has_feature(features, VIRTIO_F_IOMMU_PLATFORM) &&
2143 !(virtio_has_feature(dev->protocol_features,
2144 VHOST_USER_PROTOCOL_F_SLAVE_REQ) &&
2145 virtio_has_feature(dev->protocol_features,
2146 VHOST_USER_PROTOCOL_F_REPLY_ACK))) {
2147 error_setg(errp, "IOMMU support requires reply-ack and "
2148 "slave-req protocol features.");
2149 return -EINVAL;
2150 }
2151
2152 /* get max memory regions if backend supports configurable RAM slots */
2153 if (!virtio_has_feature(dev->protocol_features,
2154 VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS)) {
2155 u->user->memory_slots = VHOST_MEMORY_BASELINE_NREGIONS;
2156 } else {
2157 err = vhost_user_get_max_memslots(dev, &ram_slots);
2158 if (err < 0) {
2159 error_setg_errno(errp, EPROTO, "vhost_backend_init failed");
2160 return -EPROTO;
2161 }
2162
2163 if (ram_slots < u->user->memory_slots) {
2164 error_setg(errp, "The backend specified a max ram slots limit "
2165 "of %" PRIu64", when the prior validated limit was "
2166 "%d. This limit should never decrease.", ram_slots,
2167 u->user->memory_slots);
2168 return -EINVAL;
2169 }
2170
2171 u->user->memory_slots = MIN(ram_slots, VHOST_USER_MAX_RAM_SLOTS);
2172 }
2173 }
2174
2175 if (dev->migration_blocker == NULL &&
2176 !virtio_has_feature(dev->protocol_features,
2177 VHOST_USER_PROTOCOL_F_LOG_SHMFD)) {
2178 error_setg(&dev->migration_blocker,
2179 "Migration disabled: vhost-user backend lacks "
2180 "VHOST_USER_PROTOCOL_F_LOG_SHMFD feature.");
2181 }
2182
2183 if (dev->vq_index == 0) {
2184 err = vhost_setup_slave_channel(dev);
2185 if (err < 0) {
2186 error_setg_errno(errp, EPROTO, "vhost_backend_init failed");
2187 return -EPROTO;
2188 }
2189 }
2190
2191 u->postcopy_notifier.notify = vhost_user_postcopy_notifier;
2192 postcopy_add_notifier(&u->postcopy_notifier);
2193
2194 return 0;
2195 }
2196
2197 static int vhost_user_backend_cleanup(struct vhost_dev *dev)
2198 {
2199 struct vhost_user *u;
2200
2201 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
2202
2203 u = dev->opaque;
2204 if (u->postcopy_notifier.notify) {
2205 postcopy_remove_notifier(&u->postcopy_notifier);
2206 u->postcopy_notifier.notify = NULL;
2207 }
2208 u->postcopy_listen = false;
2209 if (u->postcopy_fd.handler) {
2210 postcopy_unregister_shared_ufd(&u->postcopy_fd);
2211 close(u->postcopy_fd.fd);
2212 u->postcopy_fd.handler = NULL;
2213 }
2214 if (u->slave_ioc) {
2215 close_slave_channel(u);
2216 }
2217 g_free(u->region_rb);
2218 u->region_rb = NULL;
2219 g_free(u->region_rb_offset);
2220 u->region_rb_offset = NULL;
2221 u->region_rb_len = 0;
2222 g_free(u);
2223 dev->opaque = 0;
2224
2225 return 0;
2226 }
2227
2228 static int vhost_user_get_vq_index(struct vhost_dev *dev, int idx)
2229 {
2230 assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
2231
2232 return idx;
2233 }
2234
2235 static int vhost_user_memslots_limit(struct vhost_dev *dev)
2236 {
2237 struct vhost_user *u = dev->opaque;
2238
2239 return u->user->memory_slots;
2240 }
2241
2242 static bool vhost_user_requires_shm_log(struct vhost_dev *dev)
2243 {
2244 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
2245
2246 return virtio_has_feature(dev->protocol_features,
2247 VHOST_USER_PROTOCOL_F_LOG_SHMFD);
2248 }
2249
2250 static int vhost_user_migration_done(struct vhost_dev *dev, char* mac_addr)
2251 {
2252 VhostUserMsg msg = { };
2253
2254 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
2255
2256 /* If guest supports GUEST_ANNOUNCE do nothing */
2257 if (virtio_has_feature(dev->acked_features, VIRTIO_NET_F_GUEST_ANNOUNCE)) {
2258 return 0;
2259 }
2260
2261 /* if backend supports VHOST_USER_PROTOCOL_F_RARP ask it to send the RARP */
2262 if (virtio_has_feature(dev->protocol_features,
2263 VHOST_USER_PROTOCOL_F_RARP)) {
2264 msg.hdr.request = VHOST_USER_SEND_RARP;
2265 msg.hdr.flags = VHOST_USER_VERSION;
2266 memcpy((char *)&msg.payload.u64, mac_addr, 6);
2267 msg.hdr.size = sizeof(msg.payload.u64);
2268
2269 return vhost_user_write(dev, &msg, NULL, 0);
2270 }
2271 return -ENOTSUP;
2272 }
2273
2274 static bool vhost_user_can_merge(struct vhost_dev *dev,
2275 uint64_t start1, uint64_t size1,
2276 uint64_t start2, uint64_t size2)
2277 {
2278 ram_addr_t offset;
2279 int mfd, rfd;
2280
2281 (void)vhost_user_get_mr_data(start1, &offset, &mfd);
2282 (void)vhost_user_get_mr_data(start2, &offset, &rfd);
2283
2284 return mfd == rfd;
2285 }
2286
2287 static int vhost_user_net_set_mtu(struct vhost_dev *dev, uint16_t mtu)
2288 {
2289 VhostUserMsg msg;
2290 bool reply_supported = virtio_has_feature(dev->protocol_features,
2291 VHOST_USER_PROTOCOL_F_REPLY_ACK);
2292 int ret;
2293
2294 if (!(dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_NET_MTU))) {
2295 return 0;
2296 }
2297
2298 msg.hdr.request = VHOST_USER_NET_SET_MTU;
2299 msg.payload.u64 = mtu;
2300 msg.hdr.size = sizeof(msg.payload.u64);
2301 msg.hdr.flags = VHOST_USER_VERSION;
2302 if (reply_supported) {
2303 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
2304 }
2305
2306 ret = vhost_user_write(dev, &msg, NULL, 0);
2307 if (ret < 0) {
2308 return ret;
2309 }
2310
2311 /* If reply_ack supported, slave has to ack specified MTU is valid */
2312 if (reply_supported) {
2313 return process_message_reply(dev, &msg);
2314 }
2315
2316 return 0;
2317 }
2318
2319 static int vhost_user_send_device_iotlb_msg(struct vhost_dev *dev,
2320 struct vhost_iotlb_msg *imsg)
2321 {
2322 int ret;
2323 VhostUserMsg msg = {
2324 .hdr.request = VHOST_USER_IOTLB_MSG,
2325 .hdr.size = sizeof(msg.payload.iotlb),
2326 .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
2327 .payload.iotlb = *imsg,
2328 };
2329
2330 ret = vhost_user_write(dev, &msg, NULL, 0);
2331 if (ret < 0) {
2332 return ret;
2333 }
2334
2335 return process_message_reply(dev, &msg);
2336 }
2337
2338
2339 static void vhost_user_set_iotlb_callback(struct vhost_dev *dev, int enabled)
2340 {
2341 /* No-op as the receive channel is not dedicated to IOTLB messages. */
2342 }
2343
2344 static int vhost_user_get_config(struct vhost_dev *dev, uint8_t *config,
2345 uint32_t config_len, Error **errp)
2346 {
2347 int ret;
2348 VhostUserMsg msg = {
2349 .hdr.request = VHOST_USER_GET_CONFIG,
2350 .hdr.flags = VHOST_USER_VERSION,
2351 .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + config_len,
2352 };
2353
2354 if (!virtio_has_feature(dev->protocol_features,
2355 VHOST_USER_PROTOCOL_F_CONFIG)) {
2356 error_setg(errp, "VHOST_USER_PROTOCOL_F_CONFIG not supported");
2357 return -EINVAL;
2358 }
2359
2360 assert(config_len <= VHOST_USER_MAX_CONFIG_SIZE);
2361
2362 msg.payload.config.offset = 0;
2363 msg.payload.config.size = config_len;
2364 ret = vhost_user_write(dev, &msg, NULL, 0);
2365 if (ret < 0) {
2366 error_setg_errno(errp, -ret, "vhost_get_config failed");
2367 return ret;
2368 }
2369
2370 ret = vhost_user_read(dev, &msg);
2371 if (ret < 0) {
2372 error_setg_errno(errp, -ret, "vhost_get_config failed");
2373 return ret;
2374 }
2375
2376 if (msg.hdr.request != VHOST_USER_GET_CONFIG) {
2377 error_setg(errp,
2378 "Received unexpected msg type. Expected %d received %d",
2379 VHOST_USER_GET_CONFIG, msg.hdr.request);
2380 return -EPROTO;
2381 }
2382
2383 if (msg.hdr.size != VHOST_USER_CONFIG_HDR_SIZE + config_len) {
2384 error_setg(errp, "Received bad msg size.");
2385 return -EPROTO;
2386 }
2387
2388 memcpy(config, msg.payload.config.region, config_len);
2389
2390 return 0;
2391 }
2392
2393 static int vhost_user_set_config(struct vhost_dev *dev, const uint8_t *data,
2394 uint32_t offset, uint32_t size, uint32_t flags)
2395 {
2396 int ret;
2397 uint8_t *p;
2398 bool reply_supported = virtio_has_feature(dev->protocol_features,
2399 VHOST_USER_PROTOCOL_F_REPLY_ACK);
2400
2401 VhostUserMsg msg = {
2402 .hdr.request = VHOST_USER_SET_CONFIG,
2403 .hdr.flags = VHOST_USER_VERSION,
2404 .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + size,
2405 };
2406
2407 if (!virtio_has_feature(dev->protocol_features,
2408 VHOST_USER_PROTOCOL_F_CONFIG)) {
2409 return -ENOTSUP;
2410 }
2411
2412 if (reply_supported) {
2413 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
2414 }
2415
2416 if (size > VHOST_USER_MAX_CONFIG_SIZE) {
2417 return -EINVAL;
2418 }
2419
2420 msg.payload.config.offset = offset,
2421 msg.payload.config.size = size,
2422 msg.payload.config.flags = flags,
2423 p = msg.payload.config.region;
2424 memcpy(p, data, size);
2425
2426 ret = vhost_user_write(dev, &msg, NULL, 0);
2427 if (ret < 0) {
2428 return ret;
2429 }
2430
2431 if (reply_supported) {
2432 return process_message_reply(dev, &msg);
2433 }
2434
2435 return 0;
2436 }
2437
2438 static int vhost_user_crypto_create_session(struct vhost_dev *dev,
2439 void *session_info,
2440 uint64_t *session_id)
2441 {
2442 int ret;
2443 bool crypto_session = virtio_has_feature(dev->protocol_features,
2444 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION);
2445 CryptoDevBackendSymSessionInfo *sess_info = session_info;
2446 VhostUserMsg msg = {
2447 .hdr.request = VHOST_USER_CREATE_CRYPTO_SESSION,
2448 .hdr.flags = VHOST_USER_VERSION,
2449 .hdr.size = sizeof(msg.payload.session),
2450 };
2451
2452 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
2453
2454 if (!crypto_session) {
2455 error_report("vhost-user trying to send unhandled ioctl");
2456 return -ENOTSUP;
2457 }
2458
2459 memcpy(&msg.payload.session.session_setup_data, sess_info,
2460 sizeof(CryptoDevBackendSymSessionInfo));
2461 if (sess_info->key_len) {
2462 memcpy(&msg.payload.session.key, sess_info->cipher_key,
2463 sess_info->key_len);
2464 }
2465 if (sess_info->auth_key_len > 0) {
2466 memcpy(&msg.payload.session.auth_key, sess_info->auth_key,
2467 sess_info->auth_key_len);
2468 }
2469 ret = vhost_user_write(dev, &msg, NULL, 0);
2470 if (ret < 0) {
2471 error_report("vhost_user_write() return %d, create session failed",
2472 ret);
2473 return ret;
2474 }
2475
2476 ret = vhost_user_read(dev, &msg);
2477 if (ret < 0) {
2478 error_report("vhost_user_read() return %d, create session failed",
2479 ret);
2480 return ret;
2481 }
2482
2483 if (msg.hdr.request != VHOST_USER_CREATE_CRYPTO_SESSION) {
2484 error_report("Received unexpected msg type. Expected %d received %d",
2485 VHOST_USER_CREATE_CRYPTO_SESSION, msg.hdr.request);
2486 return -EPROTO;
2487 }
2488
2489 if (msg.hdr.size != sizeof(msg.payload.session)) {
2490 error_report("Received bad msg size.");
2491 return -EPROTO;
2492 }
2493
2494 if (msg.payload.session.session_id < 0) {
2495 error_report("Bad session id: %" PRId64 "",
2496 msg.payload.session.session_id);
2497 return -EINVAL;
2498 }
2499 *session_id = msg.payload.session.session_id;
2500
2501 return 0;
2502 }
2503
2504 static int
2505 vhost_user_crypto_close_session(struct vhost_dev *dev, uint64_t session_id)
2506 {
2507 int ret;
2508 bool crypto_session = virtio_has_feature(dev->protocol_features,
2509 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION);
2510 VhostUserMsg msg = {
2511 .hdr.request = VHOST_USER_CLOSE_CRYPTO_SESSION,
2512 .hdr.flags = VHOST_USER_VERSION,
2513 .hdr.size = sizeof(msg.payload.u64),
2514 };
2515 msg.payload.u64 = session_id;
2516
2517 if (!crypto_session) {
2518 error_report("vhost-user trying to send unhandled ioctl");
2519 return -ENOTSUP;
2520 }
2521
2522 ret = vhost_user_write(dev, &msg, NULL, 0);
2523 if (ret < 0) {
2524 error_report("vhost_user_write() return %d, close session failed",
2525 ret);
2526 return ret;
2527 }
2528
2529 return 0;
2530 }
2531
2532 static bool vhost_user_mem_section_filter(struct vhost_dev *dev,
2533 MemoryRegionSection *section)
2534 {
2535 bool result;
2536
2537 result = memory_region_get_fd(section->mr) >= 0;
2538
2539 return result;
2540 }
2541
2542 static int vhost_user_get_inflight_fd(struct vhost_dev *dev,
2543 uint16_t queue_size,
2544 struct vhost_inflight *inflight)
2545 {
2546 void *addr;
2547 int fd;
2548 int ret;
2549 struct vhost_user *u = dev->opaque;
2550 CharBackend *chr = u->user->chr;
2551 VhostUserMsg msg = {
2552 .hdr.request = VHOST_USER_GET_INFLIGHT_FD,
2553 .hdr.flags = VHOST_USER_VERSION,
2554 .payload.inflight.num_queues = dev->nvqs,
2555 .payload.inflight.queue_size = queue_size,
2556 .hdr.size = sizeof(msg.payload.inflight),
2557 };
2558
2559 if (!virtio_has_feature(dev->protocol_features,
2560 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
2561 return 0;
2562 }
2563
2564 ret = vhost_user_write(dev, &msg, NULL, 0);
2565 if (ret < 0) {
2566 return ret;
2567 }
2568
2569 ret = vhost_user_read(dev, &msg);
2570 if (ret < 0) {
2571 return ret;
2572 }
2573
2574 if (msg.hdr.request != VHOST_USER_GET_INFLIGHT_FD) {
2575 error_report("Received unexpected msg type. "
2576 "Expected %d received %d",
2577 VHOST_USER_GET_INFLIGHT_FD, msg.hdr.request);
2578 return -EPROTO;
2579 }
2580
2581 if (msg.hdr.size != sizeof(msg.payload.inflight)) {
2582 error_report("Received bad msg size.");
2583 return -EPROTO;
2584 }
2585
2586 if (!msg.payload.inflight.mmap_size) {
2587 return 0;
2588 }
2589
2590 fd = qemu_chr_fe_get_msgfd(chr);
2591 if (fd < 0) {
2592 error_report("Failed to get mem fd");
2593 return -EIO;
2594 }
2595
2596 addr = mmap(0, msg.payload.inflight.mmap_size, PROT_READ | PROT_WRITE,
2597 MAP_SHARED, fd, msg.payload.inflight.mmap_offset);
2598
2599 if (addr == MAP_FAILED) {
2600 error_report("Failed to mmap mem fd");
2601 close(fd);
2602 return -EFAULT;
2603 }
2604
2605 inflight->addr = addr;
2606 inflight->fd = fd;
2607 inflight->size = msg.payload.inflight.mmap_size;
2608 inflight->offset = msg.payload.inflight.mmap_offset;
2609 inflight->queue_size = queue_size;
2610
2611 return 0;
2612 }
2613
2614 static int vhost_user_set_inflight_fd(struct vhost_dev *dev,
2615 struct vhost_inflight *inflight)
2616 {
2617 VhostUserMsg msg = {
2618 .hdr.request = VHOST_USER_SET_INFLIGHT_FD,
2619 .hdr.flags = VHOST_USER_VERSION,
2620 .payload.inflight.mmap_size = inflight->size,
2621 .payload.inflight.mmap_offset = inflight->offset,
2622 .payload.inflight.num_queues = dev->nvqs,
2623 .payload.inflight.queue_size = inflight->queue_size,
2624 .hdr.size = sizeof(msg.payload.inflight),
2625 };
2626
2627 if (!virtio_has_feature(dev->protocol_features,
2628 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
2629 return 0;
2630 }
2631
2632 return vhost_user_write(dev, &msg, &inflight->fd, 1);
2633 }
2634
2635 static void vhost_user_state_destroy(gpointer data)
2636 {
2637 VhostUserHostNotifier *n = (VhostUserHostNotifier *) data;
2638 if (n) {
2639 vhost_user_host_notifier_remove(n, NULL);
2640 object_unparent(OBJECT(&n->mr));
2641 /*
2642 * We can't free until vhost_user_host_notifier_remove has
2643 * done it's thing so schedule the free with RCU.
2644 */
2645 g_free_rcu(n, rcu);
2646 }
2647 }
2648
2649 bool vhost_user_init(VhostUserState *user, CharBackend *chr, Error **errp)
2650 {
2651 if (user->chr) {
2652 error_setg(errp, "Cannot initialize vhost-user state");
2653 return false;
2654 }
2655 user->chr = chr;
2656 user->memory_slots = 0;
2657 user->notifiers = g_ptr_array_new_full(VIRTIO_QUEUE_MAX / 4,
2658 &vhost_user_state_destroy);
2659 return true;
2660 }
2661
2662 void vhost_user_cleanup(VhostUserState *user)
2663 {
2664 if (!user->chr) {
2665 return;
2666 }
2667 memory_region_transaction_begin();
2668 user->notifiers = (GPtrArray *) g_ptr_array_free(user->notifiers, true);
2669 memory_region_transaction_commit();
2670 user->chr = NULL;
2671 }
2672
2673 static int vhost_user_dev_start(struct vhost_dev *dev, bool started)
2674 {
2675 if (!virtio_has_feature(dev->protocol_features,
2676 VHOST_USER_PROTOCOL_F_STATUS)) {
2677 return 0;
2678 }
2679
2680 /* Set device status only for last queue pair */
2681 if (dev->vq_index + dev->nvqs != dev->vq_index_end) {
2682 return 0;
2683 }
2684
2685 if (started) {
2686 return vhost_user_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE |
2687 VIRTIO_CONFIG_S_DRIVER |
2688 VIRTIO_CONFIG_S_DRIVER_OK);
2689 } else {
2690 return vhost_user_set_status(dev, 0);
2691 }
2692 }
2693
2694 const VhostOps user_ops = {
2695 .backend_type = VHOST_BACKEND_TYPE_USER,
2696 .vhost_backend_init = vhost_user_backend_init,
2697 .vhost_backend_cleanup = vhost_user_backend_cleanup,
2698 .vhost_backend_memslots_limit = vhost_user_memslots_limit,
2699 .vhost_set_log_base = vhost_user_set_log_base,
2700 .vhost_set_mem_table = vhost_user_set_mem_table,
2701 .vhost_set_vring_addr = vhost_user_set_vring_addr,
2702 .vhost_set_vring_endian = vhost_user_set_vring_endian,
2703 .vhost_set_vring_num = vhost_user_set_vring_num,
2704 .vhost_set_vring_base = vhost_user_set_vring_base,
2705 .vhost_get_vring_base = vhost_user_get_vring_base,
2706 .vhost_set_vring_kick = vhost_user_set_vring_kick,
2707 .vhost_set_vring_call = vhost_user_set_vring_call,
2708 .vhost_set_vring_err = vhost_user_set_vring_err,
2709 .vhost_set_features = vhost_user_set_features,
2710 .vhost_get_features = vhost_user_get_features,
2711 .vhost_set_owner = vhost_user_set_owner,
2712 .vhost_reset_device = vhost_user_reset_device,
2713 .vhost_get_vq_index = vhost_user_get_vq_index,
2714 .vhost_set_vring_enable = vhost_user_set_vring_enable,
2715 .vhost_requires_shm_log = vhost_user_requires_shm_log,
2716 .vhost_migration_done = vhost_user_migration_done,
2717 .vhost_backend_can_merge = vhost_user_can_merge,
2718 .vhost_net_set_mtu = vhost_user_net_set_mtu,
2719 .vhost_set_iotlb_callback = vhost_user_set_iotlb_callback,
2720 .vhost_send_device_iotlb_msg = vhost_user_send_device_iotlb_msg,
2721 .vhost_get_config = vhost_user_get_config,
2722 .vhost_set_config = vhost_user_set_config,
2723 .vhost_crypto_create_session = vhost_user_crypto_create_session,
2724 .vhost_crypto_close_session = vhost_user_crypto_close_session,
2725 .vhost_backend_mem_section_filter = vhost_user_mem_section_filter,
2726 .vhost_get_inflight_fd = vhost_user_get_inflight_fd,
2727 .vhost_set_inflight_fd = vhost_user_set_inflight_fd,
2728 .vhost_dev_start = vhost_user_dev_start,
2729 };
AR7 emulation for QEMU