4 * Copyright (c) 2013 Virtual Open Systems Sarl.
6 * This work is licensed under the terms of the GNU GPL, version 2 or later.
7 * See the COPYING file in the top-level directory.
11 #include "qemu/osdep.h"
12 #include "qapi/error.h"
13 #include "hw/virtio/virtio-dmabuf.h"
14 #include "hw/virtio/vhost.h"
15 #include "hw/virtio/virtio-crypto.h"
16 #include "hw/virtio/vhost-user.h"
17 #include "hw/virtio/vhost-backend.h"
18 #include "hw/virtio/virtio.h"
19 #include "hw/virtio/virtio-net.h"
20 #include "chardev/char-fe.h"
21 #include "io/channel-socket.h"
22 #include "sysemu/kvm.h"
23 #include "qemu/error-report.h"
24 #include "qemu/main-loop.h"
25 #include "qemu/uuid.h"
26 #include "qemu/sockets.h"
27 #include "sysemu/runstate.h"
28 #include "sysemu/cryptodev.h"
29 #include "migration/postcopy-ram.h"
31 #include "exec/ramblock.h"
33 #include <sys/ioctl.h>
34 #include <sys/socket.h>
37 #include "standard-headers/linux/vhost_types.h"
40 #include <linux/userfaultfd.h>
43 #define VHOST_MEMORY_BASELINE_NREGIONS 8
44 #define VHOST_USER_F_PROTOCOL_FEATURES 30
45 #define VHOST_USER_BACKEND_MAX_FDS 8
47 #if defined(TARGET_PPC) || defined(TARGET_PPC64)
48 #include "hw/ppc/spapr.h"
49 #define VHOST_USER_MAX_RAM_SLOTS SPAPR_MAX_RAM_SLOTS
52 #define VHOST_USER_MAX_RAM_SLOTS 512
56 * Maximum size of virtio device config space
58 #define VHOST_USER_MAX_CONFIG_SIZE 256
60 #define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1)
62 typedef enum VhostUserRequest
{
64 VHOST_USER_GET_FEATURES
= 1,
65 VHOST_USER_SET_FEATURES
= 2,
66 VHOST_USER_SET_OWNER
= 3,
67 VHOST_USER_RESET_OWNER
= 4,
68 VHOST_USER_SET_MEM_TABLE
= 5,
69 VHOST_USER_SET_LOG_BASE
= 6,
70 VHOST_USER_SET_LOG_FD
= 7,
71 VHOST_USER_SET_VRING_NUM
= 8,
72 VHOST_USER_SET_VRING_ADDR
= 9,
73 VHOST_USER_SET_VRING_BASE
= 10,
74 VHOST_USER_GET_VRING_BASE
= 11,
75 VHOST_USER_SET_VRING_KICK
= 12,
76 VHOST_USER_SET_VRING_CALL
= 13,
77 VHOST_USER_SET_VRING_ERR
= 14,
78 VHOST_USER_GET_PROTOCOL_FEATURES
= 15,
79 VHOST_USER_SET_PROTOCOL_FEATURES
= 16,
80 VHOST_USER_GET_QUEUE_NUM
= 17,
81 VHOST_USER_SET_VRING_ENABLE
= 18,
82 VHOST_USER_SEND_RARP
= 19,
83 VHOST_USER_NET_SET_MTU
= 20,
84 VHOST_USER_SET_BACKEND_REQ_FD
= 21,
85 VHOST_USER_IOTLB_MSG
= 22,
86 VHOST_USER_SET_VRING_ENDIAN
= 23,
87 VHOST_USER_GET_CONFIG
= 24,
88 VHOST_USER_SET_CONFIG
= 25,
89 VHOST_USER_CREATE_CRYPTO_SESSION
= 26,
90 VHOST_USER_CLOSE_CRYPTO_SESSION
= 27,
91 VHOST_USER_POSTCOPY_ADVISE
= 28,
92 VHOST_USER_POSTCOPY_LISTEN
= 29,
93 VHOST_USER_POSTCOPY_END
= 30,
94 VHOST_USER_GET_INFLIGHT_FD
= 31,
95 VHOST_USER_SET_INFLIGHT_FD
= 32,
96 VHOST_USER_GPU_SET_SOCKET
= 33,
97 VHOST_USER_RESET_DEVICE
= 34,
98 /* Message number 35 reserved for VHOST_USER_VRING_KICK. */
99 VHOST_USER_GET_MAX_MEM_SLOTS
= 36,
100 VHOST_USER_ADD_MEM_REG
= 37,
101 VHOST_USER_REM_MEM_REG
= 38,
102 VHOST_USER_SET_STATUS
= 39,
103 VHOST_USER_GET_STATUS
= 40,
104 VHOST_USER_GET_SHARED_OBJECT
= 41,
105 VHOST_USER_SET_DEVICE_STATE_FD
= 42,
106 VHOST_USER_CHECK_DEVICE_STATE
= 43,
110 typedef enum VhostUserBackendRequest
{
111 VHOST_USER_BACKEND_NONE
= 0,
112 VHOST_USER_BACKEND_IOTLB_MSG
= 1,
113 VHOST_USER_BACKEND_CONFIG_CHANGE_MSG
= 2,
114 VHOST_USER_BACKEND_VRING_HOST_NOTIFIER_MSG
= 3,
115 VHOST_USER_BACKEND_SHARED_OBJECT_ADD
= 6,
116 VHOST_USER_BACKEND_SHARED_OBJECT_REMOVE
= 7,
117 VHOST_USER_BACKEND_SHARED_OBJECT_LOOKUP
= 8,
118 VHOST_USER_BACKEND_MAX
119 } VhostUserBackendRequest
;
121 typedef struct VhostUserMemoryRegion
{
122 uint64_t guest_phys_addr
;
123 uint64_t memory_size
;
124 uint64_t userspace_addr
;
125 uint64_t mmap_offset
;
126 } VhostUserMemoryRegion
;
128 typedef struct VhostUserMemory
{
131 VhostUserMemoryRegion regions
[VHOST_MEMORY_BASELINE_NREGIONS
];
134 typedef struct VhostUserMemRegMsg
{
136 VhostUserMemoryRegion region
;
137 } VhostUserMemRegMsg
;
139 typedef struct VhostUserLog
{
141 uint64_t mmap_offset
;
144 typedef struct VhostUserConfig
{
148 uint8_t region
[VHOST_USER_MAX_CONFIG_SIZE
];
151 #define VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN 512
152 #define VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN 64
153 #define VHOST_CRYPTO_ASYM_MAX_KEY_LEN 1024
155 typedef struct VhostUserCryptoSession
{
159 CryptoDevBackendSymSessionInfo session_setup_data
;
160 uint8_t key
[VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN
];
161 uint8_t auth_key
[VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN
];
164 CryptoDevBackendAsymSessionInfo session_setup_data
;
165 uint8_t key
[VHOST_CRYPTO_ASYM_MAX_KEY_LEN
];
169 /* session id for success, -1 on errors */
171 } VhostUserCryptoSession
;
173 static VhostUserConfig c
__attribute__ ((unused
));
174 #define VHOST_USER_CONFIG_HDR_SIZE (sizeof(c.offset) \
178 typedef struct VhostUserVringArea
{
182 } VhostUserVringArea
;
184 typedef struct VhostUserInflight
{
186 uint64_t mmap_offset
;
191 typedef struct VhostUserShared
{
192 unsigned char uuid
[16];
196 VhostUserRequest request
;
198 #define VHOST_USER_VERSION_MASK (0x3)
199 #define VHOST_USER_REPLY_MASK (0x1 << 2)
200 #define VHOST_USER_NEED_REPLY_MASK (0x1 << 3)
202 uint32_t size
; /* the following payload size */
203 } QEMU_PACKED VhostUserHeader
;
205 /* Request payload of VHOST_USER_SET_DEVICE_STATE_FD */
206 typedef struct VhostUserTransferDeviceState
{
209 } VhostUserTransferDeviceState
;
212 #define VHOST_USER_VRING_IDX_MASK (0xff)
213 #define VHOST_USER_VRING_NOFD_MASK (0x1 << 8)
215 struct vhost_vring_state state
;
216 struct vhost_vring_addr addr
;
217 VhostUserMemory memory
;
218 VhostUserMemRegMsg mem_reg
;
220 struct vhost_iotlb_msg iotlb
;
221 VhostUserConfig config
;
222 VhostUserCryptoSession session
;
223 VhostUserVringArea area
;
224 VhostUserInflight inflight
;
225 VhostUserShared object
;
226 VhostUserTransferDeviceState transfer_state
;
229 typedef struct VhostUserMsg
{
231 VhostUserPayload payload
;
232 } QEMU_PACKED VhostUserMsg
;
234 static VhostUserMsg m
__attribute__ ((unused
));
235 #define VHOST_USER_HDR_SIZE (sizeof(VhostUserHeader))
237 #define VHOST_USER_PAYLOAD_SIZE (sizeof(VhostUserPayload))
239 /* The version of the protocol we support */
240 #define VHOST_USER_VERSION (0x1)
243 struct vhost_dev
*dev
;
244 /* Shared between vhost devs of the same virtio device */
245 VhostUserState
*user
;
246 QIOChannel
*backend_ioc
;
247 GSource
*backend_src
;
248 NotifierWithReturn postcopy_notifier
;
249 struct PostCopyFD postcopy_fd
;
250 uint64_t postcopy_client_bases
[VHOST_USER_MAX_RAM_SLOTS
];
251 /* Length of the region_rb and region_rb_offset arrays */
252 size_t region_rb_len
;
253 /* RAMBlock associated with a given region */
254 RAMBlock
**region_rb
;
256 * The offset from the start of the RAMBlock to the start of the
259 ram_addr_t
*region_rb_offset
;
261 /* True once we've entered postcopy_listen */
262 bool postcopy_listen
;
264 /* Our current regions */
265 int num_shadow_regions
;
266 struct vhost_memory_region shadow_regions
[VHOST_USER_MAX_RAM_SLOTS
];
269 struct scrub_regions
{
270 struct vhost_memory_region
*region
;
275 static int vhost_user_read_header(struct vhost_dev
*dev
, VhostUserMsg
*msg
)
277 struct vhost_user
*u
= dev
->opaque
;
278 CharBackend
*chr
= u
->user
->chr
;
279 uint8_t *p
= (uint8_t *) msg
;
280 int r
, size
= VHOST_USER_HDR_SIZE
;
282 r
= qemu_chr_fe_read_all(chr
, p
, size
);
284 int saved_errno
= errno
;
285 error_report("Failed to read msg header. Read %d instead of %d."
286 " Original request %d.", r
, size
, msg
->hdr
.request
);
287 return r
< 0 ? -saved_errno
: -EIO
;
290 /* validate received flags */
291 if (msg
->hdr
.flags
!= (VHOST_USER_REPLY_MASK
| VHOST_USER_VERSION
)) {
292 error_report("Failed to read msg header."
293 " Flags 0x%x instead of 0x%x.", msg
->hdr
.flags
,
294 VHOST_USER_REPLY_MASK
| VHOST_USER_VERSION
);
298 trace_vhost_user_read(msg
->hdr
.request
, msg
->hdr
.flags
);
303 static int vhost_user_read(struct vhost_dev
*dev
, VhostUserMsg
*msg
)
305 struct vhost_user
*u
= dev
->opaque
;
306 CharBackend
*chr
= u
->user
->chr
;
307 uint8_t *p
= (uint8_t *) msg
;
310 r
= vhost_user_read_header(dev
, msg
);
315 /* validate message size is sane */
316 if (msg
->hdr
.size
> VHOST_USER_PAYLOAD_SIZE
) {
317 error_report("Failed to read msg header."
318 " Size %d exceeds the maximum %zu.", msg
->hdr
.size
,
319 VHOST_USER_PAYLOAD_SIZE
);
324 p
+= VHOST_USER_HDR_SIZE
;
325 size
= msg
->hdr
.size
;
326 r
= qemu_chr_fe_read_all(chr
, p
, size
);
328 int saved_errno
= errno
;
329 error_report("Failed to read msg payload."
330 " Read %d instead of %d.", r
, msg
->hdr
.size
);
331 return r
< 0 ? -saved_errno
: -EIO
;
338 static int process_message_reply(struct vhost_dev
*dev
,
339 const VhostUserMsg
*msg
)
342 VhostUserMsg msg_reply
;
344 if ((msg
->hdr
.flags
& VHOST_USER_NEED_REPLY_MASK
) == 0) {
348 ret
= vhost_user_read(dev
, &msg_reply
);
353 if (msg_reply
.hdr
.request
!= msg
->hdr
.request
) {
354 error_report("Received unexpected msg type. "
355 "Expected %d received %d",
356 msg
->hdr
.request
, msg_reply
.hdr
.request
);
360 return msg_reply
.payload
.u64
? -EIO
: 0;
363 static bool vhost_user_per_device_request(VhostUserRequest request
)
366 case VHOST_USER_SET_OWNER
:
367 case VHOST_USER_RESET_OWNER
:
368 case VHOST_USER_SET_MEM_TABLE
:
369 case VHOST_USER_GET_QUEUE_NUM
:
370 case VHOST_USER_NET_SET_MTU
:
371 case VHOST_USER_RESET_DEVICE
:
372 case VHOST_USER_ADD_MEM_REG
:
373 case VHOST_USER_REM_MEM_REG
:
374 case VHOST_USER_SET_LOG_BASE
:
381 /* most non-init callers ignore the error */
382 static int vhost_user_write(struct vhost_dev
*dev
, VhostUserMsg
*msg
,
383 int *fds
, int fd_num
)
385 struct vhost_user
*u
= dev
->opaque
;
386 CharBackend
*chr
= u
->user
->chr
;
387 int ret
, size
= VHOST_USER_HDR_SIZE
+ msg
->hdr
.size
;
390 * Some devices, like virtio-scsi, are implemented as a single vhost_dev,
391 * while others, like virtio-net, contain multiple vhost_devs. For
392 * operations such as configuring device memory mappings or issuing device
393 * resets, which affect the whole device instead of individual VQs,
394 * vhost-user messages should only be sent once.
396 * Devices with multiple vhost_devs are given an associated dev->vq_index
397 * so per_device requests are only sent if vq_index is 0.
399 if (vhost_user_per_device_request(msg
->hdr
.request
)
400 && dev
->vq_index
!= 0) {
401 msg
->hdr
.flags
&= ~VHOST_USER_NEED_REPLY_MASK
;
405 if (qemu_chr_fe_set_msgfds(chr
, fds
, fd_num
) < 0) {
406 error_report("Failed to set msg fds.");
410 ret
= qemu_chr_fe_write_all(chr
, (const uint8_t *) msg
, size
);
412 int saved_errno
= errno
;
413 error_report("Failed to write msg."
414 " Wrote %d instead of %d.", ret
, size
);
415 return ret
< 0 ? -saved_errno
: -EIO
;
418 trace_vhost_user_write(msg
->hdr
.request
, msg
->hdr
.flags
);
423 int vhost_user_gpu_set_socket(struct vhost_dev
*dev
, int fd
)
426 .hdr
.request
= VHOST_USER_GPU_SET_SOCKET
,
427 .hdr
.flags
= VHOST_USER_VERSION
,
430 return vhost_user_write(dev
, &msg
, &fd
, 1);
433 static int vhost_user_set_log_base(struct vhost_dev
*dev
, uint64_t base
,
434 struct vhost_log
*log
)
436 int fds
[VHOST_USER_MAX_RAM_SLOTS
];
438 bool shmfd
= virtio_has_feature(dev
->protocol_features
,
439 VHOST_USER_PROTOCOL_F_LOG_SHMFD
);
442 .hdr
.request
= VHOST_USER_SET_LOG_BASE
,
443 .hdr
.flags
= VHOST_USER_VERSION
,
444 .payload
.log
.mmap_size
= log
->size
* sizeof(*(log
->log
)),
445 .payload
.log
.mmap_offset
= 0,
446 .hdr
.size
= sizeof(msg
.payload
.log
),
449 /* Send only once with first queue pair */
450 if (dev
->vq_index
!= 0) {
454 if (shmfd
&& log
->fd
!= -1) {
455 fds
[fd_num
++] = log
->fd
;
458 ret
= vhost_user_write(dev
, &msg
, fds
, fd_num
);
465 ret
= vhost_user_read(dev
, &msg
);
470 if (msg
.hdr
.request
!= VHOST_USER_SET_LOG_BASE
) {
471 error_report("Received unexpected msg type. "
472 "Expected %d received %d",
473 VHOST_USER_SET_LOG_BASE
, msg
.hdr
.request
);
481 static MemoryRegion
*vhost_user_get_mr_data(uint64_t addr
, ram_addr_t
*offset
,
486 assert((uintptr_t)addr
== addr
);
487 mr
= memory_region_from_host((void *)(uintptr_t)addr
, offset
);
488 *fd
= memory_region_get_fd(mr
);
489 *offset
+= mr
->ram_block
->fd_offset
;
494 static void vhost_user_fill_msg_region(VhostUserMemoryRegion
*dst
,
495 struct vhost_memory_region
*src
,
496 uint64_t mmap_offset
)
498 assert(src
!= NULL
&& dst
!= NULL
);
499 dst
->userspace_addr
= src
->userspace_addr
;
500 dst
->memory_size
= src
->memory_size
;
501 dst
->guest_phys_addr
= src
->guest_phys_addr
;
502 dst
->mmap_offset
= mmap_offset
;
505 static int vhost_user_fill_set_mem_table_msg(struct vhost_user
*u
,
506 struct vhost_dev
*dev
,
508 int *fds
, size_t *fd_num
,
509 bool track_ramblocks
)
514 struct vhost_memory_region
*reg
;
515 VhostUserMemoryRegion region_buffer
;
517 msg
->hdr
.request
= VHOST_USER_SET_MEM_TABLE
;
519 for (i
= 0; i
< dev
->mem
->nregions
; ++i
) {
520 reg
= dev
->mem
->regions
+ i
;
522 mr
= vhost_user_get_mr_data(reg
->userspace_addr
, &offset
, &fd
);
524 if (track_ramblocks
) {
525 assert(*fd_num
< VHOST_MEMORY_BASELINE_NREGIONS
);
526 trace_vhost_user_set_mem_table_withfd(*fd_num
, mr
->name
,
528 reg
->guest_phys_addr
,
531 u
->region_rb_offset
[i
] = offset
;
532 u
->region_rb
[i
] = mr
->ram_block
;
533 } else if (*fd_num
== VHOST_MEMORY_BASELINE_NREGIONS
) {
534 error_report("Failed preparing vhost-user memory table msg");
537 vhost_user_fill_msg_region(®ion_buffer
, reg
, offset
);
538 msg
->payload
.memory
.regions
[*fd_num
] = region_buffer
;
539 fds
[(*fd_num
)++] = fd
;
540 } else if (track_ramblocks
) {
541 u
->region_rb_offset
[i
] = 0;
542 u
->region_rb
[i
] = NULL
;
546 msg
->payload
.memory
.nregions
= *fd_num
;
549 error_report("Failed initializing vhost-user memory map, "
550 "consider using -object memory-backend-file share=on");
554 msg
->hdr
.size
= sizeof(msg
->payload
.memory
.nregions
);
555 msg
->hdr
.size
+= sizeof(msg
->payload
.memory
.padding
);
556 msg
->hdr
.size
+= *fd_num
* sizeof(VhostUserMemoryRegion
);
561 static inline bool reg_equal(struct vhost_memory_region
*shadow_reg
,
562 struct vhost_memory_region
*vdev_reg
)
564 return shadow_reg
->guest_phys_addr
== vdev_reg
->guest_phys_addr
&&
565 shadow_reg
->userspace_addr
== vdev_reg
->userspace_addr
&&
566 shadow_reg
->memory_size
== vdev_reg
->memory_size
;
569 static void scrub_shadow_regions(struct vhost_dev
*dev
,
570 struct scrub_regions
*add_reg
,
572 struct scrub_regions
*rem_reg
,
573 int *nr_rem_reg
, uint64_t *shadow_pcb
,
574 bool track_ramblocks
)
576 struct vhost_user
*u
= dev
->opaque
;
577 bool found
[VHOST_USER_MAX_RAM_SLOTS
] = {};
578 struct vhost_memory_region
*reg
, *shadow_reg
;
579 int i
, j
, fd
, add_idx
= 0, rm_idx
= 0, fd_num
= 0;
585 * Find memory regions present in our shadow state which are not in
586 * the device's current memory state.
588 * Mark regions in both the shadow and device state as "found".
590 for (i
= 0; i
< u
->num_shadow_regions
; i
++) {
591 shadow_reg
= &u
->shadow_regions
[i
];
594 for (j
= 0; j
< dev
->mem
->nregions
; j
++) {
595 reg
= &dev
->mem
->regions
[j
];
597 mr
= vhost_user_get_mr_data(reg
->userspace_addr
, &offset
, &fd
);
599 if (reg_equal(shadow_reg
, reg
)) {
602 if (track_ramblocks
) {
604 * Reset postcopy client bases, region_rb, and
605 * region_rb_offset in case regions are removed.
608 u
->region_rb_offset
[j
] = offset
;
609 u
->region_rb
[j
] = mr
->ram_block
;
610 shadow_pcb
[j
] = u
->postcopy_client_bases
[i
];
612 u
->region_rb_offset
[j
] = 0;
613 u
->region_rb
[j
] = NULL
;
621 * If the region was not found in the current device memory state
622 * create an entry for it in the removed list.
625 rem_reg
[rm_idx
].region
= shadow_reg
;
626 rem_reg
[rm_idx
++].reg_idx
= i
;
631 * For regions not marked "found", create entries in the added list.
633 * Note their indexes in the device memory state and the indexes of their
636 for (i
= 0; i
< dev
->mem
->nregions
; i
++) {
637 reg
= &dev
->mem
->regions
[i
];
638 vhost_user_get_mr_data(reg
->userspace_addr
, &offset
, &fd
);
644 * If the region was in both the shadow and device state we don't
645 * need to send a VHOST_USER_ADD_MEM_REG message for it.
651 add_reg
[add_idx
].region
= reg
;
652 add_reg
[add_idx
].reg_idx
= i
;
653 add_reg
[add_idx
++].fd_idx
= fd_num
;
655 *nr_rem_reg
= rm_idx
;
656 *nr_add_reg
= add_idx
;
661 static int send_remove_regions(struct vhost_dev
*dev
,
662 struct scrub_regions
*remove_reg
,
663 int nr_rem_reg
, VhostUserMsg
*msg
,
664 bool reply_supported
)
666 struct vhost_user
*u
= dev
->opaque
;
667 struct vhost_memory_region
*shadow_reg
;
668 int i
, fd
, shadow_reg_idx
, ret
;
670 VhostUserMemoryRegion region_buffer
;
673 * The regions in remove_reg appear in the same order they do in the
674 * shadow table. Therefore we can minimize memory copies by iterating
675 * through remove_reg backwards.
677 for (i
= nr_rem_reg
- 1; i
>= 0; i
--) {
678 shadow_reg
= remove_reg
[i
].region
;
679 shadow_reg_idx
= remove_reg
[i
].reg_idx
;
681 vhost_user_get_mr_data(shadow_reg
->userspace_addr
, &offset
, &fd
);
684 msg
->hdr
.request
= VHOST_USER_REM_MEM_REG
;
685 vhost_user_fill_msg_region(®ion_buffer
, shadow_reg
, 0);
686 msg
->payload
.mem_reg
.region
= region_buffer
;
688 ret
= vhost_user_write(dev
, msg
, NULL
, 0);
693 if (reply_supported
) {
694 ret
= process_message_reply(dev
, msg
);
702 * At this point we know the backend has unmapped the region. It is now
703 * safe to remove it from the shadow table.
705 memmove(&u
->shadow_regions
[shadow_reg_idx
],
706 &u
->shadow_regions
[shadow_reg_idx
+ 1],
707 sizeof(struct vhost_memory_region
) *
708 (u
->num_shadow_regions
- shadow_reg_idx
- 1));
709 u
->num_shadow_regions
--;
715 static int send_add_regions(struct vhost_dev
*dev
,
716 struct scrub_regions
*add_reg
, int nr_add_reg
,
717 VhostUserMsg
*msg
, uint64_t *shadow_pcb
,
718 bool reply_supported
, bool track_ramblocks
)
720 struct vhost_user
*u
= dev
->opaque
;
721 int i
, fd
, ret
, reg_idx
, reg_fd_idx
;
722 struct vhost_memory_region
*reg
;
725 VhostUserMsg msg_reply
;
726 VhostUserMemoryRegion region_buffer
;
728 for (i
= 0; i
< nr_add_reg
; i
++) {
729 reg
= add_reg
[i
].region
;
730 reg_idx
= add_reg
[i
].reg_idx
;
731 reg_fd_idx
= add_reg
[i
].fd_idx
;
733 mr
= vhost_user_get_mr_data(reg
->userspace_addr
, &offset
, &fd
);
736 if (track_ramblocks
) {
737 trace_vhost_user_set_mem_table_withfd(reg_fd_idx
, mr
->name
,
739 reg
->guest_phys_addr
,
742 u
->region_rb_offset
[reg_idx
] = offset
;
743 u
->region_rb
[reg_idx
] = mr
->ram_block
;
745 msg
->hdr
.request
= VHOST_USER_ADD_MEM_REG
;
746 vhost_user_fill_msg_region(®ion_buffer
, reg
, offset
);
747 msg
->payload
.mem_reg
.region
= region_buffer
;
749 ret
= vhost_user_write(dev
, msg
, &fd
, 1);
754 if (track_ramblocks
) {
757 ret
= vhost_user_read(dev
, &msg_reply
);
762 reply_gpa
= msg_reply
.payload
.mem_reg
.region
.guest_phys_addr
;
764 if (msg_reply
.hdr
.request
!= VHOST_USER_ADD_MEM_REG
) {
765 error_report("%s: Received unexpected msg type."
766 "Expected %d received %d", __func__
,
767 VHOST_USER_ADD_MEM_REG
,
768 msg_reply
.hdr
.request
);
773 * We're using the same structure, just reusing one of the
774 * fields, so it should be the same size.
776 if (msg_reply
.hdr
.size
!= msg
->hdr
.size
) {
777 error_report("%s: Unexpected size for postcopy reply "
778 "%d vs %d", __func__
, msg_reply
.hdr
.size
,
783 /* Get the postcopy client base from the backend's reply. */
784 if (reply_gpa
== dev
->mem
->regions
[reg_idx
].guest_phys_addr
) {
785 shadow_pcb
[reg_idx
] =
786 msg_reply
.payload
.mem_reg
.region
.userspace_addr
;
787 trace_vhost_user_set_mem_table_postcopy(
788 msg_reply
.payload
.mem_reg
.region
.userspace_addr
,
789 msg
->payload
.mem_reg
.region
.userspace_addr
,
790 reg_fd_idx
, reg_idx
);
792 error_report("%s: invalid postcopy reply for region. "
793 "Got guest physical address %" PRIX64
", expected "
794 "%" PRIX64
, __func__
, reply_gpa
,
795 dev
->mem
->regions
[reg_idx
].guest_phys_addr
);
798 } else if (reply_supported
) {
799 ret
= process_message_reply(dev
, msg
);
804 } else if (track_ramblocks
) {
805 u
->region_rb_offset
[reg_idx
] = 0;
806 u
->region_rb
[reg_idx
] = NULL
;
810 * At this point, we know the backend has mapped in the new
811 * region, if the region has a valid file descriptor.
813 * The region should now be added to the shadow table.
815 u
->shadow_regions
[u
->num_shadow_regions
].guest_phys_addr
=
816 reg
->guest_phys_addr
;
817 u
->shadow_regions
[u
->num_shadow_regions
].userspace_addr
=
819 u
->shadow_regions
[u
->num_shadow_regions
].memory_size
=
821 u
->num_shadow_regions
++;
827 static int vhost_user_add_remove_regions(struct vhost_dev
*dev
,
829 bool reply_supported
,
830 bool track_ramblocks
)
832 struct vhost_user
*u
= dev
->opaque
;
833 struct scrub_regions add_reg
[VHOST_USER_MAX_RAM_SLOTS
];
834 struct scrub_regions rem_reg
[VHOST_USER_MAX_RAM_SLOTS
];
835 uint64_t shadow_pcb
[VHOST_USER_MAX_RAM_SLOTS
] = {};
836 int nr_add_reg
, nr_rem_reg
;
839 msg
->hdr
.size
= sizeof(msg
->payload
.mem_reg
);
841 /* Find the regions which need to be removed or added. */
842 scrub_shadow_regions(dev
, add_reg
, &nr_add_reg
, rem_reg
, &nr_rem_reg
,
843 shadow_pcb
, track_ramblocks
);
846 ret
= send_remove_regions(dev
, rem_reg
, nr_rem_reg
, msg
,
854 ret
= send_add_regions(dev
, add_reg
, nr_add_reg
, msg
, shadow_pcb
,
855 reply_supported
, track_ramblocks
);
861 if (track_ramblocks
) {
862 memcpy(u
->postcopy_client_bases
, shadow_pcb
,
863 sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS
);
865 * Now we've registered this with the postcopy code, we ack to the
866 * client, because now we're in the position to be able to deal with
867 * any faults it generates.
869 /* TODO: Use this for failure cases as well with a bad value. */
870 msg
->hdr
.size
= sizeof(msg
->payload
.u64
);
871 msg
->payload
.u64
= 0; /* OK */
873 ret
= vhost_user_write(dev
, msg
, NULL
, 0);
882 if (track_ramblocks
) {
883 memcpy(u
->postcopy_client_bases
, shadow_pcb
,
884 sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS
);
890 static int vhost_user_set_mem_table_postcopy(struct vhost_dev
*dev
,
891 struct vhost_memory
*mem
,
892 bool reply_supported
,
893 bool config_mem_slots
)
895 struct vhost_user
*u
= dev
->opaque
;
896 int fds
[VHOST_MEMORY_BASELINE_NREGIONS
];
898 VhostUserMsg msg_reply
;
903 .hdr
.flags
= VHOST_USER_VERSION
,
906 if (u
->region_rb_len
< dev
->mem
->nregions
) {
907 u
->region_rb
= g_renew(RAMBlock
*, u
->region_rb
, dev
->mem
->nregions
);
908 u
->region_rb_offset
= g_renew(ram_addr_t
, u
->region_rb_offset
,
910 memset(&(u
->region_rb
[u
->region_rb_len
]), '\0',
911 sizeof(RAMBlock
*) * (dev
->mem
->nregions
- u
->region_rb_len
));
912 memset(&(u
->region_rb_offset
[u
->region_rb_len
]), '\0',
913 sizeof(ram_addr_t
) * (dev
->mem
->nregions
- u
->region_rb_len
));
914 u
->region_rb_len
= dev
->mem
->nregions
;
917 if (config_mem_slots
) {
918 ret
= vhost_user_add_remove_regions(dev
, &msg
, reply_supported
, true);
923 ret
= vhost_user_fill_set_mem_table_msg(u
, dev
, &msg
, fds
, &fd_num
,
929 ret
= vhost_user_write(dev
, &msg
, fds
, fd_num
);
934 ret
= vhost_user_read(dev
, &msg_reply
);
939 if (msg_reply
.hdr
.request
!= VHOST_USER_SET_MEM_TABLE
) {
940 error_report("%s: Received unexpected msg type."
941 "Expected %d received %d", __func__
,
942 VHOST_USER_SET_MEM_TABLE
, msg_reply
.hdr
.request
);
947 * We're using the same structure, just reusing one of the
948 * fields, so it should be the same size.
950 if (msg_reply
.hdr
.size
!= msg
.hdr
.size
) {
951 error_report("%s: Unexpected size for postcopy reply "
952 "%d vs %d", __func__
, msg_reply
.hdr
.size
,
957 memset(u
->postcopy_client_bases
, 0,
958 sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS
);
961 * They're in the same order as the regions that were sent
962 * but some of the regions were skipped (above) if they
965 for (msg_i
= 0, region_i
= 0;
966 region_i
< dev
->mem
->nregions
;
968 if (msg_i
< fd_num
&&
969 msg_reply
.payload
.memory
.regions
[msg_i
].guest_phys_addr
==
970 dev
->mem
->regions
[region_i
].guest_phys_addr
) {
971 u
->postcopy_client_bases
[region_i
] =
972 msg_reply
.payload
.memory
.regions
[msg_i
].userspace_addr
;
973 trace_vhost_user_set_mem_table_postcopy(
974 msg_reply
.payload
.memory
.regions
[msg_i
].userspace_addr
,
975 msg
.payload
.memory
.regions
[msg_i
].userspace_addr
,
980 if (msg_i
!= fd_num
) {
981 error_report("%s: postcopy reply not fully consumed "
983 __func__
, msg_i
, fd_num
);
988 * Now we've registered this with the postcopy code, we ack to the
989 * client, because now we're in the position to be able to deal
990 * with any faults it generates.
992 /* TODO: Use this for failure cases as well with a bad value. */
993 msg
.hdr
.size
= sizeof(msg
.payload
.u64
);
994 msg
.payload
.u64
= 0; /* OK */
995 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
1004 static int vhost_user_set_mem_table(struct vhost_dev
*dev
,
1005 struct vhost_memory
*mem
)
1007 struct vhost_user
*u
= dev
->opaque
;
1008 int fds
[VHOST_MEMORY_BASELINE_NREGIONS
];
1010 bool do_postcopy
= u
->postcopy_listen
&& u
->postcopy_fd
.handler
;
1011 bool reply_supported
= virtio_has_feature(dev
->protocol_features
,
1012 VHOST_USER_PROTOCOL_F_REPLY_ACK
);
1013 bool config_mem_slots
=
1014 virtio_has_feature(dev
->protocol_features
,
1015 VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS
);
1020 * Postcopy has enough differences that it's best done in it's own
1023 return vhost_user_set_mem_table_postcopy(dev
, mem
, reply_supported
,
1027 VhostUserMsg msg
= {
1028 .hdr
.flags
= VHOST_USER_VERSION
,
1031 if (reply_supported
) {
1032 msg
.hdr
.flags
|= VHOST_USER_NEED_REPLY_MASK
;
1035 if (config_mem_slots
) {
1036 ret
= vhost_user_add_remove_regions(dev
, &msg
, reply_supported
, false);
1041 ret
= vhost_user_fill_set_mem_table_msg(u
, dev
, &msg
, fds
, &fd_num
,
1047 ret
= vhost_user_write(dev
, &msg
, fds
, fd_num
);
1052 if (reply_supported
) {
1053 return process_message_reply(dev
, &msg
);
1060 static int vhost_user_set_vring_endian(struct vhost_dev
*dev
,
1061 struct vhost_vring_state
*ring
)
1063 bool cross_endian
= virtio_has_feature(dev
->protocol_features
,
1064 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN
);
1065 VhostUserMsg msg
= {
1066 .hdr
.request
= VHOST_USER_SET_VRING_ENDIAN
,
1067 .hdr
.flags
= VHOST_USER_VERSION
,
1068 .payload
.state
= *ring
,
1069 .hdr
.size
= sizeof(msg
.payload
.state
),
1072 if (!cross_endian
) {
1073 error_report("vhost-user trying to send unhandled ioctl");
1077 return vhost_user_write(dev
, &msg
, NULL
, 0);
1080 static int vhost_user_get_u64(struct vhost_dev
*dev
, int request
, uint64_t *u64
)
1083 VhostUserMsg msg
= {
1084 .hdr
.request
= request
,
1085 .hdr
.flags
= VHOST_USER_VERSION
,
1088 if (vhost_user_per_device_request(request
) && dev
->vq_index
!= 0) {
1092 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
1097 ret
= vhost_user_read(dev
, &msg
);
1102 if (msg
.hdr
.request
!= request
) {
1103 error_report("Received unexpected msg type. Expected %d received %d",
1104 request
, msg
.hdr
.request
);
1108 if (msg
.hdr
.size
!= sizeof(msg
.payload
.u64
)) {
1109 error_report("Received bad msg size.");
1113 *u64
= msg
.payload
.u64
;
1118 static int vhost_user_get_features(struct vhost_dev
*dev
, uint64_t *features
)
1120 if (vhost_user_get_u64(dev
, VHOST_USER_GET_FEATURES
, features
) < 0) {
1127 /* Note: "msg->hdr.flags" may be modified. */
1128 static int vhost_user_write_sync(struct vhost_dev
*dev
, VhostUserMsg
*msg
,
1129 bool wait_for_reply
)
1133 if (wait_for_reply
) {
1134 bool reply_supported
= virtio_has_feature(dev
->protocol_features
,
1135 VHOST_USER_PROTOCOL_F_REPLY_ACK
);
1136 if (reply_supported
) {
1137 msg
->hdr
.flags
|= VHOST_USER_NEED_REPLY_MASK
;
1141 ret
= vhost_user_write(dev
, msg
, NULL
, 0);
1146 if (wait_for_reply
) {
1149 if (msg
->hdr
.flags
& VHOST_USER_NEED_REPLY_MASK
) {
1150 return process_message_reply(dev
, msg
);
1154 * We need to wait for a reply but the backend does not
1155 * support replies for the command we just sent.
1156 * Send VHOST_USER_GET_FEATURES which makes all backends
1159 return vhost_user_get_features(dev
, &dummy
);
1165 static int vhost_set_vring(struct vhost_dev
*dev
,
1166 unsigned long int request
,
1167 struct vhost_vring_state
*ring
,
1168 bool wait_for_reply
)
1170 VhostUserMsg msg
= {
1171 .hdr
.request
= request
,
1172 .hdr
.flags
= VHOST_USER_VERSION
,
1173 .payload
.state
= *ring
,
1174 .hdr
.size
= sizeof(msg
.payload
.state
),
1177 return vhost_user_write_sync(dev
, &msg
, wait_for_reply
);
1180 static int vhost_user_set_vring_num(struct vhost_dev
*dev
,
1181 struct vhost_vring_state
*ring
)
1183 return vhost_set_vring(dev
, VHOST_USER_SET_VRING_NUM
, ring
, false);
1186 static void vhost_user_host_notifier_free(VhostUserHostNotifier
*n
)
1188 assert(n
&& n
->unmap_addr
);
1189 munmap(n
->unmap_addr
, qemu_real_host_page_size());
1190 n
->unmap_addr
= NULL
;
1194 * clean-up function for notifier, will finally free the structure
1197 static void vhost_user_host_notifier_remove(VhostUserHostNotifier
*n
,
1202 virtio_queue_set_host_notifier_mr(vdev
, n
->idx
, &n
->mr
, false);
1204 assert(!n
->unmap_addr
);
1205 n
->unmap_addr
= n
->addr
;
1207 call_rcu(n
, vhost_user_host_notifier_free
, rcu
);
1211 static int vhost_user_set_vring_base(struct vhost_dev
*dev
,
1212 struct vhost_vring_state
*ring
)
1214 return vhost_set_vring(dev
, VHOST_USER_SET_VRING_BASE
, ring
, false);
1217 static int vhost_user_set_vring_enable(struct vhost_dev
*dev
, int enable
)
1221 if (!virtio_has_feature(dev
->features
, VHOST_USER_F_PROTOCOL_FEATURES
)) {
1225 for (i
= 0; i
< dev
->nvqs
; ++i
) {
1227 struct vhost_vring_state state
= {
1228 .index
= dev
->vq_index
+ i
,
1233 * SET_VRING_ENABLE travels from guest to QEMU to vhost-user backend /
1234 * control plane thread via unix domain socket. Virtio requests travel
1235 * from guest to vhost-user backend / data plane thread via eventfd.
1236 * Even if the guest enables the ring first, and pushes its first virtio
1237 * request second (conforming to the virtio spec), the data plane thread
1238 * in the backend may see the virtio request before the control plane
1239 * thread sees the queue enablement. This causes (in fact, requires) the
1240 * data plane thread to discard the virtio request (it arrived on a
1241 * seemingly disabled queue). To prevent this out-of-order delivery,
1242 * don't let the guest proceed to pushing the virtio request until the
1243 * backend control plane acknowledges enabling the queue -- IOW, pass
1244 * wait_for_reply=true below.
1246 ret
= vhost_set_vring(dev
, VHOST_USER_SET_VRING_ENABLE
, &state
, true);
1249 * Restoring the previous state is likely infeasible, as well as
1250 * proceeding regardless the error, so just bail out and hope for
1251 * the device-level recovery.
1260 static VhostUserHostNotifier
*fetch_notifier(VhostUserState
*u
,
1263 if (idx
>= u
->notifiers
->len
) {
1266 return g_ptr_array_index(u
->notifiers
, idx
);
1269 static int vhost_user_get_vring_base(struct vhost_dev
*dev
,
1270 struct vhost_vring_state
*ring
)
1273 VhostUserMsg msg
= {
1274 .hdr
.request
= VHOST_USER_GET_VRING_BASE
,
1275 .hdr
.flags
= VHOST_USER_VERSION
,
1276 .payload
.state
= *ring
,
1277 .hdr
.size
= sizeof(msg
.payload
.state
),
1279 struct vhost_user
*u
= dev
->opaque
;
1281 VhostUserHostNotifier
*n
= fetch_notifier(u
->user
, ring
->index
);
1283 vhost_user_host_notifier_remove(n
, dev
->vdev
);
1286 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
1291 ret
= vhost_user_read(dev
, &msg
);
1296 if (msg
.hdr
.request
!= VHOST_USER_GET_VRING_BASE
) {
1297 error_report("Received unexpected msg type. Expected %d received %d",
1298 VHOST_USER_GET_VRING_BASE
, msg
.hdr
.request
);
1302 if (msg
.hdr
.size
!= sizeof(msg
.payload
.state
)) {
1303 error_report("Received bad msg size.");
1307 *ring
= msg
.payload
.state
;
1312 static int vhost_set_vring_file(struct vhost_dev
*dev
,
1313 VhostUserRequest request
,
1314 struct vhost_vring_file
*file
)
1316 int fds
[VHOST_USER_MAX_RAM_SLOTS
];
1318 VhostUserMsg msg
= {
1319 .hdr
.request
= request
,
1320 .hdr
.flags
= VHOST_USER_VERSION
,
1321 .payload
.u64
= file
->index
& VHOST_USER_VRING_IDX_MASK
,
1322 .hdr
.size
= sizeof(msg
.payload
.u64
),
1326 fds
[fd_num
++] = file
->fd
;
1328 msg
.payload
.u64
|= VHOST_USER_VRING_NOFD_MASK
;
1331 return vhost_user_write(dev
, &msg
, fds
, fd_num
);
1334 static int vhost_user_set_vring_kick(struct vhost_dev
*dev
,
1335 struct vhost_vring_file
*file
)
1337 return vhost_set_vring_file(dev
, VHOST_USER_SET_VRING_KICK
, file
);
1340 static int vhost_user_set_vring_call(struct vhost_dev
*dev
,
1341 struct vhost_vring_file
*file
)
1343 return vhost_set_vring_file(dev
, VHOST_USER_SET_VRING_CALL
, file
);
1346 static int vhost_user_set_vring_err(struct vhost_dev
*dev
,
1347 struct vhost_vring_file
*file
)
1349 return vhost_set_vring_file(dev
, VHOST_USER_SET_VRING_ERR
, file
);
1352 static int vhost_user_set_vring_addr(struct vhost_dev
*dev
,
1353 struct vhost_vring_addr
*addr
)
1355 VhostUserMsg msg
= {
1356 .hdr
.request
= VHOST_USER_SET_VRING_ADDR
,
1357 .hdr
.flags
= VHOST_USER_VERSION
,
1358 .payload
.addr
= *addr
,
1359 .hdr
.size
= sizeof(msg
.payload
.addr
),
1363 * wait for a reply if logging is enabled to make sure
1364 * backend is actually logging changes
1366 bool wait_for_reply
= addr
->flags
& (1 << VHOST_VRING_F_LOG
);
1368 return vhost_user_write_sync(dev
, &msg
, wait_for_reply
);
1371 static int vhost_user_set_u64(struct vhost_dev
*dev
, int request
, uint64_t u64
,
1372 bool wait_for_reply
)
1374 VhostUserMsg msg
= {
1375 .hdr
.request
= request
,
1376 .hdr
.flags
= VHOST_USER_VERSION
,
1378 .hdr
.size
= sizeof(msg
.payload
.u64
),
1381 return vhost_user_write_sync(dev
, &msg
, wait_for_reply
);
1384 static int vhost_user_set_status(struct vhost_dev
*dev
, uint8_t status
)
1386 return vhost_user_set_u64(dev
, VHOST_USER_SET_STATUS
, status
, false);
1389 static int vhost_user_get_status(struct vhost_dev
*dev
, uint8_t *status
)
1394 ret
= vhost_user_get_u64(dev
, VHOST_USER_GET_STATUS
, &value
);
1403 static int vhost_user_add_status(struct vhost_dev
*dev
, uint8_t status
)
1408 ret
= vhost_user_get_status(dev
, &s
);
1413 if ((s
& status
) == status
) {
1418 return vhost_user_set_status(dev
, s
);
1421 static int vhost_user_set_features(struct vhost_dev
*dev
,
1425 * wait for a reply if logging is enabled to make sure
1426 * backend is actually logging changes
1428 bool log_enabled
= features
& (0x1ULL
<< VHOST_F_LOG_ALL
);
1432 * We need to include any extra backend only feature bits that
1433 * might be needed by our device. Currently this includes the
1434 * VHOST_USER_F_PROTOCOL_FEATURES bit for enabling protocol
1437 ret
= vhost_user_set_u64(dev
, VHOST_USER_SET_FEATURES
,
1438 features
| dev
->backend_features
,
1441 if (virtio_has_feature(dev
->protocol_features
,
1442 VHOST_USER_PROTOCOL_F_STATUS
)) {
1444 return vhost_user_add_status(dev
, VIRTIO_CONFIG_S_FEATURES_OK
);
1451 static int vhost_user_set_protocol_features(struct vhost_dev
*dev
,
1454 return vhost_user_set_u64(dev
, VHOST_USER_SET_PROTOCOL_FEATURES
, features
,
1458 static int vhost_user_set_owner(struct vhost_dev
*dev
)
1460 VhostUserMsg msg
= {
1461 .hdr
.request
= VHOST_USER_SET_OWNER
,
1462 .hdr
.flags
= VHOST_USER_VERSION
,
1465 return vhost_user_write(dev
, &msg
, NULL
, 0);
1468 static int vhost_user_get_max_memslots(struct vhost_dev
*dev
,
1469 uint64_t *max_memslots
)
1471 uint64_t backend_max_memslots
;
1474 err
= vhost_user_get_u64(dev
, VHOST_USER_GET_MAX_MEM_SLOTS
,
1475 &backend_max_memslots
);
1480 *max_memslots
= backend_max_memslots
;
1485 static int vhost_user_reset_device(struct vhost_dev
*dev
)
1487 VhostUserMsg msg
= {
1488 .hdr
.flags
= VHOST_USER_VERSION
,
1489 .hdr
.request
= VHOST_USER_RESET_DEVICE
,
1493 * Historically, reset was not implemented so only reset devices
1494 * that are expecting it.
1496 if (!virtio_has_feature(dev
->protocol_features
,
1497 VHOST_USER_PROTOCOL_F_RESET_DEVICE
)) {
1501 return vhost_user_write(dev
, &msg
, NULL
, 0);
1504 static int vhost_user_backend_handle_config_change(struct vhost_dev
*dev
)
1506 if (!dev
->config_ops
|| !dev
->config_ops
->vhost_dev_config_notifier
) {
1510 return dev
->config_ops
->vhost_dev_config_notifier(dev
);
1514 * Fetch or create the notifier for a given idx. Newly created
1515 * notifiers are added to the pointer array that tracks them.
1517 static VhostUserHostNotifier
*fetch_or_create_notifier(VhostUserState
*u
,
1520 VhostUserHostNotifier
*n
= NULL
;
1521 if (idx
>= u
->notifiers
->len
) {
1522 g_ptr_array_set_size(u
->notifiers
, idx
+ 1);
1525 n
= g_ptr_array_index(u
->notifiers
, idx
);
1528 * In case notification arrive out-of-order,
1529 * make room for current index.
1531 g_ptr_array_remove_index(u
->notifiers
, idx
);
1532 n
= g_new0(VhostUserHostNotifier
, 1);
1534 g_ptr_array_insert(u
->notifiers
, idx
, n
);
1535 trace_vhost_user_create_notifier(idx
, n
);
1541 static int vhost_user_backend_handle_vring_host_notifier(struct vhost_dev
*dev
,
1542 VhostUserVringArea
*area
,
1545 int queue_idx
= area
->u64
& VHOST_USER_VRING_IDX_MASK
;
1546 size_t page_size
= qemu_real_host_page_size();
1547 struct vhost_user
*u
= dev
->opaque
;
1548 VhostUserState
*user
= u
->user
;
1549 VirtIODevice
*vdev
= dev
->vdev
;
1550 VhostUserHostNotifier
*n
;
1554 if (!virtio_has_feature(dev
->protocol_features
,
1555 VHOST_USER_PROTOCOL_F_HOST_NOTIFIER
) ||
1556 vdev
== NULL
|| queue_idx
>= virtio_get_num_queues(vdev
)) {
1561 * Fetch notifier and invalidate any old data before setting up
1562 * new mapped address.
1564 n
= fetch_or_create_notifier(user
, queue_idx
);
1565 vhost_user_host_notifier_remove(n
, vdev
);
1567 if (area
->u64
& VHOST_USER_VRING_NOFD_MASK
) {
1572 if (area
->size
!= page_size
) {
1576 addr
= mmap(NULL
, page_size
, PROT_READ
| PROT_WRITE
, MAP_SHARED
,
1578 if (addr
== MAP_FAILED
) {
1582 name
= g_strdup_printf("vhost-user/host-notifier@%p mmaps[%d]",
1584 if (!n
->mr
.ram
) { /* Don't init again after suspend. */
1585 memory_region_init_ram_device_ptr(&n
->mr
, OBJECT(vdev
), name
,
1588 n
->mr
.ram_block
->host
= addr
;
1592 if (virtio_queue_set_host_notifier_mr(vdev
, queue_idx
, &n
->mr
, true)) {
1593 object_unparent(OBJECT(&n
->mr
));
1594 munmap(addr
, page_size
);
1604 vhost_user_backend_handle_shared_object_add(struct vhost_dev
*dev
,
1605 VhostUserShared
*object
)
1609 memcpy(uuid
.data
, object
->uuid
, sizeof(object
->uuid
));
1610 return virtio_add_vhost_device(&uuid
, dev
);
1614 vhost_user_backend_handle_shared_object_remove(struct vhost_dev
*dev
,
1615 VhostUserShared
*object
)
1619 memcpy(uuid
.data
, object
->uuid
, sizeof(object
->uuid
));
1620 switch (virtio_object_type(&uuid
)) {
1621 case TYPE_VHOST_DEV
:
1623 struct vhost_dev
*owner
= virtio_lookup_vhost_device(&uuid
);
1625 /* Not allowed to remove non-owned entries */
1631 /* Not allowed to remove non-owned entries */
1635 return virtio_remove_resource(&uuid
);
1638 static bool vhost_user_send_resp(QIOChannel
*ioc
, VhostUserHeader
*hdr
,
1639 VhostUserPayload
*payload
, Error
**errp
)
1641 struct iovec iov
[] = {
1642 { .iov_base
= hdr
, .iov_len
= VHOST_USER_HDR_SIZE
},
1643 { .iov_base
= payload
, .iov_len
= hdr
->size
},
1646 hdr
->flags
&= ~VHOST_USER_NEED_REPLY_MASK
;
1647 hdr
->flags
|= VHOST_USER_REPLY_MASK
;
1649 return !qio_channel_writev_all(ioc
, iov
, ARRAY_SIZE(iov
), errp
);
1653 vhost_user_backend_send_dmabuf_fd(QIOChannel
*ioc
, VhostUserHeader
*hdr
,
1654 VhostUserPayload
*payload
, Error
**errp
)
1656 hdr
->size
= sizeof(payload
->u64
);
1657 return vhost_user_send_resp(ioc
, hdr
, payload
, errp
);
1660 int vhost_user_get_shared_object(struct vhost_dev
*dev
, unsigned char *uuid
,
1663 struct vhost_user
*u
= dev
->opaque
;
1664 CharBackend
*chr
= u
->user
->chr
;
1666 VhostUserMsg msg
= {
1667 .hdr
.request
= VHOST_USER_GET_SHARED_OBJECT
,
1668 .hdr
.flags
= VHOST_USER_VERSION
,
1670 memcpy(msg
.payload
.object
.uuid
, uuid
, sizeof(msg
.payload
.object
.uuid
));
1672 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
1677 ret
= vhost_user_read(dev
, &msg
);
1682 if (msg
.hdr
.request
!= VHOST_USER_GET_SHARED_OBJECT
) {
1683 error_report("Received unexpected msg type. "
1684 "Expected %d received %d",
1685 VHOST_USER_GET_SHARED_OBJECT
, msg
.hdr
.request
);
1689 *dmabuf_fd
= qemu_chr_fe_get_msgfd(chr
);
1690 if (*dmabuf_fd
< 0) {
1691 error_report("Failed to get dmabuf fd");
1699 vhost_user_backend_handle_shared_object_lookup(struct vhost_user
*u
,
1701 VhostUserHeader
*hdr
,
1702 VhostUserPayload
*payload
)
1705 CharBackend
*chr
= u
->user
->chr
;
1706 Error
*local_err
= NULL
;
1710 memcpy(uuid
.data
, payload
->object
.uuid
, sizeof(payload
->object
.uuid
));
1713 switch (virtio_object_type(&uuid
)) {
1715 dmabuf_fd
= virtio_lookup_dmabuf(&uuid
);
1717 case TYPE_VHOST_DEV
:
1719 struct vhost_dev
*dev
= virtio_lookup_vhost_device(&uuid
);
1721 payload
->u64
= -EINVAL
;
1724 int ret
= vhost_user_get_shared_object(dev
, uuid
.data
, &dmabuf_fd
);
1731 payload
->u64
= -EINVAL
;
1735 if (dmabuf_fd
!= -1) {
1739 if (qemu_chr_fe_set_msgfds(chr
, &dmabuf_fd
, fd_num
) < 0) {
1740 error_report("Failed to set msg fds.");
1741 payload
->u64
= -EINVAL
;
1744 if (!vhost_user_backend_send_dmabuf_fd(ioc
, hdr
, payload
, &local_err
)) {
1745 error_report_err(local_err
);
1752 static void close_backend_channel(struct vhost_user
*u
)
1754 g_source_destroy(u
->backend_src
);
1755 g_source_unref(u
->backend_src
);
1756 u
->backend_src
= NULL
;
1757 object_unref(OBJECT(u
->backend_ioc
));
1758 u
->backend_ioc
= NULL
;
1761 static gboolean
backend_read(QIOChannel
*ioc
, GIOCondition condition
,
1764 struct vhost_dev
*dev
= opaque
;
1765 struct vhost_user
*u
= dev
->opaque
;
1766 VhostUserHeader hdr
= { 0, };
1767 VhostUserPayload payload
= { 0, };
1768 Error
*local_err
= NULL
;
1769 gboolean rc
= G_SOURCE_CONTINUE
;
1772 g_autofree
int *fd
= NULL
;
1777 iov
.iov_base
= &hdr
;
1778 iov
.iov_len
= VHOST_USER_HDR_SIZE
;
1780 if (qio_channel_readv_full_all(ioc
, &iov
, 1, &fd
, &fdsize
, &local_err
)) {
1781 error_report_err(local_err
);
1785 if (hdr
.size
> VHOST_USER_PAYLOAD_SIZE
) {
1786 error_report("Failed to read msg header."
1787 " Size %d exceeds the maximum %zu.", hdr
.size
,
1788 VHOST_USER_PAYLOAD_SIZE
);
1793 if (qio_channel_read_all(ioc
, (char *) &payload
, hdr
.size
, &local_err
)) {
1794 error_report_err(local_err
);
1798 switch (hdr
.request
) {
1799 case VHOST_USER_BACKEND_IOTLB_MSG
:
1800 ret
= vhost_backend_handle_iotlb_msg(dev
, &payload
.iotlb
);
1802 case VHOST_USER_BACKEND_CONFIG_CHANGE_MSG
:
1803 ret
= vhost_user_backend_handle_config_change(dev
);
1805 case VHOST_USER_BACKEND_VRING_HOST_NOTIFIER_MSG
:
1806 ret
= vhost_user_backend_handle_vring_host_notifier(dev
, &payload
.area
,
1809 case VHOST_USER_BACKEND_SHARED_OBJECT_ADD
:
1810 ret
= vhost_user_backend_handle_shared_object_add(dev
, &payload
.object
);
1812 case VHOST_USER_BACKEND_SHARED_OBJECT_REMOVE
:
1813 ret
= vhost_user_backend_handle_shared_object_remove(dev
,
1816 case VHOST_USER_BACKEND_SHARED_OBJECT_LOOKUP
:
1817 ret
= vhost_user_backend_handle_shared_object_lookup(dev
->opaque
, ioc
,
1821 error_report("Received unexpected msg type: %d.", hdr
.request
);
1826 * REPLY_ACK feature handling. Other reply types has to be managed
1827 * directly in their request handlers.
1829 if (hdr
.flags
& VHOST_USER_NEED_REPLY_MASK
) {
1830 payload
.u64
= !!ret
;
1831 hdr
.size
= sizeof(payload
.u64
);
1833 if (!vhost_user_send_resp(ioc
, &hdr
, &payload
, &local_err
)) {
1834 error_report_err(local_err
);
1842 close_backend_channel(u
);
1843 rc
= G_SOURCE_REMOVE
;
1847 for (i
= 0; i
< fdsize
; i
++) {
1854 static int vhost_setup_backend_channel(struct vhost_dev
*dev
)
1856 VhostUserMsg msg
= {
1857 .hdr
.request
= VHOST_USER_SET_BACKEND_REQ_FD
,
1858 .hdr
.flags
= VHOST_USER_VERSION
,
1860 struct vhost_user
*u
= dev
->opaque
;
1862 bool reply_supported
= virtio_has_feature(dev
->protocol_features
,
1863 VHOST_USER_PROTOCOL_F_REPLY_ACK
);
1864 Error
*local_err
= NULL
;
1867 if (!virtio_has_feature(dev
->protocol_features
,
1868 VHOST_USER_PROTOCOL_F_BACKEND_REQ
)) {
1872 if (qemu_socketpair(PF_UNIX
, SOCK_STREAM
, 0, sv
) == -1) {
1873 int saved_errno
= errno
;
1874 error_report("socketpair() failed");
1875 return -saved_errno
;
1878 ioc
= QIO_CHANNEL(qio_channel_socket_new_fd(sv
[0], &local_err
));
1880 error_report_err(local_err
);
1881 return -ECONNREFUSED
;
1883 u
->backend_ioc
= ioc
;
1884 u
->backend_src
= qio_channel_add_watch_source(u
->backend_ioc
,
1886 backend_read
, dev
, NULL
, NULL
);
1888 if (reply_supported
) {
1889 msg
.hdr
.flags
|= VHOST_USER_NEED_REPLY_MASK
;
1892 ret
= vhost_user_write(dev
, &msg
, &sv
[1], 1);
1897 if (reply_supported
) {
1898 ret
= process_message_reply(dev
, &msg
);
1904 close_backend_channel(u
);
1912 * Called back from the postcopy fault thread when a fault is received on our
1914 * TODO: This is Linux specific
1916 static int vhost_user_postcopy_fault_handler(struct PostCopyFD
*pcfd
,
1919 struct vhost_dev
*dev
= pcfd
->data
;
1920 struct vhost_user
*u
= dev
->opaque
;
1921 struct uffd_msg
*msg
= ufd
;
1922 uint64_t faultaddr
= msg
->arg
.pagefault
.address
;
1923 RAMBlock
*rb
= NULL
;
1927 trace_vhost_user_postcopy_fault_handler(pcfd
->idstr
, faultaddr
,
1928 dev
->mem
->nregions
);
1929 for (i
= 0; i
< MIN(dev
->mem
->nregions
, u
->region_rb_len
); i
++) {
1930 trace_vhost_user_postcopy_fault_handler_loop(i
,
1931 u
->postcopy_client_bases
[i
], dev
->mem
->regions
[i
].memory_size
);
1932 if (faultaddr
>= u
->postcopy_client_bases
[i
]) {
1933 /* Ofset of the fault address in the vhost region */
1934 uint64_t region_offset
= faultaddr
- u
->postcopy_client_bases
[i
];
1935 if (region_offset
< dev
->mem
->regions
[i
].memory_size
) {
1936 rb_offset
= region_offset
+ u
->region_rb_offset
[i
];
1937 trace_vhost_user_postcopy_fault_handler_found(i
,
1938 region_offset
, rb_offset
);
1939 rb
= u
->region_rb
[i
];
1940 return postcopy_request_shared_page(pcfd
, rb
, faultaddr
,
1945 error_report("%s: Failed to find region for fault %" PRIx64
,
1946 __func__
, faultaddr
);
1950 static int vhost_user_postcopy_waker(struct PostCopyFD
*pcfd
, RAMBlock
*rb
,
1953 struct vhost_dev
*dev
= pcfd
->data
;
1954 struct vhost_user
*u
= dev
->opaque
;
1957 trace_vhost_user_postcopy_waker(qemu_ram_get_idstr(rb
), offset
);
1962 /* Translate the offset into an address in the clients address space */
1963 for (i
= 0; i
< MIN(dev
->mem
->nregions
, u
->region_rb_len
); i
++) {
1964 if (u
->region_rb
[i
] == rb
&&
1965 offset
>= u
->region_rb_offset
[i
] &&
1966 offset
< (u
->region_rb_offset
[i
] +
1967 dev
->mem
->regions
[i
].memory_size
)) {
1968 uint64_t client_addr
= (offset
- u
->region_rb_offset
[i
]) +
1969 u
->postcopy_client_bases
[i
];
1970 trace_vhost_user_postcopy_waker_found(client_addr
);
1971 return postcopy_wake_shared(pcfd
, client_addr
, rb
);
1975 trace_vhost_user_postcopy_waker_nomatch(qemu_ram_get_idstr(rb
), offset
);
1981 * Called at the start of an inbound postcopy on reception of the
1984 static int vhost_user_postcopy_advise(struct vhost_dev
*dev
, Error
**errp
)
1987 struct vhost_user
*u
= dev
->opaque
;
1988 CharBackend
*chr
= u
->user
->chr
;
1991 VhostUserMsg msg
= {
1992 .hdr
.request
= VHOST_USER_POSTCOPY_ADVISE
,
1993 .hdr
.flags
= VHOST_USER_VERSION
,
1996 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
1998 error_setg(errp
, "Failed to send postcopy_advise to vhost");
2002 ret
= vhost_user_read(dev
, &msg
);
2004 error_setg(errp
, "Failed to get postcopy_advise reply from vhost");
2008 if (msg
.hdr
.request
!= VHOST_USER_POSTCOPY_ADVISE
) {
2009 error_setg(errp
, "Unexpected msg type. Expected %d received %d",
2010 VHOST_USER_POSTCOPY_ADVISE
, msg
.hdr
.request
);
2015 error_setg(errp
, "Received bad msg size.");
2018 ufd
= qemu_chr_fe_get_msgfd(chr
);
2020 error_setg(errp
, "%s: Failed to get ufd", __func__
);
2023 qemu_socket_set_nonblock(ufd
);
2025 /* register ufd with userfault thread */
2026 u
->postcopy_fd
.fd
= ufd
;
2027 u
->postcopy_fd
.data
= dev
;
2028 u
->postcopy_fd
.handler
= vhost_user_postcopy_fault_handler
;
2029 u
->postcopy_fd
.waker
= vhost_user_postcopy_waker
;
2030 u
->postcopy_fd
.idstr
= "vhost-user"; /* Need to find unique name */
2031 postcopy_register_shared_ufd(&u
->postcopy_fd
);
2034 error_setg(errp
, "Postcopy not supported on non-Linux systems");
2040 * Called at the switch to postcopy on reception of the 'listen' command.
2042 static int vhost_user_postcopy_listen(struct vhost_dev
*dev
, Error
**errp
)
2044 struct vhost_user
*u
= dev
->opaque
;
2046 VhostUserMsg msg
= {
2047 .hdr
.request
= VHOST_USER_POSTCOPY_LISTEN
,
2048 .hdr
.flags
= VHOST_USER_VERSION
| VHOST_USER_NEED_REPLY_MASK
,
2050 u
->postcopy_listen
= true;
2052 trace_vhost_user_postcopy_listen();
2054 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
2056 error_setg(errp
, "Failed to send postcopy_listen to vhost");
2060 ret
= process_message_reply(dev
, &msg
);
2062 error_setg(errp
, "Failed to receive reply to postcopy_listen");
2070 * Called at the end of postcopy
2072 static int vhost_user_postcopy_end(struct vhost_dev
*dev
, Error
**errp
)
2074 VhostUserMsg msg
= {
2075 .hdr
.request
= VHOST_USER_POSTCOPY_END
,
2076 .hdr
.flags
= VHOST_USER_VERSION
| VHOST_USER_NEED_REPLY_MASK
,
2079 struct vhost_user
*u
= dev
->opaque
;
2081 trace_vhost_user_postcopy_end_entry();
2083 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
2085 error_setg(errp
, "Failed to send postcopy_end to vhost");
2089 ret
= process_message_reply(dev
, &msg
);
2091 error_setg(errp
, "Failed to receive reply to postcopy_end");
2094 postcopy_unregister_shared_ufd(&u
->postcopy_fd
);
2095 close(u
->postcopy_fd
.fd
);
2096 u
->postcopy_fd
.handler
= NULL
;
2098 trace_vhost_user_postcopy_end_exit();
2103 static int vhost_user_postcopy_notifier(NotifierWithReturn
*notifier
,
2104 void *opaque
, Error
**errp
)
2106 struct PostcopyNotifyData
*pnd
= opaque
;
2107 struct vhost_user
*u
= container_of(notifier
, struct vhost_user
,
2109 struct vhost_dev
*dev
= u
->dev
;
2111 switch (pnd
->reason
) {
2112 case POSTCOPY_NOTIFY_PROBE
:
2113 if (!virtio_has_feature(dev
->protocol_features
,
2114 VHOST_USER_PROTOCOL_F_PAGEFAULT
)) {
2115 /* TODO: Get the device name into this error somehow */
2117 "vhost-user backend not capable of postcopy");
2122 case POSTCOPY_NOTIFY_INBOUND_ADVISE
:
2123 return vhost_user_postcopy_advise(dev
, errp
);
2125 case POSTCOPY_NOTIFY_INBOUND_LISTEN
:
2126 return vhost_user_postcopy_listen(dev
, errp
);
2128 case POSTCOPY_NOTIFY_INBOUND_END
:
2129 return vhost_user_postcopy_end(dev
, errp
);
2132 /* We ignore notifications we don't know */
2139 static int vhost_user_backend_init(struct vhost_dev
*dev
, void *opaque
,
2142 uint64_t features
, ram_slots
;
2143 struct vhost_user
*u
;
2144 VhostUserState
*vus
= (VhostUserState
*) opaque
;
2147 assert(dev
->vhost_ops
->backend_type
== VHOST_BACKEND_TYPE_USER
);
2149 u
= g_new0(struct vhost_user
, 1);
2154 err
= vhost_user_get_features(dev
, &features
);
2156 error_setg_errno(errp
, -err
, "vhost_backend_init failed");
2160 if (virtio_has_feature(features
, VHOST_USER_F_PROTOCOL_FEATURES
)) {
2161 bool supports_f_config
= vus
->supports_config
||
2162 (dev
->config_ops
&& dev
->config_ops
->vhost_dev_config_notifier
);
2163 uint64_t protocol_features
;
2165 dev
->backend_features
|= 1ULL << VHOST_USER_F_PROTOCOL_FEATURES
;
2167 err
= vhost_user_get_u64(dev
, VHOST_USER_GET_PROTOCOL_FEATURES
,
2168 &protocol_features
);
2170 error_setg_errno(errp
, EPROTO
, "vhost_backend_init failed");
2175 * We will use all the protocol features we support - although
2176 * we suppress F_CONFIG if we know QEMUs internal code can not support
2179 protocol_features
&= VHOST_USER_PROTOCOL_FEATURE_MASK
;
2181 if (supports_f_config
) {
2182 if (!virtio_has_feature(protocol_features
,
2183 VHOST_USER_PROTOCOL_F_CONFIG
)) {
2184 error_setg(errp
, "vhost-user device expecting "
2185 "VHOST_USER_PROTOCOL_F_CONFIG but the vhost-user backend does "
2190 if (virtio_has_feature(protocol_features
,
2191 VHOST_USER_PROTOCOL_F_CONFIG
)) {
2192 warn_report("vhost-user backend supports "
2193 "VHOST_USER_PROTOCOL_F_CONFIG but QEMU does not.");
2194 protocol_features
&= ~(1ULL << VHOST_USER_PROTOCOL_F_CONFIG
);
2198 /* final set of protocol features */
2199 dev
->protocol_features
= protocol_features
;
2200 err
= vhost_user_set_protocol_features(dev
, dev
->protocol_features
);
2202 error_setg_errno(errp
, EPROTO
, "vhost_backend_init failed");
2206 /* query the max queues we support if backend supports Multiple Queue */
2207 if (dev
->protocol_features
& (1ULL << VHOST_USER_PROTOCOL_F_MQ
)) {
2208 err
= vhost_user_get_u64(dev
, VHOST_USER_GET_QUEUE_NUM
,
2211 error_setg_errno(errp
, EPROTO
, "vhost_backend_init failed");
2215 dev
->max_queues
= 1;
2218 if (dev
->num_queues
&& dev
->max_queues
< dev
->num_queues
) {
2219 error_setg(errp
, "The maximum number of queues supported by the "
2220 "backend is %" PRIu64
, dev
->max_queues
);
2224 if (virtio_has_feature(features
, VIRTIO_F_IOMMU_PLATFORM
) &&
2225 !(virtio_has_feature(dev
->protocol_features
,
2226 VHOST_USER_PROTOCOL_F_BACKEND_REQ
) &&
2227 virtio_has_feature(dev
->protocol_features
,
2228 VHOST_USER_PROTOCOL_F_REPLY_ACK
))) {
2229 error_setg(errp
, "IOMMU support requires reply-ack and "
2230 "backend-req protocol features.");
2234 /* get max memory regions if backend supports configurable RAM slots */
2235 if (!virtio_has_feature(dev
->protocol_features
,
2236 VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS
)) {
2237 u
->user
->memory_slots
= VHOST_MEMORY_BASELINE_NREGIONS
;
2239 err
= vhost_user_get_max_memslots(dev
, &ram_slots
);
2241 error_setg_errno(errp
, EPROTO
, "vhost_backend_init failed");
2245 if (ram_slots
< u
->user
->memory_slots
) {
2246 error_setg(errp
, "The backend specified a max ram slots limit "
2247 "of %" PRIu64
", when the prior validated limit was "
2248 "%d. This limit should never decrease.", ram_slots
,
2249 u
->user
->memory_slots
);
2253 u
->user
->memory_slots
= MIN(ram_slots
, VHOST_USER_MAX_RAM_SLOTS
);
2257 if (dev
->migration_blocker
== NULL
&&
2258 !virtio_has_feature(dev
->protocol_features
,
2259 VHOST_USER_PROTOCOL_F_LOG_SHMFD
)) {
2260 error_setg(&dev
->migration_blocker
,
2261 "Migration disabled: vhost-user backend lacks "
2262 "VHOST_USER_PROTOCOL_F_LOG_SHMFD feature.");
2265 if (dev
->vq_index
== 0) {
2266 err
= vhost_setup_backend_channel(dev
);
2268 error_setg_errno(errp
, EPROTO
, "vhost_backend_init failed");
2273 u
->postcopy_notifier
.notify
= vhost_user_postcopy_notifier
;
2274 postcopy_add_notifier(&u
->postcopy_notifier
);
2279 static int vhost_user_backend_cleanup(struct vhost_dev
*dev
)
2281 struct vhost_user
*u
;
2283 assert(dev
->vhost_ops
->backend_type
== VHOST_BACKEND_TYPE_USER
);
2286 if (u
->postcopy_notifier
.notify
) {
2287 postcopy_remove_notifier(&u
->postcopy_notifier
);
2288 u
->postcopy_notifier
.notify
= NULL
;
2290 u
->postcopy_listen
= false;
2291 if (u
->postcopy_fd
.handler
) {
2292 postcopy_unregister_shared_ufd(&u
->postcopy_fd
);
2293 close(u
->postcopy_fd
.fd
);
2294 u
->postcopy_fd
.handler
= NULL
;
2296 if (u
->backend_ioc
) {
2297 close_backend_channel(u
);
2299 g_free(u
->region_rb
);
2300 u
->region_rb
= NULL
;
2301 g_free(u
->region_rb_offset
);
2302 u
->region_rb_offset
= NULL
;
2303 u
->region_rb_len
= 0;
2310 static int vhost_user_get_vq_index(struct vhost_dev
*dev
, int idx
)
2312 assert(idx
>= dev
->vq_index
&& idx
< dev
->vq_index
+ dev
->nvqs
);
2317 static int vhost_user_memslots_limit(struct vhost_dev
*dev
)
2319 struct vhost_user
*u
= dev
->opaque
;
2321 return u
->user
->memory_slots
;
2324 static bool vhost_user_requires_shm_log(struct vhost_dev
*dev
)
2326 assert(dev
->vhost_ops
->backend_type
== VHOST_BACKEND_TYPE_USER
);
2328 return virtio_has_feature(dev
->protocol_features
,
2329 VHOST_USER_PROTOCOL_F_LOG_SHMFD
);
2332 static int vhost_user_migration_done(struct vhost_dev
*dev
, char* mac_addr
)
2334 VhostUserMsg msg
= { };
2336 assert(dev
->vhost_ops
->backend_type
== VHOST_BACKEND_TYPE_USER
);
2338 /* If guest supports GUEST_ANNOUNCE do nothing */
2339 if (virtio_has_feature(dev
->acked_features
, VIRTIO_NET_F_GUEST_ANNOUNCE
)) {
2343 /* if backend supports VHOST_USER_PROTOCOL_F_RARP ask it to send the RARP */
2344 if (virtio_has_feature(dev
->protocol_features
,
2345 VHOST_USER_PROTOCOL_F_RARP
)) {
2346 msg
.hdr
.request
= VHOST_USER_SEND_RARP
;
2347 msg
.hdr
.flags
= VHOST_USER_VERSION
;
2348 memcpy((char *)&msg
.payload
.u64
, mac_addr
, 6);
2349 msg
.hdr
.size
= sizeof(msg
.payload
.u64
);
2351 return vhost_user_write(dev
, &msg
, NULL
, 0);
2356 static int vhost_user_net_set_mtu(struct vhost_dev
*dev
, uint16_t mtu
)
2359 bool reply_supported
= virtio_has_feature(dev
->protocol_features
,
2360 VHOST_USER_PROTOCOL_F_REPLY_ACK
);
2363 if (!(dev
->protocol_features
& (1ULL << VHOST_USER_PROTOCOL_F_NET_MTU
))) {
2367 msg
.hdr
.request
= VHOST_USER_NET_SET_MTU
;
2368 msg
.payload
.u64
= mtu
;
2369 msg
.hdr
.size
= sizeof(msg
.payload
.u64
);
2370 msg
.hdr
.flags
= VHOST_USER_VERSION
;
2371 if (reply_supported
) {
2372 msg
.hdr
.flags
|= VHOST_USER_NEED_REPLY_MASK
;
2375 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
2380 /* If reply_ack supported, backend has to ack specified MTU is valid */
2381 if (reply_supported
) {
2382 return process_message_reply(dev
, &msg
);
2388 static int vhost_user_send_device_iotlb_msg(struct vhost_dev
*dev
,
2389 struct vhost_iotlb_msg
*imsg
)
2392 VhostUserMsg msg
= {
2393 .hdr
.request
= VHOST_USER_IOTLB_MSG
,
2394 .hdr
.size
= sizeof(msg
.payload
.iotlb
),
2395 .hdr
.flags
= VHOST_USER_VERSION
| VHOST_USER_NEED_REPLY_MASK
,
2396 .payload
.iotlb
= *imsg
,
2399 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
2404 return process_message_reply(dev
, &msg
);
2408 static void vhost_user_set_iotlb_callback(struct vhost_dev
*dev
, int enabled
)
2410 /* No-op as the receive channel is not dedicated to IOTLB messages. */
2413 static int vhost_user_get_config(struct vhost_dev
*dev
, uint8_t *config
,
2414 uint32_t config_len
, Error
**errp
)
2417 VhostUserMsg msg
= {
2418 .hdr
.request
= VHOST_USER_GET_CONFIG
,
2419 .hdr
.flags
= VHOST_USER_VERSION
,
2420 .hdr
.size
= VHOST_USER_CONFIG_HDR_SIZE
+ config_len
,
2423 if (!virtio_has_feature(dev
->protocol_features
,
2424 VHOST_USER_PROTOCOL_F_CONFIG
)) {
2425 error_setg(errp
, "VHOST_USER_PROTOCOL_F_CONFIG not supported");
2429 assert(config_len
<= VHOST_USER_MAX_CONFIG_SIZE
);
2431 msg
.payload
.config
.offset
= 0;
2432 msg
.payload
.config
.size
= config_len
;
2433 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
2435 error_setg_errno(errp
, -ret
, "vhost_get_config failed");
2439 ret
= vhost_user_read(dev
, &msg
);
2441 error_setg_errno(errp
, -ret
, "vhost_get_config failed");
2445 if (msg
.hdr
.request
!= VHOST_USER_GET_CONFIG
) {
2447 "Received unexpected msg type. Expected %d received %d",
2448 VHOST_USER_GET_CONFIG
, msg
.hdr
.request
);
2452 if (msg
.hdr
.size
!= VHOST_USER_CONFIG_HDR_SIZE
+ config_len
) {
2453 error_setg(errp
, "Received bad msg size.");
2457 memcpy(config
, msg
.payload
.config
.region
, config_len
);
2462 static int vhost_user_set_config(struct vhost_dev
*dev
, const uint8_t *data
,
2463 uint32_t offset
, uint32_t size
, uint32_t flags
)
2467 bool reply_supported
= virtio_has_feature(dev
->protocol_features
,
2468 VHOST_USER_PROTOCOL_F_REPLY_ACK
);
2470 VhostUserMsg msg
= {
2471 .hdr
.request
= VHOST_USER_SET_CONFIG
,
2472 .hdr
.flags
= VHOST_USER_VERSION
,
2473 .hdr
.size
= VHOST_USER_CONFIG_HDR_SIZE
+ size
,
2476 if (!virtio_has_feature(dev
->protocol_features
,
2477 VHOST_USER_PROTOCOL_F_CONFIG
)) {
2481 if (reply_supported
) {
2482 msg
.hdr
.flags
|= VHOST_USER_NEED_REPLY_MASK
;
2485 if (size
> VHOST_USER_MAX_CONFIG_SIZE
) {
2489 msg
.payload
.config
.offset
= offset
,
2490 msg
.payload
.config
.size
= size
,
2491 msg
.payload
.config
.flags
= flags
,
2492 p
= msg
.payload
.config
.region
;
2493 memcpy(p
, data
, size
);
2495 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
2500 if (reply_supported
) {
2501 return process_message_reply(dev
, &msg
);
2507 static int vhost_user_crypto_create_session(struct vhost_dev
*dev
,
2509 uint64_t *session_id
)
2512 bool crypto_session
= virtio_has_feature(dev
->protocol_features
,
2513 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION
);
2514 CryptoDevBackendSessionInfo
*backend_info
= session_info
;
2515 VhostUserMsg msg
= {
2516 .hdr
.request
= VHOST_USER_CREATE_CRYPTO_SESSION
,
2517 .hdr
.flags
= VHOST_USER_VERSION
,
2518 .hdr
.size
= sizeof(msg
.payload
.session
),
2521 assert(dev
->vhost_ops
->backend_type
== VHOST_BACKEND_TYPE_USER
);
2523 if (!crypto_session
) {
2524 error_report("vhost-user trying to send unhandled ioctl");
2528 if (backend_info
->op_code
== VIRTIO_CRYPTO_AKCIPHER_CREATE_SESSION
) {
2529 CryptoDevBackendAsymSessionInfo
*sess
= &backend_info
->u
.asym_sess_info
;
2532 memcpy(&msg
.payload
.session
.u
.asym
.session_setup_data
, sess
,
2533 sizeof(CryptoDevBackendAsymSessionInfo
));
2535 keylen
= sizeof(msg
.payload
.session
.u
.asym
.key
);
2536 if (sess
->keylen
> keylen
) {
2537 error_report("Unsupported asymmetric key size");
2541 memcpy(&msg
.payload
.session
.u
.asym
.key
, sess
->key
,
2545 CryptoDevBackendSymSessionInfo
*sess
= &backend_info
->u
.sym_sess_info
;
2548 memcpy(&msg
.payload
.session
.u
.sym
.session_setup_data
, sess
,
2549 sizeof(CryptoDevBackendSymSessionInfo
));
2550 if (sess
->key_len
) {
2551 keylen
= sizeof(msg
.payload
.session
.u
.sym
.key
);
2552 if (sess
->key_len
> keylen
) {
2553 error_report("Unsupported cipher key size");
2557 memcpy(&msg
.payload
.session
.u
.sym
.key
, sess
->cipher_key
,
2561 if (sess
->auth_key_len
> 0) {
2562 keylen
= sizeof(msg
.payload
.session
.u
.sym
.auth_key
);
2563 if (sess
->auth_key_len
> keylen
) {
2564 error_report("Unsupported auth key size");
2568 memcpy(&msg
.payload
.session
.u
.sym
.auth_key
, sess
->auth_key
,
2569 sess
->auth_key_len
);
2573 msg
.payload
.session
.op_code
= backend_info
->op_code
;
2574 msg
.payload
.session
.session_id
= backend_info
->session_id
;
2575 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
2577 error_report("vhost_user_write() return %d, create session failed",
2582 ret
= vhost_user_read(dev
, &msg
);
2584 error_report("vhost_user_read() return %d, create session failed",
2589 if (msg
.hdr
.request
!= VHOST_USER_CREATE_CRYPTO_SESSION
) {
2590 error_report("Received unexpected msg type. Expected %d received %d",
2591 VHOST_USER_CREATE_CRYPTO_SESSION
, msg
.hdr
.request
);
2595 if (msg
.hdr
.size
!= sizeof(msg
.payload
.session
)) {
2596 error_report("Received bad msg size.");
2600 if (msg
.payload
.session
.session_id
< 0) {
2601 error_report("Bad session id: %" PRId64
"",
2602 msg
.payload
.session
.session_id
);
2605 *session_id
= msg
.payload
.session
.session_id
;
2611 vhost_user_crypto_close_session(struct vhost_dev
*dev
, uint64_t session_id
)
2614 bool crypto_session
= virtio_has_feature(dev
->protocol_features
,
2615 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION
);
2616 VhostUserMsg msg
= {
2617 .hdr
.request
= VHOST_USER_CLOSE_CRYPTO_SESSION
,
2618 .hdr
.flags
= VHOST_USER_VERSION
,
2619 .hdr
.size
= sizeof(msg
.payload
.u64
),
2621 msg
.payload
.u64
= session_id
;
2623 if (!crypto_session
) {
2624 error_report("vhost-user trying to send unhandled ioctl");
2628 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
2630 error_report("vhost_user_write() return %d, close session failed",
2638 static bool vhost_user_no_private_memslots(struct vhost_dev
*dev
)
2643 static int vhost_user_get_inflight_fd(struct vhost_dev
*dev
,
2644 uint16_t queue_size
,
2645 struct vhost_inflight
*inflight
)
2650 struct vhost_user
*u
= dev
->opaque
;
2651 CharBackend
*chr
= u
->user
->chr
;
2652 VhostUserMsg msg
= {
2653 .hdr
.request
= VHOST_USER_GET_INFLIGHT_FD
,
2654 .hdr
.flags
= VHOST_USER_VERSION
,
2655 .payload
.inflight
.num_queues
= dev
->nvqs
,
2656 .payload
.inflight
.queue_size
= queue_size
,
2657 .hdr
.size
= sizeof(msg
.payload
.inflight
),
2660 if (!virtio_has_feature(dev
->protocol_features
,
2661 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD
)) {
2665 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
2670 ret
= vhost_user_read(dev
, &msg
);
2675 if (msg
.hdr
.request
!= VHOST_USER_GET_INFLIGHT_FD
) {
2676 error_report("Received unexpected msg type. "
2677 "Expected %d received %d",
2678 VHOST_USER_GET_INFLIGHT_FD
, msg
.hdr
.request
);
2682 if (msg
.hdr
.size
!= sizeof(msg
.payload
.inflight
)) {
2683 error_report("Received bad msg size.");
2687 if (!msg
.payload
.inflight
.mmap_size
) {
2691 fd
= qemu_chr_fe_get_msgfd(chr
);
2693 error_report("Failed to get mem fd");
2697 addr
= mmap(0, msg
.payload
.inflight
.mmap_size
, PROT_READ
| PROT_WRITE
,
2698 MAP_SHARED
, fd
, msg
.payload
.inflight
.mmap_offset
);
2700 if (addr
== MAP_FAILED
) {
2701 error_report("Failed to mmap mem fd");
2706 inflight
->addr
= addr
;
2708 inflight
->size
= msg
.payload
.inflight
.mmap_size
;
2709 inflight
->offset
= msg
.payload
.inflight
.mmap_offset
;
2710 inflight
->queue_size
= queue_size
;
2715 static int vhost_user_set_inflight_fd(struct vhost_dev
*dev
,
2716 struct vhost_inflight
*inflight
)
2718 VhostUserMsg msg
= {
2719 .hdr
.request
= VHOST_USER_SET_INFLIGHT_FD
,
2720 .hdr
.flags
= VHOST_USER_VERSION
,
2721 .payload
.inflight
.mmap_size
= inflight
->size
,
2722 .payload
.inflight
.mmap_offset
= inflight
->offset
,
2723 .payload
.inflight
.num_queues
= dev
->nvqs
,
2724 .payload
.inflight
.queue_size
= inflight
->queue_size
,
2725 .hdr
.size
= sizeof(msg
.payload
.inflight
),
2728 if (!virtio_has_feature(dev
->protocol_features
,
2729 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD
)) {
2733 return vhost_user_write(dev
, &msg
, &inflight
->fd
, 1);
2736 static void vhost_user_state_destroy(gpointer data
)
2738 VhostUserHostNotifier
*n
= (VhostUserHostNotifier
*) data
;
2740 vhost_user_host_notifier_remove(n
, NULL
);
2741 object_unparent(OBJECT(&n
->mr
));
2743 * We can't free until vhost_user_host_notifier_remove has
2744 * done it's thing so schedule the free with RCU.
2750 bool vhost_user_init(VhostUserState
*user
, CharBackend
*chr
, Error
**errp
)
2753 error_setg(errp
, "Cannot initialize vhost-user state");
2757 user
->memory_slots
= 0;
2758 user
->notifiers
= g_ptr_array_new_full(VIRTIO_QUEUE_MAX
/ 4,
2759 &vhost_user_state_destroy
);
2763 void vhost_user_cleanup(VhostUserState
*user
)
2768 memory_region_transaction_begin();
2769 user
->notifiers
= (GPtrArray
*) g_ptr_array_free(user
->notifiers
, true);
2770 memory_region_transaction_commit();
2776 vu_async_close_fn cb
;
2779 struct vhost_dev
*vhost
;
2780 } VhostAsyncCallback
;
2782 static void vhost_user_async_close_bh(void *opaque
)
2784 VhostAsyncCallback
*data
= opaque
;
2786 data
->cb(data
->dev
);
2792 * We only schedule the work if the machine is running. If suspended
2793 * we want to keep all the in-flight data as is for migration
2796 void vhost_user_async_close(DeviceState
*d
,
2797 CharBackend
*chardev
, struct vhost_dev
*vhost
,
2798 vu_async_close_fn cb
)
2800 if (!runstate_check(RUN_STATE_SHUTDOWN
)) {
2802 * A close event may happen during a read/write, but vhost
2803 * code assumes the vhost_dev remains setup, so delay the
2806 AioContext
*ctx
= qemu_get_current_aio_context();
2807 VhostAsyncCallback
*data
= g_new0(VhostAsyncCallback
, 1);
2809 /* Save data for the callback */
2813 data
->vhost
= vhost
;
2815 /* Disable any further notifications on the chardev */
2816 qemu_chr_fe_set_handlers(chardev
,
2817 NULL
, NULL
, NULL
, NULL
, NULL
, NULL
,
2820 aio_bh_schedule_oneshot(ctx
, vhost_user_async_close_bh
, data
);
2823 * Move vhost device to the stopped state. The vhost-user device
2824 * will be clean up and disconnected in BH. This can be useful in
2825 * the vhost migration code. If disconnect was caught there is an
2826 * option for the general vhost code to get the dev state without
2827 * knowing its type (in this case vhost-user).
2829 * Note if the vhost device is fully cleared by the time we
2830 * execute the bottom half we won't continue with the cleanup.
2832 vhost
->started
= false;
2836 static int vhost_user_dev_start(struct vhost_dev
*dev
, bool started
)
2838 if (!virtio_has_feature(dev
->protocol_features
,
2839 VHOST_USER_PROTOCOL_F_STATUS
)) {
2843 /* Set device status only for last queue pair */
2844 if (dev
->vq_index
+ dev
->nvqs
!= dev
->vq_index_end
) {
2849 return vhost_user_add_status(dev
, VIRTIO_CONFIG_S_ACKNOWLEDGE
|
2850 VIRTIO_CONFIG_S_DRIVER
|
2851 VIRTIO_CONFIG_S_DRIVER_OK
);
2857 static void vhost_user_reset_status(struct vhost_dev
*dev
)
2859 /* Set device status only for last queue pair */
2860 if (dev
->vq_index
+ dev
->nvqs
!= dev
->vq_index_end
) {
2864 if (virtio_has_feature(dev
->protocol_features
,
2865 VHOST_USER_PROTOCOL_F_STATUS
)) {
2866 vhost_user_set_status(dev
, 0);
2870 static bool vhost_user_supports_device_state(struct vhost_dev
*dev
)
2872 return virtio_has_feature(dev
->protocol_features
,
2873 VHOST_USER_PROTOCOL_F_DEVICE_STATE
);
2876 static int vhost_user_set_device_state_fd(struct vhost_dev
*dev
,
2877 VhostDeviceStateDirection direction
,
2878 VhostDeviceStatePhase phase
,
2884 struct vhost_user
*vu
= dev
->opaque
;
2885 VhostUserMsg msg
= {
2887 .request
= VHOST_USER_SET_DEVICE_STATE_FD
,
2888 .flags
= VHOST_USER_VERSION
,
2889 .size
= sizeof(msg
.payload
.transfer_state
),
2891 .payload
.transfer_state
= {
2892 .direction
= direction
,
2899 if (!vhost_user_supports_device_state(dev
)) {
2901 error_setg(errp
, "Back-end does not support migration state transfer");
2905 ret
= vhost_user_write(dev
, &msg
, &fd
, 1);
2908 error_setg_errno(errp
, -ret
,
2909 "Failed to send SET_DEVICE_STATE_FD message");
2913 ret
= vhost_user_read(dev
, &msg
);
2915 error_setg_errno(errp
, -ret
,
2916 "Failed to receive SET_DEVICE_STATE_FD reply");
2920 if (msg
.hdr
.request
!= VHOST_USER_SET_DEVICE_STATE_FD
) {
2922 "Received unexpected message type, expected %d, received %d",
2923 VHOST_USER_SET_DEVICE_STATE_FD
, msg
.hdr
.request
);
2927 if (msg
.hdr
.size
!= sizeof(msg
.payload
.u64
)) {
2929 "Received bad message size, expected %zu, received %" PRIu32
,
2930 sizeof(msg
.payload
.u64
), msg
.hdr
.size
);
2934 if ((msg
.payload
.u64
& 0xff) != 0) {
2935 error_setg(errp
, "Back-end did not accept migration state transfer");
2939 if (!(msg
.payload
.u64
& VHOST_USER_VRING_NOFD_MASK
)) {
2940 *reply_fd
= qemu_chr_fe_get_msgfd(vu
->user
->chr
);
2941 if (*reply_fd
< 0) {
2943 "Failed to get back-end-provided transfer pipe FD");
2952 static int vhost_user_check_device_state(struct vhost_dev
*dev
, Error
**errp
)
2955 VhostUserMsg msg
= {
2957 .request
= VHOST_USER_CHECK_DEVICE_STATE
,
2958 .flags
= VHOST_USER_VERSION
,
2963 if (!vhost_user_supports_device_state(dev
)) {
2964 error_setg(errp
, "Back-end does not support migration state transfer");
2968 ret
= vhost_user_write(dev
, &msg
, NULL
, 0);
2970 error_setg_errno(errp
, -ret
,
2971 "Failed to send CHECK_DEVICE_STATE message");
2975 ret
= vhost_user_read(dev
, &msg
);
2977 error_setg_errno(errp
, -ret
,
2978 "Failed to receive CHECK_DEVICE_STATE reply");
2982 if (msg
.hdr
.request
!= VHOST_USER_CHECK_DEVICE_STATE
) {
2984 "Received unexpected message type, expected %d, received %d",
2985 VHOST_USER_CHECK_DEVICE_STATE
, msg
.hdr
.request
);
2989 if (msg
.hdr
.size
!= sizeof(msg
.payload
.u64
)) {
2991 "Received bad message size, expected %zu, received %" PRIu32
,
2992 sizeof(msg
.payload
.u64
), msg
.hdr
.size
);
2996 if (msg
.payload
.u64
!= 0) {
2997 error_setg(errp
, "Back-end failed to process its internal state");
3004 const VhostOps user_ops
= {
3005 .backend_type
= VHOST_BACKEND_TYPE_USER
,
3006 .vhost_backend_init
= vhost_user_backend_init
,
3007 .vhost_backend_cleanup
= vhost_user_backend_cleanup
,
3008 .vhost_backend_memslots_limit
= vhost_user_memslots_limit
,
3009 .vhost_backend_no_private_memslots
= vhost_user_no_private_memslots
,
3010 .vhost_set_log_base
= vhost_user_set_log_base
,
3011 .vhost_set_mem_table
= vhost_user_set_mem_table
,
3012 .vhost_set_vring_addr
= vhost_user_set_vring_addr
,
3013 .vhost_set_vring_endian
= vhost_user_set_vring_endian
,
3014 .vhost_set_vring_num
= vhost_user_set_vring_num
,
3015 .vhost_set_vring_base
= vhost_user_set_vring_base
,
3016 .vhost_get_vring_base
= vhost_user_get_vring_base
,
3017 .vhost_set_vring_kick
= vhost_user_set_vring_kick
,
3018 .vhost_set_vring_call
= vhost_user_set_vring_call
,
3019 .vhost_set_vring_err
= vhost_user_set_vring_err
,
3020 .vhost_set_features
= vhost_user_set_features
,
3021 .vhost_get_features
= vhost_user_get_features
,
3022 .vhost_set_owner
= vhost_user_set_owner
,
3023 .vhost_reset_device
= vhost_user_reset_device
,
3024 .vhost_get_vq_index
= vhost_user_get_vq_index
,
3025 .vhost_set_vring_enable
= vhost_user_set_vring_enable
,
3026 .vhost_requires_shm_log
= vhost_user_requires_shm_log
,
3027 .vhost_migration_done
= vhost_user_migration_done
,
3028 .vhost_net_set_mtu
= vhost_user_net_set_mtu
,
3029 .vhost_set_iotlb_callback
= vhost_user_set_iotlb_callback
,
3030 .vhost_send_device_iotlb_msg
= vhost_user_send_device_iotlb_msg
,
3031 .vhost_get_config
= vhost_user_get_config
,
3032 .vhost_set_config
= vhost_user_set_config
,
3033 .vhost_crypto_create_session
= vhost_user_crypto_create_session
,
3034 .vhost_crypto_close_session
= vhost_user_crypto_close_session
,
3035 .vhost_get_inflight_fd
= vhost_user_get_inflight_fd
,
3036 .vhost_set_inflight_fd
= vhost_user_set_inflight_fd
,
3037 .vhost_dev_start
= vhost_user_dev_start
,
3038 .vhost_reset_status
= vhost_user_reset_status
,
3039 .vhost_supports_device_state
= vhost_user_supports_device_state
,
3040 .vhost_set_device_state_fd
= vhost_user_set_device_state_fd
,
3041 .vhost_check_device_state
= vhost_user_check_device_state
,