4 * Copyright (c) 2013 Virtual Open Systems Sarl.
6 * This work is licensed under the terms of the GNU GPL, version 2 or later.
7 * See the COPYING file in the top-level directory.
11 #include "qemu/osdep.h"
12 #include "qapi/error.h"
13 #include "hw/virtio/vhost.h"
14 #include "hw/virtio/vhost-user.h"
15 #include "hw/virtio/vhost-backend.h"
16 #include "hw/virtio/virtio.h"
17 #include "hw/virtio/virtio-net.h"
18 #include "chardev/char-fe.h"
19 #include "sysemu/kvm.h"
20 #include "qemu/error-report.h"
21 #include "qemu/main-loop.h"
22 #include "qemu/sockets.h"
23 #include "sysemu/cryptodev.h"
24 #include "migration/migration.h"
25 #include "migration/postcopy-ram.h"
28 #include <sys/ioctl.h>
29 #include <sys/socket.h>
32 #include "standard-headers/linux/vhost_types.h"
35 #include <linux/userfaultfd.h>
38 #define VHOST_MEMORY_BASELINE_NREGIONS 8
39 #define VHOST_USER_F_PROTOCOL_FEATURES 30
40 #define VHOST_USER_SLAVE_MAX_FDS 8
43 * Set maximum number of RAM slots supported to
44 * the maximum number supported by the target
47 #if defined(TARGET_X86) || defined(TARGET_X86_64) || \
48 defined(TARGET_ARM) || defined(TARGET_ARM_64)
49 #include "hw/acpi/acpi.h"
50 #define VHOST_USER_MAX_RAM_SLOTS ACPI_MAX_RAM_SLOTS
52 #elif defined(TARGET_PPC) || defined(TARGET_PPC_64)
53 #include "hw/ppc/spapr.h"
54 #define VHOST_USER_MAX_RAM_SLOTS SPAPR_MAX_RAM_SLOTS
57 #define VHOST_USER_MAX_RAM_SLOTS 512
61 * Maximum size of virtio device config space
63 #define VHOST_USER_MAX_CONFIG_SIZE 256
65 enum VhostUserProtocolFeature
{
66 VHOST_USER_PROTOCOL_F_MQ
= 0,
67 VHOST_USER_PROTOCOL_F_LOG_SHMFD
= 1,
68 VHOST_USER_PROTOCOL_F_RARP
= 2,
69 VHOST_USER_PROTOCOL_F_REPLY_ACK
= 3,
70 VHOST_USER_PROTOCOL_F_NET_MTU
= 4,
71 VHOST_USER_PROTOCOL_F_SLAVE_REQ
= 5,
72 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN
= 6,
73 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION
= 7,
74 VHOST_USER_PROTOCOL_F_PAGEFAULT
= 8,
75 VHOST_USER_PROTOCOL_F_CONFIG
= 9,
76 VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD
= 10,
77 VHOST_USER_PROTOCOL_F_HOST_NOTIFIER
= 11,
78 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD
= 12,
79 VHOST_USER_PROTOCOL_F_RESET_DEVICE
= 13,
80 /* Feature 14 reserved for VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS. */
81 VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS
= 15,
82 VHOST_USER_PROTOCOL_F_MAX
85 #define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1)
87 typedef enum VhostUserRequest
{
89 VHOST_USER_GET_FEATURES
= 1,
90 VHOST_USER_SET_FEATURES
= 2,
91 VHOST_USER_SET_OWNER
= 3,
92 VHOST_USER_RESET_OWNER
= 4,
93 VHOST_USER_SET_MEM_TABLE
= 5,
94 VHOST_USER_SET_LOG_BASE
= 6,
95 VHOST_USER_SET_LOG_FD
= 7,
96 VHOST_USER_SET_VRING_NUM
= 8,
97 VHOST_USER_SET_VRING_ADDR
= 9,
98 VHOST_USER_SET_VRING_BASE
= 10,
99 VHOST_USER_GET_VRING_BASE
= 11,
100 VHOST_USER_SET_VRING_KICK
= 12,
101 VHOST_USER_SET_VRING_CALL
= 13,
102 VHOST_USER_SET_VRING_ERR
= 14,
103 VHOST_USER_GET_PROTOCOL_FEATURES
= 15,
104 VHOST_USER_SET_PROTOCOL_FEATURES
= 16,
105 VHOST_USER_GET_QUEUE_NUM
= 17,
106 VHOST_USER_SET_VRING_ENABLE
= 18,
107 VHOST_USER_SEND_RARP
= 19,
108 VHOST_USER_NET_SET_MTU
= 20,
109 VHOST_USER_SET_SLAVE_REQ_FD
= 21,
110 VHOST_USER_IOTLB_MSG
= 22,
111 VHOST_USER_SET_VRING_ENDIAN
= 23,
112 VHOST_USER_GET_CONFIG
= 24,
113 VHOST_USER_SET_CONFIG
= 25,
114 VHOST_USER_CREATE_CRYPTO_SESSION
= 26,
115 VHOST_USER_CLOSE_CRYPTO_SESSION
= 27,
116 VHOST_USER_POSTCOPY_ADVISE
= 28,
117 VHOST_USER_POSTCOPY_LISTEN
= 29,
118 VHOST_USER_POSTCOPY_END
= 30,
119 VHOST_USER_GET_INFLIGHT_FD
= 31,
120 VHOST_USER_SET_INFLIGHT_FD
= 32,
121 VHOST_USER_GPU_SET_SOCKET
= 33,
122 VHOST_USER_RESET_DEVICE
= 34,
123 /* Message number 35 reserved for VHOST_USER_VRING_KICK. */
124 VHOST_USER_GET_MAX_MEM_SLOTS
= 36,
125 VHOST_USER_ADD_MEM_REG
= 37,
126 VHOST_USER_REM_MEM_REG
= 38,
130 typedef enum VhostUserSlaveRequest
{
131 VHOST_USER_SLAVE_NONE
= 0,
132 VHOST_USER_SLAVE_IOTLB_MSG
= 1,
133 VHOST_USER_SLAVE_CONFIG_CHANGE_MSG
= 2,
134 VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG
= 3,
136 } VhostUserSlaveRequest
;
138 typedef struct VhostUserMemoryRegion
{
139 uint64_t guest_phys_addr
;
140 uint64_t memory_size
;
141 uint64_t userspace_addr
;
142 uint64_t mmap_offset
;
143 } VhostUserMemoryRegion
;
145 typedef struct VhostUserMemory
{
148 VhostUserMemoryRegion regions
[VHOST_MEMORY_BASELINE_NREGIONS
];
151 typedef struct VhostUserMemRegMsg
{
153 VhostUserMemoryRegion region
;
154 } VhostUserMemRegMsg
;
156 typedef struct VhostUserLog
{
158 uint64_t mmap_offset
;
161 typedef struct VhostUserConfig
{
165 uint8_t region
[VHOST_USER_MAX_CONFIG_SIZE
];
168 #define VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN 512
169 #define VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN 64
171 typedef struct VhostUserCryptoSession
{
172 /* session id for success, -1 on errors */
174 CryptoDevBackendSymSessionInfo session_setup_data
;
175 uint8_t key
[VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN
];
176 uint8_t auth_key
[VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN
];
177 } VhostUserCryptoSession
;
179 static VhostUserConfig c
__attribute__ ((unused
));
180 #define VHOST_USER_CONFIG_HDR_SIZE (sizeof(c.offset) \
184 typedef struct VhostUserVringArea
{
188 } VhostUserVringArea
;
190 typedef struct VhostUserInflight
{
192 uint64_t mmap_offset
;
198 VhostUserRequest request
;
200 #define VHOST_USER_VERSION_MASK (0x3)
201 #define VHOST_USER_REPLY_MASK (0x1<<2)
202 #define VHOST_USER_NEED_REPLY_MASK (0x1 << 3)
204 uint32_t size
; /* the following payload size */
205 } QEMU_PACKED VhostUserHeader
;
208 #define VHOST_USER_VRING_IDX_MASK (0xff)
209 #define VHOST_USER_VRING_NOFD_MASK (0x1<<8)
211 struct vhost_vring_state state
;
212 struct vhost_vring_addr addr
;
213 VhostUserMemory memory
;
214 VhostUserMemRegMsg mem_reg
;
216 struct vhost_iotlb_msg iotlb
;
217 VhostUserConfig config
;
218 VhostUserCryptoSession session
;
219 VhostUserVringArea area
;
220 VhostUserInflight inflight
;
223 typedef struct VhostUserMsg
{
225 VhostUserPayload payload
;
226 } QEMU_PACKED VhostUserMsg
;
228 static VhostUserMsg m
__attribute__ ((unused
));
229 #define VHOST_USER_HDR_SIZE (sizeof(VhostUserHeader))
231 #define VHOST_USER_PAYLOAD_SIZE (sizeof(VhostUserPayload))
233 /* The version of the protocol we support */
234 #define VHOST_USER_VERSION (0x1)
237 struct vhost_dev
*dev
;
238 /* Shared between vhost devs of the same virtio device */
239 VhostUserState
*user
;
241 NotifierWithReturn postcopy_notifier
;
242 struct PostCopyFD postcopy_fd
;
243 uint64_t postcopy_client_bases
[VHOST_USER_MAX_RAM_SLOTS
];
244 /* Length of the region_rb and region_rb_offset arrays */
245 size_t region_rb_len
;
246 /* RAMBlock associated with a given region */
247 RAMBlock
**region_rb
;
248 /* The offset from the start of the RAMBlock to the start of the
251 ram_addr_t
*region_rb_offset
;
253 /* True once we've entered postcopy_listen */
254 bool postcopy_listen
;
256 /* Our current regions */
257 int num_shadow_regions
;
258 struct vhost_memory_region shadow_regions
[VHOST_USER_MAX_RAM_SLOTS
];
261 struct scrub_regions
{
262 struct vhost_memory_region
*region
;
267 static bool ioeventfd_enabled(void)
269 return !kvm_enabled() || kvm_eventfds_enabled();
272 static int vhost_user_read_header(struct vhost_dev
*dev
, VhostUserMsg
*msg
)
274 struct vhost_user
*u
= dev
->opaque
;
275 CharBackend
*chr
= u
->user
->chr
;
276 uint8_t *p
= (uint8_t *) msg
;
277 int r
, size
= VHOST_USER_HDR_SIZE
;
279 r
= qemu_chr_fe_read_all(chr
, p
, size
);
281 error_report("Failed to read msg header. Read %d instead of %d."
282 " Original request %d.", r
, size
, msg
->hdr
.request
);
286 /* validate received flags */
287 if (msg
->hdr
.flags
!= (VHOST_USER_REPLY_MASK
| VHOST_USER_VERSION
)) {
288 error_report("Failed to read msg header."
289 " Flags 0x%x instead of 0x%x.", msg
->hdr
.flags
,
290 VHOST_USER_REPLY_MASK
| VHOST_USER_VERSION
);
297 static int vhost_user_read(struct vhost_dev
*dev
, VhostUserMsg
*msg
)
299 struct vhost_user
*u
= dev
->opaque
;
300 CharBackend
*chr
= u
->user
->chr
;
301 uint8_t *p
= (uint8_t *) msg
;
304 if (vhost_user_read_header(dev
, msg
) < 0) {
308 /* validate message size is sane */
309 if (msg
->hdr
.size
> VHOST_USER_PAYLOAD_SIZE
) {
310 error_report("Failed to read msg header."
311 " Size %d exceeds the maximum %zu.", msg
->hdr
.size
,
312 VHOST_USER_PAYLOAD_SIZE
);
317 p
+= VHOST_USER_HDR_SIZE
;
318 size
= msg
->hdr
.size
;
319 r
= qemu_chr_fe_read_all(chr
, p
, size
);
321 error_report("Failed to read msg payload."
322 " Read %d instead of %d.", r
, msg
->hdr
.size
);
330 static int process_message_reply(struct vhost_dev
*dev
,
331 const VhostUserMsg
*msg
)
333 VhostUserMsg msg_reply
;
335 if ((msg
->hdr
.flags
& VHOST_USER_NEED_REPLY_MASK
) == 0) {
339 if (vhost_user_read(dev
, &msg_reply
) < 0) {
343 if (msg_reply
.hdr
.request
!= msg
->hdr
.request
) {
344 error_report("Received unexpected msg type."
345 "Expected %d received %d",
346 msg
->hdr
.request
, msg_reply
.hdr
.request
);
350 return msg_reply
.payload
.u64
? -1 : 0;
353 static bool vhost_user_one_time_request(VhostUserRequest request
)
356 case VHOST_USER_SET_OWNER
:
357 case VHOST_USER_RESET_OWNER
:
358 case VHOST_USER_SET_MEM_TABLE
:
359 case VHOST_USER_GET_QUEUE_NUM
:
360 case VHOST_USER_NET_SET_MTU
:
367 /* most non-init callers ignore the error */
368 static int vhost_user_write(struct vhost_dev
*dev
, VhostUserMsg
*msg
,
369 int *fds
, int fd_num
)
371 struct vhost_user
*u
= dev
->opaque
;
372 CharBackend
*chr
= u
->user
->chr
;
373 int ret
, size
= VHOST_USER_HDR_SIZE
+ msg
->hdr
.size
;
376 * For non-vring specific requests, like VHOST_USER_SET_MEM_TABLE,
377 * we just need send it once in the first time. For later such
378 * request, we just ignore it.
380 if (vhost_user_one_time_request(msg
->hdr
.request
) && dev
->vq_index
!= 0) {
381 msg
->hdr
.flags
&= ~VHOST_USER_NEED_REPLY_MASK
;
385 if (qemu_chr_fe_set_msgfds(chr
, fds
, fd_num
) < 0) {
386 error_report("Failed to set msg fds.");
390 ret
= qemu_chr_fe_write_all(chr
, (const uint8_t *) msg
, size
);
392 error_report("Failed to write msg."
393 " Wrote %d instead of %d.", ret
, size
);
400 int vhost_user_gpu_set_socket(struct vhost_dev
*dev
, int fd
)
403 .hdr
.request
= VHOST_USER_GPU_SET_SOCKET
,
404 .hdr
.flags
= VHOST_USER_VERSION
,
407 return vhost_user_write(dev
, &msg
, &fd
, 1);
410 static int vhost_user_set_log_base(struct vhost_dev
*dev
, uint64_t base
,
411 struct vhost_log
*log
)
413 int fds
[VHOST_USER_MAX_RAM_SLOTS
];
415 bool shmfd
= virtio_has_feature(dev
->protocol_features
,
416 VHOST_USER_PROTOCOL_F_LOG_SHMFD
);
418 .hdr
.request
= VHOST_USER_SET_LOG_BASE
,
419 .hdr
.flags
= VHOST_USER_VERSION
,
420 .payload
.log
.mmap_size
= log
->size
* sizeof(*(log
->log
)),
421 .payload
.log
.mmap_offset
= 0,
422 .hdr
.size
= sizeof(msg
.payload
.log
),
425 if (shmfd
&& log
->fd
!= -1) {
426 fds
[fd_num
++] = log
->fd
;
429 if (vhost_user_write(dev
, &msg
, fds
, fd_num
) < 0) {
435 if (vhost_user_read(dev
, &msg
) < 0) {
439 if (msg
.hdr
.request
!= VHOST_USER_SET_LOG_BASE
) {
440 error_report("Received unexpected msg type. "
441 "Expected %d received %d",
442 VHOST_USER_SET_LOG_BASE
, msg
.hdr
.request
);
450 static MemoryRegion
*vhost_user_get_mr_data(uint64_t addr
, ram_addr_t
*offset
,
455 assert((uintptr_t)addr
== addr
);
456 mr
= memory_region_from_host((void *)(uintptr_t)addr
, offset
);
457 *fd
= memory_region_get_fd(mr
);
462 static void vhost_user_fill_msg_region(VhostUserMemoryRegion
*dst
,
463 struct vhost_memory_region
*src
,
464 uint64_t mmap_offset
)
466 assert(src
!= NULL
&& dst
!= NULL
);
467 dst
->userspace_addr
= src
->userspace_addr
;
468 dst
->memory_size
= src
->memory_size
;
469 dst
->guest_phys_addr
= src
->guest_phys_addr
;
470 dst
->mmap_offset
= mmap_offset
;
473 static int vhost_user_fill_set_mem_table_msg(struct vhost_user
*u
,
474 struct vhost_dev
*dev
,
476 int *fds
, size_t *fd_num
,
477 bool track_ramblocks
)
482 struct vhost_memory_region
*reg
;
483 VhostUserMemoryRegion region_buffer
;
485 msg
->hdr
.request
= VHOST_USER_SET_MEM_TABLE
;
487 for (i
= 0; i
< dev
->mem
->nregions
; ++i
) {
488 reg
= dev
->mem
->regions
+ i
;
490 mr
= vhost_user_get_mr_data(reg
->userspace_addr
, &offset
, &fd
);
492 if (track_ramblocks
) {
493 assert(*fd_num
< VHOST_MEMORY_BASELINE_NREGIONS
);
494 trace_vhost_user_set_mem_table_withfd(*fd_num
, mr
->name
,
496 reg
->guest_phys_addr
,
499 u
->region_rb_offset
[i
] = offset
;
500 u
->region_rb
[i
] = mr
->ram_block
;
501 } else if (*fd_num
== VHOST_MEMORY_BASELINE_NREGIONS
) {
502 error_report("Failed preparing vhost-user memory table msg");
505 vhost_user_fill_msg_region(®ion_buffer
, reg
, offset
);
506 msg
->payload
.memory
.regions
[*fd_num
] = region_buffer
;
507 fds
[(*fd_num
)++] = fd
;
508 } else if (track_ramblocks
) {
509 u
->region_rb_offset
[i
] = 0;
510 u
->region_rb
[i
] = NULL
;
514 msg
->payload
.memory
.nregions
= *fd_num
;
517 error_report("Failed initializing vhost-user memory map, "
518 "consider using -object memory-backend-file share=on");
522 msg
->hdr
.size
= sizeof(msg
->payload
.memory
.nregions
);
523 msg
->hdr
.size
+= sizeof(msg
->payload
.memory
.padding
);
524 msg
->hdr
.size
+= *fd_num
* sizeof(VhostUserMemoryRegion
);
529 static inline bool reg_equal(struct vhost_memory_region
*shadow_reg
,
530 struct vhost_memory_region
*vdev_reg
)
532 return shadow_reg
->guest_phys_addr
== vdev_reg
->guest_phys_addr
&&
533 shadow_reg
->userspace_addr
== vdev_reg
->userspace_addr
&&
534 shadow_reg
->memory_size
== vdev_reg
->memory_size
;
537 static void scrub_shadow_regions(struct vhost_dev
*dev
,
538 struct scrub_regions
*add_reg
,
540 struct scrub_regions
*rem_reg
,
541 int *nr_rem_reg
, uint64_t *shadow_pcb
,
542 bool track_ramblocks
)
544 struct vhost_user
*u
= dev
->opaque
;
545 bool found
[VHOST_USER_MAX_RAM_SLOTS
] = {};
546 struct vhost_memory_region
*reg
, *shadow_reg
;
547 int i
, j
, fd
, add_idx
= 0, rm_idx
= 0, fd_num
= 0;
553 * Find memory regions present in our shadow state which are not in
554 * the device's current memory state.
556 * Mark regions in both the shadow and device state as "found".
558 for (i
= 0; i
< u
->num_shadow_regions
; i
++) {
559 shadow_reg
= &u
->shadow_regions
[i
];
562 for (j
= 0; j
< dev
->mem
->nregions
; j
++) {
563 reg
= &dev
->mem
->regions
[j
];
565 mr
= vhost_user_get_mr_data(reg
->userspace_addr
, &offset
, &fd
);
567 if (reg_equal(shadow_reg
, reg
)) {
570 if (track_ramblocks
) {
572 * Reset postcopy client bases, region_rb, and
573 * region_rb_offset in case regions are removed.
576 u
->region_rb_offset
[j
] = offset
;
577 u
->region_rb
[j
] = mr
->ram_block
;
578 shadow_pcb
[j
] = u
->postcopy_client_bases
[i
];
580 u
->region_rb_offset
[j
] = 0;
581 u
->region_rb
[j
] = NULL
;
589 * If the region was not found in the current device memory state
590 * create an entry for it in the removed list.
593 rem_reg
[rm_idx
].region
= shadow_reg
;
594 rem_reg
[rm_idx
++].reg_idx
= i
;
599 * For regions not marked "found", create entries in the added list.
601 * Note their indexes in the device memory state and the indexes of their
604 for (i
= 0; i
< dev
->mem
->nregions
; i
++) {
605 reg
= &dev
->mem
->regions
[i
];
606 vhost_user_get_mr_data(reg
->userspace_addr
, &offset
, &fd
);
612 * If the region was in both the shadow and device state we don't
613 * need to send a VHOST_USER_ADD_MEM_REG message for it.
619 add_reg
[add_idx
].region
= reg
;
620 add_reg
[add_idx
].reg_idx
= i
;
621 add_reg
[add_idx
++].fd_idx
= fd_num
;
623 *nr_rem_reg
= rm_idx
;
624 *nr_add_reg
= add_idx
;
629 static int send_remove_regions(struct vhost_dev
*dev
,
630 struct scrub_regions
*remove_reg
,
631 int nr_rem_reg
, VhostUserMsg
*msg
,
632 bool reply_supported
)
634 struct vhost_user
*u
= dev
->opaque
;
635 struct vhost_memory_region
*shadow_reg
;
636 int i
, fd
, shadow_reg_idx
, ret
;
638 VhostUserMemoryRegion region_buffer
;
641 * The regions in remove_reg appear in the same order they do in the
642 * shadow table. Therefore we can minimize memory copies by iterating
643 * through remove_reg backwards.
645 for (i
= nr_rem_reg
- 1; i
>= 0; i
--) {
646 shadow_reg
= remove_reg
[i
].region
;
647 shadow_reg_idx
= remove_reg
[i
].reg_idx
;
649 vhost_user_get_mr_data(shadow_reg
->userspace_addr
, &offset
, &fd
);
652 msg
->hdr
.request
= VHOST_USER_REM_MEM_REG
;
653 vhost_user_fill_msg_region(®ion_buffer
, shadow_reg
, 0);
654 msg
->payload
.mem_reg
.region
= region_buffer
;
656 if (vhost_user_write(dev
, msg
, &fd
, 1) < 0) {
660 if (reply_supported
) {
661 ret
= process_message_reply(dev
, msg
);
669 * At this point we know the backend has unmapped the region. It is now
670 * safe to remove it from the shadow table.
672 memmove(&u
->shadow_regions
[shadow_reg_idx
],
673 &u
->shadow_regions
[shadow_reg_idx
+ 1],
674 sizeof(struct vhost_memory_region
) *
675 (u
->num_shadow_regions
- shadow_reg_idx
- 1));
676 u
->num_shadow_regions
--;
682 static int send_add_regions(struct vhost_dev
*dev
,
683 struct scrub_regions
*add_reg
, int nr_add_reg
,
684 VhostUserMsg
*msg
, uint64_t *shadow_pcb
,
685 bool reply_supported
, bool track_ramblocks
)
687 struct vhost_user
*u
= dev
->opaque
;
688 int i
, fd
, ret
, reg_idx
, reg_fd_idx
;
689 struct vhost_memory_region
*reg
;
692 VhostUserMsg msg_reply
;
693 VhostUserMemoryRegion region_buffer
;
695 for (i
= 0; i
< nr_add_reg
; i
++) {
696 reg
= add_reg
[i
].region
;
697 reg_idx
= add_reg
[i
].reg_idx
;
698 reg_fd_idx
= add_reg
[i
].fd_idx
;
700 mr
= vhost_user_get_mr_data(reg
->userspace_addr
, &offset
, &fd
);
703 if (track_ramblocks
) {
704 trace_vhost_user_set_mem_table_withfd(reg_fd_idx
, mr
->name
,
706 reg
->guest_phys_addr
,
709 u
->region_rb_offset
[reg_idx
] = offset
;
710 u
->region_rb
[reg_idx
] = mr
->ram_block
;
712 msg
->hdr
.request
= VHOST_USER_ADD_MEM_REG
;
713 vhost_user_fill_msg_region(®ion_buffer
, reg
, offset
);
714 msg
->payload
.mem_reg
.region
= region_buffer
;
716 if (vhost_user_write(dev
, msg
, &fd
, 1) < 0) {
720 if (track_ramblocks
) {
723 if (vhost_user_read(dev
, &msg_reply
) < 0) {
727 reply_gpa
= msg_reply
.payload
.mem_reg
.region
.guest_phys_addr
;
729 if (msg_reply
.hdr
.request
!= VHOST_USER_ADD_MEM_REG
) {
730 error_report("%s: Received unexpected msg type."
731 "Expected %d received %d", __func__
,
732 VHOST_USER_ADD_MEM_REG
,
733 msg_reply
.hdr
.request
);
738 * We're using the same structure, just reusing one of the
739 * fields, so it should be the same size.
741 if (msg_reply
.hdr
.size
!= msg
->hdr
.size
) {
742 error_report("%s: Unexpected size for postcopy reply "
743 "%d vs %d", __func__
, msg_reply
.hdr
.size
,
748 /* Get the postcopy client base from the backend's reply. */
749 if (reply_gpa
== dev
->mem
->regions
[reg_idx
].guest_phys_addr
) {
750 shadow_pcb
[reg_idx
] =
751 msg_reply
.payload
.mem_reg
.region
.userspace_addr
;
752 trace_vhost_user_set_mem_table_postcopy(
753 msg_reply
.payload
.mem_reg
.region
.userspace_addr
,
754 msg
->payload
.mem_reg
.region
.userspace_addr
,
755 reg_fd_idx
, reg_idx
);
757 error_report("%s: invalid postcopy reply for region. "
758 "Got guest physical address %" PRIX64
", expected "
759 "%" PRIX64
, __func__
, reply_gpa
,
760 dev
->mem
->regions
[reg_idx
].guest_phys_addr
);
763 } else if (reply_supported
) {
764 ret
= process_message_reply(dev
, msg
);
769 } else if (track_ramblocks
) {
770 u
->region_rb_offset
[reg_idx
] = 0;
771 u
->region_rb
[reg_idx
] = NULL
;
775 * At this point, we know the backend has mapped in the new
776 * region, if the region has a valid file descriptor.
778 * The region should now be added to the shadow table.
780 u
->shadow_regions
[u
->num_shadow_regions
].guest_phys_addr
=
781 reg
->guest_phys_addr
;
782 u
->shadow_regions
[u
->num_shadow_regions
].userspace_addr
=
784 u
->shadow_regions
[u
->num_shadow_regions
].memory_size
=
786 u
->num_shadow_regions
++;
792 static int vhost_user_add_remove_regions(struct vhost_dev
*dev
,
794 bool reply_supported
,
795 bool track_ramblocks
)
797 struct vhost_user
*u
= dev
->opaque
;
798 struct scrub_regions add_reg
[VHOST_USER_MAX_RAM_SLOTS
];
799 struct scrub_regions rem_reg
[VHOST_USER_MAX_RAM_SLOTS
];
800 uint64_t shadow_pcb
[VHOST_USER_MAX_RAM_SLOTS
] = {};
801 int nr_add_reg
, nr_rem_reg
;
803 msg
->hdr
.size
= sizeof(msg
->payload
.mem_reg
);
805 /* Find the regions which need to be removed or added. */
806 scrub_shadow_regions(dev
, add_reg
, &nr_add_reg
, rem_reg
, &nr_rem_reg
,
807 shadow_pcb
, track_ramblocks
);
809 if (nr_rem_reg
&& send_remove_regions(dev
, rem_reg
, nr_rem_reg
, msg
,
810 reply_supported
) < 0)
815 if (nr_add_reg
&& send_add_regions(dev
, add_reg
, nr_add_reg
, msg
,
816 shadow_pcb
, reply_supported
, track_ramblocks
) < 0)
821 if (track_ramblocks
) {
822 memcpy(u
->postcopy_client_bases
, shadow_pcb
,
823 sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS
);
825 * Now we've registered this with the postcopy code, we ack to the
826 * client, because now we're in the position to be able to deal with
827 * any faults it generates.
829 /* TODO: Use this for failure cases as well with a bad value. */
830 msg
->hdr
.size
= sizeof(msg
->payload
.u64
);
831 msg
->payload
.u64
= 0; /* OK */
833 if (vhost_user_write(dev
, msg
, NULL
, 0) < 0) {
841 if (track_ramblocks
) {
842 memcpy(u
->postcopy_client_bases
, shadow_pcb
,
843 sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS
);
849 static int vhost_user_set_mem_table_postcopy(struct vhost_dev
*dev
,
850 struct vhost_memory
*mem
,
851 bool reply_supported
,
852 bool config_mem_slots
)
854 struct vhost_user
*u
= dev
->opaque
;
855 int fds
[VHOST_MEMORY_BASELINE_NREGIONS
];
857 VhostUserMsg msg_reply
;
861 .hdr
.flags
= VHOST_USER_VERSION
,
864 if (u
->region_rb_len
< dev
->mem
->nregions
) {
865 u
->region_rb
= g_renew(RAMBlock
*, u
->region_rb
, dev
->mem
->nregions
);
866 u
->region_rb_offset
= g_renew(ram_addr_t
, u
->region_rb_offset
,
868 memset(&(u
->region_rb
[u
->region_rb_len
]), '\0',
869 sizeof(RAMBlock
*) * (dev
->mem
->nregions
- u
->region_rb_len
));
870 memset(&(u
->region_rb_offset
[u
->region_rb_len
]), '\0',
871 sizeof(ram_addr_t
) * (dev
->mem
->nregions
- u
->region_rb_len
));
872 u
->region_rb_len
= dev
->mem
->nregions
;
875 if (config_mem_slots
) {
876 if (vhost_user_add_remove_regions(dev
, &msg
, reply_supported
,
881 if (vhost_user_fill_set_mem_table_msg(u
, dev
, &msg
, fds
, &fd_num
,
886 if (vhost_user_write(dev
, &msg
, fds
, fd_num
) < 0) {
890 if (vhost_user_read(dev
, &msg_reply
) < 0) {
894 if (msg_reply
.hdr
.request
!= VHOST_USER_SET_MEM_TABLE
) {
895 error_report("%s: Received unexpected msg type."
896 "Expected %d received %d", __func__
,
897 VHOST_USER_SET_MEM_TABLE
, msg_reply
.hdr
.request
);
902 * We're using the same structure, just reusing one of the
903 * fields, so it should be the same size.
905 if (msg_reply
.hdr
.size
!= msg
.hdr
.size
) {
906 error_report("%s: Unexpected size for postcopy reply "
907 "%d vs %d", __func__
, msg_reply
.hdr
.size
,
912 memset(u
->postcopy_client_bases
, 0,
913 sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS
);
916 * They're in the same order as the regions that were sent
917 * but some of the regions were skipped (above) if they
920 for (msg_i
= 0, region_i
= 0;
921 region_i
< dev
->mem
->nregions
;
923 if (msg_i
< fd_num
&&
924 msg_reply
.payload
.memory
.regions
[msg_i
].guest_phys_addr
==
925 dev
->mem
->regions
[region_i
].guest_phys_addr
) {
926 u
->postcopy_client_bases
[region_i
] =
927 msg_reply
.payload
.memory
.regions
[msg_i
].userspace_addr
;
928 trace_vhost_user_set_mem_table_postcopy(
929 msg_reply
.payload
.memory
.regions
[msg_i
].userspace_addr
,
930 msg
.payload
.memory
.regions
[msg_i
].userspace_addr
,
935 if (msg_i
!= fd_num
) {
936 error_report("%s: postcopy reply not fully consumed "
938 __func__
, msg_i
, fd_num
);
943 * Now we've registered this with the postcopy code, we ack to the
944 * client, because now we're in the position to be able to deal
945 * with any faults it generates.
947 /* TODO: Use this for failure cases as well with a bad value. */
948 msg
.hdr
.size
= sizeof(msg
.payload
.u64
);
949 msg
.payload
.u64
= 0; /* OK */
950 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
958 static int vhost_user_set_mem_table(struct vhost_dev
*dev
,
959 struct vhost_memory
*mem
)
961 struct vhost_user
*u
= dev
->opaque
;
962 int fds
[VHOST_MEMORY_BASELINE_NREGIONS
];
964 bool do_postcopy
= u
->postcopy_listen
&& u
->postcopy_fd
.handler
;
965 bool reply_supported
= virtio_has_feature(dev
->protocol_features
,
966 VHOST_USER_PROTOCOL_F_REPLY_ACK
);
967 bool config_mem_slots
=
968 virtio_has_feature(dev
->protocol_features
,
969 VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS
);
973 * Postcopy has enough differences that it's best done in it's own
976 return vhost_user_set_mem_table_postcopy(dev
, mem
, reply_supported
,
981 .hdr
.flags
= VHOST_USER_VERSION
,
984 if (reply_supported
) {
985 msg
.hdr
.flags
|= VHOST_USER_NEED_REPLY_MASK
;
988 if (config_mem_slots
) {
989 if (vhost_user_add_remove_regions(dev
, &msg
, reply_supported
,
994 if (vhost_user_fill_set_mem_table_msg(u
, dev
, &msg
, fds
, &fd_num
,
998 if (vhost_user_write(dev
, &msg
, fds
, fd_num
) < 0) {
1002 if (reply_supported
) {
1003 return process_message_reply(dev
, &msg
);
1010 static int vhost_user_set_vring_addr(struct vhost_dev
*dev
,
1011 struct vhost_vring_addr
*addr
)
1013 VhostUserMsg msg
= {
1014 .hdr
.request
= VHOST_USER_SET_VRING_ADDR
,
1015 .hdr
.flags
= VHOST_USER_VERSION
,
1016 .payload
.addr
= *addr
,
1017 .hdr
.size
= sizeof(msg
.payload
.addr
),
1020 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
1027 static int vhost_user_set_vring_endian(struct vhost_dev
*dev
,
1028 struct vhost_vring_state
*ring
)
1030 bool cross_endian
= virtio_has_feature(dev
->protocol_features
,
1031 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN
);
1032 VhostUserMsg msg
= {
1033 .hdr
.request
= VHOST_USER_SET_VRING_ENDIAN
,
1034 .hdr
.flags
= VHOST_USER_VERSION
,
1035 .payload
.state
= *ring
,
1036 .hdr
.size
= sizeof(msg
.payload
.state
),
1039 if (!cross_endian
) {
1040 error_report("vhost-user trying to send unhandled ioctl");
1044 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
1051 static int vhost_set_vring(struct vhost_dev
*dev
,
1052 unsigned long int request
,
1053 struct vhost_vring_state
*ring
)
1055 VhostUserMsg msg
= {
1056 .hdr
.request
= request
,
1057 .hdr
.flags
= VHOST_USER_VERSION
,
1058 .payload
.state
= *ring
,
1059 .hdr
.size
= sizeof(msg
.payload
.state
),
1062 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
1069 static int vhost_user_set_vring_num(struct vhost_dev
*dev
,
1070 struct vhost_vring_state
*ring
)
1072 return vhost_set_vring(dev
, VHOST_USER_SET_VRING_NUM
, ring
);
1075 static void vhost_user_host_notifier_restore(struct vhost_dev
*dev
,
1078 struct vhost_user
*u
= dev
->opaque
;
1079 VhostUserHostNotifier
*n
= &u
->user
->notifier
[queue_idx
];
1080 VirtIODevice
*vdev
= dev
->vdev
;
1082 if (n
->addr
&& !n
->set
) {
1083 virtio_queue_set_host_notifier_mr(vdev
, queue_idx
, &n
->mr
, true);
1088 static void vhost_user_host_notifier_remove(struct vhost_dev
*dev
,
1091 struct vhost_user
*u
= dev
->opaque
;
1092 VhostUserHostNotifier
*n
= &u
->user
->notifier
[queue_idx
];
1093 VirtIODevice
*vdev
= dev
->vdev
;
1095 if (n
->addr
&& n
->set
) {
1096 virtio_queue_set_host_notifier_mr(vdev
, queue_idx
, &n
->mr
, false);
1101 static int vhost_user_set_vring_base(struct vhost_dev
*dev
,
1102 struct vhost_vring_state
*ring
)
1104 vhost_user_host_notifier_restore(dev
, ring
->index
);
1106 return vhost_set_vring(dev
, VHOST_USER_SET_VRING_BASE
, ring
);
1109 static int vhost_user_set_vring_enable(struct vhost_dev
*dev
, int enable
)
1113 if (!virtio_has_feature(dev
->features
, VHOST_USER_F_PROTOCOL_FEATURES
)) {
1117 for (i
= 0; i
< dev
->nvqs
; ++i
) {
1118 struct vhost_vring_state state
= {
1119 .index
= dev
->vq_index
+ i
,
1123 vhost_set_vring(dev
, VHOST_USER_SET_VRING_ENABLE
, &state
);
1129 static int vhost_user_get_vring_base(struct vhost_dev
*dev
,
1130 struct vhost_vring_state
*ring
)
1132 VhostUserMsg msg
= {
1133 .hdr
.request
= VHOST_USER_GET_VRING_BASE
,
1134 .hdr
.flags
= VHOST_USER_VERSION
,
1135 .payload
.state
= *ring
,
1136 .hdr
.size
= sizeof(msg
.payload
.state
),
1139 vhost_user_host_notifier_remove(dev
, ring
->index
);
1141 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
1145 if (vhost_user_read(dev
, &msg
) < 0) {
1149 if (msg
.hdr
.request
!= VHOST_USER_GET_VRING_BASE
) {
1150 error_report("Received unexpected msg type. Expected %d received %d",
1151 VHOST_USER_GET_VRING_BASE
, msg
.hdr
.request
);
1155 if (msg
.hdr
.size
!= sizeof(msg
.payload
.state
)) {
1156 error_report("Received bad msg size.");
1160 *ring
= msg
.payload
.state
;
1165 static int vhost_set_vring_file(struct vhost_dev
*dev
,
1166 VhostUserRequest request
,
1167 struct vhost_vring_file
*file
)
1169 int fds
[VHOST_USER_MAX_RAM_SLOTS
];
1171 VhostUserMsg msg
= {
1172 .hdr
.request
= request
,
1173 .hdr
.flags
= VHOST_USER_VERSION
,
1174 .payload
.u64
= file
->index
& VHOST_USER_VRING_IDX_MASK
,
1175 .hdr
.size
= sizeof(msg
.payload
.u64
),
1178 if (ioeventfd_enabled() && file
->fd
> 0) {
1179 fds
[fd_num
++] = file
->fd
;
1181 msg
.payload
.u64
|= VHOST_USER_VRING_NOFD_MASK
;
1184 if (vhost_user_write(dev
, &msg
, fds
, fd_num
) < 0) {
1191 static int vhost_user_set_vring_kick(struct vhost_dev
*dev
,
1192 struct vhost_vring_file
*file
)
1194 return vhost_set_vring_file(dev
, VHOST_USER_SET_VRING_KICK
, file
);
1197 static int vhost_user_set_vring_call(struct vhost_dev
*dev
,
1198 struct vhost_vring_file
*file
)
1200 return vhost_set_vring_file(dev
, VHOST_USER_SET_VRING_CALL
, file
);
1203 static int vhost_user_set_u64(struct vhost_dev
*dev
, int request
, uint64_t u64
)
1205 VhostUserMsg msg
= {
1206 .hdr
.request
= request
,
1207 .hdr
.flags
= VHOST_USER_VERSION
,
1209 .hdr
.size
= sizeof(msg
.payload
.u64
),
1212 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
1219 static int vhost_user_set_features(struct vhost_dev
*dev
,
1222 return vhost_user_set_u64(dev
, VHOST_USER_SET_FEATURES
, features
);
1225 static int vhost_user_set_protocol_features(struct vhost_dev
*dev
,
1228 return vhost_user_set_u64(dev
, VHOST_USER_SET_PROTOCOL_FEATURES
, features
);
1231 static int vhost_user_get_u64(struct vhost_dev
*dev
, int request
, uint64_t *u64
)
1233 VhostUserMsg msg
= {
1234 .hdr
.request
= request
,
1235 .hdr
.flags
= VHOST_USER_VERSION
,
1238 if (vhost_user_one_time_request(request
) && dev
->vq_index
!= 0) {
1242 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
1246 if (vhost_user_read(dev
, &msg
) < 0) {
1250 if (msg
.hdr
.request
!= request
) {
1251 error_report("Received unexpected msg type. Expected %d received %d",
1252 request
, msg
.hdr
.request
);
1256 if (msg
.hdr
.size
!= sizeof(msg
.payload
.u64
)) {
1257 error_report("Received bad msg size.");
1261 *u64
= msg
.payload
.u64
;
1266 static int vhost_user_get_features(struct vhost_dev
*dev
, uint64_t *features
)
1268 return vhost_user_get_u64(dev
, VHOST_USER_GET_FEATURES
, features
);
1271 static int vhost_user_set_owner(struct vhost_dev
*dev
)
1273 VhostUserMsg msg
= {
1274 .hdr
.request
= VHOST_USER_SET_OWNER
,
1275 .hdr
.flags
= VHOST_USER_VERSION
,
1278 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
1285 static int vhost_user_get_max_memslots(struct vhost_dev
*dev
,
1286 uint64_t *max_memslots
)
1288 uint64_t backend_max_memslots
;
1291 err
= vhost_user_get_u64(dev
, VHOST_USER_GET_MAX_MEM_SLOTS
,
1292 &backend_max_memslots
);
1297 *max_memslots
= backend_max_memslots
;
1302 static int vhost_user_reset_device(struct vhost_dev
*dev
)
1304 VhostUserMsg msg
= {
1305 .hdr
.flags
= VHOST_USER_VERSION
,
1308 msg
.hdr
.request
= virtio_has_feature(dev
->protocol_features
,
1309 VHOST_USER_PROTOCOL_F_RESET_DEVICE
)
1310 ? VHOST_USER_RESET_DEVICE
1311 : VHOST_USER_RESET_OWNER
;
1313 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
1320 static int vhost_user_slave_handle_config_change(struct vhost_dev
*dev
)
1324 if (!dev
->config_ops
) {
1328 if (dev
->config_ops
->vhost_dev_config_notifier
) {
1329 ret
= dev
->config_ops
->vhost_dev_config_notifier(dev
);
1335 static int vhost_user_slave_handle_vring_host_notifier(struct vhost_dev
*dev
,
1336 VhostUserVringArea
*area
,
1339 int queue_idx
= area
->u64
& VHOST_USER_VRING_IDX_MASK
;
1340 size_t page_size
= qemu_real_host_page_size
;
1341 struct vhost_user
*u
= dev
->opaque
;
1342 VhostUserState
*user
= u
->user
;
1343 VirtIODevice
*vdev
= dev
->vdev
;
1344 VhostUserHostNotifier
*n
;
1348 if (!virtio_has_feature(dev
->protocol_features
,
1349 VHOST_USER_PROTOCOL_F_HOST_NOTIFIER
) ||
1350 vdev
== NULL
|| queue_idx
>= virtio_get_num_queues(vdev
)) {
1354 n
= &user
->notifier
[queue_idx
];
1357 virtio_queue_set_host_notifier_mr(vdev
, queue_idx
, &n
->mr
, false);
1358 object_unparent(OBJECT(&n
->mr
));
1359 munmap(n
->addr
, page_size
);
1363 if (area
->u64
& VHOST_USER_VRING_NOFD_MASK
) {
1368 if (area
->size
!= page_size
) {
1372 addr
= mmap(NULL
, page_size
, PROT_READ
| PROT_WRITE
, MAP_SHARED
,
1374 if (addr
== MAP_FAILED
) {
1378 name
= g_strdup_printf("vhost-user/host-notifier@%p mmaps[%d]",
1380 memory_region_init_ram_device_ptr(&n
->mr
, OBJECT(vdev
), name
,
1384 if (virtio_queue_set_host_notifier_mr(vdev
, queue_idx
, &n
->mr
, true)) {
1385 munmap(addr
, page_size
);
1395 static void slave_read(void *opaque
)
1397 struct vhost_dev
*dev
= opaque
;
1398 struct vhost_user
*u
= dev
->opaque
;
1399 VhostUserHeader hdr
= { 0, };
1400 VhostUserPayload payload
= { 0, };
1404 int fd
[VHOST_USER_SLAVE_MAX_FDS
];
1405 char control
[CMSG_SPACE(sizeof(fd
))];
1406 struct cmsghdr
*cmsg
;
1409 memset(&msgh
, 0, sizeof(msgh
));
1410 msgh
.msg_iov
= &iov
;
1411 msgh
.msg_iovlen
= 1;
1412 msgh
.msg_control
= control
;
1413 msgh
.msg_controllen
= sizeof(control
);
1415 memset(fd
, -1, sizeof(fd
));
1418 iov
.iov_base
= &hdr
;
1419 iov
.iov_len
= VHOST_USER_HDR_SIZE
;
1422 size
= recvmsg(u
->slave_fd
, &msgh
, 0);
1423 } while (size
< 0 && (errno
== EINTR
|| errno
== EAGAIN
));
1425 if (size
!= VHOST_USER_HDR_SIZE
) {
1426 error_report("Failed to read from slave.");
1430 if (msgh
.msg_flags
& MSG_CTRUNC
) {
1431 error_report("Truncated message.");
1435 for (cmsg
= CMSG_FIRSTHDR(&msgh
); cmsg
!= NULL
;
1436 cmsg
= CMSG_NXTHDR(&msgh
, cmsg
)) {
1437 if (cmsg
->cmsg_level
== SOL_SOCKET
&&
1438 cmsg
->cmsg_type
== SCM_RIGHTS
) {
1439 fdsize
= cmsg
->cmsg_len
- CMSG_LEN(0);
1440 memcpy(fd
, CMSG_DATA(cmsg
), fdsize
);
1445 if (hdr
.size
> VHOST_USER_PAYLOAD_SIZE
) {
1446 error_report("Failed to read msg header."
1447 " Size %d exceeds the maximum %zu.", hdr
.size
,
1448 VHOST_USER_PAYLOAD_SIZE
);
1454 size
= read(u
->slave_fd
, &payload
, hdr
.size
);
1455 } while (size
< 0 && (errno
== EINTR
|| errno
== EAGAIN
));
1457 if (size
!= hdr
.size
) {
1458 error_report("Failed to read payload from slave.");
1462 switch (hdr
.request
) {
1463 case VHOST_USER_SLAVE_IOTLB_MSG
:
1464 ret
= vhost_backend_handle_iotlb_msg(dev
, &payload
.iotlb
);
1466 case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG
:
1467 ret
= vhost_user_slave_handle_config_change(dev
);
1469 case VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG
:
1470 ret
= vhost_user_slave_handle_vring_host_notifier(dev
, &payload
.area
,
1474 error_report("Received unexpected msg type: %d.", hdr
.request
);
1478 /* Close the remaining file descriptors. */
1479 for (i
= 0; i
< fdsize
; i
++) {
1486 * REPLY_ACK feature handling. Other reply types has to be managed
1487 * directly in their request handlers.
1489 if (hdr
.flags
& VHOST_USER_NEED_REPLY_MASK
) {
1490 struct iovec iovec
[2];
1493 hdr
.flags
&= ~VHOST_USER_NEED_REPLY_MASK
;
1494 hdr
.flags
|= VHOST_USER_REPLY_MASK
;
1496 payload
.u64
= !!ret
;
1497 hdr
.size
= sizeof(payload
.u64
);
1499 iovec
[0].iov_base
= &hdr
;
1500 iovec
[0].iov_len
= VHOST_USER_HDR_SIZE
;
1501 iovec
[1].iov_base
= &payload
;
1502 iovec
[1].iov_len
= hdr
.size
;
1505 size
= writev(u
->slave_fd
, iovec
, ARRAY_SIZE(iovec
));
1506 } while (size
< 0 && (errno
== EINTR
|| errno
== EAGAIN
));
1508 if (size
!= VHOST_USER_HDR_SIZE
+ hdr
.size
) {
1509 error_report("Failed to send msg reply to slave.");
1517 qemu_set_fd_handler(u
->slave_fd
, NULL
, NULL
, NULL
);
1520 for (i
= 0; i
< fdsize
; i
++) {
1528 static int vhost_setup_slave_channel(struct vhost_dev
*dev
)
1530 VhostUserMsg msg
= {
1531 .hdr
.request
= VHOST_USER_SET_SLAVE_REQ_FD
,
1532 .hdr
.flags
= VHOST_USER_VERSION
,
1534 struct vhost_user
*u
= dev
->opaque
;
1536 bool reply_supported
= virtio_has_feature(dev
->protocol_features
,
1537 VHOST_USER_PROTOCOL_F_REPLY_ACK
);
1539 if (!virtio_has_feature(dev
->protocol_features
,
1540 VHOST_USER_PROTOCOL_F_SLAVE_REQ
)) {
1544 if (socketpair(PF_UNIX
, SOCK_STREAM
, 0, sv
) == -1) {
1545 error_report("socketpair() failed");
1549 u
->slave_fd
= sv
[0];
1550 qemu_set_fd_handler(u
->slave_fd
, slave_read
, NULL
, dev
);
1552 if (reply_supported
) {
1553 msg
.hdr
.flags
|= VHOST_USER_NEED_REPLY_MASK
;
1556 ret
= vhost_user_write(dev
, &msg
, &sv
[1], 1);
1561 if (reply_supported
) {
1562 ret
= process_message_reply(dev
, &msg
);
1568 qemu_set_fd_handler(u
->slave_fd
, NULL
, NULL
, NULL
);
1578 * Called back from the postcopy fault thread when a fault is received on our
1580 * TODO: This is Linux specific
1582 static int vhost_user_postcopy_fault_handler(struct PostCopyFD
*pcfd
,
1585 struct vhost_dev
*dev
= pcfd
->data
;
1586 struct vhost_user
*u
= dev
->opaque
;
1587 struct uffd_msg
*msg
= ufd
;
1588 uint64_t faultaddr
= msg
->arg
.pagefault
.address
;
1589 RAMBlock
*rb
= NULL
;
1593 trace_vhost_user_postcopy_fault_handler(pcfd
->idstr
, faultaddr
,
1594 dev
->mem
->nregions
);
1595 for (i
= 0; i
< MIN(dev
->mem
->nregions
, u
->region_rb_len
); i
++) {
1596 trace_vhost_user_postcopy_fault_handler_loop(i
,
1597 u
->postcopy_client_bases
[i
], dev
->mem
->regions
[i
].memory_size
);
1598 if (faultaddr
>= u
->postcopy_client_bases
[i
]) {
1599 /* Ofset of the fault address in the vhost region */
1600 uint64_t region_offset
= faultaddr
- u
->postcopy_client_bases
[i
];
1601 if (region_offset
< dev
->mem
->regions
[i
].memory_size
) {
1602 rb_offset
= region_offset
+ u
->region_rb_offset
[i
];
1603 trace_vhost_user_postcopy_fault_handler_found(i
,
1604 region_offset
, rb_offset
);
1605 rb
= u
->region_rb
[i
];
1606 return postcopy_request_shared_page(pcfd
, rb
, faultaddr
,
1611 error_report("%s: Failed to find region for fault %" PRIx64
,
1612 __func__
, faultaddr
);
1616 static int vhost_user_postcopy_waker(struct PostCopyFD
*pcfd
, RAMBlock
*rb
,
1619 struct vhost_dev
*dev
= pcfd
->data
;
1620 struct vhost_user
*u
= dev
->opaque
;
1623 trace_vhost_user_postcopy_waker(qemu_ram_get_idstr(rb
), offset
);
1628 /* Translate the offset into an address in the clients address space */
1629 for (i
= 0; i
< MIN(dev
->mem
->nregions
, u
->region_rb_len
); i
++) {
1630 if (u
->region_rb
[i
] == rb
&&
1631 offset
>= u
->region_rb_offset
[i
] &&
1632 offset
< (u
->region_rb_offset
[i
] +
1633 dev
->mem
->regions
[i
].memory_size
)) {
1634 uint64_t client_addr
= (offset
- u
->region_rb_offset
[i
]) +
1635 u
->postcopy_client_bases
[i
];
1636 trace_vhost_user_postcopy_waker_found(client_addr
);
1637 return postcopy_wake_shared(pcfd
, client_addr
, rb
);
1641 trace_vhost_user_postcopy_waker_nomatch(qemu_ram_get_idstr(rb
), offset
);
1647 * Called at the start of an inbound postcopy on reception of the
1650 static int vhost_user_postcopy_advise(struct vhost_dev
*dev
, Error
**errp
)
1653 struct vhost_user
*u
= dev
->opaque
;
1654 CharBackend
*chr
= u
->user
->chr
;
1656 VhostUserMsg msg
= {
1657 .hdr
.request
= VHOST_USER_POSTCOPY_ADVISE
,
1658 .hdr
.flags
= VHOST_USER_VERSION
,
1661 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
1662 error_setg(errp
, "Failed to send postcopy_advise to vhost");
1666 if (vhost_user_read(dev
, &msg
) < 0) {
1667 error_setg(errp
, "Failed to get postcopy_advise reply from vhost");
1671 if (msg
.hdr
.request
!= VHOST_USER_POSTCOPY_ADVISE
) {
1672 error_setg(errp
, "Unexpected msg type. Expected %d received %d",
1673 VHOST_USER_POSTCOPY_ADVISE
, msg
.hdr
.request
);
1678 error_setg(errp
, "Received bad msg size.");
1681 ufd
= qemu_chr_fe_get_msgfd(chr
);
1683 error_setg(errp
, "%s: Failed to get ufd", __func__
);
1686 qemu_set_nonblock(ufd
);
1688 /* register ufd with userfault thread */
1689 u
->postcopy_fd
.fd
= ufd
;
1690 u
->postcopy_fd
.data
= dev
;
1691 u
->postcopy_fd
.handler
= vhost_user_postcopy_fault_handler
;
1692 u
->postcopy_fd
.waker
= vhost_user_postcopy_waker
;
1693 u
->postcopy_fd
.idstr
= "vhost-user"; /* Need to find unique name */
1694 postcopy_register_shared_ufd(&u
->postcopy_fd
);
1697 error_setg(errp
, "Postcopy not supported on non-Linux systems");
1703 * Called at the switch to postcopy on reception of the 'listen' command.
1705 static int vhost_user_postcopy_listen(struct vhost_dev
*dev
, Error
**errp
)
1707 struct vhost_user
*u
= dev
->opaque
;
1709 VhostUserMsg msg
= {
1710 .hdr
.request
= VHOST_USER_POSTCOPY_LISTEN
,
1711 .hdr
.flags
= VHOST_USER_VERSION
| VHOST_USER_NEED_REPLY_MASK
,
1713 u
->postcopy_listen
= true;
1714 trace_vhost_user_postcopy_listen();
1715 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
1716 error_setg(errp
, "Failed to send postcopy_listen to vhost");
1720 ret
= process_message_reply(dev
, &msg
);
1722 error_setg(errp
, "Failed to receive reply to postcopy_listen");
1730 * Called at the end of postcopy
1732 static int vhost_user_postcopy_end(struct vhost_dev
*dev
, Error
**errp
)
1734 VhostUserMsg msg
= {
1735 .hdr
.request
= VHOST_USER_POSTCOPY_END
,
1736 .hdr
.flags
= VHOST_USER_VERSION
| VHOST_USER_NEED_REPLY_MASK
,
1739 struct vhost_user
*u
= dev
->opaque
;
1741 trace_vhost_user_postcopy_end_entry();
1742 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
1743 error_setg(errp
, "Failed to send postcopy_end to vhost");
1747 ret
= process_message_reply(dev
, &msg
);
1749 error_setg(errp
, "Failed to receive reply to postcopy_end");
1752 postcopy_unregister_shared_ufd(&u
->postcopy_fd
);
1753 close(u
->postcopy_fd
.fd
);
1754 u
->postcopy_fd
.handler
= NULL
;
1756 trace_vhost_user_postcopy_end_exit();
1761 static int vhost_user_postcopy_notifier(NotifierWithReturn
*notifier
,
1764 struct PostcopyNotifyData
*pnd
= opaque
;
1765 struct vhost_user
*u
= container_of(notifier
, struct vhost_user
,
1767 struct vhost_dev
*dev
= u
->dev
;
1769 switch (pnd
->reason
) {
1770 case POSTCOPY_NOTIFY_PROBE
:
1771 if (!virtio_has_feature(dev
->protocol_features
,
1772 VHOST_USER_PROTOCOL_F_PAGEFAULT
)) {
1773 /* TODO: Get the device name into this error somehow */
1774 error_setg(pnd
->errp
,
1775 "vhost-user backend not capable of postcopy");
1780 case POSTCOPY_NOTIFY_INBOUND_ADVISE
:
1781 return vhost_user_postcopy_advise(dev
, pnd
->errp
);
1783 case POSTCOPY_NOTIFY_INBOUND_LISTEN
:
1784 return vhost_user_postcopy_listen(dev
, pnd
->errp
);
1786 case POSTCOPY_NOTIFY_INBOUND_END
:
1787 return vhost_user_postcopy_end(dev
, pnd
->errp
);
1790 /* We ignore notifications we don't know */
1797 static int vhost_user_backend_init(struct vhost_dev
*dev
, void *opaque
)
1799 uint64_t features
, protocol_features
, ram_slots
;
1800 struct vhost_user
*u
;
1803 assert(dev
->vhost_ops
->backend_type
== VHOST_BACKEND_TYPE_USER
);
1805 u
= g_new0(struct vhost_user
, 1);
1811 err
= vhost_user_get_features(dev
, &features
);
1816 if (virtio_has_feature(features
, VHOST_USER_F_PROTOCOL_FEATURES
)) {
1817 dev
->backend_features
|= 1ULL << VHOST_USER_F_PROTOCOL_FEATURES
;
1819 err
= vhost_user_get_u64(dev
, VHOST_USER_GET_PROTOCOL_FEATURES
,
1820 &protocol_features
);
1825 dev
->protocol_features
=
1826 protocol_features
& VHOST_USER_PROTOCOL_FEATURE_MASK
;
1828 if (!dev
->config_ops
|| !dev
->config_ops
->vhost_dev_config_notifier
) {
1829 /* Don't acknowledge CONFIG feature if device doesn't support it */
1830 dev
->protocol_features
&= ~(1ULL << VHOST_USER_PROTOCOL_F_CONFIG
);
1831 } else if (!(protocol_features
&
1832 (1ULL << VHOST_USER_PROTOCOL_F_CONFIG
))) {
1833 error_report("Device expects VHOST_USER_PROTOCOL_F_CONFIG "
1834 "but backend does not support it.");
1838 err
= vhost_user_set_protocol_features(dev
, dev
->protocol_features
);
1843 /* query the max queues we support if backend supports Multiple Queue */
1844 if (dev
->protocol_features
& (1ULL << VHOST_USER_PROTOCOL_F_MQ
)) {
1845 err
= vhost_user_get_u64(dev
, VHOST_USER_GET_QUEUE_NUM
,
1852 if (virtio_has_feature(features
, VIRTIO_F_IOMMU_PLATFORM
) &&
1853 !(virtio_has_feature(dev
->protocol_features
,
1854 VHOST_USER_PROTOCOL_F_SLAVE_REQ
) &&
1855 virtio_has_feature(dev
->protocol_features
,
1856 VHOST_USER_PROTOCOL_F_REPLY_ACK
))) {
1857 error_report("IOMMU support requires reply-ack and "
1858 "slave-req protocol features.");
1862 /* get max memory regions if backend supports configurable RAM slots */
1863 if (!virtio_has_feature(dev
->protocol_features
,
1864 VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS
)) {
1865 u
->user
->memory_slots
= VHOST_MEMORY_BASELINE_NREGIONS
;
1867 err
= vhost_user_get_max_memslots(dev
, &ram_slots
);
1872 if (ram_slots
< u
->user
->memory_slots
) {
1873 error_report("The backend specified a max ram slots limit "
1874 "of %" PRIu64
", when the prior validated limit was %d. "
1875 "This limit should never decrease.", ram_slots
,
1876 u
->user
->memory_slots
);
1880 u
->user
->memory_slots
= MIN(ram_slots
, VHOST_USER_MAX_RAM_SLOTS
);
1884 if (dev
->migration_blocker
== NULL
&&
1885 !virtio_has_feature(dev
->protocol_features
,
1886 VHOST_USER_PROTOCOL_F_LOG_SHMFD
)) {
1887 error_setg(&dev
->migration_blocker
,
1888 "Migration disabled: vhost-user backend lacks "
1889 "VHOST_USER_PROTOCOL_F_LOG_SHMFD feature.");
1892 if (dev
->vq_index
== 0) {
1893 err
= vhost_setup_slave_channel(dev
);
1899 u
->postcopy_notifier
.notify
= vhost_user_postcopy_notifier
;
1900 postcopy_add_notifier(&u
->postcopy_notifier
);
1905 static int vhost_user_backend_cleanup(struct vhost_dev
*dev
)
1907 struct vhost_user
*u
;
1909 assert(dev
->vhost_ops
->backend_type
== VHOST_BACKEND_TYPE_USER
);
1912 if (u
->postcopy_notifier
.notify
) {
1913 postcopy_remove_notifier(&u
->postcopy_notifier
);
1914 u
->postcopy_notifier
.notify
= NULL
;
1916 u
->postcopy_listen
= false;
1917 if (u
->postcopy_fd
.handler
) {
1918 postcopy_unregister_shared_ufd(&u
->postcopy_fd
);
1919 close(u
->postcopy_fd
.fd
);
1920 u
->postcopy_fd
.handler
= NULL
;
1922 if (u
->slave_fd
>= 0) {
1923 qemu_set_fd_handler(u
->slave_fd
, NULL
, NULL
, NULL
);
1927 g_free(u
->region_rb
);
1928 u
->region_rb
= NULL
;
1929 g_free(u
->region_rb_offset
);
1930 u
->region_rb_offset
= NULL
;
1931 u
->region_rb_len
= 0;
1938 static int vhost_user_get_vq_index(struct vhost_dev
*dev
, int idx
)
1940 assert(idx
>= dev
->vq_index
&& idx
< dev
->vq_index
+ dev
->nvqs
);
1945 static int vhost_user_memslots_limit(struct vhost_dev
*dev
)
1947 struct vhost_user
*u
= dev
->opaque
;
1949 return u
->user
->memory_slots
;
1952 static bool vhost_user_requires_shm_log(struct vhost_dev
*dev
)
1954 assert(dev
->vhost_ops
->backend_type
== VHOST_BACKEND_TYPE_USER
);
1956 return virtio_has_feature(dev
->protocol_features
,
1957 VHOST_USER_PROTOCOL_F_LOG_SHMFD
);
1960 static int vhost_user_migration_done(struct vhost_dev
*dev
, char* mac_addr
)
1962 VhostUserMsg msg
= { };
1964 assert(dev
->vhost_ops
->backend_type
== VHOST_BACKEND_TYPE_USER
);
1966 /* If guest supports GUEST_ANNOUNCE do nothing */
1967 if (virtio_has_feature(dev
->acked_features
, VIRTIO_NET_F_GUEST_ANNOUNCE
)) {
1971 /* if backend supports VHOST_USER_PROTOCOL_F_RARP ask it to send the RARP */
1972 if (virtio_has_feature(dev
->protocol_features
,
1973 VHOST_USER_PROTOCOL_F_RARP
)) {
1974 msg
.hdr
.request
= VHOST_USER_SEND_RARP
;
1975 msg
.hdr
.flags
= VHOST_USER_VERSION
;
1976 memcpy((char *)&msg
.payload
.u64
, mac_addr
, 6);
1977 msg
.hdr
.size
= sizeof(msg
.payload
.u64
);
1979 return vhost_user_write(dev
, &msg
, NULL
, 0);
1984 static bool vhost_user_can_merge(struct vhost_dev
*dev
,
1985 uint64_t start1
, uint64_t size1
,
1986 uint64_t start2
, uint64_t size2
)
1991 (void)vhost_user_get_mr_data(start1
, &offset
, &mfd
);
1992 (void)vhost_user_get_mr_data(start2
, &offset
, &rfd
);
1997 static int vhost_user_net_set_mtu(struct vhost_dev
*dev
, uint16_t mtu
)
2000 bool reply_supported
= virtio_has_feature(dev
->protocol_features
,
2001 VHOST_USER_PROTOCOL_F_REPLY_ACK
);
2003 if (!(dev
->protocol_features
& (1ULL << VHOST_USER_PROTOCOL_F_NET_MTU
))) {
2007 msg
.hdr
.request
= VHOST_USER_NET_SET_MTU
;
2008 msg
.payload
.u64
= mtu
;
2009 msg
.hdr
.size
= sizeof(msg
.payload
.u64
);
2010 msg
.hdr
.flags
= VHOST_USER_VERSION
;
2011 if (reply_supported
) {
2012 msg
.hdr
.flags
|= VHOST_USER_NEED_REPLY_MASK
;
2015 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
2019 /* If reply_ack supported, slave has to ack specified MTU is valid */
2020 if (reply_supported
) {
2021 return process_message_reply(dev
, &msg
);
2027 static int vhost_user_send_device_iotlb_msg(struct vhost_dev
*dev
,
2028 struct vhost_iotlb_msg
*imsg
)
2030 VhostUserMsg msg
= {
2031 .hdr
.request
= VHOST_USER_IOTLB_MSG
,
2032 .hdr
.size
= sizeof(msg
.payload
.iotlb
),
2033 .hdr
.flags
= VHOST_USER_VERSION
| VHOST_USER_NEED_REPLY_MASK
,
2034 .payload
.iotlb
= *imsg
,
2037 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
2041 return process_message_reply(dev
, &msg
);
2045 static void vhost_user_set_iotlb_callback(struct vhost_dev
*dev
, int enabled
)
2047 /* No-op as the receive channel is not dedicated to IOTLB messages. */
2050 static int vhost_user_get_config(struct vhost_dev
*dev
, uint8_t *config
,
2051 uint32_t config_len
)
2053 VhostUserMsg msg
= {
2054 .hdr
.request
= VHOST_USER_GET_CONFIG
,
2055 .hdr
.flags
= VHOST_USER_VERSION
,
2056 .hdr
.size
= VHOST_USER_CONFIG_HDR_SIZE
+ config_len
,
2059 if (!virtio_has_feature(dev
->protocol_features
,
2060 VHOST_USER_PROTOCOL_F_CONFIG
)) {
2064 if (config_len
> VHOST_USER_MAX_CONFIG_SIZE
) {
2068 msg
.payload
.config
.offset
= 0;
2069 msg
.payload
.config
.size
= config_len
;
2070 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
2074 if (vhost_user_read(dev
, &msg
) < 0) {
2078 if (msg
.hdr
.request
!= VHOST_USER_GET_CONFIG
) {
2079 error_report("Received unexpected msg type. Expected %d received %d",
2080 VHOST_USER_GET_CONFIG
, msg
.hdr
.request
);
2084 if (msg
.hdr
.size
!= VHOST_USER_CONFIG_HDR_SIZE
+ config_len
) {
2085 error_report("Received bad msg size.");
2089 memcpy(config
, msg
.payload
.config
.region
, config_len
);
2094 static int vhost_user_set_config(struct vhost_dev
*dev
, const uint8_t *data
,
2095 uint32_t offset
, uint32_t size
, uint32_t flags
)
2098 bool reply_supported
= virtio_has_feature(dev
->protocol_features
,
2099 VHOST_USER_PROTOCOL_F_REPLY_ACK
);
2101 VhostUserMsg msg
= {
2102 .hdr
.request
= VHOST_USER_SET_CONFIG
,
2103 .hdr
.flags
= VHOST_USER_VERSION
,
2104 .hdr
.size
= VHOST_USER_CONFIG_HDR_SIZE
+ size
,
2107 if (!virtio_has_feature(dev
->protocol_features
,
2108 VHOST_USER_PROTOCOL_F_CONFIG
)) {
2112 if (reply_supported
) {
2113 msg
.hdr
.flags
|= VHOST_USER_NEED_REPLY_MASK
;
2116 if (size
> VHOST_USER_MAX_CONFIG_SIZE
) {
2120 msg
.payload
.config
.offset
= offset
,
2121 msg
.payload
.config
.size
= size
,
2122 msg
.payload
.config
.flags
= flags
,
2123 p
= msg
.payload
.config
.region
;
2124 memcpy(p
, data
, size
);
2126 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
2130 if (reply_supported
) {
2131 return process_message_reply(dev
, &msg
);
2137 static int vhost_user_crypto_create_session(struct vhost_dev
*dev
,
2139 uint64_t *session_id
)
2141 bool crypto_session
= virtio_has_feature(dev
->protocol_features
,
2142 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION
);
2143 CryptoDevBackendSymSessionInfo
*sess_info
= session_info
;
2144 VhostUserMsg msg
= {
2145 .hdr
.request
= VHOST_USER_CREATE_CRYPTO_SESSION
,
2146 .hdr
.flags
= VHOST_USER_VERSION
,
2147 .hdr
.size
= sizeof(msg
.payload
.session
),
2150 assert(dev
->vhost_ops
->backend_type
== VHOST_BACKEND_TYPE_USER
);
2152 if (!crypto_session
) {
2153 error_report("vhost-user trying to send unhandled ioctl");
2157 memcpy(&msg
.payload
.session
.session_setup_data
, sess_info
,
2158 sizeof(CryptoDevBackendSymSessionInfo
));
2159 if (sess_info
->key_len
) {
2160 memcpy(&msg
.payload
.session
.key
, sess_info
->cipher_key
,
2161 sess_info
->key_len
);
2163 if (sess_info
->auth_key_len
> 0) {
2164 memcpy(&msg
.payload
.session
.auth_key
, sess_info
->auth_key
,
2165 sess_info
->auth_key_len
);
2167 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
2168 error_report("vhost_user_write() return -1, create session failed");
2172 if (vhost_user_read(dev
, &msg
) < 0) {
2173 error_report("vhost_user_read() return -1, create session failed");
2177 if (msg
.hdr
.request
!= VHOST_USER_CREATE_CRYPTO_SESSION
) {
2178 error_report("Received unexpected msg type. Expected %d received %d",
2179 VHOST_USER_CREATE_CRYPTO_SESSION
, msg
.hdr
.request
);
2183 if (msg
.hdr
.size
!= sizeof(msg
.payload
.session
)) {
2184 error_report("Received bad msg size.");
2188 if (msg
.payload
.session
.session_id
< 0) {
2189 error_report("Bad session id: %" PRId64
"",
2190 msg
.payload
.session
.session_id
);
2193 *session_id
= msg
.payload
.session
.session_id
;
2199 vhost_user_crypto_close_session(struct vhost_dev
*dev
, uint64_t session_id
)
2201 bool crypto_session
= virtio_has_feature(dev
->protocol_features
,
2202 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION
);
2203 VhostUserMsg msg
= {
2204 .hdr
.request
= VHOST_USER_CLOSE_CRYPTO_SESSION
,
2205 .hdr
.flags
= VHOST_USER_VERSION
,
2206 .hdr
.size
= sizeof(msg
.payload
.u64
),
2208 msg
.payload
.u64
= session_id
;
2210 if (!crypto_session
) {
2211 error_report("vhost-user trying to send unhandled ioctl");
2215 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
2216 error_report("vhost_user_write() return -1, close session failed");
2223 static bool vhost_user_mem_section_filter(struct vhost_dev
*dev
,
2224 MemoryRegionSection
*section
)
2228 result
= memory_region_get_fd(section
->mr
) >= 0;
2233 static int vhost_user_get_inflight_fd(struct vhost_dev
*dev
,
2234 uint16_t queue_size
,
2235 struct vhost_inflight
*inflight
)
2239 struct vhost_user
*u
= dev
->opaque
;
2240 CharBackend
*chr
= u
->user
->chr
;
2241 VhostUserMsg msg
= {
2242 .hdr
.request
= VHOST_USER_GET_INFLIGHT_FD
,
2243 .hdr
.flags
= VHOST_USER_VERSION
,
2244 .payload
.inflight
.num_queues
= dev
->nvqs
,
2245 .payload
.inflight
.queue_size
= queue_size
,
2246 .hdr
.size
= sizeof(msg
.payload
.inflight
),
2249 if (!virtio_has_feature(dev
->protocol_features
,
2250 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD
)) {
2254 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
2258 if (vhost_user_read(dev
, &msg
) < 0) {
2262 if (msg
.hdr
.request
!= VHOST_USER_GET_INFLIGHT_FD
) {
2263 error_report("Received unexpected msg type. "
2264 "Expected %d received %d",
2265 VHOST_USER_GET_INFLIGHT_FD
, msg
.hdr
.request
);
2269 if (msg
.hdr
.size
!= sizeof(msg
.payload
.inflight
)) {
2270 error_report("Received bad msg size.");
2274 if (!msg
.payload
.inflight
.mmap_size
) {
2278 fd
= qemu_chr_fe_get_msgfd(chr
);
2280 error_report("Failed to get mem fd");
2284 addr
= mmap(0, msg
.payload
.inflight
.mmap_size
, PROT_READ
| PROT_WRITE
,
2285 MAP_SHARED
, fd
, msg
.payload
.inflight
.mmap_offset
);
2287 if (addr
== MAP_FAILED
) {
2288 error_report("Failed to mmap mem fd");
2293 inflight
->addr
= addr
;
2295 inflight
->size
= msg
.payload
.inflight
.mmap_size
;
2296 inflight
->offset
= msg
.payload
.inflight
.mmap_offset
;
2297 inflight
->queue_size
= queue_size
;
2302 static int vhost_user_set_inflight_fd(struct vhost_dev
*dev
,
2303 struct vhost_inflight
*inflight
)
2305 VhostUserMsg msg
= {
2306 .hdr
.request
= VHOST_USER_SET_INFLIGHT_FD
,
2307 .hdr
.flags
= VHOST_USER_VERSION
,
2308 .payload
.inflight
.mmap_size
= inflight
->size
,
2309 .payload
.inflight
.mmap_offset
= inflight
->offset
,
2310 .payload
.inflight
.num_queues
= dev
->nvqs
,
2311 .payload
.inflight
.queue_size
= inflight
->queue_size
,
2312 .hdr
.size
= sizeof(msg
.payload
.inflight
),
2315 if (!virtio_has_feature(dev
->protocol_features
,
2316 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD
)) {
2320 if (vhost_user_write(dev
, &msg
, &inflight
->fd
, 1) < 0) {
2327 bool vhost_user_init(VhostUserState
*user
, CharBackend
*chr
, Error
**errp
)
2330 error_setg(errp
, "Cannot initialize vhost-user state");
2334 user
->memory_slots
= 0;
2338 void vhost_user_cleanup(VhostUserState
*user
)
2346 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2347 if (user
->notifier
[i
].addr
) {
2348 object_unparent(OBJECT(&user
->notifier
[i
].mr
));
2349 munmap(user
->notifier
[i
].addr
, qemu_real_host_page_size
);
2350 user
->notifier
[i
].addr
= NULL
;
2356 const VhostOps user_ops
= {
2357 .backend_type
= VHOST_BACKEND_TYPE_USER
,
2358 .vhost_backend_init
= vhost_user_backend_init
,
2359 .vhost_backend_cleanup
= vhost_user_backend_cleanup
,
2360 .vhost_backend_memslots_limit
= vhost_user_memslots_limit
,
2361 .vhost_set_log_base
= vhost_user_set_log_base
,
2362 .vhost_set_mem_table
= vhost_user_set_mem_table
,
2363 .vhost_set_vring_addr
= vhost_user_set_vring_addr
,
2364 .vhost_set_vring_endian
= vhost_user_set_vring_endian
,
2365 .vhost_set_vring_num
= vhost_user_set_vring_num
,
2366 .vhost_set_vring_base
= vhost_user_set_vring_base
,
2367 .vhost_get_vring_base
= vhost_user_get_vring_base
,
2368 .vhost_set_vring_kick
= vhost_user_set_vring_kick
,
2369 .vhost_set_vring_call
= vhost_user_set_vring_call
,
2370 .vhost_set_features
= vhost_user_set_features
,
2371 .vhost_get_features
= vhost_user_get_features
,
2372 .vhost_set_owner
= vhost_user_set_owner
,
2373 .vhost_reset_device
= vhost_user_reset_device
,
2374 .vhost_get_vq_index
= vhost_user_get_vq_index
,
2375 .vhost_set_vring_enable
= vhost_user_set_vring_enable
,
2376 .vhost_requires_shm_log
= vhost_user_requires_shm_log
,
2377 .vhost_migration_done
= vhost_user_migration_done
,
2378 .vhost_backend_can_merge
= vhost_user_can_merge
,
2379 .vhost_net_set_mtu
= vhost_user_net_set_mtu
,
2380 .vhost_set_iotlb_callback
= vhost_user_set_iotlb_callback
,
2381 .vhost_send_device_iotlb_msg
= vhost_user_send_device_iotlb_msg
,
2382 .vhost_get_config
= vhost_user_get_config
,
2383 .vhost_set_config
= vhost_user_set_config
,
2384 .vhost_crypto_create_session
= vhost_user_crypto_create_session
,
2385 .vhost_crypto_close_session
= vhost_user_crypto_close_session
,
2386 .vhost_backend_mem_section_filter
= vhost_user_mem_section_filter
,
2387 .vhost_get_inflight_fd
= vhost_user_get_inflight_fd
,
2388 .vhost_set_inflight_fd
= vhost_user_set_inflight_fd
,