4 * Copyright (c) 2013 Virtual Open Systems Sarl.
6 * This work is licensed under the terms of the GNU GPL, version 2 or later.
7 * See the COPYING file in the top-level directory.
11 #include "qemu/osdep.h"
12 #include "qapi/error.h"
13 #include "hw/virtio/vhost.h"
14 #include "hw/virtio/vhost-user.h"
15 #include "hw/virtio/vhost-backend.h"
16 #include "hw/virtio/virtio.h"
17 #include "hw/virtio/virtio-net.h"
18 #include "chardev/char-fe.h"
19 #include "sysemu/kvm.h"
20 #include "qemu/error-report.h"
21 #include "qemu/main-loop.h"
22 #include "qemu/sockets.h"
23 #include "sysemu/cryptodev.h"
24 #include "migration/migration.h"
25 #include "migration/postcopy-ram.h"
28 #include <sys/ioctl.h>
29 #include <sys/socket.h>
32 #include "standard-headers/linux/vhost_types.h"
35 #include <linux/userfaultfd.h>
38 #define VHOST_MEMORY_MAX_NREGIONS 8
39 #define VHOST_USER_F_PROTOCOL_FEATURES 30
40 #define VHOST_USER_SLAVE_MAX_FDS 8
43 * Maximum size of virtio device config space
45 #define VHOST_USER_MAX_CONFIG_SIZE 256
47 enum VhostUserProtocolFeature
{
48 VHOST_USER_PROTOCOL_F_MQ
= 0,
49 VHOST_USER_PROTOCOL_F_LOG_SHMFD
= 1,
50 VHOST_USER_PROTOCOL_F_RARP
= 2,
51 VHOST_USER_PROTOCOL_F_REPLY_ACK
= 3,
52 VHOST_USER_PROTOCOL_F_NET_MTU
= 4,
53 VHOST_USER_PROTOCOL_F_SLAVE_REQ
= 5,
54 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN
= 6,
55 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION
= 7,
56 VHOST_USER_PROTOCOL_F_PAGEFAULT
= 8,
57 VHOST_USER_PROTOCOL_F_CONFIG
= 9,
58 VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD
= 10,
59 VHOST_USER_PROTOCOL_F_HOST_NOTIFIER
= 11,
60 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD
= 12,
61 VHOST_USER_PROTOCOL_F_RESET_DEVICE
= 13,
62 VHOST_USER_PROTOCOL_F_MAX
65 #define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1)
67 typedef enum VhostUserRequest
{
69 VHOST_USER_GET_FEATURES
= 1,
70 VHOST_USER_SET_FEATURES
= 2,
71 VHOST_USER_SET_OWNER
= 3,
72 VHOST_USER_RESET_OWNER
= 4,
73 VHOST_USER_SET_MEM_TABLE
= 5,
74 VHOST_USER_SET_LOG_BASE
= 6,
75 VHOST_USER_SET_LOG_FD
= 7,
76 VHOST_USER_SET_VRING_NUM
= 8,
77 VHOST_USER_SET_VRING_ADDR
= 9,
78 VHOST_USER_SET_VRING_BASE
= 10,
79 VHOST_USER_GET_VRING_BASE
= 11,
80 VHOST_USER_SET_VRING_KICK
= 12,
81 VHOST_USER_SET_VRING_CALL
= 13,
82 VHOST_USER_SET_VRING_ERR
= 14,
83 VHOST_USER_GET_PROTOCOL_FEATURES
= 15,
84 VHOST_USER_SET_PROTOCOL_FEATURES
= 16,
85 VHOST_USER_GET_QUEUE_NUM
= 17,
86 VHOST_USER_SET_VRING_ENABLE
= 18,
87 VHOST_USER_SEND_RARP
= 19,
88 VHOST_USER_NET_SET_MTU
= 20,
89 VHOST_USER_SET_SLAVE_REQ_FD
= 21,
90 VHOST_USER_IOTLB_MSG
= 22,
91 VHOST_USER_SET_VRING_ENDIAN
= 23,
92 VHOST_USER_GET_CONFIG
= 24,
93 VHOST_USER_SET_CONFIG
= 25,
94 VHOST_USER_CREATE_CRYPTO_SESSION
= 26,
95 VHOST_USER_CLOSE_CRYPTO_SESSION
= 27,
96 VHOST_USER_POSTCOPY_ADVISE
= 28,
97 VHOST_USER_POSTCOPY_LISTEN
= 29,
98 VHOST_USER_POSTCOPY_END
= 30,
99 VHOST_USER_GET_INFLIGHT_FD
= 31,
100 VHOST_USER_SET_INFLIGHT_FD
= 32,
101 VHOST_USER_GPU_SET_SOCKET
= 33,
102 VHOST_USER_RESET_DEVICE
= 34,
106 typedef enum VhostUserSlaveRequest
{
107 VHOST_USER_SLAVE_NONE
= 0,
108 VHOST_USER_SLAVE_IOTLB_MSG
= 1,
109 VHOST_USER_SLAVE_CONFIG_CHANGE_MSG
= 2,
110 VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG
= 3,
112 } VhostUserSlaveRequest
;
114 typedef struct VhostUserMemoryRegion
{
115 uint64_t guest_phys_addr
;
116 uint64_t memory_size
;
117 uint64_t userspace_addr
;
118 uint64_t mmap_offset
;
119 } VhostUserMemoryRegion
;
121 typedef struct VhostUserMemory
{
124 VhostUserMemoryRegion regions
[VHOST_MEMORY_MAX_NREGIONS
];
127 typedef struct VhostUserLog
{
129 uint64_t mmap_offset
;
132 typedef struct VhostUserConfig
{
136 uint8_t region
[VHOST_USER_MAX_CONFIG_SIZE
];
139 #define VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN 512
140 #define VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN 64
142 typedef struct VhostUserCryptoSession
{
143 /* session id for success, -1 on errors */
145 CryptoDevBackendSymSessionInfo session_setup_data
;
146 uint8_t key
[VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN
];
147 uint8_t auth_key
[VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN
];
148 } VhostUserCryptoSession
;
150 static VhostUserConfig c
__attribute__ ((unused
));
151 #define VHOST_USER_CONFIG_HDR_SIZE (sizeof(c.offset) \
155 typedef struct VhostUserVringArea
{
159 } VhostUserVringArea
;
161 typedef struct VhostUserInflight
{
163 uint64_t mmap_offset
;
169 VhostUserRequest request
;
171 #define VHOST_USER_VERSION_MASK (0x3)
172 #define VHOST_USER_REPLY_MASK (0x1<<2)
173 #define VHOST_USER_NEED_REPLY_MASK (0x1 << 3)
175 uint32_t size
; /* the following payload size */
176 } QEMU_PACKED VhostUserHeader
;
179 #define VHOST_USER_VRING_IDX_MASK (0xff)
180 #define VHOST_USER_VRING_NOFD_MASK (0x1<<8)
182 struct vhost_vring_state state
;
183 struct vhost_vring_addr addr
;
184 VhostUserMemory memory
;
186 struct vhost_iotlb_msg iotlb
;
187 VhostUserConfig config
;
188 VhostUserCryptoSession session
;
189 VhostUserVringArea area
;
190 VhostUserInflight inflight
;
193 typedef struct VhostUserMsg
{
195 VhostUserPayload payload
;
196 } QEMU_PACKED VhostUserMsg
;
198 static VhostUserMsg m
__attribute__ ((unused
));
199 #define VHOST_USER_HDR_SIZE (sizeof(VhostUserHeader))
201 #define VHOST_USER_PAYLOAD_SIZE (sizeof(VhostUserPayload))
203 /* The version of the protocol we support */
204 #define VHOST_USER_VERSION (0x1)
207 struct vhost_dev
*dev
;
208 /* Shared between vhost devs of the same virtio device */
209 VhostUserState
*user
;
211 NotifierWithReturn postcopy_notifier
;
212 struct PostCopyFD postcopy_fd
;
213 uint64_t postcopy_client_bases
[VHOST_MEMORY_MAX_NREGIONS
];
214 /* Length of the region_rb and region_rb_offset arrays */
215 size_t region_rb_len
;
216 /* RAMBlock associated with a given region */
217 RAMBlock
**region_rb
;
218 /* The offset from the start of the RAMBlock to the start of the
221 ram_addr_t
*region_rb_offset
;
223 /* True once we've entered postcopy_listen */
224 bool postcopy_listen
;
227 static bool ioeventfd_enabled(void)
229 return !kvm_enabled() || kvm_eventfds_enabled();
232 static int vhost_user_read_header(struct vhost_dev
*dev
, VhostUserMsg
*msg
)
234 struct vhost_user
*u
= dev
->opaque
;
235 CharBackend
*chr
= u
->user
->chr
;
236 uint8_t *p
= (uint8_t *) msg
;
237 int r
, size
= VHOST_USER_HDR_SIZE
;
239 r
= qemu_chr_fe_read_all(chr
, p
, size
);
241 error_report("Failed to read msg header. Read %d instead of %d."
242 " Original request %d.", r
, size
, msg
->hdr
.request
);
246 /* validate received flags */
247 if (msg
->hdr
.flags
!= (VHOST_USER_REPLY_MASK
| VHOST_USER_VERSION
)) {
248 error_report("Failed to read msg header."
249 " Flags 0x%x instead of 0x%x.", msg
->hdr
.flags
,
250 VHOST_USER_REPLY_MASK
| VHOST_USER_VERSION
);
257 static int vhost_user_read(struct vhost_dev
*dev
, VhostUserMsg
*msg
)
259 struct vhost_user
*u
= dev
->opaque
;
260 CharBackend
*chr
= u
->user
->chr
;
261 uint8_t *p
= (uint8_t *) msg
;
264 if (vhost_user_read_header(dev
, msg
) < 0) {
268 /* validate message size is sane */
269 if (msg
->hdr
.size
> VHOST_USER_PAYLOAD_SIZE
) {
270 error_report("Failed to read msg header."
271 " Size %d exceeds the maximum %zu.", msg
->hdr
.size
,
272 VHOST_USER_PAYLOAD_SIZE
);
277 p
+= VHOST_USER_HDR_SIZE
;
278 size
= msg
->hdr
.size
;
279 r
= qemu_chr_fe_read_all(chr
, p
, size
);
281 error_report("Failed to read msg payload."
282 " Read %d instead of %d.", r
, msg
->hdr
.size
);
290 static int process_message_reply(struct vhost_dev
*dev
,
291 const VhostUserMsg
*msg
)
293 VhostUserMsg msg_reply
;
295 if ((msg
->hdr
.flags
& VHOST_USER_NEED_REPLY_MASK
) == 0) {
299 if (vhost_user_read(dev
, &msg_reply
) < 0) {
303 if (msg_reply
.hdr
.request
!= msg
->hdr
.request
) {
304 error_report("Received unexpected msg type."
305 "Expected %d received %d",
306 msg
->hdr
.request
, msg_reply
.hdr
.request
);
310 return msg_reply
.payload
.u64
? -1 : 0;
313 static bool vhost_user_one_time_request(VhostUserRequest request
)
316 case VHOST_USER_SET_OWNER
:
317 case VHOST_USER_RESET_OWNER
:
318 case VHOST_USER_SET_MEM_TABLE
:
319 case VHOST_USER_GET_QUEUE_NUM
:
320 case VHOST_USER_NET_SET_MTU
:
327 /* most non-init callers ignore the error */
328 static int vhost_user_write(struct vhost_dev
*dev
, VhostUserMsg
*msg
,
329 int *fds
, int fd_num
)
331 struct vhost_user
*u
= dev
->opaque
;
332 CharBackend
*chr
= u
->user
->chr
;
333 int ret
, size
= VHOST_USER_HDR_SIZE
+ msg
->hdr
.size
;
336 * For non-vring specific requests, like VHOST_USER_SET_MEM_TABLE,
337 * we just need send it once in the first time. For later such
338 * request, we just ignore it.
340 if (vhost_user_one_time_request(msg
->hdr
.request
) && dev
->vq_index
!= 0) {
341 msg
->hdr
.flags
&= ~VHOST_USER_NEED_REPLY_MASK
;
345 if (qemu_chr_fe_set_msgfds(chr
, fds
, fd_num
) < 0) {
346 error_report("Failed to set msg fds.");
350 ret
= qemu_chr_fe_write_all(chr
, (const uint8_t *) msg
, size
);
352 error_report("Failed to write msg."
353 " Wrote %d instead of %d.", ret
, size
);
360 int vhost_user_gpu_set_socket(struct vhost_dev
*dev
, int fd
)
363 .hdr
.request
= VHOST_USER_GPU_SET_SOCKET
,
364 .hdr
.flags
= VHOST_USER_VERSION
,
367 return vhost_user_write(dev
, &msg
, &fd
, 1);
370 static int vhost_user_set_log_base(struct vhost_dev
*dev
, uint64_t base
,
371 struct vhost_log
*log
)
373 int fds
[VHOST_MEMORY_MAX_NREGIONS
];
375 bool shmfd
= virtio_has_feature(dev
->protocol_features
,
376 VHOST_USER_PROTOCOL_F_LOG_SHMFD
);
378 .hdr
.request
= VHOST_USER_SET_LOG_BASE
,
379 .hdr
.flags
= VHOST_USER_VERSION
,
380 .payload
.log
.mmap_size
= log
->size
* sizeof(*(log
->log
)),
381 .payload
.log
.mmap_offset
= 0,
382 .hdr
.size
= sizeof(msg
.payload
.log
),
385 if (shmfd
&& log
->fd
!= -1) {
386 fds
[fd_num
++] = log
->fd
;
389 if (vhost_user_write(dev
, &msg
, fds
, fd_num
) < 0) {
395 if (vhost_user_read(dev
, &msg
) < 0) {
399 if (msg
.hdr
.request
!= VHOST_USER_SET_LOG_BASE
) {
400 error_report("Received unexpected msg type. "
401 "Expected %d received %d",
402 VHOST_USER_SET_LOG_BASE
, msg
.hdr
.request
);
410 static int vhost_user_set_mem_table_postcopy(struct vhost_dev
*dev
,
411 struct vhost_memory
*mem
)
413 struct vhost_user
*u
= dev
->opaque
;
414 int fds
[VHOST_MEMORY_MAX_NREGIONS
];
417 VhostUserMsg msg_reply
;
421 .hdr
.request
= VHOST_USER_SET_MEM_TABLE
,
422 .hdr
.flags
= VHOST_USER_VERSION
,
425 if (u
->region_rb_len
< dev
->mem
->nregions
) {
426 u
->region_rb
= g_renew(RAMBlock
*, u
->region_rb
, dev
->mem
->nregions
);
427 u
->region_rb_offset
= g_renew(ram_addr_t
, u
->region_rb_offset
,
429 memset(&(u
->region_rb
[u
->region_rb_len
]), '\0',
430 sizeof(RAMBlock
*) * (dev
->mem
->nregions
- u
->region_rb_len
));
431 memset(&(u
->region_rb_offset
[u
->region_rb_len
]), '\0',
432 sizeof(ram_addr_t
) * (dev
->mem
->nregions
- u
->region_rb_len
));
433 u
->region_rb_len
= dev
->mem
->nregions
;
436 for (i
= 0; i
< dev
->mem
->nregions
; ++i
) {
437 struct vhost_memory_region
*reg
= dev
->mem
->regions
+ i
;
441 assert((uintptr_t)reg
->userspace_addr
== reg
->userspace_addr
);
442 mr
= memory_region_from_host((void *)(uintptr_t)reg
->userspace_addr
,
444 fd
= memory_region_get_fd(mr
);
446 assert(fd_num
< VHOST_MEMORY_MAX_NREGIONS
);
447 trace_vhost_user_set_mem_table_withfd(fd_num
, mr
->name
,
449 reg
->guest_phys_addr
,
450 reg
->userspace_addr
, offset
);
451 u
->region_rb_offset
[i
] = offset
;
452 u
->region_rb
[i
] = mr
->ram_block
;
453 msg
.payload
.memory
.regions
[fd_num
].userspace_addr
=
455 msg
.payload
.memory
.regions
[fd_num
].memory_size
= reg
->memory_size
;
456 msg
.payload
.memory
.regions
[fd_num
].guest_phys_addr
=
457 reg
->guest_phys_addr
;
458 msg
.payload
.memory
.regions
[fd_num
].mmap_offset
= offset
;
461 u
->region_rb_offset
[i
] = 0;
462 u
->region_rb
[i
] = NULL
;
466 msg
.payload
.memory
.nregions
= fd_num
;
469 error_report("Failed initializing vhost-user memory map, "
470 "consider using -object memory-backend-file share=on");
474 msg
.hdr
.size
= sizeof(msg
.payload
.memory
.nregions
);
475 msg
.hdr
.size
+= sizeof(msg
.payload
.memory
.padding
);
476 msg
.hdr
.size
+= fd_num
* sizeof(VhostUserMemoryRegion
);
478 if (vhost_user_write(dev
, &msg
, fds
, fd_num
) < 0) {
482 if (vhost_user_read(dev
, &msg_reply
) < 0) {
486 if (msg_reply
.hdr
.request
!= VHOST_USER_SET_MEM_TABLE
) {
487 error_report("%s: Received unexpected msg type."
488 "Expected %d received %d", __func__
,
489 VHOST_USER_SET_MEM_TABLE
, msg_reply
.hdr
.request
);
492 /* We're using the same structure, just reusing one of the
493 * fields, so it should be the same size.
495 if (msg_reply
.hdr
.size
!= msg
.hdr
.size
) {
496 error_report("%s: Unexpected size for postcopy reply "
497 "%d vs %d", __func__
, msg_reply
.hdr
.size
, msg
.hdr
.size
);
501 memset(u
->postcopy_client_bases
, 0,
502 sizeof(uint64_t) * VHOST_MEMORY_MAX_NREGIONS
);
504 /* They're in the same order as the regions that were sent
505 * but some of the regions were skipped (above) if they
508 for (msg_i
= 0, region_i
= 0;
509 region_i
< dev
->mem
->nregions
;
511 if (msg_i
< fd_num
&&
512 msg_reply
.payload
.memory
.regions
[msg_i
].guest_phys_addr
==
513 dev
->mem
->regions
[region_i
].guest_phys_addr
) {
514 u
->postcopy_client_bases
[region_i
] =
515 msg_reply
.payload
.memory
.regions
[msg_i
].userspace_addr
;
516 trace_vhost_user_set_mem_table_postcopy(
517 msg_reply
.payload
.memory
.regions
[msg_i
].userspace_addr
,
518 msg
.payload
.memory
.regions
[msg_i
].userspace_addr
,
523 if (msg_i
!= fd_num
) {
524 error_report("%s: postcopy reply not fully consumed "
526 __func__
, msg_i
, fd_num
);
529 /* Now we've registered this with the postcopy code, we ack to the client,
530 * because now we're in the position to be able to deal with any faults
533 /* TODO: Use this for failure cases as well with a bad value */
534 msg
.hdr
.size
= sizeof(msg
.payload
.u64
);
535 msg
.payload
.u64
= 0; /* OK */
536 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
543 static int vhost_user_set_mem_table(struct vhost_dev
*dev
,
544 struct vhost_memory
*mem
)
546 struct vhost_user
*u
= dev
->opaque
;
547 int fds
[VHOST_MEMORY_MAX_NREGIONS
];
550 bool do_postcopy
= u
->postcopy_listen
&& u
->postcopy_fd
.handler
;
551 bool reply_supported
= virtio_has_feature(dev
->protocol_features
,
552 VHOST_USER_PROTOCOL_F_REPLY_ACK
);
555 /* Postcopy has enough differences that it's best done in it's own
558 return vhost_user_set_mem_table_postcopy(dev
, mem
);
562 .hdr
.request
= VHOST_USER_SET_MEM_TABLE
,
563 .hdr
.flags
= VHOST_USER_VERSION
,
566 if (reply_supported
) {
567 msg
.hdr
.flags
|= VHOST_USER_NEED_REPLY_MASK
;
570 for (i
= 0; i
< dev
->mem
->nregions
; ++i
) {
571 struct vhost_memory_region
*reg
= dev
->mem
->regions
+ i
;
575 assert((uintptr_t)reg
->userspace_addr
== reg
->userspace_addr
);
576 mr
= memory_region_from_host((void *)(uintptr_t)reg
->userspace_addr
,
578 fd
= memory_region_get_fd(mr
);
580 if (fd_num
== VHOST_MEMORY_MAX_NREGIONS
) {
581 error_report("Failed preparing vhost-user memory table msg");
584 msg
.payload
.memory
.regions
[fd_num
].userspace_addr
=
586 msg
.payload
.memory
.regions
[fd_num
].memory_size
= reg
->memory_size
;
587 msg
.payload
.memory
.regions
[fd_num
].guest_phys_addr
=
588 reg
->guest_phys_addr
;
589 msg
.payload
.memory
.regions
[fd_num
].mmap_offset
= offset
;
594 msg
.payload
.memory
.nregions
= fd_num
;
597 error_report("Failed initializing vhost-user memory map, "
598 "consider using -object memory-backend-file share=on");
602 msg
.hdr
.size
= sizeof(msg
.payload
.memory
.nregions
);
603 msg
.hdr
.size
+= sizeof(msg
.payload
.memory
.padding
);
604 msg
.hdr
.size
+= fd_num
* sizeof(VhostUserMemoryRegion
);
606 if (vhost_user_write(dev
, &msg
, fds
, fd_num
) < 0) {
610 if (reply_supported
) {
611 return process_message_reply(dev
, &msg
);
617 static int vhost_user_set_vring_addr(struct vhost_dev
*dev
,
618 struct vhost_vring_addr
*addr
)
621 .hdr
.request
= VHOST_USER_SET_VRING_ADDR
,
622 .hdr
.flags
= VHOST_USER_VERSION
,
623 .payload
.addr
= *addr
,
624 .hdr
.size
= sizeof(msg
.payload
.addr
),
627 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
634 static int vhost_user_set_vring_endian(struct vhost_dev
*dev
,
635 struct vhost_vring_state
*ring
)
637 bool cross_endian
= virtio_has_feature(dev
->protocol_features
,
638 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN
);
640 .hdr
.request
= VHOST_USER_SET_VRING_ENDIAN
,
641 .hdr
.flags
= VHOST_USER_VERSION
,
642 .payload
.state
= *ring
,
643 .hdr
.size
= sizeof(msg
.payload
.state
),
647 error_report("vhost-user trying to send unhandled ioctl");
651 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
658 static int vhost_set_vring(struct vhost_dev
*dev
,
659 unsigned long int request
,
660 struct vhost_vring_state
*ring
)
663 .hdr
.request
= request
,
664 .hdr
.flags
= VHOST_USER_VERSION
,
665 .payload
.state
= *ring
,
666 .hdr
.size
= sizeof(msg
.payload
.state
),
669 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
676 static int vhost_user_set_vring_num(struct vhost_dev
*dev
,
677 struct vhost_vring_state
*ring
)
679 return vhost_set_vring(dev
, VHOST_USER_SET_VRING_NUM
, ring
);
682 static void vhost_user_host_notifier_restore(struct vhost_dev
*dev
,
685 struct vhost_user
*u
= dev
->opaque
;
686 VhostUserHostNotifier
*n
= &u
->user
->notifier
[queue_idx
];
687 VirtIODevice
*vdev
= dev
->vdev
;
689 if (n
->addr
&& !n
->set
) {
690 virtio_queue_set_host_notifier_mr(vdev
, queue_idx
, &n
->mr
, true);
695 static void vhost_user_host_notifier_remove(struct vhost_dev
*dev
,
698 struct vhost_user
*u
= dev
->opaque
;
699 VhostUserHostNotifier
*n
= &u
->user
->notifier
[queue_idx
];
700 VirtIODevice
*vdev
= dev
->vdev
;
702 if (n
->addr
&& n
->set
) {
703 virtio_queue_set_host_notifier_mr(vdev
, queue_idx
, &n
->mr
, false);
708 static int vhost_user_set_vring_base(struct vhost_dev
*dev
,
709 struct vhost_vring_state
*ring
)
711 vhost_user_host_notifier_restore(dev
, ring
->index
);
713 return vhost_set_vring(dev
, VHOST_USER_SET_VRING_BASE
, ring
);
716 static int vhost_user_set_vring_enable(struct vhost_dev
*dev
, int enable
)
720 if (!virtio_has_feature(dev
->features
, VHOST_USER_F_PROTOCOL_FEATURES
)) {
724 for (i
= 0; i
< dev
->nvqs
; ++i
) {
725 struct vhost_vring_state state
= {
726 .index
= dev
->vq_index
+ i
,
730 vhost_set_vring(dev
, VHOST_USER_SET_VRING_ENABLE
, &state
);
736 static int vhost_user_get_vring_base(struct vhost_dev
*dev
,
737 struct vhost_vring_state
*ring
)
740 .hdr
.request
= VHOST_USER_GET_VRING_BASE
,
741 .hdr
.flags
= VHOST_USER_VERSION
,
742 .payload
.state
= *ring
,
743 .hdr
.size
= sizeof(msg
.payload
.state
),
746 vhost_user_host_notifier_remove(dev
, ring
->index
);
748 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
752 if (vhost_user_read(dev
, &msg
) < 0) {
756 if (msg
.hdr
.request
!= VHOST_USER_GET_VRING_BASE
) {
757 error_report("Received unexpected msg type. Expected %d received %d",
758 VHOST_USER_GET_VRING_BASE
, msg
.hdr
.request
);
762 if (msg
.hdr
.size
!= sizeof(msg
.payload
.state
)) {
763 error_report("Received bad msg size.");
767 *ring
= msg
.payload
.state
;
772 static int vhost_set_vring_file(struct vhost_dev
*dev
,
773 VhostUserRequest request
,
774 struct vhost_vring_file
*file
)
776 int fds
[VHOST_MEMORY_MAX_NREGIONS
];
779 .hdr
.request
= request
,
780 .hdr
.flags
= VHOST_USER_VERSION
,
781 .payload
.u64
= file
->index
& VHOST_USER_VRING_IDX_MASK
,
782 .hdr
.size
= sizeof(msg
.payload
.u64
),
785 if (ioeventfd_enabled() && file
->fd
> 0) {
786 fds
[fd_num
++] = file
->fd
;
788 msg
.payload
.u64
|= VHOST_USER_VRING_NOFD_MASK
;
791 if (vhost_user_write(dev
, &msg
, fds
, fd_num
) < 0) {
798 static int vhost_user_set_vring_kick(struct vhost_dev
*dev
,
799 struct vhost_vring_file
*file
)
801 return vhost_set_vring_file(dev
, VHOST_USER_SET_VRING_KICK
, file
);
804 static int vhost_user_set_vring_call(struct vhost_dev
*dev
,
805 struct vhost_vring_file
*file
)
807 return vhost_set_vring_file(dev
, VHOST_USER_SET_VRING_CALL
, file
);
810 static int vhost_user_set_u64(struct vhost_dev
*dev
, int request
, uint64_t u64
)
813 .hdr
.request
= request
,
814 .hdr
.flags
= VHOST_USER_VERSION
,
816 .hdr
.size
= sizeof(msg
.payload
.u64
),
819 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
826 static int vhost_user_set_features(struct vhost_dev
*dev
,
829 return vhost_user_set_u64(dev
, VHOST_USER_SET_FEATURES
, features
);
832 static int vhost_user_set_protocol_features(struct vhost_dev
*dev
,
835 return vhost_user_set_u64(dev
, VHOST_USER_SET_PROTOCOL_FEATURES
, features
);
838 static int vhost_user_get_u64(struct vhost_dev
*dev
, int request
, uint64_t *u64
)
841 .hdr
.request
= request
,
842 .hdr
.flags
= VHOST_USER_VERSION
,
845 if (vhost_user_one_time_request(request
) && dev
->vq_index
!= 0) {
849 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
853 if (vhost_user_read(dev
, &msg
) < 0) {
857 if (msg
.hdr
.request
!= request
) {
858 error_report("Received unexpected msg type. Expected %d received %d",
859 request
, msg
.hdr
.request
);
863 if (msg
.hdr
.size
!= sizeof(msg
.payload
.u64
)) {
864 error_report("Received bad msg size.");
868 *u64
= msg
.payload
.u64
;
873 static int vhost_user_get_features(struct vhost_dev
*dev
, uint64_t *features
)
875 return vhost_user_get_u64(dev
, VHOST_USER_GET_FEATURES
, features
);
878 static int vhost_user_set_owner(struct vhost_dev
*dev
)
881 .hdr
.request
= VHOST_USER_SET_OWNER
,
882 .hdr
.flags
= VHOST_USER_VERSION
,
885 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
892 static int vhost_user_reset_device(struct vhost_dev
*dev
)
895 .hdr
.flags
= VHOST_USER_VERSION
,
898 msg
.hdr
.request
= virtio_has_feature(dev
->protocol_features
,
899 VHOST_USER_PROTOCOL_F_RESET_DEVICE
)
900 ? VHOST_USER_RESET_DEVICE
901 : VHOST_USER_RESET_OWNER
;
903 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
910 static int vhost_user_slave_handle_config_change(struct vhost_dev
*dev
)
914 if (!dev
->config_ops
) {
918 if (dev
->config_ops
->vhost_dev_config_notifier
) {
919 ret
= dev
->config_ops
->vhost_dev_config_notifier(dev
);
925 static int vhost_user_slave_handle_vring_host_notifier(struct vhost_dev
*dev
,
926 VhostUserVringArea
*area
,
929 int queue_idx
= area
->u64
& VHOST_USER_VRING_IDX_MASK
;
930 size_t page_size
= qemu_real_host_page_size
;
931 struct vhost_user
*u
= dev
->opaque
;
932 VhostUserState
*user
= u
->user
;
933 VirtIODevice
*vdev
= dev
->vdev
;
934 VhostUserHostNotifier
*n
;
938 if (!virtio_has_feature(dev
->protocol_features
,
939 VHOST_USER_PROTOCOL_F_HOST_NOTIFIER
) ||
940 vdev
== NULL
|| queue_idx
>= virtio_get_num_queues(vdev
)) {
944 n
= &user
->notifier
[queue_idx
];
947 virtio_queue_set_host_notifier_mr(vdev
, queue_idx
, &n
->mr
, false);
948 object_unparent(OBJECT(&n
->mr
));
949 munmap(n
->addr
, page_size
);
953 if (area
->u64
& VHOST_USER_VRING_NOFD_MASK
) {
958 if (area
->size
!= page_size
) {
962 addr
= mmap(NULL
, page_size
, PROT_READ
| PROT_WRITE
, MAP_SHARED
,
964 if (addr
== MAP_FAILED
) {
968 name
= g_strdup_printf("vhost-user/host-notifier@%p mmaps[%d]",
970 memory_region_init_ram_device_ptr(&n
->mr
, OBJECT(vdev
), name
,
974 if (virtio_queue_set_host_notifier_mr(vdev
, queue_idx
, &n
->mr
, true)) {
975 munmap(addr
, page_size
);
985 static void slave_read(void *opaque
)
987 struct vhost_dev
*dev
= opaque
;
988 struct vhost_user
*u
= dev
->opaque
;
989 VhostUserHeader hdr
= { 0, };
990 VhostUserPayload payload
= { 0, };
994 int fd
[VHOST_USER_SLAVE_MAX_FDS
];
995 char control
[CMSG_SPACE(sizeof(fd
))];
996 struct cmsghdr
*cmsg
;
999 memset(&msgh
, 0, sizeof(msgh
));
1000 msgh
.msg_iov
= &iov
;
1001 msgh
.msg_iovlen
= 1;
1002 msgh
.msg_control
= control
;
1003 msgh
.msg_controllen
= sizeof(control
);
1005 memset(fd
, -1, sizeof(fd
));
1008 iov
.iov_base
= &hdr
;
1009 iov
.iov_len
= VHOST_USER_HDR_SIZE
;
1012 size
= recvmsg(u
->slave_fd
, &msgh
, 0);
1013 } while (size
< 0 && (errno
== EINTR
|| errno
== EAGAIN
));
1015 if (size
!= VHOST_USER_HDR_SIZE
) {
1016 error_report("Failed to read from slave.");
1020 if (msgh
.msg_flags
& MSG_CTRUNC
) {
1021 error_report("Truncated message.");
1025 for (cmsg
= CMSG_FIRSTHDR(&msgh
); cmsg
!= NULL
;
1026 cmsg
= CMSG_NXTHDR(&msgh
, cmsg
)) {
1027 if (cmsg
->cmsg_level
== SOL_SOCKET
&&
1028 cmsg
->cmsg_type
== SCM_RIGHTS
) {
1029 fdsize
= cmsg
->cmsg_len
- CMSG_LEN(0);
1030 memcpy(fd
, CMSG_DATA(cmsg
), fdsize
);
1035 if (hdr
.size
> VHOST_USER_PAYLOAD_SIZE
) {
1036 error_report("Failed to read msg header."
1037 " Size %d exceeds the maximum %zu.", hdr
.size
,
1038 VHOST_USER_PAYLOAD_SIZE
);
1044 size
= read(u
->slave_fd
, &payload
, hdr
.size
);
1045 } while (size
< 0 && (errno
== EINTR
|| errno
== EAGAIN
));
1047 if (size
!= hdr
.size
) {
1048 error_report("Failed to read payload from slave.");
1052 switch (hdr
.request
) {
1053 case VHOST_USER_SLAVE_IOTLB_MSG
:
1054 ret
= vhost_backend_handle_iotlb_msg(dev
, &payload
.iotlb
);
1056 case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG
:
1057 ret
= vhost_user_slave_handle_config_change(dev
);
1059 case VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG
:
1060 ret
= vhost_user_slave_handle_vring_host_notifier(dev
, &payload
.area
,
1064 error_report("Received unexpected msg type: %d.", hdr
.request
);
1068 /* Close the remaining file descriptors. */
1069 for (i
= 0; i
< fdsize
; i
++) {
1076 * REPLY_ACK feature handling. Other reply types has to be managed
1077 * directly in their request handlers.
1079 if (hdr
.flags
& VHOST_USER_NEED_REPLY_MASK
) {
1080 struct iovec iovec
[2];
1083 hdr
.flags
&= ~VHOST_USER_NEED_REPLY_MASK
;
1084 hdr
.flags
|= VHOST_USER_REPLY_MASK
;
1086 payload
.u64
= !!ret
;
1087 hdr
.size
= sizeof(payload
.u64
);
1089 iovec
[0].iov_base
= &hdr
;
1090 iovec
[0].iov_len
= VHOST_USER_HDR_SIZE
;
1091 iovec
[1].iov_base
= &payload
;
1092 iovec
[1].iov_len
= hdr
.size
;
1095 size
= writev(u
->slave_fd
, iovec
, ARRAY_SIZE(iovec
));
1096 } while (size
< 0 && (errno
== EINTR
|| errno
== EAGAIN
));
1098 if (size
!= VHOST_USER_HDR_SIZE
+ hdr
.size
) {
1099 error_report("Failed to send msg reply to slave.");
1107 qemu_set_fd_handler(u
->slave_fd
, NULL
, NULL
, NULL
);
1110 for (i
= 0; i
< fdsize
; i
++) {
1118 static int vhost_setup_slave_channel(struct vhost_dev
*dev
)
1120 VhostUserMsg msg
= {
1121 .hdr
.request
= VHOST_USER_SET_SLAVE_REQ_FD
,
1122 .hdr
.flags
= VHOST_USER_VERSION
,
1124 struct vhost_user
*u
= dev
->opaque
;
1126 bool reply_supported
= virtio_has_feature(dev
->protocol_features
,
1127 VHOST_USER_PROTOCOL_F_REPLY_ACK
);
1129 if (!virtio_has_feature(dev
->protocol_features
,
1130 VHOST_USER_PROTOCOL_F_SLAVE_REQ
)) {
1134 if (socketpair(PF_UNIX
, SOCK_STREAM
, 0, sv
) == -1) {
1135 error_report("socketpair() failed");
1139 u
->slave_fd
= sv
[0];
1140 qemu_set_fd_handler(u
->slave_fd
, slave_read
, NULL
, dev
);
1142 if (reply_supported
) {
1143 msg
.hdr
.flags
|= VHOST_USER_NEED_REPLY_MASK
;
1146 ret
= vhost_user_write(dev
, &msg
, &sv
[1], 1);
1151 if (reply_supported
) {
1152 ret
= process_message_reply(dev
, &msg
);
1158 qemu_set_fd_handler(u
->slave_fd
, NULL
, NULL
, NULL
);
1168 * Called back from the postcopy fault thread when a fault is received on our
1170 * TODO: This is Linux specific
1172 static int vhost_user_postcopy_fault_handler(struct PostCopyFD
*pcfd
,
1175 struct vhost_dev
*dev
= pcfd
->data
;
1176 struct vhost_user
*u
= dev
->opaque
;
1177 struct uffd_msg
*msg
= ufd
;
1178 uint64_t faultaddr
= msg
->arg
.pagefault
.address
;
1179 RAMBlock
*rb
= NULL
;
1183 trace_vhost_user_postcopy_fault_handler(pcfd
->idstr
, faultaddr
,
1184 dev
->mem
->nregions
);
1185 for (i
= 0; i
< MIN(dev
->mem
->nregions
, u
->region_rb_len
); i
++) {
1186 trace_vhost_user_postcopy_fault_handler_loop(i
,
1187 u
->postcopy_client_bases
[i
], dev
->mem
->regions
[i
].memory_size
);
1188 if (faultaddr
>= u
->postcopy_client_bases
[i
]) {
1189 /* Ofset of the fault address in the vhost region */
1190 uint64_t region_offset
= faultaddr
- u
->postcopy_client_bases
[i
];
1191 if (region_offset
< dev
->mem
->regions
[i
].memory_size
) {
1192 rb_offset
= region_offset
+ u
->region_rb_offset
[i
];
1193 trace_vhost_user_postcopy_fault_handler_found(i
,
1194 region_offset
, rb_offset
);
1195 rb
= u
->region_rb
[i
];
1196 return postcopy_request_shared_page(pcfd
, rb
, faultaddr
,
1201 error_report("%s: Failed to find region for fault %" PRIx64
,
1202 __func__
, faultaddr
);
1206 static int vhost_user_postcopy_waker(struct PostCopyFD
*pcfd
, RAMBlock
*rb
,
1209 struct vhost_dev
*dev
= pcfd
->data
;
1210 struct vhost_user
*u
= dev
->opaque
;
1213 trace_vhost_user_postcopy_waker(qemu_ram_get_idstr(rb
), offset
);
1218 /* Translate the offset into an address in the clients address space */
1219 for (i
= 0; i
< MIN(dev
->mem
->nregions
, u
->region_rb_len
); i
++) {
1220 if (u
->region_rb
[i
] == rb
&&
1221 offset
>= u
->region_rb_offset
[i
] &&
1222 offset
< (u
->region_rb_offset
[i
] +
1223 dev
->mem
->regions
[i
].memory_size
)) {
1224 uint64_t client_addr
= (offset
- u
->region_rb_offset
[i
]) +
1225 u
->postcopy_client_bases
[i
];
1226 trace_vhost_user_postcopy_waker_found(client_addr
);
1227 return postcopy_wake_shared(pcfd
, client_addr
, rb
);
1231 trace_vhost_user_postcopy_waker_nomatch(qemu_ram_get_idstr(rb
), offset
);
1237 * Called at the start of an inbound postcopy on reception of the
1240 static int vhost_user_postcopy_advise(struct vhost_dev
*dev
, Error
**errp
)
1243 struct vhost_user
*u
= dev
->opaque
;
1244 CharBackend
*chr
= u
->user
->chr
;
1246 VhostUserMsg msg
= {
1247 .hdr
.request
= VHOST_USER_POSTCOPY_ADVISE
,
1248 .hdr
.flags
= VHOST_USER_VERSION
,
1251 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
1252 error_setg(errp
, "Failed to send postcopy_advise to vhost");
1256 if (vhost_user_read(dev
, &msg
) < 0) {
1257 error_setg(errp
, "Failed to get postcopy_advise reply from vhost");
1261 if (msg
.hdr
.request
!= VHOST_USER_POSTCOPY_ADVISE
) {
1262 error_setg(errp
, "Unexpected msg type. Expected %d received %d",
1263 VHOST_USER_POSTCOPY_ADVISE
, msg
.hdr
.request
);
1268 error_setg(errp
, "Received bad msg size.");
1271 ufd
= qemu_chr_fe_get_msgfd(chr
);
1273 error_setg(errp
, "%s: Failed to get ufd", __func__
);
1276 qemu_set_nonblock(ufd
);
1278 /* register ufd with userfault thread */
1279 u
->postcopy_fd
.fd
= ufd
;
1280 u
->postcopy_fd
.data
= dev
;
1281 u
->postcopy_fd
.handler
= vhost_user_postcopy_fault_handler
;
1282 u
->postcopy_fd
.waker
= vhost_user_postcopy_waker
;
1283 u
->postcopy_fd
.idstr
= "vhost-user"; /* Need to find unique name */
1284 postcopy_register_shared_ufd(&u
->postcopy_fd
);
1287 error_setg(errp
, "Postcopy not supported on non-Linux systems");
1293 * Called at the switch to postcopy on reception of the 'listen' command.
1295 static int vhost_user_postcopy_listen(struct vhost_dev
*dev
, Error
**errp
)
1297 struct vhost_user
*u
= dev
->opaque
;
1299 VhostUserMsg msg
= {
1300 .hdr
.request
= VHOST_USER_POSTCOPY_LISTEN
,
1301 .hdr
.flags
= VHOST_USER_VERSION
| VHOST_USER_NEED_REPLY_MASK
,
1303 u
->postcopy_listen
= true;
1304 trace_vhost_user_postcopy_listen();
1305 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
1306 error_setg(errp
, "Failed to send postcopy_listen to vhost");
1310 ret
= process_message_reply(dev
, &msg
);
1312 error_setg(errp
, "Failed to receive reply to postcopy_listen");
1320 * Called at the end of postcopy
1322 static int vhost_user_postcopy_end(struct vhost_dev
*dev
, Error
**errp
)
1324 VhostUserMsg msg
= {
1325 .hdr
.request
= VHOST_USER_POSTCOPY_END
,
1326 .hdr
.flags
= VHOST_USER_VERSION
| VHOST_USER_NEED_REPLY_MASK
,
1329 struct vhost_user
*u
= dev
->opaque
;
1331 trace_vhost_user_postcopy_end_entry();
1332 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
1333 error_setg(errp
, "Failed to send postcopy_end to vhost");
1337 ret
= process_message_reply(dev
, &msg
);
1339 error_setg(errp
, "Failed to receive reply to postcopy_end");
1342 postcopy_unregister_shared_ufd(&u
->postcopy_fd
);
1343 close(u
->postcopy_fd
.fd
);
1344 u
->postcopy_fd
.handler
= NULL
;
1346 trace_vhost_user_postcopy_end_exit();
1351 static int vhost_user_postcopy_notifier(NotifierWithReturn
*notifier
,
1354 struct PostcopyNotifyData
*pnd
= opaque
;
1355 struct vhost_user
*u
= container_of(notifier
, struct vhost_user
,
1357 struct vhost_dev
*dev
= u
->dev
;
1359 switch (pnd
->reason
) {
1360 case POSTCOPY_NOTIFY_PROBE
:
1361 if (!virtio_has_feature(dev
->protocol_features
,
1362 VHOST_USER_PROTOCOL_F_PAGEFAULT
)) {
1363 /* TODO: Get the device name into this error somehow */
1364 error_setg(pnd
->errp
,
1365 "vhost-user backend not capable of postcopy");
1370 case POSTCOPY_NOTIFY_INBOUND_ADVISE
:
1371 return vhost_user_postcopy_advise(dev
, pnd
->errp
);
1373 case POSTCOPY_NOTIFY_INBOUND_LISTEN
:
1374 return vhost_user_postcopy_listen(dev
, pnd
->errp
);
1376 case POSTCOPY_NOTIFY_INBOUND_END
:
1377 return vhost_user_postcopy_end(dev
, pnd
->errp
);
1380 /* We ignore notifications we don't know */
1387 static int vhost_user_backend_init(struct vhost_dev
*dev
, void *opaque
)
1389 uint64_t features
, protocol_features
;
1390 struct vhost_user
*u
;
1393 assert(dev
->vhost_ops
->backend_type
== VHOST_BACKEND_TYPE_USER
);
1395 u
= g_new0(struct vhost_user
, 1);
1401 err
= vhost_user_get_features(dev
, &features
);
1406 if (virtio_has_feature(features
, VHOST_USER_F_PROTOCOL_FEATURES
)) {
1407 dev
->backend_features
|= 1ULL << VHOST_USER_F_PROTOCOL_FEATURES
;
1409 err
= vhost_user_get_u64(dev
, VHOST_USER_GET_PROTOCOL_FEATURES
,
1410 &protocol_features
);
1415 dev
->protocol_features
=
1416 protocol_features
& VHOST_USER_PROTOCOL_FEATURE_MASK
;
1418 if (!dev
->config_ops
|| !dev
->config_ops
->vhost_dev_config_notifier
) {
1419 /* Don't acknowledge CONFIG feature if device doesn't support it */
1420 dev
->protocol_features
&= ~(1ULL << VHOST_USER_PROTOCOL_F_CONFIG
);
1421 } else if (!(protocol_features
&
1422 (1ULL << VHOST_USER_PROTOCOL_F_CONFIG
))) {
1423 error_report("Device expects VHOST_USER_PROTOCOL_F_CONFIG "
1424 "but backend does not support it.");
1428 err
= vhost_user_set_protocol_features(dev
, dev
->protocol_features
);
1433 /* query the max queues we support if backend supports Multiple Queue */
1434 if (dev
->protocol_features
& (1ULL << VHOST_USER_PROTOCOL_F_MQ
)) {
1435 err
= vhost_user_get_u64(dev
, VHOST_USER_GET_QUEUE_NUM
,
1442 if (virtio_has_feature(features
, VIRTIO_F_IOMMU_PLATFORM
) &&
1443 !(virtio_has_feature(dev
->protocol_features
,
1444 VHOST_USER_PROTOCOL_F_SLAVE_REQ
) &&
1445 virtio_has_feature(dev
->protocol_features
,
1446 VHOST_USER_PROTOCOL_F_REPLY_ACK
))) {
1447 error_report("IOMMU support requires reply-ack and "
1448 "slave-req protocol features.");
1453 if (dev
->migration_blocker
== NULL
&&
1454 !virtio_has_feature(dev
->protocol_features
,
1455 VHOST_USER_PROTOCOL_F_LOG_SHMFD
)) {
1456 error_setg(&dev
->migration_blocker
,
1457 "Migration disabled: vhost-user backend lacks "
1458 "VHOST_USER_PROTOCOL_F_LOG_SHMFD feature.");
1461 if (dev
->vq_index
== 0) {
1462 err
= vhost_setup_slave_channel(dev
);
1468 u
->postcopy_notifier
.notify
= vhost_user_postcopy_notifier
;
1469 postcopy_add_notifier(&u
->postcopy_notifier
);
1474 static int vhost_user_backend_cleanup(struct vhost_dev
*dev
)
1476 struct vhost_user
*u
;
1478 assert(dev
->vhost_ops
->backend_type
== VHOST_BACKEND_TYPE_USER
);
1481 if (u
->postcopy_notifier
.notify
) {
1482 postcopy_remove_notifier(&u
->postcopy_notifier
);
1483 u
->postcopy_notifier
.notify
= NULL
;
1485 u
->postcopy_listen
= false;
1486 if (u
->postcopy_fd
.handler
) {
1487 postcopy_unregister_shared_ufd(&u
->postcopy_fd
);
1488 close(u
->postcopy_fd
.fd
);
1489 u
->postcopy_fd
.handler
= NULL
;
1491 if (u
->slave_fd
>= 0) {
1492 qemu_set_fd_handler(u
->slave_fd
, NULL
, NULL
, NULL
);
1496 g_free(u
->region_rb
);
1497 u
->region_rb
= NULL
;
1498 g_free(u
->region_rb_offset
);
1499 u
->region_rb_offset
= NULL
;
1500 u
->region_rb_len
= 0;
1507 static int vhost_user_get_vq_index(struct vhost_dev
*dev
, int idx
)
1509 assert(idx
>= dev
->vq_index
&& idx
< dev
->vq_index
+ dev
->nvqs
);
1514 static int vhost_user_memslots_limit(struct vhost_dev
*dev
)
1516 return VHOST_MEMORY_MAX_NREGIONS
;
1519 static bool vhost_user_requires_shm_log(struct vhost_dev
*dev
)
1521 assert(dev
->vhost_ops
->backend_type
== VHOST_BACKEND_TYPE_USER
);
1523 return virtio_has_feature(dev
->protocol_features
,
1524 VHOST_USER_PROTOCOL_F_LOG_SHMFD
);
1527 static int vhost_user_migration_done(struct vhost_dev
*dev
, char* mac_addr
)
1529 VhostUserMsg msg
= { };
1531 assert(dev
->vhost_ops
->backend_type
== VHOST_BACKEND_TYPE_USER
);
1533 /* If guest supports GUEST_ANNOUNCE do nothing */
1534 if (virtio_has_feature(dev
->acked_features
, VIRTIO_NET_F_GUEST_ANNOUNCE
)) {
1538 /* if backend supports VHOST_USER_PROTOCOL_F_RARP ask it to send the RARP */
1539 if (virtio_has_feature(dev
->protocol_features
,
1540 VHOST_USER_PROTOCOL_F_RARP
)) {
1541 msg
.hdr
.request
= VHOST_USER_SEND_RARP
;
1542 msg
.hdr
.flags
= VHOST_USER_VERSION
;
1543 memcpy((char *)&msg
.payload
.u64
, mac_addr
, 6);
1544 msg
.hdr
.size
= sizeof(msg
.payload
.u64
);
1546 return vhost_user_write(dev
, &msg
, NULL
, 0);
1551 static bool vhost_user_can_merge(struct vhost_dev
*dev
,
1552 uint64_t start1
, uint64_t size1
,
1553 uint64_t start2
, uint64_t size2
)
1559 mr
= memory_region_from_host((void *)(uintptr_t)start1
, &offset
);
1560 mfd
= memory_region_get_fd(mr
);
1562 mr
= memory_region_from_host((void *)(uintptr_t)start2
, &offset
);
1563 rfd
= memory_region_get_fd(mr
);
1568 static int vhost_user_net_set_mtu(struct vhost_dev
*dev
, uint16_t mtu
)
1571 bool reply_supported
= virtio_has_feature(dev
->protocol_features
,
1572 VHOST_USER_PROTOCOL_F_REPLY_ACK
);
1574 if (!(dev
->protocol_features
& (1ULL << VHOST_USER_PROTOCOL_F_NET_MTU
))) {
1578 msg
.hdr
.request
= VHOST_USER_NET_SET_MTU
;
1579 msg
.payload
.u64
= mtu
;
1580 msg
.hdr
.size
= sizeof(msg
.payload
.u64
);
1581 msg
.hdr
.flags
= VHOST_USER_VERSION
;
1582 if (reply_supported
) {
1583 msg
.hdr
.flags
|= VHOST_USER_NEED_REPLY_MASK
;
1586 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
1590 /* If reply_ack supported, slave has to ack specified MTU is valid */
1591 if (reply_supported
) {
1592 return process_message_reply(dev
, &msg
);
1598 static int vhost_user_send_device_iotlb_msg(struct vhost_dev
*dev
,
1599 struct vhost_iotlb_msg
*imsg
)
1601 VhostUserMsg msg
= {
1602 .hdr
.request
= VHOST_USER_IOTLB_MSG
,
1603 .hdr
.size
= sizeof(msg
.payload
.iotlb
),
1604 .hdr
.flags
= VHOST_USER_VERSION
| VHOST_USER_NEED_REPLY_MASK
,
1605 .payload
.iotlb
= *imsg
,
1608 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
1612 return process_message_reply(dev
, &msg
);
1616 static void vhost_user_set_iotlb_callback(struct vhost_dev
*dev
, int enabled
)
1618 /* No-op as the receive channel is not dedicated to IOTLB messages. */
1621 static int vhost_user_get_config(struct vhost_dev
*dev
, uint8_t *config
,
1622 uint32_t config_len
)
1624 VhostUserMsg msg
= {
1625 .hdr
.request
= VHOST_USER_GET_CONFIG
,
1626 .hdr
.flags
= VHOST_USER_VERSION
,
1627 .hdr
.size
= VHOST_USER_CONFIG_HDR_SIZE
+ config_len
,
1630 if (!virtio_has_feature(dev
->protocol_features
,
1631 VHOST_USER_PROTOCOL_F_CONFIG
)) {
1635 if (config_len
> VHOST_USER_MAX_CONFIG_SIZE
) {
1639 msg
.payload
.config
.offset
= 0;
1640 msg
.payload
.config
.size
= config_len
;
1641 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
1645 if (vhost_user_read(dev
, &msg
) < 0) {
1649 if (msg
.hdr
.request
!= VHOST_USER_GET_CONFIG
) {
1650 error_report("Received unexpected msg type. Expected %d received %d",
1651 VHOST_USER_GET_CONFIG
, msg
.hdr
.request
);
1655 if (msg
.hdr
.size
!= VHOST_USER_CONFIG_HDR_SIZE
+ config_len
) {
1656 error_report("Received bad msg size.");
1660 memcpy(config
, msg
.payload
.config
.region
, config_len
);
1665 static int vhost_user_set_config(struct vhost_dev
*dev
, const uint8_t *data
,
1666 uint32_t offset
, uint32_t size
, uint32_t flags
)
1669 bool reply_supported
= virtio_has_feature(dev
->protocol_features
,
1670 VHOST_USER_PROTOCOL_F_REPLY_ACK
);
1672 VhostUserMsg msg
= {
1673 .hdr
.request
= VHOST_USER_SET_CONFIG
,
1674 .hdr
.flags
= VHOST_USER_VERSION
,
1675 .hdr
.size
= VHOST_USER_CONFIG_HDR_SIZE
+ size
,
1678 if (!virtio_has_feature(dev
->protocol_features
,
1679 VHOST_USER_PROTOCOL_F_CONFIG
)) {
1683 if (reply_supported
) {
1684 msg
.hdr
.flags
|= VHOST_USER_NEED_REPLY_MASK
;
1687 if (size
> VHOST_USER_MAX_CONFIG_SIZE
) {
1691 msg
.payload
.config
.offset
= offset
,
1692 msg
.payload
.config
.size
= size
,
1693 msg
.payload
.config
.flags
= flags
,
1694 p
= msg
.payload
.config
.region
;
1695 memcpy(p
, data
, size
);
1697 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
1701 if (reply_supported
) {
1702 return process_message_reply(dev
, &msg
);
1708 static int vhost_user_crypto_create_session(struct vhost_dev
*dev
,
1710 uint64_t *session_id
)
1712 bool crypto_session
= virtio_has_feature(dev
->protocol_features
,
1713 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION
);
1714 CryptoDevBackendSymSessionInfo
*sess_info
= session_info
;
1715 VhostUserMsg msg
= {
1716 .hdr
.request
= VHOST_USER_CREATE_CRYPTO_SESSION
,
1717 .hdr
.flags
= VHOST_USER_VERSION
,
1718 .hdr
.size
= sizeof(msg
.payload
.session
),
1721 assert(dev
->vhost_ops
->backend_type
== VHOST_BACKEND_TYPE_USER
);
1723 if (!crypto_session
) {
1724 error_report("vhost-user trying to send unhandled ioctl");
1728 memcpy(&msg
.payload
.session
.session_setup_data
, sess_info
,
1729 sizeof(CryptoDevBackendSymSessionInfo
));
1730 if (sess_info
->key_len
) {
1731 memcpy(&msg
.payload
.session
.key
, sess_info
->cipher_key
,
1732 sess_info
->key_len
);
1734 if (sess_info
->auth_key_len
> 0) {
1735 memcpy(&msg
.payload
.session
.auth_key
, sess_info
->auth_key
,
1736 sess_info
->auth_key_len
);
1738 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
1739 error_report("vhost_user_write() return -1, create session failed");
1743 if (vhost_user_read(dev
, &msg
) < 0) {
1744 error_report("vhost_user_read() return -1, create session failed");
1748 if (msg
.hdr
.request
!= VHOST_USER_CREATE_CRYPTO_SESSION
) {
1749 error_report("Received unexpected msg type. Expected %d received %d",
1750 VHOST_USER_CREATE_CRYPTO_SESSION
, msg
.hdr
.request
);
1754 if (msg
.hdr
.size
!= sizeof(msg
.payload
.session
)) {
1755 error_report("Received bad msg size.");
1759 if (msg
.payload
.session
.session_id
< 0) {
1760 error_report("Bad session id: %" PRId64
"",
1761 msg
.payload
.session
.session_id
);
1764 *session_id
= msg
.payload
.session
.session_id
;
1770 vhost_user_crypto_close_session(struct vhost_dev
*dev
, uint64_t session_id
)
1772 bool crypto_session
= virtio_has_feature(dev
->protocol_features
,
1773 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION
);
1774 VhostUserMsg msg
= {
1775 .hdr
.request
= VHOST_USER_CLOSE_CRYPTO_SESSION
,
1776 .hdr
.flags
= VHOST_USER_VERSION
,
1777 .hdr
.size
= sizeof(msg
.payload
.u64
),
1779 msg
.payload
.u64
= session_id
;
1781 if (!crypto_session
) {
1782 error_report("vhost-user trying to send unhandled ioctl");
1786 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
1787 error_report("vhost_user_write() return -1, close session failed");
1794 static bool vhost_user_mem_section_filter(struct vhost_dev
*dev
,
1795 MemoryRegionSection
*section
)
1799 result
= memory_region_get_fd(section
->mr
) >= 0;
1804 static int vhost_user_get_inflight_fd(struct vhost_dev
*dev
,
1805 uint16_t queue_size
,
1806 struct vhost_inflight
*inflight
)
1810 struct vhost_user
*u
= dev
->opaque
;
1811 CharBackend
*chr
= u
->user
->chr
;
1812 VhostUserMsg msg
= {
1813 .hdr
.request
= VHOST_USER_GET_INFLIGHT_FD
,
1814 .hdr
.flags
= VHOST_USER_VERSION
,
1815 .payload
.inflight
.num_queues
= dev
->nvqs
,
1816 .payload
.inflight
.queue_size
= queue_size
,
1817 .hdr
.size
= sizeof(msg
.payload
.inflight
),
1820 if (!virtio_has_feature(dev
->protocol_features
,
1821 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD
)) {
1825 if (vhost_user_write(dev
, &msg
, NULL
, 0) < 0) {
1829 if (vhost_user_read(dev
, &msg
) < 0) {
1833 if (msg
.hdr
.request
!= VHOST_USER_GET_INFLIGHT_FD
) {
1834 error_report("Received unexpected msg type. "
1835 "Expected %d received %d",
1836 VHOST_USER_GET_INFLIGHT_FD
, msg
.hdr
.request
);
1840 if (msg
.hdr
.size
!= sizeof(msg
.payload
.inflight
)) {
1841 error_report("Received bad msg size.");
1845 if (!msg
.payload
.inflight
.mmap_size
) {
1849 fd
= qemu_chr_fe_get_msgfd(chr
);
1851 error_report("Failed to get mem fd");
1855 addr
= mmap(0, msg
.payload
.inflight
.mmap_size
, PROT_READ
| PROT_WRITE
,
1856 MAP_SHARED
, fd
, msg
.payload
.inflight
.mmap_offset
);
1858 if (addr
== MAP_FAILED
) {
1859 error_report("Failed to mmap mem fd");
1864 inflight
->addr
= addr
;
1866 inflight
->size
= msg
.payload
.inflight
.mmap_size
;
1867 inflight
->offset
= msg
.payload
.inflight
.mmap_offset
;
1868 inflight
->queue_size
= queue_size
;
1873 static int vhost_user_set_inflight_fd(struct vhost_dev
*dev
,
1874 struct vhost_inflight
*inflight
)
1876 VhostUserMsg msg
= {
1877 .hdr
.request
= VHOST_USER_SET_INFLIGHT_FD
,
1878 .hdr
.flags
= VHOST_USER_VERSION
,
1879 .payload
.inflight
.mmap_size
= inflight
->size
,
1880 .payload
.inflight
.mmap_offset
= inflight
->offset
,
1881 .payload
.inflight
.num_queues
= dev
->nvqs
,
1882 .payload
.inflight
.queue_size
= inflight
->queue_size
,
1883 .hdr
.size
= sizeof(msg
.payload
.inflight
),
1886 if (!virtio_has_feature(dev
->protocol_features
,
1887 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD
)) {
1891 if (vhost_user_write(dev
, &msg
, &inflight
->fd
, 1) < 0) {
1898 bool vhost_user_init(VhostUserState
*user
, CharBackend
*chr
, Error
**errp
)
1901 error_setg(errp
, "Cannot initialize vhost-user state");
1908 void vhost_user_cleanup(VhostUserState
*user
)
1916 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
1917 if (user
->notifier
[i
].addr
) {
1918 object_unparent(OBJECT(&user
->notifier
[i
].mr
));
1919 munmap(user
->notifier
[i
].addr
, qemu_real_host_page_size
);
1920 user
->notifier
[i
].addr
= NULL
;
1926 const VhostOps user_ops
= {
1927 .backend_type
= VHOST_BACKEND_TYPE_USER
,
1928 .vhost_backend_init
= vhost_user_backend_init
,
1929 .vhost_backend_cleanup
= vhost_user_backend_cleanup
,
1930 .vhost_backend_memslots_limit
= vhost_user_memslots_limit
,
1931 .vhost_set_log_base
= vhost_user_set_log_base
,
1932 .vhost_set_mem_table
= vhost_user_set_mem_table
,
1933 .vhost_set_vring_addr
= vhost_user_set_vring_addr
,
1934 .vhost_set_vring_endian
= vhost_user_set_vring_endian
,
1935 .vhost_set_vring_num
= vhost_user_set_vring_num
,
1936 .vhost_set_vring_base
= vhost_user_set_vring_base
,
1937 .vhost_get_vring_base
= vhost_user_get_vring_base
,
1938 .vhost_set_vring_kick
= vhost_user_set_vring_kick
,
1939 .vhost_set_vring_call
= vhost_user_set_vring_call
,
1940 .vhost_set_features
= vhost_user_set_features
,
1941 .vhost_get_features
= vhost_user_get_features
,
1942 .vhost_set_owner
= vhost_user_set_owner
,
1943 .vhost_reset_device
= vhost_user_reset_device
,
1944 .vhost_get_vq_index
= vhost_user_get_vq_index
,
1945 .vhost_set_vring_enable
= vhost_user_set_vring_enable
,
1946 .vhost_requires_shm_log
= vhost_user_requires_shm_log
,
1947 .vhost_migration_done
= vhost_user_migration_done
,
1948 .vhost_backend_can_merge
= vhost_user_can_merge
,
1949 .vhost_net_set_mtu
= vhost_user_net_set_mtu
,
1950 .vhost_set_iotlb_callback
= vhost_user_set_iotlb_callback
,
1951 .vhost_send_device_iotlb_msg
= vhost_user_send_device_iotlb_msg
,
1952 .vhost_get_config
= vhost_user_get_config
,
1953 .vhost_set_config
= vhost_user_set_config
,
1954 .vhost_crypto_create_session
= vhost_user_crypto_create_session
,
1955 .vhost_crypto_close_session
= vhost_user_crypto_close_session
,
1956 .vhost_backend_mem_section_filter
= vhost_user_mem_section_filter
,
1957 .vhost_get_inflight_fd
= vhost_user_get_inflight_fd
,
1958 .vhost_set_inflight_fd
= vhost_user_set_inflight_fd
,