4 * Copyright IBM, Corp. 2007
5 * Copyright (c) 2016 Red Hat, Inc.
8 * Anthony Liguori <aliguori@us.ibm.com>
9 * Marc-André Lureau <mlureau@redhat.com>
10 * Victor Kaplansky <victork@redhat.com>
12 * This work is licensed under the terms of the GNU GPL, version 2 or
13 * later. See the COPYING file in the top-level directory.
20 /* this code avoids GLib dependency */
29 #include <sys/types.h>
30 #include <sys/socket.h>
31 #include <sys/eventfd.h>
35 /* Necessary to provide VIRTIO_F_VERSION_1 on system
36 * with older linux headers. Must appear before
37 * <linux/vhost.h> below.
39 #include "standard-headers/linux/virtio_config.h"
41 #if defined(__linux__)
42 #include <sys/syscall.h>
44 #include <sys/ioctl.h>
45 #include <linux/vhost.h>
47 #include <linux/magic.h>
49 #ifdef __NR_userfaultfd
50 #include <linux/userfaultfd.h>
55 #include "include/atomic.h"
57 #include "libvhost-user.h"
59 /* usually provided by GLib */
60 #if __GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ > 4)
61 #if !defined(__clang__) && (__GNUC__ == 4 && __GNUC_MINOR__ == 4)
62 #define G_GNUC_PRINTF(format_idx, arg_idx) \
63 __attribute__((__format__(gnu_printf, format_idx, arg_idx)))
65 #define G_GNUC_PRINTF(format_idx, arg_idx) \
66 __attribute__((__format__(__printf__, format_idx, arg_idx)))
69 #define G_GNUC_PRINTF(format_idx, arg_idx)
70 #endif /* !__GNUC__ */
72 #define MIN(x, y) ({ \
73 __typeof__(x) _min1 = (x); \
74 __typeof__(y) _min2 = (y); \
75 (void) (&_min1 == &_min2); \
76 _min1 < _min2 ? _min1 : _min2; })
79 /* Round number down to multiple */
80 #define ALIGN_DOWN(n, m) ((n) / (m) * (m))
82 /* Round number up to multiple */
83 #define ALIGN_UP(n, m) ALIGN_DOWN((n) + (m) - 1, (m))
86 #define unlikely(x) __builtin_expect(!!(x), 0)
89 /* Align each region to cache line size in inflight buffer */
90 #define INFLIGHT_ALIGNMENT 64
92 /* The version of inflight buffer */
93 #define INFLIGHT_VERSION 1
95 /* The version of the protocol we support */
96 #define VHOST_USER_VERSION 1
97 #define LIBVHOST_USER_DEBUG 0
101 if (LIBVHOST_USER_DEBUG) { \
102 fprintf(stderr, __VA_ARGS__); \
107 bool has_feature(uint64_t features
, unsigned int fbit
)
110 return !!(features
& (1ULL << fbit
));
114 bool vu_has_feature(VuDev
*dev
,
117 return has_feature(dev
->features
, fbit
);
120 static inline bool vu_has_protocol_feature(VuDev
*dev
, unsigned int fbit
)
122 return has_feature(dev
->protocol_features
, fbit
);
126 vu_request_to_string(unsigned int req
)
128 #define REQ(req) [req] = #req
129 static const char *vu_request_str
[] = {
130 REQ(VHOST_USER_NONE
),
131 REQ(VHOST_USER_GET_FEATURES
),
132 REQ(VHOST_USER_SET_FEATURES
),
133 REQ(VHOST_USER_SET_OWNER
),
134 REQ(VHOST_USER_RESET_OWNER
),
135 REQ(VHOST_USER_SET_MEM_TABLE
),
136 REQ(VHOST_USER_SET_LOG_BASE
),
137 REQ(VHOST_USER_SET_LOG_FD
),
138 REQ(VHOST_USER_SET_VRING_NUM
),
139 REQ(VHOST_USER_SET_VRING_ADDR
),
140 REQ(VHOST_USER_SET_VRING_BASE
),
141 REQ(VHOST_USER_GET_VRING_BASE
),
142 REQ(VHOST_USER_SET_VRING_KICK
),
143 REQ(VHOST_USER_SET_VRING_CALL
),
144 REQ(VHOST_USER_SET_VRING_ERR
),
145 REQ(VHOST_USER_GET_PROTOCOL_FEATURES
),
146 REQ(VHOST_USER_SET_PROTOCOL_FEATURES
),
147 REQ(VHOST_USER_GET_QUEUE_NUM
),
148 REQ(VHOST_USER_SET_VRING_ENABLE
),
149 REQ(VHOST_USER_SEND_RARP
),
150 REQ(VHOST_USER_NET_SET_MTU
),
151 REQ(VHOST_USER_SET_BACKEND_REQ_FD
),
152 REQ(VHOST_USER_IOTLB_MSG
),
153 REQ(VHOST_USER_SET_VRING_ENDIAN
),
154 REQ(VHOST_USER_GET_CONFIG
),
155 REQ(VHOST_USER_SET_CONFIG
),
156 REQ(VHOST_USER_POSTCOPY_ADVISE
),
157 REQ(VHOST_USER_POSTCOPY_LISTEN
),
158 REQ(VHOST_USER_POSTCOPY_END
),
159 REQ(VHOST_USER_GET_INFLIGHT_FD
),
160 REQ(VHOST_USER_SET_INFLIGHT_FD
),
161 REQ(VHOST_USER_GPU_SET_SOCKET
),
162 REQ(VHOST_USER_VRING_KICK
),
163 REQ(VHOST_USER_GET_MAX_MEM_SLOTS
),
164 REQ(VHOST_USER_ADD_MEM_REG
),
165 REQ(VHOST_USER_REM_MEM_REG
),
166 REQ(VHOST_USER_GET_SHARED_OBJECT
),
171 if (req
< VHOST_USER_MAX
) {
172 return vu_request_str
[req
];
178 static void G_GNUC_PRINTF(2, 3)
179 vu_panic(VuDev
*dev
, const char *msg
, ...)
185 if (vasprintf(&buf
, msg
, ap
) < 0) {
191 dev
->panic(dev
, buf
);
196 * find a way to call virtio_error, or perhaps close the connection?
200 /* Search for a memory region that covers this guest physical address. */
202 vu_gpa_to_mem_region(VuDev
*dev
, uint64_t guest_addr
)
205 int high
= dev
->nregions
- 1;
208 * Memory regions cannot overlap in guest physical address space. Each
209 * GPA belongs to exactly one memory region, so there can only be one
212 * We store our memory regions ordered by GPA and can simply perform a
215 while (low
<= high
) {
216 unsigned int mid
= low
+ (high
- low
) / 2;
217 VuDevRegion
*cur
= &dev
->regions
[mid
];
219 if (guest_addr
>= cur
->gpa
&& guest_addr
< cur
->gpa
+ cur
->size
) {
222 if (guest_addr
>= cur
->gpa
+ cur
->size
) {
225 if (guest_addr
< cur
->gpa
) {
232 /* Translate guest physical address to our virtual address. */
234 vu_gpa_to_va(VuDev
*dev
, uint64_t *plen
, uint64_t guest_addr
)
242 r
= vu_gpa_to_mem_region(dev
, guest_addr
);
247 if ((guest_addr
+ *plen
) > (r
->gpa
+ r
->size
)) {
248 *plen
= r
->gpa
+ r
->size
- guest_addr
;
250 return (void *)(uintptr_t)guest_addr
- r
->gpa
+ r
->mmap_addr
+
254 /* Translate qemu virtual address to our virtual address. */
256 qva_to_va(VuDev
*dev
, uint64_t qemu_addr
)
260 /* Find matching memory region. */
261 for (i
= 0; i
< dev
->nregions
; i
++) {
262 VuDevRegion
*r
= &dev
->regions
[i
];
264 if ((qemu_addr
>= r
->qva
) && (qemu_addr
< (r
->qva
+ r
->size
))) {
265 return (void *)(uintptr_t)
266 qemu_addr
- r
->qva
+ r
->mmap_addr
+ r
->mmap_offset
;
274 vu_remove_all_mem_regs(VuDev
*dev
)
278 for (i
= 0; i
< dev
->nregions
; i
++) {
279 VuDevRegion
*r
= &dev
->regions
[i
];
281 munmap((void *)(uintptr_t)r
->mmap_addr
, r
->size
+ r
->mmap_offset
);
287 map_ring(VuDev
*dev
, VuVirtq
*vq
)
289 vq
->vring
.desc
= qva_to_va(dev
, vq
->vra
.desc_user_addr
);
290 vq
->vring
.used
= qva_to_va(dev
, vq
->vra
.used_user_addr
);
291 vq
->vring
.avail
= qva_to_va(dev
, vq
->vra
.avail_user_addr
);
293 DPRINT("Setting virtq addresses:\n");
294 DPRINT(" vring_desc at %p\n", vq
->vring
.desc
);
295 DPRINT(" vring_used at %p\n", vq
->vring
.used
);
296 DPRINT(" vring_avail at %p\n", vq
->vring
.avail
);
298 return !(vq
->vring
.desc
&& vq
->vring
.used
&& vq
->vring
.avail
);
302 vu_is_vq_usable(VuDev
*dev
, VuVirtq
*vq
)
304 if (unlikely(dev
->broken
)) {
308 if (likely(vq
->vring
.avail
)) {
313 * In corner cases, we might temporarily remove a memory region that
314 * mapped a ring. When removing a memory region we make sure to
315 * unmap any rings that would be impacted. Let's try to remap if we
316 * already succeeded mapping this ring once.
318 if (!vq
->vra
.desc_user_addr
|| !vq
->vra
.used_user_addr
||
319 !vq
->vra
.avail_user_addr
) {
322 if (map_ring(dev
, vq
)) {
323 vu_panic(dev
, "remapping queue on access");
330 unmap_rings(VuDev
*dev
, VuDevRegion
*r
)
334 for (i
= 0; i
< dev
->max_queues
; i
++) {
335 VuVirtq
*vq
= &dev
->vq
[i
];
336 const uintptr_t desc
= (uintptr_t)vq
->vring
.desc
;
337 const uintptr_t used
= (uintptr_t)vq
->vring
.used
;
338 const uintptr_t avail
= (uintptr_t)vq
->vring
.avail
;
340 if (desc
< r
->mmap_addr
|| desc
>= r
->mmap_addr
+ r
->size
) {
343 if (used
< r
->mmap_addr
|| used
>= r
->mmap_addr
+ r
->size
) {
346 if (avail
< r
->mmap_addr
|| avail
>= r
->mmap_addr
+ r
->size
) {
350 DPRINT("Unmapping rings of queue %d\n", i
);
351 vq
->vring
.desc
= NULL
;
352 vq
->vring
.used
= NULL
;
353 vq
->vring
.avail
= NULL
;
358 get_fd_hugepagesize(int fd
)
360 #if defined(__linux__)
365 ret
= fstatfs(fd
, &fs
);
366 } while (ret
!= 0 && errno
== EINTR
);
368 if (!ret
&& (unsigned int)fs
.f_type
== HUGETLBFS_MAGIC
) {
376 _vu_add_mem_reg(VuDev
*dev
, VhostUserMemoryRegion
*msg_region
, int fd
)
378 const uint64_t start_gpa
= msg_region
->guest_phys_addr
;
379 const uint64_t end_gpa
= start_gpa
+ msg_region
->memory_size
;
380 int prot
= PROT_READ
| PROT_WRITE
;
381 uint64_t mmap_offset
, fd_offset
;
386 int high
= dev
->nregions
- 1;
389 DPRINT("Adding region %d\n", dev
->nregions
);
390 DPRINT(" guest_phys_addr: 0x%016"PRIx64
"\n",
391 msg_region
->guest_phys_addr
);
392 DPRINT(" memory_size: 0x%016"PRIx64
"\n",
393 msg_region
->memory_size
);
394 DPRINT(" userspace_addr: 0x%016"PRIx64
"\n",
395 msg_region
->userspace_addr
);
396 DPRINT(" old mmap_offset: 0x%016"PRIx64
"\n",
397 msg_region
->mmap_offset
);
399 if (dev
->postcopy_listening
) {
401 * In postcopy we're using PROT_NONE here to catch anyone
402 * accessing it before we userfault
408 * We will add memory regions into the array sorted by GPA. Perform a
409 * binary search to locate the insertion point: it will be at the low
412 while (low
<= high
) {
413 unsigned int mid
= low
+ (high
- low
) / 2;
414 VuDevRegion
*cur
= &dev
->regions
[mid
];
416 /* Overlap of GPA addresses. */
417 if (start_gpa
< cur
->gpa
+ cur
->size
&& cur
->gpa
< end_gpa
) {
418 vu_panic(dev
, "regions with overlapping guest physical addresses");
421 if (start_gpa
>= cur
->gpa
+ cur
->size
) {
424 if (start_gpa
< cur
->gpa
) {
431 * Convert most of msg_region->mmap_offset to fd_offset. In almost all
432 * cases, this will leave us with mmap_offset == 0, mmap()'ing only
433 * what we really need. Only if a memory region would partially cover
434 * hugetlb pages, we'd get mmap_offset != 0, which usually doesn't happen
435 * anymore (i.e., modern QEMU).
437 * Note that mmap() with hugetlb would fail if the offset into the file
438 * is not aligned to the huge page size.
440 hugepagesize
= get_fd_hugepagesize(fd
);
442 fd_offset
= ALIGN_DOWN(msg_region
->mmap_offset
, hugepagesize
);
443 mmap_offset
= msg_region
->mmap_offset
- fd_offset
;
445 fd_offset
= msg_region
->mmap_offset
;
449 DPRINT(" fd_offset: 0x%016"PRIx64
"\n",
451 DPRINT(" new mmap_offset: 0x%016"PRIx64
"\n",
454 mmap_addr
= mmap(0, msg_region
->memory_size
+ mmap_offset
,
455 prot
, MAP_SHARED
| MAP_NORESERVE
, fd
, fd_offset
);
456 if (mmap_addr
== MAP_FAILED
) {
457 vu_panic(dev
, "region mmap error: %s", strerror(errno
));
460 DPRINT(" mmap_addr: 0x%016"PRIx64
"\n",
461 (uint64_t)(uintptr_t)mmap_addr
);
463 #if defined(__linux__)
464 /* Don't include all guest memory in a coredump. */
465 madvise(mmap_addr
, msg_region
->memory_size
+ mmap_offset
,
469 /* Shift all affected entries by 1 to open a hole at idx. */
470 r
= &dev
->regions
[idx
];
471 memmove(r
+ 1, r
, sizeof(VuDevRegion
) * (dev
->nregions
- idx
));
472 r
->gpa
= msg_region
->guest_phys_addr
;
473 r
->size
= msg_region
->memory_size
;
474 r
->qva
= msg_region
->userspace_addr
;
475 r
->mmap_addr
= (uint64_t)(uintptr_t)mmap_addr
;
476 r
->mmap_offset
= mmap_offset
;
479 if (dev
->postcopy_listening
) {
481 * Return the address to QEMU so that it can translate the ufd
482 * fault addresses back.
484 msg_region
->userspace_addr
= r
->mmap_addr
+ r
->mmap_offset
;
489 vmsg_close_fds(VhostUserMsg
*vmsg
)
493 for (i
= 0; i
< vmsg
->fd_num
; i
++) {
498 /* Set reply payload.u64 and clear request flags and fd_num */
499 static void vmsg_set_reply_u64(VhostUserMsg
*vmsg
, uint64_t val
)
501 vmsg
->flags
= 0; /* defaults will be set by vu_send_reply() */
502 vmsg
->size
= sizeof(vmsg
->payload
.u64
);
503 vmsg
->payload
.u64
= val
;
507 /* A test to see if we have userfault available */
511 #if defined(__linux__) && defined(__NR_userfaultfd) &&\
512 defined(UFFD_FEATURE_MISSING_SHMEM) &&\
513 defined(UFFD_FEATURE_MISSING_HUGETLBFS)
514 /* Now test the kernel we're running on really has the features */
515 int ufd
= syscall(__NR_userfaultfd
, O_CLOEXEC
| O_NONBLOCK
);
516 struct uffdio_api api_struct
;
521 api_struct
.api
= UFFD_API
;
522 api_struct
.features
= UFFD_FEATURE_MISSING_SHMEM
|
523 UFFD_FEATURE_MISSING_HUGETLBFS
;
524 if (ioctl(ufd
, UFFDIO_API
, &api_struct
)) {
537 vu_message_read_default(VuDev
*dev
, int conn_fd
, VhostUserMsg
*vmsg
)
539 char control
[CMSG_SPACE(VHOST_MEMORY_BASELINE_NREGIONS
* sizeof(int))] = {};
541 .iov_base
= (char *)vmsg
,
542 .iov_len
= VHOST_USER_HDR_SIZE
,
544 struct msghdr msg
= {
547 .msg_control
= control
,
548 .msg_controllen
= sizeof(control
),
551 struct cmsghdr
*cmsg
;
555 rc
= recvmsg(conn_fd
, &msg
, 0);
556 } while (rc
< 0 && (errno
== EINTR
|| errno
== EAGAIN
));
559 vu_panic(dev
, "Error while recvmsg: %s", strerror(errno
));
564 for (cmsg
= CMSG_FIRSTHDR(&msg
);
566 cmsg
= CMSG_NXTHDR(&msg
, cmsg
))
568 if (cmsg
->cmsg_level
== SOL_SOCKET
&& cmsg
->cmsg_type
== SCM_RIGHTS
) {
569 fd_size
= cmsg
->cmsg_len
- CMSG_LEN(0);
570 vmsg
->fd_num
= fd_size
/ sizeof(int);
571 assert(fd_size
< VHOST_MEMORY_BASELINE_NREGIONS
);
572 memcpy(vmsg
->fds
, CMSG_DATA(cmsg
), fd_size
);
577 if (vmsg
->size
> sizeof(vmsg
->payload
)) {
579 "Error: too big message request: %d, size: vmsg->size: %u, "
580 "while sizeof(vmsg->payload) = %zu\n",
581 vmsg
->request
, vmsg
->size
, sizeof(vmsg
->payload
));
587 rc
= read(conn_fd
, &vmsg
->payload
, vmsg
->size
);
588 } while (rc
< 0 && (errno
== EINTR
|| errno
== EAGAIN
));
591 vu_panic(dev
, "Error while reading: %s", strerror(errno
));
595 assert((uint32_t)rc
== vmsg
->size
);
601 vmsg_close_fds(vmsg
);
607 vu_message_write(VuDev
*dev
, int conn_fd
, VhostUserMsg
*vmsg
)
610 uint8_t *p
= (uint8_t *)vmsg
;
611 char control
[CMSG_SPACE(VHOST_MEMORY_BASELINE_NREGIONS
* sizeof(int))] = {};
613 .iov_base
= (char *)vmsg
,
614 .iov_len
= VHOST_USER_HDR_SIZE
,
616 struct msghdr msg
= {
619 .msg_control
= control
,
621 struct cmsghdr
*cmsg
;
623 memset(control
, 0, sizeof(control
));
624 assert(vmsg
->fd_num
<= VHOST_MEMORY_BASELINE_NREGIONS
);
625 if (vmsg
->fd_num
> 0) {
626 size_t fdsize
= vmsg
->fd_num
* sizeof(int);
627 msg
.msg_controllen
= CMSG_SPACE(fdsize
);
628 cmsg
= CMSG_FIRSTHDR(&msg
);
629 cmsg
->cmsg_len
= CMSG_LEN(fdsize
);
630 cmsg
->cmsg_level
= SOL_SOCKET
;
631 cmsg
->cmsg_type
= SCM_RIGHTS
;
632 memcpy(CMSG_DATA(cmsg
), vmsg
->fds
, fdsize
);
634 msg
.msg_controllen
= 0;
638 rc
= sendmsg(conn_fd
, &msg
, 0);
639 } while (rc
< 0 && (errno
== EINTR
|| errno
== EAGAIN
));
644 rc
= write(conn_fd
, vmsg
->data
, vmsg
->size
);
646 rc
= write(conn_fd
, p
+ VHOST_USER_HDR_SIZE
, vmsg
->size
);
648 } while (rc
< 0 && (errno
== EINTR
|| errno
== EAGAIN
));
652 vu_panic(dev
, "Error while writing: %s", strerror(errno
));
660 vu_send_reply(VuDev
*dev
, int conn_fd
, VhostUserMsg
*vmsg
)
662 /* Set the version in the flags when sending the reply */
663 vmsg
->flags
&= ~VHOST_USER_VERSION_MASK
;
664 vmsg
->flags
|= VHOST_USER_VERSION
;
665 vmsg
->flags
|= VHOST_USER_REPLY_MASK
;
667 return vu_message_write(dev
, conn_fd
, vmsg
);
671 * Processes a reply on the backend channel.
672 * Entered with backend_mutex held and releases it before exit.
673 * Returns true on success.
676 vu_process_message_reply(VuDev
*dev
, const VhostUserMsg
*vmsg
)
678 VhostUserMsg msg_reply
;
681 if ((vmsg
->flags
& VHOST_USER_NEED_REPLY_MASK
) == 0) {
686 if (!vu_message_read_default(dev
, dev
->backend_fd
, &msg_reply
)) {
690 if (msg_reply
.request
!= vmsg
->request
) {
691 DPRINT("Received unexpected msg type. Expected %d received %d",
692 vmsg
->request
, msg_reply
.request
);
696 result
= msg_reply
.payload
.u64
== 0;
699 pthread_mutex_unlock(&dev
->backend_mutex
);
703 /* Kick the log_call_fd if required. */
705 vu_log_kick(VuDev
*dev
)
707 if (dev
->log_call_fd
!= -1) {
708 DPRINT("Kicking the QEMU's log...\n");
709 if (eventfd_write(dev
->log_call_fd
, 1) < 0) {
710 vu_panic(dev
, "Error writing eventfd: %s", strerror(errno
));
716 vu_log_page(uint8_t *log_table
, uint64_t page
)
718 DPRINT("Logged dirty guest page: %"PRId64
"\n", page
);
719 qatomic_or(&log_table
[page
/ 8], 1 << (page
% 8));
723 vu_log_write(VuDev
*dev
, uint64_t address
, uint64_t length
)
727 if (!(dev
->features
& (1ULL << VHOST_F_LOG_ALL
)) ||
728 !dev
->log_table
|| !length
) {
732 assert(dev
->log_size
> ((address
+ length
- 1) / VHOST_LOG_PAGE
/ 8));
734 page
= address
/ VHOST_LOG_PAGE
;
735 while (page
* VHOST_LOG_PAGE
< address
+ length
) {
736 vu_log_page(dev
->log_table
, page
);
744 vu_kick_cb(VuDev
*dev
, int condition
, void *data
)
746 int index
= (intptr_t)data
;
747 VuVirtq
*vq
= &dev
->vq
[index
];
748 int sock
= vq
->kick_fd
;
752 rc
= eventfd_read(sock
, &kick_data
);
754 vu_panic(dev
, "kick eventfd_read(): %s", strerror(errno
));
755 dev
->remove_watch(dev
, dev
->vq
[index
].kick_fd
);
757 DPRINT("Got kick_data: %016"PRIx64
" handler:%p idx:%d\n",
758 kick_data
, vq
->handler
, index
);
760 vq
->handler(dev
, index
);
766 vu_get_features_exec(VuDev
*dev
, VhostUserMsg
*vmsg
)
770 * The following VIRTIO feature bits are supported by our virtqueue
773 1ULL << VIRTIO_F_NOTIFY_ON_EMPTY
|
774 1ULL << VIRTIO_RING_F_INDIRECT_DESC
|
775 1ULL << VIRTIO_RING_F_EVENT_IDX
|
776 1ULL << VIRTIO_F_VERSION_1
|
778 /* vhost-user feature bits */
779 1ULL << VHOST_F_LOG_ALL
|
780 1ULL << VHOST_USER_F_PROTOCOL_FEATURES
;
782 if (dev
->iface
->get_features
) {
783 vmsg
->payload
.u64
|= dev
->iface
->get_features(dev
);
786 vmsg
->size
= sizeof(vmsg
->payload
.u64
);
789 DPRINT("Sending back to guest u64: 0x%016"PRIx64
"\n", vmsg
->payload
.u64
);
795 vu_set_enable_all_rings(VuDev
*dev
, bool enabled
)
799 for (i
= 0; i
< dev
->max_queues
; i
++) {
800 dev
->vq
[i
].enable
= enabled
;
805 vu_set_features_exec(VuDev
*dev
, VhostUserMsg
*vmsg
)
807 DPRINT("u64: 0x%016"PRIx64
"\n", vmsg
->payload
.u64
);
809 dev
->features
= vmsg
->payload
.u64
;
810 if (!vu_has_feature(dev
, VIRTIO_F_VERSION_1
)) {
812 * We only support devices conforming to VIRTIO 1.0 or
815 vu_panic(dev
, "virtio legacy devices aren't supported by libvhost-user");
819 if (!(dev
->features
& VHOST_USER_F_PROTOCOL_FEATURES
)) {
820 vu_set_enable_all_rings(dev
, true);
823 if (dev
->iface
->set_features
) {
824 dev
->iface
->set_features(dev
, dev
->features
);
831 vu_set_owner_exec(VuDev
*dev
, VhostUserMsg
*vmsg
)
837 vu_close_log(VuDev
*dev
)
839 if (dev
->log_table
) {
840 if (munmap(dev
->log_table
, dev
->log_size
) != 0) {
841 perror("close log munmap() error");
844 dev
->log_table
= NULL
;
846 if (dev
->log_call_fd
!= -1) {
847 close(dev
->log_call_fd
);
848 dev
->log_call_fd
= -1;
853 vu_reset_device_exec(VuDev
*dev
, VhostUserMsg
*vmsg
)
855 vu_set_enable_all_rings(dev
, false);
861 generate_faults(VuDev
*dev
) {
863 for (i
= 0; i
< dev
->nregions
; i
++) {
864 #ifdef UFFDIO_REGISTER
865 VuDevRegion
*dev_region
= &dev
->regions
[i
];
867 struct uffdio_register reg_struct
;
870 * We should already have an open ufd. Mark each memory
872 * Discard any mapping we have here; note I can't use MADV_REMOVE
873 * or fallocate to make the hole since I don't want to lose
874 * data that's already arrived in the shared process.
875 * TODO: How to do hugepage
877 ret
= madvise((void *)(uintptr_t)dev_region
->mmap_addr
,
878 dev_region
->size
+ dev_region
->mmap_offset
,
882 "%s: Failed to madvise(DONTNEED) region %d: %s\n",
883 __func__
, i
, strerror(errno
));
886 * Turn off transparent hugepages so we dont get lose wakeups
887 * in neighbouring pages.
888 * TODO: Turn this backon later.
890 ret
= madvise((void *)(uintptr_t)dev_region
->mmap_addr
,
891 dev_region
->size
+ dev_region
->mmap_offset
,
895 * Note: This can happen legally on kernels that are configured
896 * without madvise'able hugepages
899 "%s: Failed to madvise(NOHUGEPAGE) region %d: %s\n",
900 __func__
, i
, strerror(errno
));
903 reg_struct
.range
.start
= (uintptr_t)dev_region
->mmap_addr
;
904 reg_struct
.range
.len
= dev_region
->size
+ dev_region
->mmap_offset
;
905 reg_struct
.mode
= UFFDIO_REGISTER_MODE_MISSING
;
907 if (ioctl(dev
->postcopy_ufd
, UFFDIO_REGISTER
, ®_struct
)) {
908 vu_panic(dev
, "%s: Failed to userfault region %d "
909 "@%" PRIx64
" + size:%" PRIx64
" offset: %" PRIx64
912 dev_region
->mmap_addr
,
913 dev_region
->size
, dev_region
->mmap_offset
,
914 dev
->postcopy_ufd
, strerror(errno
));
917 if (!(reg_struct
.ioctls
& (1ULL << _UFFDIO_COPY
))) {
918 vu_panic(dev
, "%s Region (%d) doesn't support COPY",
922 DPRINT("%s: region %d: Registered userfault for %"
923 PRIx64
" + %" PRIx64
"\n", __func__
, i
,
924 (uint64_t)reg_struct
.range
.start
,
925 (uint64_t)reg_struct
.range
.len
);
926 /* Now it's registered we can let the client at it */
927 if (mprotect((void *)(uintptr_t)dev_region
->mmap_addr
,
928 dev_region
->size
+ dev_region
->mmap_offset
,
929 PROT_READ
| PROT_WRITE
)) {
930 vu_panic(dev
, "failed to mprotect region %d for postcopy (%s)",
934 /* TODO: Stash 'zero' support flags somewhere */
942 vu_add_mem_reg(VuDev
*dev
, VhostUserMsg
*vmsg
) {
943 VhostUserMemoryRegion m
= vmsg
->payload
.memreg
.region
, *msg_region
= &m
;
945 if (vmsg
->fd_num
!= 1) {
946 vmsg_close_fds(vmsg
);
947 vu_panic(dev
, "VHOST_USER_ADD_MEM_REG received %d fds - only 1 fd "
948 "should be sent for this message type", vmsg
->fd_num
);
952 if (vmsg
->size
< VHOST_USER_MEM_REG_SIZE
) {
954 vu_panic(dev
, "VHOST_USER_ADD_MEM_REG requires a message size of at "
955 "least %zu bytes and only %d bytes were received",
956 VHOST_USER_MEM_REG_SIZE
, vmsg
->size
);
960 if (dev
->nregions
== VHOST_USER_MAX_RAM_SLOTS
) {
962 vu_panic(dev
, "failing attempt to hot add memory via "
963 "VHOST_USER_ADD_MEM_REG message because the backend has "
964 "no free ram slots available");
969 * If we are in postcopy mode and we receive a u64 payload with a 0 value
970 * we know all the postcopy client bases have been received, and we
971 * should start generating faults.
973 if (dev
->postcopy_listening
&&
974 vmsg
->size
== sizeof(vmsg
->payload
.u64
) &&
975 vmsg
->payload
.u64
== 0) {
976 (void)generate_faults(dev
);
980 _vu_add_mem_reg(dev
, msg_region
, vmsg
->fds
[0]);
983 if (dev
->postcopy_listening
) {
984 /* Send the message back to qemu with the addresses filled in. */
986 DPRINT("Successfully added new region in postcopy\n");
989 DPRINT("Successfully added new region\n");
993 static inline bool reg_equal(VuDevRegion
*vudev_reg
,
994 VhostUserMemoryRegion
*msg_reg
)
996 if (vudev_reg
->gpa
== msg_reg
->guest_phys_addr
&&
997 vudev_reg
->qva
== msg_reg
->userspace_addr
&&
998 vudev_reg
->size
== msg_reg
->memory_size
) {
1006 vu_rem_mem_reg(VuDev
*dev
, VhostUserMsg
*vmsg
) {
1007 VhostUserMemoryRegion m
= vmsg
->payload
.memreg
.region
, *msg_region
= &m
;
1011 if (vmsg
->fd_num
> 1) {
1012 vmsg_close_fds(vmsg
);
1013 vu_panic(dev
, "VHOST_USER_REM_MEM_REG received %d fds - at most 1 fd "
1014 "should be sent for this message type", vmsg
->fd_num
);
1018 if (vmsg
->size
< VHOST_USER_MEM_REG_SIZE
) {
1019 vmsg_close_fds(vmsg
);
1020 vu_panic(dev
, "VHOST_USER_REM_MEM_REG requires a message size of at "
1021 "least %zu bytes and only %d bytes were received",
1022 VHOST_USER_MEM_REG_SIZE
, vmsg
->size
);
1026 DPRINT("Removing region:\n");
1027 DPRINT(" guest_phys_addr: 0x%016"PRIx64
"\n",
1028 msg_region
->guest_phys_addr
);
1029 DPRINT(" memory_size: 0x%016"PRIx64
"\n",
1030 msg_region
->memory_size
);
1031 DPRINT(" userspace_addr 0x%016"PRIx64
"\n",
1032 msg_region
->userspace_addr
);
1033 DPRINT(" mmap_offset 0x%016"PRIx64
"\n",
1034 msg_region
->mmap_offset
);
1036 r
= vu_gpa_to_mem_region(dev
, msg_region
->guest_phys_addr
);
1037 if (!r
|| !reg_equal(r
, msg_region
)) {
1038 vmsg_close_fds(vmsg
);
1039 vu_panic(dev
, "Specified region not found\n");
1044 * There might be valid cases where we temporarily remove memory regions
1045 * to readd them again, or remove memory regions and don't use the rings
1046 * anymore before we set the ring addresses and restart the device.
1048 * Unmap all affected rings, remapping them on demand later. This should
1051 unmap_rings(dev
, r
);
1053 munmap((void *)(uintptr_t)r
->mmap_addr
, r
->size
+ r
->mmap_offset
);
1055 idx
= r
- dev
->regions
;
1056 assert(idx
< dev
->nregions
);
1057 /* Shift all affected entries by 1 to close the hole. */
1058 memmove(r
, r
+ 1, sizeof(VuDevRegion
) * (dev
->nregions
- idx
- 1));
1059 DPRINT("Successfully removed a region\n");
1062 vmsg_close_fds(vmsg
);
1068 vu_get_shared_object(VuDev
*dev
, VhostUserMsg
*vmsg
)
1072 if (dev
->iface
->get_shared_object
) {
1073 dmabuf_fd
= dev
->iface
->get_shared_object(
1074 dev
, &vmsg
->payload
.object
.uuid
[0]);
1076 if (dmabuf_fd
!= -1) {
1077 DPRINT("dmabuf_fd found for requested UUID\n");
1078 vmsg
->fds
[fd_num
++] = dmabuf_fd
;
1080 vmsg
->fd_num
= fd_num
;
1086 vu_set_mem_table_exec(VuDev
*dev
, VhostUserMsg
*vmsg
)
1088 VhostUserMemory m
= vmsg
->payload
.memory
, *memory
= &m
;
1091 vu_remove_all_mem_regs(dev
);
1093 DPRINT("Nregions: %u\n", memory
->nregions
);
1094 for (i
= 0; i
< memory
->nregions
; i
++) {
1095 _vu_add_mem_reg(dev
, &memory
->regions
[i
], vmsg
->fds
[i
]);
1096 close(vmsg
->fds
[i
]);
1099 if (dev
->postcopy_listening
) {
1100 /* Send the message back to qemu with the addresses filled in */
1102 if (!vu_send_reply(dev
, dev
->sock
, vmsg
)) {
1103 vu_panic(dev
, "failed to respond to set-mem-table for postcopy");
1108 * Wait for QEMU to confirm that it's registered the handler for the
1111 if (!dev
->read_msg(dev
, dev
->sock
, vmsg
) ||
1112 vmsg
->size
!= sizeof(vmsg
->payload
.u64
) ||
1113 vmsg
->payload
.u64
!= 0) {
1114 vu_panic(dev
, "failed to receive valid ack for postcopy set-mem-table");
1118 /* OK, now we can go and register the memory and generate faults */
1119 (void)generate_faults(dev
);
1123 for (i
= 0; i
< dev
->max_queues
; i
++) {
1124 if (dev
->vq
[i
].vring
.desc
) {
1125 if (map_ring(dev
, &dev
->vq
[i
])) {
1126 vu_panic(dev
, "remapping queue %d during setmemtable", i
);
1135 vu_set_log_base_exec(VuDev
*dev
, VhostUserMsg
*vmsg
)
1138 uint64_t log_mmap_size
, log_mmap_offset
;
1141 if (vmsg
->fd_num
!= 1 ||
1142 vmsg
->size
!= sizeof(vmsg
->payload
.log
)) {
1143 vu_panic(dev
, "Invalid log_base message");
1148 log_mmap_offset
= vmsg
->payload
.log
.mmap_offset
;
1149 log_mmap_size
= vmsg
->payload
.log
.mmap_size
;
1150 DPRINT("Log mmap_offset: %"PRId64
"\n", log_mmap_offset
);
1151 DPRINT("Log mmap_size: %"PRId64
"\n", log_mmap_size
);
1153 rc
= mmap(0, log_mmap_size
, PROT_READ
| PROT_WRITE
, MAP_SHARED
, fd
,
1156 if (rc
== MAP_FAILED
) {
1157 perror("log mmap error");
1160 if (dev
->log_table
) {
1161 munmap(dev
->log_table
, dev
->log_size
);
1163 dev
->log_table
= rc
;
1164 dev
->log_size
= log_mmap_size
;
1166 vmsg
->size
= sizeof(vmsg
->payload
.u64
);
1173 vu_set_log_fd_exec(VuDev
*dev
, VhostUserMsg
*vmsg
)
1175 if (vmsg
->fd_num
!= 1) {
1176 vu_panic(dev
, "Invalid log_fd message");
1180 if (dev
->log_call_fd
!= -1) {
1181 close(dev
->log_call_fd
);
1183 dev
->log_call_fd
= vmsg
->fds
[0];
1184 DPRINT("Got log_call_fd: %d\n", vmsg
->fds
[0]);
1190 vu_set_vring_num_exec(VuDev
*dev
, VhostUserMsg
*vmsg
)
1192 unsigned int index
= vmsg
->payload
.state
.index
;
1193 unsigned int num
= vmsg
->payload
.state
.num
;
1195 DPRINT("State.index: %u\n", index
);
1196 DPRINT("State.num: %u\n", num
);
1197 dev
->vq
[index
].vring
.num
= num
;
1203 vu_set_vring_addr_exec(VuDev
*dev
, VhostUserMsg
*vmsg
)
1205 struct vhost_vring_addr addr
= vmsg
->payload
.addr
, *vra
= &addr
;
1206 unsigned int index
= vra
->index
;
1207 VuVirtq
*vq
= &dev
->vq
[index
];
1209 DPRINT("vhost_vring_addr:\n");
1210 DPRINT(" index: %d\n", vra
->index
);
1211 DPRINT(" flags: %d\n", vra
->flags
);
1212 DPRINT(" desc_user_addr: 0x%016" PRIx64
"\n", (uint64_t)vra
->desc_user_addr
);
1213 DPRINT(" used_user_addr: 0x%016" PRIx64
"\n", (uint64_t)vra
->used_user_addr
);
1214 DPRINT(" avail_user_addr: 0x%016" PRIx64
"\n", (uint64_t)vra
->avail_user_addr
);
1215 DPRINT(" log_guest_addr: 0x%016" PRIx64
"\n", (uint64_t)vra
->log_guest_addr
);
1218 vq
->vring
.flags
= vra
->flags
;
1219 vq
->vring
.log_guest_addr
= vra
->log_guest_addr
;
1222 if (map_ring(dev
, vq
)) {
1223 vu_panic(dev
, "Invalid vring_addr message");
1227 vq
->used_idx
= le16toh(vq
->vring
.used
->idx
);
1229 if (vq
->last_avail_idx
!= vq
->used_idx
) {
1230 bool resume
= dev
->iface
->queue_is_processed_in_order
&&
1231 dev
->iface
->queue_is_processed_in_order(dev
, index
);
1233 DPRINT("Last avail index != used index: %u != %u%s\n",
1234 vq
->last_avail_idx
, vq
->used_idx
,
1235 resume
? ", resuming" : "");
1238 vq
->shadow_avail_idx
= vq
->last_avail_idx
= vq
->used_idx
;
1246 vu_set_vring_base_exec(VuDev
*dev
, VhostUserMsg
*vmsg
)
1248 unsigned int index
= vmsg
->payload
.state
.index
;
1249 unsigned int num
= vmsg
->payload
.state
.num
;
1251 DPRINT("State.index: %u\n", index
);
1252 DPRINT("State.num: %u\n", num
);
1253 dev
->vq
[index
].shadow_avail_idx
= dev
->vq
[index
].last_avail_idx
= num
;
1259 vu_get_vring_base_exec(VuDev
*dev
, VhostUserMsg
*vmsg
)
1261 unsigned int index
= vmsg
->payload
.state
.index
;
1263 DPRINT("State.index: %u\n", index
);
1264 vmsg
->payload
.state
.num
= dev
->vq
[index
].last_avail_idx
;
1265 vmsg
->size
= sizeof(vmsg
->payload
.state
);
1267 dev
->vq
[index
].started
= false;
1268 if (dev
->iface
->queue_set_started
) {
1269 dev
->iface
->queue_set_started(dev
, index
, false);
1272 if (dev
->vq
[index
].call_fd
!= -1) {
1273 close(dev
->vq
[index
].call_fd
);
1274 dev
->vq
[index
].call_fd
= -1;
1276 if (dev
->vq
[index
].kick_fd
!= -1) {
1277 dev
->remove_watch(dev
, dev
->vq
[index
].kick_fd
);
1278 close(dev
->vq
[index
].kick_fd
);
1279 dev
->vq
[index
].kick_fd
= -1;
1286 vu_check_queue_msg_file(VuDev
*dev
, VhostUserMsg
*vmsg
)
1288 int index
= vmsg
->payload
.u64
& VHOST_USER_VRING_IDX_MASK
;
1289 bool nofd
= vmsg
->payload
.u64
& VHOST_USER_VRING_NOFD_MASK
;
1291 if (index
>= dev
->max_queues
) {
1292 vmsg_close_fds(vmsg
);
1293 vu_panic(dev
, "Invalid queue index: %u", index
);
1298 vmsg_close_fds(vmsg
);
1302 if (vmsg
->fd_num
!= 1) {
1303 vmsg_close_fds(vmsg
);
1304 vu_panic(dev
, "Invalid fds in request: %d", vmsg
->request
);
1312 inflight_desc_compare(const void *a
, const void *b
)
1314 VuVirtqInflightDesc
*desc0
= (VuVirtqInflightDesc
*)a
,
1315 *desc1
= (VuVirtqInflightDesc
*)b
;
1317 if (desc1
->counter
> desc0
->counter
&&
1318 (desc1
->counter
- desc0
->counter
) < VIRTQUEUE_MAX_SIZE
* 2) {
1326 vu_check_queue_inflights(VuDev
*dev
, VuVirtq
*vq
)
1330 if (!vu_has_protocol_feature(dev
, VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD
)) {
1334 if (unlikely(!vq
->inflight
)) {
1338 if (unlikely(!vq
->inflight
->version
)) {
1339 /* initialize the buffer */
1340 vq
->inflight
->version
= INFLIGHT_VERSION
;
1344 vq
->used_idx
= le16toh(vq
->vring
.used
->idx
);
1345 vq
->resubmit_num
= 0;
1346 vq
->resubmit_list
= NULL
;
1349 if (unlikely(vq
->inflight
->used_idx
!= vq
->used_idx
)) {
1350 vq
->inflight
->desc
[vq
->inflight
->last_batch_head
].inflight
= 0;
1354 vq
->inflight
->used_idx
= vq
->used_idx
;
1357 for (i
= 0; i
< vq
->inflight
->desc_num
; i
++) {
1358 if (vq
->inflight
->desc
[i
].inflight
== 1) {
1363 vq
->shadow_avail_idx
= vq
->last_avail_idx
= vq
->inuse
+ vq
->used_idx
;
1366 vq
->resubmit_list
= calloc(vq
->inuse
, sizeof(VuVirtqInflightDesc
));
1367 if (!vq
->resubmit_list
) {
1371 for (i
= 0; i
< vq
->inflight
->desc_num
; i
++) {
1372 if (vq
->inflight
->desc
[i
].inflight
) {
1373 vq
->resubmit_list
[vq
->resubmit_num
].index
= i
;
1374 vq
->resubmit_list
[vq
->resubmit_num
].counter
=
1375 vq
->inflight
->desc
[i
].counter
;
1380 if (vq
->resubmit_num
> 1) {
1381 qsort(vq
->resubmit_list
, vq
->resubmit_num
,
1382 sizeof(VuVirtqInflightDesc
), inflight_desc_compare
);
1384 vq
->counter
= vq
->resubmit_list
[0].counter
+ 1;
1387 /* in case of I/O hang after reconnecting */
1388 if (eventfd_write(vq
->kick_fd
, 1)) {
1396 vu_set_vring_kick_exec(VuDev
*dev
, VhostUserMsg
*vmsg
)
1398 int index
= vmsg
->payload
.u64
& VHOST_USER_VRING_IDX_MASK
;
1399 bool nofd
= vmsg
->payload
.u64
& VHOST_USER_VRING_NOFD_MASK
;
1401 DPRINT("u64: 0x%016"PRIx64
"\n", vmsg
->payload
.u64
);
1403 if (!vu_check_queue_msg_file(dev
, vmsg
)) {
1407 if (dev
->vq
[index
].kick_fd
!= -1) {
1408 dev
->remove_watch(dev
, dev
->vq
[index
].kick_fd
);
1409 close(dev
->vq
[index
].kick_fd
);
1410 dev
->vq
[index
].kick_fd
= -1;
1413 dev
->vq
[index
].kick_fd
= nofd
? -1 : vmsg
->fds
[0];
1414 DPRINT("Got kick_fd: %d for vq: %d\n", dev
->vq
[index
].kick_fd
, index
);
1416 dev
->vq
[index
].started
= true;
1417 if (dev
->iface
->queue_set_started
) {
1418 dev
->iface
->queue_set_started(dev
, index
, true);
1421 if (dev
->vq
[index
].kick_fd
!= -1 && dev
->vq
[index
].handler
) {
1422 dev
->set_watch(dev
, dev
->vq
[index
].kick_fd
, VU_WATCH_IN
,
1423 vu_kick_cb
, (void *)(long)index
);
1425 DPRINT("Waiting for kicks on fd: %d for vq: %d\n",
1426 dev
->vq
[index
].kick_fd
, index
);
1429 if (vu_check_queue_inflights(dev
, &dev
->vq
[index
])) {
1430 vu_panic(dev
, "Failed to check inflights for vq: %d\n", index
);
1436 void vu_set_queue_handler(VuDev
*dev
, VuVirtq
*vq
,
1437 vu_queue_handler_cb handler
)
1439 int qidx
= vq
- dev
->vq
;
1441 vq
->handler
= handler
;
1442 if (vq
->kick_fd
>= 0) {
1444 dev
->set_watch(dev
, vq
->kick_fd
, VU_WATCH_IN
,
1445 vu_kick_cb
, (void *)(long)qidx
);
1447 dev
->remove_watch(dev
, vq
->kick_fd
);
1452 bool vu_set_queue_host_notifier(VuDev
*dev
, VuVirtq
*vq
, int fd
,
1453 int size
, int offset
)
1455 int qidx
= vq
- dev
->vq
;
1457 VhostUserMsg vmsg
= {
1458 .request
= VHOST_USER_BACKEND_VRING_HOST_NOTIFIER_MSG
,
1459 .flags
= VHOST_USER_VERSION
| VHOST_USER_NEED_REPLY_MASK
,
1460 .size
= sizeof(vmsg
.payload
.area
),
1462 .u64
= qidx
& VHOST_USER_VRING_IDX_MASK
,
1469 vmsg
.payload
.area
.u64
|= VHOST_USER_VRING_NOFD_MASK
;
1471 vmsg
.fds
[fd_num
++] = fd
;
1474 vmsg
.fd_num
= fd_num
;
1476 if (!vu_has_protocol_feature(dev
, VHOST_USER_PROTOCOL_F_BACKEND_SEND_FD
)) {
1480 pthread_mutex_lock(&dev
->backend_mutex
);
1481 if (!vu_message_write(dev
, dev
->backend_fd
, &vmsg
)) {
1482 pthread_mutex_unlock(&dev
->backend_mutex
);
1486 /* Also unlocks the backend_mutex */
1487 return vu_process_message_reply(dev
, &vmsg
);
1491 vu_lookup_shared_object(VuDev
*dev
, unsigned char uuid
[UUID_LEN
],
1494 bool result
= false;
1495 VhostUserMsg msg_reply
;
1496 VhostUserMsg msg
= {
1497 .request
= VHOST_USER_BACKEND_SHARED_OBJECT_LOOKUP
,
1498 .size
= sizeof(msg
.payload
.object
),
1499 .flags
= VHOST_USER_VERSION
| VHOST_USER_NEED_REPLY_MASK
,
1502 memcpy(msg
.payload
.object
.uuid
, uuid
, sizeof(uuid
[0]) * UUID_LEN
);
1504 if (!vu_has_protocol_feature(dev
, VHOST_USER_PROTOCOL_F_SHARED_OBJECT
)) {
1508 pthread_mutex_lock(&dev
->backend_mutex
);
1509 if (!vu_message_write(dev
, dev
->backend_fd
, &msg
)) {
1513 if (!vu_message_read_default(dev
, dev
->backend_fd
, &msg_reply
)) {
1517 if (msg_reply
.request
!= msg
.request
) {
1518 DPRINT("Received unexpected msg type. Expected %d, received %d",
1519 msg
.request
, msg_reply
.request
);
1523 if (msg_reply
.fd_num
!= 1) {
1524 DPRINT("Received unexpected number of fds. Expected 1, received %d",
1529 *dmabuf_fd
= msg_reply
.fds
[0];
1530 result
= *dmabuf_fd
> 0 && msg_reply
.payload
.u64
== 0;
1532 pthread_mutex_unlock(&dev
->backend_mutex
);
1538 vu_send_message(VuDev
*dev
, VhostUserMsg
*vmsg
)
1540 bool result
= false;
1541 pthread_mutex_lock(&dev
->backend_mutex
);
1542 if (!vu_message_write(dev
, dev
->backend_fd
, vmsg
)) {
1548 pthread_mutex_unlock(&dev
->backend_mutex
);
1554 vu_add_shared_object(VuDev
*dev
, unsigned char uuid
[UUID_LEN
])
1556 VhostUserMsg msg
= {
1557 .request
= VHOST_USER_BACKEND_SHARED_OBJECT_ADD
,
1558 .size
= sizeof(msg
.payload
.object
),
1559 .flags
= VHOST_USER_VERSION
,
1562 memcpy(msg
.payload
.object
.uuid
, uuid
, sizeof(uuid
[0]) * UUID_LEN
);
1564 if (!vu_has_protocol_feature(dev
, VHOST_USER_PROTOCOL_F_SHARED_OBJECT
)) {
1568 return vu_send_message(dev
, &msg
);
1572 vu_rm_shared_object(VuDev
*dev
, unsigned char uuid
[UUID_LEN
])
1574 VhostUserMsg msg
= {
1575 .request
= VHOST_USER_BACKEND_SHARED_OBJECT_REMOVE
,
1576 .size
= sizeof(msg
.payload
.object
),
1577 .flags
= VHOST_USER_VERSION
,
1580 memcpy(msg
.payload
.object
.uuid
, uuid
, sizeof(uuid
[0]) * UUID_LEN
);
1582 if (!vu_has_protocol_feature(dev
, VHOST_USER_PROTOCOL_F_SHARED_OBJECT
)) {
1586 return vu_send_message(dev
, &msg
);
1590 vu_set_vring_call_exec(VuDev
*dev
, VhostUserMsg
*vmsg
)
1592 int index
= vmsg
->payload
.u64
& VHOST_USER_VRING_IDX_MASK
;
1593 bool nofd
= vmsg
->payload
.u64
& VHOST_USER_VRING_NOFD_MASK
;
1595 DPRINT("u64: 0x%016"PRIx64
"\n", vmsg
->payload
.u64
);
1597 if (!vu_check_queue_msg_file(dev
, vmsg
)) {
1601 if (dev
->vq
[index
].call_fd
!= -1) {
1602 close(dev
->vq
[index
].call_fd
);
1603 dev
->vq
[index
].call_fd
= -1;
1606 dev
->vq
[index
].call_fd
= nofd
? -1 : vmsg
->fds
[0];
1608 /* in case of I/O hang after reconnecting */
1609 if (dev
->vq
[index
].call_fd
!= -1 && eventfd_write(vmsg
->fds
[0], 1)) {
1613 DPRINT("Got call_fd: %d for vq: %d\n", dev
->vq
[index
].call_fd
, index
);
1619 vu_set_vring_err_exec(VuDev
*dev
, VhostUserMsg
*vmsg
)
1621 int index
= vmsg
->payload
.u64
& VHOST_USER_VRING_IDX_MASK
;
1622 bool nofd
= vmsg
->payload
.u64
& VHOST_USER_VRING_NOFD_MASK
;
1624 DPRINT("u64: 0x%016"PRIx64
"\n", vmsg
->payload
.u64
);
1626 if (!vu_check_queue_msg_file(dev
, vmsg
)) {
1630 if (dev
->vq
[index
].err_fd
!= -1) {
1631 close(dev
->vq
[index
].err_fd
);
1632 dev
->vq
[index
].err_fd
= -1;
1635 dev
->vq
[index
].err_fd
= nofd
? -1 : vmsg
->fds
[0];
1641 vu_get_protocol_features_exec(VuDev
*dev
, VhostUserMsg
*vmsg
)
1644 * Note that we support, but intentionally do not set,
1645 * VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS. This means that
1646 * a device implementation can return it in its callback
1647 * (get_protocol_features) if it wants to use this for
1648 * simulation, but it is otherwise not desirable (if even
1649 * implemented by the frontend.)
1651 uint64_t features
= 1ULL << VHOST_USER_PROTOCOL_F_MQ
|
1652 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD
|
1653 1ULL << VHOST_USER_PROTOCOL_F_BACKEND_REQ
|
1654 1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER
|
1655 1ULL << VHOST_USER_PROTOCOL_F_BACKEND_SEND_FD
|
1656 1ULL << VHOST_USER_PROTOCOL_F_REPLY_ACK
|
1657 1ULL << VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS
;
1659 if (have_userfault()) {
1660 features
|= 1ULL << VHOST_USER_PROTOCOL_F_PAGEFAULT
;
1663 if (dev
->iface
->get_config
&& dev
->iface
->set_config
) {
1664 features
|= 1ULL << VHOST_USER_PROTOCOL_F_CONFIG
;
1667 if (dev
->iface
->get_protocol_features
) {
1668 features
|= dev
->iface
->get_protocol_features(dev
);
1671 vmsg_set_reply_u64(vmsg
, features
);
1676 vu_set_protocol_features_exec(VuDev
*dev
, VhostUserMsg
*vmsg
)
1678 uint64_t features
= vmsg
->payload
.u64
;
1680 DPRINT("u64: 0x%016"PRIx64
"\n", features
);
1682 dev
->protocol_features
= vmsg
->payload
.u64
;
1684 if (vu_has_protocol_feature(dev
,
1685 VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS
) &&
1686 (!vu_has_protocol_feature(dev
, VHOST_USER_PROTOCOL_F_BACKEND_REQ
) ||
1687 !vu_has_protocol_feature(dev
, VHOST_USER_PROTOCOL_F_REPLY_ACK
))) {
1689 * The use case for using messages for kick/call is simulation, to make
1690 * the kick and call synchronous. To actually get that behaviour, both
1691 * of the other features are required.
1692 * Theoretically, one could use only kick messages, or do them without
1693 * having F_REPLY_ACK, but too many (possibly pending) messages on the
1694 * socket will eventually cause the frontend to hang, to avoid this in
1695 * scenarios where not desired enforce that the settings are in a way
1696 * that actually enables the simulation case.
1699 "F_IN_BAND_NOTIFICATIONS requires F_BACKEND_REQ && F_REPLY_ACK");
1703 if (dev
->iface
->set_protocol_features
) {
1704 dev
->iface
->set_protocol_features(dev
, features
);
1711 vu_get_queue_num_exec(VuDev
*dev
, VhostUserMsg
*vmsg
)
1713 vmsg_set_reply_u64(vmsg
, dev
->max_queues
);
1718 vu_set_vring_enable_exec(VuDev
*dev
, VhostUserMsg
*vmsg
)
1720 unsigned int index
= vmsg
->payload
.state
.index
;
1721 unsigned int enable
= vmsg
->payload
.state
.num
;
1723 DPRINT("State.index: %u\n", index
);
1724 DPRINT("State.enable: %u\n", enable
);
1726 if (index
>= dev
->max_queues
) {
1727 vu_panic(dev
, "Invalid vring_enable index: %u", index
);
1731 dev
->vq
[index
].enable
= enable
;
1736 vu_set_backend_req_fd(VuDev
*dev
, VhostUserMsg
*vmsg
)
1738 if (vmsg
->fd_num
!= 1) {
1739 vu_panic(dev
, "Invalid backend_req_fd message (%d fd's)", vmsg
->fd_num
);
1743 if (dev
->backend_fd
!= -1) {
1744 close(dev
->backend_fd
);
1746 dev
->backend_fd
= vmsg
->fds
[0];
1747 DPRINT("Got backend_fd: %d\n", vmsg
->fds
[0]);
1753 vu_get_config(VuDev
*dev
, VhostUserMsg
*vmsg
)
1757 if (dev
->iface
->get_config
) {
1758 ret
= dev
->iface
->get_config(dev
, vmsg
->payload
.config
.region
,
1759 vmsg
->payload
.config
.size
);
1763 /* resize to zero to indicate an error to frontend */
1771 vu_set_config(VuDev
*dev
, VhostUserMsg
*vmsg
)
1775 if (dev
->iface
->set_config
) {
1776 ret
= dev
->iface
->set_config(dev
, vmsg
->payload
.config
.region
,
1777 vmsg
->payload
.config
.offset
,
1778 vmsg
->payload
.config
.size
,
1779 vmsg
->payload
.config
.flags
);
1781 vu_panic(dev
, "Set virtio configuration space failed");
1789 vu_set_postcopy_advise(VuDev
*dev
, VhostUserMsg
*vmsg
)
1792 struct uffdio_api api_struct
;
1794 dev
->postcopy_ufd
= syscall(__NR_userfaultfd
, O_CLOEXEC
| O_NONBLOCK
);
1797 dev
->postcopy_ufd
= -1;
1800 if (dev
->postcopy_ufd
== -1) {
1801 vu_panic(dev
, "Userfaultfd not available: %s", strerror(errno
));
1806 api_struct
.api
= UFFD_API
;
1807 api_struct
.features
= 0;
1808 if (ioctl(dev
->postcopy_ufd
, UFFDIO_API
, &api_struct
)) {
1809 vu_panic(dev
, "Failed UFFDIO_API: %s", strerror(errno
));
1810 close(dev
->postcopy_ufd
);
1811 dev
->postcopy_ufd
= -1;
1814 /* TODO: Stash feature flags somewhere */
1818 /* Return a ufd to the QEMU */
1820 vmsg
->fds
[0] = dev
->postcopy_ufd
;
1821 return true; /* = send a reply */
1825 vu_set_postcopy_listen(VuDev
*dev
, VhostUserMsg
*vmsg
)
1827 if (dev
->nregions
) {
1828 vu_panic(dev
, "Regions already registered at postcopy-listen");
1829 vmsg_set_reply_u64(vmsg
, -1);
1832 dev
->postcopy_listening
= true;
1834 vmsg_set_reply_u64(vmsg
, 0);
1839 vu_set_postcopy_end(VuDev
*dev
, VhostUserMsg
*vmsg
)
1841 DPRINT("%s: Entry\n", __func__
);
1842 dev
->postcopy_listening
= false;
1843 if (dev
->postcopy_ufd
> 0) {
1844 close(dev
->postcopy_ufd
);
1845 dev
->postcopy_ufd
= -1;
1846 DPRINT("%s: Done close\n", __func__
);
1849 vmsg_set_reply_u64(vmsg
, 0);
1850 DPRINT("%s: exit\n", __func__
);
1854 static inline uint64_t
1855 vu_inflight_queue_size(uint16_t queue_size
)
1857 return ALIGN_UP(sizeof(VuDescStateSplit
) * queue_size
+
1858 sizeof(uint16_t), INFLIGHT_ALIGNMENT
);
1861 #ifdef MFD_ALLOW_SEALING
1863 memfd_alloc(const char *name
, size_t size
, unsigned int flags
, int *fd
)
1868 *fd
= memfd_create(name
, MFD_ALLOW_SEALING
);
1873 ret
= ftruncate(*fd
, size
);
1879 ret
= fcntl(*fd
, F_ADD_SEALS
, flags
);
1885 ptr
= mmap(0, size
, PROT_READ
| PROT_WRITE
, MAP_SHARED
, *fd
, 0);
1886 if (ptr
== MAP_FAILED
) {
1896 vu_get_inflight_fd(VuDev
*dev
, VhostUserMsg
*vmsg
)
1901 uint16_t num_queues
, queue_size
;
1903 if (vmsg
->size
!= sizeof(vmsg
->payload
.inflight
)) {
1904 vu_panic(dev
, "Invalid get_inflight_fd message:%d", vmsg
->size
);
1905 vmsg
->payload
.inflight
.mmap_size
= 0;
1909 num_queues
= vmsg
->payload
.inflight
.num_queues
;
1910 queue_size
= vmsg
->payload
.inflight
.queue_size
;
1912 DPRINT("set_inflight_fd num_queues: %"PRId16
"\n", num_queues
);
1913 DPRINT("set_inflight_fd queue_size: %"PRId16
"\n", queue_size
);
1915 mmap_size
= vu_inflight_queue_size(queue_size
) * num_queues
;
1917 #ifdef MFD_ALLOW_SEALING
1918 addr
= memfd_alloc("vhost-inflight", mmap_size
,
1919 F_SEAL_GROW
| F_SEAL_SHRINK
| F_SEAL_SEAL
,
1922 vu_panic(dev
, "Not implemented: memfd support is missing");
1926 vu_panic(dev
, "Failed to alloc vhost inflight area");
1927 vmsg
->payload
.inflight
.mmap_size
= 0;
1931 memset(addr
, 0, mmap_size
);
1933 dev
->inflight_info
.addr
= addr
;
1934 dev
->inflight_info
.size
= vmsg
->payload
.inflight
.mmap_size
= mmap_size
;
1935 dev
->inflight_info
.fd
= vmsg
->fds
[0] = fd
;
1937 vmsg
->payload
.inflight
.mmap_offset
= 0;
1939 DPRINT("send inflight mmap_size: %"PRId64
"\n",
1940 vmsg
->payload
.inflight
.mmap_size
);
1941 DPRINT("send inflight mmap offset: %"PRId64
"\n",
1942 vmsg
->payload
.inflight
.mmap_offset
);
1948 vu_set_inflight_fd(VuDev
*dev
, VhostUserMsg
*vmsg
)
1951 uint64_t mmap_size
, mmap_offset
;
1952 uint16_t num_queues
, queue_size
;
1955 if (vmsg
->fd_num
!= 1 ||
1956 vmsg
->size
!= sizeof(vmsg
->payload
.inflight
)) {
1957 vu_panic(dev
, "Invalid set_inflight_fd message size:%d fds:%d",
1958 vmsg
->size
, vmsg
->fd_num
);
1963 mmap_size
= vmsg
->payload
.inflight
.mmap_size
;
1964 mmap_offset
= vmsg
->payload
.inflight
.mmap_offset
;
1965 num_queues
= vmsg
->payload
.inflight
.num_queues
;
1966 queue_size
= vmsg
->payload
.inflight
.queue_size
;
1968 DPRINT("set_inflight_fd mmap_size: %"PRId64
"\n", mmap_size
);
1969 DPRINT("set_inflight_fd mmap_offset: %"PRId64
"\n", mmap_offset
);
1970 DPRINT("set_inflight_fd num_queues: %"PRId16
"\n", num_queues
);
1971 DPRINT("set_inflight_fd queue_size: %"PRId16
"\n", queue_size
);
1973 rc
= mmap(0, mmap_size
, PROT_READ
| PROT_WRITE
, MAP_SHARED
,
1976 if (rc
== MAP_FAILED
) {
1977 vu_panic(dev
, "set_inflight_fd mmap error: %s", strerror(errno
));
1981 if (dev
->inflight_info
.fd
) {
1982 close(dev
->inflight_info
.fd
);
1985 if (dev
->inflight_info
.addr
) {
1986 munmap(dev
->inflight_info
.addr
, dev
->inflight_info
.size
);
1989 dev
->inflight_info
.fd
= fd
;
1990 dev
->inflight_info
.addr
= rc
;
1991 dev
->inflight_info
.size
= mmap_size
;
1993 for (i
= 0; i
< num_queues
; i
++) {
1994 dev
->vq
[i
].inflight
= (VuVirtqInflight
*)rc
;
1995 dev
->vq
[i
].inflight
->desc_num
= queue_size
;
1996 rc
= (void *)((char *)rc
+ vu_inflight_queue_size(queue_size
));
2003 vu_handle_vring_kick(VuDev
*dev
, VhostUserMsg
*vmsg
)
2005 unsigned int index
= vmsg
->payload
.state
.index
;
2007 if (index
>= dev
->max_queues
) {
2008 vu_panic(dev
, "Invalid queue index: %u", index
);
2012 DPRINT("Got kick message: handler:%p idx:%u\n",
2013 dev
->vq
[index
].handler
, index
);
2015 if (!dev
->vq
[index
].started
) {
2016 dev
->vq
[index
].started
= true;
2018 if (dev
->iface
->queue_set_started
) {
2019 dev
->iface
->queue_set_started(dev
, index
, true);
2023 if (dev
->vq
[index
].handler
) {
2024 dev
->vq
[index
].handler(dev
, index
);
2030 static bool vu_handle_get_max_memslots(VuDev
*dev
, VhostUserMsg
*vmsg
)
2032 vmsg_set_reply_u64(vmsg
, VHOST_USER_MAX_RAM_SLOTS
);
2034 DPRINT("u64: 0x%016"PRIx64
"\n", (uint64_t) VHOST_USER_MAX_RAM_SLOTS
);
2040 vu_process_message(VuDev
*dev
, VhostUserMsg
*vmsg
)
2044 /* Print out generic part of the request. */
2045 DPRINT("================ Vhost user message ================\n");
2046 DPRINT("Request: %s (%d)\n", vu_request_to_string(vmsg
->request
),
2048 DPRINT("Flags: 0x%x\n", vmsg
->flags
);
2049 DPRINT("Size: %u\n", vmsg
->size
);
2054 for (i
= 0; i
< vmsg
->fd_num
; i
++) {
2055 DPRINT(" %d", vmsg
->fds
[i
]);
2060 if (dev
->iface
->process_msg
&&
2061 dev
->iface
->process_msg(dev
, vmsg
, &do_reply
)) {
2065 switch (vmsg
->request
) {
2066 case VHOST_USER_GET_FEATURES
:
2067 return vu_get_features_exec(dev
, vmsg
);
2068 case VHOST_USER_SET_FEATURES
:
2069 return vu_set_features_exec(dev
, vmsg
);
2070 case VHOST_USER_GET_PROTOCOL_FEATURES
:
2071 return vu_get_protocol_features_exec(dev
, vmsg
);
2072 case VHOST_USER_SET_PROTOCOL_FEATURES
:
2073 return vu_set_protocol_features_exec(dev
, vmsg
);
2074 case VHOST_USER_SET_OWNER
:
2075 return vu_set_owner_exec(dev
, vmsg
);
2076 case VHOST_USER_RESET_OWNER
:
2077 return vu_reset_device_exec(dev
, vmsg
);
2078 case VHOST_USER_SET_MEM_TABLE
:
2079 return vu_set_mem_table_exec(dev
, vmsg
);
2080 case VHOST_USER_SET_LOG_BASE
:
2081 return vu_set_log_base_exec(dev
, vmsg
);
2082 case VHOST_USER_SET_LOG_FD
:
2083 return vu_set_log_fd_exec(dev
, vmsg
);
2084 case VHOST_USER_SET_VRING_NUM
:
2085 return vu_set_vring_num_exec(dev
, vmsg
);
2086 case VHOST_USER_SET_VRING_ADDR
:
2087 return vu_set_vring_addr_exec(dev
, vmsg
);
2088 case VHOST_USER_SET_VRING_BASE
:
2089 return vu_set_vring_base_exec(dev
, vmsg
);
2090 case VHOST_USER_GET_VRING_BASE
:
2091 return vu_get_vring_base_exec(dev
, vmsg
);
2092 case VHOST_USER_SET_VRING_KICK
:
2093 return vu_set_vring_kick_exec(dev
, vmsg
);
2094 case VHOST_USER_SET_VRING_CALL
:
2095 return vu_set_vring_call_exec(dev
, vmsg
);
2096 case VHOST_USER_SET_VRING_ERR
:
2097 return vu_set_vring_err_exec(dev
, vmsg
);
2098 case VHOST_USER_GET_QUEUE_NUM
:
2099 return vu_get_queue_num_exec(dev
, vmsg
);
2100 case VHOST_USER_SET_VRING_ENABLE
:
2101 return vu_set_vring_enable_exec(dev
, vmsg
);
2102 case VHOST_USER_SET_BACKEND_REQ_FD
:
2103 return vu_set_backend_req_fd(dev
, vmsg
);
2104 case VHOST_USER_GET_CONFIG
:
2105 return vu_get_config(dev
, vmsg
);
2106 case VHOST_USER_SET_CONFIG
:
2107 return vu_set_config(dev
, vmsg
);
2108 case VHOST_USER_NONE
:
2109 /* if you need processing before exit, override iface->process_msg */
2111 case VHOST_USER_POSTCOPY_ADVISE
:
2112 return vu_set_postcopy_advise(dev
, vmsg
);
2113 case VHOST_USER_POSTCOPY_LISTEN
:
2114 return vu_set_postcopy_listen(dev
, vmsg
);
2115 case VHOST_USER_POSTCOPY_END
:
2116 return vu_set_postcopy_end(dev
, vmsg
);
2117 case VHOST_USER_GET_INFLIGHT_FD
:
2118 return vu_get_inflight_fd(dev
, vmsg
);
2119 case VHOST_USER_SET_INFLIGHT_FD
:
2120 return vu_set_inflight_fd(dev
, vmsg
);
2121 case VHOST_USER_VRING_KICK
:
2122 return vu_handle_vring_kick(dev
, vmsg
);
2123 case VHOST_USER_GET_MAX_MEM_SLOTS
:
2124 return vu_handle_get_max_memslots(dev
, vmsg
);
2125 case VHOST_USER_ADD_MEM_REG
:
2126 return vu_add_mem_reg(dev
, vmsg
);
2127 case VHOST_USER_REM_MEM_REG
:
2128 return vu_rem_mem_reg(dev
, vmsg
);
2129 case VHOST_USER_GET_SHARED_OBJECT
:
2130 return vu_get_shared_object(dev
, vmsg
);
2132 vmsg_close_fds(vmsg
);
2133 vu_panic(dev
, "Unhandled request: %d", vmsg
->request
);
2140 vu_dispatch(VuDev
*dev
)
2142 VhostUserMsg vmsg
= { 0, };
2143 int reply_requested
;
2144 bool need_reply
, success
= false;
2146 if (!dev
->read_msg(dev
, dev
->sock
, &vmsg
)) {
2150 need_reply
= vmsg
.flags
& VHOST_USER_NEED_REPLY_MASK
;
2152 reply_requested
= vu_process_message(dev
, &vmsg
);
2153 if (!reply_requested
&& need_reply
) {
2154 vmsg_set_reply_u64(&vmsg
, 0);
2155 reply_requested
= 1;
2158 if (!reply_requested
) {
2163 if (!vu_send_reply(dev
, dev
->sock
, &vmsg
)) {
2175 vu_deinit(VuDev
*dev
)
2179 vu_remove_all_mem_regs(dev
);
2181 for (i
= 0; i
< dev
->max_queues
; i
++) {
2182 VuVirtq
*vq
= &dev
->vq
[i
];
2184 if (vq
->call_fd
!= -1) {
2189 if (vq
->kick_fd
!= -1) {
2190 dev
->remove_watch(dev
, vq
->kick_fd
);
2195 if (vq
->err_fd
!= -1) {
2200 if (vq
->resubmit_list
) {
2201 free(vq
->resubmit_list
);
2202 vq
->resubmit_list
= NULL
;
2205 vq
->inflight
= NULL
;
2208 if (dev
->inflight_info
.addr
) {
2209 munmap(dev
->inflight_info
.addr
, dev
->inflight_info
.size
);
2210 dev
->inflight_info
.addr
= NULL
;
2213 if (dev
->inflight_info
.fd
> 0) {
2214 close(dev
->inflight_info
.fd
);
2215 dev
->inflight_info
.fd
= -1;
2219 if (dev
->backend_fd
!= -1) {
2220 close(dev
->backend_fd
);
2221 dev
->backend_fd
= -1;
2223 pthread_mutex_destroy(&dev
->backend_mutex
);
2225 if (dev
->sock
!= -1) {
2232 dev
->regions
= NULL
;
2237 uint16_t max_queues
,
2240 vu_read_msg_cb read_msg
,
2241 vu_set_watch_cb set_watch
,
2242 vu_remove_watch_cb remove_watch
,
2243 const VuDevIface
*iface
)
2247 assert(max_queues
> 0);
2248 assert(socket
>= 0);
2250 assert(remove_watch
);
2254 memset(dev
, 0, sizeof(*dev
));
2258 dev
->read_msg
= read_msg
? read_msg
: vu_message_read_default
;
2259 dev
->set_watch
= set_watch
;
2260 dev
->remove_watch
= remove_watch
;
2262 dev
->log_call_fd
= -1;
2263 pthread_mutex_init(&dev
->backend_mutex
, NULL
);
2264 dev
->backend_fd
= -1;
2265 dev
->max_queues
= max_queues
;
2267 dev
->regions
= malloc(VHOST_USER_MAX_RAM_SLOTS
* sizeof(dev
->regions
[0]));
2268 if (!dev
->regions
) {
2269 DPRINT("%s: failed to malloc mem regions\n", __func__
);
2273 dev
->vq
= malloc(max_queues
* sizeof(dev
->vq
[0]));
2275 DPRINT("%s: failed to malloc virtqueues\n", __func__
);
2277 dev
->regions
= NULL
;
2281 for (i
= 0; i
< max_queues
; i
++) {
2282 dev
->vq
[i
] = (VuVirtq
) {
2283 .call_fd
= -1, .kick_fd
= -1, .err_fd
= -1,
2284 .notification
= true,
2292 vu_get_queue(VuDev
*dev
, int qidx
)
2294 assert(qidx
< dev
->max_queues
);
2295 return &dev
->vq
[qidx
];
2299 vu_queue_enabled(VuDev
*dev
, VuVirtq
*vq
)
2305 vu_queue_started(const VuDev
*dev
, const VuVirtq
*vq
)
2310 static inline uint16_t
2311 vring_avail_flags(VuVirtq
*vq
)
2313 return le16toh(vq
->vring
.avail
->flags
);
2316 static inline uint16_t
2317 vring_avail_idx(VuVirtq
*vq
)
2319 vq
->shadow_avail_idx
= le16toh(vq
->vring
.avail
->idx
);
2321 return vq
->shadow_avail_idx
;
2324 static inline uint16_t
2325 vring_avail_ring(VuVirtq
*vq
, int i
)
2327 return le16toh(vq
->vring
.avail
->ring
[i
]);
2330 static inline uint16_t
2331 vring_get_used_event(VuVirtq
*vq
)
2333 return vring_avail_ring(vq
, vq
->vring
.num
);
2337 virtqueue_num_heads(VuDev
*dev
, VuVirtq
*vq
, unsigned int idx
)
2339 uint16_t num_heads
= vring_avail_idx(vq
) - idx
;
2341 /* Check it isn't doing very strange things with descriptor numbers. */
2342 if (num_heads
> vq
->vring
.num
) {
2343 vu_panic(dev
, "Guest moved used index from %u to %u",
2344 idx
, vq
->shadow_avail_idx
);
2348 /* On success, callers read a descriptor at vq->last_avail_idx.
2349 * Make sure descriptor read does not bypass avail index read. */
2357 virtqueue_get_head(VuDev
*dev
, VuVirtq
*vq
,
2358 unsigned int idx
, unsigned int *head
)
2360 /* Grab the next descriptor number they're advertising, and increment
2361 * the index we've seen. */
2362 *head
= vring_avail_ring(vq
, idx
% vq
->vring
.num
);
2364 /* If their number is silly, that's a fatal mistake. */
2365 if (*head
>= vq
->vring
.num
) {
2366 vu_panic(dev
, "Guest says index %u is available", *head
);
2374 virtqueue_read_indirect_desc(VuDev
*dev
, struct vring_desc
*desc
,
2375 uint64_t addr
, size_t len
)
2377 struct vring_desc
*ori_desc
;
2380 if (len
> (VIRTQUEUE_MAX_SIZE
* sizeof(struct vring_desc
))) {
2390 ori_desc
= vu_gpa_to_va(dev
, &read_len
, addr
);
2395 memcpy(desc
, ori_desc
, read_len
);
2405 VIRTQUEUE_READ_DESC_ERROR
= -1,
2406 VIRTQUEUE_READ_DESC_DONE
= 0, /* end of chain */
2407 VIRTQUEUE_READ_DESC_MORE
= 1, /* more buffers in chain */
2411 virtqueue_read_next_desc(VuDev
*dev
, struct vring_desc
*desc
,
2412 int i
, unsigned int max
, unsigned int *next
)
2414 /* If this descriptor says it doesn't chain, we're done. */
2415 if (!(le16toh(desc
[i
].flags
) & VRING_DESC_F_NEXT
)) {
2416 return VIRTQUEUE_READ_DESC_DONE
;
2419 /* Check they're not leading us off end of descriptors. */
2420 *next
= le16toh(desc
[i
].next
);
2421 /* Make sure compiler knows to grab that: we don't want it changing! */
2425 vu_panic(dev
, "Desc next is %u", *next
);
2426 return VIRTQUEUE_READ_DESC_ERROR
;
2429 return VIRTQUEUE_READ_DESC_MORE
;
2433 vu_queue_get_avail_bytes(VuDev
*dev
, VuVirtq
*vq
, unsigned int *in_bytes
,
2434 unsigned int *out_bytes
,
2435 unsigned max_in_bytes
, unsigned max_out_bytes
)
2438 unsigned int total_bufs
, in_total
, out_total
;
2441 idx
= vq
->last_avail_idx
;
2443 total_bufs
= in_total
= out_total
= 0;
2444 if (!vu_is_vq_usable(dev
, vq
)) {
2448 while ((rc
= virtqueue_num_heads(dev
, vq
, idx
)) > 0) {
2449 unsigned int max
, desc_len
, num_bufs
, indirect
= 0;
2450 uint64_t desc_addr
, read_len
;
2451 struct vring_desc
*desc
;
2452 struct vring_desc desc_buf
[VIRTQUEUE_MAX_SIZE
];
2455 max
= vq
->vring
.num
;
2456 num_bufs
= total_bufs
;
2457 if (!virtqueue_get_head(dev
, vq
, idx
++, &i
)) {
2460 desc
= vq
->vring
.desc
;
2462 if (le16toh(desc
[i
].flags
) & VRING_DESC_F_INDIRECT
) {
2463 if (le32toh(desc
[i
].len
) % sizeof(struct vring_desc
)) {
2464 vu_panic(dev
, "Invalid size for indirect buffer table");
2468 /* If we've got too many, that implies a descriptor loop. */
2469 if (num_bufs
>= max
) {
2470 vu_panic(dev
, "Looped descriptor");
2474 /* loop over the indirect descriptor table */
2476 desc_addr
= le64toh(desc
[i
].addr
);
2477 desc_len
= le32toh(desc
[i
].len
);
2478 max
= desc_len
/ sizeof(struct vring_desc
);
2479 read_len
= desc_len
;
2480 desc
= vu_gpa_to_va(dev
, &read_len
, desc_addr
);
2481 if (unlikely(desc
&& read_len
!= desc_len
)) {
2482 /* Failed to use zero copy */
2484 if (!virtqueue_read_indirect_desc(dev
, desc_buf
,
2491 vu_panic(dev
, "Invalid indirect buffer table");
2498 /* If we've got too many, that implies a descriptor loop. */
2499 if (++num_bufs
> max
) {
2500 vu_panic(dev
, "Looped descriptor");
2504 if (le16toh(desc
[i
].flags
) & VRING_DESC_F_WRITE
) {
2505 in_total
+= le32toh(desc
[i
].len
);
2507 out_total
+= le32toh(desc
[i
].len
);
2509 if (in_total
>= max_in_bytes
&& out_total
>= max_out_bytes
) {
2512 rc
= virtqueue_read_next_desc(dev
, desc
, i
, max
, &i
);
2513 } while (rc
== VIRTQUEUE_READ_DESC_MORE
);
2515 if (rc
== VIRTQUEUE_READ_DESC_ERROR
) {
2520 total_bufs
= num_bufs
;
2530 *in_bytes
= in_total
;
2533 *out_bytes
= out_total
;
2538 in_total
= out_total
= 0;
2543 vu_queue_avail_bytes(VuDev
*dev
, VuVirtq
*vq
, unsigned int in_bytes
,
2544 unsigned int out_bytes
)
2546 unsigned int in_total
, out_total
;
2548 vu_queue_get_avail_bytes(dev
, vq
, &in_total
, &out_total
,
2549 in_bytes
, out_bytes
);
2551 return in_bytes
<= in_total
&& out_bytes
<= out_total
;
2554 /* Fetch avail_idx from VQ memory only when we really need to know if
2555 * guest has added some buffers. */
2557 vu_queue_empty(VuDev
*dev
, VuVirtq
*vq
)
2559 if (!vu_is_vq_usable(dev
, vq
)) {
2563 if (vq
->shadow_avail_idx
!= vq
->last_avail_idx
) {
2567 return vring_avail_idx(vq
) == vq
->last_avail_idx
;
2571 vring_notify(VuDev
*dev
, VuVirtq
*vq
)
2576 /* We need to expose used array entries before checking used event. */
2579 /* Always notify when queue is empty (when feature acknowledge) */
2580 if (vu_has_feature(dev
, VIRTIO_F_NOTIFY_ON_EMPTY
) &&
2581 !vq
->inuse
&& vu_queue_empty(dev
, vq
)) {
2585 if (!vu_has_feature(dev
, VIRTIO_RING_F_EVENT_IDX
)) {
2586 return !(vring_avail_flags(vq
) & VRING_AVAIL_F_NO_INTERRUPT
);
2589 v
= vq
->signalled_used_valid
;
2590 vq
->signalled_used_valid
= true;
2591 old
= vq
->signalled_used
;
2592 new = vq
->signalled_used
= vq
->used_idx
;
2593 return !v
|| vring_need_event(vring_get_used_event(vq
), new, old
);
2596 static void _vu_queue_notify(VuDev
*dev
, VuVirtq
*vq
, bool sync
)
2598 if (!vu_is_vq_usable(dev
, vq
)) {
2602 if (!vring_notify(dev
, vq
)) {
2603 DPRINT("skipped notify...\n");
2607 if (vq
->call_fd
< 0 &&
2608 vu_has_protocol_feature(dev
,
2609 VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS
) &&
2610 vu_has_protocol_feature(dev
, VHOST_USER_PROTOCOL_F_BACKEND_REQ
)) {
2611 VhostUserMsg vmsg
= {
2612 .request
= VHOST_USER_BACKEND_VRING_CALL
,
2613 .flags
= VHOST_USER_VERSION
,
2614 .size
= sizeof(vmsg
.payload
.state
),
2616 .index
= vq
- dev
->vq
,
2620 vu_has_protocol_feature(dev
,
2621 VHOST_USER_PROTOCOL_F_REPLY_ACK
);
2624 vmsg
.flags
|= VHOST_USER_NEED_REPLY_MASK
;
2627 vu_message_write(dev
, dev
->backend_fd
, &vmsg
);
2629 vu_message_read_default(dev
, dev
->backend_fd
, &vmsg
);
2634 if (eventfd_write(vq
->call_fd
, 1) < 0) {
2635 vu_panic(dev
, "Error writing eventfd: %s", strerror(errno
));
2639 void vu_queue_notify(VuDev
*dev
, VuVirtq
*vq
)
2641 _vu_queue_notify(dev
, vq
, false);
2644 void vu_queue_notify_sync(VuDev
*dev
, VuVirtq
*vq
)
2646 _vu_queue_notify(dev
, vq
, true);
2649 void vu_config_change_msg(VuDev
*dev
)
2651 VhostUserMsg vmsg
= {
2652 .request
= VHOST_USER_BACKEND_CONFIG_CHANGE_MSG
,
2653 .flags
= VHOST_USER_VERSION
,
2656 vu_message_write(dev
, dev
->backend_fd
, &vmsg
);
2660 vring_used_flags_set_bit(VuVirtq
*vq
, int mask
)
2664 flags
= (uint16_t *)((char*)vq
->vring
.used
+
2665 offsetof(struct vring_used
, flags
));
2666 *flags
= htole16(le16toh(*flags
) | mask
);
2670 vring_used_flags_unset_bit(VuVirtq
*vq
, int mask
)
2674 flags
= (uint16_t *)((char*)vq
->vring
.used
+
2675 offsetof(struct vring_used
, flags
));
2676 *flags
= htole16(le16toh(*flags
) & ~mask
);
2680 vring_set_avail_event(VuVirtq
*vq
, uint16_t val
)
2682 uint16_t val_le
= htole16(val
);
2684 if (!vq
->notification
) {
2688 memcpy(&vq
->vring
.used
->ring
[vq
->vring
.num
], &val_le
, sizeof(uint16_t));
2692 vu_queue_set_notification(VuDev
*dev
, VuVirtq
*vq
, int enable
)
2694 vq
->notification
= enable
;
2695 if (vu_has_feature(dev
, VIRTIO_RING_F_EVENT_IDX
)) {
2696 vring_set_avail_event(vq
, vring_avail_idx(vq
));
2697 } else if (enable
) {
2698 vring_used_flags_unset_bit(vq
, VRING_USED_F_NO_NOTIFY
);
2700 vring_used_flags_set_bit(vq
, VRING_USED_F_NO_NOTIFY
);
2703 /* Expose avail event/used flags before caller checks the avail idx. */
2709 virtqueue_map_desc(VuDev
*dev
,
2710 unsigned int *p_num_sg
, struct iovec
*iov
,
2711 unsigned int max_num_sg
, bool is_write
,
2712 uint64_t pa
, size_t sz
)
2714 unsigned num_sg
= *p_num_sg
;
2716 assert(num_sg
<= max_num_sg
);
2719 vu_panic(dev
, "virtio: zero sized buffers are not allowed");
2726 if (num_sg
== max_num_sg
) {
2727 vu_panic(dev
, "virtio: too many descriptors in indirect table");
2731 iov
[num_sg
].iov_base
= vu_gpa_to_va(dev
, &len
, pa
);
2732 if (iov
[num_sg
].iov_base
== NULL
) {
2733 vu_panic(dev
, "virtio: invalid address for buffers");
2736 iov
[num_sg
].iov_len
= len
;
2747 virtqueue_alloc_element(size_t sz
,
2748 unsigned out_num
, unsigned in_num
)
2750 VuVirtqElement
*elem
;
2751 size_t in_sg_ofs
= ALIGN_UP(sz
, __alignof__(elem
->in_sg
[0]));
2752 size_t out_sg_ofs
= in_sg_ofs
+ in_num
* sizeof(elem
->in_sg
[0]);
2753 size_t out_sg_end
= out_sg_ofs
+ out_num
* sizeof(elem
->out_sg
[0]);
2755 assert(sz
>= sizeof(VuVirtqElement
));
2756 elem
= malloc(out_sg_end
);
2758 DPRINT("%s: failed to malloc virtqueue element\n", __func__
);
2761 elem
->out_num
= out_num
;
2762 elem
->in_num
= in_num
;
2763 elem
->in_sg
= (void *)elem
+ in_sg_ofs
;
2764 elem
->out_sg
= (void *)elem
+ out_sg_ofs
;
2769 vu_queue_map_desc(VuDev
*dev
, VuVirtq
*vq
, unsigned int idx
, size_t sz
)
2771 struct vring_desc
*desc
= vq
->vring
.desc
;
2772 uint64_t desc_addr
, read_len
;
2773 unsigned int desc_len
;
2774 unsigned int max
= vq
->vring
.num
;
2775 unsigned int i
= idx
;
2776 VuVirtqElement
*elem
;
2777 unsigned int out_num
= 0, in_num
= 0;
2778 struct iovec iov
[VIRTQUEUE_MAX_SIZE
];
2779 struct vring_desc desc_buf
[VIRTQUEUE_MAX_SIZE
];
2782 if (le16toh(desc
[i
].flags
) & VRING_DESC_F_INDIRECT
) {
2783 if (le32toh(desc
[i
].len
) % sizeof(struct vring_desc
)) {
2784 vu_panic(dev
, "Invalid size for indirect buffer table");
2788 /* loop over the indirect descriptor table */
2789 desc_addr
= le64toh(desc
[i
].addr
);
2790 desc_len
= le32toh(desc
[i
].len
);
2791 max
= desc_len
/ sizeof(struct vring_desc
);
2792 read_len
= desc_len
;
2793 desc
= vu_gpa_to_va(dev
, &read_len
, desc_addr
);
2794 if (unlikely(desc
&& read_len
!= desc_len
)) {
2795 /* Failed to use zero copy */
2797 if (!virtqueue_read_indirect_desc(dev
, desc_buf
,
2804 vu_panic(dev
, "Invalid indirect buffer table");
2810 /* Collect all the descriptors */
2812 if (le16toh(desc
[i
].flags
) & VRING_DESC_F_WRITE
) {
2813 if (!virtqueue_map_desc(dev
, &in_num
, iov
+ out_num
,
2814 VIRTQUEUE_MAX_SIZE
- out_num
, true,
2815 le64toh(desc
[i
].addr
),
2816 le32toh(desc
[i
].len
))) {
2821 vu_panic(dev
, "Incorrect order for descriptors");
2824 if (!virtqueue_map_desc(dev
, &out_num
, iov
,
2825 VIRTQUEUE_MAX_SIZE
, false,
2826 le64toh(desc
[i
].addr
),
2827 le32toh(desc
[i
].len
))) {
2832 /* If we've got too many, that implies a descriptor loop. */
2833 if ((in_num
+ out_num
) > max
) {
2834 vu_panic(dev
, "Looped descriptor");
2837 rc
= virtqueue_read_next_desc(dev
, desc
, i
, max
, &i
);
2838 } while (rc
== VIRTQUEUE_READ_DESC_MORE
);
2840 if (rc
== VIRTQUEUE_READ_DESC_ERROR
) {
2841 vu_panic(dev
, "read descriptor error");
2845 /* Now copy what we have collected and mapped */
2846 elem
= virtqueue_alloc_element(sz
, out_num
, in_num
);
2851 for (i
= 0; i
< out_num
; i
++) {
2852 elem
->out_sg
[i
] = iov
[i
];
2854 for (i
= 0; i
< in_num
; i
++) {
2855 elem
->in_sg
[i
] = iov
[out_num
+ i
];
2862 vu_queue_inflight_get(VuDev
*dev
, VuVirtq
*vq
, int desc_idx
)
2864 if (!vu_has_protocol_feature(dev
, VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD
)) {
2868 if (unlikely(!vq
->inflight
)) {
2872 vq
->inflight
->desc
[desc_idx
].counter
= vq
->counter
++;
2873 vq
->inflight
->desc
[desc_idx
].inflight
= 1;
2879 vu_queue_inflight_pre_put(VuDev
*dev
, VuVirtq
*vq
, int desc_idx
)
2881 if (!vu_has_protocol_feature(dev
, VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD
)) {
2885 if (unlikely(!vq
->inflight
)) {
2889 vq
->inflight
->last_batch_head
= desc_idx
;
2895 vu_queue_inflight_post_put(VuDev
*dev
, VuVirtq
*vq
, int desc_idx
)
2897 if (!vu_has_protocol_feature(dev
, VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD
)) {
2901 if (unlikely(!vq
->inflight
)) {
2907 vq
->inflight
->desc
[desc_idx
].inflight
= 0;
2911 vq
->inflight
->used_idx
= vq
->used_idx
;
2917 vu_queue_pop(VuDev
*dev
, VuVirtq
*vq
, size_t sz
)
2921 VuVirtqElement
*elem
;
2923 if (!vu_is_vq_usable(dev
, vq
)) {
2927 if (unlikely(vq
->resubmit_list
&& vq
->resubmit_num
> 0)) {
2928 i
= (--vq
->resubmit_num
);
2929 elem
= vu_queue_map_desc(dev
, vq
, vq
->resubmit_list
[i
].index
, sz
);
2931 if (!vq
->resubmit_num
) {
2932 free(vq
->resubmit_list
);
2933 vq
->resubmit_list
= NULL
;
2939 if (vu_queue_empty(dev
, vq
)) {
2943 * Needed after virtio_queue_empty(), see comment in
2944 * virtqueue_num_heads().
2948 if (vq
->inuse
>= vq
->vring
.num
) {
2949 vu_panic(dev
, "Virtqueue size exceeded");
2953 if (!virtqueue_get_head(dev
, vq
, vq
->last_avail_idx
++, &head
)) {
2957 if (vu_has_feature(dev
, VIRTIO_RING_F_EVENT_IDX
)) {
2958 vring_set_avail_event(vq
, vq
->last_avail_idx
);
2961 elem
= vu_queue_map_desc(dev
, vq
, head
, sz
);
2969 vu_queue_inflight_get(dev
, vq
, head
);
2975 vu_queue_detach_element(VuDev
*dev
, VuVirtq
*vq
, VuVirtqElement
*elem
,
2979 /* unmap, when DMA support is added */
2983 vu_queue_unpop(VuDev
*dev
, VuVirtq
*vq
, VuVirtqElement
*elem
,
2986 vq
->last_avail_idx
--;
2987 vu_queue_detach_element(dev
, vq
, elem
, len
);
2991 vu_queue_rewind(VuDev
*dev
, VuVirtq
*vq
, unsigned int num
)
2993 if (num
> vq
->inuse
) {
2996 vq
->last_avail_idx
-= num
;
3002 void vring_used_write(VuDev
*dev
, VuVirtq
*vq
,
3003 struct vring_used_elem
*uelem
, int i
)
3005 struct vring_used
*used
= vq
->vring
.used
;
3007 used
->ring
[i
] = *uelem
;
3008 vu_log_write(dev
, vq
->vring
.log_guest_addr
+
3009 offsetof(struct vring_used
, ring
[i
]),
3010 sizeof(used
->ring
[i
]));
3015 vu_log_queue_fill(VuDev
*dev
, VuVirtq
*vq
,
3016 const VuVirtqElement
*elem
,
3019 struct vring_desc
*desc
= vq
->vring
.desc
;
3020 unsigned int i
, max
, min
, desc_len
;
3021 uint64_t desc_addr
, read_len
;
3022 struct vring_desc desc_buf
[VIRTQUEUE_MAX_SIZE
];
3023 unsigned num_bufs
= 0;
3025 max
= vq
->vring
.num
;
3028 if (le16toh(desc
[i
].flags
) & VRING_DESC_F_INDIRECT
) {
3029 if (le32toh(desc
[i
].len
) % sizeof(struct vring_desc
)) {
3030 vu_panic(dev
, "Invalid size for indirect buffer table");
3034 /* loop over the indirect descriptor table */
3035 desc_addr
= le64toh(desc
[i
].addr
);
3036 desc_len
= le32toh(desc
[i
].len
);
3037 max
= desc_len
/ sizeof(struct vring_desc
);
3038 read_len
= desc_len
;
3039 desc
= vu_gpa_to_va(dev
, &read_len
, desc_addr
);
3040 if (unlikely(desc
&& read_len
!= desc_len
)) {
3041 /* Failed to use zero copy */
3043 if (!virtqueue_read_indirect_desc(dev
, desc_buf
,
3050 vu_panic(dev
, "Invalid indirect buffer table");
3057 if (++num_bufs
> max
) {
3058 vu_panic(dev
, "Looped descriptor");
3062 if (le16toh(desc
[i
].flags
) & VRING_DESC_F_WRITE
) {
3063 min
= MIN(le32toh(desc
[i
].len
), len
);
3064 vu_log_write(dev
, le64toh(desc
[i
].addr
), min
);
3069 (virtqueue_read_next_desc(dev
, desc
, i
, max
, &i
)
3070 == VIRTQUEUE_READ_DESC_MORE
));
3074 vu_queue_fill(VuDev
*dev
, VuVirtq
*vq
,
3075 const VuVirtqElement
*elem
,
3076 unsigned int len
, unsigned int idx
)
3078 struct vring_used_elem uelem
;
3080 if (!vu_is_vq_usable(dev
, vq
)) {
3084 vu_log_queue_fill(dev
, vq
, elem
, len
);
3086 idx
= (idx
+ vq
->used_idx
) % vq
->vring
.num
;
3088 uelem
.id
= htole32(elem
->index
);
3089 uelem
.len
= htole32(len
);
3090 vring_used_write(dev
, vq
, &uelem
, idx
);
3094 void vring_used_idx_set(VuDev
*dev
, VuVirtq
*vq
, uint16_t val
)
3096 vq
->vring
.used
->idx
= htole16(val
);
3098 vq
->vring
.log_guest_addr
+ offsetof(struct vring_used
, idx
),
3099 sizeof(vq
->vring
.used
->idx
));
3105 vu_queue_flush(VuDev
*dev
, VuVirtq
*vq
, unsigned int count
)
3109 if (!vu_is_vq_usable(dev
, vq
)) {
3113 /* Make sure buffer is written before we update index. */
3118 vring_used_idx_set(dev
, vq
, new);
3120 if (unlikely((int16_t)(new - vq
->signalled_used
) < (uint16_t)(new - old
))) {
3121 vq
->signalled_used_valid
= false;
3126 vu_queue_push(VuDev
*dev
, VuVirtq
*vq
,
3127 const VuVirtqElement
*elem
, unsigned int len
)
3129 vu_queue_fill(dev
, vq
, elem
, len
, 0);
3130 vu_queue_inflight_pre_put(dev
, vq
, elem
->index
);
3131 vu_queue_flush(dev
, vq
, 1);
3132 vu_queue_inflight_post_put(dev
, vq
, elem
->index
);