?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
libvhost-user: Factor out vq usability check
[qemu/ar7.git] / subprojects / libvhost-user / libvhost-user.c
blobed0a978d4f67a23396b4c240083981db2a988ab4
1 /*
2 * Vhost User library
3 *
4 * Copyright IBM, Corp. 2007
5 * Copyright (c) 2016 Red Hat, Inc.
6 *
7 * Authors:
8 * Anthony Liguori <aliguori@us.ibm.com>
9 * Marc-André Lureau <mlureau@redhat.com>
10 * Victor Kaplansky <victork@redhat.com>
11 *
12 * This work is licensed under the terms of the GNU GPL, version 2 or
13 * later. See the COPYING file in the top-level directory.
14 */
15
16 #ifndef _GNU_SOURCE
17 #define _GNU_SOURCE
18 #endif
19
20 /* this code avoids GLib dependency */
21 #include <stdlib.h>
22 #include <stdio.h>
23 #include <unistd.h>
24 #include <stdarg.h>
25 #include <errno.h>
26 #include <string.h>
27 #include <assert.h>
28 #include <inttypes.h>
29 #include <sys/types.h>
30 #include <sys/socket.h>
31 #include <sys/eventfd.h>
32 #include <sys/mman.h>
33 #include <endian.h>
34
35 /* Necessary to provide VIRTIO_F_VERSION_1 on system
36 * with older linux headers. Must appear before
37 * <linux/vhost.h> below.
38 */
39 #include "standard-headers/linux/virtio_config.h"
40
41 #if defined(__linux__)
42 #include <sys/syscall.h>
43 #include <fcntl.h>
44 #include <sys/ioctl.h>
45 #include <linux/vhost.h>
46 #include <sys/vfs.h>
47 #include <linux/magic.h>
48
49 #ifdef __NR_userfaultfd
50 #include <linux/userfaultfd.h>
51 #endif
52
53 #endif
54
55 #include "include/atomic.h"
56
57 #include "libvhost-user.h"
58
59 /* usually provided by GLib */
60 #if __GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ > 4)
61 #if !defined(__clang__) && (__GNUC__ == 4 && __GNUC_MINOR__ == 4)
62 #define G_GNUC_PRINTF(format_idx, arg_idx) \
63 __attribute__((__format__(gnu_printf, format_idx, arg_idx)))
64 #else
65 #define G_GNUC_PRINTF(format_idx, arg_idx) \
66 __attribute__((__format__(__printf__, format_idx, arg_idx)))
67 #endif
68 #else /* !__GNUC__ */
69 #define G_GNUC_PRINTF(format_idx, arg_idx)
70 #endif /* !__GNUC__ */
71 #ifndef MIN
72 #define MIN(x, y) ({ \
73 __typeof__(x) _min1 = (x); \
74 __typeof__(y) _min2 = (y); \
75 (void) (&_min1 == &_min2); \
76 _min1 < _min2 ? _min1 : _min2; })
77 #endif
78
79 /* Round number down to multiple */
80 #define ALIGN_DOWN(n, m) ((n) / (m) * (m))
81
82 /* Round number up to multiple */
83 #define ALIGN_UP(n, m) ALIGN_DOWN((n) + (m) - 1, (m))
84
85 #ifndef unlikely
86 #define unlikely(x) __builtin_expect(!!(x), 0)
87 #endif
88
89 /* Align each region to cache line size in inflight buffer */
90 #define INFLIGHT_ALIGNMENT 64
91
92 /* The version of inflight buffer */
93 #define INFLIGHT_VERSION 1
94
95 /* The version of the protocol we support */
96 #define VHOST_USER_VERSION 1
97 #define LIBVHOST_USER_DEBUG 0
98
99 #define DPRINT(...) \
100 do { \
101 if (LIBVHOST_USER_DEBUG) { \
102 fprintf(stderr, __VA_ARGS__); \
103 } \
104 } while (0)
105
106 static inline
107 bool has_feature(uint64_t features, unsigned int fbit)
108 {
109 assert(fbit < 64);
110 return !!(features & (1ULL << fbit));
111 }
112
113 static inline
114 bool vu_has_feature(VuDev *dev,
115 unsigned int fbit)
116 {
117 return has_feature(dev->features, fbit);
118 }
119
120 static inline bool vu_has_protocol_feature(VuDev *dev, unsigned int fbit)
121 {
122 return has_feature(dev->protocol_features, fbit);
123 }
124
125 const char *
126 vu_request_to_string(unsigned int req)
127 {
128 #define REQ(req) [req] = #req
129 static const char *vu_request_str[] = {
130 REQ(VHOST_USER_NONE),
131 REQ(VHOST_USER_GET_FEATURES),
132 REQ(VHOST_USER_SET_FEATURES),
133 REQ(VHOST_USER_SET_OWNER),
134 REQ(VHOST_USER_RESET_OWNER),
135 REQ(VHOST_USER_SET_MEM_TABLE),
136 REQ(VHOST_USER_SET_LOG_BASE),
137 REQ(VHOST_USER_SET_LOG_FD),
138 REQ(VHOST_USER_SET_VRING_NUM),
139 REQ(VHOST_USER_SET_VRING_ADDR),
140 REQ(VHOST_USER_SET_VRING_BASE),
141 REQ(VHOST_USER_GET_VRING_BASE),
142 REQ(VHOST_USER_SET_VRING_KICK),
143 REQ(VHOST_USER_SET_VRING_CALL),
144 REQ(VHOST_USER_SET_VRING_ERR),
145 REQ(VHOST_USER_GET_PROTOCOL_FEATURES),
146 REQ(VHOST_USER_SET_PROTOCOL_FEATURES),
147 REQ(VHOST_USER_GET_QUEUE_NUM),
148 REQ(VHOST_USER_SET_VRING_ENABLE),
149 REQ(VHOST_USER_SEND_RARP),
150 REQ(VHOST_USER_NET_SET_MTU),
151 REQ(VHOST_USER_SET_BACKEND_REQ_FD),
152 REQ(VHOST_USER_IOTLB_MSG),
153 REQ(VHOST_USER_SET_VRING_ENDIAN),
154 REQ(VHOST_USER_GET_CONFIG),
155 REQ(VHOST_USER_SET_CONFIG),
156 REQ(VHOST_USER_POSTCOPY_ADVISE),
157 REQ(VHOST_USER_POSTCOPY_LISTEN),
158 REQ(VHOST_USER_POSTCOPY_END),
159 REQ(VHOST_USER_GET_INFLIGHT_FD),
160 REQ(VHOST_USER_SET_INFLIGHT_FD),
161 REQ(VHOST_USER_GPU_SET_SOCKET),
162 REQ(VHOST_USER_VRING_KICK),
163 REQ(VHOST_USER_GET_MAX_MEM_SLOTS),
164 REQ(VHOST_USER_ADD_MEM_REG),
165 REQ(VHOST_USER_REM_MEM_REG),
166 REQ(VHOST_USER_GET_SHARED_OBJECT),
167 REQ(VHOST_USER_MAX),
168 };
169 #undef REQ
170
171 if (req < VHOST_USER_MAX) {
172 return vu_request_str[req];
173 } else {
174 return "unknown";
175 }
176 }
177
178 static void G_GNUC_PRINTF(2, 3)
179 vu_panic(VuDev *dev, const char *msg, ...)
180 {
181 char *buf = NULL;
182 va_list ap;
183
184 va_start(ap, msg);
185 if (vasprintf(&buf, msg, ap) < 0) {
186 buf = NULL;
187 }
188 va_end(ap);
189
190 dev->broken = true;
191 dev->panic(dev, buf);
192 free(buf);
193
194 /*
195 * FIXME:
196 * find a way to call virtio_error, or perhaps close the connection?
197 */
198 }
199
200 /* Search for a memory region that covers this guest physical address. */
201 static VuDevRegion *
202 vu_gpa_to_mem_region(VuDev *dev, uint64_t guest_addr)
203 {
204 int low = 0;
205 int high = dev->nregions - 1;
206
207 /*
208 * Memory regions cannot overlap in guest physical address space. Each
209 * GPA belongs to exactly one memory region, so there can only be one
210 * match.
211 *
212 * We store our memory regions ordered by GPA and can simply perform a
213 * binary search.
214 */
215 while (low <= high) {
216 unsigned int mid = low + (high - low) / 2;
217 VuDevRegion *cur = &dev->regions[mid];
218
219 if (guest_addr >= cur->gpa && guest_addr < cur->gpa + cur->size) {
220 return cur;
221 }
222 if (guest_addr >= cur->gpa + cur->size) {
223 low = mid + 1;
224 }
225 if (guest_addr < cur->gpa) {
226 high = mid - 1;
227 }
228 }
229 return NULL;
230 }
231
232 /* Translate guest physical address to our virtual address. */
233 void *
234 vu_gpa_to_va(VuDev *dev, uint64_t *plen, uint64_t guest_addr)
235 {
236 VuDevRegion *r;
237
238 if (*plen == 0) {
239 return NULL;
240 }
241
242 r = vu_gpa_to_mem_region(dev, guest_addr);
243 if (!r) {
244 return NULL;
245 }
246
247 if ((guest_addr + *plen) > (r->gpa + r->size)) {
248 *plen = r->gpa + r->size - guest_addr;
249 }
250 return (void *)(uintptr_t)guest_addr - r->gpa + r->mmap_addr +
251 r->mmap_offset;
252 }
253
254 /* Translate qemu virtual address to our virtual address. */
255 static void *
256 qva_to_va(VuDev *dev, uint64_t qemu_addr)
257 {
258 unsigned int i;
259
260 /* Find matching memory region. */
261 for (i = 0; i < dev->nregions; i++) {
262 VuDevRegion *r = &dev->regions[i];
263
264 if ((qemu_addr >= r->qva) && (qemu_addr < (r->qva + r->size))) {
265 return (void *)(uintptr_t)
266 qemu_addr - r->qva + r->mmap_addr + r->mmap_offset;
267 }
268 }
269
270 return NULL;
271 }
272
273 static void
274 vu_remove_all_mem_regs(VuDev *dev)
275 {
276 unsigned int i;
277
278 for (i = 0; i < dev->nregions; i++) {
279 VuDevRegion *r = &dev->regions[i];
280
281 munmap((void *)(uintptr_t)r->mmap_addr, r->size + r->mmap_offset);
282 }
283 dev->nregions = 0;
284 }
285
286 static bool
287 vu_is_vq_usable(VuDev *dev, VuVirtq *vq)
288 {
289 return likely(!dev->broken) && likely(vq->vring.avail);
290 }
291
292 static size_t
293 get_fd_hugepagesize(int fd)
294 {
295 #if defined(__linux__)
296 struct statfs fs;
297 int ret;
298
299 do {
300 ret = fstatfs(fd, &fs);
301 } while (ret != 0 && errno == EINTR);
302
303 if (!ret && (unsigned int)fs.f_type == HUGETLBFS_MAGIC) {
304 return fs.f_bsize;
305 }
306 #endif
307 return 0;
308 }
309
310 static void
311 _vu_add_mem_reg(VuDev *dev, VhostUserMemoryRegion *msg_region, int fd)
312 {
313 const uint64_t start_gpa = msg_region->guest_phys_addr;
314 const uint64_t end_gpa = start_gpa + msg_region->memory_size;
315 int prot = PROT_READ | PROT_WRITE;
316 uint64_t mmap_offset, fd_offset;
317 size_t hugepagesize;
318 VuDevRegion *r;
319 void *mmap_addr;
320 int low = 0;
321 int high = dev->nregions - 1;
322 unsigned int idx;
323
324 DPRINT("Adding region %d\n", dev->nregions);
325 DPRINT(" guest_phys_addr: 0x%016"PRIx64"\n",
326 msg_region->guest_phys_addr);
327 DPRINT(" memory_size: 0x%016"PRIx64"\n",
328 msg_region->memory_size);
329 DPRINT(" userspace_addr: 0x%016"PRIx64"\n",
330 msg_region->userspace_addr);
331 DPRINT(" old mmap_offset: 0x%016"PRIx64"\n",
332 msg_region->mmap_offset);
333
334 if (dev->postcopy_listening) {
335 /*
336 * In postcopy we're using PROT_NONE here to catch anyone
337 * accessing it before we userfault
338 */
339 prot = PROT_NONE;
340 }
341
342 /*
343 * We will add memory regions into the array sorted by GPA. Perform a
344 * binary search to locate the insertion point: it will be at the low
345 * index.
346 */
347 while (low <= high) {
348 unsigned int mid = low + (high - low) / 2;
349 VuDevRegion *cur = &dev->regions[mid];
350
351 /* Overlap of GPA addresses. */
352 if (start_gpa < cur->gpa + cur->size && cur->gpa < end_gpa) {
353 vu_panic(dev, "regions with overlapping guest physical addresses");
354 return;
355 }
356 if (start_gpa >= cur->gpa + cur->size) {
357 low = mid + 1;
358 }
359 if (start_gpa < cur->gpa) {
360 high = mid - 1;
361 }
362 }
363 idx = low;
364
365 /*
366 * Convert most of msg_region->mmap_offset to fd_offset. In almost all
367 * cases, this will leave us with mmap_offset == 0, mmap()'ing only
368 * what we really need. Only if a memory region would partially cover
369 * hugetlb pages, we'd get mmap_offset != 0, which usually doesn't happen
370 * anymore (i.e., modern QEMU).
371 *
372 * Note that mmap() with hugetlb would fail if the offset into the file
373 * is not aligned to the huge page size.
374 */
375 hugepagesize = get_fd_hugepagesize(fd);
376 if (hugepagesize) {
377 fd_offset = ALIGN_DOWN(msg_region->mmap_offset, hugepagesize);
378 mmap_offset = msg_region->mmap_offset - fd_offset;
379 } else {
380 fd_offset = msg_region->mmap_offset;
381 mmap_offset = 0;
382 }
383
384 DPRINT(" fd_offset: 0x%016"PRIx64"\n",
385 fd_offset);
386 DPRINT(" new mmap_offset: 0x%016"PRIx64"\n",
387 mmap_offset);
388
389 mmap_addr = mmap(0, msg_region->memory_size + mmap_offset,
390 prot, MAP_SHARED | MAP_NORESERVE, fd, fd_offset);
391 if (mmap_addr == MAP_FAILED) {
392 vu_panic(dev, "region mmap error: %s", strerror(errno));
393 return;
394 }
395 DPRINT(" mmap_addr: 0x%016"PRIx64"\n",
396 (uint64_t)(uintptr_t)mmap_addr);
397
398 /* Shift all affected entries by 1 to open a hole at idx. */
399 r = &dev->regions[idx];
400 memmove(r + 1, r, sizeof(VuDevRegion) * (dev->nregions - idx));
401 r->gpa = msg_region->guest_phys_addr;
402 r->size = msg_region->memory_size;
403 r->qva = msg_region->userspace_addr;
404 r->mmap_addr = (uint64_t)(uintptr_t)mmap_addr;
405 r->mmap_offset = mmap_offset;
406 dev->nregions++;
407
408 if (dev->postcopy_listening) {
409 /*
410 * Return the address to QEMU so that it can translate the ufd
411 * fault addresses back.
412 */
413 msg_region->userspace_addr = r->mmap_addr + r->mmap_offset;
414 }
415 }
416
417 static void
418 vmsg_close_fds(VhostUserMsg *vmsg)
419 {
420 int i;
421
422 for (i = 0; i < vmsg->fd_num; i++) {
423 close(vmsg->fds[i]);
424 }
425 }
426
427 /* Set reply payload.u64 and clear request flags and fd_num */
428 static void vmsg_set_reply_u64(VhostUserMsg *vmsg, uint64_t val)
429 {
430 vmsg->flags = 0; /* defaults will be set by vu_send_reply() */
431 vmsg->size = sizeof(vmsg->payload.u64);
432 vmsg->payload.u64 = val;
433 vmsg->fd_num = 0;
434 }
435
436 /* A test to see if we have userfault available */
437 static bool
438 have_userfault(void)
439 {
440 #if defined(__linux__) && defined(__NR_userfaultfd) &&\
441 defined(UFFD_FEATURE_MISSING_SHMEM) &&\
442 defined(UFFD_FEATURE_MISSING_HUGETLBFS)
443 /* Now test the kernel we're running on really has the features */
444 int ufd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
445 struct uffdio_api api_struct;
446 if (ufd < 0) {
447 return false;
448 }
449
450 api_struct.api = UFFD_API;
451 api_struct.features = UFFD_FEATURE_MISSING_SHMEM |
452 UFFD_FEATURE_MISSING_HUGETLBFS;
453 if (ioctl(ufd, UFFDIO_API, &api_struct)) {
454 close(ufd);
455 return false;
456 }
457 close(ufd);
458 return true;
459
460 #else
461 return false;
462 #endif
463 }
464
465 static bool
466 vu_message_read_default(VuDev *dev, int conn_fd, VhostUserMsg *vmsg)
467 {
468 char control[CMSG_SPACE(VHOST_MEMORY_BASELINE_NREGIONS * sizeof(int))] = {};
469 struct iovec iov = {
470 .iov_base = (char *)vmsg,
471 .iov_len = VHOST_USER_HDR_SIZE,
472 };
473 struct msghdr msg = {
474 .msg_iov = &iov,
475 .msg_iovlen = 1,
476 .msg_control = control,
477 .msg_controllen = sizeof(control),
478 };
479 size_t fd_size;
480 struct cmsghdr *cmsg;
481 int rc;
482
483 do {
484 rc = recvmsg(conn_fd, &msg, 0);
485 } while (rc < 0 && (errno == EINTR || errno == EAGAIN));
486
487 if (rc < 0) {
488 vu_panic(dev, "Error while recvmsg: %s", strerror(errno));
489 return false;
490 }
491
492 vmsg->fd_num = 0;
493 for (cmsg = CMSG_FIRSTHDR(&msg);
494 cmsg != NULL;
495 cmsg = CMSG_NXTHDR(&msg, cmsg))
496 {
497 if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS) {
498 fd_size = cmsg->cmsg_len - CMSG_LEN(0);
499 vmsg->fd_num = fd_size / sizeof(int);
500 assert(fd_size < VHOST_MEMORY_BASELINE_NREGIONS);
501 memcpy(vmsg->fds, CMSG_DATA(cmsg), fd_size);
502 break;
503 }
504 }
505
506 if (vmsg->size > sizeof(vmsg->payload)) {
507 vu_panic(dev,
508 "Error: too big message request: %d, size: vmsg->size: %u, "
509 "while sizeof(vmsg->payload) = %zu\n",
510 vmsg->request, vmsg->size, sizeof(vmsg->payload));
511 goto fail;
512 }
513
514 if (vmsg->size) {
515 do {
516 rc = read(conn_fd, &vmsg->payload, vmsg->size);
517 } while (rc < 0 && (errno == EINTR || errno == EAGAIN));
518
519 if (rc <= 0) {
520 vu_panic(dev, "Error while reading: %s", strerror(errno));
521 goto fail;
522 }
523
524 assert((uint32_t)rc == vmsg->size);
525 }
526
527 return true;
528
529 fail:
530 vmsg_close_fds(vmsg);
531
532 return false;
533 }
534
535 static bool
536 vu_message_write(VuDev *dev, int conn_fd, VhostUserMsg *vmsg)
537 {
538 int rc;
539 uint8_t *p = (uint8_t *)vmsg;
540 char control[CMSG_SPACE(VHOST_MEMORY_BASELINE_NREGIONS * sizeof(int))] = {};
541 struct iovec iov = {
542 .iov_base = (char *)vmsg,
543 .iov_len = VHOST_USER_HDR_SIZE,
544 };
545 struct msghdr msg = {
546 .msg_iov = &iov,
547 .msg_iovlen = 1,
548 .msg_control = control,
549 };
550 struct cmsghdr *cmsg;
551
552 memset(control, 0, sizeof(control));
553 assert(vmsg->fd_num <= VHOST_MEMORY_BASELINE_NREGIONS);
554 if (vmsg->fd_num > 0) {
555 size_t fdsize = vmsg->fd_num * sizeof(int);
556 msg.msg_controllen = CMSG_SPACE(fdsize);
557 cmsg = CMSG_FIRSTHDR(&msg);
558 cmsg->cmsg_len = CMSG_LEN(fdsize);
559 cmsg->cmsg_level = SOL_SOCKET;
560 cmsg->cmsg_type = SCM_RIGHTS;
561 memcpy(CMSG_DATA(cmsg), vmsg->fds, fdsize);
562 } else {
563 msg.msg_controllen = 0;
564 }
565
566 do {
567 rc = sendmsg(conn_fd, &msg, 0);
568 } while (rc < 0 && (errno == EINTR || errno == EAGAIN));
569
570 if (vmsg->size) {
571 do {
572 if (vmsg->data) {
573 rc = write(conn_fd, vmsg->data, vmsg->size);
574 } else {
575 rc = write(conn_fd, p + VHOST_USER_HDR_SIZE, vmsg->size);
576 }
577 } while (rc < 0 && (errno == EINTR || errno == EAGAIN));
578 }
579
580 if (rc <= 0) {
581 vu_panic(dev, "Error while writing: %s", strerror(errno));
582 return false;
583 }
584
585 return true;
586 }
587
588 static bool
589 vu_send_reply(VuDev *dev, int conn_fd, VhostUserMsg *vmsg)
590 {
591 /* Set the version in the flags when sending the reply */
592 vmsg->flags &= ~VHOST_USER_VERSION_MASK;
593 vmsg->flags |= VHOST_USER_VERSION;
594 vmsg->flags |= VHOST_USER_REPLY_MASK;
595
596 return vu_message_write(dev, conn_fd, vmsg);
597 }
598
599 /*
600 * Processes a reply on the backend channel.
601 * Entered with backend_mutex held and releases it before exit.
602 * Returns true on success.
603 */
604 static bool
605 vu_process_message_reply(VuDev *dev, const VhostUserMsg *vmsg)
606 {
607 VhostUserMsg msg_reply;
608 bool result = false;
609
610 if ((vmsg->flags & VHOST_USER_NEED_REPLY_MASK) == 0) {
611 result = true;
612 goto out;
613 }
614
615 if (!vu_message_read_default(dev, dev->backend_fd, &msg_reply)) {
616 goto out;
617 }
618
619 if (msg_reply.request != vmsg->request) {
620 DPRINT("Received unexpected msg type. Expected %d received %d",
621 vmsg->request, msg_reply.request);
622 goto out;
623 }
624
625 result = msg_reply.payload.u64 == 0;
626
627 out:
628 pthread_mutex_unlock(&dev->backend_mutex);
629 return result;
630 }
631
632 /* Kick the log_call_fd if required. */
633 static void
634 vu_log_kick(VuDev *dev)
635 {
636 if (dev->log_call_fd != -1) {
637 DPRINT("Kicking the QEMU's log...\n");
638 if (eventfd_write(dev->log_call_fd, 1) < 0) {
639 vu_panic(dev, "Error writing eventfd: %s", strerror(errno));
640 }
641 }
642 }
643
644 static void
645 vu_log_page(uint8_t *log_table, uint64_t page)
646 {
647 DPRINT("Logged dirty guest page: %"PRId64"\n", page);
648 qatomic_or(&log_table[page / 8], 1 << (page % 8));
649 }
650
651 static void
652 vu_log_write(VuDev *dev, uint64_t address, uint64_t length)
653 {
654 uint64_t page;
655
656 if (!(dev->features & (1ULL << VHOST_F_LOG_ALL)) ||
657 !dev->log_table || !length) {
658 return;
659 }
660
661 assert(dev->log_size > ((address + length - 1) / VHOST_LOG_PAGE / 8));
662
663 page = address / VHOST_LOG_PAGE;
664 while (page * VHOST_LOG_PAGE < address + length) {
665 vu_log_page(dev->log_table, page);
666 page += 1;
667 }
668
669 vu_log_kick(dev);
670 }
671
672 static void
673 vu_kick_cb(VuDev *dev, int condition, void *data)
674 {
675 int index = (intptr_t)data;
676 VuVirtq *vq = &dev->vq[index];
677 int sock = vq->kick_fd;
678 eventfd_t kick_data;
679 ssize_t rc;
680
681 rc = eventfd_read(sock, &kick_data);
682 if (rc == -1) {
683 vu_panic(dev, "kick eventfd_read(): %s", strerror(errno));
684 dev->remove_watch(dev, dev->vq[index].kick_fd);
685 } else {
686 DPRINT("Got kick_data: %016"PRIx64" handler:%p idx:%d\n",
687 kick_data, vq->handler, index);
688 if (vq->handler) {
689 vq->handler(dev, index);
690 }
691 }
692 }
693
694 static bool
695 vu_get_features_exec(VuDev *dev, VhostUserMsg *vmsg)
696 {
697 vmsg->payload.u64 =
698 /*
699 * The following VIRTIO feature bits are supported by our virtqueue
700 * implementation:
701 */
702 1ULL << VIRTIO_F_NOTIFY_ON_EMPTY |
703 1ULL << VIRTIO_RING_F_INDIRECT_DESC |
704 1ULL << VIRTIO_RING_F_EVENT_IDX |
705 1ULL << VIRTIO_F_VERSION_1 |
706
707 /* vhost-user feature bits */
708 1ULL << VHOST_F_LOG_ALL |
709 1ULL << VHOST_USER_F_PROTOCOL_FEATURES;
710
711 if (dev->iface->get_features) {
712 vmsg->payload.u64 |= dev->iface->get_features(dev);
713 }
714
715 vmsg->size = sizeof(vmsg->payload.u64);
716 vmsg->fd_num = 0;
717
718 DPRINT("Sending back to guest u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
719
720 return true;
721 }
722
723 static void
724 vu_set_enable_all_rings(VuDev *dev, bool enabled)
725 {
726 uint16_t i;
727
728 for (i = 0; i < dev->max_queues; i++) {
729 dev->vq[i].enable = enabled;
730 }
731 }
732
733 static bool
734 vu_set_features_exec(VuDev *dev, VhostUserMsg *vmsg)
735 {
736 DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
737
738 dev->features = vmsg->payload.u64;
739 if (!vu_has_feature(dev, VIRTIO_F_VERSION_1)) {
740 /*
741 * We only support devices conforming to VIRTIO 1.0 or
742 * later
743 */
744 vu_panic(dev, "virtio legacy devices aren't supported by libvhost-user");
745 return false;
746 }
747
748 if (!(dev->features & VHOST_USER_F_PROTOCOL_FEATURES)) {
749 vu_set_enable_all_rings(dev, true);
750 }
751
752 if (dev->iface->set_features) {
753 dev->iface->set_features(dev, dev->features);
754 }
755
756 return false;
757 }
758
759 static bool
760 vu_set_owner_exec(VuDev *dev, VhostUserMsg *vmsg)
761 {
762 return false;
763 }
764
765 static void
766 vu_close_log(VuDev *dev)
767 {
768 if (dev->log_table) {
769 if (munmap(dev->log_table, dev->log_size) != 0) {
770 perror("close log munmap() error");
771 }
772
773 dev->log_table = NULL;
774 }
775 if (dev->log_call_fd != -1) {
776 close(dev->log_call_fd);
777 dev->log_call_fd = -1;
778 }
779 }
780
781 static bool
782 vu_reset_device_exec(VuDev *dev, VhostUserMsg *vmsg)
783 {
784 vu_set_enable_all_rings(dev, false);
785
786 return false;
787 }
788
789 static bool
790 map_ring(VuDev *dev, VuVirtq *vq)
791 {
792 vq->vring.desc = qva_to_va(dev, vq->vra.desc_user_addr);
793 vq->vring.used = qva_to_va(dev, vq->vra.used_user_addr);
794 vq->vring.avail = qva_to_va(dev, vq->vra.avail_user_addr);
795
796 DPRINT("Setting virtq addresses:\n");
797 DPRINT(" vring_desc at %p\n", vq->vring.desc);
798 DPRINT(" vring_used at %p\n", vq->vring.used);
799 DPRINT(" vring_avail at %p\n", vq->vring.avail);
800
801 return !(vq->vring.desc && vq->vring.used && vq->vring.avail);
802 }
803
804 static bool
805 generate_faults(VuDev *dev) {
806 unsigned int i;
807 for (i = 0; i < dev->nregions; i++) {
808 #ifdef UFFDIO_REGISTER
809 VuDevRegion *dev_region = &dev->regions[i];
810 int ret;
811 struct uffdio_register reg_struct;
812
813 /*
814 * We should already have an open ufd. Mark each memory
815 * range as ufd.
816 * Discard any mapping we have here; note I can't use MADV_REMOVE
817 * or fallocate to make the hole since I don't want to lose
818 * data that's already arrived in the shared process.
819 * TODO: How to do hugepage
820 */
821 ret = madvise((void *)(uintptr_t)dev_region->mmap_addr,
822 dev_region->size + dev_region->mmap_offset,
823 MADV_DONTNEED);
824 if (ret) {
825 fprintf(stderr,
826 "%s: Failed to madvise(DONTNEED) region %d: %s\n",
827 __func__, i, strerror(errno));
828 }
829 /*
830 * Turn off transparent hugepages so we dont get lose wakeups
831 * in neighbouring pages.
832 * TODO: Turn this backon later.
833 */
834 ret = madvise((void *)(uintptr_t)dev_region->mmap_addr,
835 dev_region->size + dev_region->mmap_offset,
836 MADV_NOHUGEPAGE);
837 if (ret) {
838 /*
839 * Note: This can happen legally on kernels that are configured
840 * without madvise'able hugepages
841 */
842 fprintf(stderr,
843 "%s: Failed to madvise(NOHUGEPAGE) region %d: %s\n",
844 __func__, i, strerror(errno));
845 }
846
847 reg_struct.range.start = (uintptr_t)dev_region->mmap_addr;
848 reg_struct.range.len = dev_region->size + dev_region->mmap_offset;
849 reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING;
850
851 if (ioctl(dev->postcopy_ufd, UFFDIO_REGISTER, &reg_struct)) {
852 vu_panic(dev, "%s: Failed to userfault region %d "
853 "@%" PRIx64 " + size:%" PRIx64 " offset: %" PRIx64
854 ": (ufd=%d)%s\n",
855 __func__, i,
856 dev_region->mmap_addr,
857 dev_region->size, dev_region->mmap_offset,
858 dev->postcopy_ufd, strerror(errno));
859 return false;
860 }
861 if (!(reg_struct.ioctls & (1ULL << _UFFDIO_COPY))) {
862 vu_panic(dev, "%s Region (%d) doesn't support COPY",
863 __func__, i);
864 return false;
865 }
866 DPRINT("%s: region %d: Registered userfault for %"
867 PRIx64 " + %" PRIx64 "\n", __func__, i,
868 (uint64_t)reg_struct.range.start,
869 (uint64_t)reg_struct.range.len);
870 /* Now it's registered we can let the client at it */
871 if (mprotect((void *)(uintptr_t)dev_region->mmap_addr,
872 dev_region->size + dev_region->mmap_offset,
873 PROT_READ | PROT_WRITE)) {
874 vu_panic(dev, "failed to mprotect region %d for postcopy (%s)",
875 i, strerror(errno));
876 return false;
877 }
878 /* TODO: Stash 'zero' support flags somewhere */
879 #endif
880 }
881
882 return true;
883 }
884
885 static bool
886 vu_add_mem_reg(VuDev *dev, VhostUserMsg *vmsg) {
887 int i;
888 VhostUserMemoryRegion m = vmsg->payload.memreg.region, *msg_region = &m;
889
890 if (vmsg->fd_num != 1) {
891 vmsg_close_fds(vmsg);
892 vu_panic(dev, "VHOST_USER_ADD_MEM_REG received %d fds - only 1 fd "
893 "should be sent for this message type", vmsg->fd_num);
894 return false;
895 }
896
897 if (vmsg->size < VHOST_USER_MEM_REG_SIZE) {
898 close(vmsg->fds[0]);
899 vu_panic(dev, "VHOST_USER_ADD_MEM_REG requires a message size of at "
900 "least %zu bytes and only %d bytes were received",
901 VHOST_USER_MEM_REG_SIZE, vmsg->size);
902 return false;
903 }
904
905 if (dev->nregions == VHOST_USER_MAX_RAM_SLOTS) {
906 close(vmsg->fds[0]);
907 vu_panic(dev, "failing attempt to hot add memory via "
908 "VHOST_USER_ADD_MEM_REG message because the backend has "
909 "no free ram slots available");
910 return false;
911 }
912
913 /*
914 * If we are in postcopy mode and we receive a u64 payload with a 0 value
915 * we know all the postcopy client bases have been received, and we
916 * should start generating faults.
917 */
918 if (dev->postcopy_listening &&
919 vmsg->size == sizeof(vmsg->payload.u64) &&
920 vmsg->payload.u64 == 0) {
921 (void)generate_faults(dev);
922 return false;
923 }
924
925 _vu_add_mem_reg(dev, msg_region, vmsg->fds[0]);
926 close(vmsg->fds[0]);
927
928 if (dev->postcopy_listening) {
929 /* Send the message back to qemu with the addresses filled in. */
930 vmsg->fd_num = 0;
931 DPRINT("Successfully added new region in postcopy\n");
932 return true;
933 } else {
934 for (i = 0; i < dev->max_queues; i++) {
935 if (dev->vq[i].vring.desc) {
936 if (map_ring(dev, &dev->vq[i])) {
937 vu_panic(dev, "remapping queue %d for new memory region",
938 i);
939 }
940 }
941 }
942
943 DPRINT("Successfully added new region\n");
944 return false;
945 }
946 }
947
948 static inline bool reg_equal(VuDevRegion *vudev_reg,
949 VhostUserMemoryRegion *msg_reg)
950 {
951 if (vudev_reg->gpa == msg_reg->guest_phys_addr &&
952 vudev_reg->qva == msg_reg->userspace_addr &&
953 vudev_reg->size == msg_reg->memory_size) {
954 return true;
955 }
956
957 return false;
958 }
959
960 static bool
961 vu_rem_mem_reg(VuDev *dev, VhostUserMsg *vmsg) {
962 VhostUserMemoryRegion m = vmsg->payload.memreg.region, *msg_region = &m;
963 unsigned int idx;
964 VuDevRegion *r;
965
966 if (vmsg->fd_num > 1) {
967 vmsg_close_fds(vmsg);
968 vu_panic(dev, "VHOST_USER_REM_MEM_REG received %d fds - at most 1 fd "
969 "should be sent for this message type", vmsg->fd_num);
970 return false;
971 }
972
973 if (vmsg->size < VHOST_USER_MEM_REG_SIZE) {
974 vmsg_close_fds(vmsg);
975 vu_panic(dev, "VHOST_USER_REM_MEM_REG requires a message size of at "
976 "least %zu bytes and only %d bytes were received",
977 VHOST_USER_MEM_REG_SIZE, vmsg->size);
978 return false;
979 }
980
981 DPRINT("Removing region:\n");
982 DPRINT(" guest_phys_addr: 0x%016"PRIx64"\n",
983 msg_region->guest_phys_addr);
984 DPRINT(" memory_size: 0x%016"PRIx64"\n",
985 msg_region->memory_size);
986 DPRINT(" userspace_addr 0x%016"PRIx64"\n",
987 msg_region->userspace_addr);
988 DPRINT(" mmap_offset 0x%016"PRIx64"\n",
989 msg_region->mmap_offset);
990
991 r = vu_gpa_to_mem_region(dev, msg_region->guest_phys_addr);
992 if (!r || !reg_equal(r, msg_region)) {
993 vmsg_close_fds(vmsg);
994 vu_panic(dev, "Specified region not found\n");
995 return false;
996 }
997
998 munmap((void *)(uintptr_t)r->mmap_addr, r->size + r->mmap_offset);
999
1000 idx = r - dev->regions;
1001 assert(idx < dev->nregions);
1002 /* Shift all affected entries by 1 to close the hole. */
1003 memmove(r, r + 1, sizeof(VuDevRegion) * (dev->nregions - idx - 1));
1004 DPRINT("Successfully removed a region\n");
1005 dev->nregions--;
1006
1007 vmsg_close_fds(vmsg);
1008
1009 return false;
1010 }
1011
1012 static bool
1013 vu_get_shared_object(VuDev *dev, VhostUserMsg *vmsg)
1014 {
1015 int fd_num = 0;
1016 int dmabuf_fd = -1;
1017 if (dev->iface->get_shared_object) {
1018 dmabuf_fd = dev->iface->get_shared_object(
1019 dev, &vmsg->payload.object.uuid[0]);
1020 }
1021 if (dmabuf_fd != -1) {
1022 DPRINT("dmabuf_fd found for requested UUID\n");
1023 vmsg->fds[fd_num++] = dmabuf_fd;
1024 }
1025 vmsg->fd_num = fd_num;
1026
1027 return true;
1028 }
1029
1030 static bool
1031 vu_set_mem_table_exec(VuDev *dev, VhostUserMsg *vmsg)
1032 {
1033 VhostUserMemory m = vmsg->payload.memory, *memory = &m;
1034 unsigned int i;
1035
1036 vu_remove_all_mem_regs(dev);
1037
1038 DPRINT("Nregions: %u\n", memory->nregions);
1039 for (i = 0; i < memory->nregions; i++) {
1040 _vu_add_mem_reg(dev, &memory->regions[i], vmsg->fds[i]);
1041 close(vmsg->fds[i]);
1042 }
1043
1044 if (dev->postcopy_listening) {
1045 /* Send the message back to qemu with the addresses filled in */
1046 vmsg->fd_num = 0;
1047 if (!vu_send_reply(dev, dev->sock, vmsg)) {
1048 vu_panic(dev, "failed to respond to set-mem-table for postcopy");
1049 return false;
1050 }
1051
1052 /*
1053 * Wait for QEMU to confirm that it's registered the handler for the
1054 * faults.
1055 */
1056 if (!dev->read_msg(dev, dev->sock, vmsg) ||
1057 vmsg->size != sizeof(vmsg->payload.u64) ||
1058 vmsg->payload.u64 != 0) {
1059 vu_panic(dev, "failed to receive valid ack for postcopy set-mem-table");
1060 return false;
1061 }
1062
1063 /* OK, now we can go and register the memory and generate faults */
1064 (void)generate_faults(dev);
1065 return false;
1066 }
1067
1068 for (i = 0; i < dev->max_queues; i++) {
1069 if (dev->vq[i].vring.desc) {
1070 if (map_ring(dev, &dev->vq[i])) {
1071 vu_panic(dev, "remapping queue %d during setmemtable", i);
1072 }
1073 }
1074 }
1075
1076 return false;
1077 }
1078
1079 static bool
1080 vu_set_log_base_exec(VuDev *dev, VhostUserMsg *vmsg)
1081 {
1082 int fd;
1083 uint64_t log_mmap_size, log_mmap_offset;
1084 void *rc;
1085
1086 if (vmsg->fd_num != 1 ||
1087 vmsg->size != sizeof(vmsg->payload.log)) {
1088 vu_panic(dev, "Invalid log_base message");
1089 return true;
1090 }
1091
1092 fd = vmsg->fds[0];
1093 log_mmap_offset = vmsg->payload.log.mmap_offset;
1094 log_mmap_size = vmsg->payload.log.mmap_size;
1095 DPRINT("Log mmap_offset: %"PRId64"\n", log_mmap_offset);
1096 DPRINT("Log mmap_size: %"PRId64"\n", log_mmap_size);
1097
1098 rc = mmap(0, log_mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd,
1099 log_mmap_offset);
1100 close(fd);
1101 if (rc == MAP_FAILED) {
1102 perror("log mmap error");
1103 }
1104
1105 if (dev->log_table) {
1106 munmap(dev->log_table, dev->log_size);
1107 }
1108 dev->log_table = rc;
1109 dev->log_size = log_mmap_size;
1110
1111 vmsg->size = sizeof(vmsg->payload.u64);
1112 vmsg->fd_num = 0;
1113
1114 return true;
1115 }
1116
1117 static bool
1118 vu_set_log_fd_exec(VuDev *dev, VhostUserMsg *vmsg)
1119 {
1120 if (vmsg->fd_num != 1) {
1121 vu_panic(dev, "Invalid log_fd message");
1122 return false;
1123 }
1124
1125 if (dev->log_call_fd != -1) {
1126 close(dev->log_call_fd);
1127 }
1128 dev->log_call_fd = vmsg->fds[0];
1129 DPRINT("Got log_call_fd: %d\n", vmsg->fds[0]);
1130
1131 return false;
1132 }
1133
1134 static bool
1135 vu_set_vring_num_exec(VuDev *dev, VhostUserMsg *vmsg)
1136 {
1137 unsigned int index = vmsg->payload.state.index;
1138 unsigned int num = vmsg->payload.state.num;
1139
1140 DPRINT("State.index: %u\n", index);
1141 DPRINT("State.num: %u\n", num);
1142 dev->vq[index].vring.num = num;
1143
1144 return false;
1145 }
1146
1147 static bool
1148 vu_set_vring_addr_exec(VuDev *dev, VhostUserMsg *vmsg)
1149 {
1150 struct vhost_vring_addr addr = vmsg->payload.addr, *vra = &addr;
1151 unsigned int index = vra->index;
1152 VuVirtq *vq = &dev->vq[index];
1153
1154 DPRINT("vhost_vring_addr:\n");
1155 DPRINT(" index: %d\n", vra->index);
1156 DPRINT(" flags: %d\n", vra->flags);
1157 DPRINT(" desc_user_addr: 0x%016" PRIx64 "\n", (uint64_t)vra->desc_user_addr);
1158 DPRINT(" used_user_addr: 0x%016" PRIx64 "\n", (uint64_t)vra->used_user_addr);
1159 DPRINT(" avail_user_addr: 0x%016" PRIx64 "\n", (uint64_t)vra->avail_user_addr);
1160 DPRINT(" log_guest_addr: 0x%016" PRIx64 "\n", (uint64_t)vra->log_guest_addr);
1161
1162 vq->vra = *vra;
1163 vq->vring.flags = vra->flags;
1164 vq->vring.log_guest_addr = vra->log_guest_addr;
1165
1166
1167 if (map_ring(dev, vq)) {
1168 vu_panic(dev, "Invalid vring_addr message");
1169 return false;
1170 }
1171
1172 vq->used_idx = le16toh(vq->vring.used->idx);
1173
1174 if (vq->last_avail_idx != vq->used_idx) {
1175 bool resume = dev->iface->queue_is_processed_in_order &&
1176 dev->iface->queue_is_processed_in_order(dev, index);
1177
1178 DPRINT("Last avail index != used index: %u != %u%s\n",
1179 vq->last_avail_idx, vq->used_idx,
1180 resume ? ", resuming" : "");
1181
1182 if (resume) {
1183 vq->shadow_avail_idx = vq->last_avail_idx = vq->used_idx;
1184 }
1185 }
1186
1187 return false;
1188 }
1189
1190 static bool
1191 vu_set_vring_base_exec(VuDev *dev, VhostUserMsg *vmsg)
1192 {
1193 unsigned int index = vmsg->payload.state.index;
1194 unsigned int num = vmsg->payload.state.num;
1195
1196 DPRINT("State.index: %u\n", index);
1197 DPRINT("State.num: %u\n", num);
1198 dev->vq[index].shadow_avail_idx = dev->vq[index].last_avail_idx = num;
1199
1200 return false;
1201 }
1202
1203 static bool
1204 vu_get_vring_base_exec(VuDev *dev, VhostUserMsg *vmsg)
1205 {
1206 unsigned int index = vmsg->payload.state.index;
1207
1208 DPRINT("State.index: %u\n", index);
1209 vmsg->payload.state.num = dev->vq[index].last_avail_idx;
1210 vmsg->size = sizeof(vmsg->payload.state);
1211
1212 dev->vq[index].started = false;
1213 if (dev->iface->queue_set_started) {
1214 dev->iface->queue_set_started(dev, index, false);
1215 }
1216
1217 if (dev->vq[index].call_fd != -1) {
1218 close(dev->vq[index].call_fd);
1219 dev->vq[index].call_fd = -1;
1220 }
1221 if (dev->vq[index].kick_fd != -1) {
1222 dev->remove_watch(dev, dev->vq[index].kick_fd);
1223 close(dev->vq[index].kick_fd);
1224 dev->vq[index].kick_fd = -1;
1225 }
1226
1227 return true;
1228 }
1229
1230 static bool
1231 vu_check_queue_msg_file(VuDev *dev, VhostUserMsg *vmsg)
1232 {
1233 int index = vmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
1234 bool nofd = vmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK;
1235
1236 if (index >= dev->max_queues) {
1237 vmsg_close_fds(vmsg);
1238 vu_panic(dev, "Invalid queue index: %u", index);
1239 return false;
1240 }
1241
1242 if (nofd) {
1243 vmsg_close_fds(vmsg);
1244 return true;
1245 }
1246
1247 if (vmsg->fd_num != 1) {
1248 vmsg_close_fds(vmsg);
1249 vu_panic(dev, "Invalid fds in request: %d", vmsg->request);
1250 return false;
1251 }
1252
1253 return true;
1254 }
1255
1256 static int
1257 inflight_desc_compare(const void *a, const void *b)
1258 {
1259 VuVirtqInflightDesc *desc0 = (VuVirtqInflightDesc *)a,
1260 *desc1 = (VuVirtqInflightDesc *)b;
1261
1262 if (desc1->counter > desc0->counter &&
1263 (desc1->counter - desc0->counter) < VIRTQUEUE_MAX_SIZE * 2) {
1264 return 1;
1265 }
1266
1267 return -1;
1268 }
1269
1270 static int
1271 vu_check_queue_inflights(VuDev *dev, VuVirtq *vq)
1272 {
1273 int i = 0;
1274
1275 if (!vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
1276 return 0;
1277 }
1278
1279 if (unlikely(!vq->inflight)) {
1280 return -1;
1281 }
1282
1283 if (unlikely(!vq->inflight->version)) {
1284 /* initialize the buffer */
1285 vq->inflight->version = INFLIGHT_VERSION;
1286 return 0;
1287 }
1288
1289 vq->used_idx = le16toh(vq->vring.used->idx);
1290 vq->resubmit_num = 0;
1291 vq->resubmit_list = NULL;
1292 vq->counter = 0;
1293
1294 if (unlikely(vq->inflight->used_idx != vq->used_idx)) {
1295 vq->inflight->desc[vq->inflight->last_batch_head].inflight = 0;
1296
1297 barrier();
1298
1299 vq->inflight->used_idx = vq->used_idx;
1300 }
1301
1302 for (i = 0; i < vq->inflight->desc_num; i++) {
1303 if (vq->inflight->desc[i].inflight == 1) {
1304 vq->inuse++;
1305 }
1306 }
1307
1308 vq->shadow_avail_idx = vq->last_avail_idx = vq->inuse + vq->used_idx;
1309
1310 if (vq->inuse) {
1311 vq->resubmit_list = calloc(vq->inuse, sizeof(VuVirtqInflightDesc));
1312 if (!vq->resubmit_list) {
1313 return -1;
1314 }
1315
1316 for (i = 0; i < vq->inflight->desc_num; i++) {
1317 if (vq->inflight->desc[i].inflight) {
1318 vq->resubmit_list[vq->resubmit_num].index = i;
1319 vq->resubmit_list[vq->resubmit_num].counter =
1320 vq->inflight->desc[i].counter;
1321 vq->resubmit_num++;
1322 }
1323 }
1324
1325 if (vq->resubmit_num > 1) {
1326 qsort(vq->resubmit_list, vq->resubmit_num,
1327 sizeof(VuVirtqInflightDesc), inflight_desc_compare);
1328 }
1329 vq->counter = vq->resubmit_list[0].counter + 1;
1330 }
1331
1332 /* in case of I/O hang after reconnecting */
1333 if (eventfd_write(vq->kick_fd, 1)) {
1334 return -1;
1335 }
1336
1337 return 0;
1338 }
1339
1340 static bool
1341 vu_set_vring_kick_exec(VuDev *dev, VhostUserMsg *vmsg)
1342 {
1343 int index = vmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
1344 bool nofd = vmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK;
1345
1346 DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
1347
1348 if (!vu_check_queue_msg_file(dev, vmsg)) {
1349 return false;
1350 }
1351
1352 if (dev->vq[index].kick_fd != -1) {
1353 dev->remove_watch(dev, dev->vq[index].kick_fd);
1354 close(dev->vq[index].kick_fd);
1355 dev->vq[index].kick_fd = -1;
1356 }
1357
1358 dev->vq[index].kick_fd = nofd ? -1 : vmsg->fds[0];
1359 DPRINT("Got kick_fd: %d for vq: %d\n", dev->vq[index].kick_fd, index);
1360
1361 dev->vq[index].started = true;
1362 if (dev->iface->queue_set_started) {
1363 dev->iface->queue_set_started(dev, index, true);
1364 }
1365
1366 if (dev->vq[index].kick_fd != -1 && dev->vq[index].handler) {
1367 dev->set_watch(dev, dev->vq[index].kick_fd, VU_WATCH_IN,
1368 vu_kick_cb, (void *)(long)index);
1369
1370 DPRINT("Waiting for kicks on fd: %d for vq: %d\n",
1371 dev->vq[index].kick_fd, index);
1372 }
1373
1374 if (vu_check_queue_inflights(dev, &dev->vq[index])) {
1375 vu_panic(dev, "Failed to check inflights for vq: %d\n", index);
1376 }
1377
1378 return false;
1379 }
1380
1381 void vu_set_queue_handler(VuDev *dev, VuVirtq *vq,
1382 vu_queue_handler_cb handler)
1383 {
1384 int qidx = vq - dev->vq;
1385
1386 vq->handler = handler;
1387 if (vq->kick_fd >= 0) {
1388 if (handler) {
1389 dev->set_watch(dev, vq->kick_fd, VU_WATCH_IN,
1390 vu_kick_cb, (void *)(long)qidx);
1391 } else {
1392 dev->remove_watch(dev, vq->kick_fd);
1393 }
1394 }
1395 }
1396
1397 bool vu_set_queue_host_notifier(VuDev *dev, VuVirtq *vq, int fd,
1398 int size, int offset)
1399 {
1400 int qidx = vq - dev->vq;
1401 int fd_num = 0;
1402 VhostUserMsg vmsg = {
1403 .request = VHOST_USER_BACKEND_VRING_HOST_NOTIFIER_MSG,
1404 .flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
1405 .size = sizeof(vmsg.payload.area),
1406 .payload.area = {
1407 .u64 = qidx & VHOST_USER_VRING_IDX_MASK,
1408 .size = size,
1409 .offset = offset,
1410 },
1411 };
1412
1413 if (fd == -1) {
1414 vmsg.payload.area.u64 |= VHOST_USER_VRING_NOFD_MASK;
1415 } else {
1416 vmsg.fds[fd_num++] = fd;
1417 }
1418
1419 vmsg.fd_num = fd_num;
1420
1421 if (!vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_BACKEND_SEND_FD)) {
1422 return false;
1423 }
1424
1425 pthread_mutex_lock(&dev->backend_mutex);
1426 if (!vu_message_write(dev, dev->backend_fd, &vmsg)) {
1427 pthread_mutex_unlock(&dev->backend_mutex);
1428 return false;
1429 }
1430
1431 /* Also unlocks the backend_mutex */
1432 return vu_process_message_reply(dev, &vmsg);
1433 }
1434
1435 bool
1436 vu_lookup_shared_object(VuDev *dev, unsigned char uuid[UUID_LEN],
1437 int *dmabuf_fd)
1438 {
1439 bool result = false;
1440 VhostUserMsg msg_reply;
1441 VhostUserMsg msg = {
1442 .request = VHOST_USER_BACKEND_SHARED_OBJECT_LOOKUP,
1443 .size = sizeof(msg.payload.object),
1444 .flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
1445 };
1446
1447 memcpy(msg.payload.object.uuid, uuid, sizeof(uuid[0]) * UUID_LEN);
1448
1449 if (!vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_SHARED_OBJECT)) {
1450 return false;
1451 }
1452
1453 pthread_mutex_lock(&dev->backend_mutex);
1454 if (!vu_message_write(dev, dev->backend_fd, &msg)) {
1455 goto out;
1456 }
1457
1458 if (!vu_message_read_default(dev, dev->backend_fd, &msg_reply)) {
1459 goto out;
1460 }
1461
1462 if (msg_reply.request != msg.request) {
1463 DPRINT("Received unexpected msg type. Expected %d, received %d",
1464 msg.request, msg_reply.request);
1465 goto out;
1466 }
1467
1468 if (msg_reply.fd_num != 1) {
1469 DPRINT("Received unexpected number of fds. Expected 1, received %d",
1470 msg_reply.fd_num);
1471 goto out;
1472 }
1473
1474 *dmabuf_fd = msg_reply.fds[0];
1475 result = *dmabuf_fd > 0 && msg_reply.payload.u64 == 0;
1476 out:
1477 pthread_mutex_unlock(&dev->backend_mutex);
1478
1479 return result;
1480 }
1481
1482 static bool
1483 vu_send_message(VuDev *dev, VhostUserMsg *vmsg)
1484 {
1485 bool result = false;
1486 pthread_mutex_lock(&dev->backend_mutex);
1487 if (!vu_message_write(dev, dev->backend_fd, vmsg)) {
1488 goto out;
1489 }
1490
1491 result = true;
1492 out:
1493 pthread_mutex_unlock(&dev->backend_mutex);
1494
1495 return result;
1496 }
1497
1498 bool
1499 vu_add_shared_object(VuDev *dev, unsigned char uuid[UUID_LEN])
1500 {
1501 VhostUserMsg msg = {
1502 .request = VHOST_USER_BACKEND_SHARED_OBJECT_ADD,
1503 .size = sizeof(msg.payload.object),
1504 .flags = VHOST_USER_VERSION,
1505 };
1506
1507 memcpy(msg.payload.object.uuid, uuid, sizeof(uuid[0]) * UUID_LEN);
1508
1509 if (!vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_SHARED_OBJECT)) {
1510 return false;
1511 }
1512
1513 return vu_send_message(dev, &msg);
1514 }
1515
1516 bool
1517 vu_rm_shared_object(VuDev *dev, unsigned char uuid[UUID_LEN])
1518 {
1519 VhostUserMsg msg = {
1520 .request = VHOST_USER_BACKEND_SHARED_OBJECT_REMOVE,
1521 .size = sizeof(msg.payload.object),
1522 .flags = VHOST_USER_VERSION,
1523 };
1524
1525 memcpy(msg.payload.object.uuid, uuid, sizeof(uuid[0]) * UUID_LEN);
1526
1527 if (!vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_SHARED_OBJECT)) {
1528 return false;
1529 }
1530
1531 return vu_send_message(dev, &msg);
1532 }
1533
1534 static bool
1535 vu_set_vring_call_exec(VuDev *dev, VhostUserMsg *vmsg)
1536 {
1537 int index = vmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
1538 bool nofd = vmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK;
1539
1540 DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
1541
1542 if (!vu_check_queue_msg_file(dev, vmsg)) {
1543 return false;
1544 }
1545
1546 if (dev->vq[index].call_fd != -1) {
1547 close(dev->vq[index].call_fd);
1548 dev->vq[index].call_fd = -1;
1549 }
1550
1551 dev->vq[index].call_fd = nofd ? -1 : vmsg->fds[0];
1552
1553 /* in case of I/O hang after reconnecting */
1554 if (dev->vq[index].call_fd != -1 && eventfd_write(vmsg->fds[0], 1)) {
1555 return -1;
1556 }
1557
1558 DPRINT("Got call_fd: %d for vq: %d\n", dev->vq[index].call_fd, index);
1559
1560 return false;
1561 }
1562
1563 static bool
1564 vu_set_vring_err_exec(VuDev *dev, VhostUserMsg *vmsg)
1565 {
1566 int index = vmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
1567 bool nofd = vmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK;
1568
1569 DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
1570
1571 if (!vu_check_queue_msg_file(dev, vmsg)) {
1572 return false;
1573 }
1574
1575 if (dev->vq[index].err_fd != -1) {
1576 close(dev->vq[index].err_fd);
1577 dev->vq[index].err_fd = -1;
1578 }
1579
1580 dev->vq[index].err_fd = nofd ? -1 : vmsg->fds[0];
1581
1582 return false;
1583 }
1584
1585 static bool
1586 vu_get_protocol_features_exec(VuDev *dev, VhostUserMsg *vmsg)
1587 {
1588 /*
1589 * Note that we support, but intentionally do not set,
1590 * VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS. This means that
1591 * a device implementation can return it in its callback
1592 * (get_protocol_features) if it wants to use this for
1593 * simulation, but it is otherwise not desirable (if even
1594 * implemented by the frontend.)
1595 */
1596 uint64_t features = 1ULL << VHOST_USER_PROTOCOL_F_MQ |
1597 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD |
1598 1ULL << VHOST_USER_PROTOCOL_F_BACKEND_REQ |
1599 1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER |
1600 1ULL << VHOST_USER_PROTOCOL_F_BACKEND_SEND_FD |
1601 1ULL << VHOST_USER_PROTOCOL_F_REPLY_ACK |
1602 1ULL << VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS;
1603
1604 if (have_userfault()) {
1605 features |= 1ULL << VHOST_USER_PROTOCOL_F_PAGEFAULT;
1606 }
1607
1608 if (dev->iface->get_config && dev->iface->set_config) {
1609 features |= 1ULL << VHOST_USER_PROTOCOL_F_CONFIG;
1610 }
1611
1612 if (dev->iface->get_protocol_features) {
1613 features |= dev->iface->get_protocol_features(dev);
1614 }
1615
1616 vmsg_set_reply_u64(vmsg, features);
1617 return true;
1618 }
1619
1620 static bool
1621 vu_set_protocol_features_exec(VuDev *dev, VhostUserMsg *vmsg)
1622 {
1623 uint64_t features = vmsg->payload.u64;
1624
1625 DPRINT("u64: 0x%016"PRIx64"\n", features);
1626
1627 dev->protocol_features = vmsg->payload.u64;
1628
1629 if (vu_has_protocol_feature(dev,
1630 VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS) &&
1631 (!vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_BACKEND_REQ) ||
1632 !vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_REPLY_ACK))) {
1633 /*
1634 * The use case for using messages for kick/call is simulation, to make
1635 * the kick and call synchronous. To actually get that behaviour, both
1636 * of the other features are required.
1637 * Theoretically, one could use only kick messages, or do them without
1638 * having F_REPLY_ACK, but too many (possibly pending) messages on the
1639 * socket will eventually cause the frontend to hang, to avoid this in
1640 * scenarios where not desired enforce that the settings are in a way
1641 * that actually enables the simulation case.
1642 */
1643 vu_panic(dev,
1644 "F_IN_BAND_NOTIFICATIONS requires F_BACKEND_REQ && F_REPLY_ACK");
1645 return false;
1646 }
1647
1648 if (dev->iface->set_protocol_features) {
1649 dev->iface->set_protocol_features(dev, features);
1650 }
1651
1652 return false;
1653 }
1654
1655 static bool
1656 vu_get_queue_num_exec(VuDev *dev, VhostUserMsg *vmsg)
1657 {
1658 vmsg_set_reply_u64(vmsg, dev->max_queues);
1659 return true;
1660 }
1661
1662 static bool
1663 vu_set_vring_enable_exec(VuDev *dev, VhostUserMsg *vmsg)
1664 {
1665 unsigned int index = vmsg->payload.state.index;
1666 unsigned int enable = vmsg->payload.state.num;
1667
1668 DPRINT("State.index: %u\n", index);
1669 DPRINT("State.enable: %u\n", enable);
1670
1671 if (index >= dev->max_queues) {
1672 vu_panic(dev, "Invalid vring_enable index: %u", index);
1673 return false;
1674 }
1675
1676 dev->vq[index].enable = enable;
1677 return false;
1678 }
1679
1680 static bool
1681 vu_set_backend_req_fd(VuDev *dev, VhostUserMsg *vmsg)
1682 {
1683 if (vmsg->fd_num != 1) {
1684 vu_panic(dev, "Invalid backend_req_fd message (%d fd's)", vmsg->fd_num);
1685 return false;
1686 }
1687
1688 if (dev->backend_fd != -1) {
1689 close(dev->backend_fd);
1690 }
1691 dev->backend_fd = vmsg->fds[0];
1692 DPRINT("Got backend_fd: %d\n", vmsg->fds[0]);
1693
1694 return false;
1695 }
1696
1697 static bool
1698 vu_get_config(VuDev *dev, VhostUserMsg *vmsg)
1699 {
1700 int ret = -1;
1701
1702 if (dev->iface->get_config) {
1703 ret = dev->iface->get_config(dev, vmsg->payload.config.region,
1704 vmsg->payload.config.size);
1705 }
1706
1707 if (ret) {
1708 /* resize to zero to indicate an error to frontend */
1709 vmsg->size = 0;
1710 }
1711
1712 return true;
1713 }
1714
1715 static bool
1716 vu_set_config(VuDev *dev, VhostUserMsg *vmsg)
1717 {
1718 int ret = -1;
1719
1720 if (dev->iface->set_config) {
1721 ret = dev->iface->set_config(dev, vmsg->payload.config.region,
1722 vmsg->payload.config.offset,
1723 vmsg->payload.config.size,
1724 vmsg->payload.config.flags);
1725 if (ret) {
1726 vu_panic(dev, "Set virtio configuration space failed");
1727 }
1728 }
1729
1730 return false;
1731 }
1732
1733 static bool
1734 vu_set_postcopy_advise(VuDev *dev, VhostUserMsg *vmsg)
1735 {
1736 #ifdef UFFDIO_API
1737 struct uffdio_api api_struct;
1738
1739 dev->postcopy_ufd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
1740 vmsg->size = 0;
1741 #else
1742 dev->postcopy_ufd = -1;
1743 #endif
1744
1745 if (dev->postcopy_ufd == -1) {
1746 vu_panic(dev, "Userfaultfd not available: %s", strerror(errno));
1747 goto out;
1748 }
1749
1750 #ifdef UFFDIO_API
1751 api_struct.api = UFFD_API;
1752 api_struct.features = 0;
1753 if (ioctl(dev->postcopy_ufd, UFFDIO_API, &api_struct)) {
1754 vu_panic(dev, "Failed UFFDIO_API: %s", strerror(errno));
1755 close(dev->postcopy_ufd);
1756 dev->postcopy_ufd = -1;
1757 goto out;
1758 }
1759 /* TODO: Stash feature flags somewhere */
1760 #endif
1761
1762 out:
1763 /* Return a ufd to the QEMU */
1764 vmsg->fd_num = 1;
1765 vmsg->fds[0] = dev->postcopy_ufd;
1766 return true; /* = send a reply */
1767 }
1768
1769 static bool
1770 vu_set_postcopy_listen(VuDev *dev, VhostUserMsg *vmsg)
1771 {
1772 if (dev->nregions) {
1773 vu_panic(dev, "Regions already registered at postcopy-listen");
1774 vmsg_set_reply_u64(vmsg, -1);
1775 return true;
1776 }
1777 dev->postcopy_listening = true;
1778
1779 vmsg_set_reply_u64(vmsg, 0);
1780 return true;
1781 }
1782
1783 static bool
1784 vu_set_postcopy_end(VuDev *dev, VhostUserMsg *vmsg)
1785 {
1786 DPRINT("%s: Entry\n", __func__);
1787 dev->postcopy_listening = false;
1788 if (dev->postcopy_ufd > 0) {
1789 close(dev->postcopy_ufd);
1790 dev->postcopy_ufd = -1;
1791 DPRINT("%s: Done close\n", __func__);
1792 }
1793
1794 vmsg_set_reply_u64(vmsg, 0);
1795 DPRINT("%s: exit\n", __func__);
1796 return true;
1797 }
1798
1799 static inline uint64_t
1800 vu_inflight_queue_size(uint16_t queue_size)
1801 {
1802 return ALIGN_UP(sizeof(VuDescStateSplit) * queue_size +
1803 sizeof(uint16_t), INFLIGHT_ALIGNMENT);
1804 }
1805
1806 #ifdef MFD_ALLOW_SEALING
1807 static void *
1808 memfd_alloc(const char *name, size_t size, unsigned int flags, int *fd)
1809 {
1810 void *ptr;
1811 int ret;
1812
1813 *fd = memfd_create(name, MFD_ALLOW_SEALING);
1814 if (*fd < 0) {
1815 return NULL;
1816 }
1817
1818 ret = ftruncate(*fd, size);
1819 if (ret < 0) {
1820 close(*fd);
1821 return NULL;
1822 }
1823
1824 ret = fcntl(*fd, F_ADD_SEALS, flags);
1825 if (ret < 0) {
1826 close(*fd);
1827 return NULL;
1828 }
1829
1830 ptr = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, *fd, 0);
1831 if (ptr == MAP_FAILED) {
1832 close(*fd);
1833 return NULL;
1834 }
1835
1836 return ptr;
1837 }
1838 #endif
1839
1840 static bool
1841 vu_get_inflight_fd(VuDev *dev, VhostUserMsg *vmsg)
1842 {
1843 int fd = -1;
1844 void *addr = NULL;
1845 uint64_t mmap_size;
1846 uint16_t num_queues, queue_size;
1847
1848 if (vmsg->size != sizeof(vmsg->payload.inflight)) {
1849 vu_panic(dev, "Invalid get_inflight_fd message:%d", vmsg->size);
1850 vmsg->payload.inflight.mmap_size = 0;
1851 return true;
1852 }
1853
1854 num_queues = vmsg->payload.inflight.num_queues;
1855 queue_size = vmsg->payload.inflight.queue_size;
1856
1857 DPRINT("set_inflight_fd num_queues: %"PRId16"\n", num_queues);
1858 DPRINT("set_inflight_fd queue_size: %"PRId16"\n", queue_size);
1859
1860 mmap_size = vu_inflight_queue_size(queue_size) * num_queues;
1861
1862 #ifdef MFD_ALLOW_SEALING
1863 addr = memfd_alloc("vhost-inflight", mmap_size,
1864 F_SEAL_GROW | F_SEAL_SHRINK | F_SEAL_SEAL,
1865 &fd);
1866 #else
1867 vu_panic(dev, "Not implemented: memfd support is missing");
1868 #endif
1869
1870 if (!addr) {
1871 vu_panic(dev, "Failed to alloc vhost inflight area");
1872 vmsg->payload.inflight.mmap_size = 0;
1873 return true;
1874 }
1875
1876 memset(addr, 0, mmap_size);
1877
1878 dev->inflight_info.addr = addr;
1879 dev->inflight_info.size = vmsg->payload.inflight.mmap_size = mmap_size;
1880 dev->inflight_info.fd = vmsg->fds[0] = fd;
1881 vmsg->fd_num = 1;
1882 vmsg->payload.inflight.mmap_offset = 0;
1883
1884 DPRINT("send inflight mmap_size: %"PRId64"\n",
1885 vmsg->payload.inflight.mmap_size);
1886 DPRINT("send inflight mmap offset: %"PRId64"\n",
1887 vmsg->payload.inflight.mmap_offset);
1888
1889 return true;
1890 }
1891
1892 static bool
1893 vu_set_inflight_fd(VuDev *dev, VhostUserMsg *vmsg)
1894 {
1895 int fd, i;
1896 uint64_t mmap_size, mmap_offset;
1897 uint16_t num_queues, queue_size;
1898 void *rc;
1899
1900 if (vmsg->fd_num != 1 ||
1901 vmsg->size != sizeof(vmsg->payload.inflight)) {
1902 vu_panic(dev, "Invalid set_inflight_fd message size:%d fds:%d",
1903 vmsg->size, vmsg->fd_num);
1904 return false;
1905 }
1906
1907 fd = vmsg->fds[0];
1908 mmap_size = vmsg->payload.inflight.mmap_size;
1909 mmap_offset = vmsg->payload.inflight.mmap_offset;
1910 num_queues = vmsg->payload.inflight.num_queues;
1911 queue_size = vmsg->payload.inflight.queue_size;
1912
1913 DPRINT("set_inflight_fd mmap_size: %"PRId64"\n", mmap_size);
1914 DPRINT("set_inflight_fd mmap_offset: %"PRId64"\n", mmap_offset);
1915 DPRINT("set_inflight_fd num_queues: %"PRId16"\n", num_queues);
1916 DPRINT("set_inflight_fd queue_size: %"PRId16"\n", queue_size);
1917
1918 rc = mmap(0, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
1919 fd, mmap_offset);
1920
1921 if (rc == MAP_FAILED) {
1922 vu_panic(dev, "set_inflight_fd mmap error: %s", strerror(errno));
1923 return false;
1924 }
1925
1926 if (dev->inflight_info.fd) {
1927 close(dev->inflight_info.fd);
1928 }
1929
1930 if (dev->inflight_info.addr) {
1931 munmap(dev->inflight_info.addr, dev->inflight_info.size);
1932 }
1933
1934 dev->inflight_info.fd = fd;
1935 dev->inflight_info.addr = rc;
1936 dev->inflight_info.size = mmap_size;
1937
1938 for (i = 0; i < num_queues; i++) {
1939 dev->vq[i].inflight = (VuVirtqInflight *)rc;
1940 dev->vq[i].inflight->desc_num = queue_size;
1941 rc = (void *)((char *)rc + vu_inflight_queue_size(queue_size));
1942 }
1943
1944 return false;
1945 }
1946
1947 static bool
1948 vu_handle_vring_kick(VuDev *dev, VhostUserMsg *vmsg)
1949 {
1950 unsigned int index = vmsg->payload.state.index;
1951
1952 if (index >= dev->max_queues) {
1953 vu_panic(dev, "Invalid queue index: %u", index);
1954 return false;
1955 }
1956
1957 DPRINT("Got kick message: handler:%p idx:%u\n",
1958 dev->vq[index].handler, index);
1959
1960 if (!dev->vq[index].started) {
1961 dev->vq[index].started = true;
1962
1963 if (dev->iface->queue_set_started) {
1964 dev->iface->queue_set_started(dev, index, true);
1965 }
1966 }
1967
1968 if (dev->vq[index].handler) {
1969 dev->vq[index].handler(dev, index);
1970 }
1971
1972 return false;
1973 }
1974
1975 static bool vu_handle_get_max_memslots(VuDev *dev, VhostUserMsg *vmsg)
1976 {
1977 vmsg_set_reply_u64(vmsg, VHOST_USER_MAX_RAM_SLOTS);
1978
1979 DPRINT("u64: 0x%016"PRIx64"\n", (uint64_t) VHOST_USER_MAX_RAM_SLOTS);
1980
1981 return true;
1982 }
1983
1984 static bool
1985 vu_process_message(VuDev *dev, VhostUserMsg *vmsg)
1986 {
1987 int do_reply = 0;
1988
1989 /* Print out generic part of the request. */
1990 DPRINT("================ Vhost user message ================\n");
1991 DPRINT("Request: %s (%d)\n", vu_request_to_string(vmsg->request),
1992 vmsg->request);
1993 DPRINT("Flags: 0x%x\n", vmsg->flags);
1994 DPRINT("Size: %u\n", vmsg->size);
1995
1996 if (vmsg->fd_num) {
1997 int i;
1998 DPRINT("Fds:");
1999 for (i = 0; i < vmsg->fd_num; i++) {
2000 DPRINT(" %d", vmsg->fds[i]);
2001 }
2002 DPRINT("\n");
2003 }
2004
2005 if (dev->iface->process_msg &&
2006 dev->iface->process_msg(dev, vmsg, &do_reply)) {
2007 return do_reply;
2008 }
2009
2010 switch (vmsg->request) {
2011 case VHOST_USER_GET_FEATURES:
2012 return vu_get_features_exec(dev, vmsg);
2013 case VHOST_USER_SET_FEATURES:
2014 return vu_set_features_exec(dev, vmsg);
2015 case VHOST_USER_GET_PROTOCOL_FEATURES:
2016 return vu_get_protocol_features_exec(dev, vmsg);
2017 case VHOST_USER_SET_PROTOCOL_FEATURES:
2018 return vu_set_protocol_features_exec(dev, vmsg);
2019 case VHOST_USER_SET_OWNER:
2020 return vu_set_owner_exec(dev, vmsg);
2021 case VHOST_USER_RESET_OWNER:
2022 return vu_reset_device_exec(dev, vmsg);
2023 case VHOST_USER_SET_MEM_TABLE:
2024 return vu_set_mem_table_exec(dev, vmsg);
2025 case VHOST_USER_SET_LOG_BASE:
2026 return vu_set_log_base_exec(dev, vmsg);
2027 case VHOST_USER_SET_LOG_FD:
2028 return vu_set_log_fd_exec(dev, vmsg);
2029 case VHOST_USER_SET_VRING_NUM:
2030 return vu_set_vring_num_exec(dev, vmsg);
2031 case VHOST_USER_SET_VRING_ADDR:
2032 return vu_set_vring_addr_exec(dev, vmsg);
2033 case VHOST_USER_SET_VRING_BASE:
2034 return vu_set_vring_base_exec(dev, vmsg);
2035 case VHOST_USER_GET_VRING_BASE:
2036 return vu_get_vring_base_exec(dev, vmsg);
2037 case VHOST_USER_SET_VRING_KICK:
2038 return vu_set_vring_kick_exec(dev, vmsg);
2039 case VHOST_USER_SET_VRING_CALL:
2040 return vu_set_vring_call_exec(dev, vmsg);
2041 case VHOST_USER_SET_VRING_ERR:
2042 return vu_set_vring_err_exec(dev, vmsg);
2043 case VHOST_USER_GET_QUEUE_NUM:
2044 return vu_get_queue_num_exec(dev, vmsg);
2045 case VHOST_USER_SET_VRING_ENABLE:
2046 return vu_set_vring_enable_exec(dev, vmsg);
2047 case VHOST_USER_SET_BACKEND_REQ_FD:
2048 return vu_set_backend_req_fd(dev, vmsg);
2049 case VHOST_USER_GET_CONFIG:
2050 return vu_get_config(dev, vmsg);
2051 case VHOST_USER_SET_CONFIG:
2052 return vu_set_config(dev, vmsg);
2053 case VHOST_USER_NONE:
2054 /* if you need processing before exit, override iface->process_msg */
2055 exit(0);
2056 case VHOST_USER_POSTCOPY_ADVISE:
2057 return vu_set_postcopy_advise(dev, vmsg);
2058 case VHOST_USER_POSTCOPY_LISTEN:
2059 return vu_set_postcopy_listen(dev, vmsg);
2060 case VHOST_USER_POSTCOPY_END:
2061 return vu_set_postcopy_end(dev, vmsg);
2062 case VHOST_USER_GET_INFLIGHT_FD:
2063 return vu_get_inflight_fd(dev, vmsg);
2064 case VHOST_USER_SET_INFLIGHT_FD:
2065 return vu_set_inflight_fd(dev, vmsg);
2066 case VHOST_USER_VRING_KICK:
2067 return vu_handle_vring_kick(dev, vmsg);
2068 case VHOST_USER_GET_MAX_MEM_SLOTS:
2069 return vu_handle_get_max_memslots(dev, vmsg);
2070 case VHOST_USER_ADD_MEM_REG:
2071 return vu_add_mem_reg(dev, vmsg);
2072 case VHOST_USER_REM_MEM_REG:
2073 return vu_rem_mem_reg(dev, vmsg);
2074 case VHOST_USER_GET_SHARED_OBJECT:
2075 return vu_get_shared_object(dev, vmsg);
2076 default:
2077 vmsg_close_fds(vmsg);
2078 vu_panic(dev, "Unhandled request: %d", vmsg->request);
2079 }
2080
2081 return false;
2082 }
2083
2084 bool
2085 vu_dispatch(VuDev *dev)
2086 {
2087 VhostUserMsg vmsg = { 0, };
2088 int reply_requested;
2089 bool need_reply, success = false;
2090
2091 if (!dev->read_msg(dev, dev->sock, &vmsg)) {
2092 goto end;
2093 }
2094
2095 need_reply = vmsg.flags & VHOST_USER_NEED_REPLY_MASK;
2096
2097 reply_requested = vu_process_message(dev, &vmsg);
2098 if (!reply_requested && need_reply) {
2099 vmsg_set_reply_u64(&vmsg, 0);
2100 reply_requested = 1;
2101 }
2102
2103 if (!reply_requested) {
2104 success = true;
2105 goto end;
2106 }
2107
2108 if (!vu_send_reply(dev, dev->sock, &vmsg)) {
2109 goto end;
2110 }
2111
2112 success = true;
2113
2114 end:
2115 free(vmsg.data);
2116 return success;
2117 }
2118
2119 void
2120 vu_deinit(VuDev *dev)
2121 {
2122 unsigned int i;
2123
2124 vu_remove_all_mem_regs(dev);
2125
2126 for (i = 0; i < dev->max_queues; i++) {
2127 VuVirtq *vq = &dev->vq[i];
2128
2129 if (vq->call_fd != -1) {
2130 close(vq->call_fd);
2131 vq->call_fd = -1;
2132 }
2133
2134 if (vq->kick_fd != -1) {
2135 dev->remove_watch(dev, vq->kick_fd);
2136 close(vq->kick_fd);
2137 vq->kick_fd = -1;
2138 }
2139
2140 if (vq->err_fd != -1) {
2141 close(vq->err_fd);
2142 vq->err_fd = -1;
2143 }
2144
2145 if (vq->resubmit_list) {
2146 free(vq->resubmit_list);
2147 vq->resubmit_list = NULL;
2148 }
2149
2150 vq->inflight = NULL;
2151 }
2152
2153 if (dev->inflight_info.addr) {
2154 munmap(dev->inflight_info.addr, dev->inflight_info.size);
2155 dev->inflight_info.addr = NULL;
2156 }
2157
2158 if (dev->inflight_info.fd > 0) {
2159 close(dev->inflight_info.fd);
2160 dev->inflight_info.fd = -1;
2161 }
2162
2163 vu_close_log(dev);
2164 if (dev->backend_fd != -1) {
2165 close(dev->backend_fd);
2166 dev->backend_fd = -1;
2167 }
2168 pthread_mutex_destroy(&dev->backend_mutex);
2169
2170 if (dev->sock != -1) {
2171 close(dev->sock);
2172 }
2173
2174 free(dev->vq);
2175 dev->vq = NULL;
2176 free(dev->regions);
2177 dev->regions = NULL;
2178 }
2179
2180 bool
2181 vu_init(VuDev *dev,
2182 uint16_t max_queues,
2183 int socket,
2184 vu_panic_cb panic,
2185 vu_read_msg_cb read_msg,
2186 vu_set_watch_cb set_watch,
2187 vu_remove_watch_cb remove_watch,
2188 const VuDevIface *iface)
2189 {
2190 uint16_t i;
2191
2192 assert(max_queues > 0);
2193 assert(socket >= 0);
2194 assert(set_watch);
2195 assert(remove_watch);
2196 assert(iface);
2197 assert(panic);
2198
2199 memset(dev, 0, sizeof(*dev));
2200
2201 dev->sock = socket;
2202 dev->panic = panic;
2203 dev->read_msg = read_msg ? read_msg : vu_message_read_default;
2204 dev->set_watch = set_watch;
2205 dev->remove_watch = remove_watch;
2206 dev->iface = iface;
2207 dev->log_call_fd = -1;
2208 pthread_mutex_init(&dev->backend_mutex, NULL);
2209 dev->backend_fd = -1;
2210 dev->max_queues = max_queues;
2211
2212 dev->regions = malloc(VHOST_USER_MAX_RAM_SLOTS * sizeof(dev->regions[0]));
2213 if (!dev->regions) {
2214 DPRINT("%s: failed to malloc mem regions\n", __func__);
2215 return false;
2216 }
2217
2218 dev->vq = malloc(max_queues * sizeof(dev->vq[0]));
2219 if (!dev->vq) {
2220 DPRINT("%s: failed to malloc virtqueues\n", __func__);
2221 free(dev->regions);
2222 dev->regions = NULL;
2223 return false;
2224 }
2225
2226 for (i = 0; i < max_queues; i++) {
2227 dev->vq[i] = (VuVirtq) {
2228 .call_fd = -1, .kick_fd = -1, .err_fd = -1,
2229 .notification = true,
2230 };
2231 }
2232
2233 return true;
2234 }
2235
2236 VuVirtq *
2237 vu_get_queue(VuDev *dev, int qidx)
2238 {
2239 assert(qidx < dev->max_queues);
2240 return &dev->vq[qidx];
2241 }
2242
2243 bool
2244 vu_queue_enabled(VuDev *dev, VuVirtq *vq)
2245 {
2246 return vq->enable;
2247 }
2248
2249 bool
2250 vu_queue_started(const VuDev *dev, const VuVirtq *vq)
2251 {
2252 return vq->started;
2253 }
2254
2255 static inline uint16_t
2256 vring_avail_flags(VuVirtq *vq)
2257 {
2258 return le16toh(vq->vring.avail->flags);
2259 }
2260
2261 static inline uint16_t
2262 vring_avail_idx(VuVirtq *vq)
2263 {
2264 vq->shadow_avail_idx = le16toh(vq->vring.avail->idx);
2265
2266 return vq->shadow_avail_idx;
2267 }
2268
2269 static inline uint16_t
2270 vring_avail_ring(VuVirtq *vq, int i)
2271 {
2272 return le16toh(vq->vring.avail->ring[i]);
2273 }
2274
2275 static inline uint16_t
2276 vring_get_used_event(VuVirtq *vq)
2277 {
2278 return vring_avail_ring(vq, vq->vring.num);
2279 }
2280
2281 static int
2282 virtqueue_num_heads(VuDev *dev, VuVirtq *vq, unsigned int idx)
2283 {
2284 uint16_t num_heads = vring_avail_idx(vq) - idx;
2285
2286 /* Check it isn't doing very strange things with descriptor numbers. */
2287 if (num_heads > vq->vring.num) {
2288 vu_panic(dev, "Guest moved used index from %u to %u",
2289 idx, vq->shadow_avail_idx);
2290 return -1;
2291 }
2292 if (num_heads) {
2293 /* On success, callers read a descriptor at vq->last_avail_idx.
2294 * Make sure descriptor read does not bypass avail index read. */
2295 smp_rmb();
2296 }
2297
2298 return num_heads;
2299 }
2300
2301 static bool
2302 virtqueue_get_head(VuDev *dev, VuVirtq *vq,
2303 unsigned int idx, unsigned int *head)
2304 {
2305 /* Grab the next descriptor number they're advertising, and increment
2306 * the index we've seen. */
2307 *head = vring_avail_ring(vq, idx % vq->vring.num);
2308
2309 /* If their number is silly, that's a fatal mistake. */
2310 if (*head >= vq->vring.num) {
2311 vu_panic(dev, "Guest says index %u is available", *head);
2312 return false;
2313 }
2314
2315 return true;
2316 }
2317
2318 static int
2319 virtqueue_read_indirect_desc(VuDev *dev, struct vring_desc *desc,
2320 uint64_t addr, size_t len)
2321 {
2322 struct vring_desc *ori_desc;
2323 uint64_t read_len;
2324
2325 if (len > (VIRTQUEUE_MAX_SIZE * sizeof(struct vring_desc))) {
2326 return -1;
2327 }
2328
2329 if (len == 0) {
2330 return -1;
2331 }
2332
2333 while (len) {
2334 read_len = len;
2335 ori_desc = vu_gpa_to_va(dev, &read_len, addr);
2336 if (!ori_desc) {
2337 return -1;
2338 }
2339
2340 memcpy(desc, ori_desc, read_len);
2341 len -= read_len;
2342 addr += read_len;
2343 desc += read_len;
2344 }
2345
2346 return 0;
2347 }
2348
2349 enum {
2350 VIRTQUEUE_READ_DESC_ERROR = -1,
2351 VIRTQUEUE_READ_DESC_DONE = 0, /* end of chain */
2352 VIRTQUEUE_READ_DESC_MORE = 1, /* more buffers in chain */
2353 };
2354
2355 static int
2356 virtqueue_read_next_desc(VuDev *dev, struct vring_desc *desc,
2357 int i, unsigned int max, unsigned int *next)
2358 {
2359 /* If this descriptor says it doesn't chain, we're done. */
2360 if (!(le16toh(desc[i].flags) & VRING_DESC_F_NEXT)) {
2361 return VIRTQUEUE_READ_DESC_DONE;
2362 }
2363
2364 /* Check they're not leading us off end of descriptors. */
2365 *next = le16toh(desc[i].next);
2366 /* Make sure compiler knows to grab that: we don't want it changing! */
2367 smp_wmb();
2368
2369 if (*next >= max) {
2370 vu_panic(dev, "Desc next is %u", *next);
2371 return VIRTQUEUE_READ_DESC_ERROR;
2372 }
2373
2374 return VIRTQUEUE_READ_DESC_MORE;
2375 }
2376
2377 void
2378 vu_queue_get_avail_bytes(VuDev *dev, VuVirtq *vq, unsigned int *in_bytes,
2379 unsigned int *out_bytes,
2380 unsigned max_in_bytes, unsigned max_out_bytes)
2381 {
2382 unsigned int idx;
2383 unsigned int total_bufs, in_total, out_total;
2384 int rc;
2385
2386 idx = vq->last_avail_idx;
2387
2388 total_bufs = in_total = out_total = 0;
2389 if (!vu_is_vq_usable(dev, vq)) {
2390 goto done;
2391 }
2392
2393 while ((rc = virtqueue_num_heads(dev, vq, idx)) > 0) {
2394 unsigned int max, desc_len, num_bufs, indirect = 0;
2395 uint64_t desc_addr, read_len;
2396 struct vring_desc *desc;
2397 struct vring_desc desc_buf[VIRTQUEUE_MAX_SIZE];
2398 unsigned int i;
2399
2400 max = vq->vring.num;
2401 num_bufs = total_bufs;
2402 if (!virtqueue_get_head(dev, vq, idx++, &i)) {
2403 goto err;
2404 }
2405 desc = vq->vring.desc;
2406
2407 if (le16toh(desc[i].flags) & VRING_DESC_F_INDIRECT) {
2408 if (le32toh(desc[i].len) % sizeof(struct vring_desc)) {
2409 vu_panic(dev, "Invalid size for indirect buffer table");
2410 goto err;
2411 }
2412
2413 /* If we've got too many, that implies a descriptor loop. */
2414 if (num_bufs >= max) {
2415 vu_panic(dev, "Looped descriptor");
2416 goto err;
2417 }
2418
2419 /* loop over the indirect descriptor table */
2420 indirect = 1;
2421 desc_addr = le64toh(desc[i].addr);
2422 desc_len = le32toh(desc[i].len);
2423 max = desc_len / sizeof(struct vring_desc);
2424 read_len = desc_len;
2425 desc = vu_gpa_to_va(dev, &read_len, desc_addr);
2426 if (unlikely(desc && read_len != desc_len)) {
2427 /* Failed to use zero copy */
2428 desc = NULL;
2429 if (!virtqueue_read_indirect_desc(dev, desc_buf,
2430 desc_addr,
2431 desc_len)) {
2432 desc = desc_buf;
2433 }
2434 }
2435 if (!desc) {
2436 vu_panic(dev, "Invalid indirect buffer table");
2437 goto err;
2438 }
2439 num_bufs = i = 0;
2440 }
2441
2442 do {
2443 /* If we've got too many, that implies a descriptor loop. */
2444 if (++num_bufs > max) {
2445 vu_panic(dev, "Looped descriptor");
2446 goto err;
2447 }
2448
2449 if (le16toh(desc[i].flags) & VRING_DESC_F_WRITE) {
2450 in_total += le32toh(desc[i].len);
2451 } else {
2452 out_total += le32toh(desc[i].len);
2453 }
2454 if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
2455 goto done;
2456 }
2457 rc = virtqueue_read_next_desc(dev, desc, i, max, &i);
2458 } while (rc == VIRTQUEUE_READ_DESC_MORE);
2459
2460 if (rc == VIRTQUEUE_READ_DESC_ERROR) {
2461 goto err;
2462 }
2463
2464 if (!indirect) {
2465 total_bufs = num_bufs;
2466 } else {
2467 total_bufs++;
2468 }
2469 }
2470 if (rc < 0) {
2471 goto err;
2472 }
2473 done:
2474 if (in_bytes) {
2475 *in_bytes = in_total;
2476 }
2477 if (out_bytes) {
2478 *out_bytes = out_total;
2479 }
2480 return;
2481
2482 err:
2483 in_total = out_total = 0;
2484 goto done;
2485 }
2486
2487 bool
2488 vu_queue_avail_bytes(VuDev *dev, VuVirtq *vq, unsigned int in_bytes,
2489 unsigned int out_bytes)
2490 {
2491 unsigned int in_total, out_total;
2492
2493 vu_queue_get_avail_bytes(dev, vq, &in_total, &out_total,
2494 in_bytes, out_bytes);
2495
2496 return in_bytes <= in_total && out_bytes <= out_total;
2497 }
2498
2499 /* Fetch avail_idx from VQ memory only when we really need to know if
2500 * guest has added some buffers. */
2501 bool
2502 vu_queue_empty(VuDev *dev, VuVirtq *vq)
2503 {
2504 if (!vu_is_vq_usable(dev, vq)) {
2505 return true;
2506 }
2507
2508 if (vq->shadow_avail_idx != vq->last_avail_idx) {
2509 return false;
2510 }
2511
2512 return vring_avail_idx(vq) == vq->last_avail_idx;
2513 }
2514
2515 static bool
2516 vring_notify(VuDev *dev, VuVirtq *vq)
2517 {
2518 uint16_t old, new;
2519 bool v;
2520
2521 /* We need to expose used array entries before checking used event. */
2522 smp_mb();
2523
2524 /* Always notify when queue is empty (when feature acknowledge) */
2525 if (vu_has_feature(dev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
2526 !vq->inuse && vu_queue_empty(dev, vq)) {
2527 return true;
2528 }
2529
2530 if (!vu_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
2531 return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT);
2532 }
2533
2534 v = vq->signalled_used_valid;
2535 vq->signalled_used_valid = true;
2536 old = vq->signalled_used;
2537 new = vq->signalled_used = vq->used_idx;
2538 return !v || vring_need_event(vring_get_used_event(vq), new, old);
2539 }
2540
2541 static void _vu_queue_notify(VuDev *dev, VuVirtq *vq, bool sync)
2542 {
2543 if (!vu_is_vq_usable(dev, vq)) {
2544 return;
2545 }
2546
2547 if (!vring_notify(dev, vq)) {
2548 DPRINT("skipped notify...\n");
2549 return;
2550 }
2551
2552 if (vq->call_fd < 0 &&
2553 vu_has_protocol_feature(dev,
2554 VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS) &&
2555 vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_BACKEND_REQ)) {
2556 VhostUserMsg vmsg = {
2557 .request = VHOST_USER_BACKEND_VRING_CALL,
2558 .flags = VHOST_USER_VERSION,
2559 .size = sizeof(vmsg.payload.state),
2560 .payload.state = {
2561 .index = vq - dev->vq,
2562 },
2563 };
2564 bool ack = sync &&
2565 vu_has_protocol_feature(dev,
2566 VHOST_USER_PROTOCOL_F_REPLY_ACK);
2567
2568 if (ack) {
2569 vmsg.flags |= VHOST_USER_NEED_REPLY_MASK;
2570 }
2571
2572 vu_message_write(dev, dev->backend_fd, &vmsg);
2573 if (ack) {
2574 vu_message_read_default(dev, dev->backend_fd, &vmsg);
2575 }
2576 return;
2577 }
2578
2579 if (eventfd_write(vq->call_fd, 1) < 0) {
2580 vu_panic(dev, "Error writing eventfd: %s", strerror(errno));
2581 }
2582 }
2583
2584 void vu_queue_notify(VuDev *dev, VuVirtq *vq)
2585 {
2586 _vu_queue_notify(dev, vq, false);
2587 }
2588
2589 void vu_queue_notify_sync(VuDev *dev, VuVirtq *vq)
2590 {
2591 _vu_queue_notify(dev, vq, true);
2592 }
2593
2594 void vu_config_change_msg(VuDev *dev)
2595 {
2596 VhostUserMsg vmsg = {
2597 .request = VHOST_USER_BACKEND_CONFIG_CHANGE_MSG,
2598 .flags = VHOST_USER_VERSION,
2599 };
2600
2601 vu_message_write(dev, dev->backend_fd, &vmsg);
2602 }
2603
2604 static inline void
2605 vring_used_flags_set_bit(VuVirtq *vq, int mask)
2606 {
2607 uint16_t *flags;
2608
2609 flags = (uint16_t *)((char*)vq->vring.used +
2610 offsetof(struct vring_used, flags));
2611 *flags = htole16(le16toh(*flags) | mask);
2612 }
2613
2614 static inline void
2615 vring_used_flags_unset_bit(VuVirtq *vq, int mask)
2616 {
2617 uint16_t *flags;
2618
2619 flags = (uint16_t *)((char*)vq->vring.used +
2620 offsetof(struct vring_used, flags));
2621 *flags = htole16(le16toh(*flags) & ~mask);
2622 }
2623
2624 static inline void
2625 vring_set_avail_event(VuVirtq *vq, uint16_t val)
2626 {
2627 uint16_t val_le = htole16(val);
2628
2629 if (!vq->notification) {
2630 return;
2631 }
2632
2633 memcpy(&vq->vring.used->ring[vq->vring.num], &val_le, sizeof(uint16_t));
2634 }
2635
2636 void
2637 vu_queue_set_notification(VuDev *dev, VuVirtq *vq, int enable)
2638 {
2639 vq->notification = enable;
2640 if (vu_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
2641 vring_set_avail_event(vq, vring_avail_idx(vq));
2642 } else if (enable) {
2643 vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY);
2644 } else {
2645 vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY);
2646 }
2647 if (enable) {
2648 /* Expose avail event/used flags before caller checks the avail idx. */
2649 smp_mb();
2650 }
2651 }
2652
2653 static bool
2654 virtqueue_map_desc(VuDev *dev,
2655 unsigned int *p_num_sg, struct iovec *iov,
2656 unsigned int max_num_sg, bool is_write,
2657 uint64_t pa, size_t sz)
2658 {
2659 unsigned num_sg = *p_num_sg;
2660
2661 assert(num_sg <= max_num_sg);
2662
2663 if (!sz) {
2664 vu_panic(dev, "virtio: zero sized buffers are not allowed");
2665 return false;
2666 }
2667
2668 while (sz) {
2669 uint64_t len = sz;
2670
2671 if (num_sg == max_num_sg) {
2672 vu_panic(dev, "virtio: too many descriptors in indirect table");
2673 return false;
2674 }
2675
2676 iov[num_sg].iov_base = vu_gpa_to_va(dev, &len, pa);
2677 if (iov[num_sg].iov_base == NULL) {
2678 vu_panic(dev, "virtio: invalid address for buffers");
2679 return false;
2680 }
2681 iov[num_sg].iov_len = len;
2682 num_sg++;
2683 sz -= len;
2684 pa += len;
2685 }
2686
2687 *p_num_sg = num_sg;
2688 return true;
2689 }
2690
2691 static void *
2692 virtqueue_alloc_element(size_t sz,
2693 unsigned out_num, unsigned in_num)
2694 {
2695 VuVirtqElement *elem;
2696 size_t in_sg_ofs = ALIGN_UP(sz, __alignof__(elem->in_sg[0]));
2697 size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]);
2698 size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]);
2699
2700 assert(sz >= sizeof(VuVirtqElement));
2701 elem = malloc(out_sg_end);
2702 if (!elem) {
2703 DPRINT("%s: failed to malloc virtqueue element\n", __func__);
2704 return NULL;
2705 }
2706 elem->out_num = out_num;
2707 elem->in_num = in_num;
2708 elem->in_sg = (void *)elem + in_sg_ofs;
2709 elem->out_sg = (void *)elem + out_sg_ofs;
2710 return elem;
2711 }
2712
2713 static void *
2714 vu_queue_map_desc(VuDev *dev, VuVirtq *vq, unsigned int idx, size_t sz)
2715 {
2716 struct vring_desc *desc = vq->vring.desc;
2717 uint64_t desc_addr, read_len;
2718 unsigned int desc_len;
2719 unsigned int max = vq->vring.num;
2720 unsigned int i = idx;
2721 VuVirtqElement *elem;
2722 unsigned int out_num = 0, in_num = 0;
2723 struct iovec iov[VIRTQUEUE_MAX_SIZE];
2724 struct vring_desc desc_buf[VIRTQUEUE_MAX_SIZE];
2725 int rc;
2726
2727 if (le16toh(desc[i].flags) & VRING_DESC_F_INDIRECT) {
2728 if (le32toh(desc[i].len) % sizeof(struct vring_desc)) {
2729 vu_panic(dev, "Invalid size for indirect buffer table");
2730 return NULL;
2731 }
2732
2733 /* loop over the indirect descriptor table */
2734 desc_addr = le64toh(desc[i].addr);
2735 desc_len = le32toh(desc[i].len);
2736 max = desc_len / sizeof(struct vring_desc);
2737 read_len = desc_len;
2738 desc = vu_gpa_to_va(dev, &read_len, desc_addr);
2739 if (unlikely(desc && read_len != desc_len)) {
2740 /* Failed to use zero copy */
2741 desc = NULL;
2742 if (!virtqueue_read_indirect_desc(dev, desc_buf,
2743 desc_addr,
2744 desc_len)) {
2745 desc = desc_buf;
2746 }
2747 }
2748 if (!desc) {
2749 vu_panic(dev, "Invalid indirect buffer table");
2750 return NULL;
2751 }
2752 i = 0;
2753 }
2754
2755 /* Collect all the descriptors */
2756 do {
2757 if (le16toh(desc[i].flags) & VRING_DESC_F_WRITE) {
2758 if (!virtqueue_map_desc(dev, &in_num, iov + out_num,
2759 VIRTQUEUE_MAX_SIZE - out_num, true,
2760 le64toh(desc[i].addr),
2761 le32toh(desc[i].len))) {
2762 return NULL;
2763 }
2764 } else {
2765 if (in_num) {
2766 vu_panic(dev, "Incorrect order for descriptors");
2767 return NULL;
2768 }
2769 if (!virtqueue_map_desc(dev, &out_num, iov,
2770 VIRTQUEUE_MAX_SIZE, false,
2771 le64toh(desc[i].addr),
2772 le32toh(desc[i].len))) {
2773 return NULL;
2774 }
2775 }
2776
2777 /* If we've got too many, that implies a descriptor loop. */
2778 if ((in_num + out_num) > max) {
2779 vu_panic(dev, "Looped descriptor");
2780 return NULL;
2781 }
2782 rc = virtqueue_read_next_desc(dev, desc, i, max, &i);
2783 } while (rc == VIRTQUEUE_READ_DESC_MORE);
2784
2785 if (rc == VIRTQUEUE_READ_DESC_ERROR) {
2786 vu_panic(dev, "read descriptor error");
2787 return NULL;
2788 }
2789
2790 /* Now copy what we have collected and mapped */
2791 elem = virtqueue_alloc_element(sz, out_num, in_num);
2792 if (!elem) {
2793 return NULL;
2794 }
2795 elem->index = idx;
2796 for (i = 0; i < out_num; i++) {
2797 elem->out_sg[i] = iov[i];
2798 }
2799 for (i = 0; i < in_num; i++) {
2800 elem->in_sg[i] = iov[out_num + i];
2801 }
2802
2803 return elem;
2804 }
2805
2806 static int
2807 vu_queue_inflight_get(VuDev *dev, VuVirtq *vq, int desc_idx)
2808 {
2809 if (!vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
2810 return 0;
2811 }
2812
2813 if (unlikely(!vq->inflight)) {
2814 return -1;
2815 }
2816
2817 vq->inflight->desc[desc_idx].counter = vq->counter++;
2818 vq->inflight->desc[desc_idx].inflight = 1;
2819
2820 return 0;
2821 }
2822
2823 static int
2824 vu_queue_inflight_pre_put(VuDev *dev, VuVirtq *vq, int desc_idx)
2825 {
2826 if (!vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
2827 return 0;
2828 }
2829
2830 if (unlikely(!vq->inflight)) {
2831 return -1;
2832 }
2833
2834 vq->inflight->last_batch_head = desc_idx;
2835
2836 return 0;
2837 }
2838
2839 static int
2840 vu_queue_inflight_post_put(VuDev *dev, VuVirtq *vq, int desc_idx)
2841 {
2842 if (!vu_has_protocol_feature(dev, VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
2843 return 0;
2844 }
2845
2846 if (unlikely(!vq->inflight)) {
2847 return -1;
2848 }
2849
2850 barrier();
2851
2852 vq->inflight->desc[desc_idx].inflight = 0;
2853
2854 barrier();
2855
2856 vq->inflight->used_idx = vq->used_idx;
2857
2858 return 0;
2859 }
2860
2861 void *
2862 vu_queue_pop(VuDev *dev, VuVirtq *vq, size_t sz)
2863 {
2864 int i;
2865 unsigned int head;
2866 VuVirtqElement *elem;
2867
2868 if (!vu_is_vq_usable(dev, vq)) {
2869 return NULL;
2870 }
2871
2872 if (unlikely(vq->resubmit_list && vq->resubmit_num > 0)) {
2873 i = (--vq->resubmit_num);
2874 elem = vu_queue_map_desc(dev, vq, vq->resubmit_list[i].index, sz);
2875
2876 if (!vq->resubmit_num) {
2877 free(vq->resubmit_list);
2878 vq->resubmit_list = NULL;
2879 }
2880
2881 return elem;
2882 }
2883
2884 if (vu_queue_empty(dev, vq)) {
2885 return NULL;
2886 }
2887 /*
2888 * Needed after virtio_queue_empty(), see comment in
2889 * virtqueue_num_heads().
2890 */
2891 smp_rmb();
2892
2893 if (vq->inuse >= vq->vring.num) {
2894 vu_panic(dev, "Virtqueue size exceeded");
2895 return NULL;
2896 }
2897
2898 if (!virtqueue_get_head(dev, vq, vq->last_avail_idx++, &head)) {
2899 return NULL;
2900 }
2901
2902 if (vu_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
2903 vring_set_avail_event(vq, vq->last_avail_idx);
2904 }
2905
2906 elem = vu_queue_map_desc(dev, vq, head, sz);
2907
2908 if (!elem) {
2909 return NULL;
2910 }
2911
2912 vq->inuse++;
2913
2914 vu_queue_inflight_get(dev, vq, head);
2915
2916 return elem;
2917 }
2918
2919 static void
2920 vu_queue_detach_element(VuDev *dev, VuVirtq *vq, VuVirtqElement *elem,
2921 size_t len)
2922 {
2923 vq->inuse--;
2924 /* unmap, when DMA support is added */
2925 }
2926
2927 void
2928 vu_queue_unpop(VuDev *dev, VuVirtq *vq, VuVirtqElement *elem,
2929 size_t len)
2930 {
2931 vq->last_avail_idx--;
2932 vu_queue_detach_element(dev, vq, elem, len);
2933 }
2934
2935 bool
2936 vu_queue_rewind(VuDev *dev, VuVirtq *vq, unsigned int num)
2937 {
2938 if (num > vq->inuse) {
2939 return false;
2940 }
2941 vq->last_avail_idx -= num;
2942 vq->inuse -= num;
2943 return true;
2944 }
2945
2946 static inline
2947 void vring_used_write(VuDev *dev, VuVirtq *vq,
2948 struct vring_used_elem *uelem, int i)
2949 {
2950 struct vring_used *used = vq->vring.used;
2951
2952 used->ring[i] = *uelem;
2953 vu_log_write(dev, vq->vring.log_guest_addr +
2954 offsetof(struct vring_used, ring[i]),
2955 sizeof(used->ring[i]));
2956 }
2957
2958
2959 static void
2960 vu_log_queue_fill(VuDev *dev, VuVirtq *vq,
2961 const VuVirtqElement *elem,
2962 unsigned int len)
2963 {
2964 struct vring_desc *desc = vq->vring.desc;
2965 unsigned int i, max, min, desc_len;
2966 uint64_t desc_addr, read_len;
2967 struct vring_desc desc_buf[VIRTQUEUE_MAX_SIZE];
2968 unsigned num_bufs = 0;
2969
2970 max = vq->vring.num;
2971 i = elem->index;
2972
2973 if (le16toh(desc[i].flags) & VRING_DESC_F_INDIRECT) {
2974 if (le32toh(desc[i].len) % sizeof(struct vring_desc)) {
2975 vu_panic(dev, "Invalid size for indirect buffer table");
2976 return;
2977 }
2978
2979 /* loop over the indirect descriptor table */
2980 desc_addr = le64toh(desc[i].addr);
2981 desc_len = le32toh(desc[i].len);
2982 max = desc_len / sizeof(struct vring_desc);
2983 read_len = desc_len;
2984 desc = vu_gpa_to_va(dev, &read_len, desc_addr);
2985 if (unlikely(desc && read_len != desc_len)) {
2986 /* Failed to use zero copy */
2987 desc = NULL;
2988 if (!virtqueue_read_indirect_desc(dev, desc_buf,
2989 desc_addr,
2990 desc_len)) {
2991 desc = desc_buf;
2992 }
2993 }
2994 if (!desc) {
2995 vu_panic(dev, "Invalid indirect buffer table");
2996 return;
2997 }
2998 i = 0;
2999 }
3000
3001 do {
3002 if (++num_bufs > max) {
3003 vu_panic(dev, "Looped descriptor");
3004 return;
3005 }
3006
3007 if (le16toh(desc[i].flags) & VRING_DESC_F_WRITE) {
3008 min = MIN(le32toh(desc[i].len), len);
3009 vu_log_write(dev, le64toh(desc[i].addr), min);
3010 len -= min;
3011 }
3012
3013 } while (len > 0 &&
3014 (virtqueue_read_next_desc(dev, desc, i, max, &i)
3015 == VIRTQUEUE_READ_DESC_MORE));
3016 }
3017
3018 void
3019 vu_queue_fill(VuDev *dev, VuVirtq *vq,
3020 const VuVirtqElement *elem,
3021 unsigned int len, unsigned int idx)
3022 {
3023 struct vring_used_elem uelem;
3024
3025 if (!vu_is_vq_usable(dev, vq)) {
3026 return;
3027 }
3028
3029 vu_log_queue_fill(dev, vq, elem, len);
3030
3031 idx = (idx + vq->used_idx) % vq->vring.num;
3032
3033 uelem.id = htole32(elem->index);
3034 uelem.len = htole32(len);
3035 vring_used_write(dev, vq, &uelem, idx);
3036 }
3037
3038 static inline
3039 void vring_used_idx_set(VuDev *dev, VuVirtq *vq, uint16_t val)
3040 {
3041 vq->vring.used->idx = htole16(val);
3042 vu_log_write(dev,
3043 vq->vring.log_guest_addr + offsetof(struct vring_used, idx),
3044 sizeof(vq->vring.used->idx));
3045
3046 vq->used_idx = val;
3047 }
3048
3049 void
3050 vu_queue_flush(VuDev *dev, VuVirtq *vq, unsigned int count)
3051 {
3052 uint16_t old, new;
3053
3054 if (!vu_is_vq_usable(dev, vq)) {
3055 return;
3056 }
3057
3058 /* Make sure buffer is written before we update index. */
3059 smp_wmb();
3060
3061 old = vq->used_idx;
3062 new = old + count;
3063 vring_used_idx_set(dev, vq, new);
3064 vq->inuse -= count;
3065 if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old))) {
3066 vq->signalled_used_valid = false;
3067 }
3068 }
3069
3070 void
3071 vu_queue_push(VuDev *dev, VuVirtq *vq,
3072 const VuVirtqElement *elem, unsigned int len)
3073 {
3074 vu_queue_fill(dev, vq, elem, len, 0);
3075 vu_queue_inflight_pre_put(dev, vq, elem->index);
3076 vu_queue_flush(dev, vq, 1);
3077 vu_queue_inflight_post_put(dev, vq, elem->index);
3078 }
AR7 emulation for QEMU