?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
virtio: Use started flag in virtio_vmstate_change()
[qemu.git] / contrib / libvhost-user / libvhost-user.c
blob74d42177c5c8bd9f6a1d0395ec23b51a93cbb79c
1 /*
2 * Vhost User library
3 *
4 * Copyright IBM, Corp. 2007
5 * Copyright (c) 2016 Red Hat, Inc.
6 *
7 * Authors:
8 * Anthony Liguori <aliguori@us.ibm.com>
9 * Marc-André Lureau <mlureau@redhat.com>
10 * Victor Kaplansky <victork@redhat.com>
11 *
12 * This work is licensed under the terms of the GNU GPL, version 2 or
13 * later. See the COPYING file in the top-level directory.
14 */
15
16 /* this code avoids GLib dependency */
17 #include <stdlib.h>
18 #include <stdio.h>
19 #include <unistd.h>
20 #include <stdarg.h>
21 #include <errno.h>
22 #include <string.h>
23 #include <assert.h>
24 #include <inttypes.h>
25 #include <sys/types.h>
26 #include <sys/socket.h>
27 #include <sys/eventfd.h>
28 #include <sys/mman.h>
29 #include "qemu/compiler.h"
30
31 #if defined(__linux__)
32 #include <sys/syscall.h>
33 #include <fcntl.h>
34 #include <sys/ioctl.h>
35 #include <linux/vhost.h>
36
37 #ifdef __NR_userfaultfd
38 #include <linux/userfaultfd.h>
39 #endif
40
41 #endif
42
43 #include "qemu/atomic.h"
44 #include "qemu/osdep.h"
45 #include "qemu/memfd.h"
46
47 #include "libvhost-user.h"
48
49 /* usually provided by GLib */
50 #ifndef MIN
51 #define MIN(x, y) ({ \
52 typeof(x) _min1 = (x); \
53 typeof(y) _min2 = (y); \
54 (void) (&_min1 == &_min2); \
55 _min1 < _min2 ? _min1 : _min2; })
56 #endif
57
58 /* Round number down to multiple */
59 #define ALIGN_DOWN(n, m) ((n) / (m) * (m))
60
61 /* Round number up to multiple */
62 #define ALIGN_UP(n, m) ALIGN_DOWN((n) + (m) - 1, (m))
63
64 /* Align each region to cache line size in inflight buffer */
65 #define INFLIGHT_ALIGNMENT 64
66
67 /* The version of inflight buffer */
68 #define INFLIGHT_VERSION 1
69
70 #define VHOST_USER_HDR_SIZE offsetof(VhostUserMsg, payload.u64)
71
72 /* The version of the protocol we support */
73 #define VHOST_USER_VERSION 1
74 #define LIBVHOST_USER_DEBUG 0
75
76 #define DPRINT(...) \
77 do { \
78 if (LIBVHOST_USER_DEBUG) { \
79 fprintf(stderr, __VA_ARGS__); \
80 } \
81 } while (0)
82
83 static inline
84 bool has_feature(uint64_t features, unsigned int fbit)
85 {
86 assert(fbit < 64);
87 return !!(features & (1ULL << fbit));
88 }
89
90 static inline
91 bool vu_has_feature(VuDev *dev,
92 unsigned int fbit)
93 {
94 return has_feature(dev->features, fbit);
95 }
96
97 static const char *
98 vu_request_to_string(unsigned int req)
99 {
100 #define REQ(req) [req] = #req
101 static const char *vu_request_str[] = {
102 REQ(VHOST_USER_NONE),
103 REQ(VHOST_USER_GET_FEATURES),
104 REQ(VHOST_USER_SET_FEATURES),
105 REQ(VHOST_USER_SET_OWNER),
106 REQ(VHOST_USER_RESET_OWNER),
107 REQ(VHOST_USER_SET_MEM_TABLE),
108 REQ(VHOST_USER_SET_LOG_BASE),
109 REQ(VHOST_USER_SET_LOG_FD),
110 REQ(VHOST_USER_SET_VRING_NUM),
111 REQ(VHOST_USER_SET_VRING_ADDR),
112 REQ(VHOST_USER_SET_VRING_BASE),
113 REQ(VHOST_USER_GET_VRING_BASE),
114 REQ(VHOST_USER_SET_VRING_KICK),
115 REQ(VHOST_USER_SET_VRING_CALL),
116 REQ(VHOST_USER_SET_VRING_ERR),
117 REQ(VHOST_USER_GET_PROTOCOL_FEATURES),
118 REQ(VHOST_USER_SET_PROTOCOL_FEATURES),
119 REQ(VHOST_USER_GET_QUEUE_NUM),
120 REQ(VHOST_USER_SET_VRING_ENABLE),
121 REQ(VHOST_USER_SEND_RARP),
122 REQ(VHOST_USER_NET_SET_MTU),
123 REQ(VHOST_USER_SET_SLAVE_REQ_FD),
124 REQ(VHOST_USER_IOTLB_MSG),
125 REQ(VHOST_USER_SET_VRING_ENDIAN),
126 REQ(VHOST_USER_GET_CONFIG),
127 REQ(VHOST_USER_SET_CONFIG),
128 REQ(VHOST_USER_POSTCOPY_ADVISE),
129 REQ(VHOST_USER_POSTCOPY_LISTEN),
130 REQ(VHOST_USER_POSTCOPY_END),
131 REQ(VHOST_USER_GET_INFLIGHT_FD),
132 REQ(VHOST_USER_SET_INFLIGHT_FD),
133 REQ(VHOST_USER_MAX),
134 };
135 #undef REQ
136
137 if (req < VHOST_USER_MAX) {
138 return vu_request_str[req];
139 } else {
140 return "unknown";
141 }
142 }
143
144 static void
145 vu_panic(VuDev *dev, const char *msg, ...)
146 {
147 char *buf = NULL;
148 va_list ap;
149
150 va_start(ap, msg);
151 if (vasprintf(&buf, msg, ap) < 0) {
152 buf = NULL;
153 }
154 va_end(ap);
155
156 dev->broken = true;
157 dev->panic(dev, buf);
158 free(buf);
159
160 /* FIXME: find a way to call virtio_error? */
161 }
162
163 /* Translate guest physical address to our virtual address. */
164 void *
165 vu_gpa_to_va(VuDev *dev, uint64_t *plen, uint64_t guest_addr)
166 {
167 int i;
168
169 if (*plen == 0) {
170 return NULL;
171 }
172
173 /* Find matching memory region. */
174 for (i = 0; i < dev->nregions; i++) {
175 VuDevRegion *r = &dev->regions[i];
176
177 if ((guest_addr >= r->gpa) && (guest_addr < (r->gpa + r->size))) {
178 if ((guest_addr + *plen) > (r->gpa + r->size)) {
179 *plen = r->gpa + r->size - guest_addr;
180 }
181 return (void *)(uintptr_t)
182 guest_addr - r->gpa + r->mmap_addr + r->mmap_offset;
183 }
184 }
185
186 return NULL;
187 }
188
189 /* Translate qemu virtual address to our virtual address. */
190 static void *
191 qva_to_va(VuDev *dev, uint64_t qemu_addr)
192 {
193 int i;
194
195 /* Find matching memory region. */
196 for (i = 0; i < dev->nregions; i++) {
197 VuDevRegion *r = &dev->regions[i];
198
199 if ((qemu_addr >= r->qva) && (qemu_addr < (r->qva + r->size))) {
200 return (void *)(uintptr_t)
201 qemu_addr - r->qva + r->mmap_addr + r->mmap_offset;
202 }
203 }
204
205 return NULL;
206 }
207
208 static void
209 vmsg_close_fds(VhostUserMsg *vmsg)
210 {
211 int i;
212
213 for (i = 0; i < vmsg->fd_num; i++) {
214 close(vmsg->fds[i]);
215 }
216 }
217
218 /* A test to see if we have userfault available */
219 static bool
220 have_userfault(void)
221 {
222 #if defined(__linux__) && defined(__NR_userfaultfd) &&\
223 defined(UFFD_FEATURE_MISSING_SHMEM) &&\
224 defined(UFFD_FEATURE_MISSING_HUGETLBFS)
225 /* Now test the kernel we're running on really has the features */
226 int ufd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
227 struct uffdio_api api_struct;
228 if (ufd < 0) {
229 return false;
230 }
231
232 api_struct.api = UFFD_API;
233 api_struct.features = UFFD_FEATURE_MISSING_SHMEM |
234 UFFD_FEATURE_MISSING_HUGETLBFS;
235 if (ioctl(ufd, UFFDIO_API, &api_struct)) {
236 close(ufd);
237 return false;
238 }
239 close(ufd);
240 return true;
241
242 #else
243 return false;
244 #endif
245 }
246
247 static bool
248 vu_message_read(VuDev *dev, int conn_fd, VhostUserMsg *vmsg)
249 {
250 char control[CMSG_SPACE(VHOST_MEMORY_MAX_NREGIONS * sizeof(int))] = { };
251 struct iovec iov = {
252 .iov_base = (char *)vmsg,
253 .iov_len = VHOST_USER_HDR_SIZE,
254 };
255 struct msghdr msg = {
256 .msg_iov = &iov,
257 .msg_iovlen = 1,
258 .msg_control = control,
259 .msg_controllen = sizeof(control),
260 };
261 size_t fd_size;
262 struct cmsghdr *cmsg;
263 int rc;
264
265 do {
266 rc = recvmsg(conn_fd, &msg, 0);
267 } while (rc < 0 && (errno == EINTR || errno == EAGAIN));
268
269 if (rc < 0) {
270 vu_panic(dev, "Error while recvmsg: %s", strerror(errno));
271 return false;
272 }
273
274 vmsg->fd_num = 0;
275 for (cmsg = CMSG_FIRSTHDR(&msg);
276 cmsg != NULL;
277 cmsg = CMSG_NXTHDR(&msg, cmsg))
278 {
279 if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS) {
280 fd_size = cmsg->cmsg_len - CMSG_LEN(0);
281 vmsg->fd_num = fd_size / sizeof(int);
282 memcpy(vmsg->fds, CMSG_DATA(cmsg), fd_size);
283 break;
284 }
285 }
286
287 if (vmsg->size > sizeof(vmsg->payload)) {
288 vu_panic(dev,
289 "Error: too big message request: %d, size: vmsg->size: %u, "
290 "while sizeof(vmsg->payload) = %zu\n",
291 vmsg->request, vmsg->size, sizeof(vmsg->payload));
292 goto fail;
293 }
294
295 if (vmsg->size) {
296 do {
297 rc = read(conn_fd, &vmsg->payload, vmsg->size);
298 } while (rc < 0 && (errno == EINTR || errno == EAGAIN));
299
300 if (rc <= 0) {
301 vu_panic(dev, "Error while reading: %s", strerror(errno));
302 goto fail;
303 }
304
305 assert(rc == vmsg->size);
306 }
307
308 return true;
309
310 fail:
311 vmsg_close_fds(vmsg);
312
313 return false;
314 }
315
316 static bool
317 vu_message_write(VuDev *dev, int conn_fd, VhostUserMsg *vmsg)
318 {
319 int rc;
320 uint8_t *p = (uint8_t *)vmsg;
321 char control[CMSG_SPACE(VHOST_MEMORY_MAX_NREGIONS * sizeof(int))] = { };
322 struct iovec iov = {
323 .iov_base = (char *)vmsg,
324 .iov_len = VHOST_USER_HDR_SIZE,
325 };
326 struct msghdr msg = {
327 .msg_iov = &iov,
328 .msg_iovlen = 1,
329 .msg_control = control,
330 };
331 struct cmsghdr *cmsg;
332
333 memset(control, 0, sizeof(control));
334 assert(vmsg->fd_num <= VHOST_MEMORY_MAX_NREGIONS);
335 if (vmsg->fd_num > 0) {
336 size_t fdsize = vmsg->fd_num * sizeof(int);
337 msg.msg_controllen = CMSG_SPACE(fdsize);
338 cmsg = CMSG_FIRSTHDR(&msg);
339 cmsg->cmsg_len = CMSG_LEN(fdsize);
340 cmsg->cmsg_level = SOL_SOCKET;
341 cmsg->cmsg_type = SCM_RIGHTS;
342 memcpy(CMSG_DATA(cmsg), vmsg->fds, fdsize);
343 } else {
344 msg.msg_controllen = 0;
345 }
346
347 do {
348 rc = sendmsg(conn_fd, &msg, 0);
349 } while (rc < 0 && (errno == EINTR || errno == EAGAIN));
350
351 if (vmsg->size) {
352 do {
353 if (vmsg->data) {
354 rc = write(conn_fd, vmsg->data, vmsg->size);
355 } else {
356 rc = write(conn_fd, p + VHOST_USER_HDR_SIZE, vmsg->size);
357 }
358 } while (rc < 0 && (errno == EINTR || errno == EAGAIN));
359 }
360
361 if (rc <= 0) {
362 vu_panic(dev, "Error while writing: %s", strerror(errno));
363 return false;
364 }
365
366 return true;
367 }
368
369 static bool
370 vu_send_reply(VuDev *dev, int conn_fd, VhostUserMsg *vmsg)
371 {
372 /* Set the version in the flags when sending the reply */
373 vmsg->flags &= ~VHOST_USER_VERSION_MASK;
374 vmsg->flags |= VHOST_USER_VERSION;
375 vmsg->flags |= VHOST_USER_REPLY_MASK;
376
377 return vu_message_write(dev, conn_fd, vmsg);
378 }
379
380 static bool
381 vu_process_message_reply(VuDev *dev, const VhostUserMsg *vmsg)
382 {
383 VhostUserMsg msg_reply;
384
385 if ((vmsg->flags & VHOST_USER_NEED_REPLY_MASK) == 0) {
386 return true;
387 }
388
389 if (!vu_message_read(dev, dev->slave_fd, &msg_reply)) {
390 return false;
391 }
392
393 if (msg_reply.request != vmsg->request) {
394 DPRINT("Received unexpected msg type. Expected %d received %d",
395 vmsg->request, msg_reply.request);
396 return false;
397 }
398
399 return msg_reply.payload.u64 == 0;
400 }
401
402 /* Kick the log_call_fd if required. */
403 static void
404 vu_log_kick(VuDev *dev)
405 {
406 if (dev->log_call_fd != -1) {
407 DPRINT("Kicking the QEMU's log...\n");
408 if (eventfd_write(dev->log_call_fd, 1) < 0) {
409 vu_panic(dev, "Error writing eventfd: %s", strerror(errno));
410 }
411 }
412 }
413
414 static void
415 vu_log_page(uint8_t *log_table, uint64_t page)
416 {
417 DPRINT("Logged dirty guest page: %"PRId64"\n", page);
418 atomic_or(&log_table[page / 8], 1 << (page % 8));
419 }
420
421 static void
422 vu_log_write(VuDev *dev, uint64_t address, uint64_t length)
423 {
424 uint64_t page;
425
426 if (!(dev->features & (1ULL << VHOST_F_LOG_ALL)) ||
427 !dev->log_table || !length) {
428 return;
429 }
430
431 assert(dev->log_size > ((address + length - 1) / VHOST_LOG_PAGE / 8));
432
433 page = address / VHOST_LOG_PAGE;
434 while (page * VHOST_LOG_PAGE < address + length) {
435 vu_log_page(dev->log_table, page);
436 page += VHOST_LOG_PAGE;
437 }
438
439 vu_log_kick(dev);
440 }
441
442 static void
443 vu_kick_cb(VuDev *dev, int condition, void *data)
444 {
445 int index = (intptr_t)data;
446 VuVirtq *vq = &dev->vq[index];
447 int sock = vq->kick_fd;
448 eventfd_t kick_data;
449 ssize_t rc;
450
451 rc = eventfd_read(sock, &kick_data);
452 if (rc == -1) {
453 vu_panic(dev, "kick eventfd_read(): %s", strerror(errno));
454 dev->remove_watch(dev, dev->vq[index].kick_fd);
455 } else {
456 DPRINT("Got kick_data: %016"PRIx64" handler:%p idx:%d\n",
457 kick_data, vq->handler, index);
458 if (vq->handler) {
459 vq->handler(dev, index);
460 }
461 }
462 }
463
464 static bool
465 vu_get_features_exec(VuDev *dev, VhostUserMsg *vmsg)
466 {
467 vmsg->payload.u64 =
468 1ULL << VHOST_F_LOG_ALL |
469 1ULL << VHOST_USER_F_PROTOCOL_FEATURES;
470
471 if (dev->iface->get_features) {
472 vmsg->payload.u64 |= dev->iface->get_features(dev);
473 }
474
475 vmsg->size = sizeof(vmsg->payload.u64);
476 vmsg->fd_num = 0;
477
478 DPRINT("Sending back to guest u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
479
480 return true;
481 }
482
483 static void
484 vu_set_enable_all_rings(VuDev *dev, bool enabled)
485 {
486 int i;
487
488 for (i = 0; i < VHOST_MAX_NR_VIRTQUEUE; i++) {
489 dev->vq[i].enable = enabled;
490 }
491 }
492
493 static bool
494 vu_set_features_exec(VuDev *dev, VhostUserMsg *vmsg)
495 {
496 DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
497
498 dev->features = vmsg->payload.u64;
499
500 if (!(dev->features & VHOST_USER_F_PROTOCOL_FEATURES)) {
501 vu_set_enable_all_rings(dev, true);
502 }
503
504 if (dev->iface->set_features) {
505 dev->iface->set_features(dev, dev->features);
506 }
507
508 return false;
509 }
510
511 static bool
512 vu_set_owner_exec(VuDev *dev, VhostUserMsg *vmsg)
513 {
514 return false;
515 }
516
517 static void
518 vu_close_log(VuDev *dev)
519 {
520 if (dev->log_table) {
521 if (munmap(dev->log_table, dev->log_size) != 0) {
522 perror("close log munmap() error");
523 }
524
525 dev->log_table = NULL;
526 }
527 if (dev->log_call_fd != -1) {
528 close(dev->log_call_fd);
529 dev->log_call_fd = -1;
530 }
531 }
532
533 static bool
534 vu_reset_device_exec(VuDev *dev, VhostUserMsg *vmsg)
535 {
536 vu_set_enable_all_rings(dev, false);
537
538 return false;
539 }
540
541 static bool
542 vu_set_mem_table_exec_postcopy(VuDev *dev, VhostUserMsg *vmsg)
543 {
544 int i;
545 VhostUserMemory m = vmsg->payload.memory, *memory = &m;
546 dev->nregions = memory->nregions;
547
548 DPRINT("Nregions: %d\n", memory->nregions);
549 for (i = 0; i < dev->nregions; i++) {
550 void *mmap_addr;
551 VhostUserMemoryRegion *msg_region = &memory->regions[i];
552 VuDevRegion *dev_region = &dev->regions[i];
553
554 DPRINT("Region %d\n", i);
555 DPRINT(" guest_phys_addr: 0x%016"PRIx64"\n",
556 msg_region->guest_phys_addr);
557 DPRINT(" memory_size: 0x%016"PRIx64"\n",
558 msg_region->memory_size);
559 DPRINT(" userspace_addr 0x%016"PRIx64"\n",
560 msg_region->userspace_addr);
561 DPRINT(" mmap_offset 0x%016"PRIx64"\n",
562 msg_region->mmap_offset);
563
564 dev_region->gpa = msg_region->guest_phys_addr;
565 dev_region->size = msg_region->memory_size;
566 dev_region->qva = msg_region->userspace_addr;
567 dev_region->mmap_offset = msg_region->mmap_offset;
568
569 /* We don't use offset argument of mmap() since the
570 * mapped address has to be page aligned, and we use huge
571 * pages.
572 * In postcopy we're using PROT_NONE here to catch anyone
573 * accessing it before we userfault
574 */
575 mmap_addr = mmap(0, dev_region->size + dev_region->mmap_offset,
576 PROT_NONE, MAP_SHARED,
577 vmsg->fds[i], 0);
578
579 if (mmap_addr == MAP_FAILED) {
580 vu_panic(dev, "region mmap error: %s", strerror(errno));
581 } else {
582 dev_region->mmap_addr = (uint64_t)(uintptr_t)mmap_addr;
583 DPRINT(" mmap_addr: 0x%016"PRIx64"\n",
584 dev_region->mmap_addr);
585 }
586
587 /* Return the address to QEMU so that it can translate the ufd
588 * fault addresses back.
589 */
590 msg_region->userspace_addr = (uintptr_t)(mmap_addr +
591 dev_region->mmap_offset);
592 close(vmsg->fds[i]);
593 }
594
595 /* Send the message back to qemu with the addresses filled in */
596 vmsg->fd_num = 0;
597 if (!vu_send_reply(dev, dev->sock, vmsg)) {
598 vu_panic(dev, "failed to respond to set-mem-table for postcopy");
599 return false;
600 }
601
602 /* Wait for QEMU to confirm that it's registered the handler for the
603 * faults.
604 */
605 if (!vu_message_read(dev, dev->sock, vmsg) ||
606 vmsg->size != sizeof(vmsg->payload.u64) ||
607 vmsg->payload.u64 != 0) {
608 vu_panic(dev, "failed to receive valid ack for postcopy set-mem-table");
609 return false;
610 }
611
612 /* OK, now we can go and register the memory and generate faults */
613 for (i = 0; i < dev->nregions; i++) {
614 VuDevRegion *dev_region = &dev->regions[i];
615 int ret;
616 #ifdef UFFDIO_REGISTER
617 /* We should already have an open ufd. Mark each memory
618 * range as ufd.
619 * Discard any mapping we have here; note I can't use MADV_REMOVE
620 * or fallocate to make the hole since I don't want to lose
621 * data that's already arrived in the shared process.
622 * TODO: How to do hugepage
623 */
624 ret = madvise((void *)dev_region->mmap_addr,
625 dev_region->size + dev_region->mmap_offset,
626 MADV_DONTNEED);
627 if (ret) {
628 fprintf(stderr,
629 "%s: Failed to madvise(DONTNEED) region %d: %s\n",
630 __func__, i, strerror(errno));
631 }
632 /* Turn off transparent hugepages so we dont get lose wakeups
633 * in neighbouring pages.
634 * TODO: Turn this backon later.
635 */
636 ret = madvise((void *)dev_region->mmap_addr,
637 dev_region->size + dev_region->mmap_offset,
638 MADV_NOHUGEPAGE);
639 if (ret) {
640 /* Note: This can happen legally on kernels that are configured
641 * without madvise'able hugepages
642 */
643 fprintf(stderr,
644 "%s: Failed to madvise(NOHUGEPAGE) region %d: %s\n",
645 __func__, i, strerror(errno));
646 }
647 struct uffdio_register reg_struct;
648 reg_struct.range.start = (uintptr_t)dev_region->mmap_addr;
649 reg_struct.range.len = dev_region->size + dev_region->mmap_offset;
650 reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING;
651
652 if (ioctl(dev->postcopy_ufd, UFFDIO_REGISTER, &reg_struct)) {
653 vu_panic(dev, "%s: Failed to userfault region %d "
654 "@%p + size:%zx offset: %zx: (ufd=%d)%s\n",
655 __func__, i,
656 dev_region->mmap_addr,
657 dev_region->size, dev_region->mmap_offset,
658 dev->postcopy_ufd, strerror(errno));
659 return false;
660 }
661 if (!(reg_struct.ioctls & ((__u64)1 << _UFFDIO_COPY))) {
662 vu_panic(dev, "%s Region (%d) doesn't support COPY",
663 __func__, i);
664 return false;
665 }
666 DPRINT("%s: region %d: Registered userfault for %llx + %llx\n",
667 __func__, i, reg_struct.range.start, reg_struct.range.len);
668 /* Now it's registered we can let the client at it */
669 if (mprotect((void *)dev_region->mmap_addr,
670 dev_region->size + dev_region->mmap_offset,
671 PROT_READ | PROT_WRITE)) {
672 vu_panic(dev, "failed to mprotect region %d for postcopy (%s)",
673 i, strerror(errno));
674 return false;
675 }
676 /* TODO: Stash 'zero' support flags somewhere */
677 #endif
678 }
679
680 return false;
681 }
682
683 static bool
684 vu_set_mem_table_exec(VuDev *dev, VhostUserMsg *vmsg)
685 {
686 int i;
687 VhostUserMemory m = vmsg->payload.memory, *memory = &m;
688
689 for (i = 0; i < dev->nregions; i++) {
690 VuDevRegion *r = &dev->regions[i];
691 void *m = (void *) (uintptr_t) r->mmap_addr;
692
693 if (m) {
694 munmap(m, r->size + r->mmap_offset);
695 }
696 }
697 dev->nregions = memory->nregions;
698
699 if (dev->postcopy_listening) {
700 return vu_set_mem_table_exec_postcopy(dev, vmsg);
701 }
702
703 DPRINT("Nregions: %d\n", memory->nregions);
704 for (i = 0; i < dev->nregions; i++) {
705 void *mmap_addr;
706 VhostUserMemoryRegion *msg_region = &memory->regions[i];
707 VuDevRegion *dev_region = &dev->regions[i];
708
709 DPRINT("Region %d\n", i);
710 DPRINT(" guest_phys_addr: 0x%016"PRIx64"\n",
711 msg_region->guest_phys_addr);
712 DPRINT(" memory_size: 0x%016"PRIx64"\n",
713 msg_region->memory_size);
714 DPRINT(" userspace_addr 0x%016"PRIx64"\n",
715 msg_region->userspace_addr);
716 DPRINT(" mmap_offset 0x%016"PRIx64"\n",
717 msg_region->mmap_offset);
718
719 dev_region->gpa = msg_region->guest_phys_addr;
720 dev_region->size = msg_region->memory_size;
721 dev_region->qva = msg_region->userspace_addr;
722 dev_region->mmap_offset = msg_region->mmap_offset;
723
724 /* We don't use offset argument of mmap() since the
725 * mapped address has to be page aligned, and we use huge
726 * pages. */
727 mmap_addr = mmap(0, dev_region->size + dev_region->mmap_offset,
728 PROT_READ | PROT_WRITE, MAP_SHARED,
729 vmsg->fds[i], 0);
730
731 if (mmap_addr == MAP_FAILED) {
732 vu_panic(dev, "region mmap error: %s", strerror(errno));
733 } else {
734 dev_region->mmap_addr = (uint64_t)(uintptr_t)mmap_addr;
735 DPRINT(" mmap_addr: 0x%016"PRIx64"\n",
736 dev_region->mmap_addr);
737 }
738
739 close(vmsg->fds[i]);
740 }
741
742 return false;
743 }
744
745 static bool
746 vu_set_log_base_exec(VuDev *dev, VhostUserMsg *vmsg)
747 {
748 int fd;
749 uint64_t log_mmap_size, log_mmap_offset;
750 void *rc;
751
752 if (vmsg->fd_num != 1 ||
753 vmsg->size != sizeof(vmsg->payload.log)) {
754 vu_panic(dev, "Invalid log_base message");
755 return true;
756 }
757
758 fd = vmsg->fds[0];
759 log_mmap_offset = vmsg->payload.log.mmap_offset;
760 log_mmap_size = vmsg->payload.log.mmap_size;
761 DPRINT("Log mmap_offset: %"PRId64"\n", log_mmap_offset);
762 DPRINT("Log mmap_size: %"PRId64"\n", log_mmap_size);
763
764 rc = mmap(0, log_mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd,
765 log_mmap_offset);
766 close(fd);
767 if (rc == MAP_FAILED) {
768 perror("log mmap error");
769 }
770
771 if (dev->log_table) {
772 munmap(dev->log_table, dev->log_size);
773 }
774 dev->log_table = rc;
775 dev->log_size = log_mmap_size;
776
777 vmsg->size = sizeof(vmsg->payload.u64);
778 vmsg->fd_num = 0;
779
780 return true;
781 }
782
783 static bool
784 vu_set_log_fd_exec(VuDev *dev, VhostUserMsg *vmsg)
785 {
786 if (vmsg->fd_num != 1) {
787 vu_panic(dev, "Invalid log_fd message");
788 return false;
789 }
790
791 if (dev->log_call_fd != -1) {
792 close(dev->log_call_fd);
793 }
794 dev->log_call_fd = vmsg->fds[0];
795 DPRINT("Got log_call_fd: %d\n", vmsg->fds[0]);
796
797 return false;
798 }
799
800 static bool
801 vu_set_vring_num_exec(VuDev *dev, VhostUserMsg *vmsg)
802 {
803 unsigned int index = vmsg->payload.state.index;
804 unsigned int num = vmsg->payload.state.num;
805
806 DPRINT("State.index: %d\n", index);
807 DPRINT("State.num: %d\n", num);
808 dev->vq[index].vring.num = num;
809
810 return false;
811 }
812
813 static bool
814 vu_set_vring_addr_exec(VuDev *dev, VhostUserMsg *vmsg)
815 {
816 struct vhost_vring_addr addr = vmsg->payload.addr, *vra = &addr;
817 unsigned int index = vra->index;
818 VuVirtq *vq = &dev->vq[index];
819
820 DPRINT("vhost_vring_addr:\n");
821 DPRINT(" index: %d\n", vra->index);
822 DPRINT(" flags: %d\n", vra->flags);
823 DPRINT(" desc_user_addr: 0x%016" PRIx64 "\n", vra->desc_user_addr);
824 DPRINT(" used_user_addr: 0x%016" PRIx64 "\n", vra->used_user_addr);
825 DPRINT(" avail_user_addr: 0x%016" PRIx64 "\n", vra->avail_user_addr);
826 DPRINT(" log_guest_addr: 0x%016" PRIx64 "\n", vra->log_guest_addr);
827
828 vq->vring.flags = vra->flags;
829 vq->vring.desc = qva_to_va(dev, vra->desc_user_addr);
830 vq->vring.used = qva_to_va(dev, vra->used_user_addr);
831 vq->vring.avail = qva_to_va(dev, vra->avail_user_addr);
832 vq->vring.log_guest_addr = vra->log_guest_addr;
833
834 DPRINT("Setting virtq addresses:\n");
835 DPRINT(" vring_desc at %p\n", vq->vring.desc);
836 DPRINT(" vring_used at %p\n", vq->vring.used);
837 DPRINT(" vring_avail at %p\n", vq->vring.avail);
838
839 if (!(vq->vring.desc && vq->vring.used && vq->vring.avail)) {
840 vu_panic(dev, "Invalid vring_addr message");
841 return false;
842 }
843
844 vq->used_idx = vq->vring.used->idx;
845
846 if (vq->last_avail_idx != vq->used_idx) {
847 bool resume = dev->iface->queue_is_processed_in_order &&
848 dev->iface->queue_is_processed_in_order(dev, index);
849
850 DPRINT("Last avail index != used index: %u != %u%s\n",
851 vq->last_avail_idx, vq->used_idx,
852 resume ? ", resuming" : "");
853
854 if (resume) {
855 vq->shadow_avail_idx = vq->last_avail_idx = vq->used_idx;
856 }
857 }
858
859 return false;
860 }
861
862 static bool
863 vu_set_vring_base_exec(VuDev *dev, VhostUserMsg *vmsg)
864 {
865 unsigned int index = vmsg->payload.state.index;
866 unsigned int num = vmsg->payload.state.num;
867
868 DPRINT("State.index: %d\n", index);
869 DPRINT("State.num: %d\n", num);
870 dev->vq[index].shadow_avail_idx = dev->vq[index].last_avail_idx = num;
871
872 return false;
873 }
874
875 static bool
876 vu_get_vring_base_exec(VuDev *dev, VhostUserMsg *vmsg)
877 {
878 unsigned int index = vmsg->payload.state.index;
879
880 DPRINT("State.index: %d\n", index);
881 vmsg->payload.state.num = dev->vq[index].last_avail_idx;
882 vmsg->size = sizeof(vmsg->payload.state);
883
884 dev->vq[index].started = false;
885 if (dev->iface->queue_set_started) {
886 dev->iface->queue_set_started(dev, index, false);
887 }
888
889 if (dev->vq[index].call_fd != -1) {
890 close(dev->vq[index].call_fd);
891 dev->vq[index].call_fd = -1;
892 }
893 if (dev->vq[index].kick_fd != -1) {
894 dev->remove_watch(dev, dev->vq[index].kick_fd);
895 close(dev->vq[index].kick_fd);
896 dev->vq[index].kick_fd = -1;
897 }
898
899 return true;
900 }
901
902 static bool
903 vu_check_queue_msg_file(VuDev *dev, VhostUserMsg *vmsg)
904 {
905 int index = vmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
906
907 if (index >= VHOST_MAX_NR_VIRTQUEUE) {
908 vmsg_close_fds(vmsg);
909 vu_panic(dev, "Invalid queue index: %u", index);
910 return false;
911 }
912
913 if (vmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK ||
914 vmsg->fd_num != 1) {
915 vmsg_close_fds(vmsg);
916 vu_panic(dev, "Invalid fds in request: %d", vmsg->request);
917 return false;
918 }
919
920 return true;
921 }
922
923 static int
924 inflight_desc_compare(const void *a, const void *b)
925 {
926 VuVirtqInflightDesc *desc0 = (VuVirtqInflightDesc *)a,
927 *desc1 = (VuVirtqInflightDesc *)b;
928
929 if (desc1->counter > desc0->counter &&
930 (desc1->counter - desc0->counter) < VIRTQUEUE_MAX_SIZE * 2) {
931 return 1;
932 }
933
934 return -1;
935 }
936
937 static int
938 vu_check_queue_inflights(VuDev *dev, VuVirtq *vq)
939 {
940 int i = 0;
941
942 if (!has_feature(dev->protocol_features,
943 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
944 return 0;
945 }
946
947 if (unlikely(!vq->inflight)) {
948 return -1;
949 }
950
951 if (unlikely(!vq->inflight->version)) {
952 /* initialize the buffer */
953 vq->inflight->version = INFLIGHT_VERSION;
954 return 0;
955 }
956
957 vq->used_idx = vq->vring.used->idx;
958 vq->resubmit_num = 0;
959 vq->resubmit_list = NULL;
960 vq->counter = 0;
961
962 if (unlikely(vq->inflight->used_idx != vq->used_idx)) {
963 vq->inflight->desc[vq->inflight->last_batch_head].inflight = 0;
964
965 barrier();
966
967 vq->inflight->used_idx = vq->used_idx;
968 }
969
970 for (i = 0; i < vq->inflight->desc_num; i++) {
971 if (vq->inflight->desc[i].inflight == 1) {
972 vq->inuse++;
973 }
974 }
975
976 vq->shadow_avail_idx = vq->last_avail_idx = vq->inuse + vq->used_idx;
977
978 if (vq->inuse) {
979 vq->resubmit_list = malloc(sizeof(VuVirtqInflightDesc) * vq->inuse);
980 if (!vq->resubmit_list) {
981 return -1;
982 }
983
984 for (i = 0; i < vq->inflight->desc_num; i++) {
985 if (vq->inflight->desc[i].inflight) {
986 vq->resubmit_list[vq->resubmit_num].index = i;
987 vq->resubmit_list[vq->resubmit_num].counter =
988 vq->inflight->desc[i].counter;
989 vq->resubmit_num++;
990 }
991 }
992
993 if (vq->resubmit_num > 1) {
994 qsort(vq->resubmit_list, vq->resubmit_num,
995 sizeof(VuVirtqInflightDesc), inflight_desc_compare);
996 }
997 vq->counter = vq->resubmit_list[0].counter + 1;
998 }
999
1000 /* in case of I/O hang after reconnecting */
1001 if (eventfd_write(vq->kick_fd, 1)) {
1002 return -1;
1003 }
1004
1005 return 0;
1006 }
1007
1008 static bool
1009 vu_set_vring_kick_exec(VuDev *dev, VhostUserMsg *vmsg)
1010 {
1011 int index = vmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
1012
1013 DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
1014
1015 if (!vu_check_queue_msg_file(dev, vmsg)) {
1016 return false;
1017 }
1018
1019 if (dev->vq[index].kick_fd != -1) {
1020 dev->remove_watch(dev, dev->vq[index].kick_fd);
1021 close(dev->vq[index].kick_fd);
1022 dev->vq[index].kick_fd = -1;
1023 }
1024
1025 dev->vq[index].kick_fd = vmsg->fds[0];
1026 DPRINT("Got kick_fd: %d for vq: %d\n", vmsg->fds[0], index);
1027
1028 dev->vq[index].started = true;
1029 if (dev->iface->queue_set_started) {
1030 dev->iface->queue_set_started(dev, index, true);
1031 }
1032
1033 if (dev->vq[index].kick_fd != -1 && dev->vq[index].handler) {
1034 dev->set_watch(dev, dev->vq[index].kick_fd, VU_WATCH_IN,
1035 vu_kick_cb, (void *)(long)index);
1036
1037 DPRINT("Waiting for kicks on fd: %d for vq: %d\n",
1038 dev->vq[index].kick_fd, index);
1039 }
1040
1041 if (vu_check_queue_inflights(dev, &dev->vq[index])) {
1042 vu_panic(dev, "Failed to check inflights for vq: %d\n", index);
1043 }
1044
1045 return false;
1046 }
1047
1048 void vu_set_queue_handler(VuDev *dev, VuVirtq *vq,
1049 vu_queue_handler_cb handler)
1050 {
1051 int qidx = vq - dev->vq;
1052
1053 vq->handler = handler;
1054 if (vq->kick_fd >= 0) {
1055 if (handler) {
1056 dev->set_watch(dev, vq->kick_fd, VU_WATCH_IN,
1057 vu_kick_cb, (void *)(long)qidx);
1058 } else {
1059 dev->remove_watch(dev, vq->kick_fd);
1060 }
1061 }
1062 }
1063
1064 bool vu_set_queue_host_notifier(VuDev *dev, VuVirtq *vq, int fd,
1065 int size, int offset)
1066 {
1067 int qidx = vq - dev->vq;
1068 int fd_num = 0;
1069 VhostUserMsg vmsg = {
1070 .request = VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG,
1071 .flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
1072 .size = sizeof(vmsg.payload.area),
1073 .payload.area = {
1074 .u64 = qidx & VHOST_USER_VRING_IDX_MASK,
1075 .size = size,
1076 .offset = offset,
1077 },
1078 };
1079
1080 if (fd == -1) {
1081 vmsg.payload.area.u64 |= VHOST_USER_VRING_NOFD_MASK;
1082 } else {
1083 vmsg.fds[fd_num++] = fd;
1084 }
1085
1086 vmsg.fd_num = fd_num;
1087
1088 if ((dev->protocol_features & VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD) == 0) {
1089 return false;
1090 }
1091
1092 if (!vu_message_write(dev, dev->slave_fd, &vmsg)) {
1093 return false;
1094 }
1095
1096 return vu_process_message_reply(dev, &vmsg);
1097 }
1098
1099 static bool
1100 vu_set_vring_call_exec(VuDev *dev, VhostUserMsg *vmsg)
1101 {
1102 int index = vmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
1103
1104 DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
1105
1106 if (!vu_check_queue_msg_file(dev, vmsg)) {
1107 return false;
1108 }
1109
1110 if (dev->vq[index].call_fd != -1) {
1111 close(dev->vq[index].call_fd);
1112 dev->vq[index].call_fd = -1;
1113 }
1114
1115 dev->vq[index].call_fd = vmsg->fds[0];
1116
1117 /* in case of I/O hang after reconnecting */
1118 if (eventfd_write(vmsg->fds[0], 1)) {
1119 return -1;
1120 }
1121
1122 DPRINT("Got call_fd: %d for vq: %d\n", vmsg->fds[0], index);
1123
1124 return false;
1125 }
1126
1127 static bool
1128 vu_set_vring_err_exec(VuDev *dev, VhostUserMsg *vmsg)
1129 {
1130 int index = vmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
1131
1132 DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
1133
1134 if (!vu_check_queue_msg_file(dev, vmsg)) {
1135 return false;
1136 }
1137
1138 if (dev->vq[index].err_fd != -1) {
1139 close(dev->vq[index].err_fd);
1140 dev->vq[index].err_fd = -1;
1141 }
1142
1143 dev->vq[index].err_fd = vmsg->fds[0];
1144
1145 return false;
1146 }
1147
1148 static bool
1149 vu_get_protocol_features_exec(VuDev *dev, VhostUserMsg *vmsg)
1150 {
1151 uint64_t features = 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD |
1152 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ |
1153 1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER |
1154 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD;
1155
1156 if (have_userfault()) {
1157 features |= 1ULL << VHOST_USER_PROTOCOL_F_PAGEFAULT;
1158 }
1159
1160 if (dev->iface->get_config && dev->iface->set_config) {
1161 features |= 1ULL << VHOST_USER_PROTOCOL_F_CONFIG;
1162 }
1163
1164 if (dev->iface->get_protocol_features) {
1165 features |= dev->iface->get_protocol_features(dev);
1166 }
1167
1168 vmsg->payload.u64 = features;
1169 vmsg->size = sizeof(vmsg->payload.u64);
1170 vmsg->fd_num = 0;
1171
1172 return true;
1173 }
1174
1175 static bool
1176 vu_set_protocol_features_exec(VuDev *dev, VhostUserMsg *vmsg)
1177 {
1178 uint64_t features = vmsg->payload.u64;
1179
1180 DPRINT("u64: 0x%016"PRIx64"\n", features);
1181
1182 dev->protocol_features = vmsg->payload.u64;
1183
1184 if (dev->iface->set_protocol_features) {
1185 dev->iface->set_protocol_features(dev, features);
1186 }
1187
1188 return false;
1189 }
1190
1191 static bool
1192 vu_get_queue_num_exec(VuDev *dev, VhostUserMsg *vmsg)
1193 {
1194 DPRINT("Function %s() not implemented yet.\n", __func__);
1195 return false;
1196 }
1197
1198 static bool
1199 vu_set_vring_enable_exec(VuDev *dev, VhostUserMsg *vmsg)
1200 {
1201 unsigned int index = vmsg->payload.state.index;
1202 unsigned int enable = vmsg->payload.state.num;
1203
1204 DPRINT("State.index: %d\n", index);
1205 DPRINT("State.enable: %d\n", enable);
1206
1207 if (index >= VHOST_MAX_NR_VIRTQUEUE) {
1208 vu_panic(dev, "Invalid vring_enable index: %u", index);
1209 return false;
1210 }
1211
1212 dev->vq[index].enable = enable;
1213 return false;
1214 }
1215
1216 static bool
1217 vu_set_slave_req_fd(VuDev *dev, VhostUserMsg *vmsg)
1218 {
1219 if (vmsg->fd_num != 1) {
1220 vu_panic(dev, "Invalid slave_req_fd message (%d fd's)", vmsg->fd_num);
1221 return false;
1222 }
1223
1224 if (dev->slave_fd != -1) {
1225 close(dev->slave_fd);
1226 }
1227 dev->slave_fd = vmsg->fds[0];
1228 DPRINT("Got slave_fd: %d\n", vmsg->fds[0]);
1229
1230 return false;
1231 }
1232
1233 static bool
1234 vu_get_config(VuDev *dev, VhostUserMsg *vmsg)
1235 {
1236 int ret = -1;
1237
1238 if (dev->iface->get_config) {
1239 ret = dev->iface->get_config(dev, vmsg->payload.config.region,
1240 vmsg->payload.config.size);
1241 }
1242
1243 if (ret) {
1244 /* resize to zero to indicate an error to master */
1245 vmsg->size = 0;
1246 }
1247
1248 return true;
1249 }
1250
1251 static bool
1252 vu_set_config(VuDev *dev, VhostUserMsg *vmsg)
1253 {
1254 int ret = -1;
1255
1256 if (dev->iface->set_config) {
1257 ret = dev->iface->set_config(dev, vmsg->payload.config.region,
1258 vmsg->payload.config.offset,
1259 vmsg->payload.config.size,
1260 vmsg->payload.config.flags);
1261 if (ret) {
1262 vu_panic(dev, "Set virtio configuration space failed");
1263 }
1264 }
1265
1266 return false;
1267 }
1268
1269 static bool
1270 vu_set_postcopy_advise(VuDev *dev, VhostUserMsg *vmsg)
1271 {
1272 dev->postcopy_ufd = -1;
1273 #ifdef UFFDIO_API
1274 struct uffdio_api api_struct;
1275
1276 dev->postcopy_ufd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
1277 vmsg->size = 0;
1278 #endif
1279
1280 if (dev->postcopy_ufd == -1) {
1281 vu_panic(dev, "Userfaultfd not available: %s", strerror(errno));
1282 goto out;
1283 }
1284
1285 #ifdef UFFDIO_API
1286 api_struct.api = UFFD_API;
1287 api_struct.features = 0;
1288 if (ioctl(dev->postcopy_ufd, UFFDIO_API, &api_struct)) {
1289 vu_panic(dev, "Failed UFFDIO_API: %s", strerror(errno));
1290 close(dev->postcopy_ufd);
1291 dev->postcopy_ufd = -1;
1292 goto out;
1293 }
1294 /* TODO: Stash feature flags somewhere */
1295 #endif
1296
1297 out:
1298 /* Return a ufd to the QEMU */
1299 vmsg->fd_num = 1;
1300 vmsg->fds[0] = dev->postcopy_ufd;
1301 return true; /* = send a reply */
1302 }
1303
1304 static bool
1305 vu_set_postcopy_listen(VuDev *dev, VhostUserMsg *vmsg)
1306 {
1307 vmsg->payload.u64 = -1;
1308 vmsg->size = sizeof(vmsg->payload.u64);
1309
1310 if (dev->nregions) {
1311 vu_panic(dev, "Regions already registered at postcopy-listen");
1312 return true;
1313 }
1314 dev->postcopy_listening = true;
1315
1316 vmsg->flags = VHOST_USER_VERSION | VHOST_USER_REPLY_MASK;
1317 vmsg->payload.u64 = 0; /* Success */
1318 return true;
1319 }
1320
1321 static bool
1322 vu_set_postcopy_end(VuDev *dev, VhostUserMsg *vmsg)
1323 {
1324 DPRINT("%s: Entry\n", __func__);
1325 dev->postcopy_listening = false;
1326 if (dev->postcopy_ufd > 0) {
1327 close(dev->postcopy_ufd);
1328 dev->postcopy_ufd = -1;
1329 DPRINT("%s: Done close\n", __func__);
1330 }
1331
1332 vmsg->fd_num = 0;
1333 vmsg->payload.u64 = 0;
1334 vmsg->size = sizeof(vmsg->payload.u64);
1335 vmsg->flags = VHOST_USER_VERSION | VHOST_USER_REPLY_MASK;
1336 DPRINT("%s: exit\n", __func__);
1337 return true;
1338 }
1339
1340 static inline uint64_t
1341 vu_inflight_queue_size(uint16_t queue_size)
1342 {
1343 return ALIGN_UP(sizeof(VuDescStateSplit) * queue_size +
1344 sizeof(uint16_t), INFLIGHT_ALIGNMENT);
1345 }
1346
1347 static bool
1348 vu_get_inflight_fd(VuDev *dev, VhostUserMsg *vmsg)
1349 {
1350 int fd;
1351 void *addr;
1352 uint64_t mmap_size;
1353 uint16_t num_queues, queue_size;
1354
1355 if (vmsg->size != sizeof(vmsg->payload.inflight)) {
1356 vu_panic(dev, "Invalid get_inflight_fd message:%d", vmsg->size);
1357 vmsg->payload.inflight.mmap_size = 0;
1358 return true;
1359 }
1360
1361 num_queues = vmsg->payload.inflight.num_queues;
1362 queue_size = vmsg->payload.inflight.queue_size;
1363
1364 DPRINT("set_inflight_fd num_queues: %"PRId16"\n", num_queues);
1365 DPRINT("set_inflight_fd queue_size: %"PRId16"\n", queue_size);
1366
1367 mmap_size = vu_inflight_queue_size(queue_size) * num_queues;
1368
1369 addr = qemu_memfd_alloc("vhost-inflight", mmap_size,
1370 F_SEAL_GROW | F_SEAL_SHRINK | F_SEAL_SEAL,
1371 &fd, NULL);
1372
1373 if (!addr) {
1374 vu_panic(dev, "Failed to alloc vhost inflight area");
1375 vmsg->payload.inflight.mmap_size = 0;
1376 return true;
1377 }
1378
1379 memset(addr, 0, mmap_size);
1380
1381 dev->inflight_info.addr = addr;
1382 dev->inflight_info.size = vmsg->payload.inflight.mmap_size = mmap_size;
1383 dev->inflight_info.fd = vmsg->fds[0] = fd;
1384 vmsg->fd_num = 1;
1385 vmsg->payload.inflight.mmap_offset = 0;
1386
1387 DPRINT("send inflight mmap_size: %"PRId64"\n",
1388 vmsg->payload.inflight.mmap_size);
1389 DPRINT("send inflight mmap offset: %"PRId64"\n",
1390 vmsg->payload.inflight.mmap_offset);
1391
1392 return true;
1393 }
1394
1395 static bool
1396 vu_set_inflight_fd(VuDev *dev, VhostUserMsg *vmsg)
1397 {
1398 int fd, i;
1399 uint64_t mmap_size, mmap_offset;
1400 uint16_t num_queues, queue_size;
1401 void *rc;
1402
1403 if (vmsg->fd_num != 1 ||
1404 vmsg->size != sizeof(vmsg->payload.inflight)) {
1405 vu_panic(dev, "Invalid set_inflight_fd message size:%d fds:%d",
1406 vmsg->size, vmsg->fd_num);
1407 return false;
1408 }
1409
1410 fd = vmsg->fds[0];
1411 mmap_size = vmsg->payload.inflight.mmap_size;
1412 mmap_offset = vmsg->payload.inflight.mmap_offset;
1413 num_queues = vmsg->payload.inflight.num_queues;
1414 queue_size = vmsg->payload.inflight.queue_size;
1415
1416 DPRINT("set_inflight_fd mmap_size: %"PRId64"\n", mmap_size);
1417 DPRINT("set_inflight_fd mmap_offset: %"PRId64"\n", mmap_offset);
1418 DPRINT("set_inflight_fd num_queues: %"PRId16"\n", num_queues);
1419 DPRINT("set_inflight_fd queue_size: %"PRId16"\n", queue_size);
1420
1421 rc = mmap(0, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
1422 fd, mmap_offset);
1423
1424 if (rc == MAP_FAILED) {
1425 vu_panic(dev, "set_inflight_fd mmap error: %s", strerror(errno));
1426 return false;
1427 }
1428
1429 if (dev->inflight_info.fd) {
1430 close(dev->inflight_info.fd);
1431 }
1432
1433 if (dev->inflight_info.addr) {
1434 munmap(dev->inflight_info.addr, dev->inflight_info.size);
1435 }
1436
1437 dev->inflight_info.fd = fd;
1438 dev->inflight_info.addr = rc;
1439 dev->inflight_info.size = mmap_size;
1440
1441 for (i = 0; i < num_queues; i++) {
1442 dev->vq[i].inflight = (VuVirtqInflight *)rc;
1443 dev->vq[i].inflight->desc_num = queue_size;
1444 rc = (void *)((char *)rc + vu_inflight_queue_size(queue_size));
1445 }
1446
1447 return false;
1448 }
1449
1450 static bool
1451 vu_process_message(VuDev *dev, VhostUserMsg *vmsg)
1452 {
1453 int do_reply = 0;
1454
1455 /* Print out generic part of the request. */
1456 DPRINT("================ Vhost user message ================\n");
1457 DPRINT("Request: %s (%d)\n", vu_request_to_string(vmsg->request),
1458 vmsg->request);
1459 DPRINT("Flags: 0x%x\n", vmsg->flags);
1460 DPRINT("Size: %d\n", vmsg->size);
1461
1462 if (vmsg->fd_num) {
1463 int i;
1464 DPRINT("Fds:");
1465 for (i = 0; i < vmsg->fd_num; i++) {
1466 DPRINT(" %d", vmsg->fds[i]);
1467 }
1468 DPRINT("\n");
1469 }
1470
1471 if (dev->iface->process_msg &&
1472 dev->iface->process_msg(dev, vmsg, &do_reply)) {
1473 return do_reply;
1474 }
1475
1476 switch (vmsg->request) {
1477 case VHOST_USER_GET_FEATURES:
1478 return vu_get_features_exec(dev, vmsg);
1479 case VHOST_USER_SET_FEATURES:
1480 return vu_set_features_exec(dev, vmsg);
1481 case VHOST_USER_GET_PROTOCOL_FEATURES:
1482 return vu_get_protocol_features_exec(dev, vmsg);
1483 case VHOST_USER_SET_PROTOCOL_FEATURES:
1484 return vu_set_protocol_features_exec(dev, vmsg);
1485 case VHOST_USER_SET_OWNER:
1486 return vu_set_owner_exec(dev, vmsg);
1487 case VHOST_USER_RESET_OWNER:
1488 return vu_reset_device_exec(dev, vmsg);
1489 case VHOST_USER_SET_MEM_TABLE:
1490 return vu_set_mem_table_exec(dev, vmsg);
1491 case VHOST_USER_SET_LOG_BASE:
1492 return vu_set_log_base_exec(dev, vmsg);
1493 case VHOST_USER_SET_LOG_FD:
1494 return vu_set_log_fd_exec(dev, vmsg);
1495 case VHOST_USER_SET_VRING_NUM:
1496 return vu_set_vring_num_exec(dev, vmsg);
1497 case VHOST_USER_SET_VRING_ADDR:
1498 return vu_set_vring_addr_exec(dev, vmsg);
1499 case VHOST_USER_SET_VRING_BASE:
1500 return vu_set_vring_base_exec(dev, vmsg);
1501 case VHOST_USER_GET_VRING_BASE:
1502 return vu_get_vring_base_exec(dev, vmsg);
1503 case VHOST_USER_SET_VRING_KICK:
1504 return vu_set_vring_kick_exec(dev, vmsg);
1505 case VHOST_USER_SET_VRING_CALL:
1506 return vu_set_vring_call_exec(dev, vmsg);
1507 case VHOST_USER_SET_VRING_ERR:
1508 return vu_set_vring_err_exec(dev, vmsg);
1509 case VHOST_USER_GET_QUEUE_NUM:
1510 return vu_get_queue_num_exec(dev, vmsg);
1511 case VHOST_USER_SET_VRING_ENABLE:
1512 return vu_set_vring_enable_exec(dev, vmsg);
1513 case VHOST_USER_SET_SLAVE_REQ_FD:
1514 return vu_set_slave_req_fd(dev, vmsg);
1515 case VHOST_USER_GET_CONFIG:
1516 return vu_get_config(dev, vmsg);
1517 case VHOST_USER_SET_CONFIG:
1518 return vu_set_config(dev, vmsg);
1519 case VHOST_USER_NONE:
1520 /* if you need processing before exit, override iface->process_msg */
1521 exit(0);
1522 case VHOST_USER_POSTCOPY_ADVISE:
1523 return vu_set_postcopy_advise(dev, vmsg);
1524 case VHOST_USER_POSTCOPY_LISTEN:
1525 return vu_set_postcopy_listen(dev, vmsg);
1526 case VHOST_USER_POSTCOPY_END:
1527 return vu_set_postcopy_end(dev, vmsg);
1528 case VHOST_USER_GET_INFLIGHT_FD:
1529 return vu_get_inflight_fd(dev, vmsg);
1530 case VHOST_USER_SET_INFLIGHT_FD:
1531 return vu_set_inflight_fd(dev, vmsg);
1532 default:
1533 vmsg_close_fds(vmsg);
1534 vu_panic(dev, "Unhandled request: %d", vmsg->request);
1535 }
1536
1537 return false;
1538 }
1539
1540 bool
1541 vu_dispatch(VuDev *dev)
1542 {
1543 VhostUserMsg vmsg = { 0, };
1544 int reply_requested;
1545 bool success = false;
1546
1547 if (!vu_message_read(dev, dev->sock, &vmsg)) {
1548 goto end;
1549 }
1550
1551 reply_requested = vu_process_message(dev, &vmsg);
1552 if (!reply_requested) {
1553 success = true;
1554 goto end;
1555 }
1556
1557 if (!vu_send_reply(dev, dev->sock, &vmsg)) {
1558 goto end;
1559 }
1560
1561 success = true;
1562
1563 end:
1564 free(vmsg.data);
1565 return success;
1566 }
1567
1568 void
1569 vu_deinit(VuDev *dev)
1570 {
1571 int i;
1572
1573 for (i = 0; i < dev->nregions; i++) {
1574 VuDevRegion *r = &dev->regions[i];
1575 void *m = (void *) (uintptr_t) r->mmap_addr;
1576 if (m != MAP_FAILED) {
1577 munmap(m, r->size + r->mmap_offset);
1578 }
1579 }
1580 dev->nregions = 0;
1581
1582 for (i = 0; i < VHOST_MAX_NR_VIRTQUEUE; i++) {
1583 VuVirtq *vq = &dev->vq[i];
1584
1585 if (vq->call_fd != -1) {
1586 close(vq->call_fd);
1587 vq->call_fd = -1;
1588 }
1589
1590 if (vq->kick_fd != -1) {
1591 close(vq->kick_fd);
1592 vq->kick_fd = -1;
1593 }
1594
1595 if (vq->err_fd != -1) {
1596 close(vq->err_fd);
1597 vq->err_fd = -1;
1598 }
1599
1600 if (vq->resubmit_list) {
1601 free(vq->resubmit_list);
1602 vq->resubmit_list = NULL;
1603 }
1604
1605 vq->inflight = NULL;
1606 }
1607
1608 if (dev->inflight_info.addr) {
1609 munmap(dev->inflight_info.addr, dev->inflight_info.size);
1610 dev->inflight_info.addr = NULL;
1611 }
1612
1613 if (dev->inflight_info.fd > 0) {
1614 close(dev->inflight_info.fd);
1615 dev->inflight_info.fd = -1;
1616 }
1617
1618 vu_close_log(dev);
1619 if (dev->slave_fd != -1) {
1620 close(dev->slave_fd);
1621 dev->slave_fd = -1;
1622 }
1623
1624 if (dev->sock != -1) {
1625 close(dev->sock);
1626 }
1627 }
1628
1629 void
1630 vu_init(VuDev *dev,
1631 int socket,
1632 vu_panic_cb panic,
1633 vu_set_watch_cb set_watch,
1634 vu_remove_watch_cb remove_watch,
1635 const VuDevIface *iface)
1636 {
1637 int i;
1638
1639 assert(socket >= 0);
1640 assert(set_watch);
1641 assert(remove_watch);
1642 assert(iface);
1643 assert(panic);
1644
1645 memset(dev, 0, sizeof(*dev));
1646
1647 dev->sock = socket;
1648 dev->panic = panic;
1649 dev->set_watch = set_watch;
1650 dev->remove_watch = remove_watch;
1651 dev->iface = iface;
1652 dev->log_call_fd = -1;
1653 dev->slave_fd = -1;
1654 for (i = 0; i < VHOST_MAX_NR_VIRTQUEUE; i++) {
1655 dev->vq[i] = (VuVirtq) {
1656 .call_fd = -1, .kick_fd = -1, .err_fd = -1,
1657 .notification = true,
1658 };
1659 }
1660 }
1661
1662 VuVirtq *
1663 vu_get_queue(VuDev *dev, int qidx)
1664 {
1665 assert(qidx < VHOST_MAX_NR_VIRTQUEUE);
1666 return &dev->vq[qidx];
1667 }
1668
1669 bool
1670 vu_queue_enabled(VuDev *dev, VuVirtq *vq)
1671 {
1672 return vq->enable;
1673 }
1674
1675 bool
1676 vu_queue_started(const VuDev *dev, const VuVirtq *vq)
1677 {
1678 return vq->started;
1679 }
1680
1681 static inline uint16_t
1682 vring_avail_flags(VuVirtq *vq)
1683 {
1684 return vq->vring.avail->flags;
1685 }
1686
1687 static inline uint16_t
1688 vring_avail_idx(VuVirtq *vq)
1689 {
1690 vq->shadow_avail_idx = vq->vring.avail->idx;
1691
1692 return vq->shadow_avail_idx;
1693 }
1694
1695 static inline uint16_t
1696 vring_avail_ring(VuVirtq *vq, int i)
1697 {
1698 return vq->vring.avail->ring[i];
1699 }
1700
1701 static inline uint16_t
1702 vring_get_used_event(VuVirtq *vq)
1703 {
1704 return vring_avail_ring(vq, vq->vring.num);
1705 }
1706
1707 static int
1708 virtqueue_num_heads(VuDev *dev, VuVirtq *vq, unsigned int idx)
1709 {
1710 uint16_t num_heads = vring_avail_idx(vq) - idx;
1711
1712 /* Check it isn't doing very strange things with descriptor numbers. */
1713 if (num_heads > vq->vring.num) {
1714 vu_panic(dev, "Guest moved used index from %u to %u",
1715 idx, vq->shadow_avail_idx);
1716 return -1;
1717 }
1718 if (num_heads) {
1719 /* On success, callers read a descriptor at vq->last_avail_idx.
1720 * Make sure descriptor read does not bypass avail index read. */
1721 smp_rmb();
1722 }
1723
1724 return num_heads;
1725 }
1726
1727 static bool
1728 virtqueue_get_head(VuDev *dev, VuVirtq *vq,
1729 unsigned int idx, unsigned int *head)
1730 {
1731 /* Grab the next descriptor number they're advertising, and increment
1732 * the index we've seen. */
1733 *head = vring_avail_ring(vq, idx % vq->vring.num);
1734
1735 /* If their number is silly, that's a fatal mistake. */
1736 if (*head >= vq->vring.num) {
1737 vu_panic(dev, "Guest says index %u is available", head);
1738 return false;
1739 }
1740
1741 return true;
1742 }
1743
1744 static int
1745 virtqueue_read_indirect_desc(VuDev *dev, struct vring_desc *desc,
1746 uint64_t addr, size_t len)
1747 {
1748 struct vring_desc *ori_desc;
1749 uint64_t read_len;
1750
1751 if (len > (VIRTQUEUE_MAX_SIZE * sizeof(struct vring_desc))) {
1752 return -1;
1753 }
1754
1755 if (len == 0) {
1756 return -1;
1757 }
1758
1759 while (len) {
1760 read_len = len;
1761 ori_desc = vu_gpa_to_va(dev, &read_len, addr);
1762 if (!ori_desc) {
1763 return -1;
1764 }
1765
1766 memcpy(desc, ori_desc, read_len);
1767 len -= read_len;
1768 addr += read_len;
1769 desc += read_len;
1770 }
1771
1772 return 0;
1773 }
1774
1775 enum {
1776 VIRTQUEUE_READ_DESC_ERROR = -1,
1777 VIRTQUEUE_READ_DESC_DONE = 0, /* end of chain */
1778 VIRTQUEUE_READ_DESC_MORE = 1, /* more buffers in chain */
1779 };
1780
1781 static int
1782 virtqueue_read_next_desc(VuDev *dev, struct vring_desc *desc,
1783 int i, unsigned int max, unsigned int *next)
1784 {
1785 /* If this descriptor says it doesn't chain, we're done. */
1786 if (!(desc[i].flags & VRING_DESC_F_NEXT)) {
1787 return VIRTQUEUE_READ_DESC_DONE;
1788 }
1789
1790 /* Check they're not leading us off end of descriptors. */
1791 *next = desc[i].next;
1792 /* Make sure compiler knows to grab that: we don't want it changing! */
1793 smp_wmb();
1794
1795 if (*next >= max) {
1796 vu_panic(dev, "Desc next is %u", next);
1797 return VIRTQUEUE_READ_DESC_ERROR;
1798 }
1799
1800 return VIRTQUEUE_READ_DESC_MORE;
1801 }
1802
1803 void
1804 vu_queue_get_avail_bytes(VuDev *dev, VuVirtq *vq, unsigned int *in_bytes,
1805 unsigned int *out_bytes,
1806 unsigned max_in_bytes, unsigned max_out_bytes)
1807 {
1808 unsigned int idx;
1809 unsigned int total_bufs, in_total, out_total;
1810 int rc;
1811
1812 idx = vq->last_avail_idx;
1813
1814 total_bufs = in_total = out_total = 0;
1815 if (unlikely(dev->broken) ||
1816 unlikely(!vq->vring.avail)) {
1817 goto done;
1818 }
1819
1820 while ((rc = virtqueue_num_heads(dev, vq, idx)) > 0) {
1821 unsigned int max, desc_len, num_bufs, indirect = 0;
1822 uint64_t desc_addr, read_len;
1823 struct vring_desc *desc;
1824 struct vring_desc desc_buf[VIRTQUEUE_MAX_SIZE];
1825 unsigned int i;
1826
1827 max = vq->vring.num;
1828 num_bufs = total_bufs;
1829 if (!virtqueue_get_head(dev, vq, idx++, &i)) {
1830 goto err;
1831 }
1832 desc = vq->vring.desc;
1833
1834 if (desc[i].flags & VRING_DESC_F_INDIRECT) {
1835 if (desc[i].len % sizeof(struct vring_desc)) {
1836 vu_panic(dev, "Invalid size for indirect buffer table");
1837 goto err;
1838 }
1839
1840 /* If we've got too many, that implies a descriptor loop. */
1841 if (num_bufs >= max) {
1842 vu_panic(dev, "Looped descriptor");
1843 goto err;
1844 }
1845
1846 /* loop over the indirect descriptor table */
1847 indirect = 1;
1848 desc_addr = desc[i].addr;
1849 desc_len = desc[i].len;
1850 max = desc_len / sizeof(struct vring_desc);
1851 read_len = desc_len;
1852 desc = vu_gpa_to_va(dev, &read_len, desc_addr);
1853 if (unlikely(desc && read_len != desc_len)) {
1854 /* Failed to use zero copy */
1855 desc = NULL;
1856 if (!virtqueue_read_indirect_desc(dev, desc_buf,
1857 desc_addr,
1858 desc_len)) {
1859 desc = desc_buf;
1860 }
1861 }
1862 if (!desc) {
1863 vu_panic(dev, "Invalid indirect buffer table");
1864 goto err;
1865 }
1866 num_bufs = i = 0;
1867 }
1868
1869 do {
1870 /* If we've got too many, that implies a descriptor loop. */
1871 if (++num_bufs > max) {
1872 vu_panic(dev, "Looped descriptor");
1873 goto err;
1874 }
1875
1876 if (desc[i].flags & VRING_DESC_F_WRITE) {
1877 in_total += desc[i].len;
1878 } else {
1879 out_total += desc[i].len;
1880 }
1881 if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
1882 goto done;
1883 }
1884 rc = virtqueue_read_next_desc(dev, desc, i, max, &i);
1885 } while (rc == VIRTQUEUE_READ_DESC_MORE);
1886
1887 if (rc == VIRTQUEUE_READ_DESC_ERROR) {
1888 goto err;
1889 }
1890
1891 if (!indirect) {
1892 total_bufs = num_bufs;
1893 } else {
1894 total_bufs++;
1895 }
1896 }
1897 if (rc < 0) {
1898 goto err;
1899 }
1900 done:
1901 if (in_bytes) {
1902 *in_bytes = in_total;
1903 }
1904 if (out_bytes) {
1905 *out_bytes = out_total;
1906 }
1907 return;
1908
1909 err:
1910 in_total = out_total = 0;
1911 goto done;
1912 }
1913
1914 bool
1915 vu_queue_avail_bytes(VuDev *dev, VuVirtq *vq, unsigned int in_bytes,
1916 unsigned int out_bytes)
1917 {
1918 unsigned int in_total, out_total;
1919
1920 vu_queue_get_avail_bytes(dev, vq, &in_total, &out_total,
1921 in_bytes, out_bytes);
1922
1923 return in_bytes <= in_total && out_bytes <= out_total;
1924 }
1925
1926 /* Fetch avail_idx from VQ memory only when we really need to know if
1927 * guest has added some buffers. */
1928 bool
1929 vu_queue_empty(VuDev *dev, VuVirtq *vq)
1930 {
1931 if (unlikely(dev->broken) ||
1932 unlikely(!vq->vring.avail)) {
1933 return true;
1934 }
1935
1936 if (vq->shadow_avail_idx != vq->last_avail_idx) {
1937 return false;
1938 }
1939
1940 return vring_avail_idx(vq) == vq->last_avail_idx;
1941 }
1942
1943 static bool
1944 vring_notify(VuDev *dev, VuVirtq *vq)
1945 {
1946 uint16_t old, new;
1947 bool v;
1948
1949 /* We need to expose used array entries before checking used event. */
1950 smp_mb();
1951
1952 /* Always notify when queue is empty (when feature acknowledge) */
1953 if (vu_has_feature(dev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
1954 !vq->inuse && vu_queue_empty(dev, vq)) {
1955 return true;
1956 }
1957
1958 if (!vu_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
1959 return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT);
1960 }
1961
1962 v = vq->signalled_used_valid;
1963 vq->signalled_used_valid = true;
1964 old = vq->signalled_used;
1965 new = vq->signalled_used = vq->used_idx;
1966 return !v || vring_need_event(vring_get_used_event(vq), new, old);
1967 }
1968
1969 void
1970 vu_queue_notify(VuDev *dev, VuVirtq *vq)
1971 {
1972 if (unlikely(dev->broken) ||
1973 unlikely(!vq->vring.avail)) {
1974 return;
1975 }
1976
1977 if (!vring_notify(dev, vq)) {
1978 DPRINT("skipped notify...\n");
1979 return;
1980 }
1981
1982 if (eventfd_write(vq->call_fd, 1) < 0) {
1983 vu_panic(dev, "Error writing eventfd: %s", strerror(errno));
1984 }
1985 }
1986
1987 static inline void
1988 vring_used_flags_set_bit(VuVirtq *vq, int mask)
1989 {
1990 uint16_t *flags;
1991
1992 flags = (uint16_t *)((char*)vq->vring.used +
1993 offsetof(struct vring_used, flags));
1994 *flags |= mask;
1995 }
1996
1997 static inline void
1998 vring_used_flags_unset_bit(VuVirtq *vq, int mask)
1999 {
2000 uint16_t *flags;
2001
2002 flags = (uint16_t *)((char*)vq->vring.used +
2003 offsetof(struct vring_used, flags));
2004 *flags &= ~mask;
2005 }
2006
2007 static inline void
2008 vring_set_avail_event(VuVirtq *vq, uint16_t val)
2009 {
2010 if (!vq->notification) {
2011 return;
2012 }
2013
2014 *((uint16_t *) &vq->vring.used->ring[vq->vring.num]) = val;
2015 }
2016
2017 void
2018 vu_queue_set_notification(VuDev *dev, VuVirtq *vq, int enable)
2019 {
2020 vq->notification = enable;
2021 if (vu_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
2022 vring_set_avail_event(vq, vring_avail_idx(vq));
2023 } else if (enable) {
2024 vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY);
2025 } else {
2026 vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY);
2027 }
2028 if (enable) {
2029 /* Expose avail event/used flags before caller checks the avail idx. */
2030 smp_mb();
2031 }
2032 }
2033
2034 static void
2035 virtqueue_map_desc(VuDev *dev,
2036 unsigned int *p_num_sg, struct iovec *iov,
2037 unsigned int max_num_sg, bool is_write,
2038 uint64_t pa, size_t sz)
2039 {
2040 unsigned num_sg = *p_num_sg;
2041
2042 assert(num_sg <= max_num_sg);
2043
2044 if (!sz) {
2045 vu_panic(dev, "virtio: zero sized buffers are not allowed");
2046 return;
2047 }
2048
2049 while (sz) {
2050 uint64_t len = sz;
2051
2052 if (num_sg == max_num_sg) {
2053 vu_panic(dev, "virtio: too many descriptors in indirect table");
2054 return;
2055 }
2056
2057 iov[num_sg].iov_base = vu_gpa_to_va(dev, &len, pa);
2058 if (iov[num_sg].iov_base == NULL) {
2059 vu_panic(dev, "virtio: invalid address for buffers");
2060 return;
2061 }
2062 iov[num_sg].iov_len = len;
2063 num_sg++;
2064 sz -= len;
2065 pa += len;
2066 }
2067
2068 *p_num_sg = num_sg;
2069 }
2070
2071 static void *
2072 virtqueue_alloc_element(size_t sz,
2073 unsigned out_num, unsigned in_num)
2074 {
2075 VuVirtqElement *elem;
2076 size_t in_sg_ofs = ALIGN_UP(sz, __alignof__(elem->in_sg[0]));
2077 size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]);
2078 size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]);
2079
2080 assert(sz >= sizeof(VuVirtqElement));
2081 elem = malloc(out_sg_end);
2082 elem->out_num = out_num;
2083 elem->in_num = in_num;
2084 elem->in_sg = (void *)elem + in_sg_ofs;
2085 elem->out_sg = (void *)elem + out_sg_ofs;
2086 return elem;
2087 }
2088
2089 static void *
2090 vu_queue_map_desc(VuDev *dev, VuVirtq *vq, unsigned int idx, size_t sz)
2091 {
2092 struct vring_desc *desc = vq->vring.desc;
2093 uint64_t desc_addr, read_len;
2094 unsigned int desc_len;
2095 unsigned int max = vq->vring.num;
2096 unsigned int i = idx;
2097 VuVirtqElement *elem;
2098 unsigned int out_num = 0, in_num = 0;
2099 struct iovec iov[VIRTQUEUE_MAX_SIZE];
2100 struct vring_desc desc_buf[VIRTQUEUE_MAX_SIZE];
2101 int rc;
2102
2103 if (desc[i].flags & VRING_DESC_F_INDIRECT) {
2104 if (desc[i].len % sizeof(struct vring_desc)) {
2105 vu_panic(dev, "Invalid size for indirect buffer table");
2106 }
2107
2108 /* loop over the indirect descriptor table */
2109 desc_addr = desc[i].addr;
2110 desc_len = desc[i].len;
2111 max = desc_len / sizeof(struct vring_desc);
2112 read_len = desc_len;
2113 desc = vu_gpa_to_va(dev, &read_len, desc_addr);
2114 if (unlikely(desc && read_len != desc_len)) {
2115 /* Failed to use zero copy */
2116 desc = NULL;
2117 if (!virtqueue_read_indirect_desc(dev, desc_buf,
2118 desc_addr,
2119 desc_len)) {
2120 desc = desc_buf;
2121 }
2122 }
2123 if (!desc) {
2124 vu_panic(dev, "Invalid indirect buffer table");
2125 return NULL;
2126 }
2127 i = 0;
2128 }
2129
2130 /* Collect all the descriptors */
2131 do {
2132 if (desc[i].flags & VRING_DESC_F_WRITE) {
2133 virtqueue_map_desc(dev, &in_num, iov + out_num,
2134 VIRTQUEUE_MAX_SIZE - out_num, true,
2135 desc[i].addr, desc[i].len);
2136 } else {
2137 if (in_num) {
2138 vu_panic(dev, "Incorrect order for descriptors");
2139 return NULL;
2140 }
2141 virtqueue_map_desc(dev, &out_num, iov,
2142 VIRTQUEUE_MAX_SIZE, false,
2143 desc[i].addr, desc[i].len);
2144 }
2145
2146 /* If we've got too many, that implies a descriptor loop. */
2147 if ((in_num + out_num) > max) {
2148 vu_panic(dev, "Looped descriptor");
2149 }
2150 rc = virtqueue_read_next_desc(dev, desc, i, max, &i);
2151 } while (rc == VIRTQUEUE_READ_DESC_MORE);
2152
2153 if (rc == VIRTQUEUE_READ_DESC_ERROR) {
2154 vu_panic(dev, "read descriptor error");
2155 return NULL;
2156 }
2157
2158 /* Now copy what we have collected and mapped */
2159 elem = virtqueue_alloc_element(sz, out_num, in_num);
2160 elem->index = idx;
2161 for (i = 0; i < out_num; i++) {
2162 elem->out_sg[i] = iov[i];
2163 }
2164 for (i = 0; i < in_num; i++) {
2165 elem->in_sg[i] = iov[out_num + i];
2166 }
2167
2168 return elem;
2169 }
2170
2171 static int
2172 vu_queue_inflight_get(VuDev *dev, VuVirtq *vq, int desc_idx)
2173 {
2174 if (!has_feature(dev->protocol_features,
2175 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
2176 return 0;
2177 }
2178
2179 if (unlikely(!vq->inflight)) {
2180 return -1;
2181 }
2182
2183 vq->inflight->desc[desc_idx].counter = vq->counter++;
2184 vq->inflight->desc[desc_idx].inflight = 1;
2185
2186 return 0;
2187 }
2188
2189 static int
2190 vu_queue_inflight_pre_put(VuDev *dev, VuVirtq *vq, int desc_idx)
2191 {
2192 if (!has_feature(dev->protocol_features,
2193 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
2194 return 0;
2195 }
2196
2197 if (unlikely(!vq->inflight)) {
2198 return -1;
2199 }
2200
2201 vq->inflight->last_batch_head = desc_idx;
2202
2203 return 0;
2204 }
2205
2206 static int
2207 vu_queue_inflight_post_put(VuDev *dev, VuVirtq *vq, int desc_idx)
2208 {
2209 if (!has_feature(dev->protocol_features,
2210 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
2211 return 0;
2212 }
2213
2214 if (unlikely(!vq->inflight)) {
2215 return -1;
2216 }
2217
2218 barrier();
2219
2220 vq->inflight->desc[desc_idx].inflight = 0;
2221
2222 barrier();
2223
2224 vq->inflight->used_idx = vq->used_idx;
2225
2226 return 0;
2227 }
2228
2229 void *
2230 vu_queue_pop(VuDev *dev, VuVirtq *vq, size_t sz)
2231 {
2232 int i;
2233 unsigned int head;
2234 VuVirtqElement *elem;
2235
2236 if (unlikely(dev->broken) ||
2237 unlikely(!vq->vring.avail)) {
2238 return NULL;
2239 }
2240
2241 if (unlikely(vq->resubmit_list && vq->resubmit_num > 0)) {
2242 i = (--vq->resubmit_num);
2243 elem = vu_queue_map_desc(dev, vq, vq->resubmit_list[i].index, sz);
2244
2245 if (!vq->resubmit_num) {
2246 free(vq->resubmit_list);
2247 vq->resubmit_list = NULL;
2248 }
2249
2250 return elem;
2251 }
2252
2253 if (vu_queue_empty(dev, vq)) {
2254 return NULL;
2255 }
2256 /*
2257 * Needed after virtio_queue_empty(), see comment in
2258 * virtqueue_num_heads().
2259 */
2260 smp_rmb();
2261
2262 if (vq->inuse >= vq->vring.num) {
2263 vu_panic(dev, "Virtqueue size exceeded");
2264 return NULL;
2265 }
2266
2267 if (!virtqueue_get_head(dev, vq, vq->last_avail_idx++, &head)) {
2268 return NULL;
2269 }
2270
2271 if (vu_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
2272 vring_set_avail_event(vq, vq->last_avail_idx);
2273 }
2274
2275 elem = vu_queue_map_desc(dev, vq, head, sz);
2276
2277 if (!elem) {
2278 return NULL;
2279 }
2280
2281 vq->inuse++;
2282
2283 vu_queue_inflight_get(dev, vq, head);
2284
2285 return elem;
2286 }
2287
2288 static void
2289 vu_queue_detach_element(VuDev *dev, VuVirtq *vq, VuVirtqElement *elem,
2290 size_t len)
2291 {
2292 vq->inuse--;
2293 /* unmap, when DMA support is added */
2294 }
2295
2296 void
2297 vu_queue_unpop(VuDev *dev, VuVirtq *vq, VuVirtqElement *elem,
2298 size_t len)
2299 {
2300 vq->last_avail_idx--;
2301 vu_queue_detach_element(dev, vq, elem, len);
2302 }
2303
2304 bool
2305 vu_queue_rewind(VuDev *dev, VuVirtq *vq, unsigned int num)
2306 {
2307 if (num > vq->inuse) {
2308 return false;
2309 }
2310 vq->last_avail_idx -= num;
2311 vq->inuse -= num;
2312 return true;
2313 }
2314
2315 static inline
2316 void vring_used_write(VuDev *dev, VuVirtq *vq,
2317 struct vring_used_elem *uelem, int i)
2318 {
2319 struct vring_used *used = vq->vring.used;
2320
2321 used->ring[i] = *uelem;
2322 vu_log_write(dev, vq->vring.log_guest_addr +
2323 offsetof(struct vring_used, ring[i]),
2324 sizeof(used->ring[i]));
2325 }
2326
2327
2328 static void
2329 vu_log_queue_fill(VuDev *dev, VuVirtq *vq,
2330 const VuVirtqElement *elem,
2331 unsigned int len)
2332 {
2333 struct vring_desc *desc = vq->vring.desc;
2334 unsigned int i, max, min, desc_len;
2335 uint64_t desc_addr, read_len;
2336 struct vring_desc desc_buf[VIRTQUEUE_MAX_SIZE];
2337 unsigned num_bufs = 0;
2338
2339 max = vq->vring.num;
2340 i = elem->index;
2341
2342 if (desc[i].flags & VRING_DESC_F_INDIRECT) {
2343 if (desc[i].len % sizeof(struct vring_desc)) {
2344 vu_panic(dev, "Invalid size for indirect buffer table");
2345 }
2346
2347 /* loop over the indirect descriptor table */
2348 desc_addr = desc[i].addr;
2349 desc_len = desc[i].len;
2350 max = desc_len / sizeof(struct vring_desc);
2351 read_len = desc_len;
2352 desc = vu_gpa_to_va(dev, &read_len, desc_addr);
2353 if (unlikely(desc && read_len != desc_len)) {
2354 /* Failed to use zero copy */
2355 desc = NULL;
2356 if (!virtqueue_read_indirect_desc(dev, desc_buf,
2357 desc_addr,
2358 desc_len)) {
2359 desc = desc_buf;
2360 }
2361 }
2362 if (!desc) {
2363 vu_panic(dev, "Invalid indirect buffer table");
2364 return;
2365 }
2366 i = 0;
2367 }
2368
2369 do {
2370 if (++num_bufs > max) {
2371 vu_panic(dev, "Looped descriptor");
2372 return;
2373 }
2374
2375 if (desc[i].flags & VRING_DESC_F_WRITE) {
2376 min = MIN(desc[i].len, len);
2377 vu_log_write(dev, desc[i].addr, min);
2378 len -= min;
2379 }
2380
2381 } while (len > 0 &&
2382 (virtqueue_read_next_desc(dev, desc, i, max, &i)
2383 == VIRTQUEUE_READ_DESC_MORE));
2384 }
2385
2386 void
2387 vu_queue_fill(VuDev *dev, VuVirtq *vq,
2388 const VuVirtqElement *elem,
2389 unsigned int len, unsigned int idx)
2390 {
2391 struct vring_used_elem uelem;
2392
2393 if (unlikely(dev->broken) ||
2394 unlikely(!vq->vring.avail)) {
2395 return;
2396 }
2397
2398 vu_log_queue_fill(dev, vq, elem, len);
2399
2400 idx = (idx + vq->used_idx) % vq->vring.num;
2401
2402 uelem.id = elem->index;
2403 uelem.len = len;
2404 vring_used_write(dev, vq, &uelem, idx);
2405 }
2406
2407 static inline
2408 void vring_used_idx_set(VuDev *dev, VuVirtq *vq, uint16_t val)
2409 {
2410 vq->vring.used->idx = val;
2411 vu_log_write(dev,
2412 vq->vring.log_guest_addr + offsetof(struct vring_used, idx),
2413 sizeof(vq->vring.used->idx));
2414
2415 vq->used_idx = val;
2416 }
2417
2418 void
2419 vu_queue_flush(VuDev *dev, VuVirtq *vq, unsigned int count)
2420 {
2421 uint16_t old, new;
2422
2423 if (unlikely(dev->broken) ||
2424 unlikely(!vq->vring.avail)) {
2425 return;
2426 }
2427
2428 /* Make sure buffer is written before we update index. */
2429 smp_wmb();
2430
2431 old = vq->used_idx;
2432 new = old + count;
2433 vring_used_idx_set(dev, vq, new);
2434 vq->inuse -= count;
2435 if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old))) {
2436 vq->signalled_used_valid = false;
2437 }
2438 }
2439
2440 void
2441 vu_queue_push(VuDev *dev, VuVirtq *vq,
2442 const VuVirtqElement *elem, unsigned int len)
2443 {
2444 vu_queue_fill(dev, vq, elem, len, 0);
2445 vu_queue_inflight_pre_put(dev, vq, elem->index);
2446 vu_queue_flush(dev, vq, 1);
2447 vu_queue_inflight_post_put(dev, vq, elem->index);
2448 }
QEmu