?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
libvhost-user: support many virtqueues
[qemu/ar7.git] / contrib / libvhost-user / libvhost-user.c
blob90bea856ddbc580d97cccefb6088b6e8b4e5b53f
1 /*
2 * Vhost User library
3 *
4 * Copyright IBM, Corp. 2007
5 * Copyright (c) 2016 Red Hat, Inc.
6 *
7 * Authors:
8 * Anthony Liguori <aliguori@us.ibm.com>
9 * Marc-André Lureau <mlureau@redhat.com>
10 * Victor Kaplansky <victork@redhat.com>
11 *
12 * This work is licensed under the terms of the GNU GPL, version 2 or
13 * later. See the COPYING file in the top-level directory.
14 */
15
16 /* this code avoids GLib dependency */
17 #include <stdlib.h>
18 #include <stdio.h>
19 #include <unistd.h>
20 #include <stdarg.h>
21 #include <errno.h>
22 #include <string.h>
23 #include <assert.h>
24 #include <inttypes.h>
25 #include <sys/types.h>
26 #include <sys/socket.h>
27 #include <sys/eventfd.h>
28 #include <sys/mman.h>
29 #include "qemu/compiler.h"
30
31 #if defined(__linux__)
32 #include <sys/syscall.h>
33 #include <fcntl.h>
34 #include <sys/ioctl.h>
35 #include <linux/vhost.h>
36
37 #ifdef __NR_userfaultfd
38 #include <linux/userfaultfd.h>
39 #endif
40
41 #endif
42
43 #include "qemu/atomic.h"
44 #include "qemu/osdep.h"
45 #include "qemu/memfd.h"
46
47 #include "libvhost-user.h"
48
49 /* usually provided by GLib */
50 #ifndef MIN
51 #define MIN(x, y) ({ \
52 typeof(x) _min1 = (x); \
53 typeof(y) _min2 = (y); \
54 (void) (&_min1 == &_min2); \
55 _min1 < _min2 ? _min1 : _min2; })
56 #endif
57
58 /* Round number down to multiple */
59 #define ALIGN_DOWN(n, m) ((n) / (m) * (m))
60
61 /* Round number up to multiple */
62 #define ALIGN_UP(n, m) ALIGN_DOWN((n) + (m) - 1, (m))
63
64 /* Align each region to cache line size in inflight buffer */
65 #define INFLIGHT_ALIGNMENT 64
66
67 /* The version of inflight buffer */
68 #define INFLIGHT_VERSION 1
69
70 #define VHOST_USER_HDR_SIZE offsetof(VhostUserMsg, payload.u64)
71
72 /* The version of the protocol we support */
73 #define VHOST_USER_VERSION 1
74 #define LIBVHOST_USER_DEBUG 0
75
76 #define DPRINT(...) \
77 do { \
78 if (LIBVHOST_USER_DEBUG) { \
79 fprintf(stderr, __VA_ARGS__); \
80 } \
81 } while (0)
82
83 static inline
84 bool has_feature(uint64_t features, unsigned int fbit)
85 {
86 assert(fbit < 64);
87 return !!(features & (1ULL << fbit));
88 }
89
90 static inline
91 bool vu_has_feature(VuDev *dev,
92 unsigned int fbit)
93 {
94 return has_feature(dev->features, fbit);
95 }
96
97 static const char *
98 vu_request_to_string(unsigned int req)
99 {
100 #define REQ(req) [req] = #req
101 static const char *vu_request_str[] = {
102 REQ(VHOST_USER_NONE),
103 REQ(VHOST_USER_GET_FEATURES),
104 REQ(VHOST_USER_SET_FEATURES),
105 REQ(VHOST_USER_SET_OWNER),
106 REQ(VHOST_USER_RESET_OWNER),
107 REQ(VHOST_USER_SET_MEM_TABLE),
108 REQ(VHOST_USER_SET_LOG_BASE),
109 REQ(VHOST_USER_SET_LOG_FD),
110 REQ(VHOST_USER_SET_VRING_NUM),
111 REQ(VHOST_USER_SET_VRING_ADDR),
112 REQ(VHOST_USER_SET_VRING_BASE),
113 REQ(VHOST_USER_GET_VRING_BASE),
114 REQ(VHOST_USER_SET_VRING_KICK),
115 REQ(VHOST_USER_SET_VRING_CALL),
116 REQ(VHOST_USER_SET_VRING_ERR),
117 REQ(VHOST_USER_GET_PROTOCOL_FEATURES),
118 REQ(VHOST_USER_SET_PROTOCOL_FEATURES),
119 REQ(VHOST_USER_GET_QUEUE_NUM),
120 REQ(VHOST_USER_SET_VRING_ENABLE),
121 REQ(VHOST_USER_SEND_RARP),
122 REQ(VHOST_USER_NET_SET_MTU),
123 REQ(VHOST_USER_SET_SLAVE_REQ_FD),
124 REQ(VHOST_USER_IOTLB_MSG),
125 REQ(VHOST_USER_SET_VRING_ENDIAN),
126 REQ(VHOST_USER_GET_CONFIG),
127 REQ(VHOST_USER_SET_CONFIG),
128 REQ(VHOST_USER_POSTCOPY_ADVISE),
129 REQ(VHOST_USER_POSTCOPY_LISTEN),
130 REQ(VHOST_USER_POSTCOPY_END),
131 REQ(VHOST_USER_GET_INFLIGHT_FD),
132 REQ(VHOST_USER_SET_INFLIGHT_FD),
133 REQ(VHOST_USER_GPU_SET_SOCKET),
134 REQ(VHOST_USER_MAX),
135 };
136 #undef REQ
137
138 if (req < VHOST_USER_MAX) {
139 return vu_request_str[req];
140 } else {
141 return "unknown";
142 }
143 }
144
145 static void
146 vu_panic(VuDev *dev, const char *msg, ...)
147 {
148 char *buf = NULL;
149 va_list ap;
150
151 va_start(ap, msg);
152 if (vasprintf(&buf, msg, ap) < 0) {
153 buf = NULL;
154 }
155 va_end(ap);
156
157 dev->broken = true;
158 dev->panic(dev, buf);
159 free(buf);
160
161 /* FIXME: find a way to call virtio_error? */
162 }
163
164 /* Translate guest physical address to our virtual address. */
165 void *
166 vu_gpa_to_va(VuDev *dev, uint64_t *plen, uint64_t guest_addr)
167 {
168 int i;
169
170 if (*plen == 0) {
171 return NULL;
172 }
173
174 /* Find matching memory region. */
175 for (i = 0; i < dev->nregions; i++) {
176 VuDevRegion *r = &dev->regions[i];
177
178 if ((guest_addr >= r->gpa) && (guest_addr < (r->gpa + r->size))) {
179 if ((guest_addr + *plen) > (r->gpa + r->size)) {
180 *plen = r->gpa + r->size - guest_addr;
181 }
182 return (void *)(uintptr_t)
183 guest_addr - r->gpa + r->mmap_addr + r->mmap_offset;
184 }
185 }
186
187 return NULL;
188 }
189
190 /* Translate qemu virtual address to our virtual address. */
191 static void *
192 qva_to_va(VuDev *dev, uint64_t qemu_addr)
193 {
194 int i;
195
196 /* Find matching memory region. */
197 for (i = 0; i < dev->nregions; i++) {
198 VuDevRegion *r = &dev->regions[i];
199
200 if ((qemu_addr >= r->qva) && (qemu_addr < (r->qva + r->size))) {
201 return (void *)(uintptr_t)
202 qemu_addr - r->qva + r->mmap_addr + r->mmap_offset;
203 }
204 }
205
206 return NULL;
207 }
208
209 static void
210 vmsg_close_fds(VhostUserMsg *vmsg)
211 {
212 int i;
213
214 for (i = 0; i < vmsg->fd_num; i++) {
215 close(vmsg->fds[i]);
216 }
217 }
218
219 /* Set reply payload.u64 and clear request flags and fd_num */
220 static void vmsg_set_reply_u64(VhostUserMsg *vmsg, uint64_t val)
221 {
222 vmsg->flags = 0; /* defaults will be set by vu_send_reply() */
223 vmsg->size = sizeof(vmsg->payload.u64);
224 vmsg->payload.u64 = val;
225 vmsg->fd_num = 0;
226 }
227
228 /* A test to see if we have userfault available */
229 static bool
230 have_userfault(void)
231 {
232 #if defined(__linux__) && defined(__NR_userfaultfd) &&\
233 defined(UFFD_FEATURE_MISSING_SHMEM) &&\
234 defined(UFFD_FEATURE_MISSING_HUGETLBFS)
235 /* Now test the kernel we're running on really has the features */
236 int ufd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
237 struct uffdio_api api_struct;
238 if (ufd < 0) {
239 return false;
240 }
241
242 api_struct.api = UFFD_API;
243 api_struct.features = UFFD_FEATURE_MISSING_SHMEM |
244 UFFD_FEATURE_MISSING_HUGETLBFS;
245 if (ioctl(ufd, UFFDIO_API, &api_struct)) {
246 close(ufd);
247 return false;
248 }
249 close(ufd);
250 return true;
251
252 #else
253 return false;
254 #endif
255 }
256
257 static bool
258 vu_message_read(VuDev *dev, int conn_fd, VhostUserMsg *vmsg)
259 {
260 char control[CMSG_SPACE(VHOST_MEMORY_MAX_NREGIONS * sizeof(int))] = { };
261 struct iovec iov = {
262 .iov_base = (char *)vmsg,
263 .iov_len = VHOST_USER_HDR_SIZE,
264 };
265 struct msghdr msg = {
266 .msg_iov = &iov,
267 .msg_iovlen = 1,
268 .msg_control = control,
269 .msg_controllen = sizeof(control),
270 };
271 size_t fd_size;
272 struct cmsghdr *cmsg;
273 int rc;
274
275 do {
276 rc = recvmsg(conn_fd, &msg, 0);
277 } while (rc < 0 && (errno == EINTR || errno == EAGAIN));
278
279 if (rc < 0) {
280 vu_panic(dev, "Error while recvmsg: %s", strerror(errno));
281 return false;
282 }
283
284 vmsg->fd_num = 0;
285 for (cmsg = CMSG_FIRSTHDR(&msg);
286 cmsg != NULL;
287 cmsg = CMSG_NXTHDR(&msg, cmsg))
288 {
289 if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS) {
290 fd_size = cmsg->cmsg_len - CMSG_LEN(0);
291 vmsg->fd_num = fd_size / sizeof(int);
292 memcpy(vmsg->fds, CMSG_DATA(cmsg), fd_size);
293 break;
294 }
295 }
296
297 if (vmsg->size > sizeof(vmsg->payload)) {
298 vu_panic(dev,
299 "Error: too big message request: %d, size: vmsg->size: %u, "
300 "while sizeof(vmsg->payload) = %zu\n",
301 vmsg->request, vmsg->size, sizeof(vmsg->payload));
302 goto fail;
303 }
304
305 if (vmsg->size) {
306 do {
307 rc = read(conn_fd, &vmsg->payload, vmsg->size);
308 } while (rc < 0 && (errno == EINTR || errno == EAGAIN));
309
310 if (rc <= 0) {
311 vu_panic(dev, "Error while reading: %s", strerror(errno));
312 goto fail;
313 }
314
315 assert(rc == vmsg->size);
316 }
317
318 return true;
319
320 fail:
321 vmsg_close_fds(vmsg);
322
323 return false;
324 }
325
326 static bool
327 vu_message_write(VuDev *dev, int conn_fd, VhostUserMsg *vmsg)
328 {
329 int rc;
330 uint8_t *p = (uint8_t *)vmsg;
331 char control[CMSG_SPACE(VHOST_MEMORY_MAX_NREGIONS * sizeof(int))] = { };
332 struct iovec iov = {
333 .iov_base = (char *)vmsg,
334 .iov_len = VHOST_USER_HDR_SIZE,
335 };
336 struct msghdr msg = {
337 .msg_iov = &iov,
338 .msg_iovlen = 1,
339 .msg_control = control,
340 };
341 struct cmsghdr *cmsg;
342
343 memset(control, 0, sizeof(control));
344 assert(vmsg->fd_num <= VHOST_MEMORY_MAX_NREGIONS);
345 if (vmsg->fd_num > 0) {
346 size_t fdsize = vmsg->fd_num * sizeof(int);
347 msg.msg_controllen = CMSG_SPACE(fdsize);
348 cmsg = CMSG_FIRSTHDR(&msg);
349 cmsg->cmsg_len = CMSG_LEN(fdsize);
350 cmsg->cmsg_level = SOL_SOCKET;
351 cmsg->cmsg_type = SCM_RIGHTS;
352 memcpy(CMSG_DATA(cmsg), vmsg->fds, fdsize);
353 } else {
354 msg.msg_controllen = 0;
355 }
356
357 do {
358 rc = sendmsg(conn_fd, &msg, 0);
359 } while (rc < 0 && (errno == EINTR || errno == EAGAIN));
360
361 if (vmsg->size) {
362 do {
363 if (vmsg->data) {
364 rc = write(conn_fd, vmsg->data, vmsg->size);
365 } else {
366 rc = write(conn_fd, p + VHOST_USER_HDR_SIZE, vmsg->size);
367 }
368 } while (rc < 0 && (errno == EINTR || errno == EAGAIN));
369 }
370
371 if (rc <= 0) {
372 vu_panic(dev, "Error while writing: %s", strerror(errno));
373 return false;
374 }
375
376 return true;
377 }
378
379 static bool
380 vu_send_reply(VuDev *dev, int conn_fd, VhostUserMsg *vmsg)
381 {
382 /* Set the version in the flags when sending the reply */
383 vmsg->flags &= ~VHOST_USER_VERSION_MASK;
384 vmsg->flags |= VHOST_USER_VERSION;
385 vmsg->flags |= VHOST_USER_REPLY_MASK;
386
387 return vu_message_write(dev, conn_fd, vmsg);
388 }
389
390 static bool
391 vu_process_message_reply(VuDev *dev, const VhostUserMsg *vmsg)
392 {
393 VhostUserMsg msg_reply;
394
395 if ((vmsg->flags & VHOST_USER_NEED_REPLY_MASK) == 0) {
396 return true;
397 }
398
399 if (!vu_message_read(dev, dev->slave_fd, &msg_reply)) {
400 return false;
401 }
402
403 if (msg_reply.request != vmsg->request) {
404 DPRINT("Received unexpected msg type. Expected %d received %d",
405 vmsg->request, msg_reply.request);
406 return false;
407 }
408
409 return msg_reply.payload.u64 == 0;
410 }
411
412 /* Kick the log_call_fd if required. */
413 static void
414 vu_log_kick(VuDev *dev)
415 {
416 if (dev->log_call_fd != -1) {
417 DPRINT("Kicking the QEMU's log...\n");
418 if (eventfd_write(dev->log_call_fd, 1) < 0) {
419 vu_panic(dev, "Error writing eventfd: %s", strerror(errno));
420 }
421 }
422 }
423
424 static void
425 vu_log_page(uint8_t *log_table, uint64_t page)
426 {
427 DPRINT("Logged dirty guest page: %"PRId64"\n", page);
428 atomic_or(&log_table[page / 8], 1 << (page % 8));
429 }
430
431 static void
432 vu_log_write(VuDev *dev, uint64_t address, uint64_t length)
433 {
434 uint64_t page;
435
436 if (!(dev->features & (1ULL << VHOST_F_LOG_ALL)) ||
437 !dev->log_table || !length) {
438 return;
439 }
440
441 assert(dev->log_size > ((address + length - 1) / VHOST_LOG_PAGE / 8));
442
443 page = address / VHOST_LOG_PAGE;
444 while (page * VHOST_LOG_PAGE < address + length) {
445 vu_log_page(dev->log_table, page);
446 page += 1;
447 }
448
449 vu_log_kick(dev);
450 }
451
452 static void
453 vu_kick_cb(VuDev *dev, int condition, void *data)
454 {
455 int index = (intptr_t)data;
456 VuVirtq *vq = &dev->vq[index];
457 int sock = vq->kick_fd;
458 eventfd_t kick_data;
459 ssize_t rc;
460
461 rc = eventfd_read(sock, &kick_data);
462 if (rc == -1) {
463 vu_panic(dev, "kick eventfd_read(): %s", strerror(errno));
464 dev->remove_watch(dev, dev->vq[index].kick_fd);
465 } else {
466 DPRINT("Got kick_data: %016"PRIx64" handler:%p idx:%d\n",
467 kick_data, vq->handler, index);
468 if (vq->handler) {
469 vq->handler(dev, index);
470 }
471 }
472 }
473
474 static bool
475 vu_get_features_exec(VuDev *dev, VhostUserMsg *vmsg)
476 {
477 vmsg->payload.u64 =
478 1ULL << VHOST_F_LOG_ALL |
479 1ULL << VHOST_USER_F_PROTOCOL_FEATURES;
480
481 if (dev->iface->get_features) {
482 vmsg->payload.u64 |= dev->iface->get_features(dev);
483 }
484
485 vmsg->size = sizeof(vmsg->payload.u64);
486 vmsg->fd_num = 0;
487
488 DPRINT("Sending back to guest u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
489
490 return true;
491 }
492
493 static void
494 vu_set_enable_all_rings(VuDev *dev, bool enabled)
495 {
496 uint16_t i;
497
498 for (i = 0; i < dev->max_queues; i++) {
499 dev->vq[i].enable = enabled;
500 }
501 }
502
503 static bool
504 vu_set_features_exec(VuDev *dev, VhostUserMsg *vmsg)
505 {
506 DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
507
508 dev->features = vmsg->payload.u64;
509
510 if (!(dev->features & VHOST_USER_F_PROTOCOL_FEATURES)) {
511 vu_set_enable_all_rings(dev, true);
512 }
513
514 if (dev->iface->set_features) {
515 dev->iface->set_features(dev, dev->features);
516 }
517
518 return false;
519 }
520
521 static bool
522 vu_set_owner_exec(VuDev *dev, VhostUserMsg *vmsg)
523 {
524 return false;
525 }
526
527 static void
528 vu_close_log(VuDev *dev)
529 {
530 if (dev->log_table) {
531 if (munmap(dev->log_table, dev->log_size) != 0) {
532 perror("close log munmap() error");
533 }
534
535 dev->log_table = NULL;
536 }
537 if (dev->log_call_fd != -1) {
538 close(dev->log_call_fd);
539 dev->log_call_fd = -1;
540 }
541 }
542
543 static bool
544 vu_reset_device_exec(VuDev *dev, VhostUserMsg *vmsg)
545 {
546 vu_set_enable_all_rings(dev, false);
547
548 return false;
549 }
550
551 static bool
552 vu_set_mem_table_exec_postcopy(VuDev *dev, VhostUserMsg *vmsg)
553 {
554 int i;
555 VhostUserMemory m = vmsg->payload.memory, *memory = &m;
556 dev->nregions = memory->nregions;
557
558 DPRINT("Nregions: %d\n", memory->nregions);
559 for (i = 0; i < dev->nregions; i++) {
560 void *mmap_addr;
561 VhostUserMemoryRegion *msg_region = &memory->regions[i];
562 VuDevRegion *dev_region = &dev->regions[i];
563
564 DPRINT("Region %d\n", i);
565 DPRINT(" guest_phys_addr: 0x%016"PRIx64"\n",
566 msg_region->guest_phys_addr);
567 DPRINT(" memory_size: 0x%016"PRIx64"\n",
568 msg_region->memory_size);
569 DPRINT(" userspace_addr 0x%016"PRIx64"\n",
570 msg_region->userspace_addr);
571 DPRINT(" mmap_offset 0x%016"PRIx64"\n",
572 msg_region->mmap_offset);
573
574 dev_region->gpa = msg_region->guest_phys_addr;
575 dev_region->size = msg_region->memory_size;
576 dev_region->qva = msg_region->userspace_addr;
577 dev_region->mmap_offset = msg_region->mmap_offset;
578
579 /* We don't use offset argument of mmap() since the
580 * mapped address has to be page aligned, and we use huge
581 * pages.
582 * In postcopy we're using PROT_NONE here to catch anyone
583 * accessing it before we userfault
584 */
585 mmap_addr = mmap(0, dev_region->size + dev_region->mmap_offset,
586 PROT_NONE, MAP_SHARED,
587 vmsg->fds[i], 0);
588
589 if (mmap_addr == MAP_FAILED) {
590 vu_panic(dev, "region mmap error: %s", strerror(errno));
591 } else {
592 dev_region->mmap_addr = (uint64_t)(uintptr_t)mmap_addr;
593 DPRINT(" mmap_addr: 0x%016"PRIx64"\n",
594 dev_region->mmap_addr);
595 }
596
597 /* Return the address to QEMU so that it can translate the ufd
598 * fault addresses back.
599 */
600 msg_region->userspace_addr = (uintptr_t)(mmap_addr +
601 dev_region->mmap_offset);
602 close(vmsg->fds[i]);
603 }
604
605 /* Send the message back to qemu with the addresses filled in */
606 vmsg->fd_num = 0;
607 if (!vu_send_reply(dev, dev->sock, vmsg)) {
608 vu_panic(dev, "failed to respond to set-mem-table for postcopy");
609 return false;
610 }
611
612 /* Wait for QEMU to confirm that it's registered the handler for the
613 * faults.
614 */
615 if (!vu_message_read(dev, dev->sock, vmsg) ||
616 vmsg->size != sizeof(vmsg->payload.u64) ||
617 vmsg->payload.u64 != 0) {
618 vu_panic(dev, "failed to receive valid ack for postcopy set-mem-table");
619 return false;
620 }
621
622 /* OK, now we can go and register the memory and generate faults */
623 for (i = 0; i < dev->nregions; i++) {
624 VuDevRegion *dev_region = &dev->regions[i];
625 int ret;
626 #ifdef UFFDIO_REGISTER
627 /* We should already have an open ufd. Mark each memory
628 * range as ufd.
629 * Discard any mapping we have here; note I can't use MADV_REMOVE
630 * or fallocate to make the hole since I don't want to lose
631 * data that's already arrived in the shared process.
632 * TODO: How to do hugepage
633 */
634 ret = madvise((void *)(uintptr_t)dev_region->mmap_addr,
635 dev_region->size + dev_region->mmap_offset,
636 MADV_DONTNEED);
637 if (ret) {
638 fprintf(stderr,
639 "%s: Failed to madvise(DONTNEED) region %d: %s\n",
640 __func__, i, strerror(errno));
641 }
642 /* Turn off transparent hugepages so we dont get lose wakeups
643 * in neighbouring pages.
644 * TODO: Turn this backon later.
645 */
646 ret = madvise((void *)(uintptr_t)dev_region->mmap_addr,
647 dev_region->size + dev_region->mmap_offset,
648 MADV_NOHUGEPAGE);
649 if (ret) {
650 /* Note: This can happen legally on kernels that are configured
651 * without madvise'able hugepages
652 */
653 fprintf(stderr,
654 "%s: Failed to madvise(NOHUGEPAGE) region %d: %s\n",
655 __func__, i, strerror(errno));
656 }
657 struct uffdio_register reg_struct;
658 reg_struct.range.start = (uintptr_t)dev_region->mmap_addr;
659 reg_struct.range.len = dev_region->size + dev_region->mmap_offset;
660 reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING;
661
662 if (ioctl(dev->postcopy_ufd, UFFDIO_REGISTER, &reg_struct)) {
663 vu_panic(dev, "%s: Failed to userfault region %d "
664 "@%p + size:%zx offset: %zx: (ufd=%d)%s\n",
665 __func__, i,
666 dev_region->mmap_addr,
667 dev_region->size, dev_region->mmap_offset,
668 dev->postcopy_ufd, strerror(errno));
669 return false;
670 }
671 if (!(reg_struct.ioctls & ((__u64)1 << _UFFDIO_COPY))) {
672 vu_panic(dev, "%s Region (%d) doesn't support COPY",
673 __func__, i);
674 return false;
675 }
676 DPRINT("%s: region %d: Registered userfault for %"
677 PRIx64 " + %" PRIx64 "\n", __func__, i,
678 (uint64_t)reg_struct.range.start,
679 (uint64_t)reg_struct.range.len);
680 /* Now it's registered we can let the client at it */
681 if (mprotect((void *)(uintptr_t)dev_region->mmap_addr,
682 dev_region->size + dev_region->mmap_offset,
683 PROT_READ | PROT_WRITE)) {
684 vu_panic(dev, "failed to mprotect region %d for postcopy (%s)",
685 i, strerror(errno));
686 return false;
687 }
688 /* TODO: Stash 'zero' support flags somewhere */
689 #endif
690 }
691
692 return false;
693 }
694
695 static bool
696 vu_set_mem_table_exec(VuDev *dev, VhostUserMsg *vmsg)
697 {
698 int i;
699 VhostUserMemory m = vmsg->payload.memory, *memory = &m;
700
701 for (i = 0; i < dev->nregions; i++) {
702 VuDevRegion *r = &dev->regions[i];
703 void *m = (void *) (uintptr_t) r->mmap_addr;
704
705 if (m) {
706 munmap(m, r->size + r->mmap_offset);
707 }
708 }
709 dev->nregions = memory->nregions;
710
711 if (dev->postcopy_listening) {
712 return vu_set_mem_table_exec_postcopy(dev, vmsg);
713 }
714
715 DPRINT("Nregions: %d\n", memory->nregions);
716 for (i = 0; i < dev->nregions; i++) {
717 void *mmap_addr;
718 VhostUserMemoryRegion *msg_region = &memory->regions[i];
719 VuDevRegion *dev_region = &dev->regions[i];
720
721 DPRINT("Region %d\n", i);
722 DPRINT(" guest_phys_addr: 0x%016"PRIx64"\n",
723 msg_region->guest_phys_addr);
724 DPRINT(" memory_size: 0x%016"PRIx64"\n",
725 msg_region->memory_size);
726 DPRINT(" userspace_addr 0x%016"PRIx64"\n",
727 msg_region->userspace_addr);
728 DPRINT(" mmap_offset 0x%016"PRIx64"\n",
729 msg_region->mmap_offset);
730
731 dev_region->gpa = msg_region->guest_phys_addr;
732 dev_region->size = msg_region->memory_size;
733 dev_region->qva = msg_region->userspace_addr;
734 dev_region->mmap_offset = msg_region->mmap_offset;
735
736 /* We don't use offset argument of mmap() since the
737 * mapped address has to be page aligned, and we use huge
738 * pages. */
739 mmap_addr = mmap(0, dev_region->size + dev_region->mmap_offset,
740 PROT_READ | PROT_WRITE, MAP_SHARED,
741 vmsg->fds[i], 0);
742
743 if (mmap_addr == MAP_FAILED) {
744 vu_panic(dev, "region mmap error: %s", strerror(errno));
745 } else {
746 dev_region->mmap_addr = (uint64_t)(uintptr_t)mmap_addr;
747 DPRINT(" mmap_addr: 0x%016"PRIx64"\n",
748 dev_region->mmap_addr);
749 }
750
751 close(vmsg->fds[i]);
752 }
753
754 return false;
755 }
756
757 static bool
758 vu_set_log_base_exec(VuDev *dev, VhostUserMsg *vmsg)
759 {
760 int fd;
761 uint64_t log_mmap_size, log_mmap_offset;
762 void *rc;
763
764 if (vmsg->fd_num != 1 ||
765 vmsg->size != sizeof(vmsg->payload.log)) {
766 vu_panic(dev, "Invalid log_base message");
767 return true;
768 }
769
770 fd = vmsg->fds[0];
771 log_mmap_offset = vmsg->payload.log.mmap_offset;
772 log_mmap_size = vmsg->payload.log.mmap_size;
773 DPRINT("Log mmap_offset: %"PRId64"\n", log_mmap_offset);
774 DPRINT("Log mmap_size: %"PRId64"\n", log_mmap_size);
775
776 rc = mmap(0, log_mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd,
777 log_mmap_offset);
778 close(fd);
779 if (rc == MAP_FAILED) {
780 perror("log mmap error");
781 }
782
783 if (dev->log_table) {
784 munmap(dev->log_table, dev->log_size);
785 }
786 dev->log_table = rc;
787 dev->log_size = log_mmap_size;
788
789 vmsg->size = sizeof(vmsg->payload.u64);
790 vmsg->fd_num = 0;
791
792 return true;
793 }
794
795 static bool
796 vu_set_log_fd_exec(VuDev *dev, VhostUserMsg *vmsg)
797 {
798 if (vmsg->fd_num != 1) {
799 vu_panic(dev, "Invalid log_fd message");
800 return false;
801 }
802
803 if (dev->log_call_fd != -1) {
804 close(dev->log_call_fd);
805 }
806 dev->log_call_fd = vmsg->fds[0];
807 DPRINT("Got log_call_fd: %d\n", vmsg->fds[0]);
808
809 return false;
810 }
811
812 static bool
813 vu_set_vring_num_exec(VuDev *dev, VhostUserMsg *vmsg)
814 {
815 unsigned int index = vmsg->payload.state.index;
816 unsigned int num = vmsg->payload.state.num;
817
818 DPRINT("State.index: %d\n", index);
819 DPRINT("State.num: %d\n", num);
820 dev->vq[index].vring.num = num;
821
822 return false;
823 }
824
825 static bool
826 vu_set_vring_addr_exec(VuDev *dev, VhostUserMsg *vmsg)
827 {
828 struct vhost_vring_addr addr = vmsg->payload.addr, *vra = &addr;
829 unsigned int index = vra->index;
830 VuVirtq *vq = &dev->vq[index];
831
832 DPRINT("vhost_vring_addr:\n");
833 DPRINT(" index: %d\n", vra->index);
834 DPRINT(" flags: %d\n", vra->flags);
835 DPRINT(" desc_user_addr: 0x%016" PRIx64 "\n", vra->desc_user_addr);
836 DPRINT(" used_user_addr: 0x%016" PRIx64 "\n", vra->used_user_addr);
837 DPRINT(" avail_user_addr: 0x%016" PRIx64 "\n", vra->avail_user_addr);
838 DPRINT(" log_guest_addr: 0x%016" PRIx64 "\n", vra->log_guest_addr);
839
840 vq->vring.flags = vra->flags;
841 vq->vring.desc = qva_to_va(dev, vra->desc_user_addr);
842 vq->vring.used = qva_to_va(dev, vra->used_user_addr);
843 vq->vring.avail = qva_to_va(dev, vra->avail_user_addr);
844 vq->vring.log_guest_addr = vra->log_guest_addr;
845
846 DPRINT("Setting virtq addresses:\n");
847 DPRINT(" vring_desc at %p\n", vq->vring.desc);
848 DPRINT(" vring_used at %p\n", vq->vring.used);
849 DPRINT(" vring_avail at %p\n", vq->vring.avail);
850
851 if (!(vq->vring.desc && vq->vring.used && vq->vring.avail)) {
852 vu_panic(dev, "Invalid vring_addr message");
853 return false;
854 }
855
856 vq->used_idx = vq->vring.used->idx;
857
858 if (vq->last_avail_idx != vq->used_idx) {
859 bool resume = dev->iface->queue_is_processed_in_order &&
860 dev->iface->queue_is_processed_in_order(dev, index);
861
862 DPRINT("Last avail index != used index: %u != %u%s\n",
863 vq->last_avail_idx, vq->used_idx,
864 resume ? ", resuming" : "");
865
866 if (resume) {
867 vq->shadow_avail_idx = vq->last_avail_idx = vq->used_idx;
868 }
869 }
870
871 return false;
872 }
873
874 static bool
875 vu_set_vring_base_exec(VuDev *dev, VhostUserMsg *vmsg)
876 {
877 unsigned int index = vmsg->payload.state.index;
878 unsigned int num = vmsg->payload.state.num;
879
880 DPRINT("State.index: %d\n", index);
881 DPRINT("State.num: %d\n", num);
882 dev->vq[index].shadow_avail_idx = dev->vq[index].last_avail_idx = num;
883
884 return false;
885 }
886
887 static bool
888 vu_get_vring_base_exec(VuDev *dev, VhostUserMsg *vmsg)
889 {
890 unsigned int index = vmsg->payload.state.index;
891
892 DPRINT("State.index: %d\n", index);
893 vmsg->payload.state.num = dev->vq[index].last_avail_idx;
894 vmsg->size = sizeof(vmsg->payload.state);
895
896 dev->vq[index].started = false;
897 if (dev->iface->queue_set_started) {
898 dev->iface->queue_set_started(dev, index, false);
899 }
900
901 if (dev->vq[index].call_fd != -1) {
902 close(dev->vq[index].call_fd);
903 dev->vq[index].call_fd = -1;
904 }
905 if (dev->vq[index].kick_fd != -1) {
906 dev->remove_watch(dev, dev->vq[index].kick_fd);
907 close(dev->vq[index].kick_fd);
908 dev->vq[index].kick_fd = -1;
909 }
910
911 return true;
912 }
913
914 static bool
915 vu_check_queue_msg_file(VuDev *dev, VhostUserMsg *vmsg)
916 {
917 int index = vmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
918
919 if (index >= dev->max_queues) {
920 vmsg_close_fds(vmsg);
921 vu_panic(dev, "Invalid queue index: %u", index);
922 return false;
923 }
924
925 if (vmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK ||
926 vmsg->fd_num != 1) {
927 vmsg_close_fds(vmsg);
928 vu_panic(dev, "Invalid fds in request: %d", vmsg->request);
929 return false;
930 }
931
932 return true;
933 }
934
935 static int
936 inflight_desc_compare(const void *a, const void *b)
937 {
938 VuVirtqInflightDesc *desc0 = (VuVirtqInflightDesc *)a,
939 *desc1 = (VuVirtqInflightDesc *)b;
940
941 if (desc1->counter > desc0->counter &&
942 (desc1->counter - desc0->counter) < VIRTQUEUE_MAX_SIZE * 2) {
943 return 1;
944 }
945
946 return -1;
947 }
948
949 static int
950 vu_check_queue_inflights(VuDev *dev, VuVirtq *vq)
951 {
952 int i = 0;
953
954 if (!has_feature(dev->protocol_features,
955 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
956 return 0;
957 }
958
959 if (unlikely(!vq->inflight)) {
960 return -1;
961 }
962
963 if (unlikely(!vq->inflight->version)) {
964 /* initialize the buffer */
965 vq->inflight->version = INFLIGHT_VERSION;
966 return 0;
967 }
968
969 vq->used_idx = vq->vring.used->idx;
970 vq->resubmit_num = 0;
971 vq->resubmit_list = NULL;
972 vq->counter = 0;
973
974 if (unlikely(vq->inflight->used_idx != vq->used_idx)) {
975 vq->inflight->desc[vq->inflight->last_batch_head].inflight = 0;
976
977 barrier();
978
979 vq->inflight->used_idx = vq->used_idx;
980 }
981
982 for (i = 0; i < vq->inflight->desc_num; i++) {
983 if (vq->inflight->desc[i].inflight == 1) {
984 vq->inuse++;
985 }
986 }
987
988 vq->shadow_avail_idx = vq->last_avail_idx = vq->inuse + vq->used_idx;
989
990 if (vq->inuse) {
991 vq->resubmit_list = malloc(sizeof(VuVirtqInflightDesc) * vq->inuse);
992 if (!vq->resubmit_list) {
993 return -1;
994 }
995
996 for (i = 0; i < vq->inflight->desc_num; i++) {
997 if (vq->inflight->desc[i].inflight) {
998 vq->resubmit_list[vq->resubmit_num].index = i;
999 vq->resubmit_list[vq->resubmit_num].counter =
1000 vq->inflight->desc[i].counter;
1001 vq->resubmit_num++;
1002 }
1003 }
1004
1005 if (vq->resubmit_num > 1) {
1006 qsort(vq->resubmit_list, vq->resubmit_num,
1007 sizeof(VuVirtqInflightDesc), inflight_desc_compare);
1008 }
1009 vq->counter = vq->resubmit_list[0].counter + 1;
1010 }
1011
1012 /* in case of I/O hang after reconnecting */
1013 if (eventfd_write(vq->kick_fd, 1)) {
1014 return -1;
1015 }
1016
1017 return 0;
1018 }
1019
1020 static bool
1021 vu_set_vring_kick_exec(VuDev *dev, VhostUserMsg *vmsg)
1022 {
1023 int index = vmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
1024
1025 DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
1026
1027 if (!vu_check_queue_msg_file(dev, vmsg)) {
1028 return false;
1029 }
1030
1031 if (dev->vq[index].kick_fd != -1) {
1032 dev->remove_watch(dev, dev->vq[index].kick_fd);
1033 close(dev->vq[index].kick_fd);
1034 dev->vq[index].kick_fd = -1;
1035 }
1036
1037 dev->vq[index].kick_fd = vmsg->fds[0];
1038 DPRINT("Got kick_fd: %d for vq: %d\n", vmsg->fds[0], index);
1039
1040 dev->vq[index].started = true;
1041 if (dev->iface->queue_set_started) {
1042 dev->iface->queue_set_started(dev, index, true);
1043 }
1044
1045 if (dev->vq[index].kick_fd != -1 && dev->vq[index].handler) {
1046 dev->set_watch(dev, dev->vq[index].kick_fd, VU_WATCH_IN,
1047 vu_kick_cb, (void *)(long)index);
1048
1049 DPRINT("Waiting for kicks on fd: %d for vq: %d\n",
1050 dev->vq[index].kick_fd, index);
1051 }
1052
1053 if (vu_check_queue_inflights(dev, &dev->vq[index])) {
1054 vu_panic(dev, "Failed to check inflights for vq: %d\n", index);
1055 }
1056
1057 return false;
1058 }
1059
1060 void vu_set_queue_handler(VuDev *dev, VuVirtq *vq,
1061 vu_queue_handler_cb handler)
1062 {
1063 int qidx = vq - dev->vq;
1064
1065 vq->handler = handler;
1066 if (vq->kick_fd >= 0) {
1067 if (handler) {
1068 dev->set_watch(dev, vq->kick_fd, VU_WATCH_IN,
1069 vu_kick_cb, (void *)(long)qidx);
1070 } else {
1071 dev->remove_watch(dev, vq->kick_fd);
1072 }
1073 }
1074 }
1075
1076 bool vu_set_queue_host_notifier(VuDev *dev, VuVirtq *vq, int fd,
1077 int size, int offset)
1078 {
1079 int qidx = vq - dev->vq;
1080 int fd_num = 0;
1081 VhostUserMsg vmsg = {
1082 .request = VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG,
1083 .flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
1084 .size = sizeof(vmsg.payload.area),
1085 .payload.area = {
1086 .u64 = qidx & VHOST_USER_VRING_IDX_MASK,
1087 .size = size,
1088 .offset = offset,
1089 },
1090 };
1091
1092 if (fd == -1) {
1093 vmsg.payload.area.u64 |= VHOST_USER_VRING_NOFD_MASK;
1094 } else {
1095 vmsg.fds[fd_num++] = fd;
1096 }
1097
1098 vmsg.fd_num = fd_num;
1099
1100 if ((dev->protocol_features & VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD) == 0) {
1101 return false;
1102 }
1103
1104 if (!vu_message_write(dev, dev->slave_fd, &vmsg)) {
1105 return false;
1106 }
1107
1108 return vu_process_message_reply(dev, &vmsg);
1109 }
1110
1111 static bool
1112 vu_set_vring_call_exec(VuDev *dev, VhostUserMsg *vmsg)
1113 {
1114 int index = vmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
1115
1116 DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
1117
1118 if (!vu_check_queue_msg_file(dev, vmsg)) {
1119 return false;
1120 }
1121
1122 if (dev->vq[index].call_fd != -1) {
1123 close(dev->vq[index].call_fd);
1124 dev->vq[index].call_fd = -1;
1125 }
1126
1127 dev->vq[index].call_fd = vmsg->fds[0];
1128
1129 /* in case of I/O hang after reconnecting */
1130 if (eventfd_write(vmsg->fds[0], 1)) {
1131 return -1;
1132 }
1133
1134 DPRINT("Got call_fd: %d for vq: %d\n", vmsg->fds[0], index);
1135
1136 return false;
1137 }
1138
1139 static bool
1140 vu_set_vring_err_exec(VuDev *dev, VhostUserMsg *vmsg)
1141 {
1142 int index = vmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
1143
1144 DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
1145
1146 if (!vu_check_queue_msg_file(dev, vmsg)) {
1147 return false;
1148 }
1149
1150 if (dev->vq[index].err_fd != -1) {
1151 close(dev->vq[index].err_fd);
1152 dev->vq[index].err_fd = -1;
1153 }
1154
1155 dev->vq[index].err_fd = vmsg->fds[0];
1156
1157 return false;
1158 }
1159
1160 static bool
1161 vu_get_protocol_features_exec(VuDev *dev, VhostUserMsg *vmsg)
1162 {
1163 uint64_t features = 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD |
1164 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_REQ |
1165 1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER |
1166 1ULL << VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD;
1167
1168 if (have_userfault()) {
1169 features |= 1ULL << VHOST_USER_PROTOCOL_F_PAGEFAULT;
1170 }
1171
1172 if (dev->iface->get_config && dev->iface->set_config) {
1173 features |= 1ULL << VHOST_USER_PROTOCOL_F_CONFIG;
1174 }
1175
1176 if (dev->iface->get_protocol_features) {
1177 features |= dev->iface->get_protocol_features(dev);
1178 }
1179
1180 vmsg_set_reply_u64(vmsg, features);
1181 return true;
1182 }
1183
1184 static bool
1185 vu_set_protocol_features_exec(VuDev *dev, VhostUserMsg *vmsg)
1186 {
1187 uint64_t features = vmsg->payload.u64;
1188
1189 DPRINT("u64: 0x%016"PRIx64"\n", features);
1190
1191 dev->protocol_features = vmsg->payload.u64;
1192
1193 if (dev->iface->set_protocol_features) {
1194 dev->iface->set_protocol_features(dev, features);
1195 }
1196
1197 return false;
1198 }
1199
1200 static bool
1201 vu_get_queue_num_exec(VuDev *dev, VhostUserMsg *vmsg)
1202 {
1203 DPRINT("Function %s() not implemented yet.\n", __func__);
1204 return false;
1205 }
1206
1207 static bool
1208 vu_set_vring_enable_exec(VuDev *dev, VhostUserMsg *vmsg)
1209 {
1210 unsigned int index = vmsg->payload.state.index;
1211 unsigned int enable = vmsg->payload.state.num;
1212
1213 DPRINT("State.index: %d\n", index);
1214 DPRINT("State.enable: %d\n", enable);
1215
1216 if (index >= dev->max_queues) {
1217 vu_panic(dev, "Invalid vring_enable index: %u", index);
1218 return false;
1219 }
1220
1221 dev->vq[index].enable = enable;
1222 return false;
1223 }
1224
1225 static bool
1226 vu_set_slave_req_fd(VuDev *dev, VhostUserMsg *vmsg)
1227 {
1228 if (vmsg->fd_num != 1) {
1229 vu_panic(dev, "Invalid slave_req_fd message (%d fd's)", vmsg->fd_num);
1230 return false;
1231 }
1232
1233 if (dev->slave_fd != -1) {
1234 close(dev->slave_fd);
1235 }
1236 dev->slave_fd = vmsg->fds[0];
1237 DPRINT("Got slave_fd: %d\n", vmsg->fds[0]);
1238
1239 return false;
1240 }
1241
1242 static bool
1243 vu_get_config(VuDev *dev, VhostUserMsg *vmsg)
1244 {
1245 int ret = -1;
1246
1247 if (dev->iface->get_config) {
1248 ret = dev->iface->get_config(dev, vmsg->payload.config.region,
1249 vmsg->payload.config.size);
1250 }
1251
1252 if (ret) {
1253 /* resize to zero to indicate an error to master */
1254 vmsg->size = 0;
1255 }
1256
1257 return true;
1258 }
1259
1260 static bool
1261 vu_set_config(VuDev *dev, VhostUserMsg *vmsg)
1262 {
1263 int ret = -1;
1264
1265 if (dev->iface->set_config) {
1266 ret = dev->iface->set_config(dev, vmsg->payload.config.region,
1267 vmsg->payload.config.offset,
1268 vmsg->payload.config.size,
1269 vmsg->payload.config.flags);
1270 if (ret) {
1271 vu_panic(dev, "Set virtio configuration space failed");
1272 }
1273 }
1274
1275 return false;
1276 }
1277
1278 static bool
1279 vu_set_postcopy_advise(VuDev *dev, VhostUserMsg *vmsg)
1280 {
1281 dev->postcopy_ufd = -1;
1282 #ifdef UFFDIO_API
1283 struct uffdio_api api_struct;
1284
1285 dev->postcopy_ufd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
1286 vmsg->size = 0;
1287 #endif
1288
1289 if (dev->postcopy_ufd == -1) {
1290 vu_panic(dev, "Userfaultfd not available: %s", strerror(errno));
1291 goto out;
1292 }
1293
1294 #ifdef UFFDIO_API
1295 api_struct.api = UFFD_API;
1296 api_struct.features = 0;
1297 if (ioctl(dev->postcopy_ufd, UFFDIO_API, &api_struct)) {
1298 vu_panic(dev, "Failed UFFDIO_API: %s", strerror(errno));
1299 close(dev->postcopy_ufd);
1300 dev->postcopy_ufd = -1;
1301 goto out;
1302 }
1303 /* TODO: Stash feature flags somewhere */
1304 #endif
1305
1306 out:
1307 /* Return a ufd to the QEMU */
1308 vmsg->fd_num = 1;
1309 vmsg->fds[0] = dev->postcopy_ufd;
1310 return true; /* = send a reply */
1311 }
1312
1313 static bool
1314 vu_set_postcopy_listen(VuDev *dev, VhostUserMsg *vmsg)
1315 {
1316 if (dev->nregions) {
1317 vu_panic(dev, "Regions already registered at postcopy-listen");
1318 vmsg_set_reply_u64(vmsg, -1);
1319 return true;
1320 }
1321 dev->postcopy_listening = true;
1322
1323 vmsg_set_reply_u64(vmsg, 0);
1324 return true;
1325 }
1326
1327 static bool
1328 vu_set_postcopy_end(VuDev *dev, VhostUserMsg *vmsg)
1329 {
1330 DPRINT("%s: Entry\n", __func__);
1331 dev->postcopy_listening = false;
1332 if (dev->postcopy_ufd > 0) {
1333 close(dev->postcopy_ufd);
1334 dev->postcopy_ufd = -1;
1335 DPRINT("%s: Done close\n", __func__);
1336 }
1337
1338 vmsg_set_reply_u64(vmsg, 0);
1339 DPRINT("%s: exit\n", __func__);
1340 return true;
1341 }
1342
1343 static inline uint64_t
1344 vu_inflight_queue_size(uint16_t queue_size)
1345 {
1346 return ALIGN_UP(sizeof(VuDescStateSplit) * queue_size +
1347 sizeof(uint16_t), INFLIGHT_ALIGNMENT);
1348 }
1349
1350 static bool
1351 vu_get_inflight_fd(VuDev *dev, VhostUserMsg *vmsg)
1352 {
1353 int fd;
1354 void *addr;
1355 uint64_t mmap_size;
1356 uint16_t num_queues, queue_size;
1357
1358 if (vmsg->size != sizeof(vmsg->payload.inflight)) {
1359 vu_panic(dev, "Invalid get_inflight_fd message:%d", vmsg->size);
1360 vmsg->payload.inflight.mmap_size = 0;
1361 return true;
1362 }
1363
1364 num_queues = vmsg->payload.inflight.num_queues;
1365 queue_size = vmsg->payload.inflight.queue_size;
1366
1367 DPRINT("set_inflight_fd num_queues: %"PRId16"\n", num_queues);
1368 DPRINT("set_inflight_fd queue_size: %"PRId16"\n", queue_size);
1369
1370 mmap_size = vu_inflight_queue_size(queue_size) * num_queues;
1371
1372 addr = qemu_memfd_alloc("vhost-inflight", mmap_size,
1373 F_SEAL_GROW | F_SEAL_SHRINK | F_SEAL_SEAL,
1374 &fd, NULL);
1375
1376 if (!addr) {
1377 vu_panic(dev, "Failed to alloc vhost inflight area");
1378 vmsg->payload.inflight.mmap_size = 0;
1379 return true;
1380 }
1381
1382 memset(addr, 0, mmap_size);
1383
1384 dev->inflight_info.addr = addr;
1385 dev->inflight_info.size = vmsg->payload.inflight.mmap_size = mmap_size;
1386 dev->inflight_info.fd = vmsg->fds[0] = fd;
1387 vmsg->fd_num = 1;
1388 vmsg->payload.inflight.mmap_offset = 0;
1389
1390 DPRINT("send inflight mmap_size: %"PRId64"\n",
1391 vmsg->payload.inflight.mmap_size);
1392 DPRINT("send inflight mmap offset: %"PRId64"\n",
1393 vmsg->payload.inflight.mmap_offset);
1394
1395 return true;
1396 }
1397
1398 static bool
1399 vu_set_inflight_fd(VuDev *dev, VhostUserMsg *vmsg)
1400 {
1401 int fd, i;
1402 uint64_t mmap_size, mmap_offset;
1403 uint16_t num_queues, queue_size;
1404 void *rc;
1405
1406 if (vmsg->fd_num != 1 ||
1407 vmsg->size != sizeof(vmsg->payload.inflight)) {
1408 vu_panic(dev, "Invalid set_inflight_fd message size:%d fds:%d",
1409 vmsg->size, vmsg->fd_num);
1410 return false;
1411 }
1412
1413 fd = vmsg->fds[0];
1414 mmap_size = vmsg->payload.inflight.mmap_size;
1415 mmap_offset = vmsg->payload.inflight.mmap_offset;
1416 num_queues = vmsg->payload.inflight.num_queues;
1417 queue_size = vmsg->payload.inflight.queue_size;
1418
1419 DPRINT("set_inflight_fd mmap_size: %"PRId64"\n", mmap_size);
1420 DPRINT("set_inflight_fd mmap_offset: %"PRId64"\n", mmap_offset);
1421 DPRINT("set_inflight_fd num_queues: %"PRId16"\n", num_queues);
1422 DPRINT("set_inflight_fd queue_size: %"PRId16"\n", queue_size);
1423
1424 rc = mmap(0, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
1425 fd, mmap_offset);
1426
1427 if (rc == MAP_FAILED) {
1428 vu_panic(dev, "set_inflight_fd mmap error: %s", strerror(errno));
1429 return false;
1430 }
1431
1432 if (dev->inflight_info.fd) {
1433 close(dev->inflight_info.fd);
1434 }
1435
1436 if (dev->inflight_info.addr) {
1437 munmap(dev->inflight_info.addr, dev->inflight_info.size);
1438 }
1439
1440 dev->inflight_info.fd = fd;
1441 dev->inflight_info.addr = rc;
1442 dev->inflight_info.size = mmap_size;
1443
1444 for (i = 0; i < num_queues; i++) {
1445 dev->vq[i].inflight = (VuVirtqInflight *)rc;
1446 dev->vq[i].inflight->desc_num = queue_size;
1447 rc = (void *)((char *)rc + vu_inflight_queue_size(queue_size));
1448 }
1449
1450 return false;
1451 }
1452
1453 static bool
1454 vu_process_message(VuDev *dev, VhostUserMsg *vmsg)
1455 {
1456 int do_reply = 0;
1457
1458 /* Print out generic part of the request. */
1459 DPRINT("================ Vhost user message ================\n");
1460 DPRINT("Request: %s (%d)\n", vu_request_to_string(vmsg->request),
1461 vmsg->request);
1462 DPRINT("Flags: 0x%x\n", vmsg->flags);
1463 DPRINT("Size: %d\n", vmsg->size);
1464
1465 if (vmsg->fd_num) {
1466 int i;
1467 DPRINT("Fds:");
1468 for (i = 0; i < vmsg->fd_num; i++) {
1469 DPRINT(" %d", vmsg->fds[i]);
1470 }
1471 DPRINT("\n");
1472 }
1473
1474 if (dev->iface->process_msg &&
1475 dev->iface->process_msg(dev, vmsg, &do_reply)) {
1476 return do_reply;
1477 }
1478
1479 switch (vmsg->request) {
1480 case VHOST_USER_GET_FEATURES:
1481 return vu_get_features_exec(dev, vmsg);
1482 case VHOST_USER_SET_FEATURES:
1483 return vu_set_features_exec(dev, vmsg);
1484 case VHOST_USER_GET_PROTOCOL_FEATURES:
1485 return vu_get_protocol_features_exec(dev, vmsg);
1486 case VHOST_USER_SET_PROTOCOL_FEATURES:
1487 return vu_set_protocol_features_exec(dev, vmsg);
1488 case VHOST_USER_SET_OWNER:
1489 return vu_set_owner_exec(dev, vmsg);
1490 case VHOST_USER_RESET_OWNER:
1491 return vu_reset_device_exec(dev, vmsg);
1492 case VHOST_USER_SET_MEM_TABLE:
1493 return vu_set_mem_table_exec(dev, vmsg);
1494 case VHOST_USER_SET_LOG_BASE:
1495 return vu_set_log_base_exec(dev, vmsg);
1496 case VHOST_USER_SET_LOG_FD:
1497 return vu_set_log_fd_exec(dev, vmsg);
1498 case VHOST_USER_SET_VRING_NUM:
1499 return vu_set_vring_num_exec(dev, vmsg);
1500 case VHOST_USER_SET_VRING_ADDR:
1501 return vu_set_vring_addr_exec(dev, vmsg);
1502 case VHOST_USER_SET_VRING_BASE:
1503 return vu_set_vring_base_exec(dev, vmsg);
1504 case VHOST_USER_GET_VRING_BASE:
1505 return vu_get_vring_base_exec(dev, vmsg);
1506 case VHOST_USER_SET_VRING_KICK:
1507 return vu_set_vring_kick_exec(dev, vmsg);
1508 case VHOST_USER_SET_VRING_CALL:
1509 return vu_set_vring_call_exec(dev, vmsg);
1510 case VHOST_USER_SET_VRING_ERR:
1511 return vu_set_vring_err_exec(dev, vmsg);
1512 case VHOST_USER_GET_QUEUE_NUM:
1513 return vu_get_queue_num_exec(dev, vmsg);
1514 case VHOST_USER_SET_VRING_ENABLE:
1515 return vu_set_vring_enable_exec(dev, vmsg);
1516 case VHOST_USER_SET_SLAVE_REQ_FD:
1517 return vu_set_slave_req_fd(dev, vmsg);
1518 case VHOST_USER_GET_CONFIG:
1519 return vu_get_config(dev, vmsg);
1520 case VHOST_USER_SET_CONFIG:
1521 return vu_set_config(dev, vmsg);
1522 case VHOST_USER_NONE:
1523 /* if you need processing before exit, override iface->process_msg */
1524 exit(0);
1525 case VHOST_USER_POSTCOPY_ADVISE:
1526 return vu_set_postcopy_advise(dev, vmsg);
1527 case VHOST_USER_POSTCOPY_LISTEN:
1528 return vu_set_postcopy_listen(dev, vmsg);
1529 case VHOST_USER_POSTCOPY_END:
1530 return vu_set_postcopy_end(dev, vmsg);
1531 case VHOST_USER_GET_INFLIGHT_FD:
1532 return vu_get_inflight_fd(dev, vmsg);
1533 case VHOST_USER_SET_INFLIGHT_FD:
1534 return vu_set_inflight_fd(dev, vmsg);
1535 default:
1536 vmsg_close_fds(vmsg);
1537 vu_panic(dev, "Unhandled request: %d", vmsg->request);
1538 }
1539
1540 return false;
1541 }
1542
1543 bool
1544 vu_dispatch(VuDev *dev)
1545 {
1546 VhostUserMsg vmsg = { 0, };
1547 int reply_requested;
1548 bool success = false;
1549
1550 if (!vu_message_read(dev, dev->sock, &vmsg)) {
1551 goto end;
1552 }
1553
1554 reply_requested = vu_process_message(dev, &vmsg);
1555 if (!reply_requested) {
1556 success = true;
1557 goto end;
1558 }
1559
1560 if (!vu_send_reply(dev, dev->sock, &vmsg)) {
1561 goto end;
1562 }
1563
1564 success = true;
1565
1566 end:
1567 free(vmsg.data);
1568 return success;
1569 }
1570
1571 void
1572 vu_deinit(VuDev *dev)
1573 {
1574 int i;
1575
1576 for (i = 0; i < dev->nregions; i++) {
1577 VuDevRegion *r = &dev->regions[i];
1578 void *m = (void *) (uintptr_t) r->mmap_addr;
1579 if (m != MAP_FAILED) {
1580 munmap(m, r->size + r->mmap_offset);
1581 }
1582 }
1583 dev->nregions = 0;
1584
1585 for (i = 0; i < dev->max_queues; i++) {
1586 VuVirtq *vq = &dev->vq[i];
1587
1588 if (vq->call_fd != -1) {
1589 close(vq->call_fd);
1590 vq->call_fd = -1;
1591 }
1592
1593 if (vq->kick_fd != -1) {
1594 close(vq->kick_fd);
1595 vq->kick_fd = -1;
1596 }
1597
1598 if (vq->err_fd != -1) {
1599 close(vq->err_fd);
1600 vq->err_fd = -1;
1601 }
1602
1603 if (vq->resubmit_list) {
1604 free(vq->resubmit_list);
1605 vq->resubmit_list = NULL;
1606 }
1607
1608 vq->inflight = NULL;
1609 }
1610
1611 if (dev->inflight_info.addr) {
1612 munmap(dev->inflight_info.addr, dev->inflight_info.size);
1613 dev->inflight_info.addr = NULL;
1614 }
1615
1616 if (dev->inflight_info.fd > 0) {
1617 close(dev->inflight_info.fd);
1618 dev->inflight_info.fd = -1;
1619 }
1620
1621 vu_close_log(dev);
1622 if (dev->slave_fd != -1) {
1623 close(dev->slave_fd);
1624 dev->slave_fd = -1;
1625 }
1626
1627 if (dev->sock != -1) {
1628 close(dev->sock);
1629 }
1630
1631 free(dev->vq);
1632 dev->vq = NULL;
1633 }
1634
1635 bool
1636 vu_init(VuDev *dev,
1637 uint16_t max_queues,
1638 int socket,
1639 vu_panic_cb panic,
1640 vu_set_watch_cb set_watch,
1641 vu_remove_watch_cb remove_watch,
1642 const VuDevIface *iface)
1643 {
1644 uint16_t i;
1645
1646 assert(max_queues > 0);
1647 assert(socket >= 0);
1648 assert(set_watch);
1649 assert(remove_watch);
1650 assert(iface);
1651 assert(panic);
1652
1653 memset(dev, 0, sizeof(*dev));
1654
1655 dev->sock = socket;
1656 dev->panic = panic;
1657 dev->set_watch = set_watch;
1658 dev->remove_watch = remove_watch;
1659 dev->iface = iface;
1660 dev->log_call_fd = -1;
1661 dev->slave_fd = -1;
1662 dev->max_queues = max_queues;
1663
1664 dev->vq = malloc(max_queues * sizeof(dev->vq[0]));
1665 if (!dev->vq) {
1666 DPRINT("%s: failed to malloc virtqueues\n", __func__);
1667 return false;
1668 }
1669
1670 for (i = 0; i < max_queues; i++) {
1671 dev->vq[i] = (VuVirtq) {
1672 .call_fd = -1, .kick_fd = -1, .err_fd = -1,
1673 .notification = true,
1674 };
1675 }
1676
1677 return true;
1678 }
1679
1680 VuVirtq *
1681 vu_get_queue(VuDev *dev, int qidx)
1682 {
1683 assert(qidx < dev->max_queues);
1684 return &dev->vq[qidx];
1685 }
1686
1687 bool
1688 vu_queue_enabled(VuDev *dev, VuVirtq *vq)
1689 {
1690 return vq->enable;
1691 }
1692
1693 bool
1694 vu_queue_started(const VuDev *dev, const VuVirtq *vq)
1695 {
1696 return vq->started;
1697 }
1698
1699 static inline uint16_t
1700 vring_avail_flags(VuVirtq *vq)
1701 {
1702 return vq->vring.avail->flags;
1703 }
1704
1705 static inline uint16_t
1706 vring_avail_idx(VuVirtq *vq)
1707 {
1708 vq->shadow_avail_idx = vq->vring.avail->idx;
1709
1710 return vq->shadow_avail_idx;
1711 }
1712
1713 static inline uint16_t
1714 vring_avail_ring(VuVirtq *vq, int i)
1715 {
1716 return vq->vring.avail->ring[i];
1717 }
1718
1719 static inline uint16_t
1720 vring_get_used_event(VuVirtq *vq)
1721 {
1722 return vring_avail_ring(vq, vq->vring.num);
1723 }
1724
1725 static int
1726 virtqueue_num_heads(VuDev *dev, VuVirtq *vq, unsigned int idx)
1727 {
1728 uint16_t num_heads = vring_avail_idx(vq) - idx;
1729
1730 /* Check it isn't doing very strange things with descriptor numbers. */
1731 if (num_heads > vq->vring.num) {
1732 vu_panic(dev, "Guest moved used index from %u to %u",
1733 idx, vq->shadow_avail_idx);
1734 return -1;
1735 }
1736 if (num_heads) {
1737 /* On success, callers read a descriptor at vq->last_avail_idx.
1738 * Make sure descriptor read does not bypass avail index read. */
1739 smp_rmb();
1740 }
1741
1742 return num_heads;
1743 }
1744
1745 static bool
1746 virtqueue_get_head(VuDev *dev, VuVirtq *vq,
1747 unsigned int idx, unsigned int *head)
1748 {
1749 /* Grab the next descriptor number they're advertising, and increment
1750 * the index we've seen. */
1751 *head = vring_avail_ring(vq, idx % vq->vring.num);
1752
1753 /* If their number is silly, that's a fatal mistake. */
1754 if (*head >= vq->vring.num) {
1755 vu_panic(dev, "Guest says index %u is available", head);
1756 return false;
1757 }
1758
1759 return true;
1760 }
1761
1762 static int
1763 virtqueue_read_indirect_desc(VuDev *dev, struct vring_desc *desc,
1764 uint64_t addr, size_t len)
1765 {
1766 struct vring_desc *ori_desc;
1767 uint64_t read_len;
1768
1769 if (len > (VIRTQUEUE_MAX_SIZE * sizeof(struct vring_desc))) {
1770 return -1;
1771 }
1772
1773 if (len == 0) {
1774 return -1;
1775 }
1776
1777 while (len) {
1778 read_len = len;
1779 ori_desc = vu_gpa_to_va(dev, &read_len, addr);
1780 if (!ori_desc) {
1781 return -1;
1782 }
1783
1784 memcpy(desc, ori_desc, read_len);
1785 len -= read_len;
1786 addr += read_len;
1787 desc += read_len;
1788 }
1789
1790 return 0;
1791 }
1792
1793 enum {
1794 VIRTQUEUE_READ_DESC_ERROR = -1,
1795 VIRTQUEUE_READ_DESC_DONE = 0, /* end of chain */
1796 VIRTQUEUE_READ_DESC_MORE = 1, /* more buffers in chain */
1797 };
1798
1799 static int
1800 virtqueue_read_next_desc(VuDev *dev, struct vring_desc *desc,
1801 int i, unsigned int max, unsigned int *next)
1802 {
1803 /* If this descriptor says it doesn't chain, we're done. */
1804 if (!(desc[i].flags & VRING_DESC_F_NEXT)) {
1805 return VIRTQUEUE_READ_DESC_DONE;
1806 }
1807
1808 /* Check they're not leading us off end of descriptors. */
1809 *next = desc[i].next;
1810 /* Make sure compiler knows to grab that: we don't want it changing! */
1811 smp_wmb();
1812
1813 if (*next >= max) {
1814 vu_panic(dev, "Desc next is %u", next);
1815 return VIRTQUEUE_READ_DESC_ERROR;
1816 }
1817
1818 return VIRTQUEUE_READ_DESC_MORE;
1819 }
1820
1821 void
1822 vu_queue_get_avail_bytes(VuDev *dev, VuVirtq *vq, unsigned int *in_bytes,
1823 unsigned int *out_bytes,
1824 unsigned max_in_bytes, unsigned max_out_bytes)
1825 {
1826 unsigned int idx;
1827 unsigned int total_bufs, in_total, out_total;
1828 int rc;
1829
1830 idx = vq->last_avail_idx;
1831
1832 total_bufs = in_total = out_total = 0;
1833 if (unlikely(dev->broken) ||
1834 unlikely(!vq->vring.avail)) {
1835 goto done;
1836 }
1837
1838 while ((rc = virtqueue_num_heads(dev, vq, idx)) > 0) {
1839 unsigned int max, desc_len, num_bufs, indirect = 0;
1840 uint64_t desc_addr, read_len;
1841 struct vring_desc *desc;
1842 struct vring_desc desc_buf[VIRTQUEUE_MAX_SIZE];
1843 unsigned int i;
1844
1845 max = vq->vring.num;
1846 num_bufs = total_bufs;
1847 if (!virtqueue_get_head(dev, vq, idx++, &i)) {
1848 goto err;
1849 }
1850 desc = vq->vring.desc;
1851
1852 if (desc[i].flags & VRING_DESC_F_INDIRECT) {
1853 if (desc[i].len % sizeof(struct vring_desc)) {
1854 vu_panic(dev, "Invalid size for indirect buffer table");
1855 goto err;
1856 }
1857
1858 /* If we've got too many, that implies a descriptor loop. */
1859 if (num_bufs >= max) {
1860 vu_panic(dev, "Looped descriptor");
1861 goto err;
1862 }
1863
1864 /* loop over the indirect descriptor table */
1865 indirect = 1;
1866 desc_addr = desc[i].addr;
1867 desc_len = desc[i].len;
1868 max = desc_len / sizeof(struct vring_desc);
1869 read_len = desc_len;
1870 desc = vu_gpa_to_va(dev, &read_len, desc_addr);
1871 if (unlikely(desc && read_len != desc_len)) {
1872 /* Failed to use zero copy */
1873 desc = NULL;
1874 if (!virtqueue_read_indirect_desc(dev, desc_buf,
1875 desc_addr,
1876 desc_len)) {
1877 desc = desc_buf;
1878 }
1879 }
1880 if (!desc) {
1881 vu_panic(dev, "Invalid indirect buffer table");
1882 goto err;
1883 }
1884 num_bufs = i = 0;
1885 }
1886
1887 do {
1888 /* If we've got too many, that implies a descriptor loop. */
1889 if (++num_bufs > max) {
1890 vu_panic(dev, "Looped descriptor");
1891 goto err;
1892 }
1893
1894 if (desc[i].flags & VRING_DESC_F_WRITE) {
1895 in_total += desc[i].len;
1896 } else {
1897 out_total += desc[i].len;
1898 }
1899 if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
1900 goto done;
1901 }
1902 rc = virtqueue_read_next_desc(dev, desc, i, max, &i);
1903 } while (rc == VIRTQUEUE_READ_DESC_MORE);
1904
1905 if (rc == VIRTQUEUE_READ_DESC_ERROR) {
1906 goto err;
1907 }
1908
1909 if (!indirect) {
1910 total_bufs = num_bufs;
1911 } else {
1912 total_bufs++;
1913 }
1914 }
1915 if (rc < 0) {
1916 goto err;
1917 }
1918 done:
1919 if (in_bytes) {
1920 *in_bytes = in_total;
1921 }
1922 if (out_bytes) {
1923 *out_bytes = out_total;
1924 }
1925 return;
1926
1927 err:
1928 in_total = out_total = 0;
1929 goto done;
1930 }
1931
1932 bool
1933 vu_queue_avail_bytes(VuDev *dev, VuVirtq *vq, unsigned int in_bytes,
1934 unsigned int out_bytes)
1935 {
1936 unsigned int in_total, out_total;
1937
1938 vu_queue_get_avail_bytes(dev, vq, &in_total, &out_total,
1939 in_bytes, out_bytes);
1940
1941 return in_bytes <= in_total && out_bytes <= out_total;
1942 }
1943
1944 /* Fetch avail_idx from VQ memory only when we really need to know if
1945 * guest has added some buffers. */
1946 bool
1947 vu_queue_empty(VuDev *dev, VuVirtq *vq)
1948 {
1949 if (unlikely(dev->broken) ||
1950 unlikely(!vq->vring.avail)) {
1951 return true;
1952 }
1953
1954 if (vq->shadow_avail_idx != vq->last_avail_idx) {
1955 return false;
1956 }
1957
1958 return vring_avail_idx(vq) == vq->last_avail_idx;
1959 }
1960
1961 static bool
1962 vring_notify(VuDev *dev, VuVirtq *vq)
1963 {
1964 uint16_t old, new;
1965 bool v;
1966
1967 /* We need to expose used array entries before checking used event. */
1968 smp_mb();
1969
1970 /* Always notify when queue is empty (when feature acknowledge) */
1971 if (vu_has_feature(dev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
1972 !vq->inuse && vu_queue_empty(dev, vq)) {
1973 return true;
1974 }
1975
1976 if (!vu_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
1977 return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT);
1978 }
1979
1980 v = vq->signalled_used_valid;
1981 vq->signalled_used_valid = true;
1982 old = vq->signalled_used;
1983 new = vq->signalled_used = vq->used_idx;
1984 return !v || vring_need_event(vring_get_used_event(vq), new, old);
1985 }
1986
1987 void
1988 vu_queue_notify(VuDev *dev, VuVirtq *vq)
1989 {
1990 if (unlikely(dev->broken) ||
1991 unlikely(!vq->vring.avail)) {
1992 return;
1993 }
1994
1995 if (!vring_notify(dev, vq)) {
1996 DPRINT("skipped notify...\n");
1997 return;
1998 }
1999
2000 if (eventfd_write(vq->call_fd, 1) < 0) {
2001 vu_panic(dev, "Error writing eventfd: %s", strerror(errno));
2002 }
2003 }
2004
2005 static inline void
2006 vring_used_flags_set_bit(VuVirtq *vq, int mask)
2007 {
2008 uint16_t *flags;
2009
2010 flags = (uint16_t *)((char*)vq->vring.used +
2011 offsetof(struct vring_used, flags));
2012 *flags |= mask;
2013 }
2014
2015 static inline void
2016 vring_used_flags_unset_bit(VuVirtq *vq, int mask)
2017 {
2018 uint16_t *flags;
2019
2020 flags = (uint16_t *)((char*)vq->vring.used +
2021 offsetof(struct vring_used, flags));
2022 *flags &= ~mask;
2023 }
2024
2025 static inline void
2026 vring_set_avail_event(VuVirtq *vq, uint16_t val)
2027 {
2028 if (!vq->notification) {
2029 return;
2030 }
2031
2032 *((uint16_t *) &vq->vring.used->ring[vq->vring.num]) = val;
2033 }
2034
2035 void
2036 vu_queue_set_notification(VuDev *dev, VuVirtq *vq, int enable)
2037 {
2038 vq->notification = enable;
2039 if (vu_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
2040 vring_set_avail_event(vq, vring_avail_idx(vq));
2041 } else if (enable) {
2042 vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY);
2043 } else {
2044 vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY);
2045 }
2046 if (enable) {
2047 /* Expose avail event/used flags before caller checks the avail idx. */
2048 smp_mb();
2049 }
2050 }
2051
2052 static void
2053 virtqueue_map_desc(VuDev *dev,
2054 unsigned int *p_num_sg, struct iovec *iov,
2055 unsigned int max_num_sg, bool is_write,
2056 uint64_t pa, size_t sz)
2057 {
2058 unsigned num_sg = *p_num_sg;
2059
2060 assert(num_sg <= max_num_sg);
2061
2062 if (!sz) {
2063 vu_panic(dev, "virtio: zero sized buffers are not allowed");
2064 return;
2065 }
2066
2067 while (sz) {
2068 uint64_t len = sz;
2069
2070 if (num_sg == max_num_sg) {
2071 vu_panic(dev, "virtio: too many descriptors in indirect table");
2072 return;
2073 }
2074
2075 iov[num_sg].iov_base = vu_gpa_to_va(dev, &len, pa);
2076 if (iov[num_sg].iov_base == NULL) {
2077 vu_panic(dev, "virtio: invalid address for buffers");
2078 return;
2079 }
2080 iov[num_sg].iov_len = len;
2081 num_sg++;
2082 sz -= len;
2083 pa += len;
2084 }
2085
2086 *p_num_sg = num_sg;
2087 }
2088
2089 static void *
2090 virtqueue_alloc_element(size_t sz,
2091 unsigned out_num, unsigned in_num)
2092 {
2093 VuVirtqElement *elem;
2094 size_t in_sg_ofs = ALIGN_UP(sz, __alignof__(elem->in_sg[0]));
2095 size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]);
2096 size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]);
2097
2098 assert(sz >= sizeof(VuVirtqElement));
2099 elem = malloc(out_sg_end);
2100 elem->out_num = out_num;
2101 elem->in_num = in_num;
2102 elem->in_sg = (void *)elem + in_sg_ofs;
2103 elem->out_sg = (void *)elem + out_sg_ofs;
2104 return elem;
2105 }
2106
2107 static void *
2108 vu_queue_map_desc(VuDev *dev, VuVirtq *vq, unsigned int idx, size_t sz)
2109 {
2110 struct vring_desc *desc = vq->vring.desc;
2111 uint64_t desc_addr, read_len;
2112 unsigned int desc_len;
2113 unsigned int max = vq->vring.num;
2114 unsigned int i = idx;
2115 VuVirtqElement *elem;
2116 unsigned int out_num = 0, in_num = 0;
2117 struct iovec iov[VIRTQUEUE_MAX_SIZE];
2118 struct vring_desc desc_buf[VIRTQUEUE_MAX_SIZE];
2119 int rc;
2120
2121 if (desc[i].flags & VRING_DESC_F_INDIRECT) {
2122 if (desc[i].len % sizeof(struct vring_desc)) {
2123 vu_panic(dev, "Invalid size for indirect buffer table");
2124 }
2125
2126 /* loop over the indirect descriptor table */
2127 desc_addr = desc[i].addr;
2128 desc_len = desc[i].len;
2129 max = desc_len / sizeof(struct vring_desc);
2130 read_len = desc_len;
2131 desc = vu_gpa_to_va(dev, &read_len, desc_addr);
2132 if (unlikely(desc && read_len != desc_len)) {
2133 /* Failed to use zero copy */
2134 desc = NULL;
2135 if (!virtqueue_read_indirect_desc(dev, desc_buf,
2136 desc_addr,
2137 desc_len)) {
2138 desc = desc_buf;
2139 }
2140 }
2141 if (!desc) {
2142 vu_panic(dev, "Invalid indirect buffer table");
2143 return NULL;
2144 }
2145 i = 0;
2146 }
2147
2148 /* Collect all the descriptors */
2149 do {
2150 if (desc[i].flags & VRING_DESC_F_WRITE) {
2151 virtqueue_map_desc(dev, &in_num, iov + out_num,
2152 VIRTQUEUE_MAX_SIZE - out_num, true,
2153 desc[i].addr, desc[i].len);
2154 } else {
2155 if (in_num) {
2156 vu_panic(dev, "Incorrect order for descriptors");
2157 return NULL;
2158 }
2159 virtqueue_map_desc(dev, &out_num, iov,
2160 VIRTQUEUE_MAX_SIZE, false,
2161 desc[i].addr, desc[i].len);
2162 }
2163
2164 /* If we've got too many, that implies a descriptor loop. */
2165 if ((in_num + out_num) > max) {
2166 vu_panic(dev, "Looped descriptor");
2167 }
2168 rc = virtqueue_read_next_desc(dev, desc, i, max, &i);
2169 } while (rc == VIRTQUEUE_READ_DESC_MORE);
2170
2171 if (rc == VIRTQUEUE_READ_DESC_ERROR) {
2172 vu_panic(dev, "read descriptor error");
2173 return NULL;
2174 }
2175
2176 /* Now copy what we have collected and mapped */
2177 elem = virtqueue_alloc_element(sz, out_num, in_num);
2178 elem->index = idx;
2179 for (i = 0; i < out_num; i++) {
2180 elem->out_sg[i] = iov[i];
2181 }
2182 for (i = 0; i < in_num; i++) {
2183 elem->in_sg[i] = iov[out_num + i];
2184 }
2185
2186 return elem;
2187 }
2188
2189 static int
2190 vu_queue_inflight_get(VuDev *dev, VuVirtq *vq, int desc_idx)
2191 {
2192 if (!has_feature(dev->protocol_features,
2193 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
2194 return 0;
2195 }
2196
2197 if (unlikely(!vq->inflight)) {
2198 return -1;
2199 }
2200
2201 vq->inflight->desc[desc_idx].counter = vq->counter++;
2202 vq->inflight->desc[desc_idx].inflight = 1;
2203
2204 return 0;
2205 }
2206
2207 static int
2208 vu_queue_inflight_pre_put(VuDev *dev, VuVirtq *vq, int desc_idx)
2209 {
2210 if (!has_feature(dev->protocol_features,
2211 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
2212 return 0;
2213 }
2214
2215 if (unlikely(!vq->inflight)) {
2216 return -1;
2217 }
2218
2219 vq->inflight->last_batch_head = desc_idx;
2220
2221 return 0;
2222 }
2223
2224 static int
2225 vu_queue_inflight_post_put(VuDev *dev, VuVirtq *vq, int desc_idx)
2226 {
2227 if (!has_feature(dev->protocol_features,
2228 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
2229 return 0;
2230 }
2231
2232 if (unlikely(!vq->inflight)) {
2233 return -1;
2234 }
2235
2236 barrier();
2237
2238 vq->inflight->desc[desc_idx].inflight = 0;
2239
2240 barrier();
2241
2242 vq->inflight->used_idx = vq->used_idx;
2243
2244 return 0;
2245 }
2246
2247 void *
2248 vu_queue_pop(VuDev *dev, VuVirtq *vq, size_t sz)
2249 {
2250 int i;
2251 unsigned int head;
2252 VuVirtqElement *elem;
2253
2254 if (unlikely(dev->broken) ||
2255 unlikely(!vq->vring.avail)) {
2256 return NULL;
2257 }
2258
2259 if (unlikely(vq->resubmit_list && vq->resubmit_num > 0)) {
2260 i = (--vq->resubmit_num);
2261 elem = vu_queue_map_desc(dev, vq, vq->resubmit_list[i].index, sz);
2262
2263 if (!vq->resubmit_num) {
2264 free(vq->resubmit_list);
2265 vq->resubmit_list = NULL;
2266 }
2267
2268 return elem;
2269 }
2270
2271 if (vu_queue_empty(dev, vq)) {
2272 return NULL;
2273 }
2274 /*
2275 * Needed after virtio_queue_empty(), see comment in
2276 * virtqueue_num_heads().
2277 */
2278 smp_rmb();
2279
2280 if (vq->inuse >= vq->vring.num) {
2281 vu_panic(dev, "Virtqueue size exceeded");
2282 return NULL;
2283 }
2284
2285 if (!virtqueue_get_head(dev, vq, vq->last_avail_idx++, &head)) {
2286 return NULL;
2287 }
2288
2289 if (vu_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
2290 vring_set_avail_event(vq, vq->last_avail_idx);
2291 }
2292
2293 elem = vu_queue_map_desc(dev, vq, head, sz);
2294
2295 if (!elem) {
2296 return NULL;
2297 }
2298
2299 vq->inuse++;
2300
2301 vu_queue_inflight_get(dev, vq, head);
2302
2303 return elem;
2304 }
2305
2306 static void
2307 vu_queue_detach_element(VuDev *dev, VuVirtq *vq, VuVirtqElement *elem,
2308 size_t len)
2309 {
2310 vq->inuse--;
2311 /* unmap, when DMA support is added */
2312 }
2313
2314 void
2315 vu_queue_unpop(VuDev *dev, VuVirtq *vq, VuVirtqElement *elem,
2316 size_t len)
2317 {
2318 vq->last_avail_idx--;
2319 vu_queue_detach_element(dev, vq, elem, len);
2320 }
2321
2322 bool
2323 vu_queue_rewind(VuDev *dev, VuVirtq *vq, unsigned int num)
2324 {
2325 if (num > vq->inuse) {
2326 return false;
2327 }
2328 vq->last_avail_idx -= num;
2329 vq->inuse -= num;
2330 return true;
2331 }
2332
2333 static inline
2334 void vring_used_write(VuDev *dev, VuVirtq *vq,
2335 struct vring_used_elem *uelem, int i)
2336 {
2337 struct vring_used *used = vq->vring.used;
2338
2339 used->ring[i] = *uelem;
2340 vu_log_write(dev, vq->vring.log_guest_addr +
2341 offsetof(struct vring_used, ring[i]),
2342 sizeof(used->ring[i]));
2343 }
2344
2345
2346 static void
2347 vu_log_queue_fill(VuDev *dev, VuVirtq *vq,
2348 const VuVirtqElement *elem,
2349 unsigned int len)
2350 {
2351 struct vring_desc *desc = vq->vring.desc;
2352 unsigned int i, max, min, desc_len;
2353 uint64_t desc_addr, read_len;
2354 struct vring_desc desc_buf[VIRTQUEUE_MAX_SIZE];
2355 unsigned num_bufs = 0;
2356
2357 max = vq->vring.num;
2358 i = elem->index;
2359
2360 if (desc[i].flags & VRING_DESC_F_INDIRECT) {
2361 if (desc[i].len % sizeof(struct vring_desc)) {
2362 vu_panic(dev, "Invalid size for indirect buffer table");
2363 }
2364
2365 /* loop over the indirect descriptor table */
2366 desc_addr = desc[i].addr;
2367 desc_len = desc[i].len;
2368 max = desc_len / sizeof(struct vring_desc);
2369 read_len = desc_len;
2370 desc = vu_gpa_to_va(dev, &read_len, desc_addr);
2371 if (unlikely(desc && read_len != desc_len)) {
2372 /* Failed to use zero copy */
2373 desc = NULL;
2374 if (!virtqueue_read_indirect_desc(dev, desc_buf,
2375 desc_addr,
2376 desc_len)) {
2377 desc = desc_buf;
2378 }
2379 }
2380 if (!desc) {
2381 vu_panic(dev, "Invalid indirect buffer table");
2382 return;
2383 }
2384 i = 0;
2385 }
2386
2387 do {
2388 if (++num_bufs > max) {
2389 vu_panic(dev, "Looped descriptor");
2390 return;
2391 }
2392
2393 if (desc[i].flags & VRING_DESC_F_WRITE) {
2394 min = MIN(desc[i].len, len);
2395 vu_log_write(dev, desc[i].addr, min);
2396 len -= min;
2397 }
2398
2399 } while (len > 0 &&
2400 (virtqueue_read_next_desc(dev, desc, i, max, &i)
2401 == VIRTQUEUE_READ_DESC_MORE));
2402 }
2403
2404 void
2405 vu_queue_fill(VuDev *dev, VuVirtq *vq,
2406 const VuVirtqElement *elem,
2407 unsigned int len, unsigned int idx)
2408 {
2409 struct vring_used_elem uelem;
2410
2411 if (unlikely(dev->broken) ||
2412 unlikely(!vq->vring.avail)) {
2413 return;
2414 }
2415
2416 vu_log_queue_fill(dev, vq, elem, len);
2417
2418 idx = (idx + vq->used_idx) % vq->vring.num;
2419
2420 uelem.id = elem->index;
2421 uelem.len = len;
2422 vring_used_write(dev, vq, &uelem, idx);
2423 }
2424
2425 static inline
2426 void vring_used_idx_set(VuDev *dev, VuVirtq *vq, uint16_t val)
2427 {
2428 vq->vring.used->idx = val;
2429 vu_log_write(dev,
2430 vq->vring.log_guest_addr + offsetof(struct vring_used, idx),
2431 sizeof(vq->vring.used->idx));
2432
2433 vq->used_idx = val;
2434 }
2435
2436 void
2437 vu_queue_flush(VuDev *dev, VuVirtq *vq, unsigned int count)
2438 {
2439 uint16_t old, new;
2440
2441 if (unlikely(dev->broken) ||
2442 unlikely(!vq->vring.avail)) {
2443 return;
2444 }
2445
2446 /* Make sure buffer is written before we update index. */
2447 smp_wmb();
2448
2449 old = vq->used_idx;
2450 new = old + count;
2451 vring_used_idx_set(dev, vq, new);
2452 vq->inuse -= count;
2453 if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old))) {
2454 vq->signalled_used_valid = false;
2455 }
2456 }
2457
2458 void
2459 vu_queue_push(VuDev *dev, VuVirtq *vq,
2460 const VuVirtqElement *elem, unsigned int len)
2461 {
2462 vu_queue_fill(dev, vq, elem, len, 0);
2463 vu_queue_inflight_pre_put(dev, vq, elem->index);
2464 vu_queue_flush(dev, vq, 1);
2465 vu_queue_inflight_post_put(dev, vq, elem->index);
2466 }
AR7 emulation for QEMU