?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
libvhost-user: Update and fix feature and request lists
[qemu.git] / contrib / libvhost-user / libvhost-user.c
blobe36aed72d0bb2fa6463ee255230c095031117d75
1 /*
2 * Vhost User library
3 *
4 * Copyright IBM, Corp. 2007
5 * Copyright (c) 2016 Red Hat, Inc.
6 *
7 * Authors:
8 * Anthony Liguori <aliguori@us.ibm.com>
9 * Marc-André Lureau <mlureau@redhat.com>
10 * Victor Kaplansky <victork@redhat.com>
11 *
12 * This work is licensed under the terms of the GNU GPL, version 2 or
13 * later. See the COPYING file in the top-level directory.
14 */
15
16 /* this code avoids GLib dependency */
17 #include <stdlib.h>
18 #include <stdio.h>
19 #include <unistd.h>
20 #include <stdarg.h>
21 #include <errno.h>
22 #include <string.h>
23 #include <assert.h>
24 #include <inttypes.h>
25 #include <sys/types.h>
26 #include <sys/socket.h>
27 #include <sys/eventfd.h>
28 #include <sys/mman.h>
29 #include <linux/vhost.h>
30
31 #include "qemu/compiler.h"
32 #include "qemu/atomic.h"
33
34 #include "libvhost-user.h"
35
36 /* usually provided by GLib */
37 #ifndef MIN
38 #define MIN(x, y) ({ \
39 typeof(x) _min1 = (x); \
40 typeof(y) _min2 = (y); \
41 (void) (&_min1 == &_min2); \
42 _min1 < _min2 ? _min1 : _min2; })
43 #endif
44
45 #define VHOST_USER_HDR_SIZE offsetof(VhostUserMsg, payload.u64)
46
47 /* The version of the protocol we support */
48 #define VHOST_USER_VERSION 1
49 #define LIBVHOST_USER_DEBUG 0
50
51 #define DPRINT(...) \
52 do { \
53 if (LIBVHOST_USER_DEBUG) { \
54 fprintf(stderr, __VA_ARGS__); \
55 } \
56 } while (0)
57
58 static const char *
59 vu_request_to_string(unsigned int req)
60 {
61 #define REQ(req) [req] = #req
62 static const char *vu_request_str[] = {
63 REQ(VHOST_USER_NONE),
64 REQ(VHOST_USER_GET_FEATURES),
65 REQ(VHOST_USER_SET_FEATURES),
66 REQ(VHOST_USER_SET_OWNER),
67 REQ(VHOST_USER_RESET_OWNER),
68 REQ(VHOST_USER_SET_MEM_TABLE),
69 REQ(VHOST_USER_SET_LOG_BASE),
70 REQ(VHOST_USER_SET_LOG_FD),
71 REQ(VHOST_USER_SET_VRING_NUM),
72 REQ(VHOST_USER_SET_VRING_ADDR),
73 REQ(VHOST_USER_SET_VRING_BASE),
74 REQ(VHOST_USER_GET_VRING_BASE),
75 REQ(VHOST_USER_SET_VRING_KICK),
76 REQ(VHOST_USER_SET_VRING_CALL),
77 REQ(VHOST_USER_SET_VRING_ERR),
78 REQ(VHOST_USER_GET_PROTOCOL_FEATURES),
79 REQ(VHOST_USER_SET_PROTOCOL_FEATURES),
80 REQ(VHOST_USER_GET_QUEUE_NUM),
81 REQ(VHOST_USER_SET_VRING_ENABLE),
82 REQ(VHOST_USER_SEND_RARP),
83 REQ(VHOST_USER_NET_SET_MTU),
84 REQ(VHOST_USER_SET_SLAVE_REQ_FD),
85 REQ(VHOST_USER_IOTLB_MSG),
86 REQ(VHOST_USER_SET_VRING_ENDIAN),
87 REQ(VHOST_USER_MAX),
88 };
89 #undef REQ
90
91 if (req < VHOST_USER_MAX) {
92 return vu_request_str[req];
93 } else {
94 return "unknown";
95 }
96 }
97
98 static void
99 vu_panic(VuDev *dev, const char *msg, ...)
100 {
101 char *buf = NULL;
102 va_list ap;
103
104 va_start(ap, msg);
105 if (vasprintf(&buf, msg, ap) < 0) {
106 buf = NULL;
107 }
108 va_end(ap);
109
110 dev->broken = true;
111 dev->panic(dev, buf);
112 free(buf);
113
114 /* FIXME: find a way to call virtio_error? */
115 }
116
117 /* Translate guest physical address to our virtual address. */
118 void *
119 vu_gpa_to_va(VuDev *dev, uint64_t guest_addr)
120 {
121 int i;
122
123 /* Find matching memory region. */
124 for (i = 0; i < dev->nregions; i++) {
125 VuDevRegion *r = &dev->regions[i];
126
127 if ((guest_addr >= r->gpa) && (guest_addr < (r->gpa + r->size))) {
128 return (void *)(uintptr_t)
129 guest_addr - r->gpa + r->mmap_addr + r->mmap_offset;
130 }
131 }
132
133 return NULL;
134 }
135
136 /* Translate qemu virtual address to our virtual address. */
137 static void *
138 qva_to_va(VuDev *dev, uint64_t qemu_addr)
139 {
140 int i;
141
142 /* Find matching memory region. */
143 for (i = 0; i < dev->nregions; i++) {
144 VuDevRegion *r = &dev->regions[i];
145
146 if ((qemu_addr >= r->qva) && (qemu_addr < (r->qva + r->size))) {
147 return (void *)(uintptr_t)
148 qemu_addr - r->qva + r->mmap_addr + r->mmap_offset;
149 }
150 }
151
152 return NULL;
153 }
154
155 static void
156 vmsg_close_fds(VhostUserMsg *vmsg)
157 {
158 int i;
159
160 for (i = 0; i < vmsg->fd_num; i++) {
161 close(vmsg->fds[i]);
162 }
163 }
164
165 static bool
166 vu_message_read(VuDev *dev, int conn_fd, VhostUserMsg *vmsg)
167 {
168 char control[CMSG_SPACE(VHOST_MEMORY_MAX_NREGIONS * sizeof(int))] = { };
169 struct iovec iov = {
170 .iov_base = (char *)vmsg,
171 .iov_len = VHOST_USER_HDR_SIZE,
172 };
173 struct msghdr msg = {
174 .msg_iov = &iov,
175 .msg_iovlen = 1,
176 .msg_control = control,
177 .msg_controllen = sizeof(control),
178 };
179 size_t fd_size;
180 struct cmsghdr *cmsg;
181 int rc;
182
183 do {
184 rc = recvmsg(conn_fd, &msg, 0);
185 } while (rc < 0 && (errno == EINTR || errno == EAGAIN));
186
187 if (rc < 0) {
188 vu_panic(dev, "Error while recvmsg: %s", strerror(errno));
189 return false;
190 }
191
192 vmsg->fd_num = 0;
193 for (cmsg = CMSG_FIRSTHDR(&msg);
194 cmsg != NULL;
195 cmsg = CMSG_NXTHDR(&msg, cmsg))
196 {
197 if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS) {
198 fd_size = cmsg->cmsg_len - CMSG_LEN(0);
199 vmsg->fd_num = fd_size / sizeof(int);
200 memcpy(vmsg->fds, CMSG_DATA(cmsg), fd_size);
201 break;
202 }
203 }
204
205 if (vmsg->size > sizeof(vmsg->payload)) {
206 vu_panic(dev,
207 "Error: too big message request: %d, size: vmsg->size: %u, "
208 "while sizeof(vmsg->payload) = %zu\n",
209 vmsg->request, vmsg->size, sizeof(vmsg->payload));
210 goto fail;
211 }
212
213 if (vmsg->size) {
214 do {
215 rc = read(conn_fd, &vmsg->payload, vmsg->size);
216 } while (rc < 0 && (errno == EINTR || errno == EAGAIN));
217
218 if (rc <= 0) {
219 vu_panic(dev, "Error while reading: %s", strerror(errno));
220 goto fail;
221 }
222
223 assert(rc == vmsg->size);
224 }
225
226 return true;
227
228 fail:
229 vmsg_close_fds(vmsg);
230
231 return false;
232 }
233
234 static bool
235 vu_message_write(VuDev *dev, int conn_fd, VhostUserMsg *vmsg)
236 {
237 int rc;
238 uint8_t *p = (uint8_t *)vmsg;
239
240 /* Set the version in the flags when sending the reply */
241 vmsg->flags &= ~VHOST_USER_VERSION_MASK;
242 vmsg->flags |= VHOST_USER_VERSION;
243 vmsg->flags |= VHOST_USER_REPLY_MASK;
244
245 do {
246 rc = write(conn_fd, p, VHOST_USER_HDR_SIZE);
247 } while (rc < 0 && (errno == EINTR || errno == EAGAIN));
248
249 do {
250 if (vmsg->data) {
251 rc = write(conn_fd, vmsg->data, vmsg->size);
252 } else {
253 rc = write(conn_fd, p + VHOST_USER_HDR_SIZE, vmsg->size);
254 }
255 } while (rc < 0 && (errno == EINTR || errno == EAGAIN));
256
257 if (rc <= 0) {
258 vu_panic(dev, "Error while writing: %s", strerror(errno));
259 return false;
260 }
261
262 return true;
263 }
264
265 /* Kick the log_call_fd if required. */
266 static void
267 vu_log_kick(VuDev *dev)
268 {
269 if (dev->log_call_fd != -1) {
270 DPRINT("Kicking the QEMU's log...\n");
271 if (eventfd_write(dev->log_call_fd, 1) < 0) {
272 vu_panic(dev, "Error writing eventfd: %s", strerror(errno));
273 }
274 }
275 }
276
277 static void
278 vu_log_page(uint8_t *log_table, uint64_t page)
279 {
280 DPRINT("Logged dirty guest page: %"PRId64"\n", page);
281 atomic_or(&log_table[page / 8], 1 << (page % 8));
282 }
283
284 static void
285 vu_log_write(VuDev *dev, uint64_t address, uint64_t length)
286 {
287 uint64_t page;
288
289 if (!(dev->features & (1ULL << VHOST_F_LOG_ALL)) ||
290 !dev->log_table || !length) {
291 return;
292 }
293
294 assert(dev->log_size > ((address + length - 1) / VHOST_LOG_PAGE / 8));
295
296 page = address / VHOST_LOG_PAGE;
297 while (page * VHOST_LOG_PAGE < address + length) {
298 vu_log_page(dev->log_table, page);
299 page += VHOST_LOG_PAGE;
300 }
301
302 vu_log_kick(dev);
303 }
304
305 static void
306 vu_kick_cb(VuDev *dev, int condition, void *data)
307 {
308 int index = (intptr_t)data;
309 VuVirtq *vq = &dev->vq[index];
310 int sock = vq->kick_fd;
311 eventfd_t kick_data;
312 ssize_t rc;
313
314 rc = eventfd_read(sock, &kick_data);
315 if (rc == -1) {
316 vu_panic(dev, "kick eventfd_read(): %s", strerror(errno));
317 dev->remove_watch(dev, dev->vq[index].kick_fd);
318 } else {
319 DPRINT("Got kick_data: %016"PRIx64" handler:%p idx:%d\n",
320 kick_data, vq->handler, index);
321 if (vq->handler) {
322 vq->handler(dev, index);
323 }
324 }
325 }
326
327 static bool
328 vu_get_features_exec(VuDev *dev, VhostUserMsg *vmsg)
329 {
330 vmsg->payload.u64 =
331 1ULL << VHOST_F_LOG_ALL |
332 1ULL << VHOST_USER_F_PROTOCOL_FEATURES;
333
334 if (dev->iface->get_features) {
335 vmsg->payload.u64 |= dev->iface->get_features(dev);
336 }
337
338 vmsg->size = sizeof(vmsg->payload.u64);
339
340 DPRINT("Sending back to guest u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
341
342 return true;
343 }
344
345 static void
346 vu_set_enable_all_rings(VuDev *dev, bool enabled)
347 {
348 int i;
349
350 for (i = 0; i < VHOST_MAX_NR_VIRTQUEUE; i++) {
351 dev->vq[i].enable = enabled;
352 }
353 }
354
355 static bool
356 vu_set_features_exec(VuDev *dev, VhostUserMsg *vmsg)
357 {
358 DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
359
360 dev->features = vmsg->payload.u64;
361
362 if (!(dev->features & VHOST_USER_F_PROTOCOL_FEATURES)) {
363 vu_set_enable_all_rings(dev, true);
364 }
365
366 if (dev->iface->set_features) {
367 dev->iface->set_features(dev, dev->features);
368 }
369
370 return false;
371 }
372
373 static bool
374 vu_set_owner_exec(VuDev *dev, VhostUserMsg *vmsg)
375 {
376 return false;
377 }
378
379 static void
380 vu_close_log(VuDev *dev)
381 {
382 if (dev->log_table) {
383 if (munmap(dev->log_table, dev->log_size) != 0) {
384 perror("close log munmap() error");
385 }
386
387 dev->log_table = NULL;
388 }
389 if (dev->log_call_fd != -1) {
390 close(dev->log_call_fd);
391 dev->log_call_fd = -1;
392 }
393 }
394
395 static bool
396 vu_reset_device_exec(VuDev *dev, VhostUserMsg *vmsg)
397 {
398 vu_set_enable_all_rings(dev, false);
399
400 return false;
401 }
402
403 static bool
404 vu_set_mem_table_exec(VuDev *dev, VhostUserMsg *vmsg)
405 {
406 int i;
407 VhostUserMemory *memory = &vmsg->payload.memory;
408 dev->nregions = memory->nregions;
409
410 DPRINT("Nregions: %d\n", memory->nregions);
411 for (i = 0; i < dev->nregions; i++) {
412 void *mmap_addr;
413 VhostUserMemoryRegion *msg_region = &memory->regions[i];
414 VuDevRegion *dev_region = &dev->regions[i];
415
416 DPRINT("Region %d\n", i);
417 DPRINT(" guest_phys_addr: 0x%016"PRIx64"\n",
418 msg_region->guest_phys_addr);
419 DPRINT(" memory_size: 0x%016"PRIx64"\n",
420 msg_region->memory_size);
421 DPRINT(" userspace_addr 0x%016"PRIx64"\n",
422 msg_region->userspace_addr);
423 DPRINT(" mmap_offset 0x%016"PRIx64"\n",
424 msg_region->mmap_offset);
425
426 dev_region->gpa = msg_region->guest_phys_addr;
427 dev_region->size = msg_region->memory_size;
428 dev_region->qva = msg_region->userspace_addr;
429 dev_region->mmap_offset = msg_region->mmap_offset;
430
431 /* We don't use offset argument of mmap() since the
432 * mapped address has to be page aligned, and we use huge
433 * pages. */
434 mmap_addr = mmap(0, dev_region->size + dev_region->mmap_offset,
435 PROT_READ | PROT_WRITE, MAP_SHARED,
436 vmsg->fds[i], 0);
437
438 if (mmap_addr == MAP_FAILED) {
439 vu_panic(dev, "region mmap error: %s", strerror(errno));
440 } else {
441 dev_region->mmap_addr = (uint64_t)(uintptr_t)mmap_addr;
442 DPRINT(" mmap_addr: 0x%016"PRIx64"\n",
443 dev_region->mmap_addr);
444 }
445
446 close(vmsg->fds[i]);
447 }
448
449 return false;
450 }
451
452 static bool
453 vu_set_log_base_exec(VuDev *dev, VhostUserMsg *vmsg)
454 {
455 int fd;
456 uint64_t log_mmap_size, log_mmap_offset;
457 void *rc;
458
459 if (vmsg->fd_num != 1 ||
460 vmsg->size != sizeof(vmsg->payload.log)) {
461 vu_panic(dev, "Invalid log_base message");
462 return true;
463 }
464
465 fd = vmsg->fds[0];
466 log_mmap_offset = vmsg->payload.log.mmap_offset;
467 log_mmap_size = vmsg->payload.log.mmap_size;
468 DPRINT("Log mmap_offset: %"PRId64"\n", log_mmap_offset);
469 DPRINT("Log mmap_size: %"PRId64"\n", log_mmap_size);
470
471 rc = mmap(0, log_mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd,
472 log_mmap_offset);
473 if (rc == MAP_FAILED) {
474 perror("log mmap error");
475 }
476 dev->log_table = rc;
477 dev->log_size = log_mmap_size;
478
479 vmsg->size = sizeof(vmsg->payload.u64);
480
481 return true;
482 }
483
484 static bool
485 vu_set_log_fd_exec(VuDev *dev, VhostUserMsg *vmsg)
486 {
487 if (vmsg->fd_num != 1) {
488 vu_panic(dev, "Invalid log_fd message");
489 return false;
490 }
491
492 if (dev->log_call_fd != -1) {
493 close(dev->log_call_fd);
494 }
495 dev->log_call_fd = vmsg->fds[0];
496 DPRINT("Got log_call_fd: %d\n", vmsg->fds[0]);
497
498 return false;
499 }
500
501 static bool
502 vu_set_vring_num_exec(VuDev *dev, VhostUserMsg *vmsg)
503 {
504 unsigned int index = vmsg->payload.state.index;
505 unsigned int num = vmsg->payload.state.num;
506
507 DPRINT("State.index: %d\n", index);
508 DPRINT("State.num: %d\n", num);
509 dev->vq[index].vring.num = num;
510
511 return false;
512 }
513
514 static bool
515 vu_set_vring_addr_exec(VuDev *dev, VhostUserMsg *vmsg)
516 {
517 struct vhost_vring_addr *vra = &vmsg->payload.addr;
518 unsigned int index = vra->index;
519 VuVirtq *vq = &dev->vq[index];
520
521 DPRINT("vhost_vring_addr:\n");
522 DPRINT(" index: %d\n", vra->index);
523 DPRINT(" flags: %d\n", vra->flags);
524 DPRINT(" desc_user_addr: 0x%016llx\n", vra->desc_user_addr);
525 DPRINT(" used_user_addr: 0x%016llx\n", vra->used_user_addr);
526 DPRINT(" avail_user_addr: 0x%016llx\n", vra->avail_user_addr);
527 DPRINT(" log_guest_addr: 0x%016llx\n", vra->log_guest_addr);
528
529 vq->vring.flags = vra->flags;
530 vq->vring.desc = qva_to_va(dev, vra->desc_user_addr);
531 vq->vring.used = qva_to_va(dev, vra->used_user_addr);
532 vq->vring.avail = qva_to_va(dev, vra->avail_user_addr);
533 vq->vring.log_guest_addr = vra->log_guest_addr;
534
535 DPRINT("Setting virtq addresses:\n");
536 DPRINT(" vring_desc at %p\n", vq->vring.desc);
537 DPRINT(" vring_used at %p\n", vq->vring.used);
538 DPRINT(" vring_avail at %p\n", vq->vring.avail);
539
540 if (!(vq->vring.desc && vq->vring.used && vq->vring.avail)) {
541 vu_panic(dev, "Invalid vring_addr message");
542 return false;
543 }
544
545 vq->used_idx = vq->vring.used->idx;
546
547 if (vq->last_avail_idx != vq->used_idx) {
548 bool resume = dev->iface->queue_is_processed_in_order &&
549 dev->iface->queue_is_processed_in_order(dev, index);
550
551 DPRINT("Last avail index != used index: %u != %u%s\n",
552 vq->last_avail_idx, vq->used_idx,
553 resume ? ", resuming" : "");
554
555 if (resume) {
556 vq->shadow_avail_idx = vq->last_avail_idx = vq->used_idx;
557 }
558 }
559
560 return false;
561 }
562
563 static bool
564 vu_set_vring_base_exec(VuDev *dev, VhostUserMsg *vmsg)
565 {
566 unsigned int index = vmsg->payload.state.index;
567 unsigned int num = vmsg->payload.state.num;
568
569 DPRINT("State.index: %d\n", index);
570 DPRINT("State.num: %d\n", num);
571 dev->vq[index].shadow_avail_idx = dev->vq[index].last_avail_idx = num;
572
573 return false;
574 }
575
576 static bool
577 vu_get_vring_base_exec(VuDev *dev, VhostUserMsg *vmsg)
578 {
579 unsigned int index = vmsg->payload.state.index;
580
581 DPRINT("State.index: %d\n", index);
582 vmsg->payload.state.num = dev->vq[index].last_avail_idx;
583 vmsg->size = sizeof(vmsg->payload.state);
584
585 dev->vq[index].started = false;
586 if (dev->iface->queue_set_started) {
587 dev->iface->queue_set_started(dev, index, false);
588 }
589
590 if (dev->vq[index].call_fd != -1) {
591 close(dev->vq[index].call_fd);
592 dev->vq[index].call_fd = -1;
593 }
594 if (dev->vq[index].kick_fd != -1) {
595 dev->remove_watch(dev, dev->vq[index].kick_fd);
596 close(dev->vq[index].kick_fd);
597 dev->vq[index].kick_fd = -1;
598 }
599
600 return true;
601 }
602
603 static bool
604 vu_check_queue_msg_file(VuDev *dev, VhostUserMsg *vmsg)
605 {
606 int index = vmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
607
608 if (index >= VHOST_MAX_NR_VIRTQUEUE) {
609 vmsg_close_fds(vmsg);
610 vu_panic(dev, "Invalid queue index: %u", index);
611 return false;
612 }
613
614 if (vmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK ||
615 vmsg->fd_num != 1) {
616 vmsg_close_fds(vmsg);
617 vu_panic(dev, "Invalid fds in request: %d", vmsg->request);
618 return false;
619 }
620
621 return true;
622 }
623
624 static bool
625 vu_set_vring_kick_exec(VuDev *dev, VhostUserMsg *vmsg)
626 {
627 int index = vmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
628
629 DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
630
631 if (!vu_check_queue_msg_file(dev, vmsg)) {
632 return false;
633 }
634
635 if (dev->vq[index].kick_fd != -1) {
636 dev->remove_watch(dev, dev->vq[index].kick_fd);
637 close(dev->vq[index].kick_fd);
638 dev->vq[index].kick_fd = -1;
639 }
640
641 if (!(vmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK)) {
642 dev->vq[index].kick_fd = vmsg->fds[0];
643 DPRINT("Got kick_fd: %d for vq: %d\n", vmsg->fds[0], index);
644 }
645
646 dev->vq[index].started = true;
647 if (dev->iface->queue_set_started) {
648 dev->iface->queue_set_started(dev, index, true);
649 }
650
651 if (dev->vq[index].kick_fd != -1 && dev->vq[index].handler) {
652 dev->set_watch(dev, dev->vq[index].kick_fd, VU_WATCH_IN,
653 vu_kick_cb, (void *)(long)index);
654
655 DPRINT("Waiting for kicks on fd: %d for vq: %d\n",
656 dev->vq[index].kick_fd, index);
657 }
658
659 return false;
660 }
661
662 void vu_set_queue_handler(VuDev *dev, VuVirtq *vq,
663 vu_queue_handler_cb handler)
664 {
665 int qidx = vq - dev->vq;
666
667 vq->handler = handler;
668 if (vq->kick_fd >= 0) {
669 if (handler) {
670 dev->set_watch(dev, vq->kick_fd, VU_WATCH_IN,
671 vu_kick_cb, (void *)(long)qidx);
672 } else {
673 dev->remove_watch(dev, vq->kick_fd);
674 }
675 }
676 }
677
678 static bool
679 vu_set_vring_call_exec(VuDev *dev, VhostUserMsg *vmsg)
680 {
681 int index = vmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
682
683 DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
684
685 if (!vu_check_queue_msg_file(dev, vmsg)) {
686 return false;
687 }
688
689 if (dev->vq[index].call_fd != -1) {
690 close(dev->vq[index].call_fd);
691 dev->vq[index].call_fd = -1;
692 }
693
694 if (!(vmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK)) {
695 dev->vq[index].call_fd = vmsg->fds[0];
696 }
697
698 DPRINT("Got call_fd: %d for vq: %d\n", vmsg->fds[0], index);
699
700 return false;
701 }
702
703 static bool
704 vu_set_vring_err_exec(VuDev *dev, VhostUserMsg *vmsg)
705 {
706 int index = vmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
707
708 DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
709
710 if (!vu_check_queue_msg_file(dev, vmsg)) {
711 return false;
712 }
713
714 if (dev->vq[index].err_fd != -1) {
715 close(dev->vq[index].err_fd);
716 dev->vq[index].err_fd = -1;
717 }
718
719 if (!(vmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK)) {
720 dev->vq[index].err_fd = vmsg->fds[0];
721 }
722
723 return false;
724 }
725
726 static bool
727 vu_get_protocol_features_exec(VuDev *dev, VhostUserMsg *vmsg)
728 {
729 uint64_t features = 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD;
730
731 if (dev->iface->get_protocol_features) {
732 features |= dev->iface->get_protocol_features(dev);
733 }
734
735 vmsg->payload.u64 = features;
736 vmsg->size = sizeof(vmsg->payload.u64);
737
738 return true;
739 }
740
741 static bool
742 vu_set_protocol_features_exec(VuDev *dev, VhostUserMsg *vmsg)
743 {
744 uint64_t features = vmsg->payload.u64;
745
746 DPRINT("u64: 0x%016"PRIx64"\n", features);
747
748 dev->protocol_features = vmsg->payload.u64;
749
750 if (dev->iface->set_protocol_features) {
751 dev->iface->set_protocol_features(dev, features);
752 }
753
754 return false;
755 }
756
757 static bool
758 vu_get_queue_num_exec(VuDev *dev, VhostUserMsg *vmsg)
759 {
760 DPRINT("Function %s() not implemented yet.\n", __func__);
761 return false;
762 }
763
764 static bool
765 vu_set_vring_enable_exec(VuDev *dev, VhostUserMsg *vmsg)
766 {
767 unsigned int index = vmsg->payload.state.index;
768 unsigned int enable = vmsg->payload.state.num;
769
770 DPRINT("State.index: %d\n", index);
771 DPRINT("State.enable: %d\n", enable);
772
773 if (index >= VHOST_MAX_NR_VIRTQUEUE) {
774 vu_panic(dev, "Invalid vring_enable index: %u", index);
775 return false;
776 }
777
778 dev->vq[index].enable = enable;
779 return false;
780 }
781
782 static bool
783 vu_process_message(VuDev *dev, VhostUserMsg *vmsg)
784 {
785 int do_reply = 0;
786
787 /* Print out generic part of the request. */
788 DPRINT("================ Vhost user message ================\n");
789 DPRINT("Request: %s (%d)\n", vu_request_to_string(vmsg->request),
790 vmsg->request);
791 DPRINT("Flags: 0x%x\n", vmsg->flags);
792 DPRINT("Size: %d\n", vmsg->size);
793
794 if (vmsg->fd_num) {
795 int i;
796 DPRINT("Fds:");
797 for (i = 0; i < vmsg->fd_num; i++) {
798 DPRINT(" %d", vmsg->fds[i]);
799 }
800 DPRINT("\n");
801 }
802
803 if (dev->iface->process_msg &&
804 dev->iface->process_msg(dev, vmsg, &do_reply)) {
805 return do_reply;
806 }
807
808 switch (vmsg->request) {
809 case VHOST_USER_GET_FEATURES:
810 return vu_get_features_exec(dev, vmsg);
811 case VHOST_USER_SET_FEATURES:
812 return vu_set_features_exec(dev, vmsg);
813 case VHOST_USER_GET_PROTOCOL_FEATURES:
814 return vu_get_protocol_features_exec(dev, vmsg);
815 case VHOST_USER_SET_PROTOCOL_FEATURES:
816 return vu_set_protocol_features_exec(dev, vmsg);
817 case VHOST_USER_SET_OWNER:
818 return vu_set_owner_exec(dev, vmsg);
819 case VHOST_USER_RESET_OWNER:
820 return vu_reset_device_exec(dev, vmsg);
821 case VHOST_USER_SET_MEM_TABLE:
822 return vu_set_mem_table_exec(dev, vmsg);
823 case VHOST_USER_SET_LOG_BASE:
824 return vu_set_log_base_exec(dev, vmsg);
825 case VHOST_USER_SET_LOG_FD:
826 return vu_set_log_fd_exec(dev, vmsg);
827 case VHOST_USER_SET_VRING_NUM:
828 return vu_set_vring_num_exec(dev, vmsg);
829 case VHOST_USER_SET_VRING_ADDR:
830 return vu_set_vring_addr_exec(dev, vmsg);
831 case VHOST_USER_SET_VRING_BASE:
832 return vu_set_vring_base_exec(dev, vmsg);
833 case VHOST_USER_GET_VRING_BASE:
834 return vu_get_vring_base_exec(dev, vmsg);
835 case VHOST_USER_SET_VRING_KICK:
836 return vu_set_vring_kick_exec(dev, vmsg);
837 case VHOST_USER_SET_VRING_CALL:
838 return vu_set_vring_call_exec(dev, vmsg);
839 case VHOST_USER_SET_VRING_ERR:
840 return vu_set_vring_err_exec(dev, vmsg);
841 case VHOST_USER_GET_QUEUE_NUM:
842 return vu_get_queue_num_exec(dev, vmsg);
843 case VHOST_USER_SET_VRING_ENABLE:
844 return vu_set_vring_enable_exec(dev, vmsg);
845 case VHOST_USER_NONE:
846 break;
847 default:
848 vmsg_close_fds(vmsg);
849 vu_panic(dev, "Unhandled request: %d", vmsg->request);
850 }
851
852 return false;
853 }
854
855 bool
856 vu_dispatch(VuDev *dev)
857 {
858 VhostUserMsg vmsg = { 0, };
859 int reply_requested;
860 bool success = false;
861
862 if (!vu_message_read(dev, dev->sock, &vmsg)) {
863 goto end;
864 }
865
866 reply_requested = vu_process_message(dev, &vmsg);
867 if (!reply_requested) {
868 success = true;
869 goto end;
870 }
871
872 if (!vu_message_write(dev, dev->sock, &vmsg)) {
873 goto end;
874 }
875
876 success = true;
877
878 end:
879 free(vmsg.data);
880 return success;
881 }
882
883 void
884 vu_deinit(VuDev *dev)
885 {
886 int i;
887
888 for (i = 0; i < dev->nregions; i++) {
889 VuDevRegion *r = &dev->regions[i];
890 void *m = (void *) (uintptr_t) r->mmap_addr;
891 if (m != MAP_FAILED) {
892 munmap(m, r->size + r->mmap_offset);
893 }
894 }
895 dev->nregions = 0;
896
897 for (i = 0; i < VHOST_MAX_NR_VIRTQUEUE; i++) {
898 VuVirtq *vq = &dev->vq[i];
899
900 if (vq->call_fd != -1) {
901 close(vq->call_fd);
902 vq->call_fd = -1;
903 }
904
905 if (vq->kick_fd != -1) {
906 close(vq->kick_fd);
907 vq->kick_fd = -1;
908 }
909
910 if (vq->err_fd != -1) {
911 close(vq->err_fd);
912 vq->err_fd = -1;
913 }
914 }
915
916
917 vu_close_log(dev);
918
919 if (dev->sock != -1) {
920 close(dev->sock);
921 }
922 }
923
924 void
925 vu_init(VuDev *dev,
926 int socket,
927 vu_panic_cb panic,
928 vu_set_watch_cb set_watch,
929 vu_remove_watch_cb remove_watch,
930 const VuDevIface *iface)
931 {
932 int i;
933
934 assert(socket >= 0);
935 assert(set_watch);
936 assert(remove_watch);
937 assert(iface);
938 assert(panic);
939
940 memset(dev, 0, sizeof(*dev));
941
942 dev->sock = socket;
943 dev->panic = panic;
944 dev->set_watch = set_watch;
945 dev->remove_watch = remove_watch;
946 dev->iface = iface;
947 dev->log_call_fd = -1;
948 for (i = 0; i < VHOST_MAX_NR_VIRTQUEUE; i++) {
949 dev->vq[i] = (VuVirtq) {
950 .call_fd = -1, .kick_fd = -1, .err_fd = -1,
951 .notification = true,
952 };
953 }
954 }
955
956 VuVirtq *
957 vu_get_queue(VuDev *dev, int qidx)
958 {
959 assert(qidx < VHOST_MAX_NR_VIRTQUEUE);
960 return &dev->vq[qidx];
961 }
962
963 bool
964 vu_queue_enabled(VuDev *dev, VuVirtq *vq)
965 {
966 return vq->enable;
967 }
968
969 bool
970 vu_queue_started(const VuDev *dev, const VuVirtq *vq)
971 {
972 return vq->started;
973 }
974
975 static inline uint16_t
976 vring_avail_flags(VuVirtq *vq)
977 {
978 return vq->vring.avail->flags;
979 }
980
981 static inline uint16_t
982 vring_avail_idx(VuVirtq *vq)
983 {
984 vq->shadow_avail_idx = vq->vring.avail->idx;
985
986 return vq->shadow_avail_idx;
987 }
988
989 static inline uint16_t
990 vring_avail_ring(VuVirtq *vq, int i)
991 {
992 return vq->vring.avail->ring[i];
993 }
994
995 static inline uint16_t
996 vring_get_used_event(VuVirtq *vq)
997 {
998 return vring_avail_ring(vq, vq->vring.num);
999 }
1000
1001 static int
1002 virtqueue_num_heads(VuDev *dev, VuVirtq *vq, unsigned int idx)
1003 {
1004 uint16_t num_heads = vring_avail_idx(vq) - idx;
1005
1006 /* Check it isn't doing very strange things with descriptor numbers. */
1007 if (num_heads > vq->vring.num) {
1008 vu_panic(dev, "Guest moved used index from %u to %u",
1009 idx, vq->shadow_avail_idx);
1010 return -1;
1011 }
1012 if (num_heads) {
1013 /* On success, callers read a descriptor at vq->last_avail_idx.
1014 * Make sure descriptor read does not bypass avail index read. */
1015 smp_rmb();
1016 }
1017
1018 return num_heads;
1019 }
1020
1021 static bool
1022 virtqueue_get_head(VuDev *dev, VuVirtq *vq,
1023 unsigned int idx, unsigned int *head)
1024 {
1025 /* Grab the next descriptor number they're advertising, and increment
1026 * the index we've seen. */
1027 *head = vring_avail_ring(vq, idx % vq->vring.num);
1028
1029 /* If their number is silly, that's a fatal mistake. */
1030 if (*head >= vq->vring.num) {
1031 vu_panic(dev, "Guest says index %u is available", head);
1032 return false;
1033 }
1034
1035 return true;
1036 }
1037
1038 enum {
1039 VIRTQUEUE_READ_DESC_ERROR = -1,
1040 VIRTQUEUE_READ_DESC_DONE = 0, /* end of chain */
1041 VIRTQUEUE_READ_DESC_MORE = 1, /* more buffers in chain */
1042 };
1043
1044 static int
1045 virtqueue_read_next_desc(VuDev *dev, struct vring_desc *desc,
1046 int i, unsigned int max, unsigned int *next)
1047 {
1048 /* If this descriptor says it doesn't chain, we're done. */
1049 if (!(desc[i].flags & VRING_DESC_F_NEXT)) {
1050 return VIRTQUEUE_READ_DESC_DONE;
1051 }
1052
1053 /* Check they're not leading us off end of descriptors. */
1054 *next = desc[i].next;
1055 /* Make sure compiler knows to grab that: we don't want it changing! */
1056 smp_wmb();
1057
1058 if (*next >= max) {
1059 vu_panic(dev, "Desc next is %u", next);
1060 return VIRTQUEUE_READ_DESC_ERROR;
1061 }
1062
1063 return VIRTQUEUE_READ_DESC_MORE;
1064 }
1065
1066 void
1067 vu_queue_get_avail_bytes(VuDev *dev, VuVirtq *vq, unsigned int *in_bytes,
1068 unsigned int *out_bytes,
1069 unsigned max_in_bytes, unsigned max_out_bytes)
1070 {
1071 unsigned int idx;
1072 unsigned int total_bufs, in_total, out_total;
1073 int rc;
1074
1075 idx = vq->last_avail_idx;
1076
1077 total_bufs = in_total = out_total = 0;
1078 if (unlikely(dev->broken) ||
1079 unlikely(!vq->vring.avail)) {
1080 goto done;
1081 }
1082
1083 while ((rc = virtqueue_num_heads(dev, vq, idx)) > 0) {
1084 unsigned int max, num_bufs, indirect = 0;
1085 struct vring_desc *desc;
1086 unsigned int i;
1087
1088 max = vq->vring.num;
1089 num_bufs = total_bufs;
1090 if (!virtqueue_get_head(dev, vq, idx++, &i)) {
1091 goto err;
1092 }
1093 desc = vq->vring.desc;
1094
1095 if (desc[i].flags & VRING_DESC_F_INDIRECT) {
1096 if (desc[i].len % sizeof(struct vring_desc)) {
1097 vu_panic(dev, "Invalid size for indirect buffer table");
1098 goto err;
1099 }
1100
1101 /* If we've got too many, that implies a descriptor loop. */
1102 if (num_bufs >= max) {
1103 vu_panic(dev, "Looped descriptor");
1104 goto err;
1105 }
1106
1107 /* loop over the indirect descriptor table */
1108 indirect = 1;
1109 max = desc[i].len / sizeof(struct vring_desc);
1110 desc = vu_gpa_to_va(dev, desc[i].addr);
1111 num_bufs = i = 0;
1112 }
1113
1114 do {
1115 /* If we've got too many, that implies a descriptor loop. */
1116 if (++num_bufs > max) {
1117 vu_panic(dev, "Looped descriptor");
1118 goto err;
1119 }
1120
1121 if (desc[i].flags & VRING_DESC_F_WRITE) {
1122 in_total += desc[i].len;
1123 } else {
1124 out_total += desc[i].len;
1125 }
1126 if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
1127 goto done;
1128 }
1129 rc = virtqueue_read_next_desc(dev, desc, i, max, &i);
1130 } while (rc == VIRTQUEUE_READ_DESC_MORE);
1131
1132 if (rc == VIRTQUEUE_READ_DESC_ERROR) {
1133 goto err;
1134 }
1135
1136 if (!indirect) {
1137 total_bufs = num_bufs;
1138 } else {
1139 total_bufs++;
1140 }
1141 }
1142 if (rc < 0) {
1143 goto err;
1144 }
1145 done:
1146 if (in_bytes) {
1147 *in_bytes = in_total;
1148 }
1149 if (out_bytes) {
1150 *out_bytes = out_total;
1151 }
1152 return;
1153
1154 err:
1155 in_total = out_total = 0;
1156 goto done;
1157 }
1158
1159 bool
1160 vu_queue_avail_bytes(VuDev *dev, VuVirtq *vq, unsigned int in_bytes,
1161 unsigned int out_bytes)
1162 {
1163 unsigned int in_total, out_total;
1164
1165 vu_queue_get_avail_bytes(dev, vq, &in_total, &out_total,
1166 in_bytes, out_bytes);
1167
1168 return in_bytes <= in_total && out_bytes <= out_total;
1169 }
1170
1171 /* Fetch avail_idx from VQ memory only when we really need to know if
1172 * guest has added some buffers. */
1173 bool
1174 vu_queue_empty(VuDev *dev, VuVirtq *vq)
1175 {
1176 if (unlikely(dev->broken) ||
1177 unlikely(!vq->vring.avail)) {
1178 return true;
1179 }
1180
1181 if (vq->shadow_avail_idx != vq->last_avail_idx) {
1182 return false;
1183 }
1184
1185 return vring_avail_idx(vq) == vq->last_avail_idx;
1186 }
1187
1188 static inline
1189 bool has_feature(uint64_t features, unsigned int fbit)
1190 {
1191 assert(fbit < 64);
1192 return !!(features & (1ULL << fbit));
1193 }
1194
1195 static inline
1196 bool vu_has_feature(VuDev *dev,
1197 unsigned int fbit)
1198 {
1199 return has_feature(dev->features, fbit);
1200 }
1201
1202 static bool
1203 vring_notify(VuDev *dev, VuVirtq *vq)
1204 {
1205 uint16_t old, new;
1206 bool v;
1207
1208 /* We need to expose used array entries before checking used event. */
1209 smp_mb();
1210
1211 /* Always notify when queue is empty (when feature acknowledge) */
1212 if (vu_has_feature(dev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
1213 !vq->inuse && vu_queue_empty(dev, vq)) {
1214 return true;
1215 }
1216
1217 if (!vu_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
1218 return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT);
1219 }
1220
1221 v = vq->signalled_used_valid;
1222 vq->signalled_used_valid = true;
1223 old = vq->signalled_used;
1224 new = vq->signalled_used = vq->used_idx;
1225 return !v || vring_need_event(vring_get_used_event(vq), new, old);
1226 }
1227
1228 void
1229 vu_queue_notify(VuDev *dev, VuVirtq *vq)
1230 {
1231 if (unlikely(dev->broken) ||
1232 unlikely(!vq->vring.avail)) {
1233 return;
1234 }
1235
1236 if (!vring_notify(dev, vq)) {
1237 DPRINT("skipped notify...\n");
1238 return;
1239 }
1240
1241 if (eventfd_write(vq->call_fd, 1) < 0) {
1242 vu_panic(dev, "Error writing eventfd: %s", strerror(errno));
1243 }
1244 }
1245
1246 static inline void
1247 vring_used_flags_set_bit(VuVirtq *vq, int mask)
1248 {
1249 uint16_t *flags;
1250
1251 flags = (uint16_t *)((char*)vq->vring.used +
1252 offsetof(struct vring_used, flags));
1253 *flags |= mask;
1254 }
1255
1256 static inline void
1257 vring_used_flags_unset_bit(VuVirtq *vq, int mask)
1258 {
1259 uint16_t *flags;
1260
1261 flags = (uint16_t *)((char*)vq->vring.used +
1262 offsetof(struct vring_used, flags));
1263 *flags &= ~mask;
1264 }
1265
1266 static inline void
1267 vring_set_avail_event(VuVirtq *vq, uint16_t val)
1268 {
1269 if (!vq->notification) {
1270 return;
1271 }
1272
1273 *((uint16_t *) &vq->vring.used->ring[vq->vring.num]) = val;
1274 }
1275
1276 void
1277 vu_queue_set_notification(VuDev *dev, VuVirtq *vq, int enable)
1278 {
1279 vq->notification = enable;
1280 if (vu_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
1281 vring_set_avail_event(vq, vring_avail_idx(vq));
1282 } else if (enable) {
1283 vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY);
1284 } else {
1285 vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY);
1286 }
1287 if (enable) {
1288 /* Expose avail event/used flags before caller checks the avail idx. */
1289 smp_mb();
1290 }
1291 }
1292
1293 static void
1294 virtqueue_map_desc(VuDev *dev,
1295 unsigned int *p_num_sg, struct iovec *iov,
1296 unsigned int max_num_sg, bool is_write,
1297 uint64_t pa, size_t sz)
1298 {
1299 unsigned num_sg = *p_num_sg;
1300
1301 assert(num_sg <= max_num_sg);
1302
1303 if (!sz) {
1304 vu_panic(dev, "virtio: zero sized buffers are not allowed");
1305 return;
1306 }
1307
1308 iov[num_sg].iov_base = vu_gpa_to_va(dev, pa);
1309 iov[num_sg].iov_len = sz;
1310 num_sg++;
1311
1312 *p_num_sg = num_sg;
1313 }
1314
1315 /* Round number down to multiple */
1316 #define ALIGN_DOWN(n, m) ((n) / (m) * (m))
1317
1318 /* Round number up to multiple */
1319 #define ALIGN_UP(n, m) ALIGN_DOWN((n) + (m) - 1, (m))
1320
1321 static void *
1322 virtqueue_alloc_element(size_t sz,
1323 unsigned out_num, unsigned in_num)
1324 {
1325 VuVirtqElement *elem;
1326 size_t in_sg_ofs = ALIGN_UP(sz, __alignof__(elem->in_sg[0]));
1327 size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]);
1328 size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]);
1329
1330 assert(sz >= sizeof(VuVirtqElement));
1331 elem = malloc(out_sg_end);
1332 elem->out_num = out_num;
1333 elem->in_num = in_num;
1334 elem->in_sg = (void *)elem + in_sg_ofs;
1335 elem->out_sg = (void *)elem + out_sg_ofs;
1336 return elem;
1337 }
1338
1339 void *
1340 vu_queue_pop(VuDev *dev, VuVirtq *vq, size_t sz)
1341 {
1342 unsigned int i, head, max;
1343 VuVirtqElement *elem;
1344 unsigned out_num, in_num;
1345 struct iovec iov[VIRTQUEUE_MAX_SIZE];
1346 struct vring_desc *desc;
1347 int rc;
1348
1349 if (unlikely(dev->broken) ||
1350 unlikely(!vq->vring.avail)) {
1351 return NULL;
1352 }
1353
1354 if (vu_queue_empty(dev, vq)) {
1355 return NULL;
1356 }
1357 /* Needed after virtio_queue_empty(), see comment in
1358 * virtqueue_num_heads(). */
1359 smp_rmb();
1360
1361 /* When we start there are none of either input nor output. */
1362 out_num = in_num = 0;
1363
1364 max = vq->vring.num;
1365 if (vq->inuse >= vq->vring.num) {
1366 vu_panic(dev, "Virtqueue size exceeded");
1367 return NULL;
1368 }
1369
1370 if (!virtqueue_get_head(dev, vq, vq->last_avail_idx++, &head)) {
1371 return NULL;
1372 }
1373
1374 if (vu_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) {
1375 vring_set_avail_event(vq, vq->last_avail_idx);
1376 }
1377
1378 i = head;
1379 desc = vq->vring.desc;
1380 if (desc[i].flags & VRING_DESC_F_INDIRECT) {
1381 if (desc[i].len % sizeof(struct vring_desc)) {
1382 vu_panic(dev, "Invalid size for indirect buffer table");
1383 }
1384
1385 /* loop over the indirect descriptor table */
1386 max = desc[i].len / sizeof(struct vring_desc);
1387 desc = vu_gpa_to_va(dev, desc[i].addr);
1388 i = 0;
1389 }
1390
1391 /* Collect all the descriptors */
1392 do {
1393 if (desc[i].flags & VRING_DESC_F_WRITE) {
1394 virtqueue_map_desc(dev, &in_num, iov + out_num,
1395 VIRTQUEUE_MAX_SIZE - out_num, true,
1396 desc[i].addr, desc[i].len);
1397 } else {
1398 if (in_num) {
1399 vu_panic(dev, "Incorrect order for descriptors");
1400 return NULL;
1401 }
1402 virtqueue_map_desc(dev, &out_num, iov,
1403 VIRTQUEUE_MAX_SIZE, false,
1404 desc[i].addr, desc[i].len);
1405 }
1406
1407 /* If we've got too many, that implies a descriptor loop. */
1408 if ((in_num + out_num) > max) {
1409 vu_panic(dev, "Looped descriptor");
1410 }
1411 rc = virtqueue_read_next_desc(dev, desc, i, max, &i);
1412 } while (rc == VIRTQUEUE_READ_DESC_MORE);
1413
1414 if (rc == VIRTQUEUE_READ_DESC_ERROR) {
1415 return NULL;
1416 }
1417
1418 /* Now copy what we have collected and mapped */
1419 elem = virtqueue_alloc_element(sz, out_num, in_num);
1420 elem->index = head;
1421 for (i = 0; i < out_num; i++) {
1422 elem->out_sg[i] = iov[i];
1423 }
1424 for (i = 0; i < in_num; i++) {
1425 elem->in_sg[i] = iov[out_num + i];
1426 }
1427
1428 vq->inuse++;
1429
1430 return elem;
1431 }
1432
1433 bool
1434 vu_queue_rewind(VuDev *dev, VuVirtq *vq, unsigned int num)
1435 {
1436 if (num > vq->inuse) {
1437 return false;
1438 }
1439 vq->last_avail_idx -= num;
1440 vq->inuse -= num;
1441 return true;
1442 }
1443
1444 static inline
1445 void vring_used_write(VuDev *dev, VuVirtq *vq,
1446 struct vring_used_elem *uelem, int i)
1447 {
1448 struct vring_used *used = vq->vring.used;
1449
1450 used->ring[i] = *uelem;
1451 vu_log_write(dev, vq->vring.log_guest_addr +
1452 offsetof(struct vring_used, ring[i]),
1453 sizeof(used->ring[i]));
1454 }
1455
1456
1457 static void
1458 vu_log_queue_fill(VuDev *dev, VuVirtq *vq,
1459 const VuVirtqElement *elem,
1460 unsigned int len)
1461 {
1462 struct vring_desc *desc = vq->vring.desc;
1463 unsigned int i, max, min;
1464 unsigned num_bufs = 0;
1465
1466 max = vq->vring.num;
1467 i = elem->index;
1468
1469 if (desc[i].flags & VRING_DESC_F_INDIRECT) {
1470 if (desc[i].len % sizeof(struct vring_desc)) {
1471 vu_panic(dev, "Invalid size for indirect buffer table");
1472 }
1473
1474 /* loop over the indirect descriptor table */
1475 max = desc[i].len / sizeof(struct vring_desc);
1476 desc = vu_gpa_to_va(dev, desc[i].addr);
1477 i = 0;
1478 }
1479
1480 do {
1481 if (++num_bufs > max) {
1482 vu_panic(dev, "Looped descriptor");
1483 return;
1484 }
1485
1486 if (desc[i].flags & VRING_DESC_F_WRITE) {
1487 min = MIN(desc[i].len, len);
1488 vu_log_write(dev, desc[i].addr, min);
1489 len -= min;
1490 }
1491
1492 } while (len > 0 &&
1493 (virtqueue_read_next_desc(dev, desc, i, max, &i)
1494 == VIRTQUEUE_READ_DESC_MORE));
1495 }
1496
1497 void
1498 vu_queue_fill(VuDev *dev, VuVirtq *vq,
1499 const VuVirtqElement *elem,
1500 unsigned int len, unsigned int idx)
1501 {
1502 struct vring_used_elem uelem;
1503
1504 if (unlikely(dev->broken) ||
1505 unlikely(!vq->vring.avail)) {
1506 return;
1507 }
1508
1509 vu_log_queue_fill(dev, vq, elem, len);
1510
1511 idx = (idx + vq->used_idx) % vq->vring.num;
1512
1513 uelem.id = elem->index;
1514 uelem.len = len;
1515 vring_used_write(dev, vq, &uelem, idx);
1516 }
1517
1518 static inline
1519 void vring_used_idx_set(VuDev *dev, VuVirtq *vq, uint16_t val)
1520 {
1521 vq->vring.used->idx = val;
1522 vu_log_write(dev,
1523 vq->vring.log_guest_addr + offsetof(struct vring_used, idx),
1524 sizeof(vq->vring.used->idx));
1525
1526 vq->used_idx = val;
1527 }
1528
1529 void
1530 vu_queue_flush(VuDev *dev, VuVirtq *vq, unsigned int count)
1531 {
1532 uint16_t old, new;
1533
1534 if (unlikely(dev->broken) ||
1535 unlikely(!vq->vring.avail)) {
1536 return;
1537 }
1538
1539 /* Make sure buffer is written before we update index. */
1540 smp_wmb();
1541
1542 old = vq->used_idx;
1543 new = old + count;
1544 vring_used_idx_set(dev, vq, new);
1545 vq->inuse -= count;
1546 if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old))) {
1547 vq->signalled_used_valid = false;
1548 }
1549 }
1550
1551 void
1552 vu_queue_push(VuDev *dev, VuVirtq *vq,
1553 const VuVirtqElement *elem, unsigned int len)
1554 {
1555 vu_queue_fill(dev, vq, elem, len, 0);
1556 vu_queue_flush(dev, vq, 1);
1557 }
QEmu