4 * Copyright IBM, Corp. 2007
7 * Anthony Liguori <aliguori@us.ibm.com>
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
14 #include "qemu/osdep.h"
15 #include "qapi/error.h"
16 #include "qemu-common.h"
19 #include "exec/address-spaces.h"
20 #include "qemu/error-report.h"
21 #include "hw/virtio/virtio.h"
22 #include "qemu/atomic.h"
23 #include "hw/virtio/virtio-bus.h"
24 #include "migration/migration.h"
25 #include "hw/virtio/virtio-access.h"
26 #include "sysemu/dma.h"
29 * The alignment to use between consumer and producer parts of vring.
30 * x86 pagesize again. This is the default, used by transports like PCI
31 * which don't provide a means for the guest to tell the host the alignment.
33 #define VIRTIO_PCI_VRING_ALIGN 4096
35 typedef struct VRingDesc
43 typedef struct VRingAvail
50 typedef struct VRingUsedElem
56 typedef struct VRingUsed
60 VRingUsedElem ring
[0];
63 typedef struct VRingMemoryRegionCaches
{
65 MemoryRegionCache desc
;
66 MemoryRegionCache avail
;
67 MemoryRegionCache used
;
68 } VRingMemoryRegionCaches
;
73 unsigned int num_default
;
78 VRingMemoryRegionCaches
*caches
;
85 /* Next head to pop */
86 uint16_t last_avail_idx
;
88 /* Last avail_idx read from VQ. */
89 uint16_t shadow_avail_idx
;
93 /* Last used index value we have signalled on */
94 uint16_t signalled_used
;
96 /* Last used index value we have signalled on */
97 bool signalled_used_valid
;
99 /* Notification enabled? */
102 uint16_t queue_index
;
107 VirtIOHandleOutput handle_output
;
108 VirtIOHandleAIOOutput handle_aio_output
;
110 EventNotifier guest_notifier
;
111 EventNotifier host_notifier
;
112 QLIST_ENTRY(VirtQueue
) node
;
115 static void virtio_free_region_cache(VRingMemoryRegionCaches
*caches
)
121 address_space_cache_destroy(&caches
->desc
);
122 address_space_cache_destroy(&caches
->avail
);
123 address_space_cache_destroy(&caches
->used
);
127 static void virtio_init_region_cache(VirtIODevice
*vdev
, int n
)
129 VirtQueue
*vq
= &vdev
->vq
[n
];
130 VRingMemoryRegionCaches
*old
= vq
->vring
.caches
;
131 VRingMemoryRegionCaches
*new;
135 event_size
= virtio_vdev_has_feature(vq
->vdev
, VIRTIO_RING_F_EVENT_IDX
) ? 2 : 0;
137 addr
= vq
->vring
.desc
;
141 new = g_new0(VRingMemoryRegionCaches
, 1);
142 size
= virtio_queue_get_desc_size(vdev
, n
);
143 address_space_cache_init(&new->desc
, vdev
->dma_as
,
146 size
= virtio_queue_get_used_size(vdev
, n
) + event_size
;
147 address_space_cache_init(&new->used
, vdev
->dma_as
,
148 vq
->vring
.used
, size
, true);
150 size
= virtio_queue_get_avail_size(vdev
, n
) + event_size
;
151 address_space_cache_init(&new->avail
, vdev
->dma_as
,
152 vq
->vring
.avail
, size
, false);
154 atomic_rcu_set(&vq
->vring
.caches
, new);
156 call_rcu(old
, virtio_free_region_cache
, rcu
);
160 /* virt queue functions */
161 void virtio_queue_update_rings(VirtIODevice
*vdev
, int n
)
163 VRing
*vring
= &vdev
->vq
[n
].vring
;
166 /* not yet setup -> nothing to do */
169 vring
->avail
= vring
->desc
+ vring
->num
* sizeof(VRingDesc
);
170 vring
->used
= vring_align(vring
->avail
+
171 offsetof(VRingAvail
, ring
[vring
->num
]),
173 virtio_init_region_cache(vdev
, n
);
176 static void vring_desc_read(VirtIODevice
*vdev
, VRingDesc
*desc
,
177 MemoryRegionCache
*cache
, int i
)
179 address_space_read_cached(cache
, i
* sizeof(VRingDesc
),
180 desc
, sizeof(VRingDesc
));
181 virtio_tswap64s(vdev
, &desc
->addr
);
182 virtio_tswap32s(vdev
, &desc
->len
);
183 virtio_tswap16s(vdev
, &desc
->flags
);
184 virtio_tswap16s(vdev
, &desc
->next
);
187 static inline uint16_t vring_avail_flags(VirtQueue
*vq
)
190 pa
= vq
->vring
.avail
+ offsetof(VRingAvail
, flags
);
191 return virtio_lduw_phys(vq
->vdev
, pa
);
194 static inline uint16_t vring_avail_idx(VirtQueue
*vq
)
197 pa
= vq
->vring
.avail
+ offsetof(VRingAvail
, idx
);
198 vq
->shadow_avail_idx
= virtio_lduw_phys(vq
->vdev
, pa
);
199 return vq
->shadow_avail_idx
;
202 static inline uint16_t vring_avail_ring(VirtQueue
*vq
, int i
)
205 pa
= vq
->vring
.avail
+ offsetof(VRingAvail
, ring
[i
]);
206 return virtio_lduw_phys(vq
->vdev
, pa
);
209 static inline uint16_t vring_get_used_event(VirtQueue
*vq
)
211 return vring_avail_ring(vq
, vq
->vring
.num
);
214 static inline void vring_used_write(VirtQueue
*vq
, VRingUsedElem
*uelem
,
218 virtio_tswap32s(vq
->vdev
, &uelem
->id
);
219 virtio_tswap32s(vq
->vdev
, &uelem
->len
);
220 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, ring
[i
]);
221 address_space_write(vq
->vdev
->dma_as
, pa
, MEMTXATTRS_UNSPECIFIED
,
222 (void *)uelem
, sizeof(VRingUsedElem
));
225 static uint16_t vring_used_idx(VirtQueue
*vq
)
228 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, idx
);
229 return virtio_lduw_phys(vq
->vdev
, pa
);
232 static inline void vring_used_idx_set(VirtQueue
*vq
, uint16_t val
)
235 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, idx
);
236 virtio_stw_phys(vq
->vdev
, pa
, val
);
240 static inline void vring_used_flags_set_bit(VirtQueue
*vq
, int mask
)
242 VirtIODevice
*vdev
= vq
->vdev
;
244 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, flags
);
245 virtio_stw_phys(vdev
, pa
, virtio_lduw_phys(vdev
, pa
) | mask
);
248 static inline void vring_used_flags_unset_bit(VirtQueue
*vq
, int mask
)
250 VirtIODevice
*vdev
= vq
->vdev
;
252 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, flags
);
253 virtio_stw_phys(vdev
, pa
, virtio_lduw_phys(vdev
, pa
) & ~mask
);
256 static inline void vring_set_avail_event(VirtQueue
*vq
, uint16_t val
)
259 if (!vq
->notification
) {
262 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, ring
[vq
->vring
.num
]);
263 virtio_stw_phys(vq
->vdev
, pa
, val
);
266 void virtio_queue_set_notification(VirtQueue
*vq
, int enable
)
268 vq
->notification
= enable
;
269 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
270 vring_set_avail_event(vq
, vring_avail_idx(vq
));
272 vring_used_flags_unset_bit(vq
, VRING_USED_F_NO_NOTIFY
);
274 vring_used_flags_set_bit(vq
, VRING_USED_F_NO_NOTIFY
);
277 /* Expose avail event/used flags before caller checks the avail idx. */
282 int virtio_queue_ready(VirtQueue
*vq
)
284 return vq
->vring
.avail
!= 0;
287 /* Fetch avail_idx from VQ memory only when we really need to know if
288 * guest has added some buffers. */
289 int virtio_queue_empty(VirtQueue
*vq
)
291 if (vq
->shadow_avail_idx
!= vq
->last_avail_idx
) {
295 return vring_avail_idx(vq
) == vq
->last_avail_idx
;
298 static void virtqueue_unmap_sg(VirtQueue
*vq
, const VirtQueueElement
*elem
,
301 AddressSpace
*dma_as
= vq
->vdev
->dma_as
;
306 for (i
= 0; i
< elem
->in_num
; i
++) {
307 size_t size
= MIN(len
- offset
, elem
->in_sg
[i
].iov_len
);
309 dma_memory_unmap(dma_as
, elem
->in_sg
[i
].iov_base
,
310 elem
->in_sg
[i
].iov_len
,
311 DMA_DIRECTION_FROM_DEVICE
, size
);
316 for (i
= 0; i
< elem
->out_num
; i
++)
317 dma_memory_unmap(dma_as
, elem
->out_sg
[i
].iov_base
,
318 elem
->out_sg
[i
].iov_len
,
319 DMA_DIRECTION_TO_DEVICE
,
320 elem
->out_sg
[i
].iov_len
);
323 /* virtqueue_detach_element:
324 * @vq: The #VirtQueue
325 * @elem: The #VirtQueueElement
326 * @len: number of bytes written
328 * Detach the element from the virtqueue. This function is suitable for device
329 * reset or other situations where a #VirtQueueElement is simply freed and will
330 * not be pushed or discarded.
332 void virtqueue_detach_element(VirtQueue
*vq
, const VirtQueueElement
*elem
,
336 virtqueue_unmap_sg(vq
, elem
, len
);
340 * @vq: The #VirtQueue
341 * @elem: The #VirtQueueElement
342 * @len: number of bytes written
344 * Pretend the most recent element wasn't popped from the virtqueue. The next
345 * call to virtqueue_pop() will refetch the element.
347 void virtqueue_unpop(VirtQueue
*vq
, const VirtQueueElement
*elem
,
350 vq
->last_avail_idx
--;
351 virtqueue_detach_element(vq
, elem
, len
);
355 * @vq: The #VirtQueue
356 * @num: Number of elements to push back
358 * Pretend that elements weren't popped from the virtqueue. The next
359 * virtqueue_pop() will refetch the oldest element.
361 * Use virtqueue_unpop() instead if you have a VirtQueueElement.
363 * Returns: true on success, false if @num is greater than the number of in use
366 bool virtqueue_rewind(VirtQueue
*vq
, unsigned int num
)
368 if (num
> vq
->inuse
) {
371 vq
->last_avail_idx
-= num
;
376 void virtqueue_fill(VirtQueue
*vq
, const VirtQueueElement
*elem
,
377 unsigned int len
, unsigned int idx
)
381 trace_virtqueue_fill(vq
, elem
, len
, idx
);
383 virtqueue_unmap_sg(vq
, elem
, len
);
385 if (unlikely(vq
->vdev
->broken
)) {
389 idx
= (idx
+ vq
->used_idx
) % vq
->vring
.num
;
391 uelem
.id
= elem
->index
;
393 vring_used_write(vq
, &uelem
, idx
);
396 void virtqueue_flush(VirtQueue
*vq
, unsigned int count
)
400 if (unlikely(vq
->vdev
->broken
)) {
405 /* Make sure buffer is written before we update index. */
407 trace_virtqueue_flush(vq
, count
);
410 vring_used_idx_set(vq
, new);
412 if (unlikely((int16_t)(new - vq
->signalled_used
) < (uint16_t)(new - old
)))
413 vq
->signalled_used_valid
= false;
416 void virtqueue_push(VirtQueue
*vq
, const VirtQueueElement
*elem
,
419 virtqueue_fill(vq
, elem
, len
, 0);
420 virtqueue_flush(vq
, 1);
423 static int virtqueue_num_heads(VirtQueue
*vq
, unsigned int idx
)
425 uint16_t num_heads
= vring_avail_idx(vq
) - idx
;
427 /* Check it isn't doing very strange things with descriptor numbers. */
428 if (num_heads
> vq
->vring
.num
) {
429 virtio_error(vq
->vdev
, "Guest moved used index from %u to %u",
430 idx
, vq
->shadow_avail_idx
);
433 /* On success, callers read a descriptor at vq->last_avail_idx.
434 * Make sure descriptor read does not bypass avail index read. */
442 static bool virtqueue_get_head(VirtQueue
*vq
, unsigned int idx
,
445 /* Grab the next descriptor number they're advertising, and increment
446 * the index we've seen. */
447 *head
= vring_avail_ring(vq
, idx
% vq
->vring
.num
);
449 /* If their number is silly, that's a fatal mistake. */
450 if (*head
>= vq
->vring
.num
) {
451 virtio_error(vq
->vdev
, "Guest says index %u is available", *head
);
459 VIRTQUEUE_READ_DESC_ERROR
= -1,
460 VIRTQUEUE_READ_DESC_DONE
= 0, /* end of chain */
461 VIRTQUEUE_READ_DESC_MORE
= 1, /* more buffers in chain */
464 static int virtqueue_read_next_desc(VirtIODevice
*vdev
, VRingDesc
*desc
,
465 MemoryRegionCache
*desc_cache
, unsigned int max
,
468 /* If this descriptor says it doesn't chain, we're done. */
469 if (!(desc
->flags
& VRING_DESC_F_NEXT
)) {
470 return VIRTQUEUE_READ_DESC_DONE
;
473 /* Check they're not leading us off end of descriptors. */
475 /* Make sure compiler knows to grab that: we don't want it changing! */
479 virtio_error(vdev
, "Desc next is %u", *next
);
480 return VIRTQUEUE_READ_DESC_ERROR
;
483 vring_desc_read(vdev
, desc
, desc_cache
, *next
);
484 return VIRTQUEUE_READ_DESC_MORE
;
487 void virtqueue_get_avail_bytes(VirtQueue
*vq
, unsigned int *in_bytes
,
488 unsigned int *out_bytes
,
489 unsigned max_in_bytes
, unsigned max_out_bytes
)
491 VirtIODevice
*vdev
= vq
->vdev
;
492 unsigned int max
, idx
;
493 unsigned int total_bufs
, in_total
, out_total
;
494 VRingMemoryRegionCaches
*caches
;
495 MemoryRegionCache indirect_desc_cache
= MEMORY_REGION_CACHE_INVALID
;
500 idx
= vq
->last_avail_idx
;
501 total_bufs
= in_total
= out_total
= 0;
504 caches
= atomic_rcu_read(&vq
->vring
.caches
);
505 if (caches
->desc
.len
< max
* sizeof(VRingDesc
)) {
506 virtio_error(vdev
, "Cannot map descriptor ring");
510 while ((rc
= virtqueue_num_heads(vq
, idx
)) > 0) {
511 MemoryRegionCache
*desc_cache
= &caches
->desc
;
512 unsigned int num_bufs
;
516 num_bufs
= total_bufs
;
518 if (!virtqueue_get_head(vq
, idx
++, &i
)) {
522 vring_desc_read(vdev
, &desc
, desc_cache
, i
);
524 if (desc
.flags
& VRING_DESC_F_INDIRECT
) {
525 if (desc
.len
% sizeof(VRingDesc
)) {
526 virtio_error(vdev
, "Invalid size for indirect buffer table");
530 /* If we've got too many, that implies a descriptor loop. */
531 if (num_bufs
>= max
) {
532 virtio_error(vdev
, "Looped descriptor");
536 /* loop over the indirect descriptor table */
537 len
= address_space_cache_init(&indirect_desc_cache
,
539 desc
.addr
, desc
.len
, false);
540 desc_cache
= &indirect_desc_cache
;
541 if (len
< desc
.len
) {
542 virtio_error(vdev
, "Cannot map indirect buffer");
546 max
= desc
.len
/ sizeof(VRingDesc
);
548 vring_desc_read(vdev
, &desc
, desc_cache
, i
);
552 /* If we've got too many, that implies a descriptor loop. */
553 if (++num_bufs
> max
) {
554 virtio_error(vdev
, "Looped descriptor");
558 if (desc
.flags
& VRING_DESC_F_WRITE
) {
559 in_total
+= desc
.len
;
561 out_total
+= desc
.len
;
563 if (in_total
>= max_in_bytes
&& out_total
>= max_out_bytes
) {
567 rc
= virtqueue_read_next_desc(vdev
, &desc
, desc_cache
, max
, &i
);
568 } while (rc
== VIRTQUEUE_READ_DESC_MORE
);
570 if (rc
== VIRTQUEUE_READ_DESC_ERROR
) {
574 if (desc_cache
== &indirect_desc_cache
) {
575 address_space_cache_destroy(&indirect_desc_cache
);
578 total_bufs
= num_bufs
;
587 address_space_cache_destroy(&indirect_desc_cache
);
589 *in_bytes
= in_total
;
592 *out_bytes
= out_total
;
598 in_total
= out_total
= 0;
602 int virtqueue_avail_bytes(VirtQueue
*vq
, unsigned int in_bytes
,
603 unsigned int out_bytes
)
605 unsigned int in_total
, out_total
;
607 virtqueue_get_avail_bytes(vq
, &in_total
, &out_total
, in_bytes
, out_bytes
);
608 return in_bytes
<= in_total
&& out_bytes
<= out_total
;
611 static bool virtqueue_map_desc(VirtIODevice
*vdev
, unsigned int *p_num_sg
,
612 hwaddr
*addr
, struct iovec
*iov
,
613 unsigned int max_num_sg
, bool is_write
,
614 hwaddr pa
, size_t sz
)
617 unsigned num_sg
= *p_num_sg
;
618 assert(num_sg
<= max_num_sg
);
621 virtio_error(vdev
, "virtio: zero sized buffers are not allowed");
628 if (num_sg
== max_num_sg
) {
629 virtio_error(vdev
, "virtio: too many write descriptors in "
634 iov
[num_sg
].iov_base
= dma_memory_map(vdev
->dma_as
, pa
, &len
,
636 DMA_DIRECTION_FROM_DEVICE
:
637 DMA_DIRECTION_TO_DEVICE
);
638 if (!iov
[num_sg
].iov_base
) {
639 virtio_error(vdev
, "virtio: bogus descriptor or out of resources");
643 iov
[num_sg
].iov_len
= len
;
657 /* Only used by error code paths before we have a VirtQueueElement (therefore
658 * virtqueue_unmap_sg() can't be used). Assumes buffers weren't written to
661 static void virtqueue_undo_map_desc(unsigned int out_num
, unsigned int in_num
,
666 for (i
= 0; i
< out_num
+ in_num
; i
++) {
667 int is_write
= i
>= out_num
;
669 cpu_physical_memory_unmap(iov
->iov_base
, iov
->iov_len
, is_write
, 0);
674 static void virtqueue_map_iovec(VirtIODevice
*vdev
, struct iovec
*sg
,
675 hwaddr
*addr
, unsigned int *num_sg
,
681 for (i
= 0; i
< *num_sg
; i
++) {
683 sg
[i
].iov_base
= dma_memory_map(vdev
->dma_as
,
684 addr
[i
], &len
, is_write
?
685 DMA_DIRECTION_FROM_DEVICE
:
686 DMA_DIRECTION_TO_DEVICE
);
687 if (!sg
[i
].iov_base
) {
688 error_report("virtio: error trying to map MMIO memory");
691 if (len
!= sg
[i
].iov_len
) {
692 error_report("virtio: unexpected memory split");
698 void virtqueue_map(VirtIODevice
*vdev
, VirtQueueElement
*elem
)
700 virtqueue_map_iovec(vdev
, elem
->in_sg
, elem
->in_addr
, &elem
->in_num
, 1);
701 virtqueue_map_iovec(vdev
, elem
->out_sg
, elem
->out_addr
, &elem
->out_num
, 0);
704 static void *virtqueue_alloc_element(size_t sz
, unsigned out_num
, unsigned in_num
)
706 VirtQueueElement
*elem
;
707 size_t in_addr_ofs
= QEMU_ALIGN_UP(sz
, __alignof__(elem
->in_addr
[0]));
708 size_t out_addr_ofs
= in_addr_ofs
+ in_num
* sizeof(elem
->in_addr
[0]);
709 size_t out_addr_end
= out_addr_ofs
+ out_num
* sizeof(elem
->out_addr
[0]);
710 size_t in_sg_ofs
= QEMU_ALIGN_UP(out_addr_end
, __alignof__(elem
->in_sg
[0]));
711 size_t out_sg_ofs
= in_sg_ofs
+ in_num
* sizeof(elem
->in_sg
[0]);
712 size_t out_sg_end
= out_sg_ofs
+ out_num
* sizeof(elem
->out_sg
[0]);
714 assert(sz
>= sizeof(VirtQueueElement
));
715 elem
= g_malloc(out_sg_end
);
716 elem
->out_num
= out_num
;
717 elem
->in_num
= in_num
;
718 elem
->in_addr
= (void *)elem
+ in_addr_ofs
;
719 elem
->out_addr
= (void *)elem
+ out_addr_ofs
;
720 elem
->in_sg
= (void *)elem
+ in_sg_ofs
;
721 elem
->out_sg
= (void *)elem
+ out_sg_ofs
;
725 void *virtqueue_pop(VirtQueue
*vq
, size_t sz
)
727 unsigned int i
, head
, max
;
728 VRingMemoryRegionCaches
*caches
;
729 MemoryRegionCache indirect_desc_cache
= MEMORY_REGION_CACHE_INVALID
;
730 MemoryRegionCache
*desc_cache
;
732 VirtIODevice
*vdev
= vq
->vdev
;
733 VirtQueueElement
*elem
= NULL
;
734 unsigned out_num
, in_num
;
735 hwaddr addr
[VIRTQUEUE_MAX_SIZE
];
736 struct iovec iov
[VIRTQUEUE_MAX_SIZE
];
740 if (unlikely(vdev
->broken
)) {
743 if (virtio_queue_empty(vq
)) {
746 /* Needed after virtio_queue_empty(), see comment in
747 * virtqueue_num_heads(). */
750 /* When we start there are none of either input nor output. */
751 out_num
= in_num
= 0;
755 if (vq
->inuse
>= vq
->vring
.num
) {
756 virtio_error(vdev
, "Virtqueue size exceeded");
760 if (!virtqueue_get_head(vq
, vq
->last_avail_idx
++, &head
)) {
764 if (virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
765 vring_set_avail_event(vq
, vq
->last_avail_idx
);
771 caches
= atomic_rcu_read(&vq
->vring
.caches
);
772 if (caches
->desc
.len
< max
* sizeof(VRingDesc
)) {
773 virtio_error(vdev
, "Cannot map descriptor ring");
777 desc_cache
= &caches
->desc
;
778 vring_desc_read(vdev
, &desc
, desc_cache
, i
);
779 if (desc
.flags
& VRING_DESC_F_INDIRECT
) {
780 if (desc
.len
% sizeof(VRingDesc
)) {
781 virtio_error(vdev
, "Invalid size for indirect buffer table");
785 /* loop over the indirect descriptor table */
786 len
= address_space_cache_init(&indirect_desc_cache
, vdev
->dma_as
,
787 desc
.addr
, desc
.len
, false);
788 desc_cache
= &indirect_desc_cache
;
789 if (len
< desc
.len
) {
790 virtio_error(vdev
, "Cannot map indirect buffer");
794 max
= desc
.len
/ sizeof(VRingDesc
);
796 vring_desc_read(vdev
, &desc
, desc_cache
, i
);
799 /* Collect all the descriptors */
803 if (desc
.flags
& VRING_DESC_F_WRITE
) {
804 map_ok
= virtqueue_map_desc(vdev
, &in_num
, addr
+ out_num
,
806 VIRTQUEUE_MAX_SIZE
- out_num
, true,
807 desc
.addr
, desc
.len
);
810 virtio_error(vdev
, "Incorrect order for descriptors");
813 map_ok
= virtqueue_map_desc(vdev
, &out_num
, addr
, iov
,
814 VIRTQUEUE_MAX_SIZE
, false,
815 desc
.addr
, desc
.len
);
821 /* If we've got too many, that implies a descriptor loop. */
822 if ((in_num
+ out_num
) > max
) {
823 virtio_error(vdev
, "Looped descriptor");
827 rc
= virtqueue_read_next_desc(vdev
, &desc
, desc_cache
, max
, &i
);
828 } while (rc
== VIRTQUEUE_READ_DESC_MORE
);
830 if (rc
== VIRTQUEUE_READ_DESC_ERROR
) {
834 /* Now copy what we have collected and mapped */
835 elem
= virtqueue_alloc_element(sz
, out_num
, in_num
);
837 for (i
= 0; i
< out_num
; i
++) {
838 elem
->out_addr
[i
] = addr
[i
];
839 elem
->out_sg
[i
] = iov
[i
];
841 for (i
= 0; i
< in_num
; i
++) {
842 elem
->in_addr
[i
] = addr
[out_num
+ i
];
843 elem
->in_sg
[i
] = iov
[out_num
+ i
];
848 trace_virtqueue_pop(vq
, elem
, elem
->in_num
, elem
->out_num
);
850 address_space_cache_destroy(&indirect_desc_cache
);
856 virtqueue_undo_map_desc(out_num
, in_num
, iov
);
860 /* virtqueue_drop_all:
861 * @vq: The #VirtQueue
862 * Drops all queued buffers and indicates them to the guest
863 * as if they are done. Useful when buffers can not be
864 * processed but must be returned to the guest.
866 unsigned int virtqueue_drop_all(VirtQueue
*vq
)
868 unsigned int dropped
= 0;
869 VirtQueueElement elem
= {};
870 VirtIODevice
*vdev
= vq
->vdev
;
871 bool fEventIdx
= virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
);
873 if (unlikely(vdev
->broken
)) {
877 while (!virtio_queue_empty(vq
) && vq
->inuse
< vq
->vring
.num
) {
878 /* works similar to virtqueue_pop but does not map buffers
879 * and does not allocate any memory */
881 if (!virtqueue_get_head(vq
, vq
->last_avail_idx
, &elem
.index
)) {
885 vq
->last_avail_idx
++;
887 vring_set_avail_event(vq
, vq
->last_avail_idx
);
889 /* immediately push the element, nothing to unmap
890 * as both in_num and out_num are set to 0 */
891 virtqueue_push(vq
, &elem
, 0);
898 /* Reading and writing a structure directly to QEMUFile is *awful*, but
899 * it is what QEMU has always done by mistake. We can change it sooner
900 * or later by bumping the version number of the affected vm states.
901 * In the meanwhile, since the in-memory layout of VirtQueueElement
902 * has changed, we need to marshal to and from the layout that was
903 * used before the change.
905 typedef struct VirtQueueElementOld
{
907 unsigned int out_num
;
909 hwaddr in_addr
[VIRTQUEUE_MAX_SIZE
];
910 hwaddr out_addr
[VIRTQUEUE_MAX_SIZE
];
911 struct iovec in_sg
[VIRTQUEUE_MAX_SIZE
];
912 struct iovec out_sg
[VIRTQUEUE_MAX_SIZE
];
913 } VirtQueueElementOld
;
915 void *qemu_get_virtqueue_element(VirtIODevice
*vdev
, QEMUFile
*f
, size_t sz
)
917 VirtQueueElement
*elem
;
918 VirtQueueElementOld data
;
921 qemu_get_buffer(f
, (uint8_t *)&data
, sizeof(VirtQueueElementOld
));
923 /* TODO: teach all callers that this can fail, and return failure instead
925 * When we do, we might be able to re-enable NDEBUG below.
928 #error building with NDEBUG is not supported
930 assert(ARRAY_SIZE(data
.in_addr
) >= data
.in_num
);
931 assert(ARRAY_SIZE(data
.out_addr
) >= data
.out_num
);
933 elem
= virtqueue_alloc_element(sz
, data
.out_num
, data
.in_num
);
934 elem
->index
= data
.index
;
936 for (i
= 0; i
< elem
->in_num
; i
++) {
937 elem
->in_addr
[i
] = data
.in_addr
[i
];
940 for (i
= 0; i
< elem
->out_num
; i
++) {
941 elem
->out_addr
[i
] = data
.out_addr
[i
];
944 for (i
= 0; i
< elem
->in_num
; i
++) {
945 /* Base is overwritten by virtqueue_map. */
946 elem
->in_sg
[i
].iov_base
= 0;
947 elem
->in_sg
[i
].iov_len
= data
.in_sg
[i
].iov_len
;
950 for (i
= 0; i
< elem
->out_num
; i
++) {
951 /* Base is overwritten by virtqueue_map. */
952 elem
->out_sg
[i
].iov_base
= 0;
953 elem
->out_sg
[i
].iov_len
= data
.out_sg
[i
].iov_len
;
956 virtqueue_map(vdev
, elem
);
960 void qemu_put_virtqueue_element(QEMUFile
*f
, VirtQueueElement
*elem
)
962 VirtQueueElementOld data
;
965 memset(&data
, 0, sizeof(data
));
966 data
.index
= elem
->index
;
967 data
.in_num
= elem
->in_num
;
968 data
.out_num
= elem
->out_num
;
970 for (i
= 0; i
< elem
->in_num
; i
++) {
971 data
.in_addr
[i
] = elem
->in_addr
[i
];
974 for (i
= 0; i
< elem
->out_num
; i
++) {
975 data
.out_addr
[i
] = elem
->out_addr
[i
];
978 for (i
= 0; i
< elem
->in_num
; i
++) {
979 /* Base is overwritten by virtqueue_map when loading. Do not
980 * save it, as it would leak the QEMU address space layout. */
981 data
.in_sg
[i
].iov_len
= elem
->in_sg
[i
].iov_len
;
984 for (i
= 0; i
< elem
->out_num
; i
++) {
985 /* Do not save iov_base as above. */
986 data
.out_sg
[i
].iov_len
= elem
->out_sg
[i
].iov_len
;
988 qemu_put_buffer(f
, (uint8_t *)&data
, sizeof(VirtQueueElementOld
));
992 static void virtio_notify_vector(VirtIODevice
*vdev
, uint16_t vector
)
994 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
995 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
997 if (unlikely(vdev
->broken
)) {
1002 k
->notify(qbus
->parent
, vector
);
1006 void virtio_update_irq(VirtIODevice
*vdev
)
1008 virtio_notify_vector(vdev
, VIRTIO_NO_VECTOR
);
1011 static int virtio_validate_features(VirtIODevice
*vdev
)
1013 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1015 if (virtio_host_has_feature(vdev
, VIRTIO_F_IOMMU_PLATFORM
) &&
1016 !virtio_vdev_has_feature(vdev
, VIRTIO_F_IOMMU_PLATFORM
)) {
1020 if (k
->validate_features
) {
1021 return k
->validate_features(vdev
);
1027 int virtio_set_status(VirtIODevice
*vdev
, uint8_t val
)
1029 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1030 trace_virtio_set_status(vdev
, val
);
1032 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
1033 if (!(vdev
->status
& VIRTIO_CONFIG_S_FEATURES_OK
) &&
1034 val
& VIRTIO_CONFIG_S_FEATURES_OK
) {
1035 int ret
= virtio_validate_features(vdev
);
1042 if (k
->set_status
) {
1043 k
->set_status(vdev
, val
);
1049 bool target_words_bigendian(void);
1050 static enum virtio_device_endian
virtio_default_endian(void)
1052 if (target_words_bigendian()) {
1053 return VIRTIO_DEVICE_ENDIAN_BIG
;
1055 return VIRTIO_DEVICE_ENDIAN_LITTLE
;
1059 static enum virtio_device_endian
virtio_current_cpu_endian(void)
1061 CPUClass
*cc
= CPU_GET_CLASS(current_cpu
);
1063 if (cc
->virtio_is_big_endian(current_cpu
)) {
1064 return VIRTIO_DEVICE_ENDIAN_BIG
;
1066 return VIRTIO_DEVICE_ENDIAN_LITTLE
;
1070 void virtio_reset(void *opaque
)
1072 VirtIODevice
*vdev
= opaque
;
1073 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1076 virtio_set_status(vdev
, 0);
1078 /* Guest initiated reset */
1079 vdev
->device_endian
= virtio_current_cpu_endian();
1082 vdev
->device_endian
= virtio_default_endian();
1089 vdev
->broken
= false;
1090 vdev
->guest_features
= 0;
1091 vdev
->queue_sel
= 0;
1093 atomic_set(&vdev
->isr
, 0);
1094 vdev
->config_vector
= VIRTIO_NO_VECTOR
;
1095 virtio_notify_vector(vdev
, vdev
->config_vector
);
1097 for(i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
1098 vdev
->vq
[i
].vring
.desc
= 0;
1099 vdev
->vq
[i
].vring
.avail
= 0;
1100 vdev
->vq
[i
].vring
.used
= 0;
1101 vdev
->vq
[i
].last_avail_idx
= 0;
1102 vdev
->vq
[i
].shadow_avail_idx
= 0;
1103 vdev
->vq
[i
].used_idx
= 0;
1104 virtio_queue_set_vector(vdev
, i
, VIRTIO_NO_VECTOR
);
1105 vdev
->vq
[i
].signalled_used
= 0;
1106 vdev
->vq
[i
].signalled_used_valid
= false;
1107 vdev
->vq
[i
].notification
= true;
1108 vdev
->vq
[i
].vring
.num
= vdev
->vq
[i
].vring
.num_default
;
1109 vdev
->vq
[i
].inuse
= 0;
1113 uint32_t virtio_config_readb(VirtIODevice
*vdev
, uint32_t addr
)
1115 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1118 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1119 return (uint32_t)-1;
1122 k
->get_config(vdev
, vdev
->config
);
1124 val
= ldub_p(vdev
->config
+ addr
);
1128 uint32_t virtio_config_readw(VirtIODevice
*vdev
, uint32_t addr
)
1130 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1133 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1134 return (uint32_t)-1;
1137 k
->get_config(vdev
, vdev
->config
);
1139 val
= lduw_p(vdev
->config
+ addr
);
1143 uint32_t virtio_config_readl(VirtIODevice
*vdev
, uint32_t addr
)
1145 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1148 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1149 return (uint32_t)-1;
1152 k
->get_config(vdev
, vdev
->config
);
1154 val
= ldl_p(vdev
->config
+ addr
);
1158 void virtio_config_writeb(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
1160 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1163 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1167 stb_p(vdev
->config
+ addr
, val
);
1169 if (k
->set_config
) {
1170 k
->set_config(vdev
, vdev
->config
);
1174 void virtio_config_writew(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
1176 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1177 uint16_t val
= data
;
1179 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1183 stw_p(vdev
->config
+ addr
, val
);
1185 if (k
->set_config
) {
1186 k
->set_config(vdev
, vdev
->config
);
1190 void virtio_config_writel(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
1192 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1193 uint32_t val
= data
;
1195 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1199 stl_p(vdev
->config
+ addr
, val
);
1201 if (k
->set_config
) {
1202 k
->set_config(vdev
, vdev
->config
);
1206 uint32_t virtio_config_modern_readb(VirtIODevice
*vdev
, uint32_t addr
)
1208 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1211 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1212 return (uint32_t)-1;
1215 k
->get_config(vdev
, vdev
->config
);
1217 val
= ldub_p(vdev
->config
+ addr
);
1221 uint32_t virtio_config_modern_readw(VirtIODevice
*vdev
, uint32_t addr
)
1223 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1226 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1227 return (uint32_t)-1;
1230 k
->get_config(vdev
, vdev
->config
);
1232 val
= lduw_le_p(vdev
->config
+ addr
);
1236 uint32_t virtio_config_modern_readl(VirtIODevice
*vdev
, uint32_t addr
)
1238 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1241 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1242 return (uint32_t)-1;
1245 k
->get_config(vdev
, vdev
->config
);
1247 val
= ldl_le_p(vdev
->config
+ addr
);
1251 void virtio_config_modern_writeb(VirtIODevice
*vdev
,
1252 uint32_t addr
, uint32_t data
)
1254 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1257 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1261 stb_p(vdev
->config
+ addr
, val
);
1263 if (k
->set_config
) {
1264 k
->set_config(vdev
, vdev
->config
);
1268 void virtio_config_modern_writew(VirtIODevice
*vdev
,
1269 uint32_t addr
, uint32_t data
)
1271 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1272 uint16_t val
= data
;
1274 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1278 stw_le_p(vdev
->config
+ addr
, val
);
1280 if (k
->set_config
) {
1281 k
->set_config(vdev
, vdev
->config
);
1285 void virtio_config_modern_writel(VirtIODevice
*vdev
,
1286 uint32_t addr
, uint32_t data
)
1288 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1289 uint32_t val
= data
;
1291 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1295 stl_le_p(vdev
->config
+ addr
, val
);
1297 if (k
->set_config
) {
1298 k
->set_config(vdev
, vdev
->config
);
1302 void virtio_queue_set_addr(VirtIODevice
*vdev
, int n
, hwaddr addr
)
1304 vdev
->vq
[n
].vring
.desc
= addr
;
1305 virtio_queue_update_rings(vdev
, n
);
1308 hwaddr
virtio_queue_get_addr(VirtIODevice
*vdev
, int n
)
1310 return vdev
->vq
[n
].vring
.desc
;
1313 void virtio_queue_set_rings(VirtIODevice
*vdev
, int n
, hwaddr desc
,
1314 hwaddr avail
, hwaddr used
)
1316 vdev
->vq
[n
].vring
.desc
= desc
;
1317 vdev
->vq
[n
].vring
.avail
= avail
;
1318 vdev
->vq
[n
].vring
.used
= used
;
1319 virtio_init_region_cache(vdev
, n
);
1322 void virtio_queue_set_num(VirtIODevice
*vdev
, int n
, int num
)
1324 /* Don't allow guest to flip queue between existent and
1325 * nonexistent states, or to set it to an invalid size.
1327 if (!!num
!= !!vdev
->vq
[n
].vring
.num
||
1328 num
> VIRTQUEUE_MAX_SIZE
||
1332 vdev
->vq
[n
].vring
.num
= num
;
1335 VirtQueue
*virtio_vector_first_queue(VirtIODevice
*vdev
, uint16_t vector
)
1337 return QLIST_FIRST(&vdev
->vector_queues
[vector
]);
1340 VirtQueue
*virtio_vector_next_queue(VirtQueue
*vq
)
1342 return QLIST_NEXT(vq
, node
);
1345 int virtio_queue_get_num(VirtIODevice
*vdev
, int n
)
1347 return vdev
->vq
[n
].vring
.num
;
1350 int virtio_queue_get_max_num(VirtIODevice
*vdev
, int n
)
1352 return vdev
->vq
[n
].vring
.num_default
;
1355 int virtio_get_num_queues(VirtIODevice
*vdev
)
1359 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
1360 if (!virtio_queue_get_num(vdev
, i
)) {
1368 void virtio_queue_set_align(VirtIODevice
*vdev
, int n
, int align
)
1370 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1371 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1373 /* virtio-1 compliant devices cannot change the alignment */
1374 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
1375 error_report("tried to modify queue alignment for virtio-1 device");
1378 /* Check that the transport told us it was going to do this
1379 * (so a buggy transport will immediately assert rather than
1380 * silently failing to migrate this state)
1382 assert(k
->has_variable_vring_alignment
);
1384 vdev
->vq
[n
].vring
.align
= align
;
1385 virtio_queue_update_rings(vdev
, n
);
1388 static bool virtio_queue_notify_aio_vq(VirtQueue
*vq
)
1390 if (vq
->vring
.desc
&& vq
->handle_aio_output
) {
1391 VirtIODevice
*vdev
= vq
->vdev
;
1393 trace_virtio_queue_notify(vdev
, vq
- vdev
->vq
, vq
);
1394 return vq
->handle_aio_output(vdev
, vq
);
1400 static void virtio_queue_notify_vq(VirtQueue
*vq
)
1402 if (vq
->vring
.desc
&& vq
->handle_output
) {
1403 VirtIODevice
*vdev
= vq
->vdev
;
1405 if (unlikely(vdev
->broken
)) {
1409 trace_virtio_queue_notify(vdev
, vq
- vdev
->vq
, vq
);
1410 vq
->handle_output(vdev
, vq
);
1414 void virtio_queue_notify(VirtIODevice
*vdev
, int n
)
1416 virtio_queue_notify_vq(&vdev
->vq
[n
]);
1419 uint16_t virtio_queue_vector(VirtIODevice
*vdev
, int n
)
1421 return n
< VIRTIO_QUEUE_MAX
? vdev
->vq
[n
].vector
:
1425 void virtio_queue_set_vector(VirtIODevice
*vdev
, int n
, uint16_t vector
)
1427 VirtQueue
*vq
= &vdev
->vq
[n
];
1429 if (n
< VIRTIO_QUEUE_MAX
) {
1430 if (vdev
->vector_queues
&&
1431 vdev
->vq
[n
].vector
!= VIRTIO_NO_VECTOR
) {
1432 QLIST_REMOVE(vq
, node
);
1434 vdev
->vq
[n
].vector
= vector
;
1435 if (vdev
->vector_queues
&&
1436 vector
!= VIRTIO_NO_VECTOR
) {
1437 QLIST_INSERT_HEAD(&vdev
->vector_queues
[vector
], vq
, node
);
1442 VirtQueue
*virtio_add_queue(VirtIODevice
*vdev
, int queue_size
,
1443 VirtIOHandleOutput handle_output
)
1447 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
1448 if (vdev
->vq
[i
].vring
.num
== 0)
1452 if (i
== VIRTIO_QUEUE_MAX
|| queue_size
> VIRTQUEUE_MAX_SIZE
)
1455 vdev
->vq
[i
].vring
.num
= queue_size
;
1456 vdev
->vq
[i
].vring
.num_default
= queue_size
;
1457 vdev
->vq
[i
].vring
.align
= VIRTIO_PCI_VRING_ALIGN
;
1458 vdev
->vq
[i
].handle_output
= handle_output
;
1459 vdev
->vq
[i
].handle_aio_output
= NULL
;
1461 return &vdev
->vq
[i
];
1464 void virtio_del_queue(VirtIODevice
*vdev
, int n
)
1466 if (n
< 0 || n
>= VIRTIO_QUEUE_MAX
) {
1470 vdev
->vq
[n
].vring
.num
= 0;
1471 vdev
->vq
[n
].vring
.num_default
= 0;
1474 static void virtio_set_isr(VirtIODevice
*vdev
, int value
)
1476 uint8_t old
= atomic_read(&vdev
->isr
);
1478 /* Do not write ISR if it does not change, so that its cacheline remains
1479 * shared in the common case where the guest does not read it.
1481 if ((old
& value
) != value
) {
1482 atomic_or(&vdev
->isr
, value
);
1486 static bool virtio_should_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
1490 /* We need to expose used array entries before checking used event. */
1492 /* Always notify when queue is empty (when feature acknowledge) */
1493 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_NOTIFY_ON_EMPTY
) &&
1494 !vq
->inuse
&& virtio_queue_empty(vq
)) {
1498 if (!virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
1499 return !(vring_avail_flags(vq
) & VRING_AVAIL_F_NO_INTERRUPT
);
1502 v
= vq
->signalled_used_valid
;
1503 vq
->signalled_used_valid
= true;
1504 old
= vq
->signalled_used
;
1505 new = vq
->signalled_used
= vq
->used_idx
;
1506 return !v
|| vring_need_event(vring_get_used_event(vq
), new, old
);
1509 void virtio_notify_irqfd(VirtIODevice
*vdev
, VirtQueue
*vq
)
1511 if (!virtio_should_notify(vdev
, vq
)) {
1515 trace_virtio_notify_irqfd(vdev
, vq
);
1518 * virtio spec 1.0 says ISR bit 0 should be ignored with MSI, but
1519 * windows drivers included in virtio-win 1.8.0 (circa 2015) are
1520 * incorrectly polling this bit during crashdump and hibernation
1521 * in MSI mode, causing a hang if this bit is never updated.
1522 * Recent releases of Windows do not really shut down, but rather
1523 * log out and hibernate to make the next startup faster. Hence,
1524 * this manifested as a more serious hang during shutdown with
1526 * Next driver release from 2016 fixed this problem, so working around it
1527 * is not a must, but it's easy to do so let's do it here.
1529 * Note: it's safe to update ISR from any thread as it was switched
1530 * to an atomic operation.
1532 virtio_set_isr(vq
->vdev
, 0x1);
1533 event_notifier_set(&vq
->guest_notifier
);
1536 void virtio_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
1538 if (!virtio_should_notify(vdev
, vq
)) {
1542 trace_virtio_notify(vdev
, vq
);
1543 virtio_set_isr(vq
->vdev
, 0x1);
1544 virtio_notify_vector(vdev
, vq
->vector
);
1547 void virtio_notify_config(VirtIODevice
*vdev
)
1549 if (!(vdev
->status
& VIRTIO_CONFIG_S_DRIVER_OK
))
1552 virtio_set_isr(vdev
, 0x3);
1554 virtio_notify_vector(vdev
, vdev
->config_vector
);
1557 static bool virtio_device_endian_needed(void *opaque
)
1559 VirtIODevice
*vdev
= opaque
;
1561 assert(vdev
->device_endian
!= VIRTIO_DEVICE_ENDIAN_UNKNOWN
);
1562 if (!virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
1563 return vdev
->device_endian
!= virtio_default_endian();
1565 /* Devices conforming to VIRTIO 1.0 or later are always LE. */
1566 return vdev
->device_endian
!= VIRTIO_DEVICE_ENDIAN_LITTLE
;
1569 static bool virtio_64bit_features_needed(void *opaque
)
1571 VirtIODevice
*vdev
= opaque
;
1573 return (vdev
->host_features
>> 32) != 0;
1576 static bool virtio_virtqueue_needed(void *opaque
)
1578 VirtIODevice
*vdev
= opaque
;
1580 return virtio_host_has_feature(vdev
, VIRTIO_F_VERSION_1
);
1583 static bool virtio_ringsize_needed(void *opaque
)
1585 VirtIODevice
*vdev
= opaque
;
1588 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
1589 if (vdev
->vq
[i
].vring
.num
!= vdev
->vq
[i
].vring
.num_default
) {
1596 static bool virtio_extra_state_needed(void *opaque
)
1598 VirtIODevice
*vdev
= opaque
;
1599 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1600 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1602 return k
->has_extra_state
&&
1603 k
->has_extra_state(qbus
->parent
);
1606 static bool virtio_broken_needed(void *opaque
)
1608 VirtIODevice
*vdev
= opaque
;
1610 return vdev
->broken
;
1613 static const VMStateDescription vmstate_virtqueue
= {
1614 .name
= "virtqueue_state",
1616 .minimum_version_id
= 1,
1617 .fields
= (VMStateField
[]) {
1618 VMSTATE_UINT64(vring
.avail
, struct VirtQueue
),
1619 VMSTATE_UINT64(vring
.used
, struct VirtQueue
),
1620 VMSTATE_END_OF_LIST()
1624 static const VMStateDescription vmstate_virtio_virtqueues
= {
1625 .name
= "virtio/virtqueues",
1627 .minimum_version_id
= 1,
1628 .needed
= &virtio_virtqueue_needed
,
1629 .fields
= (VMStateField
[]) {
1630 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq
, struct VirtIODevice
,
1631 VIRTIO_QUEUE_MAX
, 0, vmstate_virtqueue
, VirtQueue
),
1632 VMSTATE_END_OF_LIST()
1636 static const VMStateDescription vmstate_ringsize
= {
1637 .name
= "ringsize_state",
1639 .minimum_version_id
= 1,
1640 .fields
= (VMStateField
[]) {
1641 VMSTATE_UINT32(vring
.num_default
, struct VirtQueue
),
1642 VMSTATE_END_OF_LIST()
1646 static const VMStateDescription vmstate_virtio_ringsize
= {
1647 .name
= "virtio/ringsize",
1649 .minimum_version_id
= 1,
1650 .needed
= &virtio_ringsize_needed
,
1651 .fields
= (VMStateField
[]) {
1652 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq
, struct VirtIODevice
,
1653 VIRTIO_QUEUE_MAX
, 0, vmstate_ringsize
, VirtQueue
),
1654 VMSTATE_END_OF_LIST()
1658 static int get_extra_state(QEMUFile
*f
, void *pv
, size_t size
,
1659 VMStateField
*field
)
1661 VirtIODevice
*vdev
= pv
;
1662 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1663 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1665 if (!k
->load_extra_state
) {
1668 return k
->load_extra_state(qbus
->parent
, f
);
1672 static int put_extra_state(QEMUFile
*f
, void *pv
, size_t size
,
1673 VMStateField
*field
, QJSON
*vmdesc
)
1675 VirtIODevice
*vdev
= pv
;
1676 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1677 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1679 k
->save_extra_state(qbus
->parent
, f
);
1683 static const VMStateInfo vmstate_info_extra_state
= {
1684 .name
= "virtqueue_extra_state",
1685 .get
= get_extra_state
,
1686 .put
= put_extra_state
,
1689 static const VMStateDescription vmstate_virtio_extra_state
= {
1690 .name
= "virtio/extra_state",
1692 .minimum_version_id
= 1,
1693 .needed
= &virtio_extra_state_needed
,
1694 .fields
= (VMStateField
[]) {
1696 .name
= "extra_state",
1698 .field_exists
= NULL
,
1700 .info
= &vmstate_info_extra_state
,
1701 .flags
= VMS_SINGLE
,
1704 VMSTATE_END_OF_LIST()
1708 static const VMStateDescription vmstate_virtio_device_endian
= {
1709 .name
= "virtio/device_endian",
1711 .minimum_version_id
= 1,
1712 .needed
= &virtio_device_endian_needed
,
1713 .fields
= (VMStateField
[]) {
1714 VMSTATE_UINT8(device_endian
, VirtIODevice
),
1715 VMSTATE_END_OF_LIST()
1719 static const VMStateDescription vmstate_virtio_64bit_features
= {
1720 .name
= "virtio/64bit_features",
1722 .minimum_version_id
= 1,
1723 .needed
= &virtio_64bit_features_needed
,
1724 .fields
= (VMStateField
[]) {
1725 VMSTATE_UINT64(guest_features
, VirtIODevice
),
1726 VMSTATE_END_OF_LIST()
1730 static const VMStateDescription vmstate_virtio_broken
= {
1731 .name
= "virtio/broken",
1733 .minimum_version_id
= 1,
1734 .needed
= &virtio_broken_needed
,
1735 .fields
= (VMStateField
[]) {
1736 VMSTATE_BOOL(broken
, VirtIODevice
),
1737 VMSTATE_END_OF_LIST()
1741 static const VMStateDescription vmstate_virtio
= {
1744 .minimum_version_id
= 1,
1745 .minimum_version_id_old
= 1,
1746 .fields
= (VMStateField
[]) {
1747 VMSTATE_END_OF_LIST()
1749 .subsections
= (const VMStateDescription
*[]) {
1750 &vmstate_virtio_device_endian
,
1751 &vmstate_virtio_64bit_features
,
1752 &vmstate_virtio_virtqueues
,
1753 &vmstate_virtio_ringsize
,
1754 &vmstate_virtio_broken
,
1755 &vmstate_virtio_extra_state
,
1760 void virtio_save(VirtIODevice
*vdev
, QEMUFile
*f
)
1762 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1763 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1764 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1765 uint32_t guest_features_lo
= (vdev
->guest_features
& 0xffffffff);
1768 if (k
->save_config
) {
1769 k
->save_config(qbus
->parent
, f
);
1772 qemu_put_8s(f
, &vdev
->status
);
1773 qemu_put_8s(f
, &vdev
->isr
);
1774 qemu_put_be16s(f
, &vdev
->queue_sel
);
1775 qemu_put_be32s(f
, &guest_features_lo
);
1776 qemu_put_be32(f
, vdev
->config_len
);
1777 qemu_put_buffer(f
, vdev
->config
, vdev
->config_len
);
1779 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
1780 if (vdev
->vq
[i
].vring
.num
== 0)
1784 qemu_put_be32(f
, i
);
1786 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
1787 if (vdev
->vq
[i
].vring
.num
== 0)
1790 qemu_put_be32(f
, vdev
->vq
[i
].vring
.num
);
1791 if (k
->has_variable_vring_alignment
) {
1792 qemu_put_be32(f
, vdev
->vq
[i
].vring
.align
);
1794 /* XXX virtio-1 devices */
1795 qemu_put_be64(f
, vdev
->vq
[i
].vring
.desc
);
1796 qemu_put_be16s(f
, &vdev
->vq
[i
].last_avail_idx
);
1797 if (k
->save_queue
) {
1798 k
->save_queue(qbus
->parent
, i
, f
);
1802 if (vdc
->save
!= NULL
) {
1807 vmstate_save_state(f
, vdc
->vmsd
, vdev
, NULL
);
1811 vmstate_save_state(f
, &vmstate_virtio
, vdev
, NULL
);
1814 /* A wrapper for use as a VMState .put function */
1815 static int virtio_device_put(QEMUFile
*f
, void *opaque
, size_t size
,
1816 VMStateField
*field
, QJSON
*vmdesc
)
1818 virtio_save(VIRTIO_DEVICE(opaque
), f
);
1823 /* A wrapper for use as a VMState .get function */
1824 static int virtio_device_get(QEMUFile
*f
, void *opaque
, size_t size
,
1825 VMStateField
*field
)
1827 VirtIODevice
*vdev
= VIRTIO_DEVICE(opaque
);
1828 DeviceClass
*dc
= DEVICE_CLASS(VIRTIO_DEVICE_GET_CLASS(vdev
));
1830 return virtio_load(vdev
, f
, dc
->vmsd
->version_id
);
1833 const VMStateInfo virtio_vmstate_info
= {
1835 .get
= virtio_device_get
,
1836 .put
= virtio_device_put
,
1839 static int virtio_set_features_nocheck(VirtIODevice
*vdev
, uint64_t val
)
1841 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1842 bool bad
= (val
& ~(vdev
->host_features
)) != 0;
1844 val
&= vdev
->host_features
;
1845 if (k
->set_features
) {
1846 k
->set_features(vdev
, val
);
1848 vdev
->guest_features
= val
;
1849 return bad
? -1 : 0;
1852 int virtio_set_features(VirtIODevice
*vdev
, uint64_t val
)
1855 * The driver must not attempt to set features after feature negotiation
1858 if (vdev
->status
& VIRTIO_CONFIG_S_FEATURES_OK
) {
1861 return virtio_set_features_nocheck(vdev
, val
);
1864 int virtio_load(VirtIODevice
*vdev
, QEMUFile
*f
, int version_id
)
1870 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1871 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1872 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1875 * We poison the endianness to ensure it does not get used before
1876 * subsections have been loaded.
1878 vdev
->device_endian
= VIRTIO_DEVICE_ENDIAN_UNKNOWN
;
1880 if (k
->load_config
) {
1881 ret
= k
->load_config(qbus
->parent
, f
);
1886 qemu_get_8s(f
, &vdev
->status
);
1887 qemu_get_8s(f
, &vdev
->isr
);
1888 qemu_get_be16s(f
, &vdev
->queue_sel
);
1889 if (vdev
->queue_sel
>= VIRTIO_QUEUE_MAX
) {
1892 qemu_get_be32s(f
, &features
);
1895 * Temporarily set guest_features low bits - needed by
1896 * virtio net load code testing for VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
1897 * VIRTIO_NET_F_GUEST_ANNOUNCE and VIRTIO_NET_F_CTRL_VQ.
1899 * Note: devices should always test host features in future - don't create
1900 * new dependencies like this.
1902 vdev
->guest_features
= features
;
1904 config_len
= qemu_get_be32(f
);
1907 * There are cases where the incoming config can be bigger or smaller
1908 * than what we have; so load what we have space for, and skip
1909 * any excess that's in the stream.
1911 qemu_get_buffer(f
, vdev
->config
, MIN(config_len
, vdev
->config_len
));
1913 while (config_len
> vdev
->config_len
) {
1918 num
= qemu_get_be32(f
);
1920 if (num
> VIRTIO_QUEUE_MAX
) {
1921 error_report("Invalid number of virtqueues: 0x%x", num
);
1925 for (i
= 0; i
< num
; i
++) {
1926 vdev
->vq
[i
].vring
.num
= qemu_get_be32(f
);
1927 if (k
->has_variable_vring_alignment
) {
1928 vdev
->vq
[i
].vring
.align
= qemu_get_be32(f
);
1930 vdev
->vq
[i
].vring
.desc
= qemu_get_be64(f
);
1931 qemu_get_be16s(f
, &vdev
->vq
[i
].last_avail_idx
);
1932 vdev
->vq
[i
].signalled_used_valid
= false;
1933 vdev
->vq
[i
].notification
= true;
1935 if (vdev
->vq
[i
].vring
.desc
) {
1936 /* XXX virtio-1 devices */
1937 virtio_queue_update_rings(vdev
, i
);
1938 } else if (vdev
->vq
[i
].last_avail_idx
) {
1939 error_report("VQ %d address 0x0 "
1940 "inconsistent with Host index 0x%x",
1941 i
, vdev
->vq
[i
].last_avail_idx
);
1944 if (k
->load_queue
) {
1945 ret
= k
->load_queue(qbus
->parent
, i
, f
);
1951 virtio_notify_vector(vdev
, VIRTIO_NO_VECTOR
);
1953 if (vdc
->load
!= NULL
) {
1954 ret
= vdc
->load(vdev
, f
, version_id
);
1961 ret
= vmstate_load_state(f
, vdc
->vmsd
, vdev
, version_id
);
1968 ret
= vmstate_load_state(f
, &vmstate_virtio
, vdev
, 1);
1973 if (vdev
->device_endian
== VIRTIO_DEVICE_ENDIAN_UNKNOWN
) {
1974 vdev
->device_endian
= virtio_default_endian();
1977 if (virtio_64bit_features_needed(vdev
)) {
1979 * Subsection load filled vdev->guest_features. Run them
1980 * through virtio_set_features to sanity-check them against
1983 uint64_t features64
= vdev
->guest_features
;
1984 if (virtio_set_features_nocheck(vdev
, features64
) < 0) {
1985 error_report("Features 0x%" PRIx64
" unsupported. "
1986 "Allowed features: 0x%" PRIx64
,
1987 features64
, vdev
->host_features
);
1991 if (virtio_set_features_nocheck(vdev
, features
) < 0) {
1992 error_report("Features 0x%x unsupported. "
1993 "Allowed features: 0x%" PRIx64
,
1994 features
, vdev
->host_features
);
1999 for (i
= 0; i
< num
; i
++) {
2000 if (vdev
->vq
[i
].vring
.desc
) {
2002 nheads
= vring_avail_idx(&vdev
->vq
[i
]) - vdev
->vq
[i
].last_avail_idx
;
2003 /* Check it isn't doing strange things with descriptor numbers. */
2004 if (nheads
> vdev
->vq
[i
].vring
.num
) {
2005 error_report("VQ %d size 0x%x Guest index 0x%x "
2006 "inconsistent with Host index 0x%x: delta 0x%x",
2007 i
, vdev
->vq
[i
].vring
.num
,
2008 vring_avail_idx(&vdev
->vq
[i
]),
2009 vdev
->vq
[i
].last_avail_idx
, nheads
);
2012 vdev
->vq
[i
].used_idx
= vring_used_idx(&vdev
->vq
[i
]);
2013 vdev
->vq
[i
].shadow_avail_idx
= vring_avail_idx(&vdev
->vq
[i
]);
2016 * Some devices migrate VirtQueueElements that have been popped
2017 * from the avail ring but not yet returned to the used ring.
2018 * Since max ring size < UINT16_MAX it's safe to use modulo
2019 * UINT16_MAX + 1 subtraction.
2021 vdev
->vq
[i
].inuse
= (uint16_t)(vdev
->vq
[i
].last_avail_idx
-
2022 vdev
->vq
[i
].used_idx
);
2023 if (vdev
->vq
[i
].inuse
> vdev
->vq
[i
].vring
.num
) {
2024 error_report("VQ %d size 0x%x < last_avail_idx 0x%x - "
2026 i
, vdev
->vq
[i
].vring
.num
,
2027 vdev
->vq
[i
].last_avail_idx
,
2028 vdev
->vq
[i
].used_idx
);
2037 void virtio_cleanup(VirtIODevice
*vdev
)
2039 qemu_del_vm_change_state_handler(vdev
->vmstate
);
2042 static void virtio_vmstate_change(void *opaque
, int running
, RunState state
)
2044 VirtIODevice
*vdev
= opaque
;
2045 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2046 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2047 bool backend_run
= running
&& (vdev
->status
& VIRTIO_CONFIG_S_DRIVER_OK
);
2048 vdev
->vm_running
= running
;
2051 virtio_set_status(vdev
, vdev
->status
);
2054 if (k
->vmstate_change
) {
2055 k
->vmstate_change(qbus
->parent
, backend_run
);
2059 virtio_set_status(vdev
, vdev
->status
);
2063 void virtio_instance_init_common(Object
*proxy_obj
, void *data
,
2064 size_t vdev_size
, const char *vdev_name
)
2066 DeviceState
*vdev
= data
;
2068 object_initialize(vdev
, vdev_size
, vdev_name
);
2069 object_property_add_child(proxy_obj
, "virtio-backend", OBJECT(vdev
), NULL
);
2070 object_unref(OBJECT(vdev
));
2071 qdev_alias_all_properties(vdev
, proxy_obj
);
2074 void virtio_init(VirtIODevice
*vdev
, const char *name
,
2075 uint16_t device_id
, size_t config_size
)
2077 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2078 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2080 int nvectors
= k
->query_nvectors
? k
->query_nvectors(qbus
->parent
) : 0;
2083 vdev
->vector_queues
=
2084 g_malloc0(sizeof(*vdev
->vector_queues
) * nvectors
);
2087 vdev
->device_id
= device_id
;
2089 atomic_set(&vdev
->isr
, 0);
2090 vdev
->queue_sel
= 0;
2091 vdev
->config_vector
= VIRTIO_NO_VECTOR
;
2092 vdev
->vq
= g_malloc0(sizeof(VirtQueue
) * VIRTIO_QUEUE_MAX
);
2093 vdev
->vm_running
= runstate_is_running();
2094 vdev
->broken
= false;
2095 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2096 vdev
->vq
[i
].vector
= VIRTIO_NO_VECTOR
;
2097 vdev
->vq
[i
].vdev
= vdev
;
2098 vdev
->vq
[i
].queue_index
= i
;
2102 vdev
->config_len
= config_size
;
2103 if (vdev
->config_len
) {
2104 vdev
->config
= g_malloc0(config_size
);
2106 vdev
->config
= NULL
;
2108 vdev
->vmstate
= qemu_add_vm_change_state_handler(virtio_vmstate_change
,
2110 vdev
->device_endian
= virtio_default_endian();
2111 vdev
->use_guest_notifier_mask
= true;
2114 hwaddr
virtio_queue_get_desc_addr(VirtIODevice
*vdev
, int n
)
2116 return vdev
->vq
[n
].vring
.desc
;
2119 hwaddr
virtio_queue_get_avail_addr(VirtIODevice
*vdev
, int n
)
2121 return vdev
->vq
[n
].vring
.avail
;
2124 hwaddr
virtio_queue_get_used_addr(VirtIODevice
*vdev
, int n
)
2126 return vdev
->vq
[n
].vring
.used
;
2129 hwaddr
virtio_queue_get_desc_size(VirtIODevice
*vdev
, int n
)
2131 return sizeof(VRingDesc
) * vdev
->vq
[n
].vring
.num
;
2134 hwaddr
virtio_queue_get_avail_size(VirtIODevice
*vdev
, int n
)
2136 return offsetof(VRingAvail
, ring
) +
2137 sizeof(uint16_t) * vdev
->vq
[n
].vring
.num
;
2140 hwaddr
virtio_queue_get_used_size(VirtIODevice
*vdev
, int n
)
2142 return offsetof(VRingUsed
, ring
) +
2143 sizeof(VRingUsedElem
) * vdev
->vq
[n
].vring
.num
;
2146 uint16_t virtio_queue_get_last_avail_idx(VirtIODevice
*vdev
, int n
)
2148 return vdev
->vq
[n
].last_avail_idx
;
2151 void virtio_queue_set_last_avail_idx(VirtIODevice
*vdev
, int n
, uint16_t idx
)
2153 vdev
->vq
[n
].last_avail_idx
= idx
;
2154 vdev
->vq
[n
].shadow_avail_idx
= idx
;
2157 void virtio_queue_update_used_idx(VirtIODevice
*vdev
, int n
)
2159 if (vdev
->vq
[n
].vring
.desc
) {
2160 vdev
->vq
[n
].used_idx
= vring_used_idx(&vdev
->vq
[n
]);
2164 void virtio_queue_invalidate_signalled_used(VirtIODevice
*vdev
, int n
)
2166 vdev
->vq
[n
].signalled_used_valid
= false;
2169 VirtQueue
*virtio_get_queue(VirtIODevice
*vdev
, int n
)
2171 return vdev
->vq
+ n
;
2174 uint16_t virtio_get_queue_index(VirtQueue
*vq
)
2176 return vq
->queue_index
;
2179 static void virtio_queue_guest_notifier_read(EventNotifier
*n
)
2181 VirtQueue
*vq
= container_of(n
, VirtQueue
, guest_notifier
);
2182 if (event_notifier_test_and_clear(n
)) {
2183 virtio_notify_vector(vq
->vdev
, vq
->vector
);
2187 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue
*vq
, bool assign
,
2190 if (assign
&& !with_irqfd
) {
2191 event_notifier_set_handler(&vq
->guest_notifier
,
2192 virtio_queue_guest_notifier_read
);
2194 event_notifier_set_handler(&vq
->guest_notifier
, NULL
);
2197 /* Test and clear notifier before closing it,
2198 * in case poll callback didn't have time to run. */
2199 virtio_queue_guest_notifier_read(&vq
->guest_notifier
);
2203 EventNotifier
*virtio_queue_get_guest_notifier(VirtQueue
*vq
)
2205 return &vq
->guest_notifier
;
2208 static void virtio_queue_host_notifier_aio_read(EventNotifier
*n
)
2210 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
2211 if (event_notifier_test_and_clear(n
)) {
2212 virtio_queue_notify_aio_vq(vq
);
2216 static void virtio_queue_host_notifier_aio_poll_begin(EventNotifier
*n
)
2218 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
2220 virtio_queue_set_notification(vq
, 0);
2223 static bool virtio_queue_host_notifier_aio_poll(void *opaque
)
2225 EventNotifier
*n
= opaque
;
2226 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
2229 if (virtio_queue_empty(vq
)) {
2233 progress
= virtio_queue_notify_aio_vq(vq
);
2235 /* In case the handler function re-enabled notifications */
2236 virtio_queue_set_notification(vq
, 0);
2240 static void virtio_queue_host_notifier_aio_poll_end(EventNotifier
*n
)
2242 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
2244 /* Caller polls once more after this to catch requests that race with us */
2245 virtio_queue_set_notification(vq
, 1);
2248 void virtio_queue_aio_set_host_notifier_handler(VirtQueue
*vq
, AioContext
*ctx
,
2249 VirtIOHandleAIOOutput handle_output
)
2251 if (handle_output
) {
2252 vq
->handle_aio_output
= handle_output
;
2253 aio_set_event_notifier(ctx
, &vq
->host_notifier
, true,
2254 virtio_queue_host_notifier_aio_read
,
2255 virtio_queue_host_notifier_aio_poll
);
2256 aio_set_event_notifier_poll(ctx
, &vq
->host_notifier
,
2257 virtio_queue_host_notifier_aio_poll_begin
,
2258 virtio_queue_host_notifier_aio_poll_end
);
2260 aio_set_event_notifier(ctx
, &vq
->host_notifier
, true, NULL
, NULL
);
2261 /* Test and clear notifier before after disabling event,
2262 * in case poll callback didn't have time to run. */
2263 virtio_queue_host_notifier_aio_read(&vq
->host_notifier
);
2264 vq
->handle_aio_output
= NULL
;
2268 void virtio_queue_host_notifier_read(EventNotifier
*n
)
2270 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
2271 if (event_notifier_test_and_clear(n
)) {
2272 virtio_queue_notify_vq(vq
);
2276 EventNotifier
*virtio_queue_get_host_notifier(VirtQueue
*vq
)
2278 return &vq
->host_notifier
;
2281 void virtio_device_set_child_bus_name(VirtIODevice
*vdev
, char *bus_name
)
2283 g_free(vdev
->bus_name
);
2284 vdev
->bus_name
= g_strdup(bus_name
);
2287 void GCC_FMT_ATTR(2, 3) virtio_error(VirtIODevice
*vdev
, const char *fmt
, ...)
2292 error_vreport(fmt
, ap
);
2295 vdev
->broken
= true;
2297 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
2298 virtio_set_status(vdev
, vdev
->status
| VIRTIO_CONFIG_S_NEEDS_RESET
);
2299 virtio_notify_config(vdev
);
2303 static void virtio_memory_listener_commit(MemoryListener
*listener
)
2305 VirtIODevice
*vdev
= container_of(listener
, VirtIODevice
, listener
);
2308 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2309 if (vdev
->vq
[i
].vring
.num
== 0) {
2312 virtio_init_region_cache(vdev
, i
);
2316 static void virtio_device_realize(DeviceState
*dev
, Error
**errp
)
2318 VirtIODevice
*vdev
= VIRTIO_DEVICE(dev
);
2319 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(dev
);
2322 /* Devices should either use vmsd or the load/save methods */
2323 assert(!vdc
->vmsd
|| !vdc
->load
);
2325 if (vdc
->realize
!= NULL
) {
2326 vdc
->realize(dev
, &err
);
2328 error_propagate(errp
, err
);
2333 virtio_bus_device_plugged(vdev
, &err
);
2335 error_propagate(errp
, err
);
2339 vdev
->listener
.commit
= virtio_memory_listener_commit
;
2340 memory_listener_register(&vdev
->listener
, vdev
->dma_as
);
2343 static void virtio_device_unrealize(DeviceState
*dev
, Error
**errp
)
2345 VirtIODevice
*vdev
= VIRTIO_DEVICE(dev
);
2346 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(dev
);
2349 virtio_bus_device_unplugged(vdev
);
2351 if (vdc
->unrealize
!= NULL
) {
2352 vdc
->unrealize(dev
, &err
);
2354 error_propagate(errp
, err
);
2359 g_free(vdev
->bus_name
);
2360 vdev
->bus_name
= NULL
;
2363 static void virtio_device_free_virtqueues(VirtIODevice
*vdev
)
2370 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2371 VRingMemoryRegionCaches
*caches
;
2372 if (vdev
->vq
[i
].vring
.num
== 0) {
2375 caches
= atomic_read(&vdev
->vq
[i
].vring
.caches
);
2376 atomic_set(&vdev
->vq
[i
].vring
.caches
, NULL
);
2377 virtio_free_region_cache(caches
);
2382 static void virtio_device_instance_finalize(Object
*obj
)
2384 VirtIODevice
*vdev
= VIRTIO_DEVICE(obj
);
2386 memory_listener_unregister(&vdev
->listener
);
2387 virtio_device_free_virtqueues(vdev
);
2389 g_free(vdev
->config
);
2390 g_free(vdev
->vector_queues
);
2393 static Property virtio_properties
[] = {
2394 DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice
, host_features
),
2395 DEFINE_PROP_END_OF_LIST(),
2398 static int virtio_device_start_ioeventfd_impl(VirtIODevice
*vdev
)
2400 VirtioBusState
*qbus
= VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev
)));
2403 for (n
= 0; n
< VIRTIO_QUEUE_MAX
; n
++) {
2404 VirtQueue
*vq
= &vdev
->vq
[n
];
2405 if (!virtio_queue_get_num(vdev
, n
)) {
2408 r
= virtio_bus_set_host_notifier(qbus
, n
, true);
2413 event_notifier_set_handler(&vq
->host_notifier
,
2414 virtio_queue_host_notifier_read
);
2417 for (n
= 0; n
< VIRTIO_QUEUE_MAX
; n
++) {
2418 /* Kick right away to begin processing requests already in vring */
2419 VirtQueue
*vq
= &vdev
->vq
[n
];
2420 if (!vq
->vring
.num
) {
2423 event_notifier_set(&vq
->host_notifier
);
2429 VirtQueue
*vq
= &vdev
->vq
[n
];
2430 if (!virtio_queue_get_num(vdev
, n
)) {
2434 event_notifier_set_handler(&vq
->host_notifier
, NULL
);
2435 r
= virtio_bus_set_host_notifier(qbus
, n
, false);
2441 int virtio_device_start_ioeventfd(VirtIODevice
*vdev
)
2443 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2444 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
2446 return virtio_bus_start_ioeventfd(vbus
);
2449 static void virtio_device_stop_ioeventfd_impl(VirtIODevice
*vdev
)
2451 VirtioBusState
*qbus
= VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev
)));
2454 for (n
= 0; n
< VIRTIO_QUEUE_MAX
; n
++) {
2455 VirtQueue
*vq
= &vdev
->vq
[n
];
2457 if (!virtio_queue_get_num(vdev
, n
)) {
2460 event_notifier_set_handler(&vq
->host_notifier
, NULL
);
2461 r
= virtio_bus_set_host_notifier(qbus
, n
, false);
2466 void virtio_device_stop_ioeventfd(VirtIODevice
*vdev
)
2468 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2469 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
2471 virtio_bus_stop_ioeventfd(vbus
);
2474 int virtio_device_grab_ioeventfd(VirtIODevice
*vdev
)
2476 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2477 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
2479 return virtio_bus_grab_ioeventfd(vbus
);
2482 void virtio_device_release_ioeventfd(VirtIODevice
*vdev
)
2484 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2485 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
2487 virtio_bus_release_ioeventfd(vbus
);
2490 static void virtio_device_class_init(ObjectClass
*klass
, void *data
)
2492 /* Set the default value here. */
2493 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_CLASS(klass
);
2494 DeviceClass
*dc
= DEVICE_CLASS(klass
);
2496 dc
->realize
= virtio_device_realize
;
2497 dc
->unrealize
= virtio_device_unrealize
;
2498 dc
->bus_type
= TYPE_VIRTIO_BUS
;
2499 dc
->props
= virtio_properties
;
2500 vdc
->start_ioeventfd
= virtio_device_start_ioeventfd_impl
;
2501 vdc
->stop_ioeventfd
= virtio_device_stop_ioeventfd_impl
;
2503 vdc
->legacy_features
|= VIRTIO_LEGACY_FEATURES
;
2506 bool virtio_device_ioeventfd_enabled(VirtIODevice
*vdev
)
2508 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2509 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
2511 return virtio_bus_ioeventfd_enabled(vbus
);
2514 static const TypeInfo virtio_device_info
= {
2515 .name
= TYPE_VIRTIO_DEVICE
,
2516 .parent
= TYPE_DEVICE
,
2517 .instance_size
= sizeof(VirtIODevice
),
2518 .class_init
= virtio_device_class_init
,
2519 .instance_finalize
= virtio_device_instance_finalize
,
2521 .class_size
= sizeof(VirtioDeviceClass
),
2524 static void virtio_register_types(void)
2526 type_register_static(&virtio_device_info
);
2529 type_init(virtio_register_types
)