4 * Copyright IBM, Corp. 2007
7 * Anthony Liguori <aliguori@us.ibm.com>
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
14 #include "qemu/osdep.h"
15 #include "qapi/error.h"
16 #include "qemu-common.h"
19 #include "exec/address-spaces.h"
20 #include "qemu/error-report.h"
21 #include "hw/virtio/virtio.h"
22 #include "qemu/atomic.h"
23 #include "hw/virtio/virtio-bus.h"
24 #include "hw/virtio/virtio-access.h"
25 #include "sysemu/dma.h"
28 * The alignment to use between consumer and producer parts of vring.
29 * x86 pagesize again. This is the default, used by transports like PCI
30 * which don't provide a means for the guest to tell the host the alignment.
32 #define VIRTIO_PCI_VRING_ALIGN 4096
34 typedef struct VRingDesc
42 typedef struct VRingAvail
49 typedef struct VRingUsedElem
55 typedef struct VRingUsed
59 VRingUsedElem ring
[0];
62 typedef struct VRingMemoryRegionCaches
{
64 MemoryRegionCache desc
;
65 MemoryRegionCache avail
;
66 MemoryRegionCache used
;
67 } VRingMemoryRegionCaches
;
72 unsigned int num_default
;
77 VRingMemoryRegionCaches
*caches
;
84 /* Next head to pop */
85 uint16_t last_avail_idx
;
87 /* Last avail_idx read from VQ. */
88 uint16_t shadow_avail_idx
;
92 /* Last used index value we have signalled on */
93 uint16_t signalled_used
;
95 /* Last used index value we have signalled on */
96 bool signalled_used_valid
;
98 /* Notification enabled? */
101 uint16_t queue_index
;
106 VirtIOHandleOutput handle_output
;
107 VirtIOHandleAIOOutput handle_aio_output
;
109 EventNotifier guest_notifier
;
110 EventNotifier host_notifier
;
111 QLIST_ENTRY(VirtQueue
) node
;
114 static void virtio_free_region_cache(VRingMemoryRegionCaches
*caches
)
120 address_space_cache_destroy(&caches
->desc
);
121 address_space_cache_destroy(&caches
->avail
);
122 address_space_cache_destroy(&caches
->used
);
126 static void virtio_init_region_cache(VirtIODevice
*vdev
, int n
)
128 VirtQueue
*vq
= &vdev
->vq
[n
];
129 VRingMemoryRegionCaches
*old
= vq
->vring
.caches
;
130 VRingMemoryRegionCaches
*new;
135 event_size
= virtio_vdev_has_feature(vq
->vdev
, VIRTIO_RING_F_EVENT_IDX
) ? 2 : 0;
137 addr
= vq
->vring
.desc
;
141 new = g_new0(VRingMemoryRegionCaches
, 1);
142 size
= virtio_queue_get_desc_size(vdev
, n
);
143 len
= address_space_cache_init(&new->desc
, vdev
->dma_as
,
146 virtio_error(vdev
, "Cannot map desc");
150 size
= virtio_queue_get_used_size(vdev
, n
) + event_size
;
151 len
= address_space_cache_init(&new->used
, vdev
->dma_as
,
152 vq
->vring
.used
, size
, true);
154 virtio_error(vdev
, "Cannot map used");
158 size
= virtio_queue_get_avail_size(vdev
, n
) + event_size
;
159 len
= address_space_cache_init(&new->avail
, vdev
->dma_as
,
160 vq
->vring
.avail
, size
, false);
162 virtio_error(vdev
, "Cannot map avail");
166 atomic_rcu_set(&vq
->vring
.caches
, new);
168 call_rcu(old
, virtio_free_region_cache
, rcu
);
173 address_space_cache_destroy(&new->used
);
175 address_space_cache_destroy(&new->desc
);
180 /* virt queue functions */
181 void virtio_queue_update_rings(VirtIODevice
*vdev
, int n
)
183 VRing
*vring
= &vdev
->vq
[n
].vring
;
185 if (!vring
->num
|| !vring
->desc
|| !vring
->align
) {
186 /* not yet setup -> nothing to do */
189 vring
->avail
= vring
->desc
+ vring
->num
* sizeof(VRingDesc
);
190 vring
->used
= vring_align(vring
->avail
+
191 offsetof(VRingAvail
, ring
[vring
->num
]),
193 virtio_init_region_cache(vdev
, n
);
196 /* Called within rcu_read_lock(). */
197 static void vring_desc_read(VirtIODevice
*vdev
, VRingDesc
*desc
,
198 MemoryRegionCache
*cache
, int i
)
200 address_space_read_cached(cache
, i
* sizeof(VRingDesc
),
201 desc
, sizeof(VRingDesc
));
202 virtio_tswap64s(vdev
, &desc
->addr
);
203 virtio_tswap32s(vdev
, &desc
->len
);
204 virtio_tswap16s(vdev
, &desc
->flags
);
205 virtio_tswap16s(vdev
, &desc
->next
);
208 static VRingMemoryRegionCaches
*vring_get_region_caches(struct VirtQueue
*vq
)
210 VRingMemoryRegionCaches
*caches
= atomic_rcu_read(&vq
->vring
.caches
);
211 assert(caches
!= NULL
);
214 /* Called within rcu_read_lock(). */
215 static inline uint16_t vring_avail_flags(VirtQueue
*vq
)
217 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
218 hwaddr pa
= offsetof(VRingAvail
, flags
);
219 return virtio_lduw_phys_cached(vq
->vdev
, &caches
->avail
, pa
);
222 /* Called within rcu_read_lock(). */
223 static inline uint16_t vring_avail_idx(VirtQueue
*vq
)
225 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
226 hwaddr pa
= offsetof(VRingAvail
, idx
);
227 vq
->shadow_avail_idx
= virtio_lduw_phys_cached(vq
->vdev
, &caches
->avail
, pa
);
228 return vq
->shadow_avail_idx
;
231 /* Called within rcu_read_lock(). */
232 static inline uint16_t vring_avail_ring(VirtQueue
*vq
, int i
)
234 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
235 hwaddr pa
= offsetof(VRingAvail
, ring
[i
]);
236 return virtio_lduw_phys_cached(vq
->vdev
, &caches
->avail
, pa
);
239 /* Called within rcu_read_lock(). */
240 static inline uint16_t vring_get_used_event(VirtQueue
*vq
)
242 return vring_avail_ring(vq
, vq
->vring
.num
);
245 /* Called within rcu_read_lock(). */
246 static inline void vring_used_write(VirtQueue
*vq
, VRingUsedElem
*uelem
,
249 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
250 hwaddr pa
= offsetof(VRingUsed
, ring
[i
]);
251 virtio_tswap32s(vq
->vdev
, &uelem
->id
);
252 virtio_tswap32s(vq
->vdev
, &uelem
->len
);
253 address_space_write_cached(&caches
->used
, pa
, uelem
, sizeof(VRingUsedElem
));
254 address_space_cache_invalidate(&caches
->used
, pa
, sizeof(VRingUsedElem
));
257 /* Called within rcu_read_lock(). */
258 static uint16_t vring_used_idx(VirtQueue
*vq
)
260 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
261 hwaddr pa
= offsetof(VRingUsed
, idx
);
262 return virtio_lduw_phys_cached(vq
->vdev
, &caches
->used
, pa
);
265 /* Called within rcu_read_lock(). */
266 static inline void vring_used_idx_set(VirtQueue
*vq
, uint16_t val
)
268 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
269 hwaddr pa
= offsetof(VRingUsed
, idx
);
270 virtio_stw_phys_cached(vq
->vdev
, &caches
->used
, pa
, val
);
271 address_space_cache_invalidate(&caches
->used
, pa
, sizeof(val
));
275 /* Called within rcu_read_lock(). */
276 static inline void vring_used_flags_set_bit(VirtQueue
*vq
, int mask
)
278 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
279 VirtIODevice
*vdev
= vq
->vdev
;
280 hwaddr pa
= offsetof(VRingUsed
, flags
);
281 uint16_t flags
= virtio_lduw_phys_cached(vq
->vdev
, &caches
->used
, pa
);
283 virtio_stw_phys_cached(vdev
, &caches
->used
, pa
, flags
| mask
);
284 address_space_cache_invalidate(&caches
->used
, pa
, sizeof(flags
));
287 /* Called within rcu_read_lock(). */
288 static inline void vring_used_flags_unset_bit(VirtQueue
*vq
, int mask
)
290 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
291 VirtIODevice
*vdev
= vq
->vdev
;
292 hwaddr pa
= offsetof(VRingUsed
, flags
);
293 uint16_t flags
= virtio_lduw_phys_cached(vq
->vdev
, &caches
->used
, pa
);
295 virtio_stw_phys_cached(vdev
, &caches
->used
, pa
, flags
& ~mask
);
296 address_space_cache_invalidate(&caches
->used
, pa
, sizeof(flags
));
299 /* Called within rcu_read_lock(). */
300 static inline void vring_set_avail_event(VirtQueue
*vq
, uint16_t val
)
302 VRingMemoryRegionCaches
*caches
;
304 if (!vq
->notification
) {
308 caches
= vring_get_region_caches(vq
);
309 pa
= offsetof(VRingUsed
, ring
[vq
->vring
.num
]);
310 virtio_stw_phys_cached(vq
->vdev
, &caches
->used
, pa
, val
);
311 address_space_cache_invalidate(&caches
->used
, pa
, sizeof(val
));
314 void virtio_queue_set_notification(VirtQueue
*vq
, int enable
)
316 vq
->notification
= enable
;
318 if (!vq
->vring
.desc
) {
323 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
324 vring_set_avail_event(vq
, vring_avail_idx(vq
));
326 vring_used_flags_unset_bit(vq
, VRING_USED_F_NO_NOTIFY
);
328 vring_used_flags_set_bit(vq
, VRING_USED_F_NO_NOTIFY
);
331 /* Expose avail event/used flags before caller checks the avail idx. */
337 int virtio_queue_ready(VirtQueue
*vq
)
339 return vq
->vring
.avail
!= 0;
342 /* Fetch avail_idx from VQ memory only when we really need to know if
343 * guest has added some buffers.
344 * Called within rcu_read_lock(). */
345 static int virtio_queue_empty_rcu(VirtQueue
*vq
)
347 if (unlikely(!vq
->vring
.avail
)) {
351 if (vq
->shadow_avail_idx
!= vq
->last_avail_idx
) {
355 return vring_avail_idx(vq
) == vq
->last_avail_idx
;
358 int virtio_queue_empty(VirtQueue
*vq
)
362 if (unlikely(!vq
->vring
.avail
)) {
366 if (vq
->shadow_avail_idx
!= vq
->last_avail_idx
) {
371 empty
= vring_avail_idx(vq
) == vq
->last_avail_idx
;
376 static void virtqueue_unmap_sg(VirtQueue
*vq
, const VirtQueueElement
*elem
,
379 AddressSpace
*dma_as
= vq
->vdev
->dma_as
;
384 for (i
= 0; i
< elem
->in_num
; i
++) {
385 size_t size
= MIN(len
- offset
, elem
->in_sg
[i
].iov_len
);
387 dma_memory_unmap(dma_as
, elem
->in_sg
[i
].iov_base
,
388 elem
->in_sg
[i
].iov_len
,
389 DMA_DIRECTION_FROM_DEVICE
, size
);
394 for (i
= 0; i
< elem
->out_num
; i
++)
395 dma_memory_unmap(dma_as
, elem
->out_sg
[i
].iov_base
,
396 elem
->out_sg
[i
].iov_len
,
397 DMA_DIRECTION_TO_DEVICE
,
398 elem
->out_sg
[i
].iov_len
);
401 /* virtqueue_detach_element:
402 * @vq: The #VirtQueue
403 * @elem: The #VirtQueueElement
404 * @len: number of bytes written
406 * Detach the element from the virtqueue. This function is suitable for device
407 * reset or other situations where a #VirtQueueElement is simply freed and will
408 * not be pushed or discarded.
410 void virtqueue_detach_element(VirtQueue
*vq
, const VirtQueueElement
*elem
,
414 virtqueue_unmap_sg(vq
, elem
, len
);
418 * @vq: The #VirtQueue
419 * @elem: The #VirtQueueElement
420 * @len: number of bytes written
422 * Pretend the most recent element wasn't popped from the virtqueue. The next
423 * call to virtqueue_pop() will refetch the element.
425 void virtqueue_unpop(VirtQueue
*vq
, const VirtQueueElement
*elem
,
428 vq
->last_avail_idx
--;
429 virtqueue_detach_element(vq
, elem
, len
);
433 * @vq: The #VirtQueue
434 * @num: Number of elements to push back
436 * Pretend that elements weren't popped from the virtqueue. The next
437 * virtqueue_pop() will refetch the oldest element.
439 * Use virtqueue_unpop() instead if you have a VirtQueueElement.
441 * Returns: true on success, false if @num is greater than the number of in use
444 bool virtqueue_rewind(VirtQueue
*vq
, unsigned int num
)
446 if (num
> vq
->inuse
) {
449 vq
->last_avail_idx
-= num
;
454 /* Called within rcu_read_lock(). */
455 void virtqueue_fill(VirtQueue
*vq
, const VirtQueueElement
*elem
,
456 unsigned int len
, unsigned int idx
)
460 trace_virtqueue_fill(vq
, elem
, len
, idx
);
462 virtqueue_unmap_sg(vq
, elem
, len
);
464 if (unlikely(vq
->vdev
->broken
)) {
468 if (unlikely(!vq
->vring
.used
)) {
472 idx
= (idx
+ vq
->used_idx
) % vq
->vring
.num
;
474 uelem
.id
= elem
->index
;
476 vring_used_write(vq
, &uelem
, idx
);
479 /* Called within rcu_read_lock(). */
480 void virtqueue_flush(VirtQueue
*vq
, unsigned int count
)
484 if (unlikely(vq
->vdev
->broken
)) {
489 if (unlikely(!vq
->vring
.used
)) {
493 /* Make sure buffer is written before we update index. */
495 trace_virtqueue_flush(vq
, count
);
498 vring_used_idx_set(vq
, new);
500 if (unlikely((int16_t)(new - vq
->signalled_used
) < (uint16_t)(new - old
)))
501 vq
->signalled_used_valid
= false;
504 void virtqueue_push(VirtQueue
*vq
, const VirtQueueElement
*elem
,
508 virtqueue_fill(vq
, elem
, len
, 0);
509 virtqueue_flush(vq
, 1);
513 /* Called within rcu_read_lock(). */
514 static int virtqueue_num_heads(VirtQueue
*vq
, unsigned int idx
)
516 uint16_t num_heads
= vring_avail_idx(vq
) - idx
;
518 /* Check it isn't doing very strange things with descriptor numbers. */
519 if (num_heads
> vq
->vring
.num
) {
520 virtio_error(vq
->vdev
, "Guest moved used index from %u to %u",
521 idx
, vq
->shadow_avail_idx
);
524 /* On success, callers read a descriptor at vq->last_avail_idx.
525 * Make sure descriptor read does not bypass avail index read. */
533 /* Called within rcu_read_lock(). */
534 static bool virtqueue_get_head(VirtQueue
*vq
, unsigned int idx
,
537 /* Grab the next descriptor number they're advertising, and increment
538 * the index we've seen. */
539 *head
= vring_avail_ring(vq
, idx
% vq
->vring
.num
);
541 /* If their number is silly, that's a fatal mistake. */
542 if (*head
>= vq
->vring
.num
) {
543 virtio_error(vq
->vdev
, "Guest says index %u is available", *head
);
551 VIRTQUEUE_READ_DESC_ERROR
= -1,
552 VIRTQUEUE_READ_DESC_DONE
= 0, /* end of chain */
553 VIRTQUEUE_READ_DESC_MORE
= 1, /* more buffers in chain */
556 static int virtqueue_read_next_desc(VirtIODevice
*vdev
, VRingDesc
*desc
,
557 MemoryRegionCache
*desc_cache
, unsigned int max
,
560 /* If this descriptor says it doesn't chain, we're done. */
561 if (!(desc
->flags
& VRING_DESC_F_NEXT
)) {
562 return VIRTQUEUE_READ_DESC_DONE
;
565 /* Check they're not leading us off end of descriptors. */
567 /* Make sure compiler knows to grab that: we don't want it changing! */
571 virtio_error(vdev
, "Desc next is %u", *next
);
572 return VIRTQUEUE_READ_DESC_ERROR
;
575 vring_desc_read(vdev
, desc
, desc_cache
, *next
);
576 return VIRTQUEUE_READ_DESC_MORE
;
579 void virtqueue_get_avail_bytes(VirtQueue
*vq
, unsigned int *in_bytes
,
580 unsigned int *out_bytes
,
581 unsigned max_in_bytes
, unsigned max_out_bytes
)
583 VirtIODevice
*vdev
= vq
->vdev
;
584 unsigned int max
, idx
;
585 unsigned int total_bufs
, in_total
, out_total
;
586 VRingMemoryRegionCaches
*caches
;
587 MemoryRegionCache indirect_desc_cache
= MEMORY_REGION_CACHE_INVALID
;
591 if (unlikely(!vq
->vring
.desc
)) {
602 idx
= vq
->last_avail_idx
;
603 total_bufs
= in_total
= out_total
= 0;
606 caches
= vring_get_region_caches(vq
);
607 if (caches
->desc
.len
< max
* sizeof(VRingDesc
)) {
608 virtio_error(vdev
, "Cannot map descriptor ring");
612 while ((rc
= virtqueue_num_heads(vq
, idx
)) > 0) {
613 MemoryRegionCache
*desc_cache
= &caches
->desc
;
614 unsigned int num_bufs
;
618 num_bufs
= total_bufs
;
620 if (!virtqueue_get_head(vq
, idx
++, &i
)) {
624 vring_desc_read(vdev
, &desc
, desc_cache
, i
);
626 if (desc
.flags
& VRING_DESC_F_INDIRECT
) {
627 if (desc
.len
% sizeof(VRingDesc
)) {
628 virtio_error(vdev
, "Invalid size for indirect buffer table");
632 /* If we've got too many, that implies a descriptor loop. */
633 if (num_bufs
>= max
) {
634 virtio_error(vdev
, "Looped descriptor");
638 /* loop over the indirect descriptor table */
639 len
= address_space_cache_init(&indirect_desc_cache
,
641 desc
.addr
, desc
.len
, false);
642 desc_cache
= &indirect_desc_cache
;
643 if (len
< desc
.len
) {
644 virtio_error(vdev
, "Cannot map indirect buffer");
648 max
= desc
.len
/ sizeof(VRingDesc
);
650 vring_desc_read(vdev
, &desc
, desc_cache
, i
);
654 /* If we've got too many, that implies a descriptor loop. */
655 if (++num_bufs
> max
) {
656 virtio_error(vdev
, "Looped descriptor");
660 if (desc
.flags
& VRING_DESC_F_WRITE
) {
661 in_total
+= desc
.len
;
663 out_total
+= desc
.len
;
665 if (in_total
>= max_in_bytes
&& out_total
>= max_out_bytes
) {
669 rc
= virtqueue_read_next_desc(vdev
, &desc
, desc_cache
, max
, &i
);
670 } while (rc
== VIRTQUEUE_READ_DESC_MORE
);
672 if (rc
== VIRTQUEUE_READ_DESC_ERROR
) {
676 if (desc_cache
== &indirect_desc_cache
) {
677 address_space_cache_destroy(&indirect_desc_cache
);
680 total_bufs
= num_bufs
;
689 address_space_cache_destroy(&indirect_desc_cache
);
691 *in_bytes
= in_total
;
694 *out_bytes
= out_total
;
700 in_total
= out_total
= 0;
704 int virtqueue_avail_bytes(VirtQueue
*vq
, unsigned int in_bytes
,
705 unsigned int out_bytes
)
707 unsigned int in_total
, out_total
;
709 virtqueue_get_avail_bytes(vq
, &in_total
, &out_total
, in_bytes
, out_bytes
);
710 return in_bytes
<= in_total
&& out_bytes
<= out_total
;
713 static bool virtqueue_map_desc(VirtIODevice
*vdev
, unsigned int *p_num_sg
,
714 hwaddr
*addr
, struct iovec
*iov
,
715 unsigned int max_num_sg
, bool is_write
,
716 hwaddr pa
, size_t sz
)
719 unsigned num_sg
= *p_num_sg
;
720 assert(num_sg
<= max_num_sg
);
723 virtio_error(vdev
, "virtio: zero sized buffers are not allowed");
730 if (num_sg
== max_num_sg
) {
731 virtio_error(vdev
, "virtio: too many write descriptors in "
736 iov
[num_sg
].iov_base
= dma_memory_map(vdev
->dma_as
, pa
, &len
,
738 DMA_DIRECTION_FROM_DEVICE
:
739 DMA_DIRECTION_TO_DEVICE
);
740 if (!iov
[num_sg
].iov_base
) {
741 virtio_error(vdev
, "virtio: bogus descriptor or out of resources");
745 iov
[num_sg
].iov_len
= len
;
759 /* Only used by error code paths before we have a VirtQueueElement (therefore
760 * virtqueue_unmap_sg() can't be used). Assumes buffers weren't written to
763 static void virtqueue_undo_map_desc(unsigned int out_num
, unsigned int in_num
,
768 for (i
= 0; i
< out_num
+ in_num
; i
++) {
769 int is_write
= i
>= out_num
;
771 cpu_physical_memory_unmap(iov
->iov_base
, iov
->iov_len
, is_write
, 0);
776 static void virtqueue_map_iovec(VirtIODevice
*vdev
, struct iovec
*sg
,
777 hwaddr
*addr
, unsigned int *num_sg
,
783 for (i
= 0; i
< *num_sg
; i
++) {
785 sg
[i
].iov_base
= dma_memory_map(vdev
->dma_as
,
786 addr
[i
], &len
, is_write
?
787 DMA_DIRECTION_FROM_DEVICE
:
788 DMA_DIRECTION_TO_DEVICE
);
789 if (!sg
[i
].iov_base
) {
790 error_report("virtio: error trying to map MMIO memory");
793 if (len
!= sg
[i
].iov_len
) {
794 error_report("virtio: unexpected memory split");
800 void virtqueue_map(VirtIODevice
*vdev
, VirtQueueElement
*elem
)
802 virtqueue_map_iovec(vdev
, elem
->in_sg
, elem
->in_addr
, &elem
->in_num
, 1);
803 virtqueue_map_iovec(vdev
, elem
->out_sg
, elem
->out_addr
, &elem
->out_num
, 0);
806 static void *virtqueue_alloc_element(size_t sz
, unsigned out_num
, unsigned in_num
)
808 VirtQueueElement
*elem
;
809 size_t in_addr_ofs
= QEMU_ALIGN_UP(sz
, __alignof__(elem
->in_addr
[0]));
810 size_t out_addr_ofs
= in_addr_ofs
+ in_num
* sizeof(elem
->in_addr
[0]);
811 size_t out_addr_end
= out_addr_ofs
+ out_num
* sizeof(elem
->out_addr
[0]);
812 size_t in_sg_ofs
= QEMU_ALIGN_UP(out_addr_end
, __alignof__(elem
->in_sg
[0]));
813 size_t out_sg_ofs
= in_sg_ofs
+ in_num
* sizeof(elem
->in_sg
[0]);
814 size_t out_sg_end
= out_sg_ofs
+ out_num
* sizeof(elem
->out_sg
[0]);
816 assert(sz
>= sizeof(VirtQueueElement
));
817 elem
= g_malloc(out_sg_end
);
818 trace_virtqueue_alloc_element(elem
, sz
, in_num
, out_num
);
819 elem
->out_num
= out_num
;
820 elem
->in_num
= in_num
;
821 elem
->in_addr
= (void *)elem
+ in_addr_ofs
;
822 elem
->out_addr
= (void *)elem
+ out_addr_ofs
;
823 elem
->in_sg
= (void *)elem
+ in_sg_ofs
;
824 elem
->out_sg
= (void *)elem
+ out_sg_ofs
;
828 void *virtqueue_pop(VirtQueue
*vq
, size_t sz
)
830 unsigned int i
, head
, max
;
831 VRingMemoryRegionCaches
*caches
;
832 MemoryRegionCache indirect_desc_cache
= MEMORY_REGION_CACHE_INVALID
;
833 MemoryRegionCache
*desc_cache
;
835 VirtIODevice
*vdev
= vq
->vdev
;
836 VirtQueueElement
*elem
= NULL
;
837 unsigned out_num
, in_num
, elem_entries
;
838 hwaddr addr
[VIRTQUEUE_MAX_SIZE
];
839 struct iovec iov
[VIRTQUEUE_MAX_SIZE
];
843 if (unlikely(vdev
->broken
)) {
847 if (virtio_queue_empty_rcu(vq
)) {
850 /* Needed after virtio_queue_empty(), see comment in
851 * virtqueue_num_heads(). */
854 /* When we start there are none of either input nor output. */
855 out_num
= in_num
= elem_entries
= 0;
859 if (vq
->inuse
>= vq
->vring
.num
) {
860 virtio_error(vdev
, "Virtqueue size exceeded");
864 if (!virtqueue_get_head(vq
, vq
->last_avail_idx
++, &head
)) {
868 if (virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
869 vring_set_avail_event(vq
, vq
->last_avail_idx
);
874 caches
= vring_get_region_caches(vq
);
875 if (caches
->desc
.len
< max
* sizeof(VRingDesc
)) {
876 virtio_error(vdev
, "Cannot map descriptor ring");
880 desc_cache
= &caches
->desc
;
881 vring_desc_read(vdev
, &desc
, desc_cache
, i
);
882 if (desc
.flags
& VRING_DESC_F_INDIRECT
) {
883 if (desc
.len
% sizeof(VRingDesc
)) {
884 virtio_error(vdev
, "Invalid size for indirect buffer table");
888 /* loop over the indirect descriptor table */
889 len
= address_space_cache_init(&indirect_desc_cache
, vdev
->dma_as
,
890 desc
.addr
, desc
.len
, false);
891 desc_cache
= &indirect_desc_cache
;
892 if (len
< desc
.len
) {
893 virtio_error(vdev
, "Cannot map indirect buffer");
897 max
= desc
.len
/ sizeof(VRingDesc
);
899 vring_desc_read(vdev
, &desc
, desc_cache
, i
);
902 /* Collect all the descriptors */
906 if (desc
.flags
& VRING_DESC_F_WRITE
) {
907 map_ok
= virtqueue_map_desc(vdev
, &in_num
, addr
+ out_num
,
909 VIRTQUEUE_MAX_SIZE
- out_num
, true,
910 desc
.addr
, desc
.len
);
913 virtio_error(vdev
, "Incorrect order for descriptors");
916 map_ok
= virtqueue_map_desc(vdev
, &out_num
, addr
, iov
,
917 VIRTQUEUE_MAX_SIZE
, false,
918 desc
.addr
, desc
.len
);
924 /* If we've got too many, that implies a descriptor loop. */
925 if (++elem_entries
> max
) {
926 virtio_error(vdev
, "Looped descriptor");
930 rc
= virtqueue_read_next_desc(vdev
, &desc
, desc_cache
, max
, &i
);
931 } while (rc
== VIRTQUEUE_READ_DESC_MORE
);
933 if (rc
== VIRTQUEUE_READ_DESC_ERROR
) {
937 /* Now copy what we have collected and mapped */
938 elem
= virtqueue_alloc_element(sz
, out_num
, in_num
);
940 for (i
= 0; i
< out_num
; i
++) {
941 elem
->out_addr
[i
] = addr
[i
];
942 elem
->out_sg
[i
] = iov
[i
];
944 for (i
= 0; i
< in_num
; i
++) {
945 elem
->in_addr
[i
] = addr
[out_num
+ i
];
946 elem
->in_sg
[i
] = iov
[out_num
+ i
];
951 trace_virtqueue_pop(vq
, elem
, elem
->in_num
, elem
->out_num
);
953 address_space_cache_destroy(&indirect_desc_cache
);
959 virtqueue_undo_map_desc(out_num
, in_num
, iov
);
963 /* virtqueue_drop_all:
964 * @vq: The #VirtQueue
965 * Drops all queued buffers and indicates them to the guest
966 * as if they are done. Useful when buffers can not be
967 * processed but must be returned to the guest.
969 unsigned int virtqueue_drop_all(VirtQueue
*vq
)
971 unsigned int dropped
= 0;
972 VirtQueueElement elem
= {};
973 VirtIODevice
*vdev
= vq
->vdev
;
974 bool fEventIdx
= virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
);
976 if (unlikely(vdev
->broken
)) {
980 while (!virtio_queue_empty(vq
) && vq
->inuse
< vq
->vring
.num
) {
981 /* works similar to virtqueue_pop but does not map buffers
982 * and does not allocate any memory */
984 if (!virtqueue_get_head(vq
, vq
->last_avail_idx
, &elem
.index
)) {
988 vq
->last_avail_idx
++;
990 vring_set_avail_event(vq
, vq
->last_avail_idx
);
992 /* immediately push the element, nothing to unmap
993 * as both in_num and out_num are set to 0 */
994 virtqueue_push(vq
, &elem
, 0);
1001 /* Reading and writing a structure directly to QEMUFile is *awful*, but
1002 * it is what QEMU has always done by mistake. We can change it sooner
1003 * or later by bumping the version number of the affected vm states.
1004 * In the meanwhile, since the in-memory layout of VirtQueueElement
1005 * has changed, we need to marshal to and from the layout that was
1006 * used before the change.
1008 typedef struct VirtQueueElementOld
{
1010 unsigned int out_num
;
1011 unsigned int in_num
;
1012 hwaddr in_addr
[VIRTQUEUE_MAX_SIZE
];
1013 hwaddr out_addr
[VIRTQUEUE_MAX_SIZE
];
1014 struct iovec in_sg
[VIRTQUEUE_MAX_SIZE
];
1015 struct iovec out_sg
[VIRTQUEUE_MAX_SIZE
];
1016 } VirtQueueElementOld
;
1018 void *qemu_get_virtqueue_element(VirtIODevice
*vdev
, QEMUFile
*f
, size_t sz
)
1020 VirtQueueElement
*elem
;
1021 VirtQueueElementOld data
;
1024 qemu_get_buffer(f
, (uint8_t *)&data
, sizeof(VirtQueueElementOld
));
1026 /* TODO: teach all callers that this can fail, and return failure instead
1027 * of asserting here.
1028 * This is just one thing (there are probably more) that must be
1029 * fixed before we can allow NDEBUG compilation.
1031 assert(ARRAY_SIZE(data
.in_addr
) >= data
.in_num
);
1032 assert(ARRAY_SIZE(data
.out_addr
) >= data
.out_num
);
1034 elem
= virtqueue_alloc_element(sz
, data
.out_num
, data
.in_num
);
1035 elem
->index
= data
.index
;
1037 for (i
= 0; i
< elem
->in_num
; i
++) {
1038 elem
->in_addr
[i
] = data
.in_addr
[i
];
1041 for (i
= 0; i
< elem
->out_num
; i
++) {
1042 elem
->out_addr
[i
] = data
.out_addr
[i
];
1045 for (i
= 0; i
< elem
->in_num
; i
++) {
1046 /* Base is overwritten by virtqueue_map. */
1047 elem
->in_sg
[i
].iov_base
= 0;
1048 elem
->in_sg
[i
].iov_len
= data
.in_sg
[i
].iov_len
;
1051 for (i
= 0; i
< elem
->out_num
; i
++) {
1052 /* Base is overwritten by virtqueue_map. */
1053 elem
->out_sg
[i
].iov_base
= 0;
1054 elem
->out_sg
[i
].iov_len
= data
.out_sg
[i
].iov_len
;
1057 virtqueue_map(vdev
, elem
);
1061 void qemu_put_virtqueue_element(QEMUFile
*f
, VirtQueueElement
*elem
)
1063 VirtQueueElementOld data
;
1066 memset(&data
, 0, sizeof(data
));
1067 data
.index
= elem
->index
;
1068 data
.in_num
= elem
->in_num
;
1069 data
.out_num
= elem
->out_num
;
1071 for (i
= 0; i
< elem
->in_num
; i
++) {
1072 data
.in_addr
[i
] = elem
->in_addr
[i
];
1075 for (i
= 0; i
< elem
->out_num
; i
++) {
1076 data
.out_addr
[i
] = elem
->out_addr
[i
];
1079 for (i
= 0; i
< elem
->in_num
; i
++) {
1080 /* Base is overwritten by virtqueue_map when loading. Do not
1081 * save it, as it would leak the QEMU address space layout. */
1082 data
.in_sg
[i
].iov_len
= elem
->in_sg
[i
].iov_len
;
1085 for (i
= 0; i
< elem
->out_num
; i
++) {
1086 /* Do not save iov_base as above. */
1087 data
.out_sg
[i
].iov_len
= elem
->out_sg
[i
].iov_len
;
1089 qemu_put_buffer(f
, (uint8_t *)&data
, sizeof(VirtQueueElementOld
));
1093 static void virtio_notify_vector(VirtIODevice
*vdev
, uint16_t vector
)
1095 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1096 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1098 if (unlikely(vdev
->broken
)) {
1103 k
->notify(qbus
->parent
, vector
);
1107 void virtio_update_irq(VirtIODevice
*vdev
)
1109 virtio_notify_vector(vdev
, VIRTIO_NO_VECTOR
);
1112 static int virtio_validate_features(VirtIODevice
*vdev
)
1114 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1116 if (virtio_host_has_feature(vdev
, VIRTIO_F_IOMMU_PLATFORM
) &&
1117 !virtio_vdev_has_feature(vdev
, VIRTIO_F_IOMMU_PLATFORM
)) {
1121 if (k
->validate_features
) {
1122 return k
->validate_features(vdev
);
1128 int virtio_set_status(VirtIODevice
*vdev
, uint8_t val
)
1130 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1131 trace_virtio_set_status(vdev
, val
);
1133 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
1134 if (!(vdev
->status
& VIRTIO_CONFIG_S_FEATURES_OK
) &&
1135 val
& VIRTIO_CONFIG_S_FEATURES_OK
) {
1136 int ret
= virtio_validate_features(vdev
);
1143 if (k
->set_status
) {
1144 k
->set_status(vdev
, val
);
1150 bool target_words_bigendian(void);
1151 static enum virtio_device_endian
virtio_default_endian(void)
1153 if (target_words_bigendian()) {
1154 return VIRTIO_DEVICE_ENDIAN_BIG
;
1156 return VIRTIO_DEVICE_ENDIAN_LITTLE
;
1160 static enum virtio_device_endian
virtio_current_cpu_endian(void)
1162 CPUClass
*cc
= CPU_GET_CLASS(current_cpu
);
1164 if (cc
->virtio_is_big_endian(current_cpu
)) {
1165 return VIRTIO_DEVICE_ENDIAN_BIG
;
1167 return VIRTIO_DEVICE_ENDIAN_LITTLE
;
1171 static void virtio_virtqueue_reset_region_cache(struct VirtQueue
*vq
)
1173 VRingMemoryRegionCaches
*caches
;
1175 caches
= atomic_read(&vq
->vring
.caches
);
1176 atomic_rcu_set(&vq
->vring
.caches
, NULL
);
1178 call_rcu(caches
, virtio_free_region_cache
, rcu
);
1182 void virtio_reset(void *opaque
)
1184 VirtIODevice
*vdev
= opaque
;
1185 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1188 virtio_set_status(vdev
, 0);
1190 /* Guest initiated reset */
1191 vdev
->device_endian
= virtio_current_cpu_endian();
1194 vdev
->device_endian
= virtio_default_endian();
1201 vdev
->broken
= false;
1202 vdev
->guest_features
= 0;
1203 vdev
->queue_sel
= 0;
1205 atomic_set(&vdev
->isr
, 0);
1206 vdev
->config_vector
= VIRTIO_NO_VECTOR
;
1207 virtio_notify_vector(vdev
, vdev
->config_vector
);
1209 for(i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
1210 vdev
->vq
[i
].vring
.desc
= 0;
1211 vdev
->vq
[i
].vring
.avail
= 0;
1212 vdev
->vq
[i
].vring
.used
= 0;
1213 vdev
->vq
[i
].last_avail_idx
= 0;
1214 vdev
->vq
[i
].shadow_avail_idx
= 0;
1215 vdev
->vq
[i
].used_idx
= 0;
1216 virtio_queue_set_vector(vdev
, i
, VIRTIO_NO_VECTOR
);
1217 vdev
->vq
[i
].signalled_used
= 0;
1218 vdev
->vq
[i
].signalled_used_valid
= false;
1219 vdev
->vq
[i
].notification
= true;
1220 vdev
->vq
[i
].vring
.num
= vdev
->vq
[i
].vring
.num_default
;
1221 vdev
->vq
[i
].inuse
= 0;
1222 virtio_virtqueue_reset_region_cache(&vdev
->vq
[i
]);
1226 uint32_t virtio_config_readb(VirtIODevice
*vdev
, uint32_t addr
)
1228 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1231 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1232 return (uint32_t)-1;
1235 k
->get_config(vdev
, vdev
->config
);
1237 val
= ldub_p(vdev
->config
+ addr
);
1241 uint32_t virtio_config_readw(VirtIODevice
*vdev
, uint32_t addr
)
1243 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1246 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1247 return (uint32_t)-1;
1250 k
->get_config(vdev
, vdev
->config
);
1252 val
= lduw_p(vdev
->config
+ addr
);
1256 uint32_t virtio_config_readl(VirtIODevice
*vdev
, uint32_t addr
)
1258 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1261 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1262 return (uint32_t)-1;
1265 k
->get_config(vdev
, vdev
->config
);
1267 val
= ldl_p(vdev
->config
+ addr
);
1271 void virtio_config_writeb(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
1273 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1276 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1280 stb_p(vdev
->config
+ addr
, val
);
1282 if (k
->set_config
) {
1283 k
->set_config(vdev
, vdev
->config
);
1287 void virtio_config_writew(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
1289 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1290 uint16_t val
= data
;
1292 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1296 stw_p(vdev
->config
+ addr
, val
);
1298 if (k
->set_config
) {
1299 k
->set_config(vdev
, vdev
->config
);
1303 void virtio_config_writel(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
1305 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1306 uint32_t val
= data
;
1308 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1312 stl_p(vdev
->config
+ addr
, val
);
1314 if (k
->set_config
) {
1315 k
->set_config(vdev
, vdev
->config
);
1319 uint32_t virtio_config_modern_readb(VirtIODevice
*vdev
, uint32_t addr
)
1321 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1324 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1325 return (uint32_t)-1;
1328 k
->get_config(vdev
, vdev
->config
);
1330 val
= ldub_p(vdev
->config
+ addr
);
1334 uint32_t virtio_config_modern_readw(VirtIODevice
*vdev
, uint32_t addr
)
1336 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1339 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1340 return (uint32_t)-1;
1343 k
->get_config(vdev
, vdev
->config
);
1345 val
= lduw_le_p(vdev
->config
+ addr
);
1349 uint32_t virtio_config_modern_readl(VirtIODevice
*vdev
, uint32_t addr
)
1351 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1354 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1355 return (uint32_t)-1;
1358 k
->get_config(vdev
, vdev
->config
);
1360 val
= ldl_le_p(vdev
->config
+ addr
);
1364 void virtio_config_modern_writeb(VirtIODevice
*vdev
,
1365 uint32_t addr
, uint32_t data
)
1367 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1370 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1374 stb_p(vdev
->config
+ addr
, val
);
1376 if (k
->set_config
) {
1377 k
->set_config(vdev
, vdev
->config
);
1381 void virtio_config_modern_writew(VirtIODevice
*vdev
,
1382 uint32_t addr
, uint32_t data
)
1384 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1385 uint16_t val
= data
;
1387 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1391 stw_le_p(vdev
->config
+ addr
, val
);
1393 if (k
->set_config
) {
1394 k
->set_config(vdev
, vdev
->config
);
1398 void virtio_config_modern_writel(VirtIODevice
*vdev
,
1399 uint32_t addr
, uint32_t data
)
1401 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1402 uint32_t val
= data
;
1404 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1408 stl_le_p(vdev
->config
+ addr
, val
);
1410 if (k
->set_config
) {
1411 k
->set_config(vdev
, vdev
->config
);
1415 void virtio_queue_set_addr(VirtIODevice
*vdev
, int n
, hwaddr addr
)
1417 if (!vdev
->vq
[n
].vring
.num
) {
1420 vdev
->vq
[n
].vring
.desc
= addr
;
1421 virtio_queue_update_rings(vdev
, n
);
1424 hwaddr
virtio_queue_get_addr(VirtIODevice
*vdev
, int n
)
1426 return vdev
->vq
[n
].vring
.desc
;
1429 void virtio_queue_set_rings(VirtIODevice
*vdev
, int n
, hwaddr desc
,
1430 hwaddr avail
, hwaddr used
)
1432 if (!vdev
->vq
[n
].vring
.num
) {
1435 vdev
->vq
[n
].vring
.desc
= desc
;
1436 vdev
->vq
[n
].vring
.avail
= avail
;
1437 vdev
->vq
[n
].vring
.used
= used
;
1438 virtio_init_region_cache(vdev
, n
);
1441 void virtio_queue_set_num(VirtIODevice
*vdev
, int n
, int num
)
1443 /* Don't allow guest to flip queue between existent and
1444 * nonexistent states, or to set it to an invalid size.
1446 if (!!num
!= !!vdev
->vq
[n
].vring
.num
||
1447 num
> VIRTQUEUE_MAX_SIZE
||
1451 vdev
->vq
[n
].vring
.num
= num
;
1454 VirtQueue
*virtio_vector_first_queue(VirtIODevice
*vdev
, uint16_t vector
)
1456 return QLIST_FIRST(&vdev
->vector_queues
[vector
]);
1459 VirtQueue
*virtio_vector_next_queue(VirtQueue
*vq
)
1461 return QLIST_NEXT(vq
, node
);
1464 int virtio_queue_get_num(VirtIODevice
*vdev
, int n
)
1466 return vdev
->vq
[n
].vring
.num
;
1469 int virtio_queue_get_max_num(VirtIODevice
*vdev
, int n
)
1471 return vdev
->vq
[n
].vring
.num_default
;
1474 int virtio_get_num_queues(VirtIODevice
*vdev
)
1478 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
1479 if (!virtio_queue_get_num(vdev
, i
)) {
1487 void virtio_queue_set_align(VirtIODevice
*vdev
, int n
, int align
)
1489 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1490 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1492 /* virtio-1 compliant devices cannot change the alignment */
1493 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
1494 error_report("tried to modify queue alignment for virtio-1 device");
1497 /* Check that the transport told us it was going to do this
1498 * (so a buggy transport will immediately assert rather than
1499 * silently failing to migrate this state)
1501 assert(k
->has_variable_vring_alignment
);
1504 vdev
->vq
[n
].vring
.align
= align
;
1505 virtio_queue_update_rings(vdev
, n
);
1509 static bool virtio_queue_notify_aio_vq(VirtQueue
*vq
)
1511 if (vq
->vring
.desc
&& vq
->handle_aio_output
) {
1512 VirtIODevice
*vdev
= vq
->vdev
;
1514 trace_virtio_queue_notify(vdev
, vq
- vdev
->vq
, vq
);
1515 return vq
->handle_aio_output(vdev
, vq
);
1521 static void virtio_queue_notify_vq(VirtQueue
*vq
)
1523 if (vq
->vring
.desc
&& vq
->handle_output
) {
1524 VirtIODevice
*vdev
= vq
->vdev
;
1526 if (unlikely(vdev
->broken
)) {
1530 trace_virtio_queue_notify(vdev
, vq
- vdev
->vq
, vq
);
1531 vq
->handle_output(vdev
, vq
);
1535 void virtio_queue_notify(VirtIODevice
*vdev
, int n
)
1537 VirtQueue
*vq
= &vdev
->vq
[n
];
1539 if (unlikely(!vq
->vring
.desc
|| vdev
->broken
)) {
1543 trace_virtio_queue_notify(vdev
, vq
- vdev
->vq
, vq
);
1544 if (vq
->handle_aio_output
) {
1545 event_notifier_set(&vq
->host_notifier
);
1546 } else if (vq
->handle_output
) {
1547 vq
->handle_output(vdev
, vq
);
1551 uint16_t virtio_queue_vector(VirtIODevice
*vdev
, int n
)
1553 return n
< VIRTIO_QUEUE_MAX
? vdev
->vq
[n
].vector
:
1557 void virtio_queue_set_vector(VirtIODevice
*vdev
, int n
, uint16_t vector
)
1559 VirtQueue
*vq
= &vdev
->vq
[n
];
1561 if (n
< VIRTIO_QUEUE_MAX
) {
1562 if (vdev
->vector_queues
&&
1563 vdev
->vq
[n
].vector
!= VIRTIO_NO_VECTOR
) {
1564 QLIST_REMOVE(vq
, node
);
1566 vdev
->vq
[n
].vector
= vector
;
1567 if (vdev
->vector_queues
&&
1568 vector
!= VIRTIO_NO_VECTOR
) {
1569 QLIST_INSERT_HEAD(&vdev
->vector_queues
[vector
], vq
, node
);
1574 VirtQueue
*virtio_add_queue(VirtIODevice
*vdev
, int queue_size
,
1575 VirtIOHandleOutput handle_output
)
1579 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
1580 if (vdev
->vq
[i
].vring
.num
== 0)
1584 if (i
== VIRTIO_QUEUE_MAX
|| queue_size
> VIRTQUEUE_MAX_SIZE
)
1587 vdev
->vq
[i
].vring
.num
= queue_size
;
1588 vdev
->vq
[i
].vring
.num_default
= queue_size
;
1589 vdev
->vq
[i
].vring
.align
= VIRTIO_PCI_VRING_ALIGN
;
1590 vdev
->vq
[i
].handle_output
= handle_output
;
1591 vdev
->vq
[i
].handle_aio_output
= NULL
;
1593 return &vdev
->vq
[i
];
1596 void virtio_del_queue(VirtIODevice
*vdev
, int n
)
1598 if (n
< 0 || n
>= VIRTIO_QUEUE_MAX
) {
1602 vdev
->vq
[n
].vring
.num
= 0;
1603 vdev
->vq
[n
].vring
.num_default
= 0;
1606 static void virtio_set_isr(VirtIODevice
*vdev
, int value
)
1608 uint8_t old
= atomic_read(&vdev
->isr
);
1610 /* Do not write ISR if it does not change, so that its cacheline remains
1611 * shared in the common case where the guest does not read it.
1613 if ((old
& value
) != value
) {
1614 atomic_or(&vdev
->isr
, value
);
1618 /* Called within rcu_read_lock(). */
1619 static bool virtio_should_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
1623 /* We need to expose used array entries before checking used event. */
1625 /* Always notify when queue is empty (when feature acknowledge) */
1626 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_NOTIFY_ON_EMPTY
) &&
1627 !vq
->inuse
&& virtio_queue_empty(vq
)) {
1631 if (!virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
1632 return !(vring_avail_flags(vq
) & VRING_AVAIL_F_NO_INTERRUPT
);
1635 v
= vq
->signalled_used_valid
;
1636 vq
->signalled_used_valid
= true;
1637 old
= vq
->signalled_used
;
1638 new = vq
->signalled_used
= vq
->used_idx
;
1639 return !v
|| vring_need_event(vring_get_used_event(vq
), new, old
);
1642 void virtio_notify_irqfd(VirtIODevice
*vdev
, VirtQueue
*vq
)
1646 should_notify
= virtio_should_notify(vdev
, vq
);
1649 if (!should_notify
) {
1653 trace_virtio_notify_irqfd(vdev
, vq
);
1656 * virtio spec 1.0 says ISR bit 0 should be ignored with MSI, but
1657 * windows drivers included in virtio-win 1.8.0 (circa 2015) are
1658 * incorrectly polling this bit during crashdump and hibernation
1659 * in MSI mode, causing a hang if this bit is never updated.
1660 * Recent releases of Windows do not really shut down, but rather
1661 * log out and hibernate to make the next startup faster. Hence,
1662 * this manifested as a more serious hang during shutdown with
1664 * Next driver release from 2016 fixed this problem, so working around it
1665 * is not a must, but it's easy to do so let's do it here.
1667 * Note: it's safe to update ISR from any thread as it was switched
1668 * to an atomic operation.
1670 virtio_set_isr(vq
->vdev
, 0x1);
1671 event_notifier_set(&vq
->guest_notifier
);
1674 static void virtio_irq(VirtQueue
*vq
)
1676 virtio_set_isr(vq
->vdev
, 0x1);
1677 virtio_notify_vector(vq
->vdev
, vq
->vector
);
1680 void virtio_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
1684 should_notify
= virtio_should_notify(vdev
, vq
);
1687 if (!should_notify
) {
1691 trace_virtio_notify(vdev
, vq
);
1695 void virtio_notify_config(VirtIODevice
*vdev
)
1697 if (!(vdev
->status
& VIRTIO_CONFIG_S_DRIVER_OK
))
1700 virtio_set_isr(vdev
, 0x3);
1702 virtio_notify_vector(vdev
, vdev
->config_vector
);
1705 static bool virtio_device_endian_needed(void *opaque
)
1707 VirtIODevice
*vdev
= opaque
;
1709 assert(vdev
->device_endian
!= VIRTIO_DEVICE_ENDIAN_UNKNOWN
);
1710 if (!virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
1711 return vdev
->device_endian
!= virtio_default_endian();
1713 /* Devices conforming to VIRTIO 1.0 or later are always LE. */
1714 return vdev
->device_endian
!= VIRTIO_DEVICE_ENDIAN_LITTLE
;
1717 static bool virtio_64bit_features_needed(void *opaque
)
1719 VirtIODevice
*vdev
= opaque
;
1721 return (vdev
->host_features
>> 32) != 0;
1724 static bool virtio_virtqueue_needed(void *opaque
)
1726 VirtIODevice
*vdev
= opaque
;
1728 return virtio_host_has_feature(vdev
, VIRTIO_F_VERSION_1
);
1731 static bool virtio_ringsize_needed(void *opaque
)
1733 VirtIODevice
*vdev
= opaque
;
1736 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
1737 if (vdev
->vq
[i
].vring
.num
!= vdev
->vq
[i
].vring
.num_default
) {
1744 static bool virtio_extra_state_needed(void *opaque
)
1746 VirtIODevice
*vdev
= opaque
;
1747 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1748 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1750 return k
->has_extra_state
&&
1751 k
->has_extra_state(qbus
->parent
);
1754 static bool virtio_broken_needed(void *opaque
)
1756 VirtIODevice
*vdev
= opaque
;
1758 return vdev
->broken
;
1761 static const VMStateDescription vmstate_virtqueue
= {
1762 .name
= "virtqueue_state",
1764 .minimum_version_id
= 1,
1765 .fields
= (VMStateField
[]) {
1766 VMSTATE_UINT64(vring
.avail
, struct VirtQueue
),
1767 VMSTATE_UINT64(vring
.used
, struct VirtQueue
),
1768 VMSTATE_END_OF_LIST()
1772 static const VMStateDescription vmstate_virtio_virtqueues
= {
1773 .name
= "virtio/virtqueues",
1775 .minimum_version_id
= 1,
1776 .needed
= &virtio_virtqueue_needed
,
1777 .fields
= (VMStateField
[]) {
1778 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq
, struct VirtIODevice
,
1779 VIRTIO_QUEUE_MAX
, 0, vmstate_virtqueue
, VirtQueue
),
1780 VMSTATE_END_OF_LIST()
1784 static const VMStateDescription vmstate_ringsize
= {
1785 .name
= "ringsize_state",
1787 .minimum_version_id
= 1,
1788 .fields
= (VMStateField
[]) {
1789 VMSTATE_UINT32(vring
.num_default
, struct VirtQueue
),
1790 VMSTATE_END_OF_LIST()
1794 static const VMStateDescription vmstate_virtio_ringsize
= {
1795 .name
= "virtio/ringsize",
1797 .minimum_version_id
= 1,
1798 .needed
= &virtio_ringsize_needed
,
1799 .fields
= (VMStateField
[]) {
1800 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq
, struct VirtIODevice
,
1801 VIRTIO_QUEUE_MAX
, 0, vmstate_ringsize
, VirtQueue
),
1802 VMSTATE_END_OF_LIST()
1806 static int get_extra_state(QEMUFile
*f
, void *pv
, size_t size
,
1807 VMStateField
*field
)
1809 VirtIODevice
*vdev
= pv
;
1810 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1811 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1813 if (!k
->load_extra_state
) {
1816 return k
->load_extra_state(qbus
->parent
, f
);
1820 static int put_extra_state(QEMUFile
*f
, void *pv
, size_t size
,
1821 VMStateField
*field
, QJSON
*vmdesc
)
1823 VirtIODevice
*vdev
= pv
;
1824 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1825 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1827 k
->save_extra_state(qbus
->parent
, f
);
1831 static const VMStateInfo vmstate_info_extra_state
= {
1832 .name
= "virtqueue_extra_state",
1833 .get
= get_extra_state
,
1834 .put
= put_extra_state
,
1837 static const VMStateDescription vmstate_virtio_extra_state
= {
1838 .name
= "virtio/extra_state",
1840 .minimum_version_id
= 1,
1841 .needed
= &virtio_extra_state_needed
,
1842 .fields
= (VMStateField
[]) {
1844 .name
= "extra_state",
1846 .field_exists
= NULL
,
1848 .info
= &vmstate_info_extra_state
,
1849 .flags
= VMS_SINGLE
,
1852 VMSTATE_END_OF_LIST()
1856 static const VMStateDescription vmstate_virtio_device_endian
= {
1857 .name
= "virtio/device_endian",
1859 .minimum_version_id
= 1,
1860 .needed
= &virtio_device_endian_needed
,
1861 .fields
= (VMStateField
[]) {
1862 VMSTATE_UINT8(device_endian
, VirtIODevice
),
1863 VMSTATE_END_OF_LIST()
1867 static const VMStateDescription vmstate_virtio_64bit_features
= {
1868 .name
= "virtio/64bit_features",
1870 .minimum_version_id
= 1,
1871 .needed
= &virtio_64bit_features_needed
,
1872 .fields
= (VMStateField
[]) {
1873 VMSTATE_UINT64(guest_features
, VirtIODevice
),
1874 VMSTATE_END_OF_LIST()
1878 static const VMStateDescription vmstate_virtio_broken
= {
1879 .name
= "virtio/broken",
1881 .minimum_version_id
= 1,
1882 .needed
= &virtio_broken_needed
,
1883 .fields
= (VMStateField
[]) {
1884 VMSTATE_BOOL(broken
, VirtIODevice
),
1885 VMSTATE_END_OF_LIST()
1889 static const VMStateDescription vmstate_virtio
= {
1892 .minimum_version_id
= 1,
1893 .minimum_version_id_old
= 1,
1894 .fields
= (VMStateField
[]) {
1895 VMSTATE_END_OF_LIST()
1897 .subsections
= (const VMStateDescription
*[]) {
1898 &vmstate_virtio_device_endian
,
1899 &vmstate_virtio_64bit_features
,
1900 &vmstate_virtio_virtqueues
,
1901 &vmstate_virtio_ringsize
,
1902 &vmstate_virtio_broken
,
1903 &vmstate_virtio_extra_state
,
1908 int virtio_save(VirtIODevice
*vdev
, QEMUFile
*f
)
1910 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1911 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1912 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1913 uint32_t guest_features_lo
= (vdev
->guest_features
& 0xffffffff);
1916 if (k
->save_config
) {
1917 k
->save_config(qbus
->parent
, f
);
1920 qemu_put_8s(f
, &vdev
->status
);
1921 qemu_put_8s(f
, &vdev
->isr
);
1922 qemu_put_be16s(f
, &vdev
->queue_sel
);
1923 qemu_put_be32s(f
, &guest_features_lo
);
1924 qemu_put_be32(f
, vdev
->config_len
);
1925 qemu_put_buffer(f
, vdev
->config
, vdev
->config_len
);
1927 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
1928 if (vdev
->vq
[i
].vring
.num
== 0)
1932 qemu_put_be32(f
, i
);
1934 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
1935 if (vdev
->vq
[i
].vring
.num
== 0)
1938 qemu_put_be32(f
, vdev
->vq
[i
].vring
.num
);
1939 if (k
->has_variable_vring_alignment
) {
1940 qemu_put_be32(f
, vdev
->vq
[i
].vring
.align
);
1943 * Save desc now, the rest of the ring addresses are saved in
1944 * subsections for VIRTIO-1 devices.
1946 qemu_put_be64(f
, vdev
->vq
[i
].vring
.desc
);
1947 qemu_put_be16s(f
, &vdev
->vq
[i
].last_avail_idx
);
1948 if (k
->save_queue
) {
1949 k
->save_queue(qbus
->parent
, i
, f
);
1953 if (vdc
->save
!= NULL
) {
1958 int ret
= vmstate_save_state(f
, vdc
->vmsd
, vdev
, NULL
);
1965 return vmstate_save_state(f
, &vmstate_virtio
, vdev
, NULL
);
1968 /* A wrapper for use as a VMState .put function */
1969 static int virtio_device_put(QEMUFile
*f
, void *opaque
, size_t size
,
1970 VMStateField
*field
, QJSON
*vmdesc
)
1972 return virtio_save(VIRTIO_DEVICE(opaque
), f
);
1975 /* A wrapper for use as a VMState .get function */
1976 static int virtio_device_get(QEMUFile
*f
, void *opaque
, size_t size
,
1977 VMStateField
*field
)
1979 VirtIODevice
*vdev
= VIRTIO_DEVICE(opaque
);
1980 DeviceClass
*dc
= DEVICE_CLASS(VIRTIO_DEVICE_GET_CLASS(vdev
));
1982 return virtio_load(vdev
, f
, dc
->vmsd
->version_id
);
1985 const VMStateInfo virtio_vmstate_info
= {
1987 .get
= virtio_device_get
,
1988 .put
= virtio_device_put
,
1991 static int virtio_set_features_nocheck(VirtIODevice
*vdev
, uint64_t val
)
1993 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1994 bool bad
= (val
& ~(vdev
->host_features
)) != 0;
1996 val
&= vdev
->host_features
;
1997 if (k
->set_features
) {
1998 k
->set_features(vdev
, val
);
2000 vdev
->guest_features
= val
;
2001 return bad
? -1 : 0;
2004 int virtio_set_features(VirtIODevice
*vdev
, uint64_t val
)
2007 * The driver must not attempt to set features after feature negotiation
2010 if (vdev
->status
& VIRTIO_CONFIG_S_FEATURES_OK
) {
2013 return virtio_set_features_nocheck(vdev
, val
);
2016 int virtio_load(VirtIODevice
*vdev
, QEMUFile
*f
, int version_id
)
2022 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2023 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2024 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2027 * We poison the endianness to ensure it does not get used before
2028 * subsections have been loaded.
2030 vdev
->device_endian
= VIRTIO_DEVICE_ENDIAN_UNKNOWN
;
2032 if (k
->load_config
) {
2033 ret
= k
->load_config(qbus
->parent
, f
);
2038 qemu_get_8s(f
, &vdev
->status
);
2039 qemu_get_8s(f
, &vdev
->isr
);
2040 qemu_get_be16s(f
, &vdev
->queue_sel
);
2041 if (vdev
->queue_sel
>= VIRTIO_QUEUE_MAX
) {
2044 qemu_get_be32s(f
, &features
);
2047 * Temporarily set guest_features low bits - needed by
2048 * virtio net load code testing for VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
2049 * VIRTIO_NET_F_GUEST_ANNOUNCE and VIRTIO_NET_F_CTRL_VQ.
2051 * Note: devices should always test host features in future - don't create
2052 * new dependencies like this.
2054 vdev
->guest_features
= features
;
2056 config_len
= qemu_get_be32(f
);
2059 * There are cases where the incoming config can be bigger or smaller
2060 * than what we have; so load what we have space for, and skip
2061 * any excess that's in the stream.
2063 qemu_get_buffer(f
, vdev
->config
, MIN(config_len
, vdev
->config_len
));
2065 while (config_len
> vdev
->config_len
) {
2070 num
= qemu_get_be32(f
);
2072 if (num
> VIRTIO_QUEUE_MAX
) {
2073 error_report("Invalid number of virtqueues: 0x%x", num
);
2077 for (i
= 0; i
< num
; i
++) {
2078 vdev
->vq
[i
].vring
.num
= qemu_get_be32(f
);
2079 if (k
->has_variable_vring_alignment
) {
2080 vdev
->vq
[i
].vring
.align
= qemu_get_be32(f
);
2082 vdev
->vq
[i
].vring
.desc
= qemu_get_be64(f
);
2083 qemu_get_be16s(f
, &vdev
->vq
[i
].last_avail_idx
);
2084 vdev
->vq
[i
].signalled_used_valid
= false;
2085 vdev
->vq
[i
].notification
= true;
2087 if (!vdev
->vq
[i
].vring
.desc
&& vdev
->vq
[i
].last_avail_idx
) {
2088 error_report("VQ %d address 0x0 "
2089 "inconsistent with Host index 0x%x",
2090 i
, vdev
->vq
[i
].last_avail_idx
);
2093 if (k
->load_queue
) {
2094 ret
= k
->load_queue(qbus
->parent
, i
, f
);
2100 virtio_notify_vector(vdev
, VIRTIO_NO_VECTOR
);
2102 if (vdc
->load
!= NULL
) {
2103 ret
= vdc
->load(vdev
, f
, version_id
);
2110 ret
= vmstate_load_state(f
, vdc
->vmsd
, vdev
, version_id
);
2117 ret
= vmstate_load_state(f
, &vmstate_virtio
, vdev
, 1);
2122 if (vdev
->device_endian
== VIRTIO_DEVICE_ENDIAN_UNKNOWN
) {
2123 vdev
->device_endian
= virtio_default_endian();
2126 if (virtio_64bit_features_needed(vdev
)) {
2128 * Subsection load filled vdev->guest_features. Run them
2129 * through virtio_set_features to sanity-check them against
2132 uint64_t features64
= vdev
->guest_features
;
2133 if (virtio_set_features_nocheck(vdev
, features64
) < 0) {
2134 error_report("Features 0x%" PRIx64
" unsupported. "
2135 "Allowed features: 0x%" PRIx64
,
2136 features64
, vdev
->host_features
);
2140 if (virtio_set_features_nocheck(vdev
, features
) < 0) {
2141 error_report("Features 0x%x unsupported. "
2142 "Allowed features: 0x%" PRIx64
,
2143 features
, vdev
->host_features
);
2149 for (i
= 0; i
< num
; i
++) {
2150 if (vdev
->vq
[i
].vring
.desc
) {
2154 * VIRTIO-1 devices migrate desc, used, and avail ring addresses so
2155 * only the region cache needs to be set up. Legacy devices need
2156 * to calculate used and avail ring addresses based on the desc
2159 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
2160 virtio_init_region_cache(vdev
, i
);
2162 virtio_queue_update_rings(vdev
, i
);
2165 nheads
= vring_avail_idx(&vdev
->vq
[i
]) - vdev
->vq
[i
].last_avail_idx
;
2166 /* Check it isn't doing strange things with descriptor numbers. */
2167 if (nheads
> vdev
->vq
[i
].vring
.num
) {
2168 error_report("VQ %d size 0x%x Guest index 0x%x "
2169 "inconsistent with Host index 0x%x: delta 0x%x",
2170 i
, vdev
->vq
[i
].vring
.num
,
2171 vring_avail_idx(&vdev
->vq
[i
]),
2172 vdev
->vq
[i
].last_avail_idx
, nheads
);
2175 vdev
->vq
[i
].used_idx
= vring_used_idx(&vdev
->vq
[i
]);
2176 vdev
->vq
[i
].shadow_avail_idx
= vring_avail_idx(&vdev
->vq
[i
]);
2179 * Some devices migrate VirtQueueElements that have been popped
2180 * from the avail ring but not yet returned to the used ring.
2181 * Since max ring size < UINT16_MAX it's safe to use modulo
2182 * UINT16_MAX + 1 subtraction.
2184 vdev
->vq
[i
].inuse
= (uint16_t)(vdev
->vq
[i
].last_avail_idx
-
2185 vdev
->vq
[i
].used_idx
);
2186 if (vdev
->vq
[i
].inuse
> vdev
->vq
[i
].vring
.num
) {
2187 error_report("VQ %d size 0x%x < last_avail_idx 0x%x - "
2189 i
, vdev
->vq
[i
].vring
.num
,
2190 vdev
->vq
[i
].last_avail_idx
,
2191 vdev
->vq
[i
].used_idx
);
2201 void virtio_cleanup(VirtIODevice
*vdev
)
2203 qemu_del_vm_change_state_handler(vdev
->vmstate
);
2206 static void virtio_vmstate_change(void *opaque
, int running
, RunState state
)
2208 VirtIODevice
*vdev
= opaque
;
2209 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2210 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2211 bool backend_run
= running
&& (vdev
->status
& VIRTIO_CONFIG_S_DRIVER_OK
);
2212 vdev
->vm_running
= running
;
2215 virtio_set_status(vdev
, vdev
->status
);
2218 if (k
->vmstate_change
) {
2219 k
->vmstate_change(qbus
->parent
, backend_run
);
2223 virtio_set_status(vdev
, vdev
->status
);
2227 void virtio_instance_init_common(Object
*proxy_obj
, void *data
,
2228 size_t vdev_size
, const char *vdev_name
)
2230 DeviceState
*vdev
= data
;
2232 object_initialize(vdev
, vdev_size
, vdev_name
);
2233 object_property_add_child(proxy_obj
, "virtio-backend", OBJECT(vdev
), NULL
);
2234 object_unref(OBJECT(vdev
));
2235 qdev_alias_all_properties(vdev
, proxy_obj
);
2238 void virtio_init(VirtIODevice
*vdev
, const char *name
,
2239 uint16_t device_id
, size_t config_size
)
2241 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2242 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2244 int nvectors
= k
->query_nvectors
? k
->query_nvectors(qbus
->parent
) : 0;
2247 vdev
->vector_queues
=
2248 g_malloc0(sizeof(*vdev
->vector_queues
) * nvectors
);
2251 vdev
->device_id
= device_id
;
2253 atomic_set(&vdev
->isr
, 0);
2254 vdev
->queue_sel
= 0;
2255 vdev
->config_vector
= VIRTIO_NO_VECTOR
;
2256 vdev
->vq
= g_malloc0(sizeof(VirtQueue
) * VIRTIO_QUEUE_MAX
);
2257 vdev
->vm_running
= runstate_is_running();
2258 vdev
->broken
= false;
2259 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2260 vdev
->vq
[i
].vector
= VIRTIO_NO_VECTOR
;
2261 vdev
->vq
[i
].vdev
= vdev
;
2262 vdev
->vq
[i
].queue_index
= i
;
2266 vdev
->config_len
= config_size
;
2267 if (vdev
->config_len
) {
2268 vdev
->config
= g_malloc0(config_size
);
2270 vdev
->config
= NULL
;
2272 vdev
->vmstate
= qemu_add_vm_change_state_handler(virtio_vmstate_change
,
2274 vdev
->device_endian
= virtio_default_endian();
2275 vdev
->use_guest_notifier_mask
= true;
2278 hwaddr
virtio_queue_get_desc_addr(VirtIODevice
*vdev
, int n
)
2280 return vdev
->vq
[n
].vring
.desc
;
2283 hwaddr
virtio_queue_get_avail_addr(VirtIODevice
*vdev
, int n
)
2285 return vdev
->vq
[n
].vring
.avail
;
2288 hwaddr
virtio_queue_get_used_addr(VirtIODevice
*vdev
, int n
)
2290 return vdev
->vq
[n
].vring
.used
;
2293 hwaddr
virtio_queue_get_desc_size(VirtIODevice
*vdev
, int n
)
2295 return sizeof(VRingDesc
) * vdev
->vq
[n
].vring
.num
;
2298 hwaddr
virtio_queue_get_avail_size(VirtIODevice
*vdev
, int n
)
2300 return offsetof(VRingAvail
, ring
) +
2301 sizeof(uint16_t) * vdev
->vq
[n
].vring
.num
;
2304 hwaddr
virtio_queue_get_used_size(VirtIODevice
*vdev
, int n
)
2306 return offsetof(VRingUsed
, ring
) +
2307 sizeof(VRingUsedElem
) * vdev
->vq
[n
].vring
.num
;
2310 uint16_t virtio_queue_get_last_avail_idx(VirtIODevice
*vdev
, int n
)
2312 return vdev
->vq
[n
].last_avail_idx
;
2315 void virtio_queue_set_last_avail_idx(VirtIODevice
*vdev
, int n
, uint16_t idx
)
2317 vdev
->vq
[n
].last_avail_idx
= idx
;
2318 vdev
->vq
[n
].shadow_avail_idx
= idx
;
2321 void virtio_queue_restore_last_avail_idx(VirtIODevice
*vdev
, int n
)
2324 if (vdev
->vq
[n
].vring
.desc
) {
2325 vdev
->vq
[n
].last_avail_idx
= vring_used_idx(&vdev
->vq
[n
]);
2326 vdev
->vq
[n
].shadow_avail_idx
= vdev
->vq
[n
].last_avail_idx
;
2331 void virtio_queue_update_used_idx(VirtIODevice
*vdev
, int n
)
2334 if (vdev
->vq
[n
].vring
.desc
) {
2335 vdev
->vq
[n
].used_idx
= vring_used_idx(&vdev
->vq
[n
]);
2340 void virtio_queue_invalidate_signalled_used(VirtIODevice
*vdev
, int n
)
2342 vdev
->vq
[n
].signalled_used_valid
= false;
2345 VirtQueue
*virtio_get_queue(VirtIODevice
*vdev
, int n
)
2347 return vdev
->vq
+ n
;
2350 uint16_t virtio_get_queue_index(VirtQueue
*vq
)
2352 return vq
->queue_index
;
2355 static void virtio_queue_guest_notifier_read(EventNotifier
*n
)
2357 VirtQueue
*vq
= container_of(n
, VirtQueue
, guest_notifier
);
2358 if (event_notifier_test_and_clear(n
)) {
2363 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue
*vq
, bool assign
,
2366 if (assign
&& !with_irqfd
) {
2367 event_notifier_set_handler(&vq
->guest_notifier
,
2368 virtio_queue_guest_notifier_read
);
2370 event_notifier_set_handler(&vq
->guest_notifier
, NULL
);
2373 /* Test and clear notifier before closing it,
2374 * in case poll callback didn't have time to run. */
2375 virtio_queue_guest_notifier_read(&vq
->guest_notifier
);
2379 EventNotifier
*virtio_queue_get_guest_notifier(VirtQueue
*vq
)
2381 return &vq
->guest_notifier
;
2384 static void virtio_queue_host_notifier_aio_read(EventNotifier
*n
)
2386 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
2387 if (event_notifier_test_and_clear(n
)) {
2388 virtio_queue_notify_aio_vq(vq
);
2392 static void virtio_queue_host_notifier_aio_poll_begin(EventNotifier
*n
)
2394 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
2396 virtio_queue_set_notification(vq
, 0);
2399 static bool virtio_queue_host_notifier_aio_poll(void *opaque
)
2401 EventNotifier
*n
= opaque
;
2402 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
2405 if (!vq
->vring
.desc
|| virtio_queue_empty(vq
)) {
2409 progress
= virtio_queue_notify_aio_vq(vq
);
2411 /* In case the handler function re-enabled notifications */
2412 virtio_queue_set_notification(vq
, 0);
2416 static void virtio_queue_host_notifier_aio_poll_end(EventNotifier
*n
)
2418 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
2420 /* Caller polls once more after this to catch requests that race with us */
2421 virtio_queue_set_notification(vq
, 1);
2424 void virtio_queue_aio_set_host_notifier_handler(VirtQueue
*vq
, AioContext
*ctx
,
2425 VirtIOHandleAIOOutput handle_output
)
2427 if (handle_output
) {
2428 vq
->handle_aio_output
= handle_output
;
2429 aio_set_event_notifier(ctx
, &vq
->host_notifier
, true,
2430 virtio_queue_host_notifier_aio_read
,
2431 virtio_queue_host_notifier_aio_poll
);
2432 aio_set_event_notifier_poll(ctx
, &vq
->host_notifier
,
2433 virtio_queue_host_notifier_aio_poll_begin
,
2434 virtio_queue_host_notifier_aio_poll_end
);
2436 aio_set_event_notifier(ctx
, &vq
->host_notifier
, true, NULL
, NULL
);
2437 /* Test and clear notifier before after disabling event,
2438 * in case poll callback didn't have time to run. */
2439 virtio_queue_host_notifier_aio_read(&vq
->host_notifier
);
2440 vq
->handle_aio_output
= NULL
;
2444 void virtio_queue_host_notifier_read(EventNotifier
*n
)
2446 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
2447 if (event_notifier_test_and_clear(n
)) {
2448 virtio_queue_notify_vq(vq
);
2452 EventNotifier
*virtio_queue_get_host_notifier(VirtQueue
*vq
)
2454 return &vq
->host_notifier
;
2457 void virtio_device_set_child_bus_name(VirtIODevice
*vdev
, char *bus_name
)
2459 g_free(vdev
->bus_name
);
2460 vdev
->bus_name
= g_strdup(bus_name
);
2463 void GCC_FMT_ATTR(2, 3) virtio_error(VirtIODevice
*vdev
, const char *fmt
, ...)
2468 error_vreport(fmt
, ap
);
2471 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
2472 vdev
->status
= vdev
->status
| VIRTIO_CONFIG_S_NEEDS_RESET
;
2473 virtio_notify_config(vdev
);
2476 vdev
->broken
= true;
2479 static void virtio_memory_listener_commit(MemoryListener
*listener
)
2481 VirtIODevice
*vdev
= container_of(listener
, VirtIODevice
, listener
);
2484 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2485 if (vdev
->vq
[i
].vring
.num
== 0) {
2488 virtio_init_region_cache(vdev
, i
);
2492 static void virtio_device_realize(DeviceState
*dev
, Error
**errp
)
2494 VirtIODevice
*vdev
= VIRTIO_DEVICE(dev
);
2495 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(dev
);
2498 /* Devices should either use vmsd or the load/save methods */
2499 assert(!vdc
->vmsd
|| !vdc
->load
);
2501 if (vdc
->realize
!= NULL
) {
2502 vdc
->realize(dev
, &err
);
2504 error_propagate(errp
, err
);
2509 virtio_bus_device_plugged(vdev
, &err
);
2511 error_propagate(errp
, err
);
2512 vdc
->unrealize(dev
, NULL
);
2516 vdev
->listener
.commit
= virtio_memory_listener_commit
;
2517 memory_listener_register(&vdev
->listener
, vdev
->dma_as
);
2520 static void virtio_device_unrealize(DeviceState
*dev
, Error
**errp
)
2522 VirtIODevice
*vdev
= VIRTIO_DEVICE(dev
);
2523 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(dev
);
2526 virtio_bus_device_unplugged(vdev
);
2528 if (vdc
->unrealize
!= NULL
) {
2529 vdc
->unrealize(dev
, &err
);
2531 error_propagate(errp
, err
);
2536 g_free(vdev
->bus_name
);
2537 vdev
->bus_name
= NULL
;
2540 static void virtio_device_free_virtqueues(VirtIODevice
*vdev
)
2547 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2548 if (vdev
->vq
[i
].vring
.num
== 0) {
2551 virtio_virtqueue_reset_region_cache(&vdev
->vq
[i
]);
2556 static void virtio_device_instance_finalize(Object
*obj
)
2558 VirtIODevice
*vdev
= VIRTIO_DEVICE(obj
);
2560 memory_listener_unregister(&vdev
->listener
);
2561 virtio_device_free_virtqueues(vdev
);
2563 g_free(vdev
->config
);
2564 g_free(vdev
->vector_queues
);
2567 static Property virtio_properties
[] = {
2568 DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice
, host_features
),
2569 DEFINE_PROP_END_OF_LIST(),
2572 static int virtio_device_start_ioeventfd_impl(VirtIODevice
*vdev
)
2574 VirtioBusState
*qbus
= VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev
)));
2577 for (n
= 0; n
< VIRTIO_QUEUE_MAX
; n
++) {
2578 VirtQueue
*vq
= &vdev
->vq
[n
];
2579 if (!virtio_queue_get_num(vdev
, n
)) {
2582 r
= virtio_bus_set_host_notifier(qbus
, n
, true);
2587 event_notifier_set_handler(&vq
->host_notifier
,
2588 virtio_queue_host_notifier_read
);
2591 for (n
= 0; n
< VIRTIO_QUEUE_MAX
; n
++) {
2592 /* Kick right away to begin processing requests already in vring */
2593 VirtQueue
*vq
= &vdev
->vq
[n
];
2594 if (!vq
->vring
.num
) {
2597 event_notifier_set(&vq
->host_notifier
);
2603 VirtQueue
*vq
= &vdev
->vq
[n
];
2604 if (!virtio_queue_get_num(vdev
, n
)) {
2608 event_notifier_set_handler(&vq
->host_notifier
, NULL
);
2609 r
= virtio_bus_set_host_notifier(qbus
, n
, false);
2615 int virtio_device_start_ioeventfd(VirtIODevice
*vdev
)
2617 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2618 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
2620 return virtio_bus_start_ioeventfd(vbus
);
2623 static void virtio_device_stop_ioeventfd_impl(VirtIODevice
*vdev
)
2625 VirtioBusState
*qbus
= VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev
)));
2628 for (n
= 0; n
< VIRTIO_QUEUE_MAX
; n
++) {
2629 VirtQueue
*vq
= &vdev
->vq
[n
];
2631 if (!virtio_queue_get_num(vdev
, n
)) {
2634 event_notifier_set_handler(&vq
->host_notifier
, NULL
);
2635 r
= virtio_bus_set_host_notifier(qbus
, n
, false);
2640 void virtio_device_stop_ioeventfd(VirtIODevice
*vdev
)
2642 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2643 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
2645 virtio_bus_stop_ioeventfd(vbus
);
2648 int virtio_device_grab_ioeventfd(VirtIODevice
*vdev
)
2650 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2651 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
2653 return virtio_bus_grab_ioeventfd(vbus
);
2656 void virtio_device_release_ioeventfd(VirtIODevice
*vdev
)
2658 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2659 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
2661 virtio_bus_release_ioeventfd(vbus
);
2664 static void virtio_device_class_init(ObjectClass
*klass
, void *data
)
2666 /* Set the default value here. */
2667 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_CLASS(klass
);
2668 DeviceClass
*dc
= DEVICE_CLASS(klass
);
2670 dc
->realize
= virtio_device_realize
;
2671 dc
->unrealize
= virtio_device_unrealize
;
2672 dc
->bus_type
= TYPE_VIRTIO_BUS
;
2673 dc
->props
= virtio_properties
;
2674 vdc
->start_ioeventfd
= virtio_device_start_ioeventfd_impl
;
2675 vdc
->stop_ioeventfd
= virtio_device_stop_ioeventfd_impl
;
2677 vdc
->legacy_features
|= VIRTIO_LEGACY_FEATURES
;
2680 bool virtio_device_ioeventfd_enabled(VirtIODevice
*vdev
)
2682 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2683 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
2685 return virtio_bus_ioeventfd_enabled(vbus
);
2688 static const TypeInfo virtio_device_info
= {
2689 .name
= TYPE_VIRTIO_DEVICE
,
2690 .parent
= TYPE_DEVICE
,
2691 .instance_size
= sizeof(VirtIODevice
),
2692 .class_init
= virtio_device_class_init
,
2693 .instance_finalize
= virtio_device_instance_finalize
,
2695 .class_size
= sizeof(VirtioDeviceClass
),
2698 static void virtio_register_types(void)
2700 type_register_static(&virtio_device_info
);
2703 type_init(virtio_register_types
)