4 * Copyright IBM, Corp. 2007
7 * Anthony Liguori <aliguori@us.ibm.com>
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
14 #include "qemu/osdep.h"
15 #include "qapi/error.h"
18 #include "qemu/error-report.h"
20 #include "qemu/main-loop.h"
21 #include "qemu/module.h"
22 #include "hw/virtio/virtio.h"
23 #include "migration/qemu-file-types.h"
24 #include "qemu/atomic.h"
25 #include "hw/virtio/virtio-bus.h"
26 #include "hw/qdev-properties.h"
27 #include "hw/virtio/virtio-access.h"
28 #include "sysemu/dma.h"
29 #include "sysemu/runstate.h"
30 #include "standard-headers/linux/virtio_ids.h"
33 * The alignment to use between consumer and producer parts of vring.
34 * x86 pagesize again. This is the default, used by transports like PCI
35 * which don't provide a means for the guest to tell the host the alignment.
37 #define VIRTIO_PCI_VRING_ALIGN 4096
39 typedef struct VRingDesc
47 typedef struct VRingPackedDesc
{
54 typedef struct VRingAvail
61 typedef struct VRingUsedElem
67 typedef struct VRingUsed
74 typedef struct VRingMemoryRegionCaches
{
76 MemoryRegionCache desc
;
77 MemoryRegionCache avail
;
78 MemoryRegionCache used
;
79 } VRingMemoryRegionCaches
;
84 unsigned int num_default
;
89 VRingMemoryRegionCaches
*caches
;
92 typedef struct VRingPackedDescEvent
{
95 } VRingPackedDescEvent
;
100 VirtQueueElement
*used_elems
;
102 /* Next head to pop */
103 uint16_t last_avail_idx
;
104 bool last_avail_wrap_counter
;
106 /* Last avail_idx read from VQ. */
107 uint16_t shadow_avail_idx
;
108 bool shadow_avail_wrap_counter
;
111 bool used_wrap_counter
;
113 /* Last used index value we have signalled on */
114 uint16_t signalled_used
;
116 /* Last used index value we have signalled on */
117 bool signalled_used_valid
;
119 /* Notification enabled? */
122 uint16_t queue_index
;
127 VirtIOHandleOutput handle_output
;
128 VirtIOHandleAIOOutput handle_aio_output
;
130 EventNotifier guest_notifier
;
131 EventNotifier host_notifier
;
132 bool host_notifier_enabled
;
133 QLIST_ENTRY(VirtQueue
) node
;
136 /* Called within call_rcu(). */
137 static void virtio_free_region_cache(VRingMemoryRegionCaches
*caches
)
139 assert(caches
!= NULL
);
140 address_space_cache_destroy(&caches
->desc
);
141 address_space_cache_destroy(&caches
->avail
);
142 address_space_cache_destroy(&caches
->used
);
146 static void virtio_virtqueue_reset_region_cache(struct VirtQueue
*vq
)
148 VRingMemoryRegionCaches
*caches
;
150 caches
= qatomic_read(&vq
->vring
.caches
);
151 qatomic_rcu_set(&vq
->vring
.caches
, NULL
);
153 call_rcu(caches
, virtio_free_region_cache
, rcu
);
157 static void virtio_init_region_cache(VirtIODevice
*vdev
, int n
)
159 VirtQueue
*vq
= &vdev
->vq
[n
];
160 VRingMemoryRegionCaches
*old
= vq
->vring
.caches
;
161 VRingMemoryRegionCaches
*new = NULL
;
167 addr
= vq
->vring
.desc
;
171 new = g_new0(VRingMemoryRegionCaches
, 1);
172 size
= virtio_queue_get_desc_size(vdev
, n
);
173 packed
= virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
) ?
175 len
= address_space_cache_init(&new->desc
, vdev
->dma_as
,
178 virtio_error(vdev
, "Cannot map desc");
182 size
= virtio_queue_get_used_size(vdev
, n
);
183 len
= address_space_cache_init(&new->used
, vdev
->dma_as
,
184 vq
->vring
.used
, size
, true);
186 virtio_error(vdev
, "Cannot map used");
190 size
= virtio_queue_get_avail_size(vdev
, n
);
191 len
= address_space_cache_init(&new->avail
, vdev
->dma_as
,
192 vq
->vring
.avail
, size
, false);
194 virtio_error(vdev
, "Cannot map avail");
198 qatomic_rcu_set(&vq
->vring
.caches
, new);
200 call_rcu(old
, virtio_free_region_cache
, rcu
);
205 address_space_cache_destroy(&new->avail
);
207 address_space_cache_destroy(&new->used
);
209 address_space_cache_destroy(&new->desc
);
212 virtio_virtqueue_reset_region_cache(vq
);
215 /* virt queue functions */
216 void virtio_queue_update_rings(VirtIODevice
*vdev
, int n
)
218 VRing
*vring
= &vdev
->vq
[n
].vring
;
220 if (!vring
->num
|| !vring
->desc
|| !vring
->align
) {
221 /* not yet setup -> nothing to do */
224 vring
->avail
= vring
->desc
+ vring
->num
* sizeof(VRingDesc
);
225 vring
->used
= vring_align(vring
->avail
+
226 offsetof(VRingAvail
, ring
[vring
->num
]),
228 virtio_init_region_cache(vdev
, n
);
231 /* Called within rcu_read_lock(). */
232 static void vring_split_desc_read(VirtIODevice
*vdev
, VRingDesc
*desc
,
233 MemoryRegionCache
*cache
, int i
)
235 address_space_read_cached(cache
, i
* sizeof(VRingDesc
),
236 desc
, sizeof(VRingDesc
));
237 virtio_tswap64s(vdev
, &desc
->addr
);
238 virtio_tswap32s(vdev
, &desc
->len
);
239 virtio_tswap16s(vdev
, &desc
->flags
);
240 virtio_tswap16s(vdev
, &desc
->next
);
243 static void vring_packed_event_read(VirtIODevice
*vdev
,
244 MemoryRegionCache
*cache
,
245 VRingPackedDescEvent
*e
)
247 hwaddr off_off
= offsetof(VRingPackedDescEvent
, off_wrap
);
248 hwaddr off_flags
= offsetof(VRingPackedDescEvent
, flags
);
250 e
->flags
= virtio_lduw_phys_cached(vdev
, cache
, off_flags
);
251 /* Make sure flags is seen before off_wrap */
253 e
->off_wrap
= virtio_lduw_phys_cached(vdev
, cache
, off_off
);
254 virtio_tswap16s(vdev
, &e
->flags
);
257 static void vring_packed_off_wrap_write(VirtIODevice
*vdev
,
258 MemoryRegionCache
*cache
,
261 hwaddr off
= offsetof(VRingPackedDescEvent
, off_wrap
);
263 virtio_stw_phys_cached(vdev
, cache
, off
, off_wrap
);
264 address_space_cache_invalidate(cache
, off
, sizeof(off_wrap
));
267 static void vring_packed_flags_write(VirtIODevice
*vdev
,
268 MemoryRegionCache
*cache
, uint16_t flags
)
270 hwaddr off
= offsetof(VRingPackedDescEvent
, flags
);
272 virtio_stw_phys_cached(vdev
, cache
, off
, flags
);
273 address_space_cache_invalidate(cache
, off
, sizeof(flags
));
276 /* Called within rcu_read_lock(). */
277 static VRingMemoryRegionCaches
*vring_get_region_caches(struct VirtQueue
*vq
)
279 return qatomic_rcu_read(&vq
->vring
.caches
);
282 /* Called within rcu_read_lock(). */
283 static inline uint16_t vring_avail_flags(VirtQueue
*vq
)
285 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
286 hwaddr pa
= offsetof(VRingAvail
, flags
);
292 return virtio_lduw_phys_cached(vq
->vdev
, &caches
->avail
, pa
);
295 /* Called within rcu_read_lock(). */
296 static inline uint16_t vring_avail_idx(VirtQueue
*vq
)
298 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
299 hwaddr pa
= offsetof(VRingAvail
, idx
);
305 vq
->shadow_avail_idx
= virtio_lduw_phys_cached(vq
->vdev
, &caches
->avail
, pa
);
306 return vq
->shadow_avail_idx
;
309 /* Called within rcu_read_lock(). */
310 static inline uint16_t vring_avail_ring(VirtQueue
*vq
, int i
)
312 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
313 hwaddr pa
= offsetof(VRingAvail
, ring
[i
]);
319 return virtio_lduw_phys_cached(vq
->vdev
, &caches
->avail
, pa
);
322 /* Called within rcu_read_lock(). */
323 static inline uint16_t vring_get_used_event(VirtQueue
*vq
)
325 return vring_avail_ring(vq
, vq
->vring
.num
);
328 /* Called within rcu_read_lock(). */
329 static inline void vring_used_write(VirtQueue
*vq
, VRingUsedElem
*uelem
,
332 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
333 hwaddr pa
= offsetof(VRingUsed
, ring
[i
]);
339 virtio_tswap32s(vq
->vdev
, &uelem
->id
);
340 virtio_tswap32s(vq
->vdev
, &uelem
->len
);
341 address_space_write_cached(&caches
->used
, pa
, uelem
, sizeof(VRingUsedElem
));
342 address_space_cache_invalidate(&caches
->used
, pa
, sizeof(VRingUsedElem
));
345 /* Called within rcu_read_lock(). */
346 static uint16_t vring_used_idx(VirtQueue
*vq
)
348 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
349 hwaddr pa
= offsetof(VRingUsed
, idx
);
355 return virtio_lduw_phys_cached(vq
->vdev
, &caches
->used
, pa
);
358 /* Called within rcu_read_lock(). */
359 static inline void vring_used_idx_set(VirtQueue
*vq
, uint16_t val
)
361 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
362 hwaddr pa
= offsetof(VRingUsed
, idx
);
365 virtio_stw_phys_cached(vq
->vdev
, &caches
->used
, pa
, val
);
366 address_space_cache_invalidate(&caches
->used
, pa
, sizeof(val
));
372 /* Called within rcu_read_lock(). */
373 static inline void vring_used_flags_set_bit(VirtQueue
*vq
, int mask
)
375 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
376 VirtIODevice
*vdev
= vq
->vdev
;
377 hwaddr pa
= offsetof(VRingUsed
, flags
);
384 flags
= virtio_lduw_phys_cached(vq
->vdev
, &caches
->used
, pa
);
385 virtio_stw_phys_cached(vdev
, &caches
->used
, pa
, flags
| mask
);
386 address_space_cache_invalidate(&caches
->used
, pa
, sizeof(flags
));
389 /* Called within rcu_read_lock(). */
390 static inline void vring_used_flags_unset_bit(VirtQueue
*vq
, int mask
)
392 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
393 VirtIODevice
*vdev
= vq
->vdev
;
394 hwaddr pa
= offsetof(VRingUsed
, flags
);
401 flags
= virtio_lduw_phys_cached(vq
->vdev
, &caches
->used
, pa
);
402 virtio_stw_phys_cached(vdev
, &caches
->used
, pa
, flags
& ~mask
);
403 address_space_cache_invalidate(&caches
->used
, pa
, sizeof(flags
));
406 /* Called within rcu_read_lock(). */
407 static inline void vring_set_avail_event(VirtQueue
*vq
, uint16_t val
)
409 VRingMemoryRegionCaches
*caches
;
411 if (!vq
->notification
) {
415 caches
= vring_get_region_caches(vq
);
420 pa
= offsetof(VRingUsed
, ring
[vq
->vring
.num
]);
421 virtio_stw_phys_cached(vq
->vdev
, &caches
->used
, pa
, val
);
422 address_space_cache_invalidate(&caches
->used
, pa
, sizeof(val
));
425 static void virtio_queue_split_set_notification(VirtQueue
*vq
, int enable
)
427 RCU_READ_LOCK_GUARD();
429 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
430 vring_set_avail_event(vq
, vring_avail_idx(vq
));
432 vring_used_flags_unset_bit(vq
, VRING_USED_F_NO_NOTIFY
);
434 vring_used_flags_set_bit(vq
, VRING_USED_F_NO_NOTIFY
);
437 /* Expose avail event/used flags before caller checks the avail idx. */
442 static void virtio_queue_packed_set_notification(VirtQueue
*vq
, int enable
)
445 VRingPackedDescEvent e
;
446 VRingMemoryRegionCaches
*caches
;
448 RCU_READ_LOCK_GUARD();
449 caches
= vring_get_region_caches(vq
);
454 vring_packed_event_read(vq
->vdev
, &caches
->used
, &e
);
457 e
.flags
= VRING_PACKED_EVENT_FLAG_DISABLE
;
458 } else if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
459 off_wrap
= vq
->shadow_avail_idx
| vq
->shadow_avail_wrap_counter
<< 15;
460 vring_packed_off_wrap_write(vq
->vdev
, &caches
->used
, off_wrap
);
461 /* Make sure off_wrap is wrote before flags */
463 e
.flags
= VRING_PACKED_EVENT_FLAG_DESC
;
465 e
.flags
= VRING_PACKED_EVENT_FLAG_ENABLE
;
468 vring_packed_flags_write(vq
->vdev
, &caches
->used
, e
.flags
);
470 /* Expose avail event/used flags before caller checks the avail idx. */
475 bool virtio_queue_get_notification(VirtQueue
*vq
)
477 return vq
->notification
;
480 void virtio_queue_set_notification(VirtQueue
*vq
, int enable
)
482 vq
->notification
= enable
;
484 if (!vq
->vring
.desc
) {
488 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
489 virtio_queue_packed_set_notification(vq
, enable
);
491 virtio_queue_split_set_notification(vq
, enable
);
495 int virtio_queue_ready(VirtQueue
*vq
)
497 return vq
->vring
.avail
!= 0;
500 static void vring_packed_desc_read_flags(VirtIODevice
*vdev
,
502 MemoryRegionCache
*cache
,
505 hwaddr off
= i
* sizeof(VRingPackedDesc
) + offsetof(VRingPackedDesc
, flags
);
507 *flags
= virtio_lduw_phys_cached(vdev
, cache
, off
);
510 static void vring_packed_desc_read(VirtIODevice
*vdev
,
511 VRingPackedDesc
*desc
,
512 MemoryRegionCache
*cache
,
513 int i
, bool strict_order
)
515 hwaddr off
= i
* sizeof(VRingPackedDesc
);
517 vring_packed_desc_read_flags(vdev
, &desc
->flags
, cache
, i
);
520 /* Make sure flags is read before the rest fields. */
524 address_space_read_cached(cache
, off
+ offsetof(VRingPackedDesc
, addr
),
525 &desc
->addr
, sizeof(desc
->addr
));
526 address_space_read_cached(cache
, off
+ offsetof(VRingPackedDesc
, id
),
527 &desc
->id
, sizeof(desc
->id
));
528 address_space_read_cached(cache
, off
+ offsetof(VRingPackedDesc
, len
),
529 &desc
->len
, sizeof(desc
->len
));
530 virtio_tswap64s(vdev
, &desc
->addr
);
531 virtio_tswap16s(vdev
, &desc
->id
);
532 virtio_tswap32s(vdev
, &desc
->len
);
535 static void vring_packed_desc_write_data(VirtIODevice
*vdev
,
536 VRingPackedDesc
*desc
,
537 MemoryRegionCache
*cache
,
540 hwaddr off_id
= i
* sizeof(VRingPackedDesc
) +
541 offsetof(VRingPackedDesc
, id
);
542 hwaddr off_len
= i
* sizeof(VRingPackedDesc
) +
543 offsetof(VRingPackedDesc
, len
);
545 virtio_tswap32s(vdev
, &desc
->len
);
546 virtio_tswap16s(vdev
, &desc
->id
);
547 address_space_write_cached(cache
, off_id
, &desc
->id
, sizeof(desc
->id
));
548 address_space_cache_invalidate(cache
, off_id
, sizeof(desc
->id
));
549 address_space_write_cached(cache
, off_len
, &desc
->len
, sizeof(desc
->len
));
550 address_space_cache_invalidate(cache
, off_len
, sizeof(desc
->len
));
553 static void vring_packed_desc_write_flags(VirtIODevice
*vdev
,
554 VRingPackedDesc
*desc
,
555 MemoryRegionCache
*cache
,
558 hwaddr off
= i
* sizeof(VRingPackedDesc
) + offsetof(VRingPackedDesc
, flags
);
560 virtio_stw_phys_cached(vdev
, cache
, off
, desc
->flags
);
561 address_space_cache_invalidate(cache
, off
, sizeof(desc
->flags
));
564 static void vring_packed_desc_write(VirtIODevice
*vdev
,
565 VRingPackedDesc
*desc
,
566 MemoryRegionCache
*cache
,
567 int i
, bool strict_order
)
569 vring_packed_desc_write_data(vdev
, desc
, cache
, i
);
571 /* Make sure data is wrote before flags. */
574 vring_packed_desc_write_flags(vdev
, desc
, cache
, i
);
577 static inline bool is_desc_avail(uint16_t flags
, bool wrap_counter
)
581 avail
= !!(flags
& (1 << VRING_PACKED_DESC_F_AVAIL
));
582 used
= !!(flags
& (1 << VRING_PACKED_DESC_F_USED
));
583 return (avail
!= used
) && (avail
== wrap_counter
);
586 /* Fetch avail_idx from VQ memory only when we really need to know if
587 * guest has added some buffers.
588 * Called within rcu_read_lock(). */
589 static int virtio_queue_empty_rcu(VirtQueue
*vq
)
591 if (virtio_device_disabled(vq
->vdev
)) {
595 if (unlikely(!vq
->vring
.avail
)) {
599 if (vq
->shadow_avail_idx
!= vq
->last_avail_idx
) {
603 return vring_avail_idx(vq
) == vq
->last_avail_idx
;
606 static int virtio_queue_split_empty(VirtQueue
*vq
)
610 if (virtio_device_disabled(vq
->vdev
)) {
614 if (unlikely(!vq
->vring
.avail
)) {
618 if (vq
->shadow_avail_idx
!= vq
->last_avail_idx
) {
622 RCU_READ_LOCK_GUARD();
623 empty
= vring_avail_idx(vq
) == vq
->last_avail_idx
;
627 /* Called within rcu_read_lock(). */
628 static int virtio_queue_packed_empty_rcu(VirtQueue
*vq
)
630 struct VRingPackedDesc desc
;
631 VRingMemoryRegionCaches
*cache
;
633 if (unlikely(!vq
->vring
.desc
)) {
637 cache
= vring_get_region_caches(vq
);
642 vring_packed_desc_read_flags(vq
->vdev
, &desc
.flags
, &cache
->desc
,
645 return !is_desc_avail(desc
.flags
, vq
->last_avail_wrap_counter
);
648 static int virtio_queue_packed_empty(VirtQueue
*vq
)
650 RCU_READ_LOCK_GUARD();
651 return virtio_queue_packed_empty_rcu(vq
);
654 int virtio_queue_empty(VirtQueue
*vq
)
656 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
657 return virtio_queue_packed_empty(vq
);
659 return virtio_queue_split_empty(vq
);
663 static void virtqueue_unmap_sg(VirtQueue
*vq
, const VirtQueueElement
*elem
,
666 AddressSpace
*dma_as
= vq
->vdev
->dma_as
;
671 for (i
= 0; i
< elem
->in_num
; i
++) {
672 size_t size
= MIN(len
- offset
, elem
->in_sg
[i
].iov_len
);
674 dma_memory_unmap(dma_as
, elem
->in_sg
[i
].iov_base
,
675 elem
->in_sg
[i
].iov_len
,
676 DMA_DIRECTION_FROM_DEVICE
, size
);
681 for (i
= 0; i
< elem
->out_num
; i
++)
682 dma_memory_unmap(dma_as
, elem
->out_sg
[i
].iov_base
,
683 elem
->out_sg
[i
].iov_len
,
684 DMA_DIRECTION_TO_DEVICE
,
685 elem
->out_sg
[i
].iov_len
);
688 /* virtqueue_detach_element:
689 * @vq: The #VirtQueue
690 * @elem: The #VirtQueueElement
691 * @len: number of bytes written
693 * Detach the element from the virtqueue. This function is suitable for device
694 * reset or other situations where a #VirtQueueElement is simply freed and will
695 * not be pushed or discarded.
697 void virtqueue_detach_element(VirtQueue
*vq
, const VirtQueueElement
*elem
,
700 vq
->inuse
-= elem
->ndescs
;
701 virtqueue_unmap_sg(vq
, elem
, len
);
704 static void virtqueue_split_rewind(VirtQueue
*vq
, unsigned int num
)
706 vq
->last_avail_idx
-= num
;
709 static void virtqueue_packed_rewind(VirtQueue
*vq
, unsigned int num
)
711 if (vq
->last_avail_idx
< num
) {
712 vq
->last_avail_idx
= vq
->vring
.num
+ vq
->last_avail_idx
- num
;
713 vq
->last_avail_wrap_counter
^= 1;
715 vq
->last_avail_idx
-= num
;
720 * @vq: The #VirtQueue
721 * @elem: The #VirtQueueElement
722 * @len: number of bytes written
724 * Pretend the most recent element wasn't popped from the virtqueue. The next
725 * call to virtqueue_pop() will refetch the element.
727 void virtqueue_unpop(VirtQueue
*vq
, const VirtQueueElement
*elem
,
731 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
732 virtqueue_packed_rewind(vq
, 1);
734 virtqueue_split_rewind(vq
, 1);
737 virtqueue_detach_element(vq
, elem
, len
);
741 * @vq: The #VirtQueue
742 * @num: Number of elements to push back
744 * Pretend that elements weren't popped from the virtqueue. The next
745 * virtqueue_pop() will refetch the oldest element.
747 * Use virtqueue_unpop() instead if you have a VirtQueueElement.
749 * Returns: true on success, false if @num is greater than the number of in use
752 bool virtqueue_rewind(VirtQueue
*vq
, unsigned int num
)
754 if (num
> vq
->inuse
) {
759 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
760 virtqueue_packed_rewind(vq
, num
);
762 virtqueue_split_rewind(vq
, num
);
767 static void virtqueue_split_fill(VirtQueue
*vq
, const VirtQueueElement
*elem
,
768 unsigned int len
, unsigned int idx
)
772 if (unlikely(!vq
->vring
.used
)) {
776 idx
= (idx
+ vq
->used_idx
) % vq
->vring
.num
;
778 uelem
.id
= elem
->index
;
780 vring_used_write(vq
, &uelem
, idx
);
783 static void virtqueue_packed_fill(VirtQueue
*vq
, const VirtQueueElement
*elem
,
784 unsigned int len
, unsigned int idx
)
786 vq
->used_elems
[idx
].index
= elem
->index
;
787 vq
->used_elems
[idx
].len
= len
;
788 vq
->used_elems
[idx
].ndescs
= elem
->ndescs
;
791 static void virtqueue_packed_fill_desc(VirtQueue
*vq
,
792 const VirtQueueElement
*elem
,
797 VRingMemoryRegionCaches
*caches
;
798 VRingPackedDesc desc
= {
802 bool wrap_counter
= vq
->used_wrap_counter
;
804 if (unlikely(!vq
->vring
.desc
)) {
808 head
= vq
->used_idx
+ idx
;
809 if (head
>= vq
->vring
.num
) {
810 head
-= vq
->vring
.num
;
814 desc
.flags
|= (1 << VRING_PACKED_DESC_F_AVAIL
);
815 desc
.flags
|= (1 << VRING_PACKED_DESC_F_USED
);
817 desc
.flags
&= ~(1 << VRING_PACKED_DESC_F_AVAIL
);
818 desc
.flags
&= ~(1 << VRING_PACKED_DESC_F_USED
);
821 caches
= vring_get_region_caches(vq
);
826 vring_packed_desc_write(vq
->vdev
, &desc
, &caches
->desc
, head
, strict_order
);
829 /* Called within rcu_read_lock(). */
830 void virtqueue_fill(VirtQueue
*vq
, const VirtQueueElement
*elem
,
831 unsigned int len
, unsigned int idx
)
833 trace_virtqueue_fill(vq
, elem
, len
, idx
);
835 virtqueue_unmap_sg(vq
, elem
, len
);
837 if (virtio_device_disabled(vq
->vdev
)) {
841 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
842 virtqueue_packed_fill(vq
, elem
, len
, idx
);
844 virtqueue_split_fill(vq
, elem
, len
, idx
);
848 /* Called within rcu_read_lock(). */
849 static void virtqueue_split_flush(VirtQueue
*vq
, unsigned int count
)
853 if (unlikely(!vq
->vring
.used
)) {
857 /* Make sure buffer is written before we update index. */
859 trace_virtqueue_flush(vq
, count
);
862 vring_used_idx_set(vq
, new);
864 if (unlikely((int16_t)(new - vq
->signalled_used
) < (uint16_t)(new - old
)))
865 vq
->signalled_used_valid
= false;
868 static void virtqueue_packed_flush(VirtQueue
*vq
, unsigned int count
)
870 unsigned int i
, ndescs
= 0;
872 if (unlikely(!vq
->vring
.desc
)) {
876 for (i
= 1; i
< count
; i
++) {
877 virtqueue_packed_fill_desc(vq
, &vq
->used_elems
[i
], i
, false);
878 ndescs
+= vq
->used_elems
[i
].ndescs
;
880 virtqueue_packed_fill_desc(vq
, &vq
->used_elems
[0], 0, true);
881 ndescs
+= vq
->used_elems
[0].ndescs
;
884 vq
->used_idx
+= ndescs
;
885 if (vq
->used_idx
>= vq
->vring
.num
) {
886 vq
->used_idx
-= vq
->vring
.num
;
887 vq
->used_wrap_counter
^= 1;
888 vq
->signalled_used_valid
= false;
892 void virtqueue_flush(VirtQueue
*vq
, unsigned int count
)
894 if (virtio_device_disabled(vq
->vdev
)) {
899 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
900 virtqueue_packed_flush(vq
, count
);
902 virtqueue_split_flush(vq
, count
);
906 void virtqueue_push(VirtQueue
*vq
, const VirtQueueElement
*elem
,
909 RCU_READ_LOCK_GUARD();
910 virtqueue_fill(vq
, elem
, len
, 0);
911 virtqueue_flush(vq
, 1);
914 /* Called within rcu_read_lock(). */
915 static int virtqueue_num_heads(VirtQueue
*vq
, unsigned int idx
)
917 uint16_t num_heads
= vring_avail_idx(vq
) - idx
;
919 /* Check it isn't doing very strange things with descriptor numbers. */
920 if (num_heads
> vq
->vring
.num
) {
921 virtio_error(vq
->vdev
, "Guest moved used index from %u to %u",
922 idx
, vq
->shadow_avail_idx
);
925 /* On success, callers read a descriptor at vq->last_avail_idx.
926 * Make sure descriptor read does not bypass avail index read. */
934 /* Called within rcu_read_lock(). */
935 static bool virtqueue_get_head(VirtQueue
*vq
, unsigned int idx
,
938 /* Grab the next descriptor number they're advertising, and increment
939 * the index we've seen. */
940 *head
= vring_avail_ring(vq
, idx
% vq
->vring
.num
);
942 /* If their number is silly, that's a fatal mistake. */
943 if (*head
>= vq
->vring
.num
) {
944 virtio_error(vq
->vdev
, "Guest says index %u is available", *head
);
952 VIRTQUEUE_READ_DESC_ERROR
= -1,
953 VIRTQUEUE_READ_DESC_DONE
= 0, /* end of chain */
954 VIRTQUEUE_READ_DESC_MORE
= 1, /* more buffers in chain */
957 static int virtqueue_split_read_next_desc(VirtIODevice
*vdev
, VRingDesc
*desc
,
958 MemoryRegionCache
*desc_cache
,
959 unsigned int max
, unsigned int *next
)
961 /* If this descriptor says it doesn't chain, we're done. */
962 if (!(desc
->flags
& VRING_DESC_F_NEXT
)) {
963 return VIRTQUEUE_READ_DESC_DONE
;
966 /* Check they're not leading us off end of descriptors. */
968 /* Make sure compiler knows to grab that: we don't want it changing! */
972 virtio_error(vdev
, "Desc next is %u", *next
);
973 return VIRTQUEUE_READ_DESC_ERROR
;
976 vring_split_desc_read(vdev
, desc
, desc_cache
, *next
);
977 return VIRTQUEUE_READ_DESC_MORE
;
980 /* Called within rcu_read_lock(). */
981 static void virtqueue_split_get_avail_bytes(VirtQueue
*vq
,
982 unsigned int *in_bytes
, unsigned int *out_bytes
,
983 unsigned max_in_bytes
, unsigned max_out_bytes
,
984 VRingMemoryRegionCaches
*caches
)
986 VirtIODevice
*vdev
= vq
->vdev
;
987 unsigned int max
, idx
;
988 unsigned int total_bufs
, in_total
, out_total
;
989 MemoryRegionCache indirect_desc_cache
= MEMORY_REGION_CACHE_INVALID
;
993 idx
= vq
->last_avail_idx
;
994 total_bufs
= in_total
= out_total
= 0;
998 while ((rc
= virtqueue_num_heads(vq
, idx
)) > 0) {
999 MemoryRegionCache
*desc_cache
= &caches
->desc
;
1000 unsigned int num_bufs
;
1004 num_bufs
= total_bufs
;
1006 if (!virtqueue_get_head(vq
, idx
++, &i
)) {
1010 vring_split_desc_read(vdev
, &desc
, desc_cache
, i
);
1012 if (desc
.flags
& VRING_DESC_F_INDIRECT
) {
1013 if (!desc
.len
|| (desc
.len
% sizeof(VRingDesc
))) {
1014 virtio_error(vdev
, "Invalid size for indirect buffer table");
1018 /* If we've got too many, that implies a descriptor loop. */
1019 if (num_bufs
>= max
) {
1020 virtio_error(vdev
, "Looped descriptor");
1024 /* loop over the indirect descriptor table */
1025 len
= address_space_cache_init(&indirect_desc_cache
,
1027 desc
.addr
, desc
.len
, false);
1028 desc_cache
= &indirect_desc_cache
;
1029 if (len
< desc
.len
) {
1030 virtio_error(vdev
, "Cannot map indirect buffer");
1034 max
= desc
.len
/ sizeof(VRingDesc
);
1036 vring_split_desc_read(vdev
, &desc
, desc_cache
, i
);
1040 /* If we've got too many, that implies a descriptor loop. */
1041 if (++num_bufs
> max
) {
1042 virtio_error(vdev
, "Looped descriptor");
1046 if (desc
.flags
& VRING_DESC_F_WRITE
) {
1047 in_total
+= desc
.len
;
1049 out_total
+= desc
.len
;
1051 if (in_total
>= max_in_bytes
&& out_total
>= max_out_bytes
) {
1055 rc
= virtqueue_split_read_next_desc(vdev
, &desc
, desc_cache
, max
, &i
);
1056 } while (rc
== VIRTQUEUE_READ_DESC_MORE
);
1058 if (rc
== VIRTQUEUE_READ_DESC_ERROR
) {
1062 if (desc_cache
== &indirect_desc_cache
) {
1063 address_space_cache_destroy(&indirect_desc_cache
);
1066 total_bufs
= num_bufs
;
1075 address_space_cache_destroy(&indirect_desc_cache
);
1077 *in_bytes
= in_total
;
1080 *out_bytes
= out_total
;
1085 in_total
= out_total
= 0;
1089 static int virtqueue_packed_read_next_desc(VirtQueue
*vq
,
1090 VRingPackedDesc
*desc
,
1097 /* If this descriptor says it doesn't chain, we're done. */
1098 if (!indirect
&& !(desc
->flags
& VRING_DESC_F_NEXT
)) {
1099 return VIRTQUEUE_READ_DESC_DONE
;
1105 return VIRTQUEUE_READ_DESC_DONE
;
1107 (*next
) -= vq
->vring
.num
;
1111 vring_packed_desc_read(vq
->vdev
, desc
, desc_cache
, *next
, false);
1112 return VIRTQUEUE_READ_DESC_MORE
;
1115 /* Called within rcu_read_lock(). */
1116 static void virtqueue_packed_get_avail_bytes(VirtQueue
*vq
,
1117 unsigned int *in_bytes
,
1118 unsigned int *out_bytes
,
1119 unsigned max_in_bytes
,
1120 unsigned max_out_bytes
,
1121 VRingMemoryRegionCaches
*caches
)
1123 VirtIODevice
*vdev
= vq
->vdev
;
1124 unsigned int max
, idx
;
1125 unsigned int total_bufs
, in_total
, out_total
;
1126 MemoryRegionCache
*desc_cache
;
1127 MemoryRegionCache indirect_desc_cache
= MEMORY_REGION_CACHE_INVALID
;
1129 VRingPackedDesc desc
;
1132 idx
= vq
->last_avail_idx
;
1133 wrap_counter
= vq
->last_avail_wrap_counter
;
1134 total_bufs
= in_total
= out_total
= 0;
1136 max
= vq
->vring
.num
;
1139 unsigned int num_bufs
= total_bufs
;
1140 unsigned int i
= idx
;
1143 desc_cache
= &caches
->desc
;
1144 vring_packed_desc_read(vdev
, &desc
, desc_cache
, idx
, true);
1145 if (!is_desc_avail(desc
.flags
, wrap_counter
)) {
1149 if (desc
.flags
& VRING_DESC_F_INDIRECT
) {
1150 if (desc
.len
% sizeof(VRingPackedDesc
)) {
1151 virtio_error(vdev
, "Invalid size for indirect buffer table");
1155 /* If we've got too many, that implies a descriptor loop. */
1156 if (num_bufs
>= max
) {
1157 virtio_error(vdev
, "Looped descriptor");
1161 /* loop over the indirect descriptor table */
1162 len
= address_space_cache_init(&indirect_desc_cache
,
1164 desc
.addr
, desc
.len
, false);
1165 desc_cache
= &indirect_desc_cache
;
1166 if (len
< desc
.len
) {
1167 virtio_error(vdev
, "Cannot map indirect buffer");
1171 max
= desc
.len
/ sizeof(VRingPackedDesc
);
1173 vring_packed_desc_read(vdev
, &desc
, desc_cache
, i
, false);
1177 /* If we've got too many, that implies a descriptor loop. */
1178 if (++num_bufs
> max
) {
1179 virtio_error(vdev
, "Looped descriptor");
1183 if (desc
.flags
& VRING_DESC_F_WRITE
) {
1184 in_total
+= desc
.len
;
1186 out_total
+= desc
.len
;
1188 if (in_total
>= max_in_bytes
&& out_total
>= max_out_bytes
) {
1192 rc
= virtqueue_packed_read_next_desc(vq
, &desc
, desc_cache
, max
,
1194 &indirect_desc_cache
);
1195 } while (rc
== VIRTQUEUE_READ_DESC_MORE
);
1197 if (desc_cache
== &indirect_desc_cache
) {
1198 address_space_cache_destroy(&indirect_desc_cache
);
1202 idx
+= num_bufs
- total_bufs
;
1203 total_bufs
= num_bufs
;
1206 if (idx
>= vq
->vring
.num
) {
1207 idx
-= vq
->vring
.num
;
1212 /* Record the index and wrap counter for a kick we want */
1213 vq
->shadow_avail_idx
= idx
;
1214 vq
->shadow_avail_wrap_counter
= wrap_counter
;
1216 address_space_cache_destroy(&indirect_desc_cache
);
1218 *in_bytes
= in_total
;
1221 *out_bytes
= out_total
;
1226 in_total
= out_total
= 0;
1230 void virtqueue_get_avail_bytes(VirtQueue
*vq
, unsigned int *in_bytes
,
1231 unsigned int *out_bytes
,
1232 unsigned max_in_bytes
, unsigned max_out_bytes
)
1235 VRingMemoryRegionCaches
*caches
;
1237 RCU_READ_LOCK_GUARD();
1239 if (unlikely(!vq
->vring
.desc
)) {
1243 caches
= vring_get_region_caches(vq
);
1248 desc_size
= virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
) ?
1249 sizeof(VRingPackedDesc
) : sizeof(VRingDesc
);
1250 if (caches
->desc
.len
< vq
->vring
.num
* desc_size
) {
1251 virtio_error(vq
->vdev
, "Cannot map descriptor ring");
1255 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
1256 virtqueue_packed_get_avail_bytes(vq
, in_bytes
, out_bytes
,
1257 max_in_bytes
, max_out_bytes
,
1260 virtqueue_split_get_avail_bytes(vq
, in_bytes
, out_bytes
,
1261 max_in_bytes
, max_out_bytes
,
1275 int virtqueue_avail_bytes(VirtQueue
*vq
, unsigned int in_bytes
,
1276 unsigned int out_bytes
)
1278 unsigned int in_total
, out_total
;
1280 virtqueue_get_avail_bytes(vq
, &in_total
, &out_total
, in_bytes
, out_bytes
);
1281 return in_bytes
<= in_total
&& out_bytes
<= out_total
;
1284 static bool virtqueue_map_desc(VirtIODevice
*vdev
, unsigned int *p_num_sg
,
1285 hwaddr
*addr
, struct iovec
*iov
,
1286 unsigned int max_num_sg
, bool is_write
,
1287 hwaddr pa
, size_t sz
)
1290 unsigned num_sg
= *p_num_sg
;
1291 assert(num_sg
<= max_num_sg
);
1294 virtio_error(vdev
, "virtio: zero sized buffers are not allowed");
1301 if (num_sg
== max_num_sg
) {
1302 virtio_error(vdev
, "virtio: too many write descriptors in "
1307 iov
[num_sg
].iov_base
= dma_memory_map(vdev
->dma_as
, pa
, &len
,
1309 DMA_DIRECTION_FROM_DEVICE
:
1310 DMA_DIRECTION_TO_DEVICE
,
1311 MEMTXATTRS_UNSPECIFIED
);
1312 if (!iov
[num_sg
].iov_base
) {
1313 virtio_error(vdev
, "virtio: bogus descriptor or out of resources");
1317 iov
[num_sg
].iov_len
= len
;
1331 /* Only used by error code paths before we have a VirtQueueElement (therefore
1332 * virtqueue_unmap_sg() can't be used). Assumes buffers weren't written to
1335 static void virtqueue_undo_map_desc(unsigned int out_num
, unsigned int in_num
,
1340 for (i
= 0; i
< out_num
+ in_num
; i
++) {
1341 int is_write
= i
>= out_num
;
1343 cpu_physical_memory_unmap(iov
->iov_base
, iov
->iov_len
, is_write
, 0);
1348 static void virtqueue_map_iovec(VirtIODevice
*vdev
, struct iovec
*sg
,
1349 hwaddr
*addr
, unsigned int num_sg
,
1355 for (i
= 0; i
< num_sg
; i
++) {
1356 len
= sg
[i
].iov_len
;
1357 sg
[i
].iov_base
= dma_memory_map(vdev
->dma_as
,
1358 addr
[i
], &len
, is_write
?
1359 DMA_DIRECTION_FROM_DEVICE
:
1360 DMA_DIRECTION_TO_DEVICE
,
1361 MEMTXATTRS_UNSPECIFIED
);
1362 if (!sg
[i
].iov_base
) {
1363 error_report("virtio: error trying to map MMIO memory");
1366 if (len
!= sg
[i
].iov_len
) {
1367 error_report("virtio: unexpected memory split");
1373 void virtqueue_map(VirtIODevice
*vdev
, VirtQueueElement
*elem
)
1375 virtqueue_map_iovec(vdev
, elem
->in_sg
, elem
->in_addr
, elem
->in_num
, true);
1376 virtqueue_map_iovec(vdev
, elem
->out_sg
, elem
->out_addr
, elem
->out_num
,
1380 static void *virtqueue_alloc_element(size_t sz
, unsigned out_num
, unsigned in_num
)
1382 VirtQueueElement
*elem
;
1383 size_t in_addr_ofs
= QEMU_ALIGN_UP(sz
, __alignof__(elem
->in_addr
[0]));
1384 size_t out_addr_ofs
= in_addr_ofs
+ in_num
* sizeof(elem
->in_addr
[0]);
1385 size_t out_addr_end
= out_addr_ofs
+ out_num
* sizeof(elem
->out_addr
[0]);
1386 size_t in_sg_ofs
= QEMU_ALIGN_UP(out_addr_end
, __alignof__(elem
->in_sg
[0]));
1387 size_t out_sg_ofs
= in_sg_ofs
+ in_num
* sizeof(elem
->in_sg
[0]);
1388 size_t out_sg_end
= out_sg_ofs
+ out_num
* sizeof(elem
->out_sg
[0]);
1390 assert(sz
>= sizeof(VirtQueueElement
));
1391 elem
= g_malloc(out_sg_end
);
1392 trace_virtqueue_alloc_element(elem
, sz
, in_num
, out_num
);
1393 elem
->out_num
= out_num
;
1394 elem
->in_num
= in_num
;
1395 elem
->in_addr
= (void *)elem
+ in_addr_ofs
;
1396 elem
->out_addr
= (void *)elem
+ out_addr_ofs
;
1397 elem
->in_sg
= (void *)elem
+ in_sg_ofs
;
1398 elem
->out_sg
= (void *)elem
+ out_sg_ofs
;
1402 static void *virtqueue_split_pop(VirtQueue
*vq
, size_t sz
)
1404 unsigned int i
, head
, max
;
1405 VRingMemoryRegionCaches
*caches
;
1406 MemoryRegionCache indirect_desc_cache
= MEMORY_REGION_CACHE_INVALID
;
1407 MemoryRegionCache
*desc_cache
;
1409 VirtIODevice
*vdev
= vq
->vdev
;
1410 VirtQueueElement
*elem
= NULL
;
1411 unsigned out_num
, in_num
, elem_entries
;
1412 hwaddr addr
[VIRTQUEUE_MAX_SIZE
];
1413 struct iovec iov
[VIRTQUEUE_MAX_SIZE
];
1417 RCU_READ_LOCK_GUARD();
1418 if (virtio_queue_empty_rcu(vq
)) {
1421 /* Needed after virtio_queue_empty(), see comment in
1422 * virtqueue_num_heads(). */
1425 /* When we start there are none of either input nor output. */
1426 out_num
= in_num
= elem_entries
= 0;
1428 max
= vq
->vring
.num
;
1430 if (vq
->inuse
>= vq
->vring
.num
) {
1431 virtio_error(vdev
, "Virtqueue size exceeded");
1435 if (!virtqueue_get_head(vq
, vq
->last_avail_idx
++, &head
)) {
1439 if (virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
1440 vring_set_avail_event(vq
, vq
->last_avail_idx
);
1445 caches
= vring_get_region_caches(vq
);
1447 virtio_error(vdev
, "Region caches not initialized");
1451 if (caches
->desc
.len
< max
* sizeof(VRingDesc
)) {
1452 virtio_error(vdev
, "Cannot map descriptor ring");
1456 desc_cache
= &caches
->desc
;
1457 vring_split_desc_read(vdev
, &desc
, desc_cache
, i
);
1458 if (desc
.flags
& VRING_DESC_F_INDIRECT
) {
1459 if (!desc
.len
|| (desc
.len
% sizeof(VRingDesc
))) {
1460 virtio_error(vdev
, "Invalid size for indirect buffer table");
1464 /* loop over the indirect descriptor table */
1465 len
= address_space_cache_init(&indirect_desc_cache
, vdev
->dma_as
,
1466 desc
.addr
, desc
.len
, false);
1467 desc_cache
= &indirect_desc_cache
;
1468 if (len
< desc
.len
) {
1469 virtio_error(vdev
, "Cannot map indirect buffer");
1473 max
= desc
.len
/ sizeof(VRingDesc
);
1475 vring_split_desc_read(vdev
, &desc
, desc_cache
, i
);
1478 /* Collect all the descriptors */
1482 if (desc
.flags
& VRING_DESC_F_WRITE
) {
1483 map_ok
= virtqueue_map_desc(vdev
, &in_num
, addr
+ out_num
,
1485 VIRTQUEUE_MAX_SIZE
- out_num
, true,
1486 desc
.addr
, desc
.len
);
1489 virtio_error(vdev
, "Incorrect order for descriptors");
1492 map_ok
= virtqueue_map_desc(vdev
, &out_num
, addr
, iov
,
1493 VIRTQUEUE_MAX_SIZE
, false,
1494 desc
.addr
, desc
.len
);
1500 /* If we've got too many, that implies a descriptor loop. */
1501 if (++elem_entries
> max
) {
1502 virtio_error(vdev
, "Looped descriptor");
1506 rc
= virtqueue_split_read_next_desc(vdev
, &desc
, desc_cache
, max
, &i
);
1507 } while (rc
== VIRTQUEUE_READ_DESC_MORE
);
1509 if (rc
== VIRTQUEUE_READ_DESC_ERROR
) {
1513 /* Now copy what we have collected and mapped */
1514 elem
= virtqueue_alloc_element(sz
, out_num
, in_num
);
1517 for (i
= 0; i
< out_num
; i
++) {
1518 elem
->out_addr
[i
] = addr
[i
];
1519 elem
->out_sg
[i
] = iov
[i
];
1521 for (i
= 0; i
< in_num
; i
++) {
1522 elem
->in_addr
[i
] = addr
[out_num
+ i
];
1523 elem
->in_sg
[i
] = iov
[out_num
+ i
];
1528 trace_virtqueue_pop(vq
, elem
, elem
->in_num
, elem
->out_num
);
1530 address_space_cache_destroy(&indirect_desc_cache
);
1535 virtqueue_undo_map_desc(out_num
, in_num
, iov
);
1539 static void *virtqueue_packed_pop(VirtQueue
*vq
, size_t sz
)
1541 unsigned int i
, max
;
1542 VRingMemoryRegionCaches
*caches
;
1543 MemoryRegionCache indirect_desc_cache
= MEMORY_REGION_CACHE_INVALID
;
1544 MemoryRegionCache
*desc_cache
;
1546 VirtIODevice
*vdev
= vq
->vdev
;
1547 VirtQueueElement
*elem
= NULL
;
1548 unsigned out_num
, in_num
, elem_entries
;
1549 hwaddr addr
[VIRTQUEUE_MAX_SIZE
];
1550 struct iovec iov
[VIRTQUEUE_MAX_SIZE
];
1551 VRingPackedDesc desc
;
1555 RCU_READ_LOCK_GUARD();
1556 if (virtio_queue_packed_empty_rcu(vq
)) {
1560 /* When we start there are none of either input nor output. */
1561 out_num
= in_num
= elem_entries
= 0;
1563 max
= vq
->vring
.num
;
1565 if (vq
->inuse
>= vq
->vring
.num
) {
1566 virtio_error(vdev
, "Virtqueue size exceeded");
1570 i
= vq
->last_avail_idx
;
1572 caches
= vring_get_region_caches(vq
);
1574 virtio_error(vdev
, "Region caches not initialized");
1578 if (caches
->desc
.len
< max
* sizeof(VRingDesc
)) {
1579 virtio_error(vdev
, "Cannot map descriptor ring");
1583 desc_cache
= &caches
->desc
;
1584 vring_packed_desc_read(vdev
, &desc
, desc_cache
, i
, true);
1586 if (desc
.flags
& VRING_DESC_F_INDIRECT
) {
1587 if (desc
.len
% sizeof(VRingPackedDesc
)) {
1588 virtio_error(vdev
, "Invalid size for indirect buffer table");
1592 /* loop over the indirect descriptor table */
1593 len
= address_space_cache_init(&indirect_desc_cache
, vdev
->dma_as
,
1594 desc
.addr
, desc
.len
, false);
1595 desc_cache
= &indirect_desc_cache
;
1596 if (len
< desc
.len
) {
1597 virtio_error(vdev
, "Cannot map indirect buffer");
1601 max
= desc
.len
/ sizeof(VRingPackedDesc
);
1603 vring_packed_desc_read(vdev
, &desc
, desc_cache
, i
, false);
1606 /* Collect all the descriptors */
1610 if (desc
.flags
& VRING_DESC_F_WRITE
) {
1611 map_ok
= virtqueue_map_desc(vdev
, &in_num
, addr
+ out_num
,
1613 VIRTQUEUE_MAX_SIZE
- out_num
, true,
1614 desc
.addr
, desc
.len
);
1617 virtio_error(vdev
, "Incorrect order for descriptors");
1620 map_ok
= virtqueue_map_desc(vdev
, &out_num
, addr
, iov
,
1621 VIRTQUEUE_MAX_SIZE
, false,
1622 desc
.addr
, desc
.len
);
1628 /* If we've got too many, that implies a descriptor loop. */
1629 if (++elem_entries
> max
) {
1630 virtio_error(vdev
, "Looped descriptor");
1634 rc
= virtqueue_packed_read_next_desc(vq
, &desc
, desc_cache
, max
, &i
,
1636 &indirect_desc_cache
);
1637 } while (rc
== VIRTQUEUE_READ_DESC_MORE
);
1639 /* Now copy what we have collected and mapped */
1640 elem
= virtqueue_alloc_element(sz
, out_num
, in_num
);
1641 for (i
= 0; i
< out_num
; i
++) {
1642 elem
->out_addr
[i
] = addr
[i
];
1643 elem
->out_sg
[i
] = iov
[i
];
1645 for (i
= 0; i
< in_num
; i
++) {
1646 elem
->in_addr
[i
] = addr
[out_num
+ i
];
1647 elem
->in_sg
[i
] = iov
[out_num
+ i
];
1651 elem
->ndescs
= (desc_cache
== &indirect_desc_cache
) ? 1 : elem_entries
;
1652 vq
->last_avail_idx
+= elem
->ndescs
;
1653 vq
->inuse
+= elem
->ndescs
;
1655 if (vq
->last_avail_idx
>= vq
->vring
.num
) {
1656 vq
->last_avail_idx
-= vq
->vring
.num
;
1657 vq
->last_avail_wrap_counter
^= 1;
1660 vq
->shadow_avail_idx
= vq
->last_avail_idx
;
1661 vq
->shadow_avail_wrap_counter
= vq
->last_avail_wrap_counter
;
1663 trace_virtqueue_pop(vq
, elem
, elem
->in_num
, elem
->out_num
);
1665 address_space_cache_destroy(&indirect_desc_cache
);
1670 virtqueue_undo_map_desc(out_num
, in_num
, iov
);
1674 void *virtqueue_pop(VirtQueue
*vq
, size_t sz
)
1676 if (virtio_device_disabled(vq
->vdev
)) {
1680 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
1681 return virtqueue_packed_pop(vq
, sz
);
1683 return virtqueue_split_pop(vq
, sz
);
1687 static unsigned int virtqueue_packed_drop_all(VirtQueue
*vq
)
1689 VRingMemoryRegionCaches
*caches
;
1690 MemoryRegionCache
*desc_cache
;
1691 unsigned int dropped
= 0;
1692 VirtQueueElement elem
= {};
1693 VirtIODevice
*vdev
= vq
->vdev
;
1694 VRingPackedDesc desc
;
1696 RCU_READ_LOCK_GUARD();
1698 caches
= vring_get_region_caches(vq
);
1703 desc_cache
= &caches
->desc
;
1705 virtio_queue_set_notification(vq
, 0);
1707 while (vq
->inuse
< vq
->vring
.num
) {
1708 unsigned int idx
= vq
->last_avail_idx
;
1710 * works similar to virtqueue_pop but does not map buffers
1711 * and does not allocate any memory.
1713 vring_packed_desc_read(vdev
, &desc
, desc_cache
,
1714 vq
->last_avail_idx
, true);
1715 if (!is_desc_avail(desc
.flags
, vq
->last_avail_wrap_counter
)) {
1718 elem
.index
= desc
.id
;
1720 while (virtqueue_packed_read_next_desc(vq
, &desc
, desc_cache
,
1721 vq
->vring
.num
, &idx
, false)) {
1725 * immediately push the element, nothing to unmap
1726 * as both in_num and out_num are set to 0.
1728 virtqueue_push(vq
, &elem
, 0);
1730 vq
->last_avail_idx
+= elem
.ndescs
;
1731 if (vq
->last_avail_idx
>= vq
->vring
.num
) {
1732 vq
->last_avail_idx
-= vq
->vring
.num
;
1733 vq
->last_avail_wrap_counter
^= 1;
1740 static unsigned int virtqueue_split_drop_all(VirtQueue
*vq
)
1742 unsigned int dropped
= 0;
1743 VirtQueueElement elem
= {};
1744 VirtIODevice
*vdev
= vq
->vdev
;
1745 bool fEventIdx
= virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
);
1747 while (!virtio_queue_empty(vq
) && vq
->inuse
< vq
->vring
.num
) {
1748 /* works similar to virtqueue_pop but does not map buffers
1749 * and does not allocate any memory */
1751 if (!virtqueue_get_head(vq
, vq
->last_avail_idx
, &elem
.index
)) {
1755 vq
->last_avail_idx
++;
1757 vring_set_avail_event(vq
, vq
->last_avail_idx
);
1759 /* immediately push the element, nothing to unmap
1760 * as both in_num and out_num are set to 0 */
1761 virtqueue_push(vq
, &elem
, 0);
1768 /* virtqueue_drop_all:
1769 * @vq: The #VirtQueue
1770 * Drops all queued buffers and indicates them to the guest
1771 * as if they are done. Useful when buffers can not be
1772 * processed but must be returned to the guest.
1774 unsigned int virtqueue_drop_all(VirtQueue
*vq
)
1776 struct VirtIODevice
*vdev
= vq
->vdev
;
1778 if (virtio_device_disabled(vq
->vdev
)) {
1782 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
1783 return virtqueue_packed_drop_all(vq
);
1785 return virtqueue_split_drop_all(vq
);
1789 /* Reading and writing a structure directly to QEMUFile is *awful*, but
1790 * it is what QEMU has always done by mistake. We can change it sooner
1791 * or later by bumping the version number of the affected vm states.
1792 * In the meanwhile, since the in-memory layout of VirtQueueElement
1793 * has changed, we need to marshal to and from the layout that was
1794 * used before the change.
1796 typedef struct VirtQueueElementOld
{
1798 unsigned int out_num
;
1799 unsigned int in_num
;
1800 hwaddr in_addr
[VIRTQUEUE_MAX_SIZE
];
1801 hwaddr out_addr
[VIRTQUEUE_MAX_SIZE
];
1802 struct iovec in_sg
[VIRTQUEUE_MAX_SIZE
];
1803 struct iovec out_sg
[VIRTQUEUE_MAX_SIZE
];
1804 } VirtQueueElementOld
;
1806 void *qemu_get_virtqueue_element(VirtIODevice
*vdev
, QEMUFile
*f
, size_t sz
)
1808 VirtQueueElement
*elem
;
1809 VirtQueueElementOld data
;
1812 qemu_get_buffer(f
, (uint8_t *)&data
, sizeof(VirtQueueElementOld
));
1814 /* TODO: teach all callers that this can fail, and return failure instead
1815 * of asserting here.
1816 * This is just one thing (there are probably more) that must be
1817 * fixed before we can allow NDEBUG compilation.
1819 assert(ARRAY_SIZE(data
.in_addr
) >= data
.in_num
);
1820 assert(ARRAY_SIZE(data
.out_addr
) >= data
.out_num
);
1822 elem
= virtqueue_alloc_element(sz
, data
.out_num
, data
.in_num
);
1823 elem
->index
= data
.index
;
1825 for (i
= 0; i
< elem
->in_num
; i
++) {
1826 elem
->in_addr
[i
] = data
.in_addr
[i
];
1829 for (i
= 0; i
< elem
->out_num
; i
++) {
1830 elem
->out_addr
[i
] = data
.out_addr
[i
];
1833 for (i
= 0; i
< elem
->in_num
; i
++) {
1834 /* Base is overwritten by virtqueue_map. */
1835 elem
->in_sg
[i
].iov_base
= 0;
1836 elem
->in_sg
[i
].iov_len
= data
.in_sg
[i
].iov_len
;
1839 for (i
= 0; i
< elem
->out_num
; i
++) {
1840 /* Base is overwritten by virtqueue_map. */
1841 elem
->out_sg
[i
].iov_base
= 0;
1842 elem
->out_sg
[i
].iov_len
= data
.out_sg
[i
].iov_len
;
1845 if (virtio_host_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
1846 qemu_get_be32s(f
, &elem
->ndescs
);
1849 virtqueue_map(vdev
, elem
);
1853 void qemu_put_virtqueue_element(VirtIODevice
*vdev
, QEMUFile
*f
,
1854 VirtQueueElement
*elem
)
1856 VirtQueueElementOld data
;
1859 memset(&data
, 0, sizeof(data
));
1860 data
.index
= elem
->index
;
1861 data
.in_num
= elem
->in_num
;
1862 data
.out_num
= elem
->out_num
;
1864 for (i
= 0; i
< elem
->in_num
; i
++) {
1865 data
.in_addr
[i
] = elem
->in_addr
[i
];
1868 for (i
= 0; i
< elem
->out_num
; i
++) {
1869 data
.out_addr
[i
] = elem
->out_addr
[i
];
1872 for (i
= 0; i
< elem
->in_num
; i
++) {
1873 /* Base is overwritten by virtqueue_map when loading. Do not
1874 * save it, as it would leak the QEMU address space layout. */
1875 data
.in_sg
[i
].iov_len
= elem
->in_sg
[i
].iov_len
;
1878 for (i
= 0; i
< elem
->out_num
; i
++) {
1879 /* Do not save iov_base as above. */
1880 data
.out_sg
[i
].iov_len
= elem
->out_sg
[i
].iov_len
;
1883 if (virtio_host_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
1884 qemu_put_be32s(f
, &elem
->ndescs
);
1887 qemu_put_buffer(f
, (uint8_t *)&data
, sizeof(VirtQueueElementOld
));
1891 static void virtio_notify_vector(VirtIODevice
*vdev
, uint16_t vector
)
1893 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1894 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1896 if (virtio_device_disabled(vdev
)) {
1901 k
->notify(qbus
->parent
, vector
);
1905 void virtio_update_irq(VirtIODevice
*vdev
)
1907 virtio_notify_vector(vdev
, VIRTIO_NO_VECTOR
);
1910 static int virtio_validate_features(VirtIODevice
*vdev
)
1912 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1914 if (virtio_host_has_feature(vdev
, VIRTIO_F_IOMMU_PLATFORM
) &&
1915 !virtio_vdev_has_feature(vdev
, VIRTIO_F_IOMMU_PLATFORM
)) {
1919 if (k
->validate_features
) {
1920 return k
->validate_features(vdev
);
1926 int virtio_set_status(VirtIODevice
*vdev
, uint8_t val
)
1928 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1929 trace_virtio_set_status(vdev
, val
);
1931 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
1932 if (!(vdev
->status
& VIRTIO_CONFIG_S_FEATURES_OK
) &&
1933 val
& VIRTIO_CONFIG_S_FEATURES_OK
) {
1934 int ret
= virtio_validate_features(vdev
);
1942 if ((vdev
->status
& VIRTIO_CONFIG_S_DRIVER_OK
) !=
1943 (val
& VIRTIO_CONFIG_S_DRIVER_OK
)) {
1944 virtio_set_started(vdev
, val
& VIRTIO_CONFIG_S_DRIVER_OK
);
1947 if (k
->set_status
) {
1948 k
->set_status(vdev
, val
);
1955 static enum virtio_device_endian
virtio_default_endian(void)
1957 if (target_words_bigendian()) {
1958 return VIRTIO_DEVICE_ENDIAN_BIG
;
1960 return VIRTIO_DEVICE_ENDIAN_LITTLE
;
1964 static enum virtio_device_endian
virtio_current_cpu_endian(void)
1966 if (cpu_virtio_is_big_endian(current_cpu
)) {
1967 return VIRTIO_DEVICE_ENDIAN_BIG
;
1969 return VIRTIO_DEVICE_ENDIAN_LITTLE
;
1973 void virtio_reset(void *opaque
)
1975 VirtIODevice
*vdev
= opaque
;
1976 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1979 virtio_set_status(vdev
, 0);
1981 /* Guest initiated reset */
1982 vdev
->device_endian
= virtio_current_cpu_endian();
1985 vdev
->device_endian
= virtio_default_endian();
1992 vdev
->start_on_kick
= false;
1993 vdev
->started
= false;
1994 vdev
->broken
= false;
1995 vdev
->guest_features
= 0;
1996 vdev
->queue_sel
= 0;
1998 vdev
->disabled
= false;
1999 qatomic_set(&vdev
->isr
, 0);
2000 vdev
->config_vector
= VIRTIO_NO_VECTOR
;
2001 virtio_notify_vector(vdev
, vdev
->config_vector
);
2003 for(i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2004 vdev
->vq
[i
].vring
.desc
= 0;
2005 vdev
->vq
[i
].vring
.avail
= 0;
2006 vdev
->vq
[i
].vring
.used
= 0;
2007 vdev
->vq
[i
].last_avail_idx
= 0;
2008 vdev
->vq
[i
].shadow_avail_idx
= 0;
2009 vdev
->vq
[i
].used_idx
= 0;
2010 vdev
->vq
[i
].last_avail_wrap_counter
= true;
2011 vdev
->vq
[i
].shadow_avail_wrap_counter
= true;
2012 vdev
->vq
[i
].used_wrap_counter
= true;
2013 virtio_queue_set_vector(vdev
, i
, VIRTIO_NO_VECTOR
);
2014 vdev
->vq
[i
].signalled_used
= 0;
2015 vdev
->vq
[i
].signalled_used_valid
= false;
2016 vdev
->vq
[i
].notification
= true;
2017 vdev
->vq
[i
].vring
.num
= vdev
->vq
[i
].vring
.num_default
;
2018 vdev
->vq
[i
].inuse
= 0;
2019 virtio_virtqueue_reset_region_cache(&vdev
->vq
[i
]);
2023 uint32_t virtio_config_readb(VirtIODevice
*vdev
, uint32_t addr
)
2025 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2028 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2029 return (uint32_t)-1;
2032 k
->get_config(vdev
, vdev
->config
);
2034 val
= ldub_p(vdev
->config
+ addr
);
2038 uint32_t virtio_config_readw(VirtIODevice
*vdev
, uint32_t addr
)
2040 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2043 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2044 return (uint32_t)-1;
2047 k
->get_config(vdev
, vdev
->config
);
2049 val
= lduw_p(vdev
->config
+ addr
);
2053 uint32_t virtio_config_readl(VirtIODevice
*vdev
, uint32_t addr
)
2055 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2058 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2059 return (uint32_t)-1;
2062 k
->get_config(vdev
, vdev
->config
);
2064 val
= ldl_p(vdev
->config
+ addr
);
2068 void virtio_config_writeb(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
2070 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2073 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2077 stb_p(vdev
->config
+ addr
, val
);
2079 if (k
->set_config
) {
2080 k
->set_config(vdev
, vdev
->config
);
2084 void virtio_config_writew(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
2086 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2087 uint16_t val
= data
;
2089 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2093 stw_p(vdev
->config
+ addr
, val
);
2095 if (k
->set_config
) {
2096 k
->set_config(vdev
, vdev
->config
);
2100 void virtio_config_writel(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
2102 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2103 uint32_t val
= data
;
2105 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2109 stl_p(vdev
->config
+ addr
, val
);
2111 if (k
->set_config
) {
2112 k
->set_config(vdev
, vdev
->config
);
2116 uint32_t virtio_config_modern_readb(VirtIODevice
*vdev
, uint32_t addr
)
2118 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2121 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2122 return (uint32_t)-1;
2125 k
->get_config(vdev
, vdev
->config
);
2127 val
= ldub_p(vdev
->config
+ addr
);
2131 uint32_t virtio_config_modern_readw(VirtIODevice
*vdev
, uint32_t addr
)
2133 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2136 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2137 return (uint32_t)-1;
2140 k
->get_config(vdev
, vdev
->config
);
2142 val
= lduw_le_p(vdev
->config
+ addr
);
2146 uint32_t virtio_config_modern_readl(VirtIODevice
*vdev
, uint32_t addr
)
2148 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2151 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2152 return (uint32_t)-1;
2155 k
->get_config(vdev
, vdev
->config
);
2157 val
= ldl_le_p(vdev
->config
+ addr
);
2161 void virtio_config_modern_writeb(VirtIODevice
*vdev
,
2162 uint32_t addr
, uint32_t data
)
2164 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2167 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2171 stb_p(vdev
->config
+ addr
, val
);
2173 if (k
->set_config
) {
2174 k
->set_config(vdev
, vdev
->config
);
2178 void virtio_config_modern_writew(VirtIODevice
*vdev
,
2179 uint32_t addr
, uint32_t data
)
2181 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2182 uint16_t val
= data
;
2184 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2188 stw_le_p(vdev
->config
+ addr
, val
);
2190 if (k
->set_config
) {
2191 k
->set_config(vdev
, vdev
->config
);
2195 void virtio_config_modern_writel(VirtIODevice
*vdev
,
2196 uint32_t addr
, uint32_t data
)
2198 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2199 uint32_t val
= data
;
2201 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2205 stl_le_p(vdev
->config
+ addr
, val
);
2207 if (k
->set_config
) {
2208 k
->set_config(vdev
, vdev
->config
);
2212 void virtio_queue_set_addr(VirtIODevice
*vdev
, int n
, hwaddr addr
)
2214 if (!vdev
->vq
[n
].vring
.num
) {
2217 vdev
->vq
[n
].vring
.desc
= addr
;
2218 virtio_queue_update_rings(vdev
, n
);
2221 hwaddr
virtio_queue_get_addr(VirtIODevice
*vdev
, int n
)
2223 return vdev
->vq
[n
].vring
.desc
;
2226 void virtio_queue_set_rings(VirtIODevice
*vdev
, int n
, hwaddr desc
,
2227 hwaddr avail
, hwaddr used
)
2229 if (!vdev
->vq
[n
].vring
.num
) {
2232 vdev
->vq
[n
].vring
.desc
= desc
;
2233 vdev
->vq
[n
].vring
.avail
= avail
;
2234 vdev
->vq
[n
].vring
.used
= used
;
2235 virtio_init_region_cache(vdev
, n
);
2238 void virtio_queue_set_num(VirtIODevice
*vdev
, int n
, int num
)
2240 /* Don't allow guest to flip queue between existent and
2241 * nonexistent states, or to set it to an invalid size.
2243 if (!!num
!= !!vdev
->vq
[n
].vring
.num
||
2244 num
> VIRTQUEUE_MAX_SIZE
||
2248 vdev
->vq
[n
].vring
.num
= num
;
2251 VirtQueue
*virtio_vector_first_queue(VirtIODevice
*vdev
, uint16_t vector
)
2253 return QLIST_FIRST(&vdev
->vector_queues
[vector
]);
2256 VirtQueue
*virtio_vector_next_queue(VirtQueue
*vq
)
2258 return QLIST_NEXT(vq
, node
);
2261 int virtio_queue_get_num(VirtIODevice
*vdev
, int n
)
2263 return vdev
->vq
[n
].vring
.num
;
2266 int virtio_queue_get_max_num(VirtIODevice
*vdev
, int n
)
2268 return vdev
->vq
[n
].vring
.num_default
;
2271 int virtio_get_num_queues(VirtIODevice
*vdev
)
2275 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2276 if (!virtio_queue_get_num(vdev
, i
)) {
2284 void virtio_queue_set_align(VirtIODevice
*vdev
, int n
, int align
)
2286 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2287 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2289 /* virtio-1 compliant devices cannot change the alignment */
2290 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
2291 error_report("tried to modify queue alignment for virtio-1 device");
2294 /* Check that the transport told us it was going to do this
2295 * (so a buggy transport will immediately assert rather than
2296 * silently failing to migrate this state)
2298 assert(k
->has_variable_vring_alignment
);
2301 vdev
->vq
[n
].vring
.align
= align
;
2302 virtio_queue_update_rings(vdev
, n
);
2306 static bool virtio_queue_notify_aio_vq(VirtQueue
*vq
)
2310 if (vq
->vring
.desc
&& vq
->handle_aio_output
) {
2311 VirtIODevice
*vdev
= vq
->vdev
;
2313 trace_virtio_queue_notify(vdev
, vq
- vdev
->vq
, vq
);
2314 ret
= vq
->handle_aio_output(vdev
, vq
);
2316 if (unlikely(vdev
->start_on_kick
)) {
2317 virtio_set_started(vdev
, true);
2324 static void virtio_queue_notify_vq(VirtQueue
*vq
)
2326 if (vq
->vring
.desc
&& vq
->handle_output
) {
2327 VirtIODevice
*vdev
= vq
->vdev
;
2329 if (unlikely(vdev
->broken
)) {
2333 trace_virtio_queue_notify(vdev
, vq
- vdev
->vq
, vq
);
2334 vq
->handle_output(vdev
, vq
);
2336 if (unlikely(vdev
->start_on_kick
)) {
2337 virtio_set_started(vdev
, true);
2342 void virtio_queue_notify(VirtIODevice
*vdev
, int n
)
2344 VirtQueue
*vq
= &vdev
->vq
[n
];
2346 if (unlikely(!vq
->vring
.desc
|| vdev
->broken
)) {
2350 trace_virtio_queue_notify(vdev
, vq
- vdev
->vq
, vq
);
2351 if (vq
->host_notifier_enabled
) {
2352 event_notifier_set(&vq
->host_notifier
);
2353 } else if (vq
->handle_output
) {
2354 vq
->handle_output(vdev
, vq
);
2356 if (unlikely(vdev
->start_on_kick
)) {
2357 virtio_set_started(vdev
, true);
2362 uint16_t virtio_queue_vector(VirtIODevice
*vdev
, int n
)
2364 return n
< VIRTIO_QUEUE_MAX
? vdev
->vq
[n
].vector
:
2368 void virtio_queue_set_vector(VirtIODevice
*vdev
, int n
, uint16_t vector
)
2370 VirtQueue
*vq
= &vdev
->vq
[n
];
2372 if (n
< VIRTIO_QUEUE_MAX
) {
2373 if (vdev
->vector_queues
&&
2374 vdev
->vq
[n
].vector
!= VIRTIO_NO_VECTOR
) {
2375 QLIST_REMOVE(vq
, node
);
2377 vdev
->vq
[n
].vector
= vector
;
2378 if (vdev
->vector_queues
&&
2379 vector
!= VIRTIO_NO_VECTOR
) {
2380 QLIST_INSERT_HEAD(&vdev
->vector_queues
[vector
], vq
, node
);
2385 VirtQueue
*virtio_add_queue(VirtIODevice
*vdev
, int queue_size
,
2386 VirtIOHandleOutput handle_output
)
2390 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2391 if (vdev
->vq
[i
].vring
.num
== 0)
2395 if (i
== VIRTIO_QUEUE_MAX
|| queue_size
> VIRTQUEUE_MAX_SIZE
)
2398 vdev
->vq
[i
].vring
.num
= queue_size
;
2399 vdev
->vq
[i
].vring
.num_default
= queue_size
;
2400 vdev
->vq
[i
].vring
.align
= VIRTIO_PCI_VRING_ALIGN
;
2401 vdev
->vq
[i
].handle_output
= handle_output
;
2402 vdev
->vq
[i
].handle_aio_output
= NULL
;
2403 vdev
->vq
[i
].used_elems
= g_malloc0(sizeof(VirtQueueElement
) *
2406 return &vdev
->vq
[i
];
2409 void virtio_delete_queue(VirtQueue
*vq
)
2412 vq
->vring
.num_default
= 0;
2413 vq
->handle_output
= NULL
;
2414 vq
->handle_aio_output
= NULL
;
2415 g_free(vq
->used_elems
);
2416 vq
->used_elems
= NULL
;
2417 virtio_virtqueue_reset_region_cache(vq
);
2420 void virtio_del_queue(VirtIODevice
*vdev
, int n
)
2422 if (n
< 0 || n
>= VIRTIO_QUEUE_MAX
) {
2426 virtio_delete_queue(&vdev
->vq
[n
]);
2429 static void virtio_set_isr(VirtIODevice
*vdev
, int value
)
2431 uint8_t old
= qatomic_read(&vdev
->isr
);
2433 /* Do not write ISR if it does not change, so that its cacheline remains
2434 * shared in the common case where the guest does not read it.
2436 if ((old
& value
) != value
) {
2437 qatomic_or(&vdev
->isr
, value
);
2441 /* Called within rcu_read_lock(). */
2442 static bool virtio_split_should_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
2446 /* We need to expose used array entries before checking used event. */
2448 /* Always notify when queue is empty (when feature acknowledge) */
2449 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_NOTIFY_ON_EMPTY
) &&
2450 !vq
->inuse
&& virtio_queue_empty(vq
)) {
2454 if (!virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
2455 return !(vring_avail_flags(vq
) & VRING_AVAIL_F_NO_INTERRUPT
);
2458 v
= vq
->signalled_used_valid
;
2459 vq
->signalled_used_valid
= true;
2460 old
= vq
->signalled_used
;
2461 new = vq
->signalled_used
= vq
->used_idx
;
2462 return !v
|| vring_need_event(vring_get_used_event(vq
), new, old
);
2465 static bool vring_packed_need_event(VirtQueue
*vq
, bool wrap
,
2466 uint16_t off_wrap
, uint16_t new,
2469 int off
= off_wrap
& ~(1 << 15);
2471 if (wrap
!= off_wrap
>> 15) {
2472 off
-= vq
->vring
.num
;
2475 return vring_need_event(off
, new, old
);
2478 /* Called within rcu_read_lock(). */
2479 static bool virtio_packed_should_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
2481 VRingPackedDescEvent e
;
2484 VRingMemoryRegionCaches
*caches
;
2486 caches
= vring_get_region_caches(vq
);
2491 vring_packed_event_read(vdev
, &caches
->avail
, &e
);
2493 old
= vq
->signalled_used
;
2494 new = vq
->signalled_used
= vq
->used_idx
;
2495 v
= vq
->signalled_used_valid
;
2496 vq
->signalled_used_valid
= true;
2498 if (e
.flags
== VRING_PACKED_EVENT_FLAG_DISABLE
) {
2500 } else if (e
.flags
== VRING_PACKED_EVENT_FLAG_ENABLE
) {
2504 return !v
|| vring_packed_need_event(vq
, vq
->used_wrap_counter
,
2505 e
.off_wrap
, new, old
);
2508 /* Called within rcu_read_lock(). */
2509 static bool virtio_should_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
2511 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
2512 return virtio_packed_should_notify(vdev
, vq
);
2514 return virtio_split_should_notify(vdev
, vq
);
2518 void virtio_notify_irqfd(VirtIODevice
*vdev
, VirtQueue
*vq
)
2520 WITH_RCU_READ_LOCK_GUARD() {
2521 if (!virtio_should_notify(vdev
, vq
)) {
2526 trace_virtio_notify_irqfd(vdev
, vq
);
2529 * virtio spec 1.0 says ISR bit 0 should be ignored with MSI, but
2530 * windows drivers included in virtio-win 1.8.0 (circa 2015) are
2531 * incorrectly polling this bit during crashdump and hibernation
2532 * in MSI mode, causing a hang if this bit is never updated.
2533 * Recent releases of Windows do not really shut down, but rather
2534 * log out and hibernate to make the next startup faster. Hence,
2535 * this manifested as a more serious hang during shutdown with
2537 * Next driver release from 2016 fixed this problem, so working around it
2538 * is not a must, but it's easy to do so let's do it here.
2540 * Note: it's safe to update ISR from any thread as it was switched
2541 * to an atomic operation.
2543 virtio_set_isr(vq
->vdev
, 0x1);
2544 event_notifier_set(&vq
->guest_notifier
);
2547 static void virtio_irq(VirtQueue
*vq
)
2549 virtio_set_isr(vq
->vdev
, 0x1);
2550 virtio_notify_vector(vq
->vdev
, vq
->vector
);
2553 void virtio_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
2555 WITH_RCU_READ_LOCK_GUARD() {
2556 if (!virtio_should_notify(vdev
, vq
)) {
2561 trace_virtio_notify(vdev
, vq
);
2565 void virtio_notify_config(VirtIODevice
*vdev
)
2567 if (!(vdev
->status
& VIRTIO_CONFIG_S_DRIVER_OK
))
2570 virtio_set_isr(vdev
, 0x3);
2572 virtio_notify_vector(vdev
, vdev
->config_vector
);
2575 static bool virtio_device_endian_needed(void *opaque
)
2577 VirtIODevice
*vdev
= opaque
;
2579 assert(vdev
->device_endian
!= VIRTIO_DEVICE_ENDIAN_UNKNOWN
);
2580 if (!virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
2581 return vdev
->device_endian
!= virtio_default_endian();
2583 /* Devices conforming to VIRTIO 1.0 or later are always LE. */
2584 return vdev
->device_endian
!= VIRTIO_DEVICE_ENDIAN_LITTLE
;
2587 static bool virtio_64bit_features_needed(void *opaque
)
2589 VirtIODevice
*vdev
= opaque
;
2591 return (vdev
->host_features
>> 32) != 0;
2594 static bool virtio_virtqueue_needed(void *opaque
)
2596 VirtIODevice
*vdev
= opaque
;
2598 return virtio_host_has_feature(vdev
, VIRTIO_F_VERSION_1
);
2601 static bool virtio_packed_virtqueue_needed(void *opaque
)
2603 VirtIODevice
*vdev
= opaque
;
2605 return virtio_host_has_feature(vdev
, VIRTIO_F_RING_PACKED
);
2608 static bool virtio_ringsize_needed(void *opaque
)
2610 VirtIODevice
*vdev
= opaque
;
2613 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2614 if (vdev
->vq
[i
].vring
.num
!= vdev
->vq
[i
].vring
.num_default
) {
2621 static bool virtio_extra_state_needed(void *opaque
)
2623 VirtIODevice
*vdev
= opaque
;
2624 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2625 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2627 return k
->has_extra_state
&&
2628 k
->has_extra_state(qbus
->parent
);
2631 static bool virtio_broken_needed(void *opaque
)
2633 VirtIODevice
*vdev
= opaque
;
2635 return vdev
->broken
;
2638 static bool virtio_started_needed(void *opaque
)
2640 VirtIODevice
*vdev
= opaque
;
2642 return vdev
->started
;
2645 static bool virtio_disabled_needed(void *opaque
)
2647 VirtIODevice
*vdev
= opaque
;
2649 return vdev
->disabled
;
2652 static const VMStateDescription vmstate_virtqueue
= {
2653 .name
= "virtqueue_state",
2655 .minimum_version_id
= 1,
2656 .fields
= (VMStateField
[]) {
2657 VMSTATE_UINT64(vring
.avail
, struct VirtQueue
),
2658 VMSTATE_UINT64(vring
.used
, struct VirtQueue
),
2659 VMSTATE_END_OF_LIST()
2663 static const VMStateDescription vmstate_packed_virtqueue
= {
2664 .name
= "packed_virtqueue_state",
2666 .minimum_version_id
= 1,
2667 .fields
= (VMStateField
[]) {
2668 VMSTATE_UINT16(last_avail_idx
, struct VirtQueue
),
2669 VMSTATE_BOOL(last_avail_wrap_counter
, struct VirtQueue
),
2670 VMSTATE_UINT16(used_idx
, struct VirtQueue
),
2671 VMSTATE_BOOL(used_wrap_counter
, struct VirtQueue
),
2672 VMSTATE_UINT32(inuse
, struct VirtQueue
),
2673 VMSTATE_END_OF_LIST()
2677 static const VMStateDescription vmstate_virtio_virtqueues
= {
2678 .name
= "virtio/virtqueues",
2680 .minimum_version_id
= 1,
2681 .needed
= &virtio_virtqueue_needed
,
2682 .fields
= (VMStateField
[]) {
2683 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq
, struct VirtIODevice
,
2684 VIRTIO_QUEUE_MAX
, 0, vmstate_virtqueue
, VirtQueue
),
2685 VMSTATE_END_OF_LIST()
2689 static const VMStateDescription vmstate_virtio_packed_virtqueues
= {
2690 .name
= "virtio/packed_virtqueues",
2692 .minimum_version_id
= 1,
2693 .needed
= &virtio_packed_virtqueue_needed
,
2694 .fields
= (VMStateField
[]) {
2695 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq
, struct VirtIODevice
,
2696 VIRTIO_QUEUE_MAX
, 0, vmstate_packed_virtqueue
, VirtQueue
),
2697 VMSTATE_END_OF_LIST()
2701 static const VMStateDescription vmstate_ringsize
= {
2702 .name
= "ringsize_state",
2704 .minimum_version_id
= 1,
2705 .fields
= (VMStateField
[]) {
2706 VMSTATE_UINT32(vring
.num_default
, struct VirtQueue
),
2707 VMSTATE_END_OF_LIST()
2711 static const VMStateDescription vmstate_virtio_ringsize
= {
2712 .name
= "virtio/ringsize",
2714 .minimum_version_id
= 1,
2715 .needed
= &virtio_ringsize_needed
,
2716 .fields
= (VMStateField
[]) {
2717 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq
, struct VirtIODevice
,
2718 VIRTIO_QUEUE_MAX
, 0, vmstate_ringsize
, VirtQueue
),
2719 VMSTATE_END_OF_LIST()
2723 static int get_extra_state(QEMUFile
*f
, void *pv
, size_t size
,
2724 const VMStateField
*field
)
2726 VirtIODevice
*vdev
= pv
;
2727 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2728 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2730 if (!k
->load_extra_state
) {
2733 return k
->load_extra_state(qbus
->parent
, f
);
2737 static int put_extra_state(QEMUFile
*f
, void *pv
, size_t size
,
2738 const VMStateField
*field
, JSONWriter
*vmdesc
)
2740 VirtIODevice
*vdev
= pv
;
2741 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2742 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2744 k
->save_extra_state(qbus
->parent
, f
);
2748 static const VMStateInfo vmstate_info_extra_state
= {
2749 .name
= "virtqueue_extra_state",
2750 .get
= get_extra_state
,
2751 .put
= put_extra_state
,
2754 static const VMStateDescription vmstate_virtio_extra_state
= {
2755 .name
= "virtio/extra_state",
2757 .minimum_version_id
= 1,
2758 .needed
= &virtio_extra_state_needed
,
2759 .fields
= (VMStateField
[]) {
2761 .name
= "extra_state",
2763 .field_exists
= NULL
,
2765 .info
= &vmstate_info_extra_state
,
2766 .flags
= VMS_SINGLE
,
2769 VMSTATE_END_OF_LIST()
2773 static const VMStateDescription vmstate_virtio_device_endian
= {
2774 .name
= "virtio/device_endian",
2776 .minimum_version_id
= 1,
2777 .needed
= &virtio_device_endian_needed
,
2778 .fields
= (VMStateField
[]) {
2779 VMSTATE_UINT8(device_endian
, VirtIODevice
),
2780 VMSTATE_END_OF_LIST()
2784 static const VMStateDescription vmstate_virtio_64bit_features
= {
2785 .name
= "virtio/64bit_features",
2787 .minimum_version_id
= 1,
2788 .needed
= &virtio_64bit_features_needed
,
2789 .fields
= (VMStateField
[]) {
2790 VMSTATE_UINT64(guest_features
, VirtIODevice
),
2791 VMSTATE_END_OF_LIST()
2795 static const VMStateDescription vmstate_virtio_broken
= {
2796 .name
= "virtio/broken",
2798 .minimum_version_id
= 1,
2799 .needed
= &virtio_broken_needed
,
2800 .fields
= (VMStateField
[]) {
2801 VMSTATE_BOOL(broken
, VirtIODevice
),
2802 VMSTATE_END_OF_LIST()
2806 static const VMStateDescription vmstate_virtio_started
= {
2807 .name
= "virtio/started",
2809 .minimum_version_id
= 1,
2810 .needed
= &virtio_started_needed
,
2811 .fields
= (VMStateField
[]) {
2812 VMSTATE_BOOL(started
, VirtIODevice
),
2813 VMSTATE_END_OF_LIST()
2817 static const VMStateDescription vmstate_virtio_disabled
= {
2818 .name
= "virtio/disabled",
2820 .minimum_version_id
= 1,
2821 .needed
= &virtio_disabled_needed
,
2822 .fields
= (VMStateField
[]) {
2823 VMSTATE_BOOL(disabled
, VirtIODevice
),
2824 VMSTATE_END_OF_LIST()
2828 static const VMStateDescription vmstate_virtio
= {
2831 .minimum_version_id
= 1,
2832 .minimum_version_id_old
= 1,
2833 .fields
= (VMStateField
[]) {
2834 VMSTATE_END_OF_LIST()
2836 .subsections
= (const VMStateDescription
*[]) {
2837 &vmstate_virtio_device_endian
,
2838 &vmstate_virtio_64bit_features
,
2839 &vmstate_virtio_virtqueues
,
2840 &vmstate_virtio_ringsize
,
2841 &vmstate_virtio_broken
,
2842 &vmstate_virtio_extra_state
,
2843 &vmstate_virtio_started
,
2844 &vmstate_virtio_packed_virtqueues
,
2845 &vmstate_virtio_disabled
,
2850 int virtio_save(VirtIODevice
*vdev
, QEMUFile
*f
)
2852 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2853 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2854 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2855 uint32_t guest_features_lo
= (vdev
->guest_features
& 0xffffffff);
2858 if (k
->save_config
) {
2859 k
->save_config(qbus
->parent
, f
);
2862 qemu_put_8s(f
, &vdev
->status
);
2863 qemu_put_8s(f
, &vdev
->isr
);
2864 qemu_put_be16s(f
, &vdev
->queue_sel
);
2865 qemu_put_be32s(f
, &guest_features_lo
);
2866 qemu_put_be32(f
, vdev
->config_len
);
2867 qemu_put_buffer(f
, vdev
->config
, vdev
->config_len
);
2869 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2870 if (vdev
->vq
[i
].vring
.num
== 0)
2874 qemu_put_be32(f
, i
);
2876 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2877 if (vdev
->vq
[i
].vring
.num
== 0)
2880 qemu_put_be32(f
, vdev
->vq
[i
].vring
.num
);
2881 if (k
->has_variable_vring_alignment
) {
2882 qemu_put_be32(f
, vdev
->vq
[i
].vring
.align
);
2885 * Save desc now, the rest of the ring addresses are saved in
2886 * subsections for VIRTIO-1 devices.
2888 qemu_put_be64(f
, vdev
->vq
[i
].vring
.desc
);
2889 qemu_put_be16s(f
, &vdev
->vq
[i
].last_avail_idx
);
2890 if (k
->save_queue
) {
2891 k
->save_queue(qbus
->parent
, i
, f
);
2895 if (vdc
->save
!= NULL
) {
2900 int ret
= vmstate_save_state(f
, vdc
->vmsd
, vdev
, NULL
);
2907 return vmstate_save_state(f
, &vmstate_virtio
, vdev
, NULL
);
2910 /* A wrapper for use as a VMState .put function */
2911 static int virtio_device_put(QEMUFile
*f
, void *opaque
, size_t size
,
2912 const VMStateField
*field
, JSONWriter
*vmdesc
)
2914 return virtio_save(VIRTIO_DEVICE(opaque
), f
);
2917 /* A wrapper for use as a VMState .get function */
2918 static int virtio_device_get(QEMUFile
*f
, void *opaque
, size_t size
,
2919 const VMStateField
*field
)
2921 VirtIODevice
*vdev
= VIRTIO_DEVICE(opaque
);
2922 DeviceClass
*dc
= DEVICE_CLASS(VIRTIO_DEVICE_GET_CLASS(vdev
));
2924 return virtio_load(vdev
, f
, dc
->vmsd
->version_id
);
2927 const VMStateInfo virtio_vmstate_info
= {
2929 .get
= virtio_device_get
,
2930 .put
= virtio_device_put
,
2933 static int virtio_set_features_nocheck(VirtIODevice
*vdev
, uint64_t val
)
2935 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2936 bool bad
= (val
& ~(vdev
->host_features
)) != 0;
2938 val
&= vdev
->host_features
;
2939 if (k
->set_features
) {
2940 k
->set_features(vdev
, val
);
2942 vdev
->guest_features
= val
;
2943 return bad
? -1 : 0;
2946 int virtio_set_features(VirtIODevice
*vdev
, uint64_t val
)
2950 * The driver must not attempt to set features after feature negotiation
2953 if (vdev
->status
& VIRTIO_CONFIG_S_FEATURES_OK
) {
2956 ret
= virtio_set_features_nocheck(vdev
, val
);
2957 if (virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
2958 /* VIRTIO_RING_F_EVENT_IDX changes the size of the caches. */
2960 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2961 if (vdev
->vq
[i
].vring
.num
!= 0) {
2962 virtio_init_region_cache(vdev
, i
);
2967 if (!virtio_device_started(vdev
, vdev
->status
) &&
2968 !virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
2969 vdev
->start_on_kick
= true;
2975 size_t virtio_feature_get_config_size(const VirtIOFeature
*feature_sizes
,
2976 uint64_t host_features
)
2978 size_t config_size
= 0;
2981 for (i
= 0; feature_sizes
[i
].flags
!= 0; i
++) {
2982 if (host_features
& feature_sizes
[i
].flags
) {
2983 config_size
= MAX(feature_sizes
[i
].end
, config_size
);
2990 int virtio_load(VirtIODevice
*vdev
, QEMUFile
*f
, int version_id
)
2996 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2997 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2998 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(vdev
);
3001 * We poison the endianness to ensure it does not get used before
3002 * subsections have been loaded.
3004 vdev
->device_endian
= VIRTIO_DEVICE_ENDIAN_UNKNOWN
;
3006 if (k
->load_config
) {
3007 ret
= k
->load_config(qbus
->parent
, f
);
3012 qemu_get_8s(f
, &vdev
->status
);
3013 qemu_get_8s(f
, &vdev
->isr
);
3014 qemu_get_be16s(f
, &vdev
->queue_sel
);
3015 if (vdev
->queue_sel
>= VIRTIO_QUEUE_MAX
) {
3018 qemu_get_be32s(f
, &features
);
3021 * Temporarily set guest_features low bits - needed by
3022 * virtio net load code testing for VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
3023 * VIRTIO_NET_F_GUEST_ANNOUNCE and VIRTIO_NET_F_CTRL_VQ.
3025 * Note: devices should always test host features in future - don't create
3026 * new dependencies like this.
3028 vdev
->guest_features
= features
;
3030 config_len
= qemu_get_be32(f
);
3033 * There are cases where the incoming config can be bigger or smaller
3034 * than what we have; so load what we have space for, and skip
3035 * any excess that's in the stream.
3037 qemu_get_buffer(f
, vdev
->config
, MIN(config_len
, vdev
->config_len
));
3039 while (config_len
> vdev
->config_len
) {
3044 num
= qemu_get_be32(f
);
3046 if (num
> VIRTIO_QUEUE_MAX
) {
3047 error_report("Invalid number of virtqueues: 0x%x", num
);
3051 for (i
= 0; i
< num
; i
++) {
3052 vdev
->vq
[i
].vring
.num
= qemu_get_be32(f
);
3053 if (k
->has_variable_vring_alignment
) {
3054 vdev
->vq
[i
].vring
.align
= qemu_get_be32(f
);
3056 vdev
->vq
[i
].vring
.desc
= qemu_get_be64(f
);
3057 qemu_get_be16s(f
, &vdev
->vq
[i
].last_avail_idx
);
3058 vdev
->vq
[i
].signalled_used_valid
= false;
3059 vdev
->vq
[i
].notification
= true;
3061 if (!vdev
->vq
[i
].vring
.desc
&& vdev
->vq
[i
].last_avail_idx
) {
3062 error_report("VQ %d address 0x0 "
3063 "inconsistent with Host index 0x%x",
3064 i
, vdev
->vq
[i
].last_avail_idx
);
3067 if (k
->load_queue
) {
3068 ret
= k
->load_queue(qbus
->parent
, i
, f
);
3074 virtio_notify_vector(vdev
, VIRTIO_NO_VECTOR
);
3076 if (vdc
->load
!= NULL
) {
3077 ret
= vdc
->load(vdev
, f
, version_id
);
3084 ret
= vmstate_load_state(f
, vdc
->vmsd
, vdev
, version_id
);
3091 ret
= vmstate_load_state(f
, &vmstate_virtio
, vdev
, 1);
3096 if (vdev
->device_endian
== VIRTIO_DEVICE_ENDIAN_UNKNOWN
) {
3097 vdev
->device_endian
= virtio_default_endian();
3100 if (virtio_64bit_features_needed(vdev
)) {
3102 * Subsection load filled vdev->guest_features. Run them
3103 * through virtio_set_features to sanity-check them against
3106 uint64_t features64
= vdev
->guest_features
;
3107 if (virtio_set_features_nocheck(vdev
, features64
) < 0) {
3108 error_report("Features 0x%" PRIx64
" unsupported. "
3109 "Allowed features: 0x%" PRIx64
,
3110 features64
, vdev
->host_features
);
3114 if (virtio_set_features_nocheck(vdev
, features
) < 0) {
3115 error_report("Features 0x%x unsupported. "
3116 "Allowed features: 0x%" PRIx64
,
3117 features
, vdev
->host_features
);
3122 if (!virtio_device_started(vdev
, vdev
->status
) &&
3123 !virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
3124 vdev
->start_on_kick
= true;
3127 RCU_READ_LOCK_GUARD();
3128 for (i
= 0; i
< num
; i
++) {
3129 if (vdev
->vq
[i
].vring
.desc
) {
3133 * VIRTIO-1 devices migrate desc, used, and avail ring addresses so
3134 * only the region cache needs to be set up. Legacy devices need
3135 * to calculate used and avail ring addresses based on the desc
3138 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
3139 virtio_init_region_cache(vdev
, i
);
3141 virtio_queue_update_rings(vdev
, i
);
3144 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3145 vdev
->vq
[i
].shadow_avail_idx
= vdev
->vq
[i
].last_avail_idx
;
3146 vdev
->vq
[i
].shadow_avail_wrap_counter
=
3147 vdev
->vq
[i
].last_avail_wrap_counter
;
3151 nheads
= vring_avail_idx(&vdev
->vq
[i
]) - vdev
->vq
[i
].last_avail_idx
;
3152 /* Check it isn't doing strange things with descriptor numbers. */
3153 if (nheads
> vdev
->vq
[i
].vring
.num
) {
3154 virtio_error(vdev
, "VQ %d size 0x%x Guest index 0x%x "
3155 "inconsistent with Host index 0x%x: delta 0x%x",
3156 i
, vdev
->vq
[i
].vring
.num
,
3157 vring_avail_idx(&vdev
->vq
[i
]),
3158 vdev
->vq
[i
].last_avail_idx
, nheads
);
3159 vdev
->vq
[i
].used_idx
= 0;
3160 vdev
->vq
[i
].shadow_avail_idx
= 0;
3161 vdev
->vq
[i
].inuse
= 0;
3164 vdev
->vq
[i
].used_idx
= vring_used_idx(&vdev
->vq
[i
]);
3165 vdev
->vq
[i
].shadow_avail_idx
= vring_avail_idx(&vdev
->vq
[i
]);
3168 * Some devices migrate VirtQueueElements that have been popped
3169 * from the avail ring but not yet returned to the used ring.
3170 * Since max ring size < UINT16_MAX it's safe to use modulo
3171 * UINT16_MAX + 1 subtraction.
3173 vdev
->vq
[i
].inuse
= (uint16_t)(vdev
->vq
[i
].last_avail_idx
-
3174 vdev
->vq
[i
].used_idx
);
3175 if (vdev
->vq
[i
].inuse
> vdev
->vq
[i
].vring
.num
) {
3176 error_report("VQ %d size 0x%x < last_avail_idx 0x%x - "
3178 i
, vdev
->vq
[i
].vring
.num
,
3179 vdev
->vq
[i
].last_avail_idx
,
3180 vdev
->vq
[i
].used_idx
);
3186 if (vdc
->post_load
) {
3187 ret
= vdc
->post_load(vdev
);
3196 void virtio_cleanup(VirtIODevice
*vdev
)
3198 qemu_del_vm_change_state_handler(vdev
->vmstate
);
3201 static void virtio_vmstate_change(void *opaque
, bool running
, RunState state
)
3203 VirtIODevice
*vdev
= opaque
;
3204 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3205 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
3206 bool backend_run
= running
&& virtio_device_started(vdev
, vdev
->status
);
3207 vdev
->vm_running
= running
;
3210 virtio_set_status(vdev
, vdev
->status
);
3213 if (k
->vmstate_change
) {
3214 k
->vmstate_change(qbus
->parent
, backend_run
);
3218 virtio_set_status(vdev
, vdev
->status
);
3222 void virtio_instance_init_common(Object
*proxy_obj
, void *data
,
3223 size_t vdev_size
, const char *vdev_name
)
3225 DeviceState
*vdev
= data
;
3227 object_initialize_child_with_props(proxy_obj
, "virtio-backend", vdev
,
3228 vdev_size
, vdev_name
, &error_abort
,
3230 qdev_alias_all_properties(vdev
, proxy_obj
);
3233 void virtio_init(VirtIODevice
*vdev
, const char *name
,
3234 uint16_t device_id
, size_t config_size
)
3236 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3237 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
3239 int nvectors
= k
->query_nvectors
? k
->query_nvectors(qbus
->parent
) : 0;
3242 vdev
->vector_queues
=
3243 g_malloc0(sizeof(*vdev
->vector_queues
) * nvectors
);
3246 vdev
->start_on_kick
= false;
3247 vdev
->started
= false;
3248 vdev
->device_id
= device_id
;
3250 qatomic_set(&vdev
->isr
, 0);
3251 vdev
->queue_sel
= 0;
3252 vdev
->config_vector
= VIRTIO_NO_VECTOR
;
3253 vdev
->vq
= g_malloc0(sizeof(VirtQueue
) * VIRTIO_QUEUE_MAX
);
3254 vdev
->vm_running
= runstate_is_running();
3255 vdev
->broken
= false;
3256 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
3257 vdev
->vq
[i
].vector
= VIRTIO_NO_VECTOR
;
3258 vdev
->vq
[i
].vdev
= vdev
;
3259 vdev
->vq
[i
].queue_index
= i
;
3260 vdev
->vq
[i
].host_notifier_enabled
= false;
3264 vdev
->config_len
= config_size
;
3265 if (vdev
->config_len
) {
3266 vdev
->config
= g_malloc0(config_size
);
3268 vdev
->config
= NULL
;
3270 vdev
->vmstate
= qdev_add_vm_change_state_handler(DEVICE(vdev
),
3271 virtio_vmstate_change
, vdev
);
3272 vdev
->device_endian
= virtio_default_endian();
3273 vdev
->use_guest_notifier_mask
= true;
3277 * Only devices that have already been around prior to defining the virtio
3278 * standard support legacy mode; this includes devices not specified in the
3279 * standard. All newer devices conform to the virtio standard only.
3281 bool virtio_legacy_allowed(VirtIODevice
*vdev
)
3283 switch (vdev
->device_id
) {
3285 case VIRTIO_ID_BLOCK
:
3286 case VIRTIO_ID_CONSOLE
:
3288 case VIRTIO_ID_BALLOON
:
3289 case VIRTIO_ID_RPMSG
:
3290 case VIRTIO_ID_SCSI
:
3292 case VIRTIO_ID_RPROC_SERIAL
:
3293 case VIRTIO_ID_CAIF
:
3300 bool virtio_legacy_check_disabled(VirtIODevice
*vdev
)
3302 return vdev
->disable_legacy_check
;
3305 hwaddr
virtio_queue_get_desc_addr(VirtIODevice
*vdev
, int n
)
3307 return vdev
->vq
[n
].vring
.desc
;
3310 bool virtio_queue_enabled_legacy(VirtIODevice
*vdev
, int n
)
3312 return virtio_queue_get_desc_addr(vdev
, n
) != 0;
3315 bool virtio_queue_enabled(VirtIODevice
*vdev
, int n
)
3317 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3318 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
3320 if (k
->queue_enabled
) {
3321 return k
->queue_enabled(qbus
->parent
, n
);
3323 return virtio_queue_enabled_legacy(vdev
, n
);
3326 hwaddr
virtio_queue_get_avail_addr(VirtIODevice
*vdev
, int n
)
3328 return vdev
->vq
[n
].vring
.avail
;
3331 hwaddr
virtio_queue_get_used_addr(VirtIODevice
*vdev
, int n
)
3333 return vdev
->vq
[n
].vring
.used
;
3336 hwaddr
virtio_queue_get_desc_size(VirtIODevice
*vdev
, int n
)
3338 return sizeof(VRingDesc
) * vdev
->vq
[n
].vring
.num
;
3341 hwaddr
virtio_queue_get_avail_size(VirtIODevice
*vdev
, int n
)
3345 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3346 return sizeof(struct VRingPackedDescEvent
);
3349 s
= virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
) ? 2 : 0;
3350 return offsetof(VRingAvail
, ring
) +
3351 sizeof(uint16_t) * vdev
->vq
[n
].vring
.num
+ s
;
3354 hwaddr
virtio_queue_get_used_size(VirtIODevice
*vdev
, int n
)
3358 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3359 return sizeof(struct VRingPackedDescEvent
);
3362 s
= virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
) ? 2 : 0;
3363 return offsetof(VRingUsed
, ring
) +
3364 sizeof(VRingUsedElem
) * vdev
->vq
[n
].vring
.num
+ s
;
3367 static unsigned int virtio_queue_packed_get_last_avail_idx(VirtIODevice
*vdev
,
3370 unsigned int avail
, used
;
3372 avail
= vdev
->vq
[n
].last_avail_idx
;
3373 avail
|= ((uint16_t)vdev
->vq
[n
].last_avail_wrap_counter
) << 15;
3375 used
= vdev
->vq
[n
].used_idx
;
3376 used
|= ((uint16_t)vdev
->vq
[n
].used_wrap_counter
) << 15;
3378 return avail
| used
<< 16;
3381 static uint16_t virtio_queue_split_get_last_avail_idx(VirtIODevice
*vdev
,
3384 return vdev
->vq
[n
].last_avail_idx
;
3387 unsigned int virtio_queue_get_last_avail_idx(VirtIODevice
*vdev
, int n
)
3389 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3390 return virtio_queue_packed_get_last_avail_idx(vdev
, n
);
3392 return virtio_queue_split_get_last_avail_idx(vdev
, n
);
3396 static void virtio_queue_packed_set_last_avail_idx(VirtIODevice
*vdev
,
3397 int n
, unsigned int idx
)
3399 struct VirtQueue
*vq
= &vdev
->vq
[n
];
3401 vq
->last_avail_idx
= vq
->shadow_avail_idx
= idx
& 0x7fff;
3402 vq
->last_avail_wrap_counter
=
3403 vq
->shadow_avail_wrap_counter
= !!(idx
& 0x8000);
3405 vq
->used_idx
= idx
& 0x7ffff;
3406 vq
->used_wrap_counter
= !!(idx
& 0x8000);
3409 static void virtio_queue_split_set_last_avail_idx(VirtIODevice
*vdev
,
3410 int n
, unsigned int idx
)
3412 vdev
->vq
[n
].last_avail_idx
= idx
;
3413 vdev
->vq
[n
].shadow_avail_idx
= idx
;
3416 void virtio_queue_set_last_avail_idx(VirtIODevice
*vdev
, int n
,
3419 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3420 virtio_queue_packed_set_last_avail_idx(vdev
, n
, idx
);
3422 virtio_queue_split_set_last_avail_idx(vdev
, n
, idx
);
3426 static void virtio_queue_packed_restore_last_avail_idx(VirtIODevice
*vdev
,
3429 /* We don't have a reference like avail idx in shared memory */
3433 static void virtio_queue_split_restore_last_avail_idx(VirtIODevice
*vdev
,
3436 RCU_READ_LOCK_GUARD();
3437 if (vdev
->vq
[n
].vring
.desc
) {
3438 vdev
->vq
[n
].last_avail_idx
= vring_used_idx(&vdev
->vq
[n
]);
3439 vdev
->vq
[n
].shadow_avail_idx
= vdev
->vq
[n
].last_avail_idx
;
3443 void virtio_queue_restore_last_avail_idx(VirtIODevice
*vdev
, int n
)
3445 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3446 virtio_queue_packed_restore_last_avail_idx(vdev
, n
);
3448 virtio_queue_split_restore_last_avail_idx(vdev
, n
);
3452 static void virtio_queue_packed_update_used_idx(VirtIODevice
*vdev
, int n
)
3454 /* used idx was updated through set_last_avail_idx() */
3458 static void virtio_split_packed_update_used_idx(VirtIODevice
*vdev
, int n
)
3460 RCU_READ_LOCK_GUARD();
3461 if (vdev
->vq
[n
].vring
.desc
) {
3462 vdev
->vq
[n
].used_idx
= vring_used_idx(&vdev
->vq
[n
]);
3466 void virtio_queue_update_used_idx(VirtIODevice
*vdev
, int n
)
3468 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3469 return virtio_queue_packed_update_used_idx(vdev
, n
);
3471 return virtio_split_packed_update_used_idx(vdev
, n
);
3475 void virtio_queue_invalidate_signalled_used(VirtIODevice
*vdev
, int n
)
3477 vdev
->vq
[n
].signalled_used_valid
= false;
3480 VirtQueue
*virtio_get_queue(VirtIODevice
*vdev
, int n
)
3482 return vdev
->vq
+ n
;
3485 uint16_t virtio_get_queue_index(VirtQueue
*vq
)
3487 return vq
->queue_index
;
3490 static void virtio_queue_guest_notifier_read(EventNotifier
*n
)
3492 VirtQueue
*vq
= container_of(n
, VirtQueue
, guest_notifier
);
3493 if (event_notifier_test_and_clear(n
)) {
3498 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue
*vq
, bool assign
,
3501 if (assign
&& !with_irqfd
) {
3502 event_notifier_set_handler(&vq
->guest_notifier
,
3503 virtio_queue_guest_notifier_read
);
3505 event_notifier_set_handler(&vq
->guest_notifier
, NULL
);
3508 /* Test and clear notifier before closing it,
3509 * in case poll callback didn't have time to run. */
3510 virtio_queue_guest_notifier_read(&vq
->guest_notifier
);
3514 EventNotifier
*virtio_queue_get_guest_notifier(VirtQueue
*vq
)
3516 return &vq
->guest_notifier
;
3519 static void virtio_queue_host_notifier_aio_read(EventNotifier
*n
)
3521 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
3522 if (event_notifier_test_and_clear(n
)) {
3523 virtio_queue_notify_aio_vq(vq
);
3527 static void virtio_queue_host_notifier_aio_poll_begin(EventNotifier
*n
)
3529 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
3531 virtio_queue_set_notification(vq
, 0);
3534 static bool virtio_queue_host_notifier_aio_poll(void *opaque
)
3536 EventNotifier
*n
= opaque
;
3537 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
3539 return vq
->vring
.desc
&& !virtio_queue_empty(vq
);
3542 static void virtio_queue_host_notifier_aio_poll_ready(EventNotifier
*n
)
3544 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
3546 virtio_queue_notify_aio_vq(vq
);
3549 static void virtio_queue_host_notifier_aio_poll_end(EventNotifier
*n
)
3551 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
3553 /* Caller polls once more after this to catch requests that race with us */
3554 virtio_queue_set_notification(vq
, 1);
3557 void virtio_queue_aio_set_host_notifier_handler(VirtQueue
*vq
, AioContext
*ctx
,
3558 VirtIOHandleAIOOutput handle_output
)
3560 if (handle_output
) {
3561 vq
->handle_aio_output
= handle_output
;
3562 aio_set_event_notifier(ctx
, &vq
->host_notifier
, true,
3563 virtio_queue_host_notifier_aio_read
,
3564 virtio_queue_host_notifier_aio_poll
,
3565 virtio_queue_host_notifier_aio_poll_ready
);
3566 aio_set_event_notifier_poll(ctx
, &vq
->host_notifier
,
3567 virtio_queue_host_notifier_aio_poll_begin
,
3568 virtio_queue_host_notifier_aio_poll_end
);
3570 aio_set_event_notifier(ctx
, &vq
->host_notifier
, true, NULL
, NULL
, NULL
);
3571 /* Test and clear notifier before after disabling event,
3572 * in case poll callback didn't have time to run. */
3573 virtio_queue_host_notifier_aio_read(&vq
->host_notifier
);
3574 vq
->handle_aio_output
= NULL
;
3578 void virtio_queue_host_notifier_read(EventNotifier
*n
)
3580 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
3581 if (event_notifier_test_and_clear(n
)) {
3582 virtio_queue_notify_vq(vq
);
3586 EventNotifier
*virtio_queue_get_host_notifier(VirtQueue
*vq
)
3588 return &vq
->host_notifier
;
3591 void virtio_queue_set_host_notifier_enabled(VirtQueue
*vq
, bool enabled
)
3593 vq
->host_notifier_enabled
= enabled
;
3596 int virtio_queue_set_host_notifier_mr(VirtIODevice
*vdev
, int n
,
3597 MemoryRegion
*mr
, bool assign
)
3599 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3600 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
3602 if (k
->set_host_notifier_mr
) {
3603 return k
->set_host_notifier_mr(qbus
->parent
, n
, mr
, assign
);
3609 void virtio_device_set_child_bus_name(VirtIODevice
*vdev
, char *bus_name
)
3611 g_free(vdev
->bus_name
);
3612 vdev
->bus_name
= g_strdup(bus_name
);
3615 void GCC_FMT_ATTR(2, 3) virtio_error(VirtIODevice
*vdev
, const char *fmt
, ...)
3620 error_vreport(fmt
, ap
);
3623 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
3624 vdev
->status
= vdev
->status
| VIRTIO_CONFIG_S_NEEDS_RESET
;
3625 virtio_notify_config(vdev
);
3628 vdev
->broken
= true;
3631 static void virtio_memory_listener_commit(MemoryListener
*listener
)
3633 VirtIODevice
*vdev
= container_of(listener
, VirtIODevice
, listener
);
3636 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
3637 if (vdev
->vq
[i
].vring
.num
== 0) {
3640 virtio_init_region_cache(vdev
, i
);
3644 static void virtio_device_realize(DeviceState
*dev
, Error
**errp
)
3646 VirtIODevice
*vdev
= VIRTIO_DEVICE(dev
);
3647 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(dev
);
3650 /* Devices should either use vmsd or the load/save methods */
3651 assert(!vdc
->vmsd
|| !vdc
->load
);
3653 if (vdc
->realize
!= NULL
) {
3654 vdc
->realize(dev
, &err
);
3656 error_propagate(errp
, err
);
3661 virtio_bus_device_plugged(vdev
, &err
);
3663 error_propagate(errp
, err
);
3664 vdc
->unrealize(dev
);
3668 vdev
->listener
.commit
= virtio_memory_listener_commit
;
3669 vdev
->listener
.name
= "virtio";
3670 memory_listener_register(&vdev
->listener
, vdev
->dma_as
);
3673 static void virtio_device_unrealize(DeviceState
*dev
)
3675 VirtIODevice
*vdev
= VIRTIO_DEVICE(dev
);
3676 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(dev
);
3678 memory_listener_unregister(&vdev
->listener
);
3679 virtio_bus_device_unplugged(vdev
);
3681 if (vdc
->unrealize
!= NULL
) {
3682 vdc
->unrealize(dev
);
3685 g_free(vdev
->bus_name
);
3686 vdev
->bus_name
= NULL
;
3689 static void virtio_device_free_virtqueues(VirtIODevice
*vdev
)
3696 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
3697 if (vdev
->vq
[i
].vring
.num
== 0) {
3700 virtio_virtqueue_reset_region_cache(&vdev
->vq
[i
]);
3705 static void virtio_device_instance_finalize(Object
*obj
)
3707 VirtIODevice
*vdev
= VIRTIO_DEVICE(obj
);
3709 virtio_device_free_virtqueues(vdev
);
3711 g_free(vdev
->config
);
3712 g_free(vdev
->vector_queues
);
3715 static Property virtio_properties
[] = {
3716 DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice
, host_features
),
3717 DEFINE_PROP_BOOL("use-started", VirtIODevice
, use_started
, true),
3718 DEFINE_PROP_BOOL("use-disabled-flag", VirtIODevice
, use_disabled_flag
, true),
3719 DEFINE_PROP_BOOL("x-disable-legacy-check", VirtIODevice
,
3720 disable_legacy_check
, false),
3721 DEFINE_PROP_END_OF_LIST(),
3724 static int virtio_device_start_ioeventfd_impl(VirtIODevice
*vdev
)
3726 VirtioBusState
*qbus
= VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev
)));
3730 * Batch all the host notifiers in a single transaction to avoid
3731 * quadratic time complexity in address_space_update_ioeventfds().
3733 memory_region_transaction_begin();
3734 for (n
= 0; n
< VIRTIO_QUEUE_MAX
; n
++) {
3735 VirtQueue
*vq
= &vdev
->vq
[n
];
3736 if (!virtio_queue_get_num(vdev
, n
)) {
3739 r
= virtio_bus_set_host_notifier(qbus
, n
, true);
3744 event_notifier_set_handler(&vq
->host_notifier
,
3745 virtio_queue_host_notifier_read
);
3748 for (n
= 0; n
< VIRTIO_QUEUE_MAX
; n
++) {
3749 /* Kick right away to begin processing requests already in vring */
3750 VirtQueue
*vq
= &vdev
->vq
[n
];
3751 if (!vq
->vring
.num
) {
3754 event_notifier_set(&vq
->host_notifier
);
3756 memory_region_transaction_commit();
3760 i
= n
; /* save n for a second iteration after transaction is committed. */
3762 VirtQueue
*vq
= &vdev
->vq
[n
];
3763 if (!virtio_queue_get_num(vdev
, n
)) {
3767 event_notifier_set_handler(&vq
->host_notifier
, NULL
);
3768 r
= virtio_bus_set_host_notifier(qbus
, n
, false);
3772 * The transaction expects the ioeventfds to be open when it
3773 * commits. Do it now, before the cleanup loop.
3775 memory_region_transaction_commit();
3778 if (!virtio_queue_get_num(vdev
, i
)) {
3781 virtio_bus_cleanup_host_notifier(qbus
, i
);
3786 int virtio_device_start_ioeventfd(VirtIODevice
*vdev
)
3788 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3789 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
3791 return virtio_bus_start_ioeventfd(vbus
);
3794 static void virtio_device_stop_ioeventfd_impl(VirtIODevice
*vdev
)
3796 VirtioBusState
*qbus
= VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev
)));
3800 * Batch all the host notifiers in a single transaction to avoid
3801 * quadratic time complexity in address_space_update_ioeventfds().
3803 memory_region_transaction_begin();
3804 for (n
= 0; n
< VIRTIO_QUEUE_MAX
; n
++) {
3805 VirtQueue
*vq
= &vdev
->vq
[n
];
3807 if (!virtio_queue_get_num(vdev
, n
)) {
3810 event_notifier_set_handler(&vq
->host_notifier
, NULL
);
3811 r
= virtio_bus_set_host_notifier(qbus
, n
, false);
3815 * The transaction expects the ioeventfds to be open when it
3816 * commits. Do it now, before the cleanup loop.
3818 memory_region_transaction_commit();
3820 for (n
= 0; n
< VIRTIO_QUEUE_MAX
; n
++) {
3821 if (!virtio_queue_get_num(vdev
, n
)) {
3824 virtio_bus_cleanup_host_notifier(qbus
, n
);
3828 int virtio_device_grab_ioeventfd(VirtIODevice
*vdev
)
3830 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3831 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
3833 return virtio_bus_grab_ioeventfd(vbus
);
3836 void virtio_device_release_ioeventfd(VirtIODevice
*vdev
)
3838 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3839 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
3841 virtio_bus_release_ioeventfd(vbus
);
3844 static void virtio_device_class_init(ObjectClass
*klass
, void *data
)
3846 /* Set the default value here. */
3847 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_CLASS(klass
);
3848 DeviceClass
*dc
= DEVICE_CLASS(klass
);
3850 dc
->realize
= virtio_device_realize
;
3851 dc
->unrealize
= virtio_device_unrealize
;
3852 dc
->bus_type
= TYPE_VIRTIO_BUS
;
3853 device_class_set_props(dc
, virtio_properties
);
3854 vdc
->start_ioeventfd
= virtio_device_start_ioeventfd_impl
;
3855 vdc
->stop_ioeventfd
= virtio_device_stop_ioeventfd_impl
;
3857 vdc
->legacy_features
|= VIRTIO_LEGACY_FEATURES
;
3860 bool virtio_device_ioeventfd_enabled(VirtIODevice
*vdev
)
3862 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3863 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
3865 return virtio_bus_ioeventfd_enabled(vbus
);
3868 static const TypeInfo virtio_device_info
= {
3869 .name
= TYPE_VIRTIO_DEVICE
,
3870 .parent
= TYPE_DEVICE
,
3871 .instance_size
= sizeof(VirtIODevice
),
3872 .class_init
= virtio_device_class_init
,
3873 .instance_finalize
= virtio_device_instance_finalize
,
3875 .class_size
= sizeof(VirtioDeviceClass
),
3878 static void virtio_register_types(void)
3880 type_register_static(&virtio_device_info
);
3883 type_init(virtio_register_types
)