4 * Copyright IBM, Corp. 2007
7 * Anthony Liguori <aliguori@us.ibm.com>
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
14 #include "qemu/osdep.h"
15 #include "qapi/error.h"
18 #include "exec/address-spaces.h"
19 #include "qemu/error-report.h"
20 #include "qemu/main-loop.h"
21 #include "qemu/module.h"
22 #include "hw/virtio/virtio.h"
23 #include "migration/qemu-file-types.h"
24 #include "qemu/atomic.h"
25 #include "hw/virtio/virtio-bus.h"
26 #include "hw/qdev-properties.h"
27 #include "hw/virtio/virtio-access.h"
28 #include "sysemu/dma.h"
29 #include "sysemu/runstate.h"
32 * The alignment to use between consumer and producer parts of vring.
33 * x86 pagesize again. This is the default, used by transports like PCI
34 * which don't provide a means for the guest to tell the host the alignment.
36 #define VIRTIO_PCI_VRING_ALIGN 4096
38 typedef struct VRingDesc
46 typedef struct VRingPackedDesc
{
53 typedef struct VRingAvail
60 typedef struct VRingUsedElem
66 typedef struct VRingUsed
70 VRingUsedElem ring
[0];
73 typedef struct VRingMemoryRegionCaches
{
75 MemoryRegionCache desc
;
76 MemoryRegionCache avail
;
77 MemoryRegionCache used
;
78 } VRingMemoryRegionCaches
;
83 unsigned int num_default
;
88 VRingMemoryRegionCaches
*caches
;
91 typedef struct VRingPackedDescEvent
{
94 } VRingPackedDescEvent
;
99 VirtQueueElement
*used_elems
;
101 /* Next head to pop */
102 uint16_t last_avail_idx
;
103 bool last_avail_wrap_counter
;
105 /* Last avail_idx read from VQ. */
106 uint16_t shadow_avail_idx
;
107 bool shadow_avail_wrap_counter
;
110 bool used_wrap_counter
;
112 /* Last used index value we have signalled on */
113 uint16_t signalled_used
;
115 /* Last used index value we have signalled on */
116 bool signalled_used_valid
;
118 /* Notification enabled? */
121 uint16_t queue_index
;
126 VirtIOHandleOutput handle_output
;
127 VirtIOHandleAIOOutput handle_aio_output
;
129 EventNotifier guest_notifier
;
130 EventNotifier host_notifier
;
131 bool host_notifier_enabled
;
132 QLIST_ENTRY(VirtQueue
) node
;
135 static void virtio_free_region_cache(VRingMemoryRegionCaches
*caches
)
141 address_space_cache_destroy(&caches
->desc
);
142 address_space_cache_destroy(&caches
->avail
);
143 address_space_cache_destroy(&caches
->used
);
147 static void virtio_virtqueue_reset_region_cache(struct VirtQueue
*vq
)
149 VRingMemoryRegionCaches
*caches
;
151 caches
= atomic_read(&vq
->vring
.caches
);
152 atomic_rcu_set(&vq
->vring
.caches
, NULL
);
154 call_rcu(caches
, virtio_free_region_cache
, rcu
);
158 static void virtio_init_region_cache(VirtIODevice
*vdev
, int n
)
160 VirtQueue
*vq
= &vdev
->vq
[n
];
161 VRingMemoryRegionCaches
*old
= vq
->vring
.caches
;
162 VRingMemoryRegionCaches
*new = NULL
;
168 addr
= vq
->vring
.desc
;
172 new = g_new0(VRingMemoryRegionCaches
, 1);
173 size
= virtio_queue_get_desc_size(vdev
, n
);
174 packed
= virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
) ?
176 len
= address_space_cache_init(&new->desc
, vdev
->dma_as
,
179 virtio_error(vdev
, "Cannot map desc");
183 size
= virtio_queue_get_used_size(vdev
, n
);
184 len
= address_space_cache_init(&new->used
, vdev
->dma_as
,
185 vq
->vring
.used
, size
, true);
187 virtio_error(vdev
, "Cannot map used");
191 size
= virtio_queue_get_avail_size(vdev
, n
);
192 len
= address_space_cache_init(&new->avail
, vdev
->dma_as
,
193 vq
->vring
.avail
, size
, false);
195 virtio_error(vdev
, "Cannot map avail");
199 atomic_rcu_set(&vq
->vring
.caches
, new);
201 call_rcu(old
, virtio_free_region_cache
, rcu
);
206 address_space_cache_destroy(&new->avail
);
208 address_space_cache_destroy(&new->used
);
210 address_space_cache_destroy(&new->desc
);
213 virtio_virtqueue_reset_region_cache(vq
);
216 /* virt queue functions */
217 void virtio_queue_update_rings(VirtIODevice
*vdev
, int n
)
219 VRing
*vring
= &vdev
->vq
[n
].vring
;
221 if (!vring
->num
|| !vring
->desc
|| !vring
->align
) {
222 /* not yet setup -> nothing to do */
225 vring
->avail
= vring
->desc
+ vring
->num
* sizeof(VRingDesc
);
226 vring
->used
= vring_align(vring
->avail
+
227 offsetof(VRingAvail
, ring
[vring
->num
]),
229 virtio_init_region_cache(vdev
, n
);
232 /* Called within rcu_read_lock(). */
233 static void vring_split_desc_read(VirtIODevice
*vdev
, VRingDesc
*desc
,
234 MemoryRegionCache
*cache
, int i
)
236 address_space_read_cached(cache
, i
* sizeof(VRingDesc
),
237 desc
, sizeof(VRingDesc
));
238 virtio_tswap64s(vdev
, &desc
->addr
);
239 virtio_tswap32s(vdev
, &desc
->len
);
240 virtio_tswap16s(vdev
, &desc
->flags
);
241 virtio_tswap16s(vdev
, &desc
->next
);
244 static void vring_packed_event_read(VirtIODevice
*vdev
,
245 MemoryRegionCache
*cache
,
246 VRingPackedDescEvent
*e
)
248 hwaddr off_off
= offsetof(VRingPackedDescEvent
, off_wrap
);
249 hwaddr off_flags
= offsetof(VRingPackedDescEvent
, flags
);
251 address_space_read_cached(cache
, off_flags
, &e
->flags
,
253 /* Make sure flags is seen before off_wrap */
255 address_space_read_cached(cache
, off_off
, &e
->off_wrap
,
256 sizeof(e
->off_wrap
));
257 virtio_tswap16s(vdev
, &e
->off_wrap
);
258 virtio_tswap16s(vdev
, &e
->flags
);
261 static void vring_packed_off_wrap_write(VirtIODevice
*vdev
,
262 MemoryRegionCache
*cache
,
265 hwaddr off
= offsetof(VRingPackedDescEvent
, off_wrap
);
267 virtio_tswap16s(vdev
, &off_wrap
);
268 address_space_write_cached(cache
, off
, &off_wrap
, sizeof(off_wrap
));
269 address_space_cache_invalidate(cache
, off
, sizeof(off_wrap
));
272 static void vring_packed_flags_write(VirtIODevice
*vdev
,
273 MemoryRegionCache
*cache
, uint16_t flags
)
275 hwaddr off
= offsetof(VRingPackedDescEvent
, flags
);
277 virtio_tswap16s(vdev
, &flags
);
278 address_space_write_cached(cache
, off
, &flags
, sizeof(flags
));
279 address_space_cache_invalidate(cache
, off
, sizeof(flags
));
282 /* Called within rcu_read_lock(). */
283 static VRingMemoryRegionCaches
*vring_get_region_caches(struct VirtQueue
*vq
)
285 VRingMemoryRegionCaches
*caches
= atomic_rcu_read(&vq
->vring
.caches
);
286 assert(caches
!= NULL
);
289 /* Called within rcu_read_lock(). */
290 static inline uint16_t vring_avail_flags(VirtQueue
*vq
)
292 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
293 hwaddr pa
= offsetof(VRingAvail
, flags
);
294 return virtio_lduw_phys_cached(vq
->vdev
, &caches
->avail
, pa
);
297 /* Called within rcu_read_lock(). */
298 static inline uint16_t vring_avail_idx(VirtQueue
*vq
)
300 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
301 hwaddr pa
= offsetof(VRingAvail
, idx
);
302 vq
->shadow_avail_idx
= virtio_lduw_phys_cached(vq
->vdev
, &caches
->avail
, pa
);
303 return vq
->shadow_avail_idx
;
306 /* Called within rcu_read_lock(). */
307 static inline uint16_t vring_avail_ring(VirtQueue
*vq
, int i
)
309 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
310 hwaddr pa
= offsetof(VRingAvail
, ring
[i
]);
311 return virtio_lduw_phys_cached(vq
->vdev
, &caches
->avail
, pa
);
314 /* Called within rcu_read_lock(). */
315 static inline uint16_t vring_get_used_event(VirtQueue
*vq
)
317 return vring_avail_ring(vq
, vq
->vring
.num
);
320 /* Called within rcu_read_lock(). */
321 static inline void vring_used_write(VirtQueue
*vq
, VRingUsedElem
*uelem
,
324 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
325 hwaddr pa
= offsetof(VRingUsed
, ring
[i
]);
326 virtio_tswap32s(vq
->vdev
, &uelem
->id
);
327 virtio_tswap32s(vq
->vdev
, &uelem
->len
);
328 address_space_write_cached(&caches
->used
, pa
, uelem
, sizeof(VRingUsedElem
));
329 address_space_cache_invalidate(&caches
->used
, pa
, sizeof(VRingUsedElem
));
332 /* Called within rcu_read_lock(). */
333 static uint16_t vring_used_idx(VirtQueue
*vq
)
335 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
336 hwaddr pa
= offsetof(VRingUsed
, idx
);
337 return virtio_lduw_phys_cached(vq
->vdev
, &caches
->used
, pa
);
340 /* Called within rcu_read_lock(). */
341 static inline void vring_used_idx_set(VirtQueue
*vq
, uint16_t val
)
343 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
344 hwaddr pa
= offsetof(VRingUsed
, idx
);
345 virtio_stw_phys_cached(vq
->vdev
, &caches
->used
, pa
, val
);
346 address_space_cache_invalidate(&caches
->used
, pa
, sizeof(val
));
350 /* Called within rcu_read_lock(). */
351 static inline void vring_used_flags_set_bit(VirtQueue
*vq
, int mask
)
353 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
354 VirtIODevice
*vdev
= vq
->vdev
;
355 hwaddr pa
= offsetof(VRingUsed
, flags
);
356 uint16_t flags
= virtio_lduw_phys_cached(vq
->vdev
, &caches
->used
, pa
);
358 virtio_stw_phys_cached(vdev
, &caches
->used
, pa
, flags
| mask
);
359 address_space_cache_invalidate(&caches
->used
, pa
, sizeof(flags
));
362 /* Called within rcu_read_lock(). */
363 static inline void vring_used_flags_unset_bit(VirtQueue
*vq
, int mask
)
365 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
366 VirtIODevice
*vdev
= vq
->vdev
;
367 hwaddr pa
= offsetof(VRingUsed
, flags
);
368 uint16_t flags
= virtio_lduw_phys_cached(vq
->vdev
, &caches
->used
, pa
);
370 virtio_stw_phys_cached(vdev
, &caches
->used
, pa
, flags
& ~mask
);
371 address_space_cache_invalidate(&caches
->used
, pa
, sizeof(flags
));
374 /* Called within rcu_read_lock(). */
375 static inline void vring_set_avail_event(VirtQueue
*vq
, uint16_t val
)
377 VRingMemoryRegionCaches
*caches
;
379 if (!vq
->notification
) {
383 caches
= vring_get_region_caches(vq
);
384 pa
= offsetof(VRingUsed
, ring
[vq
->vring
.num
]);
385 virtio_stw_phys_cached(vq
->vdev
, &caches
->used
, pa
, val
);
386 address_space_cache_invalidate(&caches
->used
, pa
, sizeof(val
));
389 static void virtio_queue_split_set_notification(VirtQueue
*vq
, int enable
)
391 RCU_READ_LOCK_GUARD();
393 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
394 vring_set_avail_event(vq
, vring_avail_idx(vq
));
396 vring_used_flags_unset_bit(vq
, VRING_USED_F_NO_NOTIFY
);
398 vring_used_flags_set_bit(vq
, VRING_USED_F_NO_NOTIFY
);
401 /* Expose avail event/used flags before caller checks the avail idx. */
406 static void virtio_queue_packed_set_notification(VirtQueue
*vq
, int enable
)
409 VRingPackedDescEvent e
;
410 VRingMemoryRegionCaches
*caches
;
412 RCU_READ_LOCK_GUARD();
413 caches
= vring_get_region_caches(vq
);
414 vring_packed_event_read(vq
->vdev
, &caches
->used
, &e
);
417 e
.flags
= VRING_PACKED_EVENT_FLAG_DISABLE
;
418 } else if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
419 off_wrap
= vq
->shadow_avail_idx
| vq
->shadow_avail_wrap_counter
<< 15;
420 vring_packed_off_wrap_write(vq
->vdev
, &caches
->used
, off_wrap
);
421 /* Make sure off_wrap is wrote before flags */
423 e
.flags
= VRING_PACKED_EVENT_FLAG_DESC
;
425 e
.flags
= VRING_PACKED_EVENT_FLAG_ENABLE
;
428 vring_packed_flags_write(vq
->vdev
, &caches
->used
, e
.flags
);
430 /* Expose avail event/used flags before caller checks the avail idx. */
435 bool virtio_queue_get_notification(VirtQueue
*vq
)
437 return vq
->notification
;
440 void virtio_queue_set_notification(VirtQueue
*vq
, int enable
)
442 vq
->notification
= enable
;
444 if (!vq
->vring
.desc
) {
448 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
449 virtio_queue_packed_set_notification(vq
, enable
);
451 virtio_queue_split_set_notification(vq
, enable
);
455 int virtio_queue_ready(VirtQueue
*vq
)
457 return vq
->vring
.avail
!= 0;
460 static void vring_packed_desc_read_flags(VirtIODevice
*vdev
,
462 MemoryRegionCache
*cache
,
465 address_space_read_cached(cache
,
466 i
* sizeof(VRingPackedDesc
) +
467 offsetof(VRingPackedDesc
, flags
),
468 flags
, sizeof(*flags
));
469 virtio_tswap16s(vdev
, flags
);
472 static void vring_packed_desc_read(VirtIODevice
*vdev
,
473 VRingPackedDesc
*desc
,
474 MemoryRegionCache
*cache
,
475 int i
, bool strict_order
)
477 hwaddr off
= i
* sizeof(VRingPackedDesc
);
479 vring_packed_desc_read_flags(vdev
, &desc
->flags
, cache
, i
);
482 /* Make sure flags is read before the rest fields. */
486 address_space_read_cached(cache
, off
+ offsetof(VRingPackedDesc
, addr
),
487 &desc
->addr
, sizeof(desc
->addr
));
488 address_space_read_cached(cache
, off
+ offsetof(VRingPackedDesc
, id
),
489 &desc
->id
, sizeof(desc
->id
));
490 address_space_read_cached(cache
, off
+ offsetof(VRingPackedDesc
, len
),
491 &desc
->len
, sizeof(desc
->len
));
492 virtio_tswap64s(vdev
, &desc
->addr
);
493 virtio_tswap16s(vdev
, &desc
->id
);
494 virtio_tswap32s(vdev
, &desc
->len
);
497 static void vring_packed_desc_write_data(VirtIODevice
*vdev
,
498 VRingPackedDesc
*desc
,
499 MemoryRegionCache
*cache
,
502 hwaddr off_id
= i
* sizeof(VRingPackedDesc
) +
503 offsetof(VRingPackedDesc
, id
);
504 hwaddr off_len
= i
* sizeof(VRingPackedDesc
) +
505 offsetof(VRingPackedDesc
, len
);
507 virtio_tswap32s(vdev
, &desc
->len
);
508 virtio_tswap16s(vdev
, &desc
->id
);
509 address_space_write_cached(cache
, off_id
, &desc
->id
, sizeof(desc
->id
));
510 address_space_cache_invalidate(cache
, off_id
, sizeof(desc
->id
));
511 address_space_write_cached(cache
, off_len
, &desc
->len
, sizeof(desc
->len
));
512 address_space_cache_invalidate(cache
, off_len
, sizeof(desc
->len
));
515 static void vring_packed_desc_write_flags(VirtIODevice
*vdev
,
516 VRingPackedDesc
*desc
,
517 MemoryRegionCache
*cache
,
520 hwaddr off
= i
* sizeof(VRingPackedDesc
) + offsetof(VRingPackedDesc
, flags
);
522 virtio_tswap16s(vdev
, &desc
->flags
);
523 address_space_write_cached(cache
, off
, &desc
->flags
, sizeof(desc
->flags
));
524 address_space_cache_invalidate(cache
, off
, sizeof(desc
->flags
));
527 static void vring_packed_desc_write(VirtIODevice
*vdev
,
528 VRingPackedDesc
*desc
,
529 MemoryRegionCache
*cache
,
530 int i
, bool strict_order
)
532 vring_packed_desc_write_data(vdev
, desc
, cache
, i
);
534 /* Make sure data is wrote before flags. */
537 vring_packed_desc_write_flags(vdev
, desc
, cache
, i
);
540 static inline bool is_desc_avail(uint16_t flags
, bool wrap_counter
)
544 avail
= !!(flags
& (1 << VRING_PACKED_DESC_F_AVAIL
));
545 used
= !!(flags
& (1 << VRING_PACKED_DESC_F_USED
));
546 return (avail
!= used
) && (avail
== wrap_counter
);
549 /* Fetch avail_idx from VQ memory only when we really need to know if
550 * guest has added some buffers.
551 * Called within rcu_read_lock(). */
552 static int virtio_queue_empty_rcu(VirtQueue
*vq
)
554 if (virtio_device_disabled(vq
->vdev
)) {
558 if (unlikely(!vq
->vring
.avail
)) {
562 if (vq
->shadow_avail_idx
!= vq
->last_avail_idx
) {
566 return vring_avail_idx(vq
) == vq
->last_avail_idx
;
569 static int virtio_queue_split_empty(VirtQueue
*vq
)
573 if (virtio_device_disabled(vq
->vdev
)) {
577 if (unlikely(!vq
->vring
.avail
)) {
581 if (vq
->shadow_avail_idx
!= vq
->last_avail_idx
) {
585 RCU_READ_LOCK_GUARD();
586 empty
= vring_avail_idx(vq
) == vq
->last_avail_idx
;
590 static int virtio_queue_packed_empty_rcu(VirtQueue
*vq
)
592 struct VRingPackedDesc desc
;
593 VRingMemoryRegionCaches
*cache
;
595 if (unlikely(!vq
->vring
.desc
)) {
599 cache
= vring_get_region_caches(vq
);
600 vring_packed_desc_read_flags(vq
->vdev
, &desc
.flags
, &cache
->desc
,
603 return !is_desc_avail(desc
.flags
, vq
->last_avail_wrap_counter
);
606 static int virtio_queue_packed_empty(VirtQueue
*vq
)
608 RCU_READ_LOCK_GUARD();
609 return virtio_queue_packed_empty_rcu(vq
);
612 int virtio_queue_empty(VirtQueue
*vq
)
614 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
615 return virtio_queue_packed_empty(vq
);
617 return virtio_queue_split_empty(vq
);
621 static void virtqueue_unmap_sg(VirtQueue
*vq
, const VirtQueueElement
*elem
,
624 AddressSpace
*dma_as
= vq
->vdev
->dma_as
;
629 for (i
= 0; i
< elem
->in_num
; i
++) {
630 size_t size
= MIN(len
- offset
, elem
->in_sg
[i
].iov_len
);
632 dma_memory_unmap(dma_as
, elem
->in_sg
[i
].iov_base
,
633 elem
->in_sg
[i
].iov_len
,
634 DMA_DIRECTION_FROM_DEVICE
, size
);
639 for (i
= 0; i
< elem
->out_num
; i
++)
640 dma_memory_unmap(dma_as
, elem
->out_sg
[i
].iov_base
,
641 elem
->out_sg
[i
].iov_len
,
642 DMA_DIRECTION_TO_DEVICE
,
643 elem
->out_sg
[i
].iov_len
);
646 /* virtqueue_detach_element:
647 * @vq: The #VirtQueue
648 * @elem: The #VirtQueueElement
649 * @len: number of bytes written
651 * Detach the element from the virtqueue. This function is suitable for device
652 * reset or other situations where a #VirtQueueElement is simply freed and will
653 * not be pushed or discarded.
655 void virtqueue_detach_element(VirtQueue
*vq
, const VirtQueueElement
*elem
,
658 vq
->inuse
-= elem
->ndescs
;
659 virtqueue_unmap_sg(vq
, elem
, len
);
662 static void virtqueue_split_rewind(VirtQueue
*vq
, unsigned int num
)
664 vq
->last_avail_idx
-= num
;
667 static void virtqueue_packed_rewind(VirtQueue
*vq
, unsigned int num
)
669 if (vq
->last_avail_idx
< num
) {
670 vq
->last_avail_idx
= vq
->vring
.num
+ vq
->last_avail_idx
- num
;
671 vq
->last_avail_wrap_counter
^= 1;
673 vq
->last_avail_idx
-= num
;
678 * @vq: The #VirtQueue
679 * @elem: The #VirtQueueElement
680 * @len: number of bytes written
682 * Pretend the most recent element wasn't popped from the virtqueue. The next
683 * call to virtqueue_pop() will refetch the element.
685 void virtqueue_unpop(VirtQueue
*vq
, const VirtQueueElement
*elem
,
689 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
690 virtqueue_packed_rewind(vq
, 1);
692 virtqueue_split_rewind(vq
, 1);
695 virtqueue_detach_element(vq
, elem
, len
);
699 * @vq: The #VirtQueue
700 * @num: Number of elements to push back
702 * Pretend that elements weren't popped from the virtqueue. The next
703 * virtqueue_pop() will refetch the oldest element.
705 * Use virtqueue_unpop() instead if you have a VirtQueueElement.
707 * Returns: true on success, false if @num is greater than the number of in use
710 bool virtqueue_rewind(VirtQueue
*vq
, unsigned int num
)
712 if (num
> vq
->inuse
) {
717 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
718 virtqueue_packed_rewind(vq
, num
);
720 virtqueue_split_rewind(vq
, num
);
725 static void virtqueue_split_fill(VirtQueue
*vq
, const VirtQueueElement
*elem
,
726 unsigned int len
, unsigned int idx
)
730 if (unlikely(!vq
->vring
.used
)) {
734 idx
= (idx
+ vq
->used_idx
) % vq
->vring
.num
;
736 uelem
.id
= elem
->index
;
738 vring_used_write(vq
, &uelem
, idx
);
741 static void virtqueue_packed_fill(VirtQueue
*vq
, const VirtQueueElement
*elem
,
742 unsigned int len
, unsigned int idx
)
744 vq
->used_elems
[idx
].index
= elem
->index
;
745 vq
->used_elems
[idx
].len
= len
;
746 vq
->used_elems
[idx
].ndescs
= elem
->ndescs
;
749 static void virtqueue_packed_fill_desc(VirtQueue
*vq
,
750 const VirtQueueElement
*elem
,
755 VRingMemoryRegionCaches
*caches
;
756 VRingPackedDesc desc
= {
760 bool wrap_counter
= vq
->used_wrap_counter
;
762 if (unlikely(!vq
->vring
.desc
)) {
766 head
= vq
->used_idx
+ idx
;
767 if (head
>= vq
->vring
.num
) {
768 head
-= vq
->vring
.num
;
772 desc
.flags
|= (1 << VRING_PACKED_DESC_F_AVAIL
);
773 desc
.flags
|= (1 << VRING_PACKED_DESC_F_USED
);
775 desc
.flags
&= ~(1 << VRING_PACKED_DESC_F_AVAIL
);
776 desc
.flags
&= ~(1 << VRING_PACKED_DESC_F_USED
);
779 caches
= vring_get_region_caches(vq
);
780 vring_packed_desc_write(vq
->vdev
, &desc
, &caches
->desc
, head
, strict_order
);
783 /* Called within rcu_read_lock(). */
784 void virtqueue_fill(VirtQueue
*vq
, const VirtQueueElement
*elem
,
785 unsigned int len
, unsigned int idx
)
787 trace_virtqueue_fill(vq
, elem
, len
, idx
);
789 virtqueue_unmap_sg(vq
, elem
, len
);
791 if (virtio_device_disabled(vq
->vdev
)) {
795 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
796 virtqueue_packed_fill(vq
, elem
, len
, idx
);
798 virtqueue_split_fill(vq
, elem
, len
, idx
);
802 /* Called within rcu_read_lock(). */
803 static void virtqueue_split_flush(VirtQueue
*vq
, unsigned int count
)
807 if (unlikely(!vq
->vring
.used
)) {
811 /* Make sure buffer is written before we update index. */
813 trace_virtqueue_flush(vq
, count
);
816 vring_used_idx_set(vq
, new);
818 if (unlikely((int16_t)(new - vq
->signalled_used
) < (uint16_t)(new - old
)))
819 vq
->signalled_used_valid
= false;
822 static void virtqueue_packed_flush(VirtQueue
*vq
, unsigned int count
)
824 unsigned int i
, ndescs
= 0;
826 if (unlikely(!vq
->vring
.desc
)) {
830 for (i
= 1; i
< count
; i
++) {
831 virtqueue_packed_fill_desc(vq
, &vq
->used_elems
[i
], i
, false);
832 ndescs
+= vq
->used_elems
[i
].ndescs
;
834 virtqueue_packed_fill_desc(vq
, &vq
->used_elems
[0], 0, true);
835 ndescs
+= vq
->used_elems
[0].ndescs
;
838 vq
->used_idx
+= ndescs
;
839 if (vq
->used_idx
>= vq
->vring
.num
) {
840 vq
->used_idx
-= vq
->vring
.num
;
841 vq
->used_wrap_counter
^= 1;
845 void virtqueue_flush(VirtQueue
*vq
, unsigned int count
)
847 if (virtio_device_disabled(vq
->vdev
)) {
852 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
853 virtqueue_packed_flush(vq
, count
);
855 virtqueue_split_flush(vq
, count
);
859 void virtqueue_push(VirtQueue
*vq
, const VirtQueueElement
*elem
,
862 RCU_READ_LOCK_GUARD();
863 virtqueue_fill(vq
, elem
, len
, 0);
864 virtqueue_flush(vq
, 1);
867 /* Called within rcu_read_lock(). */
868 static int virtqueue_num_heads(VirtQueue
*vq
, unsigned int idx
)
870 uint16_t num_heads
= vring_avail_idx(vq
) - idx
;
872 /* Check it isn't doing very strange things with descriptor numbers. */
873 if (num_heads
> vq
->vring
.num
) {
874 virtio_error(vq
->vdev
, "Guest moved used index from %u to %u",
875 idx
, vq
->shadow_avail_idx
);
878 /* On success, callers read a descriptor at vq->last_avail_idx.
879 * Make sure descriptor read does not bypass avail index read. */
887 /* Called within rcu_read_lock(). */
888 static bool virtqueue_get_head(VirtQueue
*vq
, unsigned int idx
,
891 /* Grab the next descriptor number they're advertising, and increment
892 * the index we've seen. */
893 *head
= vring_avail_ring(vq
, idx
% vq
->vring
.num
);
895 /* If their number is silly, that's a fatal mistake. */
896 if (*head
>= vq
->vring
.num
) {
897 virtio_error(vq
->vdev
, "Guest says index %u is available", *head
);
905 VIRTQUEUE_READ_DESC_ERROR
= -1,
906 VIRTQUEUE_READ_DESC_DONE
= 0, /* end of chain */
907 VIRTQUEUE_READ_DESC_MORE
= 1, /* more buffers in chain */
910 static int virtqueue_split_read_next_desc(VirtIODevice
*vdev
, VRingDesc
*desc
,
911 MemoryRegionCache
*desc_cache
,
912 unsigned int max
, unsigned int *next
)
914 /* If this descriptor says it doesn't chain, we're done. */
915 if (!(desc
->flags
& VRING_DESC_F_NEXT
)) {
916 return VIRTQUEUE_READ_DESC_DONE
;
919 /* Check they're not leading us off end of descriptors. */
921 /* Make sure compiler knows to grab that: we don't want it changing! */
925 virtio_error(vdev
, "Desc next is %u", *next
);
926 return VIRTQUEUE_READ_DESC_ERROR
;
929 vring_split_desc_read(vdev
, desc
, desc_cache
, *next
);
930 return VIRTQUEUE_READ_DESC_MORE
;
933 static void virtqueue_split_get_avail_bytes(VirtQueue
*vq
,
934 unsigned int *in_bytes
, unsigned int *out_bytes
,
935 unsigned max_in_bytes
, unsigned max_out_bytes
)
937 VirtIODevice
*vdev
= vq
->vdev
;
938 unsigned int max
, idx
;
939 unsigned int total_bufs
, in_total
, out_total
;
940 VRingMemoryRegionCaches
*caches
;
941 MemoryRegionCache indirect_desc_cache
= MEMORY_REGION_CACHE_INVALID
;
945 RCU_READ_LOCK_GUARD();
947 idx
= vq
->last_avail_idx
;
948 total_bufs
= in_total
= out_total
= 0;
951 caches
= vring_get_region_caches(vq
);
952 while ((rc
= virtqueue_num_heads(vq
, idx
)) > 0) {
953 MemoryRegionCache
*desc_cache
= &caches
->desc
;
954 unsigned int num_bufs
;
958 num_bufs
= total_bufs
;
960 if (!virtqueue_get_head(vq
, idx
++, &i
)) {
964 vring_split_desc_read(vdev
, &desc
, desc_cache
, i
);
966 if (desc
.flags
& VRING_DESC_F_INDIRECT
) {
967 if (!desc
.len
|| (desc
.len
% sizeof(VRingDesc
))) {
968 virtio_error(vdev
, "Invalid size for indirect buffer table");
972 /* If we've got too many, that implies a descriptor loop. */
973 if (num_bufs
>= max
) {
974 virtio_error(vdev
, "Looped descriptor");
978 /* loop over the indirect descriptor table */
979 len
= address_space_cache_init(&indirect_desc_cache
,
981 desc
.addr
, desc
.len
, false);
982 desc_cache
= &indirect_desc_cache
;
983 if (len
< desc
.len
) {
984 virtio_error(vdev
, "Cannot map indirect buffer");
988 max
= desc
.len
/ sizeof(VRingDesc
);
990 vring_split_desc_read(vdev
, &desc
, desc_cache
, i
);
994 /* If we've got too many, that implies a descriptor loop. */
995 if (++num_bufs
> max
) {
996 virtio_error(vdev
, "Looped descriptor");
1000 if (desc
.flags
& VRING_DESC_F_WRITE
) {
1001 in_total
+= desc
.len
;
1003 out_total
+= desc
.len
;
1005 if (in_total
>= max_in_bytes
&& out_total
>= max_out_bytes
) {
1009 rc
= virtqueue_split_read_next_desc(vdev
, &desc
, desc_cache
, max
, &i
);
1010 } while (rc
== VIRTQUEUE_READ_DESC_MORE
);
1012 if (rc
== VIRTQUEUE_READ_DESC_ERROR
) {
1016 if (desc_cache
== &indirect_desc_cache
) {
1017 address_space_cache_destroy(&indirect_desc_cache
);
1020 total_bufs
= num_bufs
;
1029 address_space_cache_destroy(&indirect_desc_cache
);
1031 *in_bytes
= in_total
;
1034 *out_bytes
= out_total
;
1039 in_total
= out_total
= 0;
1043 static int virtqueue_packed_read_next_desc(VirtQueue
*vq
,
1044 VRingPackedDesc
*desc
,
1051 /* If this descriptor says it doesn't chain, we're done. */
1052 if (!indirect
&& !(desc
->flags
& VRING_DESC_F_NEXT
)) {
1053 return VIRTQUEUE_READ_DESC_DONE
;
1059 return VIRTQUEUE_READ_DESC_DONE
;
1061 (*next
) -= vq
->vring
.num
;
1065 vring_packed_desc_read(vq
->vdev
, desc
, desc_cache
, *next
, false);
1066 return VIRTQUEUE_READ_DESC_MORE
;
1069 static void virtqueue_packed_get_avail_bytes(VirtQueue
*vq
,
1070 unsigned int *in_bytes
,
1071 unsigned int *out_bytes
,
1072 unsigned max_in_bytes
,
1073 unsigned max_out_bytes
)
1075 VirtIODevice
*vdev
= vq
->vdev
;
1076 unsigned int max
, idx
;
1077 unsigned int total_bufs
, in_total
, out_total
;
1078 MemoryRegionCache
*desc_cache
;
1079 VRingMemoryRegionCaches
*caches
;
1080 MemoryRegionCache indirect_desc_cache
= MEMORY_REGION_CACHE_INVALID
;
1082 VRingPackedDesc desc
;
1085 RCU_READ_LOCK_GUARD();
1086 idx
= vq
->last_avail_idx
;
1087 wrap_counter
= vq
->last_avail_wrap_counter
;
1088 total_bufs
= in_total
= out_total
= 0;
1090 max
= vq
->vring
.num
;
1091 caches
= vring_get_region_caches(vq
);
1094 unsigned int num_bufs
= total_bufs
;
1095 unsigned int i
= idx
;
1098 desc_cache
= &caches
->desc
;
1099 vring_packed_desc_read(vdev
, &desc
, desc_cache
, idx
, true);
1100 if (!is_desc_avail(desc
.flags
, wrap_counter
)) {
1104 if (desc
.flags
& VRING_DESC_F_INDIRECT
) {
1105 if (desc
.len
% sizeof(VRingPackedDesc
)) {
1106 virtio_error(vdev
, "Invalid size for indirect buffer table");
1110 /* If we've got too many, that implies a descriptor loop. */
1111 if (num_bufs
>= max
) {
1112 virtio_error(vdev
, "Looped descriptor");
1116 /* loop over the indirect descriptor table */
1117 len
= address_space_cache_init(&indirect_desc_cache
,
1119 desc
.addr
, desc
.len
, false);
1120 desc_cache
= &indirect_desc_cache
;
1121 if (len
< desc
.len
) {
1122 virtio_error(vdev
, "Cannot map indirect buffer");
1126 max
= desc
.len
/ sizeof(VRingPackedDesc
);
1128 vring_packed_desc_read(vdev
, &desc
, desc_cache
, i
, false);
1132 /* If we've got too many, that implies a descriptor loop. */
1133 if (++num_bufs
> max
) {
1134 virtio_error(vdev
, "Looped descriptor");
1138 if (desc
.flags
& VRING_DESC_F_WRITE
) {
1139 in_total
+= desc
.len
;
1141 out_total
+= desc
.len
;
1143 if (in_total
>= max_in_bytes
&& out_total
>= max_out_bytes
) {
1147 rc
= virtqueue_packed_read_next_desc(vq
, &desc
, desc_cache
, max
,
1149 &indirect_desc_cache
);
1150 } while (rc
== VIRTQUEUE_READ_DESC_MORE
);
1152 if (desc_cache
== &indirect_desc_cache
) {
1153 address_space_cache_destroy(&indirect_desc_cache
);
1157 idx
+= num_bufs
- total_bufs
;
1158 total_bufs
= num_bufs
;
1161 if (idx
>= vq
->vring
.num
) {
1162 idx
-= vq
->vring
.num
;
1167 /* Record the index and wrap counter for a kick we want */
1168 vq
->shadow_avail_idx
= idx
;
1169 vq
->shadow_avail_wrap_counter
= wrap_counter
;
1171 address_space_cache_destroy(&indirect_desc_cache
);
1173 *in_bytes
= in_total
;
1176 *out_bytes
= out_total
;
1181 in_total
= out_total
= 0;
1185 void virtqueue_get_avail_bytes(VirtQueue
*vq
, unsigned int *in_bytes
,
1186 unsigned int *out_bytes
,
1187 unsigned max_in_bytes
, unsigned max_out_bytes
)
1190 VRingMemoryRegionCaches
*caches
;
1192 if (unlikely(!vq
->vring
.desc
)) {
1196 caches
= vring_get_region_caches(vq
);
1197 desc_size
= virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
) ?
1198 sizeof(VRingPackedDesc
) : sizeof(VRingDesc
);
1199 if (caches
->desc
.len
< vq
->vring
.num
* desc_size
) {
1200 virtio_error(vq
->vdev
, "Cannot map descriptor ring");
1204 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
1205 virtqueue_packed_get_avail_bytes(vq
, in_bytes
, out_bytes
,
1206 max_in_bytes
, max_out_bytes
);
1208 virtqueue_split_get_avail_bytes(vq
, in_bytes
, out_bytes
,
1209 max_in_bytes
, max_out_bytes
);
1222 int virtqueue_avail_bytes(VirtQueue
*vq
, unsigned int in_bytes
,
1223 unsigned int out_bytes
)
1225 unsigned int in_total
, out_total
;
1227 virtqueue_get_avail_bytes(vq
, &in_total
, &out_total
, in_bytes
, out_bytes
);
1228 return in_bytes
<= in_total
&& out_bytes
<= out_total
;
1231 static bool virtqueue_map_desc(VirtIODevice
*vdev
, unsigned int *p_num_sg
,
1232 hwaddr
*addr
, struct iovec
*iov
,
1233 unsigned int max_num_sg
, bool is_write
,
1234 hwaddr pa
, size_t sz
)
1237 unsigned num_sg
= *p_num_sg
;
1238 assert(num_sg
<= max_num_sg
);
1241 virtio_error(vdev
, "virtio: zero sized buffers are not allowed");
1248 if (num_sg
== max_num_sg
) {
1249 virtio_error(vdev
, "virtio: too many write descriptors in "
1254 iov
[num_sg
].iov_base
= dma_memory_map(vdev
->dma_as
, pa
, &len
,
1256 DMA_DIRECTION_FROM_DEVICE
:
1257 DMA_DIRECTION_TO_DEVICE
);
1258 if (!iov
[num_sg
].iov_base
) {
1259 virtio_error(vdev
, "virtio: bogus descriptor or out of resources");
1263 iov
[num_sg
].iov_len
= len
;
1277 /* Only used by error code paths before we have a VirtQueueElement (therefore
1278 * virtqueue_unmap_sg() can't be used). Assumes buffers weren't written to
1281 static void virtqueue_undo_map_desc(unsigned int out_num
, unsigned int in_num
,
1286 for (i
= 0; i
< out_num
+ in_num
; i
++) {
1287 int is_write
= i
>= out_num
;
1289 cpu_physical_memory_unmap(iov
->iov_base
, iov
->iov_len
, is_write
, 0);
1294 static void virtqueue_map_iovec(VirtIODevice
*vdev
, struct iovec
*sg
,
1295 hwaddr
*addr
, unsigned int num_sg
,
1301 for (i
= 0; i
< num_sg
; i
++) {
1302 len
= sg
[i
].iov_len
;
1303 sg
[i
].iov_base
= dma_memory_map(vdev
->dma_as
,
1304 addr
[i
], &len
, is_write
?
1305 DMA_DIRECTION_FROM_DEVICE
:
1306 DMA_DIRECTION_TO_DEVICE
);
1307 if (!sg
[i
].iov_base
) {
1308 error_report("virtio: error trying to map MMIO memory");
1311 if (len
!= sg
[i
].iov_len
) {
1312 error_report("virtio: unexpected memory split");
1318 void virtqueue_map(VirtIODevice
*vdev
, VirtQueueElement
*elem
)
1320 virtqueue_map_iovec(vdev
, elem
->in_sg
, elem
->in_addr
, elem
->in_num
, 1);
1321 virtqueue_map_iovec(vdev
, elem
->out_sg
, elem
->out_addr
, elem
->out_num
, 0);
1324 static void *virtqueue_alloc_element(size_t sz
, unsigned out_num
, unsigned in_num
)
1326 VirtQueueElement
*elem
;
1327 size_t in_addr_ofs
= QEMU_ALIGN_UP(sz
, __alignof__(elem
->in_addr
[0]));
1328 size_t out_addr_ofs
= in_addr_ofs
+ in_num
* sizeof(elem
->in_addr
[0]);
1329 size_t out_addr_end
= out_addr_ofs
+ out_num
* sizeof(elem
->out_addr
[0]);
1330 size_t in_sg_ofs
= QEMU_ALIGN_UP(out_addr_end
, __alignof__(elem
->in_sg
[0]));
1331 size_t out_sg_ofs
= in_sg_ofs
+ in_num
* sizeof(elem
->in_sg
[0]);
1332 size_t out_sg_end
= out_sg_ofs
+ out_num
* sizeof(elem
->out_sg
[0]);
1334 assert(sz
>= sizeof(VirtQueueElement
));
1335 elem
= g_malloc(out_sg_end
);
1336 trace_virtqueue_alloc_element(elem
, sz
, in_num
, out_num
);
1337 elem
->out_num
= out_num
;
1338 elem
->in_num
= in_num
;
1339 elem
->in_addr
= (void *)elem
+ in_addr_ofs
;
1340 elem
->out_addr
= (void *)elem
+ out_addr_ofs
;
1341 elem
->in_sg
= (void *)elem
+ in_sg_ofs
;
1342 elem
->out_sg
= (void *)elem
+ out_sg_ofs
;
1346 static void *virtqueue_split_pop(VirtQueue
*vq
, size_t sz
)
1348 unsigned int i
, head
, max
;
1349 VRingMemoryRegionCaches
*caches
;
1350 MemoryRegionCache indirect_desc_cache
= MEMORY_REGION_CACHE_INVALID
;
1351 MemoryRegionCache
*desc_cache
;
1353 VirtIODevice
*vdev
= vq
->vdev
;
1354 VirtQueueElement
*elem
= NULL
;
1355 unsigned out_num
, in_num
, elem_entries
;
1356 hwaddr addr
[VIRTQUEUE_MAX_SIZE
];
1357 struct iovec iov
[VIRTQUEUE_MAX_SIZE
];
1361 RCU_READ_LOCK_GUARD();
1362 if (virtio_queue_empty_rcu(vq
)) {
1365 /* Needed after virtio_queue_empty(), see comment in
1366 * virtqueue_num_heads(). */
1369 /* When we start there are none of either input nor output. */
1370 out_num
= in_num
= elem_entries
= 0;
1372 max
= vq
->vring
.num
;
1374 if (vq
->inuse
>= vq
->vring
.num
) {
1375 virtio_error(vdev
, "Virtqueue size exceeded");
1379 if (!virtqueue_get_head(vq
, vq
->last_avail_idx
++, &head
)) {
1383 if (virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
1384 vring_set_avail_event(vq
, vq
->last_avail_idx
);
1389 caches
= vring_get_region_caches(vq
);
1390 if (caches
->desc
.len
< max
* sizeof(VRingDesc
)) {
1391 virtio_error(vdev
, "Cannot map descriptor ring");
1395 desc_cache
= &caches
->desc
;
1396 vring_split_desc_read(vdev
, &desc
, desc_cache
, i
);
1397 if (desc
.flags
& VRING_DESC_F_INDIRECT
) {
1398 if (!desc
.len
|| (desc
.len
% sizeof(VRingDesc
))) {
1399 virtio_error(vdev
, "Invalid size for indirect buffer table");
1403 /* loop over the indirect descriptor table */
1404 len
= address_space_cache_init(&indirect_desc_cache
, vdev
->dma_as
,
1405 desc
.addr
, desc
.len
, false);
1406 desc_cache
= &indirect_desc_cache
;
1407 if (len
< desc
.len
) {
1408 virtio_error(vdev
, "Cannot map indirect buffer");
1412 max
= desc
.len
/ sizeof(VRingDesc
);
1414 vring_split_desc_read(vdev
, &desc
, desc_cache
, i
);
1417 /* Collect all the descriptors */
1421 if (desc
.flags
& VRING_DESC_F_WRITE
) {
1422 map_ok
= virtqueue_map_desc(vdev
, &in_num
, addr
+ out_num
,
1424 VIRTQUEUE_MAX_SIZE
- out_num
, true,
1425 desc
.addr
, desc
.len
);
1428 virtio_error(vdev
, "Incorrect order for descriptors");
1431 map_ok
= virtqueue_map_desc(vdev
, &out_num
, addr
, iov
,
1432 VIRTQUEUE_MAX_SIZE
, false,
1433 desc
.addr
, desc
.len
);
1439 /* If we've got too many, that implies a descriptor loop. */
1440 if (++elem_entries
> max
) {
1441 virtio_error(vdev
, "Looped descriptor");
1445 rc
= virtqueue_split_read_next_desc(vdev
, &desc
, desc_cache
, max
, &i
);
1446 } while (rc
== VIRTQUEUE_READ_DESC_MORE
);
1448 if (rc
== VIRTQUEUE_READ_DESC_ERROR
) {
1452 /* Now copy what we have collected and mapped */
1453 elem
= virtqueue_alloc_element(sz
, out_num
, in_num
);
1456 for (i
= 0; i
< out_num
; i
++) {
1457 elem
->out_addr
[i
] = addr
[i
];
1458 elem
->out_sg
[i
] = iov
[i
];
1460 for (i
= 0; i
< in_num
; i
++) {
1461 elem
->in_addr
[i
] = addr
[out_num
+ i
];
1462 elem
->in_sg
[i
] = iov
[out_num
+ i
];
1467 trace_virtqueue_pop(vq
, elem
, elem
->in_num
, elem
->out_num
);
1469 address_space_cache_destroy(&indirect_desc_cache
);
1474 virtqueue_undo_map_desc(out_num
, in_num
, iov
);
1478 static void *virtqueue_packed_pop(VirtQueue
*vq
, size_t sz
)
1480 unsigned int i
, max
;
1481 VRingMemoryRegionCaches
*caches
;
1482 MemoryRegionCache indirect_desc_cache
= MEMORY_REGION_CACHE_INVALID
;
1483 MemoryRegionCache
*desc_cache
;
1485 VirtIODevice
*vdev
= vq
->vdev
;
1486 VirtQueueElement
*elem
= NULL
;
1487 unsigned out_num
, in_num
, elem_entries
;
1488 hwaddr addr
[VIRTQUEUE_MAX_SIZE
];
1489 struct iovec iov
[VIRTQUEUE_MAX_SIZE
];
1490 VRingPackedDesc desc
;
1494 RCU_READ_LOCK_GUARD();
1495 if (virtio_queue_packed_empty_rcu(vq
)) {
1499 /* When we start there are none of either input nor output. */
1500 out_num
= in_num
= elem_entries
= 0;
1502 max
= vq
->vring
.num
;
1504 if (vq
->inuse
>= vq
->vring
.num
) {
1505 virtio_error(vdev
, "Virtqueue size exceeded");
1509 i
= vq
->last_avail_idx
;
1511 caches
= vring_get_region_caches(vq
);
1512 if (caches
->desc
.len
< max
* sizeof(VRingDesc
)) {
1513 virtio_error(vdev
, "Cannot map descriptor ring");
1517 desc_cache
= &caches
->desc
;
1518 vring_packed_desc_read(vdev
, &desc
, desc_cache
, i
, true);
1520 if (desc
.flags
& VRING_DESC_F_INDIRECT
) {
1521 if (desc
.len
% sizeof(VRingPackedDesc
)) {
1522 virtio_error(vdev
, "Invalid size for indirect buffer table");
1526 /* loop over the indirect descriptor table */
1527 len
= address_space_cache_init(&indirect_desc_cache
, vdev
->dma_as
,
1528 desc
.addr
, desc
.len
, false);
1529 desc_cache
= &indirect_desc_cache
;
1530 if (len
< desc
.len
) {
1531 virtio_error(vdev
, "Cannot map indirect buffer");
1535 max
= desc
.len
/ sizeof(VRingPackedDesc
);
1537 vring_packed_desc_read(vdev
, &desc
, desc_cache
, i
, false);
1540 /* Collect all the descriptors */
1544 if (desc
.flags
& VRING_DESC_F_WRITE
) {
1545 map_ok
= virtqueue_map_desc(vdev
, &in_num
, addr
+ out_num
,
1547 VIRTQUEUE_MAX_SIZE
- out_num
, true,
1548 desc
.addr
, desc
.len
);
1551 virtio_error(vdev
, "Incorrect order for descriptors");
1554 map_ok
= virtqueue_map_desc(vdev
, &out_num
, addr
, iov
,
1555 VIRTQUEUE_MAX_SIZE
, false,
1556 desc
.addr
, desc
.len
);
1562 /* If we've got too many, that implies a descriptor loop. */
1563 if (++elem_entries
> max
) {
1564 virtio_error(vdev
, "Looped descriptor");
1568 rc
= virtqueue_packed_read_next_desc(vq
, &desc
, desc_cache
, max
, &i
,
1570 &indirect_desc_cache
);
1571 } while (rc
== VIRTQUEUE_READ_DESC_MORE
);
1573 /* Now copy what we have collected and mapped */
1574 elem
= virtqueue_alloc_element(sz
, out_num
, in_num
);
1575 for (i
= 0; i
< out_num
; i
++) {
1576 elem
->out_addr
[i
] = addr
[i
];
1577 elem
->out_sg
[i
] = iov
[i
];
1579 for (i
= 0; i
< in_num
; i
++) {
1580 elem
->in_addr
[i
] = addr
[out_num
+ i
];
1581 elem
->in_sg
[i
] = iov
[out_num
+ i
];
1585 elem
->ndescs
= (desc_cache
== &indirect_desc_cache
) ? 1 : elem_entries
;
1586 vq
->last_avail_idx
+= elem
->ndescs
;
1587 vq
->inuse
+= elem
->ndescs
;
1589 if (vq
->last_avail_idx
>= vq
->vring
.num
) {
1590 vq
->last_avail_idx
-= vq
->vring
.num
;
1591 vq
->last_avail_wrap_counter
^= 1;
1594 vq
->shadow_avail_idx
= vq
->last_avail_idx
;
1595 vq
->shadow_avail_wrap_counter
= vq
->last_avail_wrap_counter
;
1597 trace_virtqueue_pop(vq
, elem
, elem
->in_num
, elem
->out_num
);
1599 address_space_cache_destroy(&indirect_desc_cache
);
1604 virtqueue_undo_map_desc(out_num
, in_num
, iov
);
1608 void *virtqueue_pop(VirtQueue
*vq
, size_t sz
)
1610 if (virtio_device_disabled(vq
->vdev
)) {
1614 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
1615 return virtqueue_packed_pop(vq
, sz
);
1617 return virtqueue_split_pop(vq
, sz
);
1621 static unsigned int virtqueue_packed_drop_all(VirtQueue
*vq
)
1623 VRingMemoryRegionCaches
*caches
;
1624 MemoryRegionCache
*desc_cache
;
1625 unsigned int dropped
= 0;
1626 VirtQueueElement elem
= {};
1627 VirtIODevice
*vdev
= vq
->vdev
;
1628 VRingPackedDesc desc
;
1630 caches
= vring_get_region_caches(vq
);
1631 desc_cache
= &caches
->desc
;
1633 virtio_queue_set_notification(vq
, 0);
1635 while (vq
->inuse
< vq
->vring
.num
) {
1636 unsigned int idx
= vq
->last_avail_idx
;
1638 * works similar to virtqueue_pop but does not map buffers
1639 * and does not allocate any memory.
1641 vring_packed_desc_read(vdev
, &desc
, desc_cache
,
1642 vq
->last_avail_idx
, true);
1643 if (!is_desc_avail(desc
.flags
, vq
->last_avail_wrap_counter
)) {
1646 elem
.index
= desc
.id
;
1648 while (virtqueue_packed_read_next_desc(vq
, &desc
, desc_cache
,
1649 vq
->vring
.num
, &idx
, false)) {
1653 * immediately push the element, nothing to unmap
1654 * as both in_num and out_num are set to 0.
1656 virtqueue_push(vq
, &elem
, 0);
1658 vq
->last_avail_idx
+= elem
.ndescs
;
1659 if (vq
->last_avail_idx
>= vq
->vring
.num
) {
1660 vq
->last_avail_idx
-= vq
->vring
.num
;
1661 vq
->last_avail_wrap_counter
^= 1;
1668 static unsigned int virtqueue_split_drop_all(VirtQueue
*vq
)
1670 unsigned int dropped
= 0;
1671 VirtQueueElement elem
= {};
1672 VirtIODevice
*vdev
= vq
->vdev
;
1673 bool fEventIdx
= virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
);
1675 while (!virtio_queue_empty(vq
) && vq
->inuse
< vq
->vring
.num
) {
1676 /* works similar to virtqueue_pop but does not map buffers
1677 * and does not allocate any memory */
1679 if (!virtqueue_get_head(vq
, vq
->last_avail_idx
, &elem
.index
)) {
1683 vq
->last_avail_idx
++;
1685 vring_set_avail_event(vq
, vq
->last_avail_idx
);
1687 /* immediately push the element, nothing to unmap
1688 * as both in_num and out_num are set to 0 */
1689 virtqueue_push(vq
, &elem
, 0);
1696 /* virtqueue_drop_all:
1697 * @vq: The #VirtQueue
1698 * Drops all queued buffers and indicates them to the guest
1699 * as if they are done. Useful when buffers can not be
1700 * processed but must be returned to the guest.
1702 unsigned int virtqueue_drop_all(VirtQueue
*vq
)
1704 struct VirtIODevice
*vdev
= vq
->vdev
;
1706 if (virtio_device_disabled(vq
->vdev
)) {
1710 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
1711 return virtqueue_packed_drop_all(vq
);
1713 return virtqueue_split_drop_all(vq
);
1717 /* Reading and writing a structure directly to QEMUFile is *awful*, but
1718 * it is what QEMU has always done by mistake. We can change it sooner
1719 * or later by bumping the version number of the affected vm states.
1720 * In the meanwhile, since the in-memory layout of VirtQueueElement
1721 * has changed, we need to marshal to and from the layout that was
1722 * used before the change.
1724 typedef struct VirtQueueElementOld
{
1726 unsigned int out_num
;
1727 unsigned int in_num
;
1728 hwaddr in_addr
[VIRTQUEUE_MAX_SIZE
];
1729 hwaddr out_addr
[VIRTQUEUE_MAX_SIZE
];
1730 struct iovec in_sg
[VIRTQUEUE_MAX_SIZE
];
1731 struct iovec out_sg
[VIRTQUEUE_MAX_SIZE
];
1732 } VirtQueueElementOld
;
1734 void *qemu_get_virtqueue_element(VirtIODevice
*vdev
, QEMUFile
*f
, size_t sz
)
1736 VirtQueueElement
*elem
;
1737 VirtQueueElementOld data
;
1740 qemu_get_buffer(f
, (uint8_t *)&data
, sizeof(VirtQueueElementOld
));
1742 /* TODO: teach all callers that this can fail, and return failure instead
1743 * of asserting here.
1744 * This is just one thing (there are probably more) that must be
1745 * fixed before we can allow NDEBUG compilation.
1747 assert(ARRAY_SIZE(data
.in_addr
) >= data
.in_num
);
1748 assert(ARRAY_SIZE(data
.out_addr
) >= data
.out_num
);
1750 elem
= virtqueue_alloc_element(sz
, data
.out_num
, data
.in_num
);
1751 elem
->index
= data
.index
;
1753 for (i
= 0; i
< elem
->in_num
; i
++) {
1754 elem
->in_addr
[i
] = data
.in_addr
[i
];
1757 for (i
= 0; i
< elem
->out_num
; i
++) {
1758 elem
->out_addr
[i
] = data
.out_addr
[i
];
1761 for (i
= 0; i
< elem
->in_num
; i
++) {
1762 /* Base is overwritten by virtqueue_map. */
1763 elem
->in_sg
[i
].iov_base
= 0;
1764 elem
->in_sg
[i
].iov_len
= data
.in_sg
[i
].iov_len
;
1767 for (i
= 0; i
< elem
->out_num
; i
++) {
1768 /* Base is overwritten by virtqueue_map. */
1769 elem
->out_sg
[i
].iov_base
= 0;
1770 elem
->out_sg
[i
].iov_len
= data
.out_sg
[i
].iov_len
;
1773 if (virtio_host_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
1774 qemu_get_be32s(f
, &elem
->ndescs
);
1777 virtqueue_map(vdev
, elem
);
1781 void qemu_put_virtqueue_element(VirtIODevice
*vdev
, QEMUFile
*f
,
1782 VirtQueueElement
*elem
)
1784 VirtQueueElementOld data
;
1787 memset(&data
, 0, sizeof(data
));
1788 data
.index
= elem
->index
;
1789 data
.in_num
= elem
->in_num
;
1790 data
.out_num
= elem
->out_num
;
1792 for (i
= 0; i
< elem
->in_num
; i
++) {
1793 data
.in_addr
[i
] = elem
->in_addr
[i
];
1796 for (i
= 0; i
< elem
->out_num
; i
++) {
1797 data
.out_addr
[i
] = elem
->out_addr
[i
];
1800 for (i
= 0; i
< elem
->in_num
; i
++) {
1801 /* Base is overwritten by virtqueue_map when loading. Do not
1802 * save it, as it would leak the QEMU address space layout. */
1803 data
.in_sg
[i
].iov_len
= elem
->in_sg
[i
].iov_len
;
1806 for (i
= 0; i
< elem
->out_num
; i
++) {
1807 /* Do not save iov_base as above. */
1808 data
.out_sg
[i
].iov_len
= elem
->out_sg
[i
].iov_len
;
1811 if (virtio_host_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
1812 qemu_put_be32s(f
, &elem
->ndescs
);
1815 qemu_put_buffer(f
, (uint8_t *)&data
, sizeof(VirtQueueElementOld
));
1819 static void virtio_notify_vector(VirtIODevice
*vdev
, uint16_t vector
)
1821 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1822 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1824 if (virtio_device_disabled(vdev
)) {
1829 k
->notify(qbus
->parent
, vector
);
1833 void virtio_update_irq(VirtIODevice
*vdev
)
1835 virtio_notify_vector(vdev
, VIRTIO_NO_VECTOR
);
1838 static int virtio_validate_features(VirtIODevice
*vdev
)
1840 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1842 if (virtio_host_has_feature(vdev
, VIRTIO_F_IOMMU_PLATFORM
) &&
1843 !virtio_vdev_has_feature(vdev
, VIRTIO_F_IOMMU_PLATFORM
)) {
1847 if (k
->validate_features
) {
1848 return k
->validate_features(vdev
);
1854 int virtio_set_status(VirtIODevice
*vdev
, uint8_t val
)
1856 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1857 trace_virtio_set_status(vdev
, val
);
1859 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
1860 if (!(vdev
->status
& VIRTIO_CONFIG_S_FEATURES_OK
) &&
1861 val
& VIRTIO_CONFIG_S_FEATURES_OK
) {
1862 int ret
= virtio_validate_features(vdev
);
1870 if ((vdev
->status
& VIRTIO_CONFIG_S_DRIVER_OK
) !=
1871 (val
& VIRTIO_CONFIG_S_DRIVER_OK
)) {
1872 virtio_set_started(vdev
, val
& VIRTIO_CONFIG_S_DRIVER_OK
);
1875 if (k
->set_status
) {
1876 k
->set_status(vdev
, val
);
1883 static enum virtio_device_endian
virtio_default_endian(void)
1885 if (target_words_bigendian()) {
1886 return VIRTIO_DEVICE_ENDIAN_BIG
;
1888 return VIRTIO_DEVICE_ENDIAN_LITTLE
;
1892 static enum virtio_device_endian
virtio_current_cpu_endian(void)
1894 CPUClass
*cc
= CPU_GET_CLASS(current_cpu
);
1896 if (cc
->virtio_is_big_endian(current_cpu
)) {
1897 return VIRTIO_DEVICE_ENDIAN_BIG
;
1899 return VIRTIO_DEVICE_ENDIAN_LITTLE
;
1903 void virtio_reset(void *opaque
)
1905 VirtIODevice
*vdev
= opaque
;
1906 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1909 virtio_set_status(vdev
, 0);
1911 /* Guest initiated reset */
1912 vdev
->device_endian
= virtio_current_cpu_endian();
1915 vdev
->device_endian
= virtio_default_endian();
1922 vdev
->start_on_kick
= false;
1923 vdev
->started
= false;
1924 vdev
->broken
= false;
1925 vdev
->guest_features
= 0;
1926 vdev
->queue_sel
= 0;
1928 vdev
->disabled
= false;
1929 atomic_set(&vdev
->isr
, 0);
1930 vdev
->config_vector
= VIRTIO_NO_VECTOR
;
1931 virtio_notify_vector(vdev
, vdev
->config_vector
);
1933 for(i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
1934 vdev
->vq
[i
].vring
.desc
= 0;
1935 vdev
->vq
[i
].vring
.avail
= 0;
1936 vdev
->vq
[i
].vring
.used
= 0;
1937 vdev
->vq
[i
].last_avail_idx
= 0;
1938 vdev
->vq
[i
].shadow_avail_idx
= 0;
1939 vdev
->vq
[i
].used_idx
= 0;
1940 vdev
->vq
[i
].last_avail_wrap_counter
= true;
1941 vdev
->vq
[i
].shadow_avail_wrap_counter
= true;
1942 vdev
->vq
[i
].used_wrap_counter
= true;
1943 virtio_queue_set_vector(vdev
, i
, VIRTIO_NO_VECTOR
);
1944 vdev
->vq
[i
].signalled_used
= 0;
1945 vdev
->vq
[i
].signalled_used_valid
= false;
1946 vdev
->vq
[i
].notification
= true;
1947 vdev
->vq
[i
].vring
.num
= vdev
->vq
[i
].vring
.num_default
;
1948 vdev
->vq
[i
].inuse
= 0;
1949 virtio_virtqueue_reset_region_cache(&vdev
->vq
[i
]);
1953 uint32_t virtio_config_readb(VirtIODevice
*vdev
, uint32_t addr
)
1955 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1958 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1959 return (uint32_t)-1;
1962 k
->get_config(vdev
, vdev
->config
);
1964 val
= ldub_p(vdev
->config
+ addr
);
1968 uint32_t virtio_config_readw(VirtIODevice
*vdev
, uint32_t addr
)
1970 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1973 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1974 return (uint32_t)-1;
1977 k
->get_config(vdev
, vdev
->config
);
1979 val
= lduw_p(vdev
->config
+ addr
);
1983 uint32_t virtio_config_readl(VirtIODevice
*vdev
, uint32_t addr
)
1985 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1988 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1989 return (uint32_t)-1;
1992 k
->get_config(vdev
, vdev
->config
);
1994 val
= ldl_p(vdev
->config
+ addr
);
1998 void virtio_config_writeb(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
2000 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2003 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2007 stb_p(vdev
->config
+ addr
, val
);
2009 if (k
->set_config
) {
2010 k
->set_config(vdev
, vdev
->config
);
2014 void virtio_config_writew(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
2016 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2017 uint16_t val
= data
;
2019 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2023 stw_p(vdev
->config
+ addr
, val
);
2025 if (k
->set_config
) {
2026 k
->set_config(vdev
, vdev
->config
);
2030 void virtio_config_writel(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
2032 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2033 uint32_t val
= data
;
2035 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2039 stl_p(vdev
->config
+ addr
, val
);
2041 if (k
->set_config
) {
2042 k
->set_config(vdev
, vdev
->config
);
2046 uint32_t virtio_config_modern_readb(VirtIODevice
*vdev
, uint32_t addr
)
2048 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2051 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2052 return (uint32_t)-1;
2055 k
->get_config(vdev
, vdev
->config
);
2057 val
= ldub_p(vdev
->config
+ addr
);
2061 uint32_t virtio_config_modern_readw(VirtIODevice
*vdev
, uint32_t addr
)
2063 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2066 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2067 return (uint32_t)-1;
2070 k
->get_config(vdev
, vdev
->config
);
2072 val
= lduw_le_p(vdev
->config
+ addr
);
2076 uint32_t virtio_config_modern_readl(VirtIODevice
*vdev
, uint32_t addr
)
2078 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2081 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2082 return (uint32_t)-1;
2085 k
->get_config(vdev
, vdev
->config
);
2087 val
= ldl_le_p(vdev
->config
+ addr
);
2091 void virtio_config_modern_writeb(VirtIODevice
*vdev
,
2092 uint32_t addr
, uint32_t data
)
2094 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2097 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2101 stb_p(vdev
->config
+ addr
, val
);
2103 if (k
->set_config
) {
2104 k
->set_config(vdev
, vdev
->config
);
2108 void virtio_config_modern_writew(VirtIODevice
*vdev
,
2109 uint32_t addr
, uint32_t data
)
2111 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2112 uint16_t val
= data
;
2114 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2118 stw_le_p(vdev
->config
+ addr
, val
);
2120 if (k
->set_config
) {
2121 k
->set_config(vdev
, vdev
->config
);
2125 void virtio_config_modern_writel(VirtIODevice
*vdev
,
2126 uint32_t addr
, uint32_t data
)
2128 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2129 uint32_t val
= data
;
2131 if (addr
+ sizeof(val
) > vdev
->config_len
) {
2135 stl_le_p(vdev
->config
+ addr
, val
);
2137 if (k
->set_config
) {
2138 k
->set_config(vdev
, vdev
->config
);
2142 void virtio_queue_set_addr(VirtIODevice
*vdev
, int n
, hwaddr addr
)
2144 if (!vdev
->vq
[n
].vring
.num
) {
2147 vdev
->vq
[n
].vring
.desc
= addr
;
2148 virtio_queue_update_rings(vdev
, n
);
2151 hwaddr
virtio_queue_get_addr(VirtIODevice
*vdev
, int n
)
2153 return vdev
->vq
[n
].vring
.desc
;
2156 void virtio_queue_set_rings(VirtIODevice
*vdev
, int n
, hwaddr desc
,
2157 hwaddr avail
, hwaddr used
)
2159 if (!vdev
->vq
[n
].vring
.num
) {
2162 vdev
->vq
[n
].vring
.desc
= desc
;
2163 vdev
->vq
[n
].vring
.avail
= avail
;
2164 vdev
->vq
[n
].vring
.used
= used
;
2165 virtio_init_region_cache(vdev
, n
);
2168 void virtio_queue_set_num(VirtIODevice
*vdev
, int n
, int num
)
2170 /* Don't allow guest to flip queue between existent and
2171 * nonexistent states, or to set it to an invalid size.
2173 if (!!num
!= !!vdev
->vq
[n
].vring
.num
||
2174 num
> VIRTQUEUE_MAX_SIZE
||
2178 vdev
->vq
[n
].vring
.num
= num
;
2181 VirtQueue
*virtio_vector_first_queue(VirtIODevice
*vdev
, uint16_t vector
)
2183 return QLIST_FIRST(&vdev
->vector_queues
[vector
]);
2186 VirtQueue
*virtio_vector_next_queue(VirtQueue
*vq
)
2188 return QLIST_NEXT(vq
, node
);
2191 int virtio_queue_get_num(VirtIODevice
*vdev
, int n
)
2193 return vdev
->vq
[n
].vring
.num
;
2196 int virtio_queue_get_max_num(VirtIODevice
*vdev
, int n
)
2198 return vdev
->vq
[n
].vring
.num_default
;
2201 int virtio_get_num_queues(VirtIODevice
*vdev
)
2205 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2206 if (!virtio_queue_get_num(vdev
, i
)) {
2214 void virtio_queue_set_align(VirtIODevice
*vdev
, int n
, int align
)
2216 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2217 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2219 /* virtio-1 compliant devices cannot change the alignment */
2220 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
2221 error_report("tried to modify queue alignment for virtio-1 device");
2224 /* Check that the transport told us it was going to do this
2225 * (so a buggy transport will immediately assert rather than
2226 * silently failing to migrate this state)
2228 assert(k
->has_variable_vring_alignment
);
2231 vdev
->vq
[n
].vring
.align
= align
;
2232 virtio_queue_update_rings(vdev
, n
);
2236 static bool virtio_queue_notify_aio_vq(VirtQueue
*vq
)
2240 if (vq
->vring
.desc
&& vq
->handle_aio_output
) {
2241 VirtIODevice
*vdev
= vq
->vdev
;
2243 trace_virtio_queue_notify(vdev
, vq
- vdev
->vq
, vq
);
2244 ret
= vq
->handle_aio_output(vdev
, vq
);
2246 if (unlikely(vdev
->start_on_kick
)) {
2247 virtio_set_started(vdev
, true);
2254 static void virtio_queue_notify_vq(VirtQueue
*vq
)
2256 if (vq
->vring
.desc
&& vq
->handle_output
) {
2257 VirtIODevice
*vdev
= vq
->vdev
;
2259 if (unlikely(vdev
->broken
)) {
2263 trace_virtio_queue_notify(vdev
, vq
- vdev
->vq
, vq
);
2264 vq
->handle_output(vdev
, vq
);
2266 if (unlikely(vdev
->start_on_kick
)) {
2267 virtio_set_started(vdev
, true);
2272 void virtio_queue_notify(VirtIODevice
*vdev
, int n
)
2274 VirtQueue
*vq
= &vdev
->vq
[n
];
2276 if (unlikely(!vq
->vring
.desc
|| vdev
->broken
)) {
2280 trace_virtio_queue_notify(vdev
, vq
- vdev
->vq
, vq
);
2281 if (vq
->host_notifier_enabled
) {
2282 event_notifier_set(&vq
->host_notifier
);
2283 } else if (vq
->handle_output
) {
2284 vq
->handle_output(vdev
, vq
);
2286 if (unlikely(vdev
->start_on_kick
)) {
2287 virtio_set_started(vdev
, true);
2292 uint16_t virtio_queue_vector(VirtIODevice
*vdev
, int n
)
2294 return n
< VIRTIO_QUEUE_MAX
? vdev
->vq
[n
].vector
:
2298 void virtio_queue_set_vector(VirtIODevice
*vdev
, int n
, uint16_t vector
)
2300 VirtQueue
*vq
= &vdev
->vq
[n
];
2302 if (n
< VIRTIO_QUEUE_MAX
) {
2303 if (vdev
->vector_queues
&&
2304 vdev
->vq
[n
].vector
!= VIRTIO_NO_VECTOR
) {
2305 QLIST_REMOVE(vq
, node
);
2307 vdev
->vq
[n
].vector
= vector
;
2308 if (vdev
->vector_queues
&&
2309 vector
!= VIRTIO_NO_VECTOR
) {
2310 QLIST_INSERT_HEAD(&vdev
->vector_queues
[vector
], vq
, node
);
2315 VirtQueue
*virtio_add_queue(VirtIODevice
*vdev
, int queue_size
,
2316 VirtIOHandleOutput handle_output
)
2320 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2321 if (vdev
->vq
[i
].vring
.num
== 0)
2325 if (i
== VIRTIO_QUEUE_MAX
|| queue_size
> VIRTQUEUE_MAX_SIZE
)
2328 vdev
->vq
[i
].vring
.num
= queue_size
;
2329 vdev
->vq
[i
].vring
.num_default
= queue_size
;
2330 vdev
->vq
[i
].vring
.align
= VIRTIO_PCI_VRING_ALIGN
;
2331 vdev
->vq
[i
].handle_output
= handle_output
;
2332 vdev
->vq
[i
].handle_aio_output
= NULL
;
2333 vdev
->vq
[i
].used_elems
= g_malloc0(sizeof(VirtQueueElement
) *
2336 return &vdev
->vq
[i
];
2339 void virtio_delete_queue(VirtQueue
*vq
)
2342 vq
->vring
.num_default
= 0;
2343 vq
->handle_output
= NULL
;
2344 vq
->handle_aio_output
= NULL
;
2345 g_free(vq
->used_elems
);
2346 vq
->used_elems
= NULL
;
2347 virtio_virtqueue_reset_region_cache(vq
);
2350 void virtio_del_queue(VirtIODevice
*vdev
, int n
)
2352 if (n
< 0 || n
>= VIRTIO_QUEUE_MAX
) {
2356 virtio_delete_queue(&vdev
->vq
[n
]);
2359 static void virtio_set_isr(VirtIODevice
*vdev
, int value
)
2361 uint8_t old
= atomic_read(&vdev
->isr
);
2363 /* Do not write ISR if it does not change, so that its cacheline remains
2364 * shared in the common case where the guest does not read it.
2366 if ((old
& value
) != value
) {
2367 atomic_or(&vdev
->isr
, value
);
2371 static bool virtio_split_should_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
2375 /* We need to expose used array entries before checking used event. */
2377 /* Always notify when queue is empty (when feature acknowledge) */
2378 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_NOTIFY_ON_EMPTY
) &&
2379 !vq
->inuse
&& virtio_queue_empty(vq
)) {
2383 if (!virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
2384 return !(vring_avail_flags(vq
) & VRING_AVAIL_F_NO_INTERRUPT
);
2387 v
= vq
->signalled_used_valid
;
2388 vq
->signalled_used_valid
= true;
2389 old
= vq
->signalled_used
;
2390 new = vq
->signalled_used
= vq
->used_idx
;
2391 return !v
|| vring_need_event(vring_get_used_event(vq
), new, old
);
2394 static bool vring_packed_need_event(VirtQueue
*vq
, bool wrap
,
2395 uint16_t off_wrap
, uint16_t new,
2398 int off
= off_wrap
& ~(1 << 15);
2400 if (wrap
!= off_wrap
>> 15) {
2401 off
-= vq
->vring
.num
;
2404 return vring_need_event(off
, new, old
);
2407 static bool virtio_packed_should_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
2409 VRingPackedDescEvent e
;
2412 VRingMemoryRegionCaches
*caches
;
2414 caches
= vring_get_region_caches(vq
);
2415 vring_packed_event_read(vdev
, &caches
->avail
, &e
);
2417 old
= vq
->signalled_used
;
2418 new = vq
->signalled_used
= vq
->used_idx
;
2419 v
= vq
->signalled_used_valid
;
2420 vq
->signalled_used_valid
= true;
2422 if (e
.flags
== VRING_PACKED_EVENT_FLAG_DISABLE
) {
2424 } else if (e
.flags
== VRING_PACKED_EVENT_FLAG_ENABLE
) {
2428 return !v
|| vring_packed_need_event(vq
, vq
->used_wrap_counter
,
2429 e
.off_wrap
, new, old
);
2432 /* Called within rcu_read_lock(). */
2433 static bool virtio_should_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
2435 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
2436 return virtio_packed_should_notify(vdev
, vq
);
2438 return virtio_split_should_notify(vdev
, vq
);
2442 void virtio_notify_irqfd(VirtIODevice
*vdev
, VirtQueue
*vq
)
2444 WITH_RCU_READ_LOCK_GUARD() {
2445 if (!virtio_should_notify(vdev
, vq
)) {
2450 trace_virtio_notify_irqfd(vdev
, vq
);
2453 * virtio spec 1.0 says ISR bit 0 should be ignored with MSI, but
2454 * windows drivers included in virtio-win 1.8.0 (circa 2015) are
2455 * incorrectly polling this bit during crashdump and hibernation
2456 * in MSI mode, causing a hang if this bit is never updated.
2457 * Recent releases of Windows do not really shut down, but rather
2458 * log out and hibernate to make the next startup faster. Hence,
2459 * this manifested as a more serious hang during shutdown with
2461 * Next driver release from 2016 fixed this problem, so working around it
2462 * is not a must, but it's easy to do so let's do it here.
2464 * Note: it's safe to update ISR from any thread as it was switched
2465 * to an atomic operation.
2467 virtio_set_isr(vq
->vdev
, 0x1);
2468 event_notifier_set(&vq
->guest_notifier
);
2471 static void virtio_irq(VirtQueue
*vq
)
2473 virtio_set_isr(vq
->vdev
, 0x1);
2474 virtio_notify_vector(vq
->vdev
, vq
->vector
);
2477 void virtio_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
2479 WITH_RCU_READ_LOCK_GUARD() {
2480 if (!virtio_should_notify(vdev
, vq
)) {
2485 trace_virtio_notify(vdev
, vq
);
2489 void virtio_notify_config(VirtIODevice
*vdev
)
2491 if (!(vdev
->status
& VIRTIO_CONFIG_S_DRIVER_OK
))
2494 virtio_set_isr(vdev
, 0x3);
2496 virtio_notify_vector(vdev
, vdev
->config_vector
);
2499 static bool virtio_device_endian_needed(void *opaque
)
2501 VirtIODevice
*vdev
= opaque
;
2503 assert(vdev
->device_endian
!= VIRTIO_DEVICE_ENDIAN_UNKNOWN
);
2504 if (!virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
2505 return vdev
->device_endian
!= virtio_default_endian();
2507 /* Devices conforming to VIRTIO 1.0 or later are always LE. */
2508 return vdev
->device_endian
!= VIRTIO_DEVICE_ENDIAN_LITTLE
;
2511 static bool virtio_64bit_features_needed(void *opaque
)
2513 VirtIODevice
*vdev
= opaque
;
2515 return (vdev
->host_features
>> 32) != 0;
2518 static bool virtio_virtqueue_needed(void *opaque
)
2520 VirtIODevice
*vdev
= opaque
;
2522 return virtio_host_has_feature(vdev
, VIRTIO_F_VERSION_1
);
2525 static bool virtio_packed_virtqueue_needed(void *opaque
)
2527 VirtIODevice
*vdev
= opaque
;
2529 return virtio_host_has_feature(vdev
, VIRTIO_F_RING_PACKED
);
2532 static bool virtio_ringsize_needed(void *opaque
)
2534 VirtIODevice
*vdev
= opaque
;
2537 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2538 if (vdev
->vq
[i
].vring
.num
!= vdev
->vq
[i
].vring
.num_default
) {
2545 static bool virtio_extra_state_needed(void *opaque
)
2547 VirtIODevice
*vdev
= opaque
;
2548 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2549 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2551 return k
->has_extra_state
&&
2552 k
->has_extra_state(qbus
->parent
);
2555 static bool virtio_broken_needed(void *opaque
)
2557 VirtIODevice
*vdev
= opaque
;
2559 return vdev
->broken
;
2562 static bool virtio_started_needed(void *opaque
)
2564 VirtIODevice
*vdev
= opaque
;
2566 return vdev
->started
;
2569 static bool virtio_disabled_needed(void *opaque
)
2571 VirtIODevice
*vdev
= opaque
;
2573 return vdev
->disabled
;
2576 static const VMStateDescription vmstate_virtqueue
= {
2577 .name
= "virtqueue_state",
2579 .minimum_version_id
= 1,
2580 .fields
= (VMStateField
[]) {
2581 VMSTATE_UINT64(vring
.avail
, struct VirtQueue
),
2582 VMSTATE_UINT64(vring
.used
, struct VirtQueue
),
2583 VMSTATE_END_OF_LIST()
2587 static const VMStateDescription vmstate_packed_virtqueue
= {
2588 .name
= "packed_virtqueue_state",
2590 .minimum_version_id
= 1,
2591 .fields
= (VMStateField
[]) {
2592 VMSTATE_UINT16(last_avail_idx
, struct VirtQueue
),
2593 VMSTATE_BOOL(last_avail_wrap_counter
, struct VirtQueue
),
2594 VMSTATE_UINT16(used_idx
, struct VirtQueue
),
2595 VMSTATE_BOOL(used_wrap_counter
, struct VirtQueue
),
2596 VMSTATE_UINT32(inuse
, struct VirtQueue
),
2597 VMSTATE_END_OF_LIST()
2601 static const VMStateDescription vmstate_virtio_virtqueues
= {
2602 .name
= "virtio/virtqueues",
2604 .minimum_version_id
= 1,
2605 .needed
= &virtio_virtqueue_needed
,
2606 .fields
= (VMStateField
[]) {
2607 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq
, struct VirtIODevice
,
2608 VIRTIO_QUEUE_MAX
, 0, vmstate_virtqueue
, VirtQueue
),
2609 VMSTATE_END_OF_LIST()
2613 static const VMStateDescription vmstate_virtio_packed_virtqueues
= {
2614 .name
= "virtio/packed_virtqueues",
2616 .minimum_version_id
= 1,
2617 .needed
= &virtio_packed_virtqueue_needed
,
2618 .fields
= (VMStateField
[]) {
2619 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq
, struct VirtIODevice
,
2620 VIRTIO_QUEUE_MAX
, 0, vmstate_packed_virtqueue
, VirtQueue
),
2621 VMSTATE_END_OF_LIST()
2625 static const VMStateDescription vmstate_ringsize
= {
2626 .name
= "ringsize_state",
2628 .minimum_version_id
= 1,
2629 .fields
= (VMStateField
[]) {
2630 VMSTATE_UINT32(vring
.num_default
, struct VirtQueue
),
2631 VMSTATE_END_OF_LIST()
2635 static const VMStateDescription vmstate_virtio_ringsize
= {
2636 .name
= "virtio/ringsize",
2638 .minimum_version_id
= 1,
2639 .needed
= &virtio_ringsize_needed
,
2640 .fields
= (VMStateField
[]) {
2641 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq
, struct VirtIODevice
,
2642 VIRTIO_QUEUE_MAX
, 0, vmstate_ringsize
, VirtQueue
),
2643 VMSTATE_END_OF_LIST()
2647 static int get_extra_state(QEMUFile
*f
, void *pv
, size_t size
,
2648 const VMStateField
*field
)
2650 VirtIODevice
*vdev
= pv
;
2651 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2652 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2654 if (!k
->load_extra_state
) {
2657 return k
->load_extra_state(qbus
->parent
, f
);
2661 static int put_extra_state(QEMUFile
*f
, void *pv
, size_t size
,
2662 const VMStateField
*field
, QJSON
*vmdesc
)
2664 VirtIODevice
*vdev
= pv
;
2665 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2666 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2668 k
->save_extra_state(qbus
->parent
, f
);
2672 static const VMStateInfo vmstate_info_extra_state
= {
2673 .name
= "virtqueue_extra_state",
2674 .get
= get_extra_state
,
2675 .put
= put_extra_state
,
2678 static const VMStateDescription vmstate_virtio_extra_state
= {
2679 .name
= "virtio/extra_state",
2681 .minimum_version_id
= 1,
2682 .needed
= &virtio_extra_state_needed
,
2683 .fields
= (VMStateField
[]) {
2685 .name
= "extra_state",
2687 .field_exists
= NULL
,
2689 .info
= &vmstate_info_extra_state
,
2690 .flags
= VMS_SINGLE
,
2693 VMSTATE_END_OF_LIST()
2697 static const VMStateDescription vmstate_virtio_device_endian
= {
2698 .name
= "virtio/device_endian",
2700 .minimum_version_id
= 1,
2701 .needed
= &virtio_device_endian_needed
,
2702 .fields
= (VMStateField
[]) {
2703 VMSTATE_UINT8(device_endian
, VirtIODevice
),
2704 VMSTATE_END_OF_LIST()
2708 static const VMStateDescription vmstate_virtio_64bit_features
= {
2709 .name
= "virtio/64bit_features",
2711 .minimum_version_id
= 1,
2712 .needed
= &virtio_64bit_features_needed
,
2713 .fields
= (VMStateField
[]) {
2714 VMSTATE_UINT64(guest_features
, VirtIODevice
),
2715 VMSTATE_END_OF_LIST()
2719 static const VMStateDescription vmstate_virtio_broken
= {
2720 .name
= "virtio/broken",
2722 .minimum_version_id
= 1,
2723 .needed
= &virtio_broken_needed
,
2724 .fields
= (VMStateField
[]) {
2725 VMSTATE_BOOL(broken
, VirtIODevice
),
2726 VMSTATE_END_OF_LIST()
2730 static const VMStateDescription vmstate_virtio_started
= {
2731 .name
= "virtio/started",
2733 .minimum_version_id
= 1,
2734 .needed
= &virtio_started_needed
,
2735 .fields
= (VMStateField
[]) {
2736 VMSTATE_BOOL(started
, VirtIODevice
),
2737 VMSTATE_END_OF_LIST()
2741 static const VMStateDescription vmstate_virtio_disabled
= {
2742 .name
= "virtio/disabled",
2744 .minimum_version_id
= 1,
2745 .needed
= &virtio_disabled_needed
,
2746 .fields
= (VMStateField
[]) {
2747 VMSTATE_BOOL(disabled
, VirtIODevice
),
2748 VMSTATE_END_OF_LIST()
2752 static const VMStateDescription vmstate_virtio
= {
2755 .minimum_version_id
= 1,
2756 .minimum_version_id_old
= 1,
2757 .fields
= (VMStateField
[]) {
2758 VMSTATE_END_OF_LIST()
2760 .subsections
= (const VMStateDescription
*[]) {
2761 &vmstate_virtio_device_endian
,
2762 &vmstate_virtio_64bit_features
,
2763 &vmstate_virtio_virtqueues
,
2764 &vmstate_virtio_ringsize
,
2765 &vmstate_virtio_broken
,
2766 &vmstate_virtio_extra_state
,
2767 &vmstate_virtio_started
,
2768 &vmstate_virtio_packed_virtqueues
,
2769 &vmstate_virtio_disabled
,
2774 int virtio_save(VirtIODevice
*vdev
, QEMUFile
*f
)
2776 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2777 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2778 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2779 uint32_t guest_features_lo
= (vdev
->guest_features
& 0xffffffff);
2782 if (k
->save_config
) {
2783 k
->save_config(qbus
->parent
, f
);
2786 qemu_put_8s(f
, &vdev
->status
);
2787 qemu_put_8s(f
, &vdev
->isr
);
2788 qemu_put_be16s(f
, &vdev
->queue_sel
);
2789 qemu_put_be32s(f
, &guest_features_lo
);
2790 qemu_put_be32(f
, vdev
->config_len
);
2791 qemu_put_buffer(f
, vdev
->config
, vdev
->config_len
);
2793 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2794 if (vdev
->vq
[i
].vring
.num
== 0)
2798 qemu_put_be32(f
, i
);
2800 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2801 if (vdev
->vq
[i
].vring
.num
== 0)
2804 qemu_put_be32(f
, vdev
->vq
[i
].vring
.num
);
2805 if (k
->has_variable_vring_alignment
) {
2806 qemu_put_be32(f
, vdev
->vq
[i
].vring
.align
);
2809 * Save desc now, the rest of the ring addresses are saved in
2810 * subsections for VIRTIO-1 devices.
2812 qemu_put_be64(f
, vdev
->vq
[i
].vring
.desc
);
2813 qemu_put_be16s(f
, &vdev
->vq
[i
].last_avail_idx
);
2814 if (k
->save_queue
) {
2815 k
->save_queue(qbus
->parent
, i
, f
);
2819 if (vdc
->save
!= NULL
) {
2824 int ret
= vmstate_save_state(f
, vdc
->vmsd
, vdev
, NULL
);
2831 return vmstate_save_state(f
, &vmstate_virtio
, vdev
, NULL
);
2834 /* A wrapper for use as a VMState .put function */
2835 static int virtio_device_put(QEMUFile
*f
, void *opaque
, size_t size
,
2836 const VMStateField
*field
, QJSON
*vmdesc
)
2838 return virtio_save(VIRTIO_DEVICE(opaque
), f
);
2841 /* A wrapper for use as a VMState .get function */
2842 static int virtio_device_get(QEMUFile
*f
, void *opaque
, size_t size
,
2843 const VMStateField
*field
)
2845 VirtIODevice
*vdev
= VIRTIO_DEVICE(opaque
);
2846 DeviceClass
*dc
= DEVICE_CLASS(VIRTIO_DEVICE_GET_CLASS(vdev
));
2848 return virtio_load(vdev
, f
, dc
->vmsd
->version_id
);
2851 const VMStateInfo virtio_vmstate_info
= {
2853 .get
= virtio_device_get
,
2854 .put
= virtio_device_put
,
2857 static int virtio_set_features_nocheck(VirtIODevice
*vdev
, uint64_t val
)
2859 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2860 bool bad
= (val
& ~(vdev
->host_features
)) != 0;
2862 val
&= vdev
->host_features
;
2863 if (k
->set_features
) {
2864 k
->set_features(vdev
, val
);
2866 vdev
->guest_features
= val
;
2867 return bad
? -1 : 0;
2870 int virtio_set_features(VirtIODevice
*vdev
, uint64_t val
)
2874 * The driver must not attempt to set features after feature negotiation
2877 if (vdev
->status
& VIRTIO_CONFIG_S_FEATURES_OK
) {
2880 ret
= virtio_set_features_nocheck(vdev
, val
);
2882 if (virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
2883 /* VIRTIO_RING_F_EVENT_IDX changes the size of the caches. */
2885 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2886 if (vdev
->vq
[i
].vring
.num
!= 0) {
2887 virtio_init_region_cache(vdev
, i
);
2892 if (!virtio_device_started(vdev
, vdev
->status
) &&
2893 !virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
2894 vdev
->start_on_kick
= true;
2900 size_t virtio_feature_get_config_size(VirtIOFeature
*feature_sizes
,
2901 uint64_t host_features
)
2903 size_t config_size
= 0;
2906 for (i
= 0; feature_sizes
[i
].flags
!= 0; i
++) {
2907 if (host_features
& feature_sizes
[i
].flags
) {
2908 config_size
= MAX(feature_sizes
[i
].end
, config_size
);
2915 int virtio_load(VirtIODevice
*vdev
, QEMUFile
*f
, int version_id
)
2921 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2922 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2923 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2926 * We poison the endianness to ensure it does not get used before
2927 * subsections have been loaded.
2929 vdev
->device_endian
= VIRTIO_DEVICE_ENDIAN_UNKNOWN
;
2931 if (k
->load_config
) {
2932 ret
= k
->load_config(qbus
->parent
, f
);
2937 qemu_get_8s(f
, &vdev
->status
);
2938 qemu_get_8s(f
, &vdev
->isr
);
2939 qemu_get_be16s(f
, &vdev
->queue_sel
);
2940 if (vdev
->queue_sel
>= VIRTIO_QUEUE_MAX
) {
2943 qemu_get_be32s(f
, &features
);
2946 * Temporarily set guest_features low bits - needed by
2947 * virtio net load code testing for VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
2948 * VIRTIO_NET_F_GUEST_ANNOUNCE and VIRTIO_NET_F_CTRL_VQ.
2950 * Note: devices should always test host features in future - don't create
2951 * new dependencies like this.
2953 vdev
->guest_features
= features
;
2955 config_len
= qemu_get_be32(f
);
2958 * There are cases where the incoming config can be bigger or smaller
2959 * than what we have; so load what we have space for, and skip
2960 * any excess that's in the stream.
2962 qemu_get_buffer(f
, vdev
->config
, MIN(config_len
, vdev
->config_len
));
2964 while (config_len
> vdev
->config_len
) {
2969 num
= qemu_get_be32(f
);
2971 if (num
> VIRTIO_QUEUE_MAX
) {
2972 error_report("Invalid number of virtqueues: 0x%x", num
);
2976 for (i
= 0; i
< num
; i
++) {
2977 vdev
->vq
[i
].vring
.num
= qemu_get_be32(f
);
2978 if (k
->has_variable_vring_alignment
) {
2979 vdev
->vq
[i
].vring
.align
= qemu_get_be32(f
);
2981 vdev
->vq
[i
].vring
.desc
= qemu_get_be64(f
);
2982 qemu_get_be16s(f
, &vdev
->vq
[i
].last_avail_idx
);
2983 vdev
->vq
[i
].signalled_used_valid
= false;
2984 vdev
->vq
[i
].notification
= true;
2986 if (!vdev
->vq
[i
].vring
.desc
&& vdev
->vq
[i
].last_avail_idx
) {
2987 error_report("VQ %d address 0x0 "
2988 "inconsistent with Host index 0x%x",
2989 i
, vdev
->vq
[i
].last_avail_idx
);
2992 if (k
->load_queue
) {
2993 ret
= k
->load_queue(qbus
->parent
, i
, f
);
2999 virtio_notify_vector(vdev
, VIRTIO_NO_VECTOR
);
3001 if (vdc
->load
!= NULL
) {
3002 ret
= vdc
->load(vdev
, f
, version_id
);
3009 ret
= vmstate_load_state(f
, vdc
->vmsd
, vdev
, version_id
);
3016 ret
= vmstate_load_state(f
, &vmstate_virtio
, vdev
, 1);
3021 if (vdev
->device_endian
== VIRTIO_DEVICE_ENDIAN_UNKNOWN
) {
3022 vdev
->device_endian
= virtio_default_endian();
3025 if (virtio_64bit_features_needed(vdev
)) {
3027 * Subsection load filled vdev->guest_features. Run them
3028 * through virtio_set_features to sanity-check them against
3031 uint64_t features64
= vdev
->guest_features
;
3032 if (virtio_set_features_nocheck(vdev
, features64
) < 0) {
3033 error_report("Features 0x%" PRIx64
" unsupported. "
3034 "Allowed features: 0x%" PRIx64
,
3035 features64
, vdev
->host_features
);
3039 if (virtio_set_features_nocheck(vdev
, features
) < 0) {
3040 error_report("Features 0x%x unsupported. "
3041 "Allowed features: 0x%" PRIx64
,
3042 features
, vdev
->host_features
);
3047 if (!virtio_device_started(vdev
, vdev
->status
) &&
3048 !virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
3049 vdev
->start_on_kick
= true;
3052 RCU_READ_LOCK_GUARD();
3053 for (i
= 0; i
< num
; i
++) {
3054 if (vdev
->vq
[i
].vring
.desc
) {
3058 * VIRTIO-1 devices migrate desc, used, and avail ring addresses so
3059 * only the region cache needs to be set up. Legacy devices need
3060 * to calculate used and avail ring addresses based on the desc
3063 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
3064 virtio_init_region_cache(vdev
, i
);
3066 virtio_queue_update_rings(vdev
, i
);
3069 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3070 vdev
->vq
[i
].shadow_avail_idx
= vdev
->vq
[i
].last_avail_idx
;
3071 vdev
->vq
[i
].shadow_avail_wrap_counter
=
3072 vdev
->vq
[i
].last_avail_wrap_counter
;
3076 nheads
= vring_avail_idx(&vdev
->vq
[i
]) - vdev
->vq
[i
].last_avail_idx
;
3077 /* Check it isn't doing strange things with descriptor numbers. */
3078 if (nheads
> vdev
->vq
[i
].vring
.num
) {
3079 error_report("VQ %d size 0x%x Guest index 0x%x "
3080 "inconsistent with Host index 0x%x: delta 0x%x",
3081 i
, vdev
->vq
[i
].vring
.num
,
3082 vring_avail_idx(&vdev
->vq
[i
]),
3083 vdev
->vq
[i
].last_avail_idx
, nheads
);
3086 vdev
->vq
[i
].used_idx
= vring_used_idx(&vdev
->vq
[i
]);
3087 vdev
->vq
[i
].shadow_avail_idx
= vring_avail_idx(&vdev
->vq
[i
]);
3090 * Some devices migrate VirtQueueElements that have been popped
3091 * from the avail ring but not yet returned to the used ring.
3092 * Since max ring size < UINT16_MAX it's safe to use modulo
3093 * UINT16_MAX + 1 subtraction.
3095 vdev
->vq
[i
].inuse
= (uint16_t)(vdev
->vq
[i
].last_avail_idx
-
3096 vdev
->vq
[i
].used_idx
);
3097 if (vdev
->vq
[i
].inuse
> vdev
->vq
[i
].vring
.num
) {
3098 error_report("VQ %d size 0x%x < last_avail_idx 0x%x - "
3100 i
, vdev
->vq
[i
].vring
.num
,
3101 vdev
->vq
[i
].last_avail_idx
,
3102 vdev
->vq
[i
].used_idx
);
3108 if (vdc
->post_load
) {
3109 ret
= vdc
->post_load(vdev
);
3118 void virtio_cleanup(VirtIODevice
*vdev
)
3120 qemu_del_vm_change_state_handler(vdev
->vmstate
);
3123 static void virtio_vmstate_change(void *opaque
, int running
, RunState state
)
3125 VirtIODevice
*vdev
= opaque
;
3126 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3127 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
3128 bool backend_run
= running
&& virtio_device_started(vdev
, vdev
->status
);
3129 vdev
->vm_running
= running
;
3132 virtio_set_status(vdev
, vdev
->status
);
3135 if (k
->vmstate_change
) {
3136 k
->vmstate_change(qbus
->parent
, backend_run
);
3140 virtio_set_status(vdev
, vdev
->status
);
3144 void virtio_instance_init_common(Object
*proxy_obj
, void *data
,
3145 size_t vdev_size
, const char *vdev_name
)
3147 DeviceState
*vdev
= data
;
3149 object_initialize_child(proxy_obj
, "virtio-backend", vdev
, vdev_size
,
3150 vdev_name
, &error_abort
, NULL
);
3151 qdev_alias_all_properties(vdev
, proxy_obj
);
3154 void virtio_init(VirtIODevice
*vdev
, const char *name
,
3155 uint16_t device_id
, size_t config_size
)
3157 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3158 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
3160 int nvectors
= k
->query_nvectors
? k
->query_nvectors(qbus
->parent
) : 0;
3163 vdev
->vector_queues
=
3164 g_malloc0(sizeof(*vdev
->vector_queues
) * nvectors
);
3167 vdev
->start_on_kick
= false;
3168 vdev
->started
= false;
3169 vdev
->device_id
= device_id
;
3171 atomic_set(&vdev
->isr
, 0);
3172 vdev
->queue_sel
= 0;
3173 vdev
->config_vector
= VIRTIO_NO_VECTOR
;
3174 vdev
->vq
= g_malloc0(sizeof(VirtQueue
) * VIRTIO_QUEUE_MAX
);
3175 vdev
->vm_running
= runstate_is_running();
3176 vdev
->broken
= false;
3177 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
3178 vdev
->vq
[i
].vector
= VIRTIO_NO_VECTOR
;
3179 vdev
->vq
[i
].vdev
= vdev
;
3180 vdev
->vq
[i
].queue_index
= i
;
3181 vdev
->vq
[i
].host_notifier_enabled
= false;
3185 vdev
->config_len
= config_size
;
3186 if (vdev
->config_len
) {
3187 vdev
->config
= g_malloc0(config_size
);
3189 vdev
->config
= NULL
;
3191 vdev
->vmstate
= qdev_add_vm_change_state_handler(DEVICE(vdev
),
3192 virtio_vmstate_change
, vdev
);
3193 vdev
->device_endian
= virtio_default_endian();
3194 vdev
->use_guest_notifier_mask
= true;
3197 hwaddr
virtio_queue_get_desc_addr(VirtIODevice
*vdev
, int n
)
3199 return vdev
->vq
[n
].vring
.desc
;
3202 bool virtio_queue_enabled(VirtIODevice
*vdev
, int n
)
3204 return virtio_queue_get_desc_addr(vdev
, n
) != 0;
3207 hwaddr
virtio_queue_get_avail_addr(VirtIODevice
*vdev
, int n
)
3209 return vdev
->vq
[n
].vring
.avail
;
3212 hwaddr
virtio_queue_get_used_addr(VirtIODevice
*vdev
, int n
)
3214 return vdev
->vq
[n
].vring
.used
;
3217 hwaddr
virtio_queue_get_desc_size(VirtIODevice
*vdev
, int n
)
3219 return sizeof(VRingDesc
) * vdev
->vq
[n
].vring
.num
;
3222 hwaddr
virtio_queue_get_avail_size(VirtIODevice
*vdev
, int n
)
3226 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3227 return sizeof(struct VRingPackedDescEvent
);
3230 s
= virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
) ? 2 : 0;
3231 return offsetof(VRingAvail
, ring
) +
3232 sizeof(uint16_t) * vdev
->vq
[n
].vring
.num
+ s
;
3235 hwaddr
virtio_queue_get_used_size(VirtIODevice
*vdev
, int n
)
3239 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3240 return sizeof(struct VRingPackedDescEvent
);
3243 s
= virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
) ? 2 : 0;
3244 return offsetof(VRingUsed
, ring
) +
3245 sizeof(VRingUsedElem
) * vdev
->vq
[n
].vring
.num
+ s
;
3248 static unsigned int virtio_queue_packed_get_last_avail_idx(VirtIODevice
*vdev
,
3251 unsigned int avail
, used
;
3253 avail
= vdev
->vq
[n
].last_avail_idx
;
3254 avail
|= ((uint16_t)vdev
->vq
[n
].last_avail_wrap_counter
) << 15;
3256 used
= vdev
->vq
[n
].used_idx
;
3257 used
|= ((uint16_t)vdev
->vq
[n
].used_wrap_counter
) << 15;
3259 return avail
| used
<< 16;
3262 static uint16_t virtio_queue_split_get_last_avail_idx(VirtIODevice
*vdev
,
3265 return vdev
->vq
[n
].last_avail_idx
;
3268 unsigned int virtio_queue_get_last_avail_idx(VirtIODevice
*vdev
, int n
)
3270 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3271 return virtio_queue_packed_get_last_avail_idx(vdev
, n
);
3273 return virtio_queue_split_get_last_avail_idx(vdev
, n
);
3277 static void virtio_queue_packed_set_last_avail_idx(VirtIODevice
*vdev
,
3278 int n
, unsigned int idx
)
3280 struct VirtQueue
*vq
= &vdev
->vq
[n
];
3282 vq
->last_avail_idx
= vq
->shadow_avail_idx
= idx
& 0x7fff;
3283 vq
->last_avail_wrap_counter
=
3284 vq
->shadow_avail_wrap_counter
= !!(idx
& 0x8000);
3286 vq
->used_idx
= idx
& 0x7ffff;
3287 vq
->used_wrap_counter
= !!(idx
& 0x8000);
3290 static void virtio_queue_split_set_last_avail_idx(VirtIODevice
*vdev
,
3291 int n
, unsigned int idx
)
3293 vdev
->vq
[n
].last_avail_idx
= idx
;
3294 vdev
->vq
[n
].shadow_avail_idx
= idx
;
3297 void virtio_queue_set_last_avail_idx(VirtIODevice
*vdev
, int n
,
3300 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3301 virtio_queue_packed_set_last_avail_idx(vdev
, n
, idx
);
3303 virtio_queue_split_set_last_avail_idx(vdev
, n
, idx
);
3307 static void virtio_queue_packed_restore_last_avail_idx(VirtIODevice
*vdev
,
3310 /* We don't have a reference like avail idx in shared memory */
3314 static void virtio_queue_split_restore_last_avail_idx(VirtIODevice
*vdev
,
3317 RCU_READ_LOCK_GUARD();
3318 if (vdev
->vq
[n
].vring
.desc
) {
3319 vdev
->vq
[n
].last_avail_idx
= vring_used_idx(&vdev
->vq
[n
]);
3320 vdev
->vq
[n
].shadow_avail_idx
= vdev
->vq
[n
].last_avail_idx
;
3324 void virtio_queue_restore_last_avail_idx(VirtIODevice
*vdev
, int n
)
3326 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3327 virtio_queue_packed_restore_last_avail_idx(vdev
, n
);
3329 virtio_queue_split_restore_last_avail_idx(vdev
, n
);
3333 static void virtio_queue_packed_update_used_idx(VirtIODevice
*vdev
, int n
)
3335 /* used idx was updated through set_last_avail_idx() */
3339 static void virtio_split_packed_update_used_idx(VirtIODevice
*vdev
, int n
)
3341 RCU_READ_LOCK_GUARD();
3342 if (vdev
->vq
[n
].vring
.desc
) {
3343 vdev
->vq
[n
].used_idx
= vring_used_idx(&vdev
->vq
[n
]);
3347 void virtio_queue_update_used_idx(VirtIODevice
*vdev
, int n
)
3349 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3350 return virtio_queue_packed_update_used_idx(vdev
, n
);
3352 return virtio_split_packed_update_used_idx(vdev
, n
);
3356 void virtio_queue_invalidate_signalled_used(VirtIODevice
*vdev
, int n
)
3358 vdev
->vq
[n
].signalled_used_valid
= false;
3361 VirtQueue
*virtio_get_queue(VirtIODevice
*vdev
, int n
)
3363 return vdev
->vq
+ n
;
3366 uint16_t virtio_get_queue_index(VirtQueue
*vq
)
3368 return vq
->queue_index
;
3371 static void virtio_queue_guest_notifier_read(EventNotifier
*n
)
3373 VirtQueue
*vq
= container_of(n
, VirtQueue
, guest_notifier
);
3374 if (event_notifier_test_and_clear(n
)) {
3379 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue
*vq
, bool assign
,
3382 if (assign
&& !with_irqfd
) {
3383 event_notifier_set_handler(&vq
->guest_notifier
,
3384 virtio_queue_guest_notifier_read
);
3386 event_notifier_set_handler(&vq
->guest_notifier
, NULL
);
3389 /* Test and clear notifier before closing it,
3390 * in case poll callback didn't have time to run. */
3391 virtio_queue_guest_notifier_read(&vq
->guest_notifier
);
3395 EventNotifier
*virtio_queue_get_guest_notifier(VirtQueue
*vq
)
3397 return &vq
->guest_notifier
;
3400 static void virtio_queue_host_notifier_aio_read(EventNotifier
*n
)
3402 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
3403 if (event_notifier_test_and_clear(n
)) {
3404 virtio_queue_notify_aio_vq(vq
);
3408 static void virtio_queue_host_notifier_aio_poll_begin(EventNotifier
*n
)
3410 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
3412 virtio_queue_set_notification(vq
, 0);
3415 static bool virtio_queue_host_notifier_aio_poll(void *opaque
)
3417 EventNotifier
*n
= opaque
;
3418 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
3420 if (!vq
->vring
.desc
|| virtio_queue_empty(vq
)) {
3424 return virtio_queue_notify_aio_vq(vq
);
3427 static void virtio_queue_host_notifier_aio_poll_end(EventNotifier
*n
)
3429 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
3431 /* Caller polls once more after this to catch requests that race with us */
3432 virtio_queue_set_notification(vq
, 1);
3435 void virtio_queue_aio_set_host_notifier_handler(VirtQueue
*vq
, AioContext
*ctx
,
3436 VirtIOHandleAIOOutput handle_output
)
3438 if (handle_output
) {
3439 vq
->handle_aio_output
= handle_output
;
3440 aio_set_event_notifier(ctx
, &vq
->host_notifier
, true,
3441 virtio_queue_host_notifier_aio_read
,
3442 virtio_queue_host_notifier_aio_poll
);
3443 aio_set_event_notifier_poll(ctx
, &vq
->host_notifier
,
3444 virtio_queue_host_notifier_aio_poll_begin
,
3445 virtio_queue_host_notifier_aio_poll_end
);
3447 aio_set_event_notifier(ctx
, &vq
->host_notifier
, true, NULL
, NULL
);
3448 /* Test and clear notifier before after disabling event,
3449 * in case poll callback didn't have time to run. */
3450 virtio_queue_host_notifier_aio_read(&vq
->host_notifier
);
3451 vq
->handle_aio_output
= NULL
;
3455 void virtio_queue_host_notifier_read(EventNotifier
*n
)
3457 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
3458 if (event_notifier_test_and_clear(n
)) {
3459 virtio_queue_notify_vq(vq
);
3463 EventNotifier
*virtio_queue_get_host_notifier(VirtQueue
*vq
)
3465 return &vq
->host_notifier
;
3468 void virtio_queue_set_host_notifier_enabled(VirtQueue
*vq
, bool enabled
)
3470 vq
->host_notifier_enabled
= enabled
;
3473 int virtio_queue_set_host_notifier_mr(VirtIODevice
*vdev
, int n
,
3474 MemoryRegion
*mr
, bool assign
)
3476 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3477 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
3479 if (k
->set_host_notifier_mr
) {
3480 return k
->set_host_notifier_mr(qbus
->parent
, n
, mr
, assign
);
3486 void virtio_device_set_child_bus_name(VirtIODevice
*vdev
, char *bus_name
)
3488 g_free(vdev
->bus_name
);
3489 vdev
->bus_name
= g_strdup(bus_name
);
3492 void GCC_FMT_ATTR(2, 3) virtio_error(VirtIODevice
*vdev
, const char *fmt
, ...)
3497 error_vreport(fmt
, ap
);
3500 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
3501 vdev
->status
= vdev
->status
| VIRTIO_CONFIG_S_NEEDS_RESET
;
3502 virtio_notify_config(vdev
);
3505 vdev
->broken
= true;
3508 static void virtio_memory_listener_commit(MemoryListener
*listener
)
3510 VirtIODevice
*vdev
= container_of(listener
, VirtIODevice
, listener
);
3513 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
3514 if (vdev
->vq
[i
].vring
.num
== 0) {
3517 virtio_init_region_cache(vdev
, i
);
3521 static void virtio_device_realize(DeviceState
*dev
, Error
**errp
)
3523 VirtIODevice
*vdev
= VIRTIO_DEVICE(dev
);
3524 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(dev
);
3527 /* Devices should either use vmsd or the load/save methods */
3528 assert(!vdc
->vmsd
|| !vdc
->load
);
3530 if (vdc
->realize
!= NULL
) {
3531 vdc
->realize(dev
, &err
);
3533 error_propagate(errp
, err
);
3538 virtio_bus_device_plugged(vdev
, &err
);
3540 error_propagate(errp
, err
);
3541 vdc
->unrealize(dev
, NULL
);
3545 vdev
->listener
.commit
= virtio_memory_listener_commit
;
3546 memory_listener_register(&vdev
->listener
, vdev
->dma_as
);
3549 static void virtio_device_unrealize(DeviceState
*dev
, Error
**errp
)
3551 VirtIODevice
*vdev
= VIRTIO_DEVICE(dev
);
3552 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(dev
);
3555 virtio_bus_device_unplugged(vdev
);
3557 if (vdc
->unrealize
!= NULL
) {
3558 vdc
->unrealize(dev
, &err
);
3560 error_propagate(errp
, err
);
3565 g_free(vdev
->bus_name
);
3566 vdev
->bus_name
= NULL
;
3569 static void virtio_device_free_virtqueues(VirtIODevice
*vdev
)
3576 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
3577 if (vdev
->vq
[i
].vring
.num
== 0) {
3580 virtio_virtqueue_reset_region_cache(&vdev
->vq
[i
]);
3585 static void virtio_device_instance_finalize(Object
*obj
)
3587 VirtIODevice
*vdev
= VIRTIO_DEVICE(obj
);
3589 memory_listener_unregister(&vdev
->listener
);
3590 virtio_device_free_virtqueues(vdev
);
3592 g_free(vdev
->config
);
3593 g_free(vdev
->vector_queues
);
3596 static Property virtio_properties
[] = {
3597 DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice
, host_features
),
3598 DEFINE_PROP_BOOL("use-started", VirtIODevice
, use_started
, true),
3599 DEFINE_PROP_BOOL("use-disabled-flag", VirtIODevice
, use_disabled_flag
, true),
3600 DEFINE_PROP_END_OF_LIST(),
3603 static int virtio_device_start_ioeventfd_impl(VirtIODevice
*vdev
)
3605 VirtioBusState
*qbus
= VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev
)));
3608 memory_region_transaction_begin();
3609 for (n
= 0; n
< VIRTIO_QUEUE_MAX
; n
++) {
3610 VirtQueue
*vq
= &vdev
->vq
[n
];
3611 if (!virtio_queue_get_num(vdev
, n
)) {
3614 r
= virtio_bus_set_host_notifier(qbus
, n
, true);
3619 event_notifier_set_handler(&vq
->host_notifier
,
3620 virtio_queue_host_notifier_read
);
3623 for (n
= 0; n
< VIRTIO_QUEUE_MAX
; n
++) {
3624 /* Kick right away to begin processing requests already in vring */
3625 VirtQueue
*vq
= &vdev
->vq
[n
];
3626 if (!vq
->vring
.num
) {
3629 event_notifier_set(&vq
->host_notifier
);
3631 memory_region_transaction_commit();
3635 i
= n
; /* save n for a second iteration after transaction is committed. */
3637 VirtQueue
*vq
= &vdev
->vq
[n
];
3638 if (!virtio_queue_get_num(vdev
, n
)) {
3642 event_notifier_set_handler(&vq
->host_notifier
, NULL
);
3643 r
= virtio_bus_set_host_notifier(qbus
, n
, false);
3646 memory_region_transaction_commit();
3649 if (!virtio_queue_get_num(vdev
, i
)) {
3652 virtio_bus_cleanup_host_notifier(qbus
, i
);
3657 int virtio_device_start_ioeventfd(VirtIODevice
*vdev
)
3659 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3660 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
3662 return virtio_bus_start_ioeventfd(vbus
);
3665 static void virtio_device_stop_ioeventfd_impl(VirtIODevice
*vdev
)
3667 VirtioBusState
*qbus
= VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev
)));
3670 memory_region_transaction_begin();
3671 for (n
= 0; n
< VIRTIO_QUEUE_MAX
; n
++) {
3672 VirtQueue
*vq
= &vdev
->vq
[n
];
3674 if (!virtio_queue_get_num(vdev
, n
)) {
3677 event_notifier_set_handler(&vq
->host_notifier
, NULL
);
3678 r
= virtio_bus_set_host_notifier(qbus
, n
, false);
3681 memory_region_transaction_commit();
3683 for (n
= 0; n
< VIRTIO_QUEUE_MAX
; n
++) {
3684 if (!virtio_queue_get_num(vdev
, n
)) {
3687 virtio_bus_cleanup_host_notifier(qbus
, n
);
3691 int virtio_device_grab_ioeventfd(VirtIODevice
*vdev
)
3693 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3694 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
3696 return virtio_bus_grab_ioeventfd(vbus
);
3699 void virtio_device_release_ioeventfd(VirtIODevice
*vdev
)
3701 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3702 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
3704 virtio_bus_release_ioeventfd(vbus
);
3707 static void virtio_device_class_init(ObjectClass
*klass
, void *data
)
3709 /* Set the default value here. */
3710 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_CLASS(klass
);
3711 DeviceClass
*dc
= DEVICE_CLASS(klass
);
3713 dc
->realize
= virtio_device_realize
;
3714 dc
->unrealize
= virtio_device_unrealize
;
3715 dc
->bus_type
= TYPE_VIRTIO_BUS
;
3716 dc
->props
= virtio_properties
;
3717 vdc
->start_ioeventfd
= virtio_device_start_ioeventfd_impl
;
3718 vdc
->stop_ioeventfd
= virtio_device_stop_ioeventfd_impl
;
3720 vdc
->legacy_features
|= VIRTIO_LEGACY_FEATURES
;
3723 bool virtio_device_ioeventfd_enabled(VirtIODevice
*vdev
)
3725 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3726 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
3728 return virtio_bus_ioeventfd_enabled(vbus
);
3731 static const TypeInfo virtio_device_info
= {
3732 .name
= TYPE_VIRTIO_DEVICE
,
3733 .parent
= TYPE_DEVICE
,
3734 .instance_size
= sizeof(VirtIODevice
),
3735 .class_init
= virtio_device_class_init
,
3736 .instance_finalize
= virtio_device_instance_finalize
,
3738 .class_size
= sizeof(VirtioDeviceClass
),
3741 static void virtio_register_types(void)
3743 type_register_static(&virtio_device_info
);
3746 type_init(virtio_register_types
)