4 * Copyright IBM, Corp. 2007
7 * Anthony Liguori <aliguori@us.ibm.com>
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
14 #include "qemu/osdep.h"
15 #include "qapi/error.h"
16 #include "qemu-common.h"
19 #include "exec/address-spaces.h"
20 #include "qemu/error-report.h"
21 #include "hw/virtio/virtio.h"
22 #include "qemu/atomic.h"
23 #include "hw/virtio/virtio-bus.h"
24 #include "migration/migration.h"
25 #include "hw/virtio/virtio-access.h"
26 #include "sysemu/dma.h"
29 * The alignment to use between consumer and producer parts of vring.
30 * x86 pagesize again. This is the default, used by transports like PCI
31 * which don't provide a means for the guest to tell the host the alignment.
33 #define VIRTIO_PCI_VRING_ALIGN 4096
35 typedef struct VRingDesc
43 typedef struct VRingAvail
50 typedef struct VRingUsedElem
56 typedef struct VRingUsed
60 VRingUsedElem ring
[0];
66 unsigned int num_default
;
77 /* Next head to pop */
78 uint16_t last_avail_idx
;
80 /* Last avail_idx read from VQ. */
81 uint16_t shadow_avail_idx
;
85 /* Last used index value we have signalled on */
86 uint16_t signalled_used
;
88 /* Last used index value we have signalled on */
89 bool signalled_used_valid
;
91 /* Notification enabled? */
99 VirtIOHandleOutput handle_output
;
100 VirtIOHandleAIOOutput handle_aio_output
;
102 EventNotifier guest_notifier
;
103 EventNotifier host_notifier
;
104 QLIST_ENTRY(VirtQueue
) node
;
107 /* virt queue functions */
108 void virtio_queue_update_rings(VirtIODevice
*vdev
, int n
)
110 VRing
*vring
= &vdev
->vq
[n
].vring
;
113 /* not yet setup -> nothing to do */
116 vring
->avail
= vring
->desc
+ vring
->num
* sizeof(VRingDesc
);
117 vring
->used
= vring_align(vring
->avail
+
118 offsetof(VRingAvail
, ring
[vring
->num
]),
122 static void vring_desc_read(VirtIODevice
*vdev
, VRingDesc
*desc
,
123 uint8_t *desc_ptr
, int i
)
125 memcpy(desc
, desc_ptr
+ i
* sizeof(VRingDesc
), sizeof(VRingDesc
));
126 virtio_tswap64s(vdev
, &desc
->addr
);
127 virtio_tswap32s(vdev
, &desc
->len
);
128 virtio_tswap16s(vdev
, &desc
->flags
);
129 virtio_tswap16s(vdev
, &desc
->next
);
132 static inline uint16_t vring_avail_flags(VirtQueue
*vq
)
135 pa
= vq
->vring
.avail
+ offsetof(VRingAvail
, flags
);
136 return virtio_lduw_phys(vq
->vdev
, pa
);
139 static inline uint16_t vring_avail_idx(VirtQueue
*vq
)
142 pa
= vq
->vring
.avail
+ offsetof(VRingAvail
, idx
);
143 vq
->shadow_avail_idx
= virtio_lduw_phys(vq
->vdev
, pa
);
144 return vq
->shadow_avail_idx
;
147 static inline uint16_t vring_avail_ring(VirtQueue
*vq
, int i
)
150 pa
= vq
->vring
.avail
+ offsetof(VRingAvail
, ring
[i
]);
151 return virtio_lduw_phys(vq
->vdev
, pa
);
154 static inline uint16_t vring_get_used_event(VirtQueue
*vq
)
156 return vring_avail_ring(vq
, vq
->vring
.num
);
159 static inline void vring_used_write(VirtQueue
*vq
, VRingUsedElem
*uelem
,
163 virtio_tswap32s(vq
->vdev
, &uelem
->id
);
164 virtio_tswap32s(vq
->vdev
, &uelem
->len
);
165 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, ring
[i
]);
166 address_space_write(vq
->vdev
->dma_as
, pa
, MEMTXATTRS_UNSPECIFIED
,
167 (void *)uelem
, sizeof(VRingUsedElem
));
170 static uint16_t vring_used_idx(VirtQueue
*vq
)
173 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, idx
);
174 return virtio_lduw_phys(vq
->vdev
, pa
);
177 static inline void vring_used_idx_set(VirtQueue
*vq
, uint16_t val
)
180 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, idx
);
181 virtio_stw_phys(vq
->vdev
, pa
, val
);
185 static inline void vring_used_flags_set_bit(VirtQueue
*vq
, int mask
)
187 VirtIODevice
*vdev
= vq
->vdev
;
189 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, flags
);
190 virtio_stw_phys(vdev
, pa
, virtio_lduw_phys(vdev
, pa
) | mask
);
193 static inline void vring_used_flags_unset_bit(VirtQueue
*vq
, int mask
)
195 VirtIODevice
*vdev
= vq
->vdev
;
197 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, flags
);
198 virtio_stw_phys(vdev
, pa
, virtio_lduw_phys(vdev
, pa
) & ~mask
);
201 static inline void vring_set_avail_event(VirtQueue
*vq
, uint16_t val
)
204 if (!vq
->notification
) {
207 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, ring
[vq
->vring
.num
]);
208 virtio_stw_phys(vq
->vdev
, pa
, val
);
211 void virtio_queue_set_notification(VirtQueue
*vq
, int enable
)
213 vq
->notification
= enable
;
214 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
215 vring_set_avail_event(vq
, vring_avail_idx(vq
));
217 vring_used_flags_unset_bit(vq
, VRING_USED_F_NO_NOTIFY
);
219 vring_used_flags_set_bit(vq
, VRING_USED_F_NO_NOTIFY
);
222 /* Expose avail event/used flags before caller checks the avail idx. */
227 int virtio_queue_ready(VirtQueue
*vq
)
229 return vq
->vring
.avail
!= 0;
232 /* Fetch avail_idx from VQ memory only when we really need to know if
233 * guest has added some buffers. */
234 int virtio_queue_empty(VirtQueue
*vq
)
236 if (vq
->shadow_avail_idx
!= vq
->last_avail_idx
) {
240 return vring_avail_idx(vq
) == vq
->last_avail_idx
;
243 static void virtqueue_unmap_sg(VirtQueue
*vq
, const VirtQueueElement
*elem
,
246 AddressSpace
*dma_as
= vq
->vdev
->dma_as
;
251 for (i
= 0; i
< elem
->in_num
; i
++) {
252 size_t size
= MIN(len
- offset
, elem
->in_sg
[i
].iov_len
);
254 dma_memory_unmap(dma_as
, elem
->in_sg
[i
].iov_base
,
255 elem
->in_sg
[i
].iov_len
,
256 DMA_DIRECTION_FROM_DEVICE
, size
);
261 for (i
= 0; i
< elem
->out_num
; i
++)
262 dma_memory_unmap(dma_as
, elem
->out_sg
[i
].iov_base
,
263 elem
->out_sg
[i
].iov_len
,
264 DMA_DIRECTION_TO_DEVICE
,
265 elem
->out_sg
[i
].iov_len
);
268 /* virtqueue_detach_element:
269 * @vq: The #VirtQueue
270 * @elem: The #VirtQueueElement
271 * @len: number of bytes written
273 * Detach the element from the virtqueue. This function is suitable for device
274 * reset or other situations where a #VirtQueueElement is simply freed and will
275 * not be pushed or discarded.
277 void virtqueue_detach_element(VirtQueue
*vq
, const VirtQueueElement
*elem
,
281 virtqueue_unmap_sg(vq
, elem
, len
);
285 * @vq: The #VirtQueue
286 * @elem: The #VirtQueueElement
287 * @len: number of bytes written
289 * Pretend the most recent element wasn't popped from the virtqueue. The next
290 * call to virtqueue_pop() will refetch the element.
292 void virtqueue_unpop(VirtQueue
*vq
, const VirtQueueElement
*elem
,
295 vq
->last_avail_idx
--;
296 virtqueue_detach_element(vq
, elem
, len
);
300 * @vq: The #VirtQueue
301 * @num: Number of elements to push back
303 * Pretend that elements weren't popped from the virtqueue. The next
304 * virtqueue_pop() will refetch the oldest element.
306 * Use virtqueue_unpop() instead if you have a VirtQueueElement.
308 * Returns: true on success, false if @num is greater than the number of in use
311 bool virtqueue_rewind(VirtQueue
*vq
, unsigned int num
)
313 if (num
> vq
->inuse
) {
316 vq
->last_avail_idx
-= num
;
321 void virtqueue_fill(VirtQueue
*vq
, const VirtQueueElement
*elem
,
322 unsigned int len
, unsigned int idx
)
326 trace_virtqueue_fill(vq
, elem
, len
, idx
);
328 virtqueue_unmap_sg(vq
, elem
, len
);
330 if (unlikely(vq
->vdev
->broken
)) {
334 idx
= (idx
+ vq
->used_idx
) % vq
->vring
.num
;
336 uelem
.id
= elem
->index
;
338 vring_used_write(vq
, &uelem
, idx
);
341 void virtqueue_flush(VirtQueue
*vq
, unsigned int count
)
345 if (unlikely(vq
->vdev
->broken
)) {
350 /* Make sure buffer is written before we update index. */
352 trace_virtqueue_flush(vq
, count
);
355 vring_used_idx_set(vq
, new);
357 if (unlikely((int16_t)(new - vq
->signalled_used
) < (uint16_t)(new - old
)))
358 vq
->signalled_used_valid
= false;
361 void virtqueue_push(VirtQueue
*vq
, const VirtQueueElement
*elem
,
364 virtqueue_fill(vq
, elem
, len
, 0);
365 virtqueue_flush(vq
, 1);
368 static int virtqueue_num_heads(VirtQueue
*vq
, unsigned int idx
)
370 uint16_t num_heads
= vring_avail_idx(vq
) - idx
;
372 /* Check it isn't doing very strange things with descriptor numbers. */
373 if (num_heads
> vq
->vring
.num
) {
374 virtio_error(vq
->vdev
, "Guest moved used index from %u to %u",
375 idx
, vq
->shadow_avail_idx
);
378 /* On success, callers read a descriptor at vq->last_avail_idx.
379 * Make sure descriptor read does not bypass avail index read. */
387 static bool virtqueue_get_head(VirtQueue
*vq
, unsigned int idx
,
390 /* Grab the next descriptor number they're advertising, and increment
391 * the index we've seen. */
392 *head
= vring_avail_ring(vq
, idx
% vq
->vring
.num
);
394 /* If their number is silly, that's a fatal mistake. */
395 if (*head
>= vq
->vring
.num
) {
396 virtio_error(vq
->vdev
, "Guest says index %u is available", *head
);
404 VIRTQUEUE_READ_DESC_ERROR
= -1,
405 VIRTQUEUE_READ_DESC_DONE
= 0, /* end of chain */
406 VIRTQUEUE_READ_DESC_MORE
= 1, /* more buffers in chain */
409 static int virtqueue_read_next_desc(VirtIODevice
*vdev
, VRingDesc
*desc
,
410 void *desc_ptr
, unsigned int max
,
413 /* If this descriptor says it doesn't chain, we're done. */
414 if (!(desc
->flags
& VRING_DESC_F_NEXT
)) {
415 return VIRTQUEUE_READ_DESC_DONE
;
418 /* Check they're not leading us off end of descriptors. */
420 /* Make sure compiler knows to grab that: we don't want it changing! */
424 virtio_error(vdev
, "Desc next is %u", *next
);
425 return VIRTQUEUE_READ_DESC_ERROR
;
428 vring_desc_read(vdev
, desc
, desc_ptr
, *next
);
429 return VIRTQUEUE_READ_DESC_MORE
;
432 void virtqueue_get_avail_bytes(VirtQueue
*vq
, unsigned int *in_bytes
,
433 unsigned int *out_bytes
,
434 unsigned max_in_bytes
, unsigned max_out_bytes
)
436 VirtIODevice
*vdev
= vq
->vdev
;
437 unsigned int max
, idx
;
438 unsigned int total_bufs
, in_total
, out_total
;
439 void *vring_desc_ptr
;
440 void *indirect_desc_ptr
= NULL
;
444 idx
= vq
->last_avail_idx
;
445 total_bufs
= in_total
= out_total
= 0;
448 len
= max
* sizeof(VRingDesc
);
449 vring_desc_ptr
= address_space_map(vdev
->dma_as
, vq
->vring
.desc
, &len
, false);
450 if (len
< max
* sizeof(VRingDesc
)) {
451 virtio_error(vdev
, "Cannot map descriptor ring");
455 while ((rc
= virtqueue_num_heads(vq
, idx
)) > 0) {
456 void *desc_ptr
= vring_desc_ptr
;
457 unsigned int num_bufs
;
461 num_bufs
= total_bufs
;
463 if (!virtqueue_get_head(vq
, idx
++, &i
)) {
467 vring_desc_read(vdev
, &desc
, desc_ptr
, i
);
469 if (desc
.flags
& VRING_DESC_F_INDIRECT
) {
471 if (desc
.len
% sizeof(VRingDesc
)) {
472 virtio_error(vdev
, "Invalid size for indirect buffer table");
476 /* If we've got too many, that implies a descriptor loop. */
477 if (num_bufs
>= max
) {
478 virtio_error(vdev
, "Looped descriptor");
482 /* loop over the indirect descriptor table */
483 indirect_desc_ptr
= address_space_map(vdev
->dma_as
, desc
.addr
,
485 desc_ptr
= indirect_desc_ptr
;
486 if (len
< desc
.len
) {
487 virtio_error(vdev
, "Cannot map indirect buffer");
491 max
= desc
.len
/ sizeof(VRingDesc
);
493 vring_desc_read(vdev
, &desc
, desc_ptr
, i
);
497 /* If we've got too many, that implies a descriptor loop. */
498 if (++num_bufs
> max
) {
499 virtio_error(vdev
, "Looped descriptor");
503 if (desc
.flags
& VRING_DESC_F_WRITE
) {
504 in_total
+= desc
.len
;
506 out_total
+= desc
.len
;
508 if (in_total
>= max_in_bytes
&& out_total
>= max_out_bytes
) {
512 rc
= virtqueue_read_next_desc(vdev
, &desc
, desc_ptr
, max
, &i
);
513 } while (rc
== VIRTQUEUE_READ_DESC_MORE
);
515 if (rc
== VIRTQUEUE_READ_DESC_ERROR
) {
519 if (desc_ptr
== indirect_desc_ptr
) {
520 address_space_unmap(vdev
->dma_as
, desc_ptr
, len
, false, 0);
521 indirect_desc_ptr
= NULL
;
524 total_bufs
= num_bufs
;
533 if (indirect_desc_ptr
) {
534 address_space_unmap(vdev
->dma_as
, indirect_desc_ptr
, len
, false, 0);
536 address_space_unmap(vdev
->dma_as
, vring_desc_ptr
, len
, false, 0);
538 *in_bytes
= in_total
;
541 *out_bytes
= out_total
;
546 in_total
= out_total
= 0;
550 int virtqueue_avail_bytes(VirtQueue
*vq
, unsigned int in_bytes
,
551 unsigned int out_bytes
)
553 unsigned int in_total
, out_total
;
555 virtqueue_get_avail_bytes(vq
, &in_total
, &out_total
, in_bytes
, out_bytes
);
556 return in_bytes
<= in_total
&& out_bytes
<= out_total
;
559 static bool virtqueue_map_desc(VirtIODevice
*vdev
, unsigned int *p_num_sg
,
560 hwaddr
*addr
, struct iovec
*iov
,
561 unsigned int max_num_sg
, bool is_write
,
562 hwaddr pa
, size_t sz
)
565 unsigned num_sg
= *p_num_sg
;
566 assert(num_sg
<= max_num_sg
);
569 virtio_error(vdev
, "virtio: zero sized buffers are not allowed");
576 if (num_sg
== max_num_sg
) {
577 virtio_error(vdev
, "virtio: too many write descriptors in "
582 iov
[num_sg
].iov_base
= dma_memory_map(vdev
->dma_as
, pa
, &len
,
584 DMA_DIRECTION_FROM_DEVICE
:
585 DMA_DIRECTION_TO_DEVICE
);
586 if (!iov
[num_sg
].iov_base
) {
587 virtio_error(vdev
, "virtio: bogus descriptor or out of resources");
591 iov
[num_sg
].iov_len
= len
;
605 /* Only used by error code paths before we have a VirtQueueElement (therefore
606 * virtqueue_unmap_sg() can't be used). Assumes buffers weren't written to
609 static void virtqueue_undo_map_desc(unsigned int out_num
, unsigned int in_num
,
614 for (i
= 0; i
< out_num
+ in_num
; i
++) {
615 int is_write
= i
>= out_num
;
617 cpu_physical_memory_unmap(iov
->iov_base
, iov
->iov_len
, is_write
, 0);
622 static void virtqueue_map_iovec(VirtIODevice
*vdev
, struct iovec
*sg
,
623 hwaddr
*addr
, unsigned int *num_sg
,
629 for (i
= 0; i
< *num_sg
; i
++) {
631 sg
[i
].iov_base
= dma_memory_map(vdev
->dma_as
,
632 addr
[i
], &len
, is_write
?
633 DMA_DIRECTION_FROM_DEVICE
:
634 DMA_DIRECTION_TO_DEVICE
);
635 if (!sg
[i
].iov_base
) {
636 error_report("virtio: error trying to map MMIO memory");
639 if (len
!= sg
[i
].iov_len
) {
640 error_report("virtio: unexpected memory split");
646 void virtqueue_map(VirtIODevice
*vdev
, VirtQueueElement
*elem
)
648 virtqueue_map_iovec(vdev
, elem
->in_sg
, elem
->in_addr
, &elem
->in_num
, 1);
649 virtqueue_map_iovec(vdev
, elem
->out_sg
, elem
->out_addr
, &elem
->out_num
, 0);
652 static void *virtqueue_alloc_element(size_t sz
, unsigned out_num
, unsigned in_num
)
654 VirtQueueElement
*elem
;
655 size_t in_addr_ofs
= QEMU_ALIGN_UP(sz
, __alignof__(elem
->in_addr
[0]));
656 size_t out_addr_ofs
= in_addr_ofs
+ in_num
* sizeof(elem
->in_addr
[0]);
657 size_t out_addr_end
= out_addr_ofs
+ out_num
* sizeof(elem
->out_addr
[0]);
658 size_t in_sg_ofs
= QEMU_ALIGN_UP(out_addr_end
, __alignof__(elem
->in_sg
[0]));
659 size_t out_sg_ofs
= in_sg_ofs
+ in_num
* sizeof(elem
->in_sg
[0]);
660 size_t out_sg_end
= out_sg_ofs
+ out_num
* sizeof(elem
->out_sg
[0]);
662 assert(sz
>= sizeof(VirtQueueElement
));
663 elem
= g_malloc(out_sg_end
);
664 elem
->out_num
= out_num
;
665 elem
->in_num
= in_num
;
666 elem
->in_addr
= (void *)elem
+ in_addr_ofs
;
667 elem
->out_addr
= (void *)elem
+ out_addr_ofs
;
668 elem
->in_sg
= (void *)elem
+ in_sg_ofs
;
669 elem
->out_sg
= (void *)elem
+ out_sg_ofs
;
673 void *virtqueue_pop(VirtQueue
*vq
, size_t sz
)
675 unsigned int i
, head
, max
;
676 void *vring_desc_ptr
;
677 void *indirect_desc_ptr
= NULL
;
680 VirtIODevice
*vdev
= vq
->vdev
;
681 VirtQueueElement
*elem
= NULL
;
682 unsigned out_num
, in_num
;
683 hwaddr addr
[VIRTQUEUE_MAX_SIZE
];
684 struct iovec iov
[VIRTQUEUE_MAX_SIZE
];
688 if (unlikely(vdev
->broken
)) {
691 if (virtio_queue_empty(vq
)) {
694 /* Needed after virtio_queue_empty(), see comment in
695 * virtqueue_num_heads(). */
698 /* When we start there are none of either input nor output. */
699 out_num
= in_num
= 0;
703 if (vq
->inuse
>= vq
->vring
.num
) {
704 virtio_error(vdev
, "Virtqueue size exceeded");
708 if (!virtqueue_get_head(vq
, vq
->last_avail_idx
++, &head
)) {
712 if (virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
713 vring_set_avail_event(vq
, vq
->last_avail_idx
);
718 len
= max
* sizeof(VRingDesc
);
719 vring_desc_ptr
= address_space_map(vdev
->dma_as
, vq
->vring
.desc
, &len
, false);
720 if (len
< max
* sizeof(VRingDesc
)) {
721 virtio_error(vdev
, "Cannot map descriptor ring");
725 desc_ptr
= vring_desc_ptr
;
726 vring_desc_read(vdev
, &desc
, desc_ptr
, i
);
727 if (desc
.flags
& VRING_DESC_F_INDIRECT
) {
728 if (desc
.len
% sizeof(VRingDesc
)) {
729 virtio_error(vdev
, "Invalid size for indirect buffer table");
733 /* loop over the indirect descriptor table */
735 indirect_desc_ptr
= address_space_map(vdev
->dma_as
, desc
.addr
, &len
, false);
736 desc_ptr
= indirect_desc_ptr
;
737 if (len
< desc
.len
) {
738 virtio_error(vdev
, "Cannot map indirect buffer");
742 max
= desc
.len
/ sizeof(VRingDesc
);
744 vring_desc_read(vdev
, &desc
, desc_ptr
, i
);
747 /* Collect all the descriptors */
751 if (desc
.flags
& VRING_DESC_F_WRITE
) {
752 map_ok
= virtqueue_map_desc(vdev
, &in_num
, addr
+ out_num
,
754 VIRTQUEUE_MAX_SIZE
- out_num
, true,
755 desc
.addr
, desc
.len
);
758 virtio_error(vdev
, "Incorrect order for descriptors");
761 map_ok
= virtqueue_map_desc(vdev
, &out_num
, addr
, iov
,
762 VIRTQUEUE_MAX_SIZE
, false,
763 desc
.addr
, desc
.len
);
769 /* If we've got too many, that implies a descriptor loop. */
770 if ((in_num
+ out_num
) > max
) {
771 virtio_error(vdev
, "Looped descriptor");
775 rc
= virtqueue_read_next_desc(vdev
, &desc
, desc_ptr
, max
, &i
);
776 } while (rc
== VIRTQUEUE_READ_DESC_MORE
);
778 if (rc
== VIRTQUEUE_READ_DESC_ERROR
) {
782 /* Now copy what we have collected and mapped */
783 elem
= virtqueue_alloc_element(sz
, out_num
, in_num
);
785 for (i
= 0; i
< out_num
; i
++) {
786 elem
->out_addr
[i
] = addr
[i
];
787 elem
->out_sg
[i
] = iov
[i
];
789 for (i
= 0; i
< in_num
; i
++) {
790 elem
->in_addr
[i
] = addr
[out_num
+ i
];
791 elem
->in_sg
[i
] = iov
[out_num
+ i
];
796 trace_virtqueue_pop(vq
, elem
, elem
->in_num
, elem
->out_num
);
798 if (indirect_desc_ptr
) {
799 address_space_unmap(vdev
->dma_as
, indirect_desc_ptr
, len
, false, 0);
801 address_space_unmap(vdev
->dma_as
, vring_desc_ptr
, len
, false, 0);
806 virtqueue_undo_map_desc(out_num
, in_num
, iov
);
810 /* virtqueue_drop_all:
811 * @vq: The #VirtQueue
812 * Drops all queued buffers and indicates them to the guest
813 * as if they are done. Useful when buffers can not be
814 * processed but must be returned to the guest.
816 unsigned int virtqueue_drop_all(VirtQueue
*vq
)
818 unsigned int dropped
= 0;
819 VirtQueueElement elem
= {};
820 VirtIODevice
*vdev
= vq
->vdev
;
821 bool fEventIdx
= virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
);
823 if (unlikely(vdev
->broken
)) {
827 while (!virtio_queue_empty(vq
) && vq
->inuse
< vq
->vring
.num
) {
828 /* works similar to virtqueue_pop but does not map buffers
829 * and does not allocate any memory */
831 if (!virtqueue_get_head(vq
, vq
->last_avail_idx
, &elem
.index
)) {
835 vq
->last_avail_idx
++;
837 vring_set_avail_event(vq
, vq
->last_avail_idx
);
839 /* immediately push the element, nothing to unmap
840 * as both in_num and out_num are set to 0 */
841 virtqueue_push(vq
, &elem
, 0);
848 /* Reading and writing a structure directly to QEMUFile is *awful*, but
849 * it is what QEMU has always done by mistake. We can change it sooner
850 * or later by bumping the version number of the affected vm states.
851 * In the meanwhile, since the in-memory layout of VirtQueueElement
852 * has changed, we need to marshal to and from the layout that was
853 * used before the change.
855 typedef struct VirtQueueElementOld
{
857 unsigned int out_num
;
859 hwaddr in_addr
[VIRTQUEUE_MAX_SIZE
];
860 hwaddr out_addr
[VIRTQUEUE_MAX_SIZE
];
861 struct iovec in_sg
[VIRTQUEUE_MAX_SIZE
];
862 struct iovec out_sg
[VIRTQUEUE_MAX_SIZE
];
863 } VirtQueueElementOld
;
865 void *qemu_get_virtqueue_element(VirtIODevice
*vdev
, QEMUFile
*f
, size_t sz
)
867 VirtQueueElement
*elem
;
868 VirtQueueElementOld data
;
871 qemu_get_buffer(f
, (uint8_t *)&data
, sizeof(VirtQueueElementOld
));
873 /* TODO: teach all callers that this can fail, and return failure instead
875 * When we do, we might be able to re-enable NDEBUG below.
878 #error building with NDEBUG is not supported
880 assert(ARRAY_SIZE(data
.in_addr
) >= data
.in_num
);
881 assert(ARRAY_SIZE(data
.out_addr
) >= data
.out_num
);
883 elem
= virtqueue_alloc_element(sz
, data
.out_num
, data
.in_num
);
884 elem
->index
= data
.index
;
886 for (i
= 0; i
< elem
->in_num
; i
++) {
887 elem
->in_addr
[i
] = data
.in_addr
[i
];
890 for (i
= 0; i
< elem
->out_num
; i
++) {
891 elem
->out_addr
[i
] = data
.out_addr
[i
];
894 for (i
= 0; i
< elem
->in_num
; i
++) {
895 /* Base is overwritten by virtqueue_map. */
896 elem
->in_sg
[i
].iov_base
= 0;
897 elem
->in_sg
[i
].iov_len
= data
.in_sg
[i
].iov_len
;
900 for (i
= 0; i
< elem
->out_num
; i
++) {
901 /* Base is overwritten by virtqueue_map. */
902 elem
->out_sg
[i
].iov_base
= 0;
903 elem
->out_sg
[i
].iov_len
= data
.out_sg
[i
].iov_len
;
906 virtqueue_map(vdev
, elem
);
910 void qemu_put_virtqueue_element(QEMUFile
*f
, VirtQueueElement
*elem
)
912 VirtQueueElementOld data
;
915 memset(&data
, 0, sizeof(data
));
916 data
.index
= elem
->index
;
917 data
.in_num
= elem
->in_num
;
918 data
.out_num
= elem
->out_num
;
920 for (i
= 0; i
< elem
->in_num
; i
++) {
921 data
.in_addr
[i
] = elem
->in_addr
[i
];
924 for (i
= 0; i
< elem
->out_num
; i
++) {
925 data
.out_addr
[i
] = elem
->out_addr
[i
];
928 for (i
= 0; i
< elem
->in_num
; i
++) {
929 /* Base is overwritten by virtqueue_map when loading. Do not
930 * save it, as it would leak the QEMU address space layout. */
931 data
.in_sg
[i
].iov_len
= elem
->in_sg
[i
].iov_len
;
934 for (i
= 0; i
< elem
->out_num
; i
++) {
935 /* Do not save iov_base as above. */
936 data
.out_sg
[i
].iov_len
= elem
->out_sg
[i
].iov_len
;
938 qemu_put_buffer(f
, (uint8_t *)&data
, sizeof(VirtQueueElementOld
));
942 static void virtio_notify_vector(VirtIODevice
*vdev
, uint16_t vector
)
944 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
945 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
947 if (unlikely(vdev
->broken
)) {
952 k
->notify(qbus
->parent
, vector
);
956 void virtio_update_irq(VirtIODevice
*vdev
)
958 virtio_notify_vector(vdev
, VIRTIO_NO_VECTOR
);
961 static int virtio_validate_features(VirtIODevice
*vdev
)
963 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
965 if (virtio_host_has_feature(vdev
, VIRTIO_F_IOMMU_PLATFORM
) &&
966 !virtio_vdev_has_feature(vdev
, VIRTIO_F_IOMMU_PLATFORM
)) {
970 if (k
->validate_features
) {
971 return k
->validate_features(vdev
);
977 int virtio_set_status(VirtIODevice
*vdev
, uint8_t val
)
979 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
980 trace_virtio_set_status(vdev
, val
);
982 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
983 if (!(vdev
->status
& VIRTIO_CONFIG_S_FEATURES_OK
) &&
984 val
& VIRTIO_CONFIG_S_FEATURES_OK
) {
985 int ret
= virtio_validate_features(vdev
);
993 k
->set_status(vdev
, val
);
999 bool target_words_bigendian(void);
1000 static enum virtio_device_endian
virtio_default_endian(void)
1002 if (target_words_bigendian()) {
1003 return VIRTIO_DEVICE_ENDIAN_BIG
;
1005 return VIRTIO_DEVICE_ENDIAN_LITTLE
;
1009 static enum virtio_device_endian
virtio_current_cpu_endian(void)
1011 CPUClass
*cc
= CPU_GET_CLASS(current_cpu
);
1013 if (cc
->virtio_is_big_endian(current_cpu
)) {
1014 return VIRTIO_DEVICE_ENDIAN_BIG
;
1016 return VIRTIO_DEVICE_ENDIAN_LITTLE
;
1020 void virtio_reset(void *opaque
)
1022 VirtIODevice
*vdev
= opaque
;
1023 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1026 virtio_set_status(vdev
, 0);
1028 /* Guest initiated reset */
1029 vdev
->device_endian
= virtio_current_cpu_endian();
1032 vdev
->device_endian
= virtio_default_endian();
1039 vdev
->broken
= false;
1040 vdev
->guest_features
= 0;
1041 vdev
->queue_sel
= 0;
1043 atomic_set(&vdev
->isr
, 0);
1044 vdev
->config_vector
= VIRTIO_NO_VECTOR
;
1045 virtio_notify_vector(vdev
, vdev
->config_vector
);
1047 for(i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
1048 vdev
->vq
[i
].vring
.desc
= 0;
1049 vdev
->vq
[i
].vring
.avail
= 0;
1050 vdev
->vq
[i
].vring
.used
= 0;
1051 vdev
->vq
[i
].last_avail_idx
= 0;
1052 vdev
->vq
[i
].shadow_avail_idx
= 0;
1053 vdev
->vq
[i
].used_idx
= 0;
1054 virtio_queue_set_vector(vdev
, i
, VIRTIO_NO_VECTOR
);
1055 vdev
->vq
[i
].signalled_used
= 0;
1056 vdev
->vq
[i
].signalled_used_valid
= false;
1057 vdev
->vq
[i
].notification
= true;
1058 vdev
->vq
[i
].vring
.num
= vdev
->vq
[i
].vring
.num_default
;
1059 vdev
->vq
[i
].inuse
= 0;
1063 uint32_t virtio_config_readb(VirtIODevice
*vdev
, uint32_t addr
)
1065 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1068 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1069 return (uint32_t)-1;
1072 k
->get_config(vdev
, vdev
->config
);
1074 val
= ldub_p(vdev
->config
+ addr
);
1078 uint32_t virtio_config_readw(VirtIODevice
*vdev
, uint32_t addr
)
1080 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1083 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1084 return (uint32_t)-1;
1087 k
->get_config(vdev
, vdev
->config
);
1089 val
= lduw_p(vdev
->config
+ addr
);
1093 uint32_t virtio_config_readl(VirtIODevice
*vdev
, uint32_t addr
)
1095 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1098 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1099 return (uint32_t)-1;
1102 k
->get_config(vdev
, vdev
->config
);
1104 val
= ldl_p(vdev
->config
+ addr
);
1108 void virtio_config_writeb(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
1110 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1113 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1117 stb_p(vdev
->config
+ addr
, val
);
1119 if (k
->set_config
) {
1120 k
->set_config(vdev
, vdev
->config
);
1124 void virtio_config_writew(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
1126 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1127 uint16_t val
= data
;
1129 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1133 stw_p(vdev
->config
+ addr
, val
);
1135 if (k
->set_config
) {
1136 k
->set_config(vdev
, vdev
->config
);
1140 void virtio_config_writel(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
1142 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1143 uint32_t val
= data
;
1145 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1149 stl_p(vdev
->config
+ addr
, val
);
1151 if (k
->set_config
) {
1152 k
->set_config(vdev
, vdev
->config
);
1156 uint32_t virtio_config_modern_readb(VirtIODevice
*vdev
, uint32_t addr
)
1158 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1161 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1162 return (uint32_t)-1;
1165 k
->get_config(vdev
, vdev
->config
);
1167 val
= ldub_p(vdev
->config
+ addr
);
1171 uint32_t virtio_config_modern_readw(VirtIODevice
*vdev
, uint32_t addr
)
1173 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1176 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1177 return (uint32_t)-1;
1180 k
->get_config(vdev
, vdev
->config
);
1182 val
= lduw_le_p(vdev
->config
+ addr
);
1186 uint32_t virtio_config_modern_readl(VirtIODevice
*vdev
, uint32_t addr
)
1188 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1191 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1192 return (uint32_t)-1;
1195 k
->get_config(vdev
, vdev
->config
);
1197 val
= ldl_le_p(vdev
->config
+ addr
);
1201 void virtio_config_modern_writeb(VirtIODevice
*vdev
,
1202 uint32_t addr
, uint32_t data
)
1204 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1207 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1211 stb_p(vdev
->config
+ addr
, val
);
1213 if (k
->set_config
) {
1214 k
->set_config(vdev
, vdev
->config
);
1218 void virtio_config_modern_writew(VirtIODevice
*vdev
,
1219 uint32_t addr
, uint32_t data
)
1221 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1222 uint16_t val
= data
;
1224 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1228 stw_le_p(vdev
->config
+ addr
, val
);
1230 if (k
->set_config
) {
1231 k
->set_config(vdev
, vdev
->config
);
1235 void virtio_config_modern_writel(VirtIODevice
*vdev
,
1236 uint32_t addr
, uint32_t data
)
1238 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1239 uint32_t val
= data
;
1241 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1245 stl_le_p(vdev
->config
+ addr
, val
);
1247 if (k
->set_config
) {
1248 k
->set_config(vdev
, vdev
->config
);
1252 void virtio_queue_set_addr(VirtIODevice
*vdev
, int n
, hwaddr addr
)
1254 vdev
->vq
[n
].vring
.desc
= addr
;
1255 virtio_queue_update_rings(vdev
, n
);
1258 hwaddr
virtio_queue_get_addr(VirtIODevice
*vdev
, int n
)
1260 return vdev
->vq
[n
].vring
.desc
;
1263 void virtio_queue_set_rings(VirtIODevice
*vdev
, int n
, hwaddr desc
,
1264 hwaddr avail
, hwaddr used
)
1266 vdev
->vq
[n
].vring
.desc
= desc
;
1267 vdev
->vq
[n
].vring
.avail
= avail
;
1268 vdev
->vq
[n
].vring
.used
= used
;
1271 void virtio_queue_set_num(VirtIODevice
*vdev
, int n
, int num
)
1273 /* Don't allow guest to flip queue between existent and
1274 * nonexistent states, or to set it to an invalid size.
1276 if (!!num
!= !!vdev
->vq
[n
].vring
.num
||
1277 num
> VIRTQUEUE_MAX_SIZE
||
1281 vdev
->vq
[n
].vring
.num
= num
;
1284 VirtQueue
*virtio_vector_first_queue(VirtIODevice
*vdev
, uint16_t vector
)
1286 return QLIST_FIRST(&vdev
->vector_queues
[vector
]);
1289 VirtQueue
*virtio_vector_next_queue(VirtQueue
*vq
)
1291 return QLIST_NEXT(vq
, node
);
1294 int virtio_queue_get_num(VirtIODevice
*vdev
, int n
)
1296 return vdev
->vq
[n
].vring
.num
;
1299 int virtio_queue_get_max_num(VirtIODevice
*vdev
, int n
)
1301 return vdev
->vq
[n
].vring
.num_default
;
1304 int virtio_get_num_queues(VirtIODevice
*vdev
)
1308 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
1309 if (!virtio_queue_get_num(vdev
, i
)) {
1317 void virtio_queue_set_align(VirtIODevice
*vdev
, int n
, int align
)
1319 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1320 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1322 /* virtio-1 compliant devices cannot change the alignment */
1323 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
1324 error_report("tried to modify queue alignment for virtio-1 device");
1327 /* Check that the transport told us it was going to do this
1328 * (so a buggy transport will immediately assert rather than
1329 * silently failing to migrate this state)
1331 assert(k
->has_variable_vring_alignment
);
1333 vdev
->vq
[n
].vring
.align
= align
;
1334 virtio_queue_update_rings(vdev
, n
);
1337 static bool virtio_queue_notify_aio_vq(VirtQueue
*vq
)
1339 if (vq
->vring
.desc
&& vq
->handle_aio_output
) {
1340 VirtIODevice
*vdev
= vq
->vdev
;
1342 trace_virtio_queue_notify(vdev
, vq
- vdev
->vq
, vq
);
1343 return vq
->handle_aio_output(vdev
, vq
);
1349 static void virtio_queue_notify_vq(VirtQueue
*vq
)
1351 if (vq
->vring
.desc
&& vq
->handle_output
) {
1352 VirtIODevice
*vdev
= vq
->vdev
;
1354 if (unlikely(vdev
->broken
)) {
1358 trace_virtio_queue_notify(vdev
, vq
- vdev
->vq
, vq
);
1359 vq
->handle_output(vdev
, vq
);
1363 void virtio_queue_notify(VirtIODevice
*vdev
, int n
)
1365 virtio_queue_notify_vq(&vdev
->vq
[n
]);
1368 uint16_t virtio_queue_vector(VirtIODevice
*vdev
, int n
)
1370 return n
< VIRTIO_QUEUE_MAX
? vdev
->vq
[n
].vector
:
1374 void virtio_queue_set_vector(VirtIODevice
*vdev
, int n
, uint16_t vector
)
1376 VirtQueue
*vq
= &vdev
->vq
[n
];
1378 if (n
< VIRTIO_QUEUE_MAX
) {
1379 if (vdev
->vector_queues
&&
1380 vdev
->vq
[n
].vector
!= VIRTIO_NO_VECTOR
) {
1381 QLIST_REMOVE(vq
, node
);
1383 vdev
->vq
[n
].vector
= vector
;
1384 if (vdev
->vector_queues
&&
1385 vector
!= VIRTIO_NO_VECTOR
) {
1386 QLIST_INSERT_HEAD(&vdev
->vector_queues
[vector
], vq
, node
);
1391 VirtQueue
*virtio_add_queue(VirtIODevice
*vdev
, int queue_size
,
1392 VirtIOHandleOutput handle_output
)
1396 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
1397 if (vdev
->vq
[i
].vring
.num
== 0)
1401 if (i
== VIRTIO_QUEUE_MAX
|| queue_size
> VIRTQUEUE_MAX_SIZE
)
1404 vdev
->vq
[i
].vring
.num
= queue_size
;
1405 vdev
->vq
[i
].vring
.num_default
= queue_size
;
1406 vdev
->vq
[i
].vring
.align
= VIRTIO_PCI_VRING_ALIGN
;
1407 vdev
->vq
[i
].handle_output
= handle_output
;
1408 vdev
->vq
[i
].handle_aio_output
= NULL
;
1410 return &vdev
->vq
[i
];
1413 void virtio_del_queue(VirtIODevice
*vdev
, int n
)
1415 if (n
< 0 || n
>= VIRTIO_QUEUE_MAX
) {
1419 vdev
->vq
[n
].vring
.num
= 0;
1420 vdev
->vq
[n
].vring
.num_default
= 0;
1423 static void virtio_set_isr(VirtIODevice
*vdev
, int value
)
1425 uint8_t old
= atomic_read(&vdev
->isr
);
1427 /* Do not write ISR if it does not change, so that its cacheline remains
1428 * shared in the common case where the guest does not read it.
1430 if ((old
& value
) != value
) {
1431 atomic_or(&vdev
->isr
, value
);
1435 static bool virtio_should_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
1439 /* We need to expose used array entries before checking used event. */
1441 /* Always notify when queue is empty (when feature acknowledge) */
1442 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_NOTIFY_ON_EMPTY
) &&
1443 !vq
->inuse
&& virtio_queue_empty(vq
)) {
1447 if (!virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
1448 return !(vring_avail_flags(vq
) & VRING_AVAIL_F_NO_INTERRUPT
);
1451 v
= vq
->signalled_used_valid
;
1452 vq
->signalled_used_valid
= true;
1453 old
= vq
->signalled_used
;
1454 new = vq
->signalled_used
= vq
->used_idx
;
1455 return !v
|| vring_need_event(vring_get_used_event(vq
), new, old
);
1458 void virtio_notify_irqfd(VirtIODevice
*vdev
, VirtQueue
*vq
)
1460 if (!virtio_should_notify(vdev
, vq
)) {
1464 trace_virtio_notify_irqfd(vdev
, vq
);
1467 * virtio spec 1.0 says ISR bit 0 should be ignored with MSI, but
1468 * windows drivers included in virtio-win 1.8.0 (circa 2015) are
1469 * incorrectly polling this bit during crashdump and hibernation
1470 * in MSI mode, causing a hang if this bit is never updated.
1471 * Recent releases of Windows do not really shut down, but rather
1472 * log out and hibernate to make the next startup faster. Hence,
1473 * this manifested as a more serious hang during shutdown with
1475 * Next driver release from 2016 fixed this problem, so working around it
1476 * is not a must, but it's easy to do so let's do it here.
1478 * Note: it's safe to update ISR from any thread as it was switched
1479 * to an atomic operation.
1481 virtio_set_isr(vq
->vdev
, 0x1);
1482 event_notifier_set(&vq
->guest_notifier
);
1485 void virtio_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
1487 if (!virtio_should_notify(vdev
, vq
)) {
1491 trace_virtio_notify(vdev
, vq
);
1492 virtio_set_isr(vq
->vdev
, 0x1);
1493 virtio_notify_vector(vdev
, vq
->vector
);
1496 void virtio_notify_config(VirtIODevice
*vdev
)
1498 if (!(vdev
->status
& VIRTIO_CONFIG_S_DRIVER_OK
))
1501 virtio_set_isr(vdev
, 0x3);
1503 virtio_notify_vector(vdev
, vdev
->config_vector
);
1506 static bool virtio_device_endian_needed(void *opaque
)
1508 VirtIODevice
*vdev
= opaque
;
1510 assert(vdev
->device_endian
!= VIRTIO_DEVICE_ENDIAN_UNKNOWN
);
1511 if (!virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
1512 return vdev
->device_endian
!= virtio_default_endian();
1514 /* Devices conforming to VIRTIO 1.0 or later are always LE. */
1515 return vdev
->device_endian
!= VIRTIO_DEVICE_ENDIAN_LITTLE
;
1518 static bool virtio_64bit_features_needed(void *opaque
)
1520 VirtIODevice
*vdev
= opaque
;
1522 return (vdev
->host_features
>> 32) != 0;
1525 static bool virtio_virtqueue_needed(void *opaque
)
1527 VirtIODevice
*vdev
= opaque
;
1529 return virtio_host_has_feature(vdev
, VIRTIO_F_VERSION_1
);
1532 static bool virtio_ringsize_needed(void *opaque
)
1534 VirtIODevice
*vdev
= opaque
;
1537 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
1538 if (vdev
->vq
[i
].vring
.num
!= vdev
->vq
[i
].vring
.num_default
) {
1545 static bool virtio_extra_state_needed(void *opaque
)
1547 VirtIODevice
*vdev
= opaque
;
1548 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1549 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1551 return k
->has_extra_state
&&
1552 k
->has_extra_state(qbus
->parent
);
1555 static bool virtio_broken_needed(void *opaque
)
1557 VirtIODevice
*vdev
= opaque
;
1559 return vdev
->broken
;
1562 static const VMStateDescription vmstate_virtqueue
= {
1563 .name
= "virtqueue_state",
1565 .minimum_version_id
= 1,
1566 .fields
= (VMStateField
[]) {
1567 VMSTATE_UINT64(vring
.avail
, struct VirtQueue
),
1568 VMSTATE_UINT64(vring
.used
, struct VirtQueue
),
1569 VMSTATE_END_OF_LIST()
1573 static const VMStateDescription vmstate_virtio_virtqueues
= {
1574 .name
= "virtio/virtqueues",
1576 .minimum_version_id
= 1,
1577 .needed
= &virtio_virtqueue_needed
,
1578 .fields
= (VMStateField
[]) {
1579 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq
, struct VirtIODevice
,
1580 VIRTIO_QUEUE_MAX
, 0, vmstate_virtqueue
, VirtQueue
),
1581 VMSTATE_END_OF_LIST()
1585 static const VMStateDescription vmstate_ringsize
= {
1586 .name
= "ringsize_state",
1588 .minimum_version_id
= 1,
1589 .fields
= (VMStateField
[]) {
1590 VMSTATE_UINT32(vring
.num_default
, struct VirtQueue
),
1591 VMSTATE_END_OF_LIST()
1595 static const VMStateDescription vmstate_virtio_ringsize
= {
1596 .name
= "virtio/ringsize",
1598 .minimum_version_id
= 1,
1599 .needed
= &virtio_ringsize_needed
,
1600 .fields
= (VMStateField
[]) {
1601 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq
, struct VirtIODevice
,
1602 VIRTIO_QUEUE_MAX
, 0, vmstate_ringsize
, VirtQueue
),
1603 VMSTATE_END_OF_LIST()
1607 static int get_extra_state(QEMUFile
*f
, void *pv
, size_t size
,
1608 VMStateField
*field
)
1610 VirtIODevice
*vdev
= pv
;
1611 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1612 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1614 if (!k
->load_extra_state
) {
1617 return k
->load_extra_state(qbus
->parent
, f
);
1621 static int put_extra_state(QEMUFile
*f
, void *pv
, size_t size
,
1622 VMStateField
*field
, QJSON
*vmdesc
)
1624 VirtIODevice
*vdev
= pv
;
1625 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1626 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1628 k
->save_extra_state(qbus
->parent
, f
);
1632 static const VMStateInfo vmstate_info_extra_state
= {
1633 .name
= "virtqueue_extra_state",
1634 .get
= get_extra_state
,
1635 .put
= put_extra_state
,
1638 static const VMStateDescription vmstate_virtio_extra_state
= {
1639 .name
= "virtio/extra_state",
1641 .minimum_version_id
= 1,
1642 .needed
= &virtio_extra_state_needed
,
1643 .fields
= (VMStateField
[]) {
1645 .name
= "extra_state",
1647 .field_exists
= NULL
,
1649 .info
= &vmstate_info_extra_state
,
1650 .flags
= VMS_SINGLE
,
1653 VMSTATE_END_OF_LIST()
1657 static const VMStateDescription vmstate_virtio_device_endian
= {
1658 .name
= "virtio/device_endian",
1660 .minimum_version_id
= 1,
1661 .needed
= &virtio_device_endian_needed
,
1662 .fields
= (VMStateField
[]) {
1663 VMSTATE_UINT8(device_endian
, VirtIODevice
),
1664 VMSTATE_END_OF_LIST()
1668 static const VMStateDescription vmstate_virtio_64bit_features
= {
1669 .name
= "virtio/64bit_features",
1671 .minimum_version_id
= 1,
1672 .needed
= &virtio_64bit_features_needed
,
1673 .fields
= (VMStateField
[]) {
1674 VMSTATE_UINT64(guest_features
, VirtIODevice
),
1675 VMSTATE_END_OF_LIST()
1679 static const VMStateDescription vmstate_virtio_broken
= {
1680 .name
= "virtio/broken",
1682 .minimum_version_id
= 1,
1683 .needed
= &virtio_broken_needed
,
1684 .fields
= (VMStateField
[]) {
1685 VMSTATE_BOOL(broken
, VirtIODevice
),
1686 VMSTATE_END_OF_LIST()
1690 static const VMStateDescription vmstate_virtio
= {
1693 .minimum_version_id
= 1,
1694 .minimum_version_id_old
= 1,
1695 .fields
= (VMStateField
[]) {
1696 VMSTATE_END_OF_LIST()
1698 .subsections
= (const VMStateDescription
*[]) {
1699 &vmstate_virtio_device_endian
,
1700 &vmstate_virtio_64bit_features
,
1701 &vmstate_virtio_virtqueues
,
1702 &vmstate_virtio_ringsize
,
1703 &vmstate_virtio_broken
,
1704 &vmstate_virtio_extra_state
,
1709 void virtio_save(VirtIODevice
*vdev
, QEMUFile
*f
)
1711 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1712 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1713 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1714 uint32_t guest_features_lo
= (vdev
->guest_features
& 0xffffffff);
1717 if (k
->save_config
) {
1718 k
->save_config(qbus
->parent
, f
);
1721 qemu_put_8s(f
, &vdev
->status
);
1722 qemu_put_8s(f
, &vdev
->isr
);
1723 qemu_put_be16s(f
, &vdev
->queue_sel
);
1724 qemu_put_be32s(f
, &guest_features_lo
);
1725 qemu_put_be32(f
, vdev
->config_len
);
1726 qemu_put_buffer(f
, vdev
->config
, vdev
->config_len
);
1728 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
1729 if (vdev
->vq
[i
].vring
.num
== 0)
1733 qemu_put_be32(f
, i
);
1735 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
1736 if (vdev
->vq
[i
].vring
.num
== 0)
1739 qemu_put_be32(f
, vdev
->vq
[i
].vring
.num
);
1740 if (k
->has_variable_vring_alignment
) {
1741 qemu_put_be32(f
, vdev
->vq
[i
].vring
.align
);
1743 /* XXX virtio-1 devices */
1744 qemu_put_be64(f
, vdev
->vq
[i
].vring
.desc
);
1745 qemu_put_be16s(f
, &vdev
->vq
[i
].last_avail_idx
);
1746 if (k
->save_queue
) {
1747 k
->save_queue(qbus
->parent
, i
, f
);
1751 if (vdc
->save
!= NULL
) {
1756 vmstate_save_state(f
, vdc
->vmsd
, vdev
, NULL
);
1760 vmstate_save_state(f
, &vmstate_virtio
, vdev
, NULL
);
1763 /* A wrapper for use as a VMState .put function */
1764 static int virtio_device_put(QEMUFile
*f
, void *opaque
, size_t size
,
1765 VMStateField
*field
, QJSON
*vmdesc
)
1767 virtio_save(VIRTIO_DEVICE(opaque
), f
);
1772 /* A wrapper for use as a VMState .get function */
1773 static int virtio_device_get(QEMUFile
*f
, void *opaque
, size_t size
,
1774 VMStateField
*field
)
1776 VirtIODevice
*vdev
= VIRTIO_DEVICE(opaque
);
1777 DeviceClass
*dc
= DEVICE_CLASS(VIRTIO_DEVICE_GET_CLASS(vdev
));
1779 return virtio_load(vdev
, f
, dc
->vmsd
->version_id
);
1782 const VMStateInfo virtio_vmstate_info
= {
1784 .get
= virtio_device_get
,
1785 .put
= virtio_device_put
,
1788 static int virtio_set_features_nocheck(VirtIODevice
*vdev
, uint64_t val
)
1790 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1791 bool bad
= (val
& ~(vdev
->host_features
)) != 0;
1793 val
&= vdev
->host_features
;
1794 if (k
->set_features
) {
1795 k
->set_features(vdev
, val
);
1797 vdev
->guest_features
= val
;
1798 return bad
? -1 : 0;
1801 int virtio_set_features(VirtIODevice
*vdev
, uint64_t val
)
1804 * The driver must not attempt to set features after feature negotiation
1807 if (vdev
->status
& VIRTIO_CONFIG_S_FEATURES_OK
) {
1810 return virtio_set_features_nocheck(vdev
, val
);
1813 int virtio_load(VirtIODevice
*vdev
, QEMUFile
*f
, int version_id
)
1819 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1820 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1821 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1824 * We poison the endianness to ensure it does not get used before
1825 * subsections have been loaded.
1827 vdev
->device_endian
= VIRTIO_DEVICE_ENDIAN_UNKNOWN
;
1829 if (k
->load_config
) {
1830 ret
= k
->load_config(qbus
->parent
, f
);
1835 qemu_get_8s(f
, &vdev
->status
);
1836 qemu_get_8s(f
, &vdev
->isr
);
1837 qemu_get_be16s(f
, &vdev
->queue_sel
);
1838 if (vdev
->queue_sel
>= VIRTIO_QUEUE_MAX
) {
1841 qemu_get_be32s(f
, &features
);
1844 * Temporarily set guest_features low bits - needed by
1845 * virtio net load code testing for VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
1846 * VIRTIO_NET_F_GUEST_ANNOUNCE and VIRTIO_NET_F_CTRL_VQ.
1848 * Note: devices should always test host features in future - don't create
1849 * new dependencies like this.
1851 vdev
->guest_features
= features
;
1853 config_len
= qemu_get_be32(f
);
1856 * There are cases where the incoming config can be bigger or smaller
1857 * than what we have; so load what we have space for, and skip
1858 * any excess that's in the stream.
1860 qemu_get_buffer(f
, vdev
->config
, MIN(config_len
, vdev
->config_len
));
1862 while (config_len
> vdev
->config_len
) {
1867 num
= qemu_get_be32(f
);
1869 if (num
> VIRTIO_QUEUE_MAX
) {
1870 error_report("Invalid number of virtqueues: 0x%x", num
);
1874 for (i
= 0; i
< num
; i
++) {
1875 vdev
->vq
[i
].vring
.num
= qemu_get_be32(f
);
1876 if (k
->has_variable_vring_alignment
) {
1877 vdev
->vq
[i
].vring
.align
= qemu_get_be32(f
);
1879 vdev
->vq
[i
].vring
.desc
= qemu_get_be64(f
);
1880 qemu_get_be16s(f
, &vdev
->vq
[i
].last_avail_idx
);
1881 vdev
->vq
[i
].signalled_used_valid
= false;
1882 vdev
->vq
[i
].notification
= true;
1884 if (vdev
->vq
[i
].vring
.desc
) {
1885 /* XXX virtio-1 devices */
1886 virtio_queue_update_rings(vdev
, i
);
1887 } else if (vdev
->vq
[i
].last_avail_idx
) {
1888 error_report("VQ %d address 0x0 "
1889 "inconsistent with Host index 0x%x",
1890 i
, vdev
->vq
[i
].last_avail_idx
);
1893 if (k
->load_queue
) {
1894 ret
= k
->load_queue(qbus
->parent
, i
, f
);
1900 virtio_notify_vector(vdev
, VIRTIO_NO_VECTOR
);
1902 if (vdc
->load
!= NULL
) {
1903 ret
= vdc
->load(vdev
, f
, version_id
);
1910 ret
= vmstate_load_state(f
, vdc
->vmsd
, vdev
, version_id
);
1917 ret
= vmstate_load_state(f
, &vmstate_virtio
, vdev
, 1);
1922 if (vdev
->device_endian
== VIRTIO_DEVICE_ENDIAN_UNKNOWN
) {
1923 vdev
->device_endian
= virtio_default_endian();
1926 if (virtio_64bit_features_needed(vdev
)) {
1928 * Subsection load filled vdev->guest_features. Run them
1929 * through virtio_set_features to sanity-check them against
1932 uint64_t features64
= vdev
->guest_features
;
1933 if (virtio_set_features_nocheck(vdev
, features64
) < 0) {
1934 error_report("Features 0x%" PRIx64
" unsupported. "
1935 "Allowed features: 0x%" PRIx64
,
1936 features64
, vdev
->host_features
);
1940 if (virtio_set_features_nocheck(vdev
, features
) < 0) {
1941 error_report("Features 0x%x unsupported. "
1942 "Allowed features: 0x%" PRIx64
,
1943 features
, vdev
->host_features
);
1948 for (i
= 0; i
< num
; i
++) {
1949 if (vdev
->vq
[i
].vring
.desc
) {
1951 nheads
= vring_avail_idx(&vdev
->vq
[i
]) - vdev
->vq
[i
].last_avail_idx
;
1952 /* Check it isn't doing strange things with descriptor numbers. */
1953 if (nheads
> vdev
->vq
[i
].vring
.num
) {
1954 error_report("VQ %d size 0x%x Guest index 0x%x "
1955 "inconsistent with Host index 0x%x: delta 0x%x",
1956 i
, vdev
->vq
[i
].vring
.num
,
1957 vring_avail_idx(&vdev
->vq
[i
]),
1958 vdev
->vq
[i
].last_avail_idx
, nheads
);
1961 vdev
->vq
[i
].used_idx
= vring_used_idx(&vdev
->vq
[i
]);
1962 vdev
->vq
[i
].shadow_avail_idx
= vring_avail_idx(&vdev
->vq
[i
]);
1965 * Some devices migrate VirtQueueElements that have been popped
1966 * from the avail ring but not yet returned to the used ring.
1967 * Since max ring size < UINT16_MAX it's safe to use modulo
1968 * UINT16_MAX + 1 subtraction.
1970 vdev
->vq
[i
].inuse
= (uint16_t)(vdev
->vq
[i
].last_avail_idx
-
1971 vdev
->vq
[i
].used_idx
);
1972 if (vdev
->vq
[i
].inuse
> vdev
->vq
[i
].vring
.num
) {
1973 error_report("VQ %d size 0x%x < last_avail_idx 0x%x - "
1975 i
, vdev
->vq
[i
].vring
.num
,
1976 vdev
->vq
[i
].last_avail_idx
,
1977 vdev
->vq
[i
].used_idx
);
1986 void virtio_cleanup(VirtIODevice
*vdev
)
1988 qemu_del_vm_change_state_handler(vdev
->vmstate
);
1989 g_free(vdev
->config
);
1991 g_free(vdev
->vector_queues
);
1994 static void virtio_vmstate_change(void *opaque
, int running
, RunState state
)
1996 VirtIODevice
*vdev
= opaque
;
1997 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1998 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1999 bool backend_run
= running
&& (vdev
->status
& VIRTIO_CONFIG_S_DRIVER_OK
);
2000 vdev
->vm_running
= running
;
2003 virtio_set_status(vdev
, vdev
->status
);
2006 if (k
->vmstate_change
) {
2007 k
->vmstate_change(qbus
->parent
, backend_run
);
2011 virtio_set_status(vdev
, vdev
->status
);
2015 void virtio_instance_init_common(Object
*proxy_obj
, void *data
,
2016 size_t vdev_size
, const char *vdev_name
)
2018 DeviceState
*vdev
= data
;
2020 object_initialize(vdev
, vdev_size
, vdev_name
);
2021 object_property_add_child(proxy_obj
, "virtio-backend", OBJECT(vdev
), NULL
);
2022 object_unref(OBJECT(vdev
));
2023 qdev_alias_all_properties(vdev
, proxy_obj
);
2026 void virtio_init(VirtIODevice
*vdev
, const char *name
,
2027 uint16_t device_id
, size_t config_size
)
2029 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2030 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2032 int nvectors
= k
->query_nvectors
? k
->query_nvectors(qbus
->parent
) : 0;
2035 vdev
->vector_queues
=
2036 g_malloc0(sizeof(*vdev
->vector_queues
) * nvectors
);
2039 vdev
->device_id
= device_id
;
2041 atomic_set(&vdev
->isr
, 0);
2042 vdev
->queue_sel
= 0;
2043 vdev
->config_vector
= VIRTIO_NO_VECTOR
;
2044 vdev
->vq
= g_malloc0(sizeof(VirtQueue
) * VIRTIO_QUEUE_MAX
);
2045 vdev
->vm_running
= runstate_is_running();
2046 vdev
->broken
= false;
2047 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2048 vdev
->vq
[i
].vector
= VIRTIO_NO_VECTOR
;
2049 vdev
->vq
[i
].vdev
= vdev
;
2050 vdev
->vq
[i
].queue_index
= i
;
2054 vdev
->config_len
= config_size
;
2055 if (vdev
->config_len
) {
2056 vdev
->config
= g_malloc0(config_size
);
2058 vdev
->config
= NULL
;
2060 vdev
->vmstate
= qemu_add_vm_change_state_handler(virtio_vmstate_change
,
2062 vdev
->device_endian
= virtio_default_endian();
2063 vdev
->use_guest_notifier_mask
= true;
2066 hwaddr
virtio_queue_get_desc_addr(VirtIODevice
*vdev
, int n
)
2068 return vdev
->vq
[n
].vring
.desc
;
2071 hwaddr
virtio_queue_get_avail_addr(VirtIODevice
*vdev
, int n
)
2073 return vdev
->vq
[n
].vring
.avail
;
2076 hwaddr
virtio_queue_get_used_addr(VirtIODevice
*vdev
, int n
)
2078 return vdev
->vq
[n
].vring
.used
;
2081 hwaddr
virtio_queue_get_desc_size(VirtIODevice
*vdev
, int n
)
2083 return sizeof(VRingDesc
) * vdev
->vq
[n
].vring
.num
;
2086 hwaddr
virtio_queue_get_avail_size(VirtIODevice
*vdev
, int n
)
2088 return offsetof(VRingAvail
, ring
) +
2089 sizeof(uint16_t) * vdev
->vq
[n
].vring
.num
;
2092 hwaddr
virtio_queue_get_used_size(VirtIODevice
*vdev
, int n
)
2094 return offsetof(VRingUsed
, ring
) +
2095 sizeof(VRingUsedElem
) * vdev
->vq
[n
].vring
.num
;
2098 uint16_t virtio_queue_get_last_avail_idx(VirtIODevice
*vdev
, int n
)
2100 return vdev
->vq
[n
].last_avail_idx
;
2103 void virtio_queue_set_last_avail_idx(VirtIODevice
*vdev
, int n
, uint16_t idx
)
2105 vdev
->vq
[n
].last_avail_idx
= idx
;
2106 vdev
->vq
[n
].shadow_avail_idx
= idx
;
2109 void virtio_queue_update_used_idx(VirtIODevice
*vdev
, int n
)
2111 vdev
->vq
[n
].used_idx
= vring_used_idx(&vdev
->vq
[n
]);
2114 void virtio_queue_invalidate_signalled_used(VirtIODevice
*vdev
, int n
)
2116 vdev
->vq
[n
].signalled_used_valid
= false;
2119 VirtQueue
*virtio_get_queue(VirtIODevice
*vdev
, int n
)
2121 return vdev
->vq
+ n
;
2124 uint16_t virtio_get_queue_index(VirtQueue
*vq
)
2126 return vq
->queue_index
;
2129 static void virtio_queue_guest_notifier_read(EventNotifier
*n
)
2131 VirtQueue
*vq
= container_of(n
, VirtQueue
, guest_notifier
);
2132 if (event_notifier_test_and_clear(n
)) {
2133 virtio_notify_vector(vq
->vdev
, vq
->vector
);
2137 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue
*vq
, bool assign
,
2140 if (assign
&& !with_irqfd
) {
2141 event_notifier_set_handler(&vq
->guest_notifier
,
2142 virtio_queue_guest_notifier_read
);
2144 event_notifier_set_handler(&vq
->guest_notifier
, NULL
);
2147 /* Test and clear notifier before closing it,
2148 * in case poll callback didn't have time to run. */
2149 virtio_queue_guest_notifier_read(&vq
->guest_notifier
);
2153 EventNotifier
*virtio_queue_get_guest_notifier(VirtQueue
*vq
)
2155 return &vq
->guest_notifier
;
2158 static void virtio_queue_host_notifier_aio_read(EventNotifier
*n
)
2160 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
2161 if (event_notifier_test_and_clear(n
)) {
2162 virtio_queue_notify_aio_vq(vq
);
2166 static void virtio_queue_host_notifier_aio_poll_begin(EventNotifier
*n
)
2168 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
2170 virtio_queue_set_notification(vq
, 0);
2173 static bool virtio_queue_host_notifier_aio_poll(void *opaque
)
2175 EventNotifier
*n
= opaque
;
2176 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
2179 if (virtio_queue_empty(vq
)) {
2183 progress
= virtio_queue_notify_aio_vq(vq
);
2185 /* In case the handler function re-enabled notifications */
2186 virtio_queue_set_notification(vq
, 0);
2190 static void virtio_queue_host_notifier_aio_poll_end(EventNotifier
*n
)
2192 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
2194 /* Caller polls once more after this to catch requests that race with us */
2195 virtio_queue_set_notification(vq
, 1);
2198 void virtio_queue_aio_set_host_notifier_handler(VirtQueue
*vq
, AioContext
*ctx
,
2199 VirtIOHandleAIOOutput handle_output
)
2201 if (handle_output
) {
2202 vq
->handle_aio_output
= handle_output
;
2203 aio_set_event_notifier(ctx
, &vq
->host_notifier
, true,
2204 virtio_queue_host_notifier_aio_read
,
2205 virtio_queue_host_notifier_aio_poll
);
2206 aio_set_event_notifier_poll(ctx
, &vq
->host_notifier
,
2207 virtio_queue_host_notifier_aio_poll_begin
,
2208 virtio_queue_host_notifier_aio_poll_end
);
2210 aio_set_event_notifier(ctx
, &vq
->host_notifier
, true, NULL
, NULL
);
2211 /* Test and clear notifier before after disabling event,
2212 * in case poll callback didn't have time to run. */
2213 virtio_queue_host_notifier_aio_read(&vq
->host_notifier
);
2214 vq
->handle_aio_output
= NULL
;
2218 void virtio_queue_host_notifier_read(EventNotifier
*n
)
2220 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
2221 if (event_notifier_test_and_clear(n
)) {
2222 virtio_queue_notify_vq(vq
);
2226 EventNotifier
*virtio_queue_get_host_notifier(VirtQueue
*vq
)
2228 return &vq
->host_notifier
;
2231 void virtio_device_set_child_bus_name(VirtIODevice
*vdev
, char *bus_name
)
2233 g_free(vdev
->bus_name
);
2234 vdev
->bus_name
= g_strdup(bus_name
);
2237 void GCC_FMT_ATTR(2, 3) virtio_error(VirtIODevice
*vdev
, const char *fmt
, ...)
2242 error_vreport(fmt
, ap
);
2245 vdev
->broken
= true;
2247 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
2248 virtio_set_status(vdev
, vdev
->status
| VIRTIO_CONFIG_S_NEEDS_RESET
);
2249 virtio_notify_config(vdev
);
2253 static void virtio_device_realize(DeviceState
*dev
, Error
**errp
)
2255 VirtIODevice
*vdev
= VIRTIO_DEVICE(dev
);
2256 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(dev
);
2259 /* Devices should either use vmsd or the load/save methods */
2260 assert(!vdc
->vmsd
|| !vdc
->load
);
2262 if (vdc
->realize
!= NULL
) {
2263 vdc
->realize(dev
, &err
);
2265 error_propagate(errp
, err
);
2270 virtio_bus_device_plugged(vdev
, &err
);
2272 error_propagate(errp
, err
);
2277 static void virtio_device_unrealize(DeviceState
*dev
, Error
**errp
)
2279 VirtIODevice
*vdev
= VIRTIO_DEVICE(dev
);
2280 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(dev
);
2283 virtio_bus_device_unplugged(vdev
);
2285 if (vdc
->unrealize
!= NULL
) {
2286 vdc
->unrealize(dev
, &err
);
2288 error_propagate(errp
, err
);
2293 g_free(vdev
->bus_name
);
2294 vdev
->bus_name
= NULL
;
2297 static Property virtio_properties
[] = {
2298 DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice
, host_features
),
2299 DEFINE_PROP_END_OF_LIST(),
2302 static int virtio_device_start_ioeventfd_impl(VirtIODevice
*vdev
)
2304 VirtioBusState
*qbus
= VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev
)));
2307 for (n
= 0; n
< VIRTIO_QUEUE_MAX
; n
++) {
2308 VirtQueue
*vq
= &vdev
->vq
[n
];
2309 if (!virtio_queue_get_num(vdev
, n
)) {
2312 r
= virtio_bus_set_host_notifier(qbus
, n
, true);
2317 event_notifier_set_handler(&vq
->host_notifier
,
2318 virtio_queue_host_notifier_read
);
2321 for (n
= 0; n
< VIRTIO_QUEUE_MAX
; n
++) {
2322 /* Kick right away to begin processing requests already in vring */
2323 VirtQueue
*vq
= &vdev
->vq
[n
];
2324 if (!vq
->vring
.num
) {
2327 event_notifier_set(&vq
->host_notifier
);
2333 VirtQueue
*vq
= &vdev
->vq
[n
];
2334 if (!virtio_queue_get_num(vdev
, n
)) {
2338 event_notifier_set_handler(&vq
->host_notifier
, NULL
);
2339 r
= virtio_bus_set_host_notifier(qbus
, n
, false);
2345 int virtio_device_start_ioeventfd(VirtIODevice
*vdev
)
2347 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2348 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
2350 return virtio_bus_start_ioeventfd(vbus
);
2353 static void virtio_device_stop_ioeventfd_impl(VirtIODevice
*vdev
)
2355 VirtioBusState
*qbus
= VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev
)));
2358 for (n
= 0; n
< VIRTIO_QUEUE_MAX
; n
++) {
2359 VirtQueue
*vq
= &vdev
->vq
[n
];
2361 if (!virtio_queue_get_num(vdev
, n
)) {
2364 event_notifier_set_handler(&vq
->host_notifier
, NULL
);
2365 r
= virtio_bus_set_host_notifier(qbus
, n
, false);
2370 void virtio_device_stop_ioeventfd(VirtIODevice
*vdev
)
2372 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2373 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
2375 virtio_bus_stop_ioeventfd(vbus
);
2378 int virtio_device_grab_ioeventfd(VirtIODevice
*vdev
)
2380 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2381 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
2383 return virtio_bus_grab_ioeventfd(vbus
);
2386 void virtio_device_release_ioeventfd(VirtIODevice
*vdev
)
2388 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2389 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
2391 virtio_bus_release_ioeventfd(vbus
);
2394 static void virtio_device_class_init(ObjectClass
*klass
, void *data
)
2396 /* Set the default value here. */
2397 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_CLASS(klass
);
2398 DeviceClass
*dc
= DEVICE_CLASS(klass
);
2400 dc
->realize
= virtio_device_realize
;
2401 dc
->unrealize
= virtio_device_unrealize
;
2402 dc
->bus_type
= TYPE_VIRTIO_BUS
;
2403 dc
->props
= virtio_properties
;
2404 vdc
->start_ioeventfd
= virtio_device_start_ioeventfd_impl
;
2405 vdc
->stop_ioeventfd
= virtio_device_stop_ioeventfd_impl
;
2407 vdc
->legacy_features
|= VIRTIO_LEGACY_FEATURES
;
2410 bool virtio_device_ioeventfd_enabled(VirtIODevice
*vdev
)
2412 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2413 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
2415 return virtio_bus_ioeventfd_enabled(vbus
);
2418 static const TypeInfo virtio_device_info
= {
2419 .name
= TYPE_VIRTIO_DEVICE
,
2420 .parent
= TYPE_DEVICE
,
2421 .instance_size
= sizeof(VirtIODevice
),
2422 .class_init
= virtio_device_class_init
,
2424 .class_size
= sizeof(VirtioDeviceClass
),
2427 static void virtio_register_types(void)
2429 type_register_static(&virtio_device_info
);
2432 type_init(virtio_register_types
)