4 * Copyright IBM, Corp. 2007
7 * Anthony Liguori <aliguori@us.ibm.com>
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
14 #include "qemu/osdep.h"
15 #include "qapi/error.h"
16 #include "qemu-common.h"
19 #include "exec/address-spaces.h"
20 #include "qemu/error-report.h"
21 #include "hw/virtio/virtio.h"
22 #include "qemu/atomic.h"
23 #include "hw/virtio/virtio-bus.h"
24 #include "migration/migration.h"
25 #include "hw/virtio/virtio-access.h"
28 * The alignment to use between consumer and producer parts of vring.
29 * x86 pagesize again. This is the default, used by transports like PCI
30 * which don't provide a means for the guest to tell the host the alignment.
32 #define VIRTIO_PCI_VRING_ALIGN 4096
34 typedef struct VRingDesc
42 typedef struct VRingAvail
49 typedef struct VRingUsedElem
55 typedef struct VRingUsed
59 VRingUsedElem ring
[0];
65 unsigned int num_default
;
76 /* Next head to pop */
77 uint16_t last_avail_idx
;
79 /* Last avail_idx read from VQ. */
80 uint16_t shadow_avail_idx
;
84 /* Last used index value we have signalled on */
85 uint16_t signalled_used
;
87 /* Last used index value we have signalled on */
88 bool signalled_used_valid
;
90 /* Notification enabled? */
98 VirtIOHandleOutput handle_output
;
99 VirtIOHandleOutput handle_aio_output
;
102 EventNotifier guest_notifier
;
103 EventNotifier host_notifier
;
104 QLIST_ENTRY(VirtQueue
) node
;
107 /* virt queue functions */
108 void virtio_queue_update_rings(VirtIODevice
*vdev
, int n
)
110 VRing
*vring
= &vdev
->vq
[n
].vring
;
113 /* not yet setup -> nothing to do */
116 vring
->avail
= vring
->desc
+ vring
->num
* sizeof(VRingDesc
);
117 vring
->used
= vring_align(vring
->avail
+
118 offsetof(VRingAvail
, ring
[vring
->num
]),
122 static void vring_desc_read(VirtIODevice
*vdev
, VRingDesc
*desc
,
123 hwaddr desc_pa
, int i
)
125 address_space_read(&address_space_memory
, desc_pa
+ i
* sizeof(VRingDesc
),
126 MEMTXATTRS_UNSPECIFIED
, (void *)desc
, sizeof(VRingDesc
));
127 virtio_tswap64s(vdev
, &desc
->addr
);
128 virtio_tswap32s(vdev
, &desc
->len
);
129 virtio_tswap16s(vdev
, &desc
->flags
);
130 virtio_tswap16s(vdev
, &desc
->next
);
133 static inline uint16_t vring_avail_flags(VirtQueue
*vq
)
136 pa
= vq
->vring
.avail
+ offsetof(VRingAvail
, flags
);
137 return virtio_lduw_phys(vq
->vdev
, pa
);
140 static inline uint16_t vring_avail_idx(VirtQueue
*vq
)
143 pa
= vq
->vring
.avail
+ offsetof(VRingAvail
, idx
);
144 vq
->shadow_avail_idx
= virtio_lduw_phys(vq
->vdev
, pa
);
145 return vq
->shadow_avail_idx
;
148 static inline uint16_t vring_avail_ring(VirtQueue
*vq
, int i
)
151 pa
= vq
->vring
.avail
+ offsetof(VRingAvail
, ring
[i
]);
152 return virtio_lduw_phys(vq
->vdev
, pa
);
155 static inline uint16_t vring_get_used_event(VirtQueue
*vq
)
157 return vring_avail_ring(vq
, vq
->vring
.num
);
160 static inline void vring_used_write(VirtQueue
*vq
, VRingUsedElem
*uelem
,
164 virtio_tswap32s(vq
->vdev
, &uelem
->id
);
165 virtio_tswap32s(vq
->vdev
, &uelem
->len
);
166 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, ring
[i
]);
167 address_space_write(&address_space_memory
, pa
, MEMTXATTRS_UNSPECIFIED
,
168 (void *)uelem
, sizeof(VRingUsedElem
));
171 static uint16_t vring_used_idx(VirtQueue
*vq
)
174 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, idx
);
175 return virtio_lduw_phys(vq
->vdev
, pa
);
178 static inline void vring_used_idx_set(VirtQueue
*vq
, uint16_t val
)
181 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, idx
);
182 virtio_stw_phys(vq
->vdev
, pa
, val
);
186 static inline void vring_used_flags_set_bit(VirtQueue
*vq
, int mask
)
188 VirtIODevice
*vdev
= vq
->vdev
;
190 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, flags
);
191 virtio_stw_phys(vdev
, pa
, virtio_lduw_phys(vdev
, pa
) | mask
);
194 static inline void vring_used_flags_unset_bit(VirtQueue
*vq
, int mask
)
196 VirtIODevice
*vdev
= vq
->vdev
;
198 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, flags
);
199 virtio_stw_phys(vdev
, pa
, virtio_lduw_phys(vdev
, pa
) & ~mask
);
202 static inline void vring_set_avail_event(VirtQueue
*vq
, uint16_t val
)
205 if (!vq
->notification
) {
208 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, ring
[vq
->vring
.num
]);
209 virtio_stw_phys(vq
->vdev
, pa
, val
);
212 void virtio_queue_set_notification(VirtQueue
*vq
, int enable
)
214 vq
->notification
= enable
;
215 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
216 vring_set_avail_event(vq
, vring_avail_idx(vq
));
218 vring_used_flags_unset_bit(vq
, VRING_USED_F_NO_NOTIFY
);
220 vring_used_flags_set_bit(vq
, VRING_USED_F_NO_NOTIFY
);
223 /* Expose avail event/used flags before caller checks the avail idx. */
228 int virtio_queue_ready(VirtQueue
*vq
)
230 return vq
->vring
.avail
!= 0;
233 /* Fetch avail_idx from VQ memory only when we really need to know if
234 * guest has added some buffers. */
235 int virtio_queue_empty(VirtQueue
*vq
)
237 if (vq
->shadow_avail_idx
!= vq
->last_avail_idx
) {
241 return vring_avail_idx(vq
) == vq
->last_avail_idx
;
244 static void virtqueue_unmap_sg(VirtQueue
*vq
, const VirtQueueElement
*elem
,
251 for (i
= 0; i
< elem
->in_num
; i
++) {
252 size_t size
= MIN(len
- offset
, elem
->in_sg
[i
].iov_len
);
254 cpu_physical_memory_unmap(elem
->in_sg
[i
].iov_base
,
255 elem
->in_sg
[i
].iov_len
,
261 for (i
= 0; i
< elem
->out_num
; i
++)
262 cpu_physical_memory_unmap(elem
->out_sg
[i
].iov_base
,
263 elem
->out_sg
[i
].iov_len
,
264 0, elem
->out_sg
[i
].iov_len
);
267 void virtqueue_discard(VirtQueue
*vq
, const VirtQueueElement
*elem
,
270 vq
->last_avail_idx
--;
272 virtqueue_unmap_sg(vq
, elem
, len
);
276 * @vq: The #VirtQueue
277 * @num: Number of elements to push back
279 * Pretend that elements weren't popped from the virtqueue. The next
280 * virtqueue_pop() will refetch the oldest element.
282 * Use virtqueue_discard() instead if you have a VirtQueueElement.
284 * Returns: true on success, false if @num is greater than the number of in use
287 bool virtqueue_rewind(VirtQueue
*vq
, unsigned int num
)
289 if (num
> vq
->inuse
) {
292 vq
->last_avail_idx
-= num
;
297 void virtqueue_fill(VirtQueue
*vq
, const VirtQueueElement
*elem
,
298 unsigned int len
, unsigned int idx
)
302 trace_virtqueue_fill(vq
, elem
, len
, idx
);
304 virtqueue_unmap_sg(vq
, elem
, len
);
306 if (unlikely(vq
->vdev
->broken
)) {
310 idx
= (idx
+ vq
->used_idx
) % vq
->vring
.num
;
312 uelem
.id
= elem
->index
;
314 vring_used_write(vq
, &uelem
, idx
);
317 void virtqueue_flush(VirtQueue
*vq
, unsigned int count
)
321 if (unlikely(vq
->vdev
->broken
)) {
326 /* Make sure buffer is written before we update index. */
328 trace_virtqueue_flush(vq
, count
);
331 vring_used_idx_set(vq
, new);
333 if (unlikely((int16_t)(new - vq
->signalled_used
) < (uint16_t)(new - old
)))
334 vq
->signalled_used_valid
= false;
337 void virtqueue_push(VirtQueue
*vq
, const VirtQueueElement
*elem
,
340 virtqueue_fill(vq
, elem
, len
, 0);
341 virtqueue_flush(vq
, 1);
344 static int virtqueue_num_heads(VirtQueue
*vq
, unsigned int idx
)
346 uint16_t num_heads
= vring_avail_idx(vq
) - idx
;
348 /* Check it isn't doing very strange things with descriptor numbers. */
349 if (num_heads
> vq
->vring
.num
) {
350 virtio_error(vq
->vdev
, "Guest moved used index from %u to %u",
351 idx
, vq
->shadow_avail_idx
);
354 /* On success, callers read a descriptor at vq->last_avail_idx.
355 * Make sure descriptor read does not bypass avail index read. */
363 static bool virtqueue_get_head(VirtQueue
*vq
, unsigned int idx
,
366 /* Grab the next descriptor number they're advertising, and increment
367 * the index we've seen. */
368 *head
= vring_avail_ring(vq
, idx
% vq
->vring
.num
);
370 /* If their number is silly, that's a fatal mistake. */
371 if (*head
>= vq
->vring
.num
) {
372 virtio_error(vq
->vdev
, "Guest says index %u is available", *head
);
380 VIRTQUEUE_READ_DESC_ERROR
= -1,
381 VIRTQUEUE_READ_DESC_DONE
= 0, /* end of chain */
382 VIRTQUEUE_READ_DESC_MORE
= 1, /* more buffers in chain */
385 static int virtqueue_read_next_desc(VirtIODevice
*vdev
, VRingDesc
*desc
,
386 hwaddr desc_pa
, unsigned int max
,
389 /* If this descriptor says it doesn't chain, we're done. */
390 if (!(desc
->flags
& VRING_DESC_F_NEXT
)) {
391 return VIRTQUEUE_READ_DESC_DONE
;
394 /* Check they're not leading us off end of descriptors. */
396 /* Make sure compiler knows to grab that: we don't want it changing! */
400 virtio_error(vdev
, "Desc next is %u", *next
);
401 return VIRTQUEUE_READ_DESC_ERROR
;
404 vring_desc_read(vdev
, desc
, desc_pa
, *next
);
405 return VIRTQUEUE_READ_DESC_MORE
;
408 void virtqueue_get_avail_bytes(VirtQueue
*vq
, unsigned int *in_bytes
,
409 unsigned int *out_bytes
,
410 unsigned max_in_bytes
, unsigned max_out_bytes
)
413 unsigned int total_bufs
, in_total
, out_total
;
416 idx
= vq
->last_avail_idx
;
418 total_bufs
= in_total
= out_total
= 0;
419 while ((rc
= virtqueue_num_heads(vq
, idx
)) > 0) {
420 VirtIODevice
*vdev
= vq
->vdev
;
421 unsigned int max
, num_bufs
, indirect
= 0;
427 num_bufs
= total_bufs
;
429 if (!virtqueue_get_head(vq
, idx
++, &i
)) {
433 desc_pa
= vq
->vring
.desc
;
434 vring_desc_read(vdev
, &desc
, desc_pa
, i
);
436 if (desc
.flags
& VRING_DESC_F_INDIRECT
) {
437 if (desc
.len
% sizeof(VRingDesc
)) {
438 virtio_error(vdev
, "Invalid size for indirect buffer table");
442 /* If we've got too many, that implies a descriptor loop. */
443 if (num_bufs
>= max
) {
444 virtio_error(vdev
, "Looped descriptor");
448 /* loop over the indirect descriptor table */
450 max
= desc
.len
/ sizeof(VRingDesc
);
453 vring_desc_read(vdev
, &desc
, desc_pa
, i
);
457 /* If we've got too many, that implies a descriptor loop. */
458 if (++num_bufs
> max
) {
459 virtio_error(vdev
, "Looped descriptor");
463 if (desc
.flags
& VRING_DESC_F_WRITE
) {
464 in_total
+= desc
.len
;
466 out_total
+= desc
.len
;
468 if (in_total
>= max_in_bytes
&& out_total
>= max_out_bytes
) {
472 rc
= virtqueue_read_next_desc(vdev
, &desc
, desc_pa
, max
, &i
);
473 } while (rc
== VIRTQUEUE_READ_DESC_MORE
);
475 if (rc
== VIRTQUEUE_READ_DESC_ERROR
) {
480 total_bufs
= num_bufs
;
491 *in_bytes
= in_total
;
494 *out_bytes
= out_total
;
499 in_total
= out_total
= 0;
503 int virtqueue_avail_bytes(VirtQueue
*vq
, unsigned int in_bytes
,
504 unsigned int out_bytes
)
506 unsigned int in_total
, out_total
;
508 virtqueue_get_avail_bytes(vq
, &in_total
, &out_total
, in_bytes
, out_bytes
);
509 return in_bytes
<= in_total
&& out_bytes
<= out_total
;
512 static bool virtqueue_map_desc(VirtIODevice
*vdev
, unsigned int *p_num_sg
,
513 hwaddr
*addr
, struct iovec
*iov
,
514 unsigned int max_num_sg
, bool is_write
,
515 hwaddr pa
, size_t sz
)
518 unsigned num_sg
= *p_num_sg
;
519 assert(num_sg
<= max_num_sg
);
522 virtio_error(vdev
, "virtio: zero sized buffers are not allowed");
529 if (num_sg
== max_num_sg
) {
530 virtio_error(vdev
, "virtio: too many write descriptors in "
535 iov
[num_sg
].iov_base
= cpu_physical_memory_map(pa
, &len
, is_write
);
536 if (!iov
[num_sg
].iov_base
) {
537 virtio_error(vdev
, "virtio: bogus descriptor or out of resources");
541 iov
[num_sg
].iov_len
= len
;
555 /* Only used by error code paths before we have a VirtQueueElement (therefore
556 * virtqueue_unmap_sg() can't be used). Assumes buffers weren't written to
559 static void virtqueue_undo_map_desc(unsigned int out_num
, unsigned int in_num
,
564 for (i
= 0; i
< out_num
+ in_num
; i
++) {
565 int is_write
= i
>= out_num
;
567 cpu_physical_memory_unmap(iov
->iov_base
, iov
->iov_len
, is_write
, 0);
572 static void virtqueue_map_iovec(struct iovec
*sg
, hwaddr
*addr
,
573 unsigned int *num_sg
, unsigned int max_size
,
579 /* Note: this function MUST validate input, some callers
580 * are passing in num_sg values received over the network.
582 /* TODO: teach all callers that this can fail, and return failure instead
584 * When we do, we might be able to re-enable NDEBUG below.
587 #error building with NDEBUG is not supported
589 assert(*num_sg
<= max_size
);
591 for (i
= 0; i
< *num_sg
; i
++) {
593 sg
[i
].iov_base
= cpu_physical_memory_map(addr
[i
], &len
, is_write
);
594 if (!sg
[i
].iov_base
) {
595 error_report("virtio: error trying to map MMIO memory");
598 if (len
!= sg
[i
].iov_len
) {
599 error_report("virtio: unexpected memory split");
605 void virtqueue_map(VirtQueueElement
*elem
)
607 virtqueue_map_iovec(elem
->in_sg
, elem
->in_addr
, &elem
->in_num
,
608 VIRTQUEUE_MAX_SIZE
, 1);
609 virtqueue_map_iovec(elem
->out_sg
, elem
->out_addr
, &elem
->out_num
,
610 VIRTQUEUE_MAX_SIZE
, 0);
613 void *virtqueue_alloc_element(size_t sz
, unsigned out_num
, unsigned in_num
)
615 VirtQueueElement
*elem
;
616 size_t in_addr_ofs
= QEMU_ALIGN_UP(sz
, __alignof__(elem
->in_addr
[0]));
617 size_t out_addr_ofs
= in_addr_ofs
+ in_num
* sizeof(elem
->in_addr
[0]);
618 size_t out_addr_end
= out_addr_ofs
+ out_num
* sizeof(elem
->out_addr
[0]);
619 size_t in_sg_ofs
= QEMU_ALIGN_UP(out_addr_end
, __alignof__(elem
->in_sg
[0]));
620 size_t out_sg_ofs
= in_sg_ofs
+ in_num
* sizeof(elem
->in_sg
[0]);
621 size_t out_sg_end
= out_sg_ofs
+ out_num
* sizeof(elem
->out_sg
[0]);
623 assert(sz
>= sizeof(VirtQueueElement
));
624 elem
= g_malloc(out_sg_end
);
625 elem
->out_num
= out_num
;
626 elem
->in_num
= in_num
;
627 elem
->in_addr
= (void *)elem
+ in_addr_ofs
;
628 elem
->out_addr
= (void *)elem
+ out_addr_ofs
;
629 elem
->in_sg
= (void *)elem
+ in_sg_ofs
;
630 elem
->out_sg
= (void *)elem
+ out_sg_ofs
;
634 void *virtqueue_pop(VirtQueue
*vq
, size_t sz
)
636 unsigned int i
, head
, max
;
637 hwaddr desc_pa
= vq
->vring
.desc
;
638 VirtIODevice
*vdev
= vq
->vdev
;
639 VirtQueueElement
*elem
;
640 unsigned out_num
, in_num
;
641 hwaddr addr
[VIRTQUEUE_MAX_SIZE
];
642 struct iovec iov
[VIRTQUEUE_MAX_SIZE
];
646 if (unlikely(vdev
->broken
)) {
649 if (virtio_queue_empty(vq
)) {
652 /* Needed after virtio_queue_empty(), see comment in
653 * virtqueue_num_heads(). */
656 /* When we start there are none of either input nor output. */
657 out_num
= in_num
= 0;
661 if (vq
->inuse
>= vq
->vring
.num
) {
662 virtio_error(vdev
, "Virtqueue size exceeded");
666 if (!virtqueue_get_head(vq
, vq
->last_avail_idx
++, &head
)) {
670 if (virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
671 vring_set_avail_event(vq
, vq
->last_avail_idx
);
675 vring_desc_read(vdev
, &desc
, desc_pa
, i
);
676 if (desc
.flags
& VRING_DESC_F_INDIRECT
) {
677 if (desc
.len
% sizeof(VRingDesc
)) {
678 virtio_error(vdev
, "Invalid size for indirect buffer table");
682 /* loop over the indirect descriptor table */
683 max
= desc
.len
/ sizeof(VRingDesc
);
686 vring_desc_read(vdev
, &desc
, desc_pa
, i
);
689 /* Collect all the descriptors */
693 if (desc
.flags
& VRING_DESC_F_WRITE
) {
694 map_ok
= virtqueue_map_desc(vdev
, &in_num
, addr
+ out_num
,
696 VIRTQUEUE_MAX_SIZE
- out_num
, true,
697 desc
.addr
, desc
.len
);
700 virtio_error(vdev
, "Incorrect order for descriptors");
703 map_ok
= virtqueue_map_desc(vdev
, &out_num
, addr
, iov
,
704 VIRTQUEUE_MAX_SIZE
, false,
705 desc
.addr
, desc
.len
);
711 /* If we've got too many, that implies a descriptor loop. */
712 if ((in_num
+ out_num
) > max
) {
713 virtio_error(vdev
, "Looped descriptor");
717 rc
= virtqueue_read_next_desc(vdev
, &desc
, desc_pa
, max
, &i
);
718 } while (rc
== VIRTQUEUE_READ_DESC_MORE
);
720 if (rc
== VIRTQUEUE_READ_DESC_ERROR
) {
724 /* Now copy what we have collected and mapped */
725 elem
= virtqueue_alloc_element(sz
, out_num
, in_num
);
727 for (i
= 0; i
< out_num
; i
++) {
728 elem
->out_addr
[i
] = addr
[i
];
729 elem
->out_sg
[i
] = iov
[i
];
731 for (i
= 0; i
< in_num
; i
++) {
732 elem
->in_addr
[i
] = addr
[out_num
+ i
];
733 elem
->in_sg
[i
] = iov
[out_num
+ i
];
738 trace_virtqueue_pop(vq
, elem
, elem
->in_num
, elem
->out_num
);
742 virtqueue_undo_map_desc(out_num
, in_num
, iov
);
746 /* Reading and writing a structure directly to QEMUFile is *awful*, but
747 * it is what QEMU has always done by mistake. We can change it sooner
748 * or later by bumping the version number of the affected vm states.
749 * In the meanwhile, since the in-memory layout of VirtQueueElement
750 * has changed, we need to marshal to and from the layout that was
751 * used before the change.
753 typedef struct VirtQueueElementOld
{
755 unsigned int out_num
;
757 hwaddr in_addr
[VIRTQUEUE_MAX_SIZE
];
758 hwaddr out_addr
[VIRTQUEUE_MAX_SIZE
];
759 struct iovec in_sg
[VIRTQUEUE_MAX_SIZE
];
760 struct iovec out_sg
[VIRTQUEUE_MAX_SIZE
];
761 } VirtQueueElementOld
;
763 void *qemu_get_virtqueue_element(QEMUFile
*f
, size_t sz
)
765 VirtQueueElement
*elem
;
766 VirtQueueElementOld data
;
769 qemu_get_buffer(f
, (uint8_t *)&data
, sizeof(VirtQueueElementOld
));
771 elem
= virtqueue_alloc_element(sz
, data
.out_num
, data
.in_num
);
772 elem
->index
= data
.index
;
774 for (i
= 0; i
< elem
->in_num
; i
++) {
775 elem
->in_addr
[i
] = data
.in_addr
[i
];
778 for (i
= 0; i
< elem
->out_num
; i
++) {
779 elem
->out_addr
[i
] = data
.out_addr
[i
];
782 for (i
= 0; i
< elem
->in_num
; i
++) {
783 /* Base is overwritten by virtqueue_map. */
784 elem
->in_sg
[i
].iov_base
= 0;
785 elem
->in_sg
[i
].iov_len
= data
.in_sg
[i
].iov_len
;
788 for (i
= 0; i
< elem
->out_num
; i
++) {
789 /* Base is overwritten by virtqueue_map. */
790 elem
->out_sg
[i
].iov_base
= 0;
791 elem
->out_sg
[i
].iov_len
= data
.out_sg
[i
].iov_len
;
798 void qemu_put_virtqueue_element(QEMUFile
*f
, VirtQueueElement
*elem
)
800 VirtQueueElementOld data
;
803 memset(&data
, 0, sizeof(data
));
804 data
.index
= elem
->index
;
805 data
.in_num
= elem
->in_num
;
806 data
.out_num
= elem
->out_num
;
808 for (i
= 0; i
< elem
->in_num
; i
++) {
809 data
.in_addr
[i
] = elem
->in_addr
[i
];
812 for (i
= 0; i
< elem
->out_num
; i
++) {
813 data
.out_addr
[i
] = elem
->out_addr
[i
];
816 for (i
= 0; i
< elem
->in_num
; i
++) {
817 /* Base is overwritten by virtqueue_map when loading. Do not
818 * save it, as it would leak the QEMU address space layout. */
819 data
.in_sg
[i
].iov_len
= elem
->in_sg
[i
].iov_len
;
822 for (i
= 0; i
< elem
->out_num
; i
++) {
823 /* Do not save iov_base as above. */
824 data
.out_sg
[i
].iov_len
= elem
->out_sg
[i
].iov_len
;
826 qemu_put_buffer(f
, (uint8_t *)&data
, sizeof(VirtQueueElementOld
));
830 static void virtio_notify_vector(VirtIODevice
*vdev
, uint16_t vector
)
832 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
833 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
835 if (unlikely(vdev
->broken
)) {
840 k
->notify(qbus
->parent
, vector
);
844 void virtio_update_irq(VirtIODevice
*vdev
)
846 virtio_notify_vector(vdev
, VIRTIO_NO_VECTOR
);
849 static int virtio_validate_features(VirtIODevice
*vdev
)
851 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
853 if (k
->validate_features
) {
854 return k
->validate_features(vdev
);
860 int virtio_set_status(VirtIODevice
*vdev
, uint8_t val
)
862 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
863 trace_virtio_set_status(vdev
, val
);
865 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
866 if (!(vdev
->status
& VIRTIO_CONFIG_S_FEATURES_OK
) &&
867 val
& VIRTIO_CONFIG_S_FEATURES_OK
) {
868 int ret
= virtio_validate_features(vdev
);
876 k
->set_status(vdev
, val
);
882 bool target_words_bigendian(void);
883 static enum virtio_device_endian
virtio_default_endian(void)
885 if (target_words_bigendian()) {
886 return VIRTIO_DEVICE_ENDIAN_BIG
;
888 return VIRTIO_DEVICE_ENDIAN_LITTLE
;
892 static enum virtio_device_endian
virtio_current_cpu_endian(void)
894 CPUClass
*cc
= CPU_GET_CLASS(current_cpu
);
896 if (cc
->virtio_is_big_endian(current_cpu
)) {
897 return VIRTIO_DEVICE_ENDIAN_BIG
;
899 return VIRTIO_DEVICE_ENDIAN_LITTLE
;
903 void virtio_reset(void *opaque
)
905 VirtIODevice
*vdev
= opaque
;
906 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
909 virtio_set_status(vdev
, 0);
911 /* Guest initiated reset */
912 vdev
->device_endian
= virtio_current_cpu_endian();
915 vdev
->device_endian
= virtio_default_endian();
922 vdev
->broken
= false;
923 vdev
->guest_features
= 0;
927 vdev
->config_vector
= VIRTIO_NO_VECTOR
;
928 virtio_notify_vector(vdev
, vdev
->config_vector
);
930 for(i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
931 vdev
->vq
[i
].vring
.desc
= 0;
932 vdev
->vq
[i
].vring
.avail
= 0;
933 vdev
->vq
[i
].vring
.used
= 0;
934 vdev
->vq
[i
].last_avail_idx
= 0;
935 vdev
->vq
[i
].shadow_avail_idx
= 0;
936 vdev
->vq
[i
].used_idx
= 0;
937 virtio_queue_set_vector(vdev
, i
, VIRTIO_NO_VECTOR
);
938 vdev
->vq
[i
].signalled_used
= 0;
939 vdev
->vq
[i
].signalled_used_valid
= false;
940 vdev
->vq
[i
].notification
= true;
941 vdev
->vq
[i
].vring
.num
= vdev
->vq
[i
].vring
.num_default
;
942 vdev
->vq
[i
].inuse
= 0;
946 uint32_t virtio_config_readb(VirtIODevice
*vdev
, uint32_t addr
)
948 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
951 if (addr
+ sizeof(val
) > vdev
->config_len
) {
955 k
->get_config(vdev
, vdev
->config
);
957 val
= ldub_p(vdev
->config
+ addr
);
961 uint32_t virtio_config_readw(VirtIODevice
*vdev
, uint32_t addr
)
963 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
966 if (addr
+ sizeof(val
) > vdev
->config_len
) {
970 k
->get_config(vdev
, vdev
->config
);
972 val
= lduw_p(vdev
->config
+ addr
);
976 uint32_t virtio_config_readl(VirtIODevice
*vdev
, uint32_t addr
)
978 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
981 if (addr
+ sizeof(val
) > vdev
->config_len
) {
985 k
->get_config(vdev
, vdev
->config
);
987 val
= ldl_p(vdev
->config
+ addr
);
991 void virtio_config_writeb(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
993 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
996 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1000 stb_p(vdev
->config
+ addr
, val
);
1002 if (k
->set_config
) {
1003 k
->set_config(vdev
, vdev
->config
);
1007 void virtio_config_writew(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
1009 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1010 uint16_t val
= data
;
1012 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1016 stw_p(vdev
->config
+ addr
, val
);
1018 if (k
->set_config
) {
1019 k
->set_config(vdev
, vdev
->config
);
1023 void virtio_config_writel(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
1025 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1026 uint32_t val
= data
;
1028 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1032 stl_p(vdev
->config
+ addr
, val
);
1034 if (k
->set_config
) {
1035 k
->set_config(vdev
, vdev
->config
);
1039 uint32_t virtio_config_modern_readb(VirtIODevice
*vdev
, uint32_t addr
)
1041 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1044 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1045 return (uint32_t)-1;
1048 k
->get_config(vdev
, vdev
->config
);
1050 val
= ldub_p(vdev
->config
+ addr
);
1054 uint32_t virtio_config_modern_readw(VirtIODevice
*vdev
, uint32_t addr
)
1056 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1059 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1060 return (uint32_t)-1;
1063 k
->get_config(vdev
, vdev
->config
);
1065 val
= lduw_le_p(vdev
->config
+ addr
);
1069 uint32_t virtio_config_modern_readl(VirtIODevice
*vdev
, uint32_t addr
)
1071 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1074 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1075 return (uint32_t)-1;
1078 k
->get_config(vdev
, vdev
->config
);
1080 val
= ldl_le_p(vdev
->config
+ addr
);
1084 void virtio_config_modern_writeb(VirtIODevice
*vdev
,
1085 uint32_t addr
, uint32_t data
)
1087 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1090 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1094 stb_p(vdev
->config
+ addr
, val
);
1096 if (k
->set_config
) {
1097 k
->set_config(vdev
, vdev
->config
);
1101 void virtio_config_modern_writew(VirtIODevice
*vdev
,
1102 uint32_t addr
, uint32_t data
)
1104 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1105 uint16_t val
= data
;
1107 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1111 stw_le_p(vdev
->config
+ addr
, val
);
1113 if (k
->set_config
) {
1114 k
->set_config(vdev
, vdev
->config
);
1118 void virtio_config_modern_writel(VirtIODevice
*vdev
,
1119 uint32_t addr
, uint32_t data
)
1121 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1122 uint32_t val
= data
;
1124 if (addr
+ sizeof(val
) > vdev
->config_len
) {
1128 stl_le_p(vdev
->config
+ addr
, val
);
1130 if (k
->set_config
) {
1131 k
->set_config(vdev
, vdev
->config
);
1135 void virtio_queue_set_addr(VirtIODevice
*vdev
, int n
, hwaddr addr
)
1137 vdev
->vq
[n
].vring
.desc
= addr
;
1138 virtio_queue_update_rings(vdev
, n
);
1141 hwaddr
virtio_queue_get_addr(VirtIODevice
*vdev
, int n
)
1143 return vdev
->vq
[n
].vring
.desc
;
1146 void virtio_queue_set_rings(VirtIODevice
*vdev
, int n
, hwaddr desc
,
1147 hwaddr avail
, hwaddr used
)
1149 vdev
->vq
[n
].vring
.desc
= desc
;
1150 vdev
->vq
[n
].vring
.avail
= avail
;
1151 vdev
->vq
[n
].vring
.used
= used
;
1154 void virtio_queue_set_num(VirtIODevice
*vdev
, int n
, int num
)
1156 /* Don't allow guest to flip queue between existent and
1157 * nonexistent states, or to set it to an invalid size.
1159 if (!!num
!= !!vdev
->vq
[n
].vring
.num
||
1160 num
> VIRTQUEUE_MAX_SIZE
||
1164 vdev
->vq
[n
].vring
.num
= num
;
1167 VirtQueue
*virtio_vector_first_queue(VirtIODevice
*vdev
, uint16_t vector
)
1169 return QLIST_FIRST(&vdev
->vector_queues
[vector
]);
1172 VirtQueue
*virtio_vector_next_queue(VirtQueue
*vq
)
1174 return QLIST_NEXT(vq
, node
);
1177 int virtio_queue_get_num(VirtIODevice
*vdev
, int n
)
1179 return vdev
->vq
[n
].vring
.num
;
1182 int virtio_get_num_queues(VirtIODevice
*vdev
)
1186 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
1187 if (!virtio_queue_get_num(vdev
, i
)) {
1195 void virtio_queue_set_align(VirtIODevice
*vdev
, int n
, int align
)
1197 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1198 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1200 /* virtio-1 compliant devices cannot change the alignment */
1201 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
1202 error_report("tried to modify queue alignment for virtio-1 device");
1205 /* Check that the transport told us it was going to do this
1206 * (so a buggy transport will immediately assert rather than
1207 * silently failing to migrate this state)
1209 assert(k
->has_variable_vring_alignment
);
1211 vdev
->vq
[n
].vring
.align
= align
;
1212 virtio_queue_update_rings(vdev
, n
);
1215 static void virtio_queue_notify_aio_vq(VirtQueue
*vq
)
1217 if (vq
->vring
.desc
&& vq
->handle_aio_output
) {
1218 VirtIODevice
*vdev
= vq
->vdev
;
1220 trace_virtio_queue_notify(vdev
, vq
- vdev
->vq
, vq
);
1221 vq
->handle_aio_output(vdev
, vq
);
1225 static void virtio_queue_notify_vq(VirtQueue
*vq
)
1227 if (vq
->vring
.desc
&& vq
->handle_output
) {
1228 VirtIODevice
*vdev
= vq
->vdev
;
1230 if (unlikely(vdev
->broken
)) {
1234 trace_virtio_queue_notify(vdev
, vq
- vdev
->vq
, vq
);
1235 vq
->handle_output(vdev
, vq
);
1239 void virtio_queue_notify(VirtIODevice
*vdev
, int n
)
1241 virtio_queue_notify_vq(&vdev
->vq
[n
]);
1244 uint16_t virtio_queue_vector(VirtIODevice
*vdev
, int n
)
1246 return n
< VIRTIO_QUEUE_MAX
? vdev
->vq
[n
].vector
:
1250 void virtio_queue_set_vector(VirtIODevice
*vdev
, int n
, uint16_t vector
)
1252 VirtQueue
*vq
= &vdev
->vq
[n
];
1254 if (n
< VIRTIO_QUEUE_MAX
) {
1255 if (vdev
->vector_queues
&&
1256 vdev
->vq
[n
].vector
!= VIRTIO_NO_VECTOR
) {
1257 QLIST_REMOVE(vq
, node
);
1259 vdev
->vq
[n
].vector
= vector
;
1260 if (vdev
->vector_queues
&&
1261 vector
!= VIRTIO_NO_VECTOR
) {
1262 QLIST_INSERT_HEAD(&vdev
->vector_queues
[vector
], vq
, node
);
1267 static VirtQueue
*virtio_add_queue_internal(VirtIODevice
*vdev
, int queue_size
,
1268 VirtIOHandleOutput handle_output
,
1273 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
1274 if (vdev
->vq
[i
].vring
.num
== 0)
1278 if (i
== VIRTIO_QUEUE_MAX
|| queue_size
> VIRTQUEUE_MAX_SIZE
)
1281 vdev
->vq
[i
].vring
.num
= queue_size
;
1282 vdev
->vq
[i
].vring
.num_default
= queue_size
;
1283 vdev
->vq
[i
].vring
.align
= VIRTIO_PCI_VRING_ALIGN
;
1284 vdev
->vq
[i
].handle_output
= handle_output
;
1285 vdev
->vq
[i
].handle_aio_output
= NULL
;
1286 vdev
->vq
[i
].use_aio
= use_aio
;
1288 return &vdev
->vq
[i
];
1291 /* Add a virt queue and mark AIO.
1292 * An AIO queue will use the AioContext based event interface instead of the
1293 * default IOHandler and EventNotifier interface.
1295 VirtQueue
*virtio_add_queue_aio(VirtIODevice
*vdev
, int queue_size
,
1296 VirtIOHandleOutput handle_output
)
1298 return virtio_add_queue_internal(vdev
, queue_size
, handle_output
, true);
1301 /* Add a normal virt queue (on the contrary to the AIO version above. */
1302 VirtQueue
*virtio_add_queue(VirtIODevice
*vdev
, int queue_size
,
1303 VirtIOHandleOutput handle_output
)
1305 return virtio_add_queue_internal(vdev
, queue_size
, handle_output
, false);
1308 void virtio_del_queue(VirtIODevice
*vdev
, int n
)
1310 if (n
< 0 || n
>= VIRTIO_QUEUE_MAX
) {
1314 vdev
->vq
[n
].vring
.num
= 0;
1315 vdev
->vq
[n
].vring
.num_default
= 0;
1318 void virtio_irq(VirtQueue
*vq
)
1320 trace_virtio_irq(vq
);
1321 vq
->vdev
->isr
|= 0x01;
1322 virtio_notify_vector(vq
->vdev
, vq
->vector
);
1325 bool virtio_should_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
1329 /* We need to expose used array entries before checking used event. */
1331 /* Always notify when queue is empty (when feature acknowledge) */
1332 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_NOTIFY_ON_EMPTY
) &&
1333 !vq
->inuse
&& virtio_queue_empty(vq
)) {
1337 if (!virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
1338 return !(vring_avail_flags(vq
) & VRING_AVAIL_F_NO_INTERRUPT
);
1341 v
= vq
->signalled_used_valid
;
1342 vq
->signalled_used_valid
= true;
1343 old
= vq
->signalled_used
;
1344 new = vq
->signalled_used
= vq
->used_idx
;
1345 return !v
|| vring_need_event(vring_get_used_event(vq
), new, old
);
1348 void virtio_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
1350 if (!virtio_should_notify(vdev
, vq
)) {
1354 trace_virtio_notify(vdev
, vq
);
1356 virtio_notify_vector(vdev
, vq
->vector
);
1359 void virtio_notify_config(VirtIODevice
*vdev
)
1361 if (!(vdev
->status
& VIRTIO_CONFIG_S_DRIVER_OK
))
1366 virtio_notify_vector(vdev
, vdev
->config_vector
);
1369 static bool virtio_device_endian_needed(void *opaque
)
1371 VirtIODevice
*vdev
= opaque
;
1373 assert(vdev
->device_endian
!= VIRTIO_DEVICE_ENDIAN_UNKNOWN
);
1374 if (!virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
1375 return vdev
->device_endian
!= virtio_default_endian();
1377 /* Devices conforming to VIRTIO 1.0 or later are always LE. */
1378 return vdev
->device_endian
!= VIRTIO_DEVICE_ENDIAN_LITTLE
;
1381 static bool virtio_64bit_features_needed(void *opaque
)
1383 VirtIODevice
*vdev
= opaque
;
1385 return (vdev
->host_features
>> 32) != 0;
1388 static bool virtio_virtqueue_needed(void *opaque
)
1390 VirtIODevice
*vdev
= opaque
;
1392 return virtio_host_has_feature(vdev
, VIRTIO_F_VERSION_1
);
1395 static bool virtio_ringsize_needed(void *opaque
)
1397 VirtIODevice
*vdev
= opaque
;
1400 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
1401 if (vdev
->vq
[i
].vring
.num
!= vdev
->vq
[i
].vring
.num_default
) {
1408 static bool virtio_extra_state_needed(void *opaque
)
1410 VirtIODevice
*vdev
= opaque
;
1411 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1412 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1414 return k
->has_extra_state
&&
1415 k
->has_extra_state(qbus
->parent
);
1418 static bool virtio_broken_needed(void *opaque
)
1420 VirtIODevice
*vdev
= opaque
;
1422 return vdev
->broken
;
1425 static const VMStateDescription vmstate_virtqueue
= {
1426 .name
= "virtqueue_state",
1428 .minimum_version_id
= 1,
1429 .fields
= (VMStateField
[]) {
1430 VMSTATE_UINT64(vring
.avail
, struct VirtQueue
),
1431 VMSTATE_UINT64(vring
.used
, struct VirtQueue
),
1432 VMSTATE_END_OF_LIST()
1436 static const VMStateDescription vmstate_virtio_virtqueues
= {
1437 .name
= "virtio/virtqueues",
1439 .minimum_version_id
= 1,
1440 .needed
= &virtio_virtqueue_needed
,
1441 .fields
= (VMStateField
[]) {
1442 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq
, struct VirtIODevice
,
1443 VIRTIO_QUEUE_MAX
, 0, vmstate_virtqueue
, VirtQueue
),
1444 VMSTATE_END_OF_LIST()
1448 static const VMStateDescription vmstate_ringsize
= {
1449 .name
= "ringsize_state",
1451 .minimum_version_id
= 1,
1452 .fields
= (VMStateField
[]) {
1453 VMSTATE_UINT32(vring
.num_default
, struct VirtQueue
),
1454 VMSTATE_END_OF_LIST()
1458 static const VMStateDescription vmstate_virtio_ringsize
= {
1459 .name
= "virtio/ringsize",
1461 .minimum_version_id
= 1,
1462 .needed
= &virtio_ringsize_needed
,
1463 .fields
= (VMStateField
[]) {
1464 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq
, struct VirtIODevice
,
1465 VIRTIO_QUEUE_MAX
, 0, vmstate_ringsize
, VirtQueue
),
1466 VMSTATE_END_OF_LIST()
1470 static int get_extra_state(QEMUFile
*f
, void *pv
, size_t size
)
1472 VirtIODevice
*vdev
= pv
;
1473 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1474 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1476 if (!k
->load_extra_state
) {
1479 return k
->load_extra_state(qbus
->parent
, f
);
1483 static void put_extra_state(QEMUFile
*f
, void *pv
, size_t size
)
1485 VirtIODevice
*vdev
= pv
;
1486 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1487 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1489 k
->save_extra_state(qbus
->parent
, f
);
1492 static const VMStateInfo vmstate_info_extra_state
= {
1493 .name
= "virtqueue_extra_state",
1494 .get
= get_extra_state
,
1495 .put
= put_extra_state
,
1498 static const VMStateDescription vmstate_virtio_extra_state
= {
1499 .name
= "virtio/extra_state",
1501 .minimum_version_id
= 1,
1502 .needed
= &virtio_extra_state_needed
,
1503 .fields
= (VMStateField
[]) {
1505 .name
= "extra_state",
1507 .field_exists
= NULL
,
1509 .info
= &vmstate_info_extra_state
,
1510 .flags
= VMS_SINGLE
,
1513 VMSTATE_END_OF_LIST()
1517 static const VMStateDescription vmstate_virtio_device_endian
= {
1518 .name
= "virtio/device_endian",
1520 .minimum_version_id
= 1,
1521 .needed
= &virtio_device_endian_needed
,
1522 .fields
= (VMStateField
[]) {
1523 VMSTATE_UINT8(device_endian
, VirtIODevice
),
1524 VMSTATE_END_OF_LIST()
1528 static const VMStateDescription vmstate_virtio_64bit_features
= {
1529 .name
= "virtio/64bit_features",
1531 .minimum_version_id
= 1,
1532 .needed
= &virtio_64bit_features_needed
,
1533 .fields
= (VMStateField
[]) {
1534 VMSTATE_UINT64(guest_features
, VirtIODevice
),
1535 VMSTATE_END_OF_LIST()
1539 static const VMStateDescription vmstate_virtio_broken
= {
1540 .name
= "virtio/broken",
1542 .minimum_version_id
= 1,
1543 .needed
= &virtio_broken_needed
,
1544 .fields
= (VMStateField
[]) {
1545 VMSTATE_BOOL(broken
, VirtIODevice
),
1546 VMSTATE_END_OF_LIST()
1550 static const VMStateDescription vmstate_virtio
= {
1553 .minimum_version_id
= 1,
1554 .minimum_version_id_old
= 1,
1555 .fields
= (VMStateField
[]) {
1556 VMSTATE_END_OF_LIST()
1558 .subsections
= (const VMStateDescription
*[]) {
1559 &vmstate_virtio_device_endian
,
1560 &vmstate_virtio_64bit_features
,
1561 &vmstate_virtio_virtqueues
,
1562 &vmstate_virtio_ringsize
,
1563 &vmstate_virtio_broken
,
1564 &vmstate_virtio_extra_state
,
1569 void virtio_save(VirtIODevice
*vdev
, QEMUFile
*f
)
1571 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1572 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1573 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1574 uint32_t guest_features_lo
= (vdev
->guest_features
& 0xffffffff);
1577 if (k
->save_config
) {
1578 k
->save_config(qbus
->parent
, f
);
1581 qemu_put_8s(f
, &vdev
->status
);
1582 qemu_put_8s(f
, &vdev
->isr
);
1583 qemu_put_be16s(f
, &vdev
->queue_sel
);
1584 qemu_put_be32s(f
, &guest_features_lo
);
1585 qemu_put_be32(f
, vdev
->config_len
);
1586 qemu_put_buffer(f
, vdev
->config
, vdev
->config_len
);
1588 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
1589 if (vdev
->vq
[i
].vring
.num
== 0)
1593 qemu_put_be32(f
, i
);
1595 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
1596 if (vdev
->vq
[i
].vring
.num
== 0)
1599 qemu_put_be32(f
, vdev
->vq
[i
].vring
.num
);
1600 if (k
->has_variable_vring_alignment
) {
1601 qemu_put_be32(f
, vdev
->vq
[i
].vring
.align
);
1603 /* XXX virtio-1 devices */
1604 qemu_put_be64(f
, vdev
->vq
[i
].vring
.desc
);
1605 qemu_put_be16s(f
, &vdev
->vq
[i
].last_avail_idx
);
1606 if (k
->save_queue
) {
1607 k
->save_queue(qbus
->parent
, i
, f
);
1611 if (vdc
->save
!= NULL
) {
1616 vmstate_save_state(f
, &vmstate_virtio
, vdev
, NULL
);
1619 /* A wrapper for use as a VMState .put function */
1620 void virtio_vmstate_save(QEMUFile
*f
, void *opaque
, size_t size
)
1622 virtio_save(VIRTIO_DEVICE(opaque
), f
);
1625 static int virtio_set_features_nocheck(VirtIODevice
*vdev
, uint64_t val
)
1627 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1628 bool bad
= (val
& ~(vdev
->host_features
)) != 0;
1630 val
&= vdev
->host_features
;
1631 if (k
->set_features
) {
1632 k
->set_features(vdev
, val
);
1634 vdev
->guest_features
= val
;
1635 return bad
? -1 : 0;
1638 int virtio_set_features(VirtIODevice
*vdev
, uint64_t val
)
1641 * The driver must not attempt to set features after feature negotiation
1644 if (vdev
->status
& VIRTIO_CONFIG_S_FEATURES_OK
) {
1647 return virtio_set_features_nocheck(vdev
, val
);
1650 int virtio_load(VirtIODevice
*vdev
, QEMUFile
*f
, int version_id
)
1656 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1657 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1658 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1661 * We poison the endianness to ensure it does not get used before
1662 * subsections have been loaded.
1664 vdev
->device_endian
= VIRTIO_DEVICE_ENDIAN_UNKNOWN
;
1666 if (k
->load_config
) {
1667 ret
= k
->load_config(qbus
->parent
, f
);
1672 qemu_get_8s(f
, &vdev
->status
);
1673 qemu_get_8s(f
, &vdev
->isr
);
1674 qemu_get_be16s(f
, &vdev
->queue_sel
);
1675 if (vdev
->queue_sel
>= VIRTIO_QUEUE_MAX
) {
1678 qemu_get_be32s(f
, &features
);
1681 * Temporarily set guest_features low bits - needed by
1682 * virtio net load code testing for VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
1683 * VIRTIO_NET_F_GUEST_ANNOUNCE and VIRTIO_NET_F_CTRL_VQ.
1685 * Note: devices should always test host features in future - don't create
1686 * new dependencies like this.
1688 vdev
->guest_features
= features
;
1690 config_len
= qemu_get_be32(f
);
1693 * There are cases where the incoming config can be bigger or smaller
1694 * than what we have; so load what we have space for, and skip
1695 * any excess that's in the stream.
1697 qemu_get_buffer(f
, vdev
->config
, MIN(config_len
, vdev
->config_len
));
1699 while (config_len
> vdev
->config_len
) {
1704 num
= qemu_get_be32(f
);
1706 if (num
> VIRTIO_QUEUE_MAX
) {
1707 error_report("Invalid number of virtqueues: 0x%x", num
);
1711 for (i
= 0; i
< num
; i
++) {
1712 vdev
->vq
[i
].vring
.num
= qemu_get_be32(f
);
1713 if (k
->has_variable_vring_alignment
) {
1714 vdev
->vq
[i
].vring
.align
= qemu_get_be32(f
);
1716 vdev
->vq
[i
].vring
.desc
= qemu_get_be64(f
);
1717 qemu_get_be16s(f
, &vdev
->vq
[i
].last_avail_idx
);
1718 vdev
->vq
[i
].signalled_used_valid
= false;
1719 vdev
->vq
[i
].notification
= true;
1721 if (vdev
->vq
[i
].vring
.desc
) {
1722 /* XXX virtio-1 devices */
1723 virtio_queue_update_rings(vdev
, i
);
1724 } else if (vdev
->vq
[i
].last_avail_idx
) {
1725 error_report("VQ %d address 0x0 "
1726 "inconsistent with Host index 0x%x",
1727 i
, vdev
->vq
[i
].last_avail_idx
);
1730 if (k
->load_queue
) {
1731 ret
= k
->load_queue(qbus
->parent
, i
, f
);
1737 virtio_notify_vector(vdev
, VIRTIO_NO_VECTOR
);
1739 if (vdc
->load
!= NULL
) {
1740 ret
= vdc
->load(vdev
, f
, version_id
);
1747 ret
= vmstate_load_state(f
, &vmstate_virtio
, vdev
, 1);
1752 if (vdev
->device_endian
== VIRTIO_DEVICE_ENDIAN_UNKNOWN
) {
1753 vdev
->device_endian
= virtio_default_endian();
1756 if (virtio_64bit_features_needed(vdev
)) {
1758 * Subsection load filled vdev->guest_features. Run them
1759 * through virtio_set_features to sanity-check them against
1762 uint64_t features64
= vdev
->guest_features
;
1763 if (virtio_set_features_nocheck(vdev
, features64
) < 0) {
1764 error_report("Features 0x%" PRIx64
" unsupported. "
1765 "Allowed features: 0x%" PRIx64
,
1766 features64
, vdev
->host_features
);
1770 if (virtio_set_features_nocheck(vdev
, features
) < 0) {
1771 error_report("Features 0x%x unsupported. "
1772 "Allowed features: 0x%" PRIx64
,
1773 features
, vdev
->host_features
);
1778 for (i
= 0; i
< num
; i
++) {
1779 if (vdev
->vq
[i
].vring
.desc
) {
1781 nheads
= vring_avail_idx(&vdev
->vq
[i
]) - vdev
->vq
[i
].last_avail_idx
;
1782 /* Check it isn't doing strange things with descriptor numbers. */
1783 if (nheads
> vdev
->vq
[i
].vring
.num
) {
1784 error_report("VQ %d size 0x%x Guest index 0x%x "
1785 "inconsistent with Host index 0x%x: delta 0x%x",
1786 i
, vdev
->vq
[i
].vring
.num
,
1787 vring_avail_idx(&vdev
->vq
[i
]),
1788 vdev
->vq
[i
].last_avail_idx
, nheads
);
1791 vdev
->vq
[i
].used_idx
= vring_used_idx(&vdev
->vq
[i
]);
1792 vdev
->vq
[i
].shadow_avail_idx
= vring_avail_idx(&vdev
->vq
[i
]);
1795 * Some devices migrate VirtQueueElements that have been popped
1796 * from the avail ring but not yet returned to the used ring.
1798 vdev
->vq
[i
].inuse
= vdev
->vq
[i
].last_avail_idx
-
1799 vdev
->vq
[i
].used_idx
;
1800 if (vdev
->vq
[i
].inuse
> vdev
->vq
[i
].vring
.num
) {
1801 error_report("VQ %d size 0x%x < last_avail_idx 0x%x - "
1803 i
, vdev
->vq
[i
].vring
.num
,
1804 vdev
->vq
[i
].last_avail_idx
,
1805 vdev
->vq
[i
].used_idx
);
1814 void virtio_cleanup(VirtIODevice
*vdev
)
1816 qemu_del_vm_change_state_handler(vdev
->vmstate
);
1817 g_free(vdev
->config
);
1819 g_free(vdev
->vector_queues
);
1822 static void virtio_vmstate_change(void *opaque
, int running
, RunState state
)
1824 VirtIODevice
*vdev
= opaque
;
1825 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1826 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1827 bool backend_run
= running
&& (vdev
->status
& VIRTIO_CONFIG_S_DRIVER_OK
);
1828 vdev
->vm_running
= running
;
1831 virtio_set_status(vdev
, vdev
->status
);
1834 if (k
->vmstate_change
) {
1835 k
->vmstate_change(qbus
->parent
, backend_run
);
1839 virtio_set_status(vdev
, vdev
->status
);
1843 void virtio_instance_init_common(Object
*proxy_obj
, void *data
,
1844 size_t vdev_size
, const char *vdev_name
)
1846 DeviceState
*vdev
= data
;
1848 object_initialize(vdev
, vdev_size
, vdev_name
);
1849 object_property_add_child(proxy_obj
, "virtio-backend", OBJECT(vdev
), NULL
);
1850 object_unref(OBJECT(vdev
));
1851 qdev_alias_all_properties(vdev
, proxy_obj
);
1854 void virtio_init(VirtIODevice
*vdev
, const char *name
,
1855 uint16_t device_id
, size_t config_size
)
1857 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1858 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1860 int nvectors
= k
->query_nvectors
? k
->query_nvectors(qbus
->parent
) : 0;
1863 vdev
->vector_queues
=
1864 g_malloc0(sizeof(*vdev
->vector_queues
) * nvectors
);
1867 vdev
->device_id
= device_id
;
1870 vdev
->queue_sel
= 0;
1871 vdev
->config_vector
= VIRTIO_NO_VECTOR
;
1872 vdev
->vq
= g_malloc0(sizeof(VirtQueue
) * VIRTIO_QUEUE_MAX
);
1873 vdev
->vm_running
= runstate_is_running();
1874 vdev
->broken
= false;
1875 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
1876 vdev
->vq
[i
].vector
= VIRTIO_NO_VECTOR
;
1877 vdev
->vq
[i
].vdev
= vdev
;
1878 vdev
->vq
[i
].queue_index
= i
;
1882 vdev
->config_len
= config_size
;
1883 if (vdev
->config_len
) {
1884 vdev
->config
= g_malloc0(config_size
);
1886 vdev
->config
= NULL
;
1888 vdev
->vmstate
= qemu_add_vm_change_state_handler(virtio_vmstate_change
,
1890 vdev
->device_endian
= virtio_default_endian();
1891 vdev
->use_guest_notifier_mask
= true;
1894 hwaddr
virtio_queue_get_desc_addr(VirtIODevice
*vdev
, int n
)
1896 return vdev
->vq
[n
].vring
.desc
;
1899 hwaddr
virtio_queue_get_avail_addr(VirtIODevice
*vdev
, int n
)
1901 return vdev
->vq
[n
].vring
.avail
;
1904 hwaddr
virtio_queue_get_used_addr(VirtIODevice
*vdev
, int n
)
1906 return vdev
->vq
[n
].vring
.used
;
1909 hwaddr
virtio_queue_get_ring_addr(VirtIODevice
*vdev
, int n
)
1911 return vdev
->vq
[n
].vring
.desc
;
1914 hwaddr
virtio_queue_get_desc_size(VirtIODevice
*vdev
, int n
)
1916 return sizeof(VRingDesc
) * vdev
->vq
[n
].vring
.num
;
1919 hwaddr
virtio_queue_get_avail_size(VirtIODevice
*vdev
, int n
)
1921 return offsetof(VRingAvail
, ring
) +
1922 sizeof(uint16_t) * vdev
->vq
[n
].vring
.num
;
1925 hwaddr
virtio_queue_get_used_size(VirtIODevice
*vdev
, int n
)
1927 return offsetof(VRingUsed
, ring
) +
1928 sizeof(VRingUsedElem
) * vdev
->vq
[n
].vring
.num
;
1931 hwaddr
virtio_queue_get_ring_size(VirtIODevice
*vdev
, int n
)
1933 return vdev
->vq
[n
].vring
.used
- vdev
->vq
[n
].vring
.desc
+
1934 virtio_queue_get_used_size(vdev
, n
);
1937 uint16_t virtio_queue_get_last_avail_idx(VirtIODevice
*vdev
, int n
)
1939 return vdev
->vq
[n
].last_avail_idx
;
1942 void virtio_queue_set_last_avail_idx(VirtIODevice
*vdev
, int n
, uint16_t idx
)
1944 vdev
->vq
[n
].last_avail_idx
= idx
;
1945 vdev
->vq
[n
].shadow_avail_idx
= idx
;
1948 void virtio_queue_invalidate_signalled_used(VirtIODevice
*vdev
, int n
)
1950 vdev
->vq
[n
].signalled_used_valid
= false;
1953 VirtQueue
*virtio_get_queue(VirtIODevice
*vdev
, int n
)
1955 return vdev
->vq
+ n
;
1958 uint16_t virtio_get_queue_index(VirtQueue
*vq
)
1960 return vq
->queue_index
;
1963 static void virtio_queue_guest_notifier_read(EventNotifier
*n
)
1965 VirtQueue
*vq
= container_of(n
, VirtQueue
, guest_notifier
);
1966 if (event_notifier_test_and_clear(n
)) {
1971 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue
*vq
, bool assign
,
1974 if (assign
&& !with_irqfd
) {
1975 event_notifier_set_handler(&vq
->guest_notifier
, false,
1976 virtio_queue_guest_notifier_read
);
1978 event_notifier_set_handler(&vq
->guest_notifier
, false, NULL
);
1981 /* Test and clear notifier before closing it,
1982 * in case poll callback didn't have time to run. */
1983 virtio_queue_guest_notifier_read(&vq
->guest_notifier
);
1987 EventNotifier
*virtio_queue_get_guest_notifier(VirtQueue
*vq
)
1989 return &vq
->guest_notifier
;
1992 static void virtio_queue_host_notifier_aio_read(EventNotifier
*n
)
1994 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
1995 if (event_notifier_test_and_clear(n
)) {
1996 virtio_queue_notify_aio_vq(vq
);
2000 void virtio_queue_aio_set_host_notifier_handler(VirtQueue
*vq
, AioContext
*ctx
,
2001 VirtIOHandleOutput handle_output
)
2003 if (handle_output
) {
2004 vq
->handle_aio_output
= handle_output
;
2005 aio_set_event_notifier(ctx
, &vq
->host_notifier
, true,
2006 virtio_queue_host_notifier_aio_read
);
2008 aio_set_event_notifier(ctx
, &vq
->host_notifier
, true, NULL
);
2009 /* Test and clear notifier before after disabling event,
2010 * in case poll callback didn't have time to run. */
2011 virtio_queue_host_notifier_aio_read(&vq
->host_notifier
);
2012 vq
->handle_aio_output
= NULL
;
2016 static void virtio_queue_host_notifier_read(EventNotifier
*n
)
2018 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
2019 if (event_notifier_test_and_clear(n
)) {
2020 virtio_queue_notify_vq(vq
);
2024 void virtio_queue_set_host_notifier_fd_handler(VirtQueue
*vq
, bool assign
,
2027 AioContext
*ctx
= qemu_get_aio_context();
2028 if (assign
&& set_handler
) {
2030 aio_set_event_notifier(ctx
, &vq
->host_notifier
, true,
2031 virtio_queue_host_notifier_read
);
2033 event_notifier_set_handler(&vq
->host_notifier
, true,
2034 virtio_queue_host_notifier_read
);
2038 aio_set_event_notifier(ctx
, &vq
->host_notifier
, true, NULL
);
2040 event_notifier_set_handler(&vq
->host_notifier
, true, NULL
);
2044 /* Test and clear notifier before after disabling event,
2045 * in case poll callback didn't have time to run. */
2046 virtio_queue_host_notifier_read(&vq
->host_notifier
);
2050 EventNotifier
*virtio_queue_get_host_notifier(VirtQueue
*vq
)
2052 return &vq
->host_notifier
;
2055 void virtio_device_set_child_bus_name(VirtIODevice
*vdev
, char *bus_name
)
2057 g_free(vdev
->bus_name
);
2058 vdev
->bus_name
= g_strdup(bus_name
);
2061 void GCC_FMT_ATTR(2, 3) virtio_error(VirtIODevice
*vdev
, const char *fmt
, ...)
2066 error_vreport(fmt
, ap
);
2069 vdev
->broken
= true;
2071 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
2072 virtio_set_status(vdev
, vdev
->status
| VIRTIO_CONFIG_S_NEEDS_RESET
);
2073 virtio_notify_config(vdev
);
2077 static void virtio_device_realize(DeviceState
*dev
, Error
**errp
)
2079 VirtIODevice
*vdev
= VIRTIO_DEVICE(dev
);
2080 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(dev
);
2083 if (vdc
->realize
!= NULL
) {
2084 vdc
->realize(dev
, &err
);
2086 error_propagate(errp
, err
);
2091 virtio_bus_device_plugged(vdev
, &err
);
2093 error_propagate(errp
, err
);
2098 static void virtio_device_unrealize(DeviceState
*dev
, Error
**errp
)
2100 VirtIODevice
*vdev
= VIRTIO_DEVICE(dev
);
2101 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(dev
);
2104 virtio_bus_device_unplugged(vdev
);
2106 if (vdc
->unrealize
!= NULL
) {
2107 vdc
->unrealize(dev
, &err
);
2109 error_propagate(errp
, err
);
2114 g_free(vdev
->bus_name
);
2115 vdev
->bus_name
= NULL
;
2118 static Property virtio_properties
[] = {
2119 DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice
, host_features
),
2120 DEFINE_PROP_END_OF_LIST(),
2123 static void virtio_device_class_init(ObjectClass
*klass
, void *data
)
2125 /* Set the default value here. */
2126 DeviceClass
*dc
= DEVICE_CLASS(klass
);
2128 dc
->realize
= virtio_device_realize
;
2129 dc
->unrealize
= virtio_device_unrealize
;
2130 dc
->bus_type
= TYPE_VIRTIO_BUS
;
2131 dc
->props
= virtio_properties
;
2134 static const TypeInfo virtio_device_info
= {
2135 .name
= TYPE_VIRTIO_DEVICE
,
2136 .parent
= TYPE_DEVICE
,
2137 .instance_size
= sizeof(VirtIODevice
),
2138 .class_init
= virtio_device_class_init
,
2140 .class_size
= sizeof(VirtioDeviceClass
),
2143 static void virtio_register_types(void)
2145 type_register_static(&virtio_device_info
);
2148 type_init(virtio_register_types
)