4 * Copyright IBM, Corp. 2007
7 * Anthony Liguori <aliguori@us.ibm.com>
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
17 #include "exec/address-spaces.h"
18 #include "qemu/error-report.h"
19 #include "hw/virtio/virtio.h"
20 #include "qemu/atomic.h"
21 #include "hw/virtio/virtio-bus.h"
22 #include "migration/migration.h"
23 #include "hw/virtio/virtio-access.h"
26 * The alignment to use between consumer and producer parts of vring.
27 * x86 pagesize again. This is the default, used by transports like PCI
28 * which don't provide a means for the guest to tell the host the alignment.
30 #define VIRTIO_PCI_VRING_ALIGN 4096
32 typedef struct VRingDesc
40 typedef struct VRingAvail
47 typedef struct VRingUsedElem
53 typedef struct VRingUsed
57 VRingUsedElem ring
[0];
73 uint16_t last_avail_idx
;
74 /* Last used index value we have signalled on */
75 uint16_t signalled_used
;
77 /* Last used index value we have signalled on */
78 bool signalled_used_valid
;
80 /* Notification enabled? */
88 void (*handle_output
)(VirtIODevice
*vdev
, VirtQueue
*vq
);
90 EventNotifier guest_notifier
;
91 EventNotifier host_notifier
;
94 /* virt queue functions */
95 static void virtqueue_init(VirtQueue
*vq
)
100 vq
->vring
.avail
= pa
+ vq
->vring
.num
* sizeof(VRingDesc
);
101 vq
->vring
.used
= vring_align(vq
->vring
.avail
+
102 offsetof(VRingAvail
, ring
[vq
->vring
.num
]),
106 static inline uint64_t vring_desc_addr(VirtIODevice
*vdev
, hwaddr desc_pa
,
110 pa
= desc_pa
+ sizeof(VRingDesc
) * i
+ offsetof(VRingDesc
, addr
);
111 return virtio_ldq_phys(vdev
, pa
);
114 static inline uint32_t vring_desc_len(VirtIODevice
*vdev
, hwaddr desc_pa
, int i
)
117 pa
= desc_pa
+ sizeof(VRingDesc
) * i
+ offsetof(VRingDesc
, len
);
118 return virtio_ldl_phys(vdev
, pa
);
121 static inline uint16_t vring_desc_flags(VirtIODevice
*vdev
, hwaddr desc_pa
,
125 pa
= desc_pa
+ sizeof(VRingDesc
) * i
+ offsetof(VRingDesc
, flags
);
126 return virtio_lduw_phys(vdev
, pa
);
129 static inline uint16_t vring_desc_next(VirtIODevice
*vdev
, hwaddr desc_pa
,
133 pa
= desc_pa
+ sizeof(VRingDesc
) * i
+ offsetof(VRingDesc
, next
);
134 return virtio_lduw_phys(vdev
, pa
);
137 static inline uint16_t vring_avail_flags(VirtQueue
*vq
)
140 pa
= vq
->vring
.avail
+ offsetof(VRingAvail
, flags
);
141 return virtio_lduw_phys(vq
->vdev
, pa
);
144 static inline uint16_t vring_avail_idx(VirtQueue
*vq
)
147 pa
= vq
->vring
.avail
+ offsetof(VRingAvail
, idx
);
148 return virtio_lduw_phys(vq
->vdev
, pa
);
151 static inline uint16_t vring_avail_ring(VirtQueue
*vq
, int i
)
154 pa
= vq
->vring
.avail
+ offsetof(VRingAvail
, ring
[i
]);
155 return virtio_lduw_phys(vq
->vdev
, pa
);
158 static inline uint16_t vring_used_event(VirtQueue
*vq
)
160 return vring_avail_ring(vq
, vq
->vring
.num
);
163 static inline void vring_used_ring_id(VirtQueue
*vq
, int i
, uint32_t val
)
166 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, ring
[i
].id
);
167 virtio_stl_phys(vq
->vdev
, pa
, val
);
170 static inline void vring_used_ring_len(VirtQueue
*vq
, int i
, uint32_t val
)
173 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, ring
[i
].len
);
174 virtio_stl_phys(vq
->vdev
, pa
, val
);
177 static uint16_t vring_used_idx(VirtQueue
*vq
)
180 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, idx
);
181 return virtio_lduw_phys(vq
->vdev
, pa
);
184 static inline void vring_used_idx_set(VirtQueue
*vq
, uint16_t val
)
187 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, idx
);
188 virtio_stw_phys(vq
->vdev
, pa
, val
);
191 static inline void vring_used_flags_set_bit(VirtQueue
*vq
, int mask
)
193 VirtIODevice
*vdev
= vq
->vdev
;
195 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, flags
);
196 virtio_stw_phys(vdev
, pa
, virtio_lduw_phys(vdev
, pa
) | mask
);
199 static inline void vring_used_flags_unset_bit(VirtQueue
*vq
, int mask
)
201 VirtIODevice
*vdev
= vq
->vdev
;
203 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, flags
);
204 virtio_stw_phys(vdev
, pa
, virtio_lduw_phys(vdev
, pa
) & ~mask
);
207 static inline void vring_avail_event(VirtQueue
*vq
, uint16_t val
)
210 if (!vq
->notification
) {
213 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, ring
[vq
->vring
.num
]);
214 virtio_stw_phys(vq
->vdev
, pa
, val
);
217 void virtio_queue_set_notification(VirtQueue
*vq
, int enable
)
219 vq
->notification
= enable
;
220 if (vq
->vdev
->guest_features
& (1 << VIRTIO_RING_F_EVENT_IDX
)) {
221 vring_avail_event(vq
, vring_avail_idx(vq
));
223 vring_used_flags_unset_bit(vq
, VRING_USED_F_NO_NOTIFY
);
225 vring_used_flags_set_bit(vq
, VRING_USED_F_NO_NOTIFY
);
228 /* Expose avail event/used flags before caller checks the avail idx. */
233 int virtio_queue_ready(VirtQueue
*vq
)
235 return vq
->vring
.avail
!= 0;
238 int virtio_queue_empty(VirtQueue
*vq
)
240 return vring_avail_idx(vq
) == vq
->last_avail_idx
;
243 void virtqueue_fill(VirtQueue
*vq
, const VirtQueueElement
*elem
,
244 unsigned int len
, unsigned int idx
)
249 trace_virtqueue_fill(vq
, elem
, len
, idx
);
252 for (i
= 0; i
< elem
->in_num
; i
++) {
253 size_t size
= MIN(len
- offset
, elem
->in_sg
[i
].iov_len
);
255 cpu_physical_memory_unmap(elem
->in_sg
[i
].iov_base
,
256 elem
->in_sg
[i
].iov_len
,
262 for (i
= 0; i
< elem
->out_num
; i
++)
263 cpu_physical_memory_unmap(elem
->out_sg
[i
].iov_base
,
264 elem
->out_sg
[i
].iov_len
,
265 0, elem
->out_sg
[i
].iov_len
);
267 idx
= (idx
+ vring_used_idx(vq
)) % vq
->vring
.num
;
269 /* Get a pointer to the next entry in the used ring. */
270 vring_used_ring_id(vq
, idx
, elem
->index
);
271 vring_used_ring_len(vq
, idx
, len
);
274 void virtqueue_flush(VirtQueue
*vq
, unsigned int count
)
277 /* Make sure buffer is written before we update index. */
279 trace_virtqueue_flush(vq
, count
);
280 old
= vring_used_idx(vq
);
282 vring_used_idx_set(vq
, new);
284 if (unlikely((int16_t)(new - vq
->signalled_used
) < (uint16_t)(new - old
)))
285 vq
->signalled_used_valid
= false;
288 void virtqueue_push(VirtQueue
*vq
, const VirtQueueElement
*elem
,
291 virtqueue_fill(vq
, elem
, len
, 0);
292 virtqueue_flush(vq
, 1);
295 static int virtqueue_num_heads(VirtQueue
*vq
, unsigned int idx
)
297 uint16_t num_heads
= vring_avail_idx(vq
) - idx
;
299 /* Check it isn't doing very strange things with descriptor numbers. */
300 if (num_heads
> vq
->vring
.num
) {
301 error_report("Guest moved used index from %u to %u",
302 idx
, vring_avail_idx(vq
));
305 /* On success, callers read a descriptor at vq->last_avail_idx.
306 * Make sure descriptor read does not bypass avail index read. */
314 static unsigned int virtqueue_get_head(VirtQueue
*vq
, unsigned int idx
)
318 /* Grab the next descriptor number they're advertising, and increment
319 * the index we've seen. */
320 head
= vring_avail_ring(vq
, idx
% vq
->vring
.num
);
322 /* If their number is silly, that's a fatal mistake. */
323 if (head
>= vq
->vring
.num
) {
324 error_report("Guest says index %u is available", head
);
331 static unsigned virtqueue_next_desc(VirtIODevice
*vdev
, hwaddr desc_pa
,
332 unsigned int i
, unsigned int max
)
336 /* If this descriptor says it doesn't chain, we're done. */
337 if (!(vring_desc_flags(vdev
, desc_pa
, i
) & VRING_DESC_F_NEXT
)) {
341 /* Check they're not leading us off end of descriptors. */
342 next
= vring_desc_next(vdev
, desc_pa
, i
);
343 /* Make sure compiler knows to grab that: we don't want it changing! */
347 error_report("Desc next is %u", next
);
354 void virtqueue_get_avail_bytes(VirtQueue
*vq
, unsigned int *in_bytes
,
355 unsigned int *out_bytes
,
356 unsigned max_in_bytes
, unsigned max_out_bytes
)
359 unsigned int total_bufs
, in_total
, out_total
;
361 idx
= vq
->last_avail_idx
;
363 total_bufs
= in_total
= out_total
= 0;
364 while (virtqueue_num_heads(vq
, idx
)) {
365 VirtIODevice
*vdev
= vq
->vdev
;
366 unsigned int max
, num_bufs
, indirect
= 0;
371 num_bufs
= total_bufs
;
372 i
= virtqueue_get_head(vq
, idx
++);
373 desc_pa
= vq
->vring
.desc
;
375 if (vring_desc_flags(vdev
, desc_pa
, i
) & VRING_DESC_F_INDIRECT
) {
376 if (vring_desc_len(vdev
, desc_pa
, i
) % sizeof(VRingDesc
)) {
377 error_report("Invalid size for indirect buffer table");
381 /* If we've got too many, that implies a descriptor loop. */
382 if (num_bufs
>= max
) {
383 error_report("Looped descriptor");
387 /* loop over the indirect descriptor table */
389 max
= vring_desc_len(vdev
, desc_pa
, i
) / sizeof(VRingDesc
);
390 desc_pa
= vring_desc_addr(vdev
, desc_pa
, i
);
395 /* If we've got too many, that implies a descriptor loop. */
396 if (++num_bufs
> max
) {
397 error_report("Looped descriptor");
401 if (vring_desc_flags(vdev
, desc_pa
, i
) & VRING_DESC_F_WRITE
) {
402 in_total
+= vring_desc_len(vdev
, desc_pa
, i
);
404 out_total
+= vring_desc_len(vdev
, desc_pa
, i
);
406 if (in_total
>= max_in_bytes
&& out_total
>= max_out_bytes
) {
409 } while ((i
= virtqueue_next_desc(vdev
, desc_pa
, i
, max
)) != max
);
412 total_bufs
= num_bufs
;
418 *in_bytes
= in_total
;
421 *out_bytes
= out_total
;
425 int virtqueue_avail_bytes(VirtQueue
*vq
, unsigned int in_bytes
,
426 unsigned int out_bytes
)
428 unsigned int in_total
, out_total
;
430 virtqueue_get_avail_bytes(vq
, &in_total
, &out_total
, in_bytes
, out_bytes
);
431 return in_bytes
<= in_total
&& out_bytes
<= out_total
;
434 void virtqueue_map_sg(struct iovec
*sg
, hwaddr
*addr
,
435 size_t num_sg
, int is_write
)
440 if (num_sg
> VIRTQUEUE_MAX_SIZE
) {
441 error_report("virtio: map attempt out of bounds: %zd > %d",
442 num_sg
, VIRTQUEUE_MAX_SIZE
);
446 for (i
= 0; i
< num_sg
; i
++) {
448 sg
[i
].iov_base
= cpu_physical_memory_map(addr
[i
], &len
, is_write
);
449 if (sg
[i
].iov_base
== NULL
|| len
!= sg
[i
].iov_len
) {
450 error_report("virtio: error trying to map MMIO memory");
456 int virtqueue_pop(VirtQueue
*vq
, VirtQueueElement
*elem
)
458 unsigned int i
, head
, max
;
459 hwaddr desc_pa
= vq
->vring
.desc
;
460 VirtIODevice
*vdev
= vq
->vdev
;
462 if (!virtqueue_num_heads(vq
, vq
->last_avail_idx
))
465 /* When we start there are none of either input nor output. */
466 elem
->out_num
= elem
->in_num
= 0;
470 i
= head
= virtqueue_get_head(vq
, vq
->last_avail_idx
++);
471 if (vdev
->guest_features
& (1 << VIRTIO_RING_F_EVENT_IDX
)) {
472 vring_avail_event(vq
, vring_avail_idx(vq
));
475 if (vring_desc_flags(vdev
, desc_pa
, i
) & VRING_DESC_F_INDIRECT
) {
476 if (vring_desc_len(vdev
, desc_pa
, i
) % sizeof(VRingDesc
)) {
477 error_report("Invalid size for indirect buffer table");
481 /* loop over the indirect descriptor table */
482 max
= vring_desc_len(vdev
, desc_pa
, i
) / sizeof(VRingDesc
);
483 desc_pa
= vring_desc_addr(vdev
, desc_pa
, i
);
487 /* Collect all the descriptors */
491 if (vring_desc_flags(vdev
, desc_pa
, i
) & VRING_DESC_F_WRITE
) {
492 if (elem
->in_num
>= ARRAY_SIZE(elem
->in_sg
)) {
493 error_report("Too many write descriptors in indirect table");
496 elem
->in_addr
[elem
->in_num
] = vring_desc_addr(vdev
, desc_pa
, i
);
497 sg
= &elem
->in_sg
[elem
->in_num
++];
499 if (elem
->out_num
>= ARRAY_SIZE(elem
->out_sg
)) {
500 error_report("Too many read descriptors in indirect table");
503 elem
->out_addr
[elem
->out_num
] = vring_desc_addr(vdev
, desc_pa
, i
);
504 sg
= &elem
->out_sg
[elem
->out_num
++];
507 sg
->iov_len
= vring_desc_len(vdev
, desc_pa
, i
);
509 /* If we've got too many, that implies a descriptor loop. */
510 if ((elem
->in_num
+ elem
->out_num
) > max
) {
511 error_report("Looped descriptor");
514 } while ((i
= virtqueue_next_desc(vdev
, desc_pa
, i
, max
)) != max
);
516 /* Now map what we have collected */
517 virtqueue_map_sg(elem
->in_sg
, elem
->in_addr
, elem
->in_num
, 1);
518 virtqueue_map_sg(elem
->out_sg
, elem
->out_addr
, elem
->out_num
, 0);
524 trace_virtqueue_pop(vq
, elem
, elem
->in_num
, elem
->out_num
);
525 return elem
->in_num
+ elem
->out_num
;
529 static void virtio_notify_vector(VirtIODevice
*vdev
, uint16_t vector
)
531 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
532 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
535 k
->notify(qbus
->parent
, vector
);
539 void virtio_update_irq(VirtIODevice
*vdev
)
541 virtio_notify_vector(vdev
, VIRTIO_NO_VECTOR
);
544 void virtio_set_status(VirtIODevice
*vdev
, uint8_t val
)
546 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
547 trace_virtio_set_status(vdev
, val
);
550 k
->set_status(vdev
, val
);
555 bool target_words_bigendian(void);
556 static enum virtio_device_endian
virtio_default_endian(void)
558 if (target_words_bigendian()) {
559 return VIRTIO_DEVICE_ENDIAN_BIG
;
561 return VIRTIO_DEVICE_ENDIAN_LITTLE
;
565 static enum virtio_device_endian
virtio_current_cpu_endian(void)
567 CPUClass
*cc
= CPU_GET_CLASS(current_cpu
);
569 if (cc
->virtio_is_big_endian(current_cpu
)) {
570 return VIRTIO_DEVICE_ENDIAN_BIG
;
572 return VIRTIO_DEVICE_ENDIAN_LITTLE
;
576 void virtio_reset(void *opaque
)
578 VirtIODevice
*vdev
= opaque
;
579 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
582 virtio_set_status(vdev
, 0);
584 /* Guest initiated reset */
585 vdev
->device_endian
= virtio_current_cpu_endian();
588 vdev
->device_endian
= virtio_default_endian();
595 vdev
->guest_features
= 0;
599 vdev
->config_vector
= VIRTIO_NO_VECTOR
;
600 virtio_notify_vector(vdev
, vdev
->config_vector
);
602 for(i
= 0; i
< VIRTIO_PCI_QUEUE_MAX
; i
++) {
603 vdev
->vq
[i
].vring
.desc
= 0;
604 vdev
->vq
[i
].vring
.avail
= 0;
605 vdev
->vq
[i
].vring
.used
= 0;
606 vdev
->vq
[i
].last_avail_idx
= 0;
608 vdev
->vq
[i
].vector
= VIRTIO_NO_VECTOR
;
609 vdev
->vq
[i
].signalled_used
= 0;
610 vdev
->vq
[i
].signalled_used_valid
= false;
611 vdev
->vq
[i
].notification
= true;
615 uint32_t virtio_config_readb(VirtIODevice
*vdev
, uint32_t addr
)
617 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
620 if (addr
+ sizeof(val
) > vdev
->config_len
) {
624 k
->get_config(vdev
, vdev
->config
);
626 val
= ldub_p(vdev
->config
+ addr
);
630 uint32_t virtio_config_readw(VirtIODevice
*vdev
, uint32_t addr
)
632 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
635 if (addr
+ sizeof(val
) > vdev
->config_len
) {
639 k
->get_config(vdev
, vdev
->config
);
641 val
= lduw_p(vdev
->config
+ addr
);
645 uint32_t virtio_config_readl(VirtIODevice
*vdev
, uint32_t addr
)
647 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
650 if (addr
+ sizeof(val
) > vdev
->config_len
) {
654 k
->get_config(vdev
, vdev
->config
);
656 val
= ldl_p(vdev
->config
+ addr
);
660 void virtio_config_writeb(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
662 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
665 if (addr
+ sizeof(val
) > vdev
->config_len
) {
669 stb_p(vdev
->config
+ addr
, val
);
672 k
->set_config(vdev
, vdev
->config
);
676 void virtio_config_writew(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
678 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
681 if (addr
+ sizeof(val
) > vdev
->config_len
) {
685 stw_p(vdev
->config
+ addr
, val
);
688 k
->set_config(vdev
, vdev
->config
);
692 void virtio_config_writel(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
694 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
697 if (addr
+ sizeof(val
) > vdev
->config_len
) {
701 stl_p(vdev
->config
+ addr
, val
);
704 k
->set_config(vdev
, vdev
->config
);
708 void virtio_queue_set_addr(VirtIODevice
*vdev
, int n
, hwaddr addr
)
710 vdev
->vq
[n
].pa
= addr
;
711 virtqueue_init(&vdev
->vq
[n
]);
714 hwaddr
virtio_queue_get_addr(VirtIODevice
*vdev
, int n
)
716 return vdev
->vq
[n
].pa
;
719 void virtio_queue_set_num(VirtIODevice
*vdev
, int n
, int num
)
721 /* Don't allow guest to flip queue between existent and
722 * nonexistent states, or to set it to an invalid size.
724 if (!!num
!= !!vdev
->vq
[n
].vring
.num
||
725 num
> VIRTQUEUE_MAX_SIZE
||
729 vdev
->vq
[n
].vring
.num
= num
;
730 virtqueue_init(&vdev
->vq
[n
]);
733 int virtio_queue_get_num(VirtIODevice
*vdev
, int n
)
735 return vdev
->vq
[n
].vring
.num
;
738 int virtio_queue_get_id(VirtQueue
*vq
)
740 VirtIODevice
*vdev
= vq
->vdev
;
741 assert(vq
>= &vdev
->vq
[0] && vq
< &vdev
->vq
[VIRTIO_PCI_QUEUE_MAX
]);
742 return vq
- &vdev
->vq
[0];
745 void virtio_queue_set_align(VirtIODevice
*vdev
, int n
, int align
)
747 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
748 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
750 /* Check that the transport told us it was going to do this
751 * (so a buggy transport will immediately assert rather than
752 * silently failing to migrate this state)
754 assert(k
->has_variable_vring_alignment
);
756 vdev
->vq
[n
].vring
.align
= align
;
757 virtqueue_init(&vdev
->vq
[n
]);
760 void virtio_queue_notify_vq(VirtQueue
*vq
)
762 if (vq
->vring
.desc
) {
763 VirtIODevice
*vdev
= vq
->vdev
;
764 trace_virtio_queue_notify(vdev
, vq
- vdev
->vq
, vq
);
765 vq
->handle_output(vdev
, vq
);
769 void virtio_queue_notify(VirtIODevice
*vdev
, int n
)
771 virtio_queue_notify_vq(&vdev
->vq
[n
]);
774 uint16_t virtio_queue_vector(VirtIODevice
*vdev
, int n
)
776 return n
< VIRTIO_PCI_QUEUE_MAX
? vdev
->vq
[n
].vector
:
780 void virtio_queue_set_vector(VirtIODevice
*vdev
, int n
, uint16_t vector
)
782 if (n
< VIRTIO_PCI_QUEUE_MAX
)
783 vdev
->vq
[n
].vector
= vector
;
786 VirtQueue
*virtio_add_queue(VirtIODevice
*vdev
, int queue_size
,
787 void (*handle_output
)(VirtIODevice
*, VirtQueue
*))
791 for (i
= 0; i
< VIRTIO_PCI_QUEUE_MAX
; i
++) {
792 if (vdev
->vq
[i
].vring
.num
== 0)
796 if (i
== VIRTIO_PCI_QUEUE_MAX
|| queue_size
> VIRTQUEUE_MAX_SIZE
)
799 vdev
->vq
[i
].vring
.num
= queue_size
;
800 vdev
->vq
[i
].vring
.align
= VIRTIO_PCI_VRING_ALIGN
;
801 vdev
->vq
[i
].handle_output
= handle_output
;
806 void virtio_del_queue(VirtIODevice
*vdev
, int n
)
808 if (n
< 0 || n
>= VIRTIO_PCI_QUEUE_MAX
) {
812 vdev
->vq
[n
].vring
.num
= 0;
815 void virtio_irq(VirtQueue
*vq
)
817 trace_virtio_irq(vq
);
818 vq
->vdev
->isr
|= 0x01;
819 virtio_notify_vector(vq
->vdev
, vq
->vector
);
822 /* Assuming a given event_idx value from the other size, if
823 * we have just incremented index from old to new_idx,
824 * should we trigger an event? */
825 static inline int vring_need_event(uint16_t event
, uint16_t new, uint16_t old
)
827 /* Note: Xen has similar logic for notification hold-off
828 * in include/xen/interface/io/ring.h with req_event and req_prod
829 * corresponding to event_idx + 1 and new respectively.
830 * Note also that req_event and req_prod in Xen start at 1,
831 * event indexes in virtio start at 0. */
832 return (uint16_t)(new - event
- 1) < (uint16_t)(new - old
);
835 static bool vring_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
839 /* We need to expose used array entries before checking used event. */
841 /* Always notify when queue is empty (when feature acknowledge) */
842 if (((vdev
->guest_features
& (1 << VIRTIO_F_NOTIFY_ON_EMPTY
)) &&
843 !vq
->inuse
&& vring_avail_idx(vq
) == vq
->last_avail_idx
)) {
847 if (!(vdev
->guest_features
& (1 << VIRTIO_RING_F_EVENT_IDX
))) {
848 return !(vring_avail_flags(vq
) & VRING_AVAIL_F_NO_INTERRUPT
);
851 v
= vq
->signalled_used_valid
;
852 vq
->signalled_used_valid
= true;
853 old
= vq
->signalled_used
;
854 new = vq
->signalled_used
= vring_used_idx(vq
);
855 return !v
|| vring_need_event(vring_used_event(vq
), new, old
);
858 void virtio_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
860 if (!vring_notify(vdev
, vq
)) {
864 trace_virtio_notify(vdev
, vq
);
866 virtio_notify_vector(vdev
, vq
->vector
);
869 void virtio_notify_config(VirtIODevice
*vdev
)
871 if (!(vdev
->status
& VIRTIO_CONFIG_S_DRIVER_OK
))
875 virtio_notify_vector(vdev
, vdev
->config_vector
);
878 static bool virtio_device_endian_needed(void *opaque
)
880 VirtIODevice
*vdev
= opaque
;
882 assert(vdev
->device_endian
!= VIRTIO_DEVICE_ENDIAN_UNKNOWN
);
883 return vdev
->device_endian
!= virtio_default_endian();
886 static const VMStateDescription vmstate_virtio_device_endian
= {
887 .name
= "virtio/device_endian",
889 .minimum_version_id
= 1,
890 .fields
= (VMStateField
[]) {
891 VMSTATE_UINT8(device_endian
, VirtIODevice
),
892 VMSTATE_END_OF_LIST()
896 static const VMStateDescription vmstate_virtio
= {
899 .minimum_version_id
= 1,
900 .minimum_version_id_old
= 1,
901 .fields
= (VMStateField
[]) {
902 VMSTATE_END_OF_LIST()
904 .subsections
= (VMStateSubsection
[]) {
906 .vmsd
= &vmstate_virtio_device_endian
,
907 .needed
= &virtio_device_endian_needed
913 void virtio_save(VirtIODevice
*vdev
, QEMUFile
*f
)
915 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
916 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
917 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(vdev
);
920 if (k
->save_config
) {
921 k
->save_config(qbus
->parent
, f
);
924 qemu_put_8s(f
, &vdev
->status
);
925 qemu_put_8s(f
, &vdev
->isr
);
926 qemu_put_be16s(f
, &vdev
->queue_sel
);
927 qemu_put_be32s(f
, &vdev
->guest_features
);
928 qemu_put_be32(f
, vdev
->config_len
);
929 qemu_put_buffer(f
, vdev
->config
, vdev
->config_len
);
931 for (i
= 0; i
< VIRTIO_PCI_QUEUE_MAX
; i
++) {
932 if (vdev
->vq
[i
].vring
.num
== 0)
938 for (i
= 0; i
< VIRTIO_PCI_QUEUE_MAX
; i
++) {
939 if (vdev
->vq
[i
].vring
.num
== 0)
942 qemu_put_be32(f
, vdev
->vq
[i
].vring
.num
);
943 if (k
->has_variable_vring_alignment
) {
944 qemu_put_be32(f
, vdev
->vq
[i
].vring
.align
);
946 qemu_put_be64(f
, vdev
->vq
[i
].pa
);
947 qemu_put_be16s(f
, &vdev
->vq
[i
].last_avail_idx
);
949 k
->save_queue(qbus
->parent
, i
, f
);
953 if (vdc
->save
!= NULL
) {
958 vmstate_save_state(f
, &vmstate_virtio
, vdev
);
961 int virtio_set_features(VirtIODevice
*vdev
, uint32_t val
)
963 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
964 VirtioBusClass
*vbusk
= VIRTIO_BUS_GET_CLASS(qbus
);
965 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
966 uint32_t supported_features
= vbusk
->get_features(qbus
->parent
);
967 bool bad
= (val
& ~supported_features
) != 0;
969 val
&= supported_features
;
970 if (k
->set_features
) {
971 k
->set_features(vdev
, val
);
973 vdev
->guest_features
= val
;
977 int virtio_load(VirtIODevice
*vdev
, QEMUFile
*f
, int version_id
)
983 uint32_t supported_features
;
984 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
985 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
986 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(vdev
);
989 * We poison the endianness to ensure it does not get used before
990 * subsections have been loaded.
992 vdev
->device_endian
= VIRTIO_DEVICE_ENDIAN_UNKNOWN
;
994 if (k
->load_config
) {
995 ret
= k
->load_config(qbus
->parent
, f
);
1000 qemu_get_8s(f
, &vdev
->status
);
1001 qemu_get_8s(f
, &vdev
->isr
);
1002 qemu_get_be16s(f
, &vdev
->queue_sel
);
1003 if (vdev
->queue_sel
>= VIRTIO_PCI_QUEUE_MAX
) {
1006 qemu_get_be32s(f
, &features
);
1008 if (virtio_set_features(vdev
, features
) < 0) {
1009 supported_features
= k
->get_features(qbus
->parent
);
1010 error_report("Features 0x%x unsupported. Allowed features: 0x%x",
1011 features
, supported_features
);
1014 config_len
= qemu_get_be32(f
);
1017 * There are cases where the incoming config can be bigger or smaller
1018 * than what we have; so load what we have space for, and skip
1019 * any excess that's in the stream.
1021 qemu_get_buffer(f
, vdev
->config
, MIN(config_len
, vdev
->config_len
));
1023 while (config_len
> vdev
->config_len
) {
1028 num
= qemu_get_be32(f
);
1030 if (num
> VIRTIO_PCI_QUEUE_MAX
) {
1031 error_report("Invalid number of PCI queues: 0x%x", num
);
1035 for (i
= 0; i
< num
; i
++) {
1036 vdev
->vq
[i
].vring
.num
= qemu_get_be32(f
);
1037 if (k
->has_variable_vring_alignment
) {
1038 vdev
->vq
[i
].vring
.align
= qemu_get_be32(f
);
1040 vdev
->vq
[i
].pa
= qemu_get_be64(f
);
1041 qemu_get_be16s(f
, &vdev
->vq
[i
].last_avail_idx
);
1042 vdev
->vq
[i
].signalled_used_valid
= false;
1043 vdev
->vq
[i
].notification
= true;
1045 if (vdev
->vq
[i
].pa
) {
1046 virtqueue_init(&vdev
->vq
[i
]);
1047 } else if (vdev
->vq
[i
].last_avail_idx
) {
1048 error_report("VQ %d address 0x0 "
1049 "inconsistent with Host index 0x%x",
1050 i
, vdev
->vq
[i
].last_avail_idx
);
1053 if (k
->load_queue
) {
1054 ret
= k
->load_queue(qbus
->parent
, i
, f
);
1060 virtio_notify_vector(vdev
, VIRTIO_NO_VECTOR
);
1062 if (vdc
->load
!= NULL
) {
1063 ret
= vdc
->load(vdev
, f
, version_id
);
1070 ret
= vmstate_load_state(f
, &vmstate_virtio
, vdev
, 1);
1075 if (vdev
->device_endian
== VIRTIO_DEVICE_ENDIAN_UNKNOWN
) {
1076 vdev
->device_endian
= virtio_default_endian();
1079 for (i
= 0; i
< num
; i
++) {
1080 if (vdev
->vq
[i
].pa
) {
1082 nheads
= vring_avail_idx(&vdev
->vq
[i
]) - vdev
->vq
[i
].last_avail_idx
;
1083 /* Check it isn't doing strange things with descriptor numbers. */
1084 if (nheads
> vdev
->vq
[i
].vring
.num
) {
1085 error_report("VQ %d size 0x%x Guest index 0x%x "
1086 "inconsistent with Host index 0x%x: delta 0x%x",
1087 i
, vdev
->vq
[i
].vring
.num
,
1088 vring_avail_idx(&vdev
->vq
[i
]),
1089 vdev
->vq
[i
].last_avail_idx
, nheads
);
1098 void virtio_cleanup(VirtIODevice
*vdev
)
1100 qemu_del_vm_change_state_handler(vdev
->vmstate
);
1101 g_free(vdev
->config
);
1105 static void virtio_vmstate_change(void *opaque
, int running
, RunState state
)
1107 VirtIODevice
*vdev
= opaque
;
1108 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1109 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1110 bool backend_run
= running
&& (vdev
->status
& VIRTIO_CONFIG_S_DRIVER_OK
);
1111 vdev
->vm_running
= running
;
1114 virtio_set_status(vdev
, vdev
->status
);
1117 if (k
->vmstate_change
) {
1118 k
->vmstate_change(qbus
->parent
, backend_run
);
1122 virtio_set_status(vdev
, vdev
->status
);
1126 void virtio_init(VirtIODevice
*vdev
, const char *name
,
1127 uint16_t device_id
, size_t config_size
)
1130 vdev
->device_id
= device_id
;
1133 vdev
->queue_sel
= 0;
1134 vdev
->config_vector
= VIRTIO_NO_VECTOR
;
1135 vdev
->vq
= g_malloc0(sizeof(VirtQueue
) * VIRTIO_PCI_QUEUE_MAX
);
1136 vdev
->vm_running
= runstate_is_running();
1137 for (i
= 0; i
< VIRTIO_PCI_QUEUE_MAX
; i
++) {
1138 vdev
->vq
[i
].vector
= VIRTIO_NO_VECTOR
;
1139 vdev
->vq
[i
].vdev
= vdev
;
1140 vdev
->vq
[i
].queue_index
= i
;
1144 vdev
->config_len
= config_size
;
1145 if (vdev
->config_len
) {
1146 vdev
->config
= g_malloc0(config_size
);
1148 vdev
->config
= NULL
;
1150 vdev
->vmstate
= qemu_add_vm_change_state_handler(virtio_vmstate_change
,
1152 vdev
->device_endian
= virtio_default_endian();
1155 hwaddr
virtio_queue_get_desc_addr(VirtIODevice
*vdev
, int n
)
1157 return vdev
->vq
[n
].vring
.desc
;
1160 hwaddr
virtio_queue_get_avail_addr(VirtIODevice
*vdev
, int n
)
1162 return vdev
->vq
[n
].vring
.avail
;
1165 hwaddr
virtio_queue_get_used_addr(VirtIODevice
*vdev
, int n
)
1167 return vdev
->vq
[n
].vring
.used
;
1170 hwaddr
virtio_queue_get_ring_addr(VirtIODevice
*vdev
, int n
)
1172 return vdev
->vq
[n
].vring
.desc
;
1175 hwaddr
virtio_queue_get_desc_size(VirtIODevice
*vdev
, int n
)
1177 return sizeof(VRingDesc
) * vdev
->vq
[n
].vring
.num
;
1180 hwaddr
virtio_queue_get_avail_size(VirtIODevice
*vdev
, int n
)
1182 return offsetof(VRingAvail
, ring
) +
1183 sizeof(uint64_t) * vdev
->vq
[n
].vring
.num
;
1186 hwaddr
virtio_queue_get_used_size(VirtIODevice
*vdev
, int n
)
1188 return offsetof(VRingUsed
, ring
) +
1189 sizeof(VRingUsedElem
) * vdev
->vq
[n
].vring
.num
;
1192 hwaddr
virtio_queue_get_ring_size(VirtIODevice
*vdev
, int n
)
1194 return vdev
->vq
[n
].vring
.used
- vdev
->vq
[n
].vring
.desc
+
1195 virtio_queue_get_used_size(vdev
, n
);
1198 uint16_t virtio_queue_get_last_avail_idx(VirtIODevice
*vdev
, int n
)
1200 return vdev
->vq
[n
].last_avail_idx
;
1203 void virtio_queue_set_last_avail_idx(VirtIODevice
*vdev
, int n
, uint16_t idx
)
1205 vdev
->vq
[n
].last_avail_idx
= idx
;
1208 void virtio_queue_invalidate_signalled_used(VirtIODevice
*vdev
, int n
)
1210 vdev
->vq
[n
].signalled_used_valid
= false;
1213 VirtQueue
*virtio_get_queue(VirtIODevice
*vdev
, int n
)
1215 return vdev
->vq
+ n
;
1218 uint16_t virtio_get_queue_index(VirtQueue
*vq
)
1220 return vq
->queue_index
;
1223 static void virtio_queue_guest_notifier_read(EventNotifier
*n
)
1225 VirtQueue
*vq
= container_of(n
, VirtQueue
, guest_notifier
);
1226 if (event_notifier_test_and_clear(n
)) {
1231 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue
*vq
, bool assign
,
1234 if (assign
&& !with_irqfd
) {
1235 event_notifier_set_handler(&vq
->guest_notifier
,
1236 virtio_queue_guest_notifier_read
);
1238 event_notifier_set_handler(&vq
->guest_notifier
, NULL
);
1241 /* Test and clear notifier before closing it,
1242 * in case poll callback didn't have time to run. */
1243 virtio_queue_guest_notifier_read(&vq
->guest_notifier
);
1247 EventNotifier
*virtio_queue_get_guest_notifier(VirtQueue
*vq
)
1249 return &vq
->guest_notifier
;
1252 static void virtio_queue_host_notifier_read(EventNotifier
*n
)
1254 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
1255 if (event_notifier_test_and_clear(n
)) {
1256 virtio_queue_notify_vq(vq
);
1260 void virtio_queue_set_host_notifier_fd_handler(VirtQueue
*vq
, bool assign
,
1263 if (assign
&& set_handler
) {
1264 event_notifier_set_handler(&vq
->host_notifier
,
1265 virtio_queue_host_notifier_read
);
1267 event_notifier_set_handler(&vq
->host_notifier
, NULL
);
1270 /* Test and clear notifier before after disabling event,
1271 * in case poll callback didn't have time to run. */
1272 virtio_queue_host_notifier_read(&vq
->host_notifier
);
1276 EventNotifier
*virtio_queue_get_host_notifier(VirtQueue
*vq
)
1278 return &vq
->host_notifier
;
1281 void virtio_device_set_child_bus_name(VirtIODevice
*vdev
, char *bus_name
)
1283 g_free(vdev
->bus_name
);
1284 vdev
->bus_name
= g_strdup(bus_name
);
1287 static void virtio_device_realize(DeviceState
*dev
, Error
**errp
)
1289 VirtIODevice
*vdev
= VIRTIO_DEVICE(dev
);
1290 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(dev
);
1293 if (vdc
->realize
!= NULL
) {
1294 vdc
->realize(dev
, &err
);
1296 error_propagate(errp
, err
);
1300 virtio_bus_device_plugged(vdev
);
1303 static void virtio_device_unrealize(DeviceState
*dev
, Error
**errp
)
1305 VirtIODevice
*vdev
= VIRTIO_DEVICE(dev
);
1306 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(dev
);
1309 virtio_bus_device_unplugged(vdev
);
1311 if (vdc
->unrealize
!= NULL
) {
1312 vdc
->unrealize(dev
, &err
);
1314 error_propagate(errp
, err
);
1319 g_free(vdev
->bus_name
);
1320 vdev
->bus_name
= NULL
;
1323 static void virtio_device_class_init(ObjectClass
*klass
, void *data
)
1325 /* Set the default value here. */
1326 DeviceClass
*dc
= DEVICE_CLASS(klass
);
1328 dc
->realize
= virtio_device_realize
;
1329 dc
->unrealize
= virtio_device_unrealize
;
1330 dc
->bus_type
= TYPE_VIRTIO_BUS
;
1333 static const TypeInfo virtio_device_info
= {
1334 .name
= TYPE_VIRTIO_DEVICE
,
1335 .parent
= TYPE_DEVICE
,
1336 .instance_size
= sizeof(VirtIODevice
),
1337 .class_init
= virtio_device_class_init
,
1339 .class_size
= sizeof(VirtioDeviceClass
),
1342 static void virtio_register_types(void)
1344 type_register_static(&virtio_device_info
);
1347 type_init(virtio_register_types
)