4 * Copyright IBM, Corp. 2007
7 * Anthony Liguori <aliguori@us.ibm.com>
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
17 #include "qemu-error.h"
19 #include "qemu-barrier.h"
21 /* The alignment to use between consumer and producer parts of vring.
22 * x86 pagesize again. */
23 #define VIRTIO_PCI_VRING_ALIGN 4096
25 typedef struct VRingDesc
33 typedef struct VRingAvail
40 typedef struct VRingUsedElem
46 typedef struct VRingUsed
50 VRingUsedElem ring
[0];
56 target_phys_addr_t desc
;
57 target_phys_addr_t avail
;
58 target_phys_addr_t used
;
64 target_phys_addr_t pa
;
65 uint16_t last_avail_idx
;
66 /* Last used index value we have signalled on */
67 uint16_t signalled_used
;
69 /* Last used index value we have signalled on */
70 bool signalled_used_valid
;
72 /* Notification enabled? */
78 void (*handle_output
)(VirtIODevice
*vdev
, VirtQueue
*vq
);
80 EventNotifier guest_notifier
;
81 EventNotifier host_notifier
;
84 /* virt queue functions */
85 static void virtqueue_init(VirtQueue
*vq
)
87 target_phys_addr_t pa
= vq
->pa
;
90 vq
->vring
.avail
= pa
+ vq
->vring
.num
* sizeof(VRingDesc
);
91 vq
->vring
.used
= vring_align(vq
->vring
.avail
+
92 offsetof(VRingAvail
, ring
[vq
->vring
.num
]),
93 VIRTIO_PCI_VRING_ALIGN
);
96 static inline uint64_t vring_desc_addr(target_phys_addr_t desc_pa
, int i
)
98 target_phys_addr_t pa
;
99 pa
= desc_pa
+ sizeof(VRingDesc
) * i
+ offsetof(VRingDesc
, addr
);
103 static inline uint32_t vring_desc_len(target_phys_addr_t desc_pa
, int i
)
105 target_phys_addr_t pa
;
106 pa
= desc_pa
+ sizeof(VRingDesc
) * i
+ offsetof(VRingDesc
, len
);
110 static inline uint16_t vring_desc_flags(target_phys_addr_t desc_pa
, int i
)
112 target_phys_addr_t pa
;
113 pa
= desc_pa
+ sizeof(VRingDesc
) * i
+ offsetof(VRingDesc
, flags
);
114 return lduw_phys(pa
);
117 static inline uint16_t vring_desc_next(target_phys_addr_t desc_pa
, int i
)
119 target_phys_addr_t pa
;
120 pa
= desc_pa
+ sizeof(VRingDesc
) * i
+ offsetof(VRingDesc
, next
);
121 return lduw_phys(pa
);
124 static inline uint16_t vring_avail_flags(VirtQueue
*vq
)
126 target_phys_addr_t pa
;
127 pa
= vq
->vring
.avail
+ offsetof(VRingAvail
, flags
);
128 return lduw_phys(pa
);
131 static inline uint16_t vring_avail_idx(VirtQueue
*vq
)
133 target_phys_addr_t pa
;
134 pa
= vq
->vring
.avail
+ offsetof(VRingAvail
, idx
);
135 return lduw_phys(pa
);
138 static inline uint16_t vring_avail_ring(VirtQueue
*vq
, int i
)
140 target_phys_addr_t pa
;
141 pa
= vq
->vring
.avail
+ offsetof(VRingAvail
, ring
[i
]);
142 return lduw_phys(pa
);
145 static inline uint16_t vring_used_event(VirtQueue
*vq
)
147 return vring_avail_ring(vq
, vq
->vring
.num
);
150 static inline void vring_used_ring_id(VirtQueue
*vq
, int i
, uint32_t val
)
152 target_phys_addr_t pa
;
153 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, ring
[i
].id
);
157 static inline void vring_used_ring_len(VirtQueue
*vq
, int i
, uint32_t val
)
159 target_phys_addr_t pa
;
160 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, ring
[i
].len
);
164 static uint16_t vring_used_idx(VirtQueue
*vq
)
166 target_phys_addr_t pa
;
167 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, idx
);
168 return lduw_phys(pa
);
171 static inline void vring_used_idx_set(VirtQueue
*vq
, uint16_t val
)
173 target_phys_addr_t pa
;
174 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, idx
);
178 static inline void vring_used_flags_set_bit(VirtQueue
*vq
, int mask
)
180 target_phys_addr_t pa
;
181 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, flags
);
182 stw_phys(pa
, lduw_phys(pa
) | mask
);
185 static inline void vring_used_flags_unset_bit(VirtQueue
*vq
, int mask
)
187 target_phys_addr_t pa
;
188 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, flags
);
189 stw_phys(pa
, lduw_phys(pa
) & ~mask
);
192 static inline void vring_avail_event(VirtQueue
*vq
, uint16_t val
)
194 target_phys_addr_t pa
;
195 if (!vq
->notification
) {
198 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, ring
[vq
->vring
.num
]);
202 void virtio_queue_set_notification(VirtQueue
*vq
, int enable
)
204 vq
->notification
= enable
;
205 if (vq
->vdev
->guest_features
& (1 << VIRTIO_RING_F_EVENT_IDX
)) {
206 vring_avail_event(vq
, vring_avail_idx(vq
));
208 vring_used_flags_unset_bit(vq
, VRING_USED_F_NO_NOTIFY
);
210 vring_used_flags_set_bit(vq
, VRING_USED_F_NO_NOTIFY
);
213 /* Expose avail event/used flags before caller checks the avail idx. */
218 int virtio_queue_ready(VirtQueue
*vq
)
220 return vq
->vring
.avail
!= 0;
223 int virtio_queue_empty(VirtQueue
*vq
)
225 return vring_avail_idx(vq
) == vq
->last_avail_idx
;
228 void virtqueue_fill(VirtQueue
*vq
, const VirtQueueElement
*elem
,
229 unsigned int len
, unsigned int idx
)
234 trace_virtqueue_fill(vq
, elem
, len
, idx
);
237 for (i
= 0; i
< elem
->in_num
; i
++) {
238 size_t size
= MIN(len
- offset
, elem
->in_sg
[i
].iov_len
);
240 cpu_physical_memory_unmap(elem
->in_sg
[i
].iov_base
,
241 elem
->in_sg
[i
].iov_len
,
247 for (i
= 0; i
< elem
->out_num
; i
++)
248 cpu_physical_memory_unmap(elem
->out_sg
[i
].iov_base
,
249 elem
->out_sg
[i
].iov_len
,
250 0, elem
->out_sg
[i
].iov_len
);
252 idx
= (idx
+ vring_used_idx(vq
)) % vq
->vring
.num
;
254 /* Get a pointer to the next entry in the used ring. */
255 vring_used_ring_id(vq
, idx
, elem
->index
);
256 vring_used_ring_len(vq
, idx
, len
);
259 void virtqueue_flush(VirtQueue
*vq
, unsigned int count
)
262 /* Make sure buffer is written before we update index. */
264 trace_virtqueue_flush(vq
, count
);
265 old
= vring_used_idx(vq
);
267 vring_used_idx_set(vq
, new);
269 if (unlikely((int16_t)(new - vq
->signalled_used
) < (uint16_t)(new - old
)))
270 vq
->signalled_used_valid
= false;
273 void virtqueue_push(VirtQueue
*vq
, const VirtQueueElement
*elem
,
276 virtqueue_fill(vq
, elem
, len
, 0);
277 virtqueue_flush(vq
, 1);
280 static int virtqueue_num_heads(VirtQueue
*vq
, unsigned int idx
)
282 uint16_t num_heads
= vring_avail_idx(vq
) - idx
;
284 /* Check it isn't doing very strange things with descriptor numbers. */
285 if (num_heads
> vq
->vring
.num
) {
286 error_report("Guest moved used index from %u to %u",
287 idx
, vring_avail_idx(vq
));
290 /* On success, callers read a descriptor at vq->last_avail_idx.
291 * Make sure descriptor read does not bypass avail index read. */
299 static unsigned int virtqueue_get_head(VirtQueue
*vq
, unsigned int idx
)
303 /* Grab the next descriptor number they're advertising, and increment
304 * the index we've seen. */
305 head
= vring_avail_ring(vq
, idx
% vq
->vring
.num
);
307 /* If their number is silly, that's a fatal mistake. */
308 if (head
>= vq
->vring
.num
) {
309 error_report("Guest says index %u is available", head
);
316 static unsigned virtqueue_next_desc(target_phys_addr_t desc_pa
,
317 unsigned int i
, unsigned int max
)
321 /* If this descriptor says it doesn't chain, we're done. */
322 if (!(vring_desc_flags(desc_pa
, i
) & VRING_DESC_F_NEXT
))
325 /* Check they're not leading us off end of descriptors. */
326 next
= vring_desc_next(desc_pa
, i
);
327 /* Make sure compiler knows to grab that: we don't want it changing! */
331 error_report("Desc next is %u", next
);
338 int virtqueue_avail_bytes(VirtQueue
*vq
, int in_bytes
, int out_bytes
)
341 int total_bufs
, in_total
, out_total
;
343 idx
= vq
->last_avail_idx
;
345 total_bufs
= in_total
= out_total
= 0;
346 while (virtqueue_num_heads(vq
, idx
)) {
347 unsigned int max
, num_bufs
, indirect
= 0;
348 target_phys_addr_t desc_pa
;
352 num_bufs
= total_bufs
;
353 i
= virtqueue_get_head(vq
, idx
++);
354 desc_pa
= vq
->vring
.desc
;
356 if (vring_desc_flags(desc_pa
, i
) & VRING_DESC_F_INDIRECT
) {
357 if (vring_desc_len(desc_pa
, i
) % sizeof(VRingDesc
)) {
358 error_report("Invalid size for indirect buffer table");
362 /* If we've got too many, that implies a descriptor loop. */
363 if (num_bufs
>= max
) {
364 error_report("Looped descriptor");
368 /* loop over the indirect descriptor table */
370 max
= vring_desc_len(desc_pa
, i
) / sizeof(VRingDesc
);
372 desc_pa
= vring_desc_addr(desc_pa
, i
);
376 /* If we've got too many, that implies a descriptor loop. */
377 if (++num_bufs
> max
) {
378 error_report("Looped descriptor");
382 if (vring_desc_flags(desc_pa
, i
) & VRING_DESC_F_WRITE
) {
384 (in_total
+= vring_desc_len(desc_pa
, i
)) >= in_bytes
)
388 (out_total
+= vring_desc_len(desc_pa
, i
)) >= out_bytes
)
391 } while ((i
= virtqueue_next_desc(desc_pa
, i
, max
)) != max
);
394 total_bufs
= num_bufs
;
402 void virtqueue_map_sg(struct iovec
*sg
, target_phys_addr_t
*addr
,
403 size_t num_sg
, int is_write
)
406 target_phys_addr_t len
;
408 for (i
= 0; i
< num_sg
; i
++) {
410 sg
[i
].iov_base
= cpu_physical_memory_map(addr
[i
], &len
, is_write
);
411 if (sg
[i
].iov_base
== NULL
|| len
!= sg
[i
].iov_len
) {
412 error_report("virtio: trying to map MMIO memory");
418 int virtqueue_pop(VirtQueue
*vq
, VirtQueueElement
*elem
)
420 unsigned int i
, head
, max
;
421 target_phys_addr_t desc_pa
= vq
->vring
.desc
;
423 if (!virtqueue_num_heads(vq
, vq
->last_avail_idx
))
426 /* When we start there are none of either input nor output. */
427 elem
->out_num
= elem
->in_num
= 0;
431 i
= head
= virtqueue_get_head(vq
, vq
->last_avail_idx
++);
432 if (vq
->vdev
->guest_features
& (1 << VIRTIO_RING_F_EVENT_IDX
)) {
433 vring_avail_event(vq
, vring_avail_idx(vq
));
436 if (vring_desc_flags(desc_pa
, i
) & VRING_DESC_F_INDIRECT
) {
437 if (vring_desc_len(desc_pa
, i
) % sizeof(VRingDesc
)) {
438 error_report("Invalid size for indirect buffer table");
442 /* loop over the indirect descriptor table */
443 max
= vring_desc_len(desc_pa
, i
) / sizeof(VRingDesc
);
444 desc_pa
= vring_desc_addr(desc_pa
, i
);
448 /* Collect all the descriptors */
452 if (vring_desc_flags(desc_pa
, i
) & VRING_DESC_F_WRITE
) {
453 if (elem
->in_num
>= ARRAY_SIZE(elem
->in_sg
)) {
454 error_report("Too many write descriptors in indirect table");
457 elem
->in_addr
[elem
->in_num
] = vring_desc_addr(desc_pa
, i
);
458 sg
= &elem
->in_sg
[elem
->in_num
++];
460 if (elem
->out_num
>= ARRAY_SIZE(elem
->out_sg
)) {
461 error_report("Too many read descriptors in indirect table");
464 elem
->out_addr
[elem
->out_num
] = vring_desc_addr(desc_pa
, i
);
465 sg
= &elem
->out_sg
[elem
->out_num
++];
468 sg
->iov_len
= vring_desc_len(desc_pa
, i
);
470 /* If we've got too many, that implies a descriptor loop. */
471 if ((elem
->in_num
+ elem
->out_num
) > max
) {
472 error_report("Looped descriptor");
475 } while ((i
= virtqueue_next_desc(desc_pa
, i
, max
)) != max
);
477 /* Now map what we have collected */
478 virtqueue_map_sg(elem
->in_sg
, elem
->in_addr
, elem
->in_num
, 1);
479 virtqueue_map_sg(elem
->out_sg
, elem
->out_addr
, elem
->out_num
, 0);
485 trace_virtqueue_pop(vq
, elem
, elem
->in_num
, elem
->out_num
);
486 return elem
->in_num
+ elem
->out_num
;
490 static void virtio_notify_vector(VirtIODevice
*vdev
, uint16_t vector
)
492 if (vdev
->binding
->notify
) {
493 vdev
->binding
->notify(vdev
->binding_opaque
, vector
);
497 void virtio_update_irq(VirtIODevice
*vdev
)
499 virtio_notify_vector(vdev
, VIRTIO_NO_VECTOR
);
502 void virtio_set_status(VirtIODevice
*vdev
, uint8_t val
)
504 trace_virtio_set_status(vdev
, val
);
506 if (vdev
->set_status
) {
507 vdev
->set_status(vdev
, val
);
512 void virtio_reset(void *opaque
)
514 VirtIODevice
*vdev
= opaque
;
517 virtio_set_status(vdev
, 0);
522 vdev
->guest_features
= 0;
526 vdev
->config_vector
= VIRTIO_NO_VECTOR
;
527 virtio_notify_vector(vdev
, vdev
->config_vector
);
529 for(i
= 0; i
< VIRTIO_PCI_QUEUE_MAX
; i
++) {
530 vdev
->vq
[i
].vring
.desc
= 0;
531 vdev
->vq
[i
].vring
.avail
= 0;
532 vdev
->vq
[i
].vring
.used
= 0;
533 vdev
->vq
[i
].last_avail_idx
= 0;
535 vdev
->vq
[i
].vector
= VIRTIO_NO_VECTOR
;
536 vdev
->vq
[i
].signalled_used
= 0;
537 vdev
->vq
[i
].signalled_used_valid
= false;
538 vdev
->vq
[i
].notification
= true;
542 uint32_t virtio_config_readb(VirtIODevice
*vdev
, uint32_t addr
)
546 vdev
->get_config(vdev
, vdev
->config
);
548 if (addr
> (vdev
->config_len
- sizeof(val
)))
551 val
= ldub_p(vdev
->config
+ addr
);
555 uint32_t virtio_config_readw(VirtIODevice
*vdev
, uint32_t addr
)
559 vdev
->get_config(vdev
, vdev
->config
);
561 if (addr
> (vdev
->config_len
- sizeof(val
)))
564 val
= lduw_p(vdev
->config
+ addr
);
568 uint32_t virtio_config_readl(VirtIODevice
*vdev
, uint32_t addr
)
572 vdev
->get_config(vdev
, vdev
->config
);
574 if (addr
> (vdev
->config_len
- sizeof(val
)))
577 val
= ldl_p(vdev
->config
+ addr
);
581 void virtio_config_writeb(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
585 if (addr
> (vdev
->config_len
- sizeof(val
)))
588 stb_p(vdev
->config
+ addr
, val
);
590 if (vdev
->set_config
)
591 vdev
->set_config(vdev
, vdev
->config
);
594 void virtio_config_writew(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
598 if (addr
> (vdev
->config_len
- sizeof(val
)))
601 stw_p(vdev
->config
+ addr
, val
);
603 if (vdev
->set_config
)
604 vdev
->set_config(vdev
, vdev
->config
);
607 void virtio_config_writel(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
611 if (addr
> (vdev
->config_len
- sizeof(val
)))
614 stl_p(vdev
->config
+ addr
, val
);
616 if (vdev
->set_config
)
617 vdev
->set_config(vdev
, vdev
->config
);
620 void virtio_queue_set_addr(VirtIODevice
*vdev
, int n
, target_phys_addr_t addr
)
622 vdev
->vq
[n
].pa
= addr
;
623 virtqueue_init(&vdev
->vq
[n
]);
626 target_phys_addr_t
virtio_queue_get_addr(VirtIODevice
*vdev
, int n
)
628 return vdev
->vq
[n
].pa
;
631 int virtio_queue_get_num(VirtIODevice
*vdev
, int n
)
633 return vdev
->vq
[n
].vring
.num
;
636 int virtio_queue_get_id(VirtQueue
*vq
)
638 VirtIODevice
*vdev
= vq
->vdev
;
639 assert(vq
>= &vdev
->vq
[0] && vq
< &vdev
->vq
[VIRTIO_PCI_QUEUE_MAX
]);
640 return vq
- &vdev
->vq
[0];
643 void virtio_queue_notify_vq(VirtQueue
*vq
)
645 if (vq
->vring
.desc
) {
646 VirtIODevice
*vdev
= vq
->vdev
;
647 trace_virtio_queue_notify(vdev
, vq
- vdev
->vq
, vq
);
648 vq
->handle_output(vdev
, vq
);
652 void virtio_queue_notify(VirtIODevice
*vdev
, int n
)
654 virtio_queue_notify_vq(&vdev
->vq
[n
]);
657 uint16_t virtio_queue_vector(VirtIODevice
*vdev
, int n
)
659 return n
< VIRTIO_PCI_QUEUE_MAX
? vdev
->vq
[n
].vector
:
663 void virtio_queue_set_vector(VirtIODevice
*vdev
, int n
, uint16_t vector
)
665 if (n
< VIRTIO_PCI_QUEUE_MAX
)
666 vdev
->vq
[n
].vector
= vector
;
669 VirtQueue
*virtio_add_queue(VirtIODevice
*vdev
, int queue_size
,
670 void (*handle_output
)(VirtIODevice
*, VirtQueue
*))
674 for (i
= 0; i
< VIRTIO_PCI_QUEUE_MAX
; i
++) {
675 if (vdev
->vq
[i
].vring
.num
== 0)
679 if (i
== VIRTIO_PCI_QUEUE_MAX
|| queue_size
> VIRTQUEUE_MAX_SIZE
)
682 vdev
->vq
[i
].vring
.num
= queue_size
;
683 vdev
->vq
[i
].handle_output
= handle_output
;
688 void virtio_irq(VirtQueue
*vq
)
690 trace_virtio_irq(vq
);
691 vq
->vdev
->isr
|= 0x01;
692 virtio_notify_vector(vq
->vdev
, vq
->vector
);
695 /* Assuming a given event_idx value from the other size, if
696 * we have just incremented index from old to new_idx,
697 * should we trigger an event? */
698 static inline int vring_need_event(uint16_t event
, uint16_t new, uint16_t old
)
700 /* Note: Xen has similar logic for notification hold-off
701 * in include/xen/interface/io/ring.h with req_event and req_prod
702 * corresponding to event_idx + 1 and new respectively.
703 * Note also that req_event and req_prod in Xen start at 1,
704 * event indexes in virtio start at 0. */
705 return (uint16_t)(new - event
- 1) < (uint16_t)(new - old
);
708 static bool vring_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
712 /* We need to expose used array entries before checking used event. */
714 /* Always notify when queue is empty (when feature acknowledge) */
715 if (((vdev
->guest_features
& (1 << VIRTIO_F_NOTIFY_ON_EMPTY
)) &&
716 !vq
->inuse
&& vring_avail_idx(vq
) == vq
->last_avail_idx
)) {
720 if (!(vdev
->guest_features
& (1 << VIRTIO_RING_F_EVENT_IDX
))) {
721 return !(vring_avail_flags(vq
) & VRING_AVAIL_F_NO_INTERRUPT
);
724 v
= vq
->signalled_used_valid
;
725 vq
->signalled_used_valid
= true;
726 old
= vq
->signalled_used
;
727 new = vq
->signalled_used
= vring_used_idx(vq
);
728 return !v
|| vring_need_event(vring_used_event(vq
), new, old
);
731 void virtio_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
733 if (!vring_notify(vdev
, vq
)) {
737 trace_virtio_notify(vdev
, vq
);
739 virtio_notify_vector(vdev
, vq
->vector
);
742 void virtio_notify_config(VirtIODevice
*vdev
)
744 if (!(vdev
->status
& VIRTIO_CONFIG_S_DRIVER_OK
))
748 virtio_notify_vector(vdev
, vdev
->config_vector
);
751 void virtio_save(VirtIODevice
*vdev
, QEMUFile
*f
)
755 if (vdev
->binding
->save_config
)
756 vdev
->binding
->save_config(vdev
->binding_opaque
, f
);
758 qemu_put_8s(f
, &vdev
->status
);
759 qemu_put_8s(f
, &vdev
->isr
);
760 qemu_put_be16s(f
, &vdev
->queue_sel
);
761 qemu_put_be32s(f
, &vdev
->guest_features
);
762 qemu_put_be32(f
, vdev
->config_len
);
763 qemu_put_buffer(f
, vdev
->config
, vdev
->config_len
);
765 for (i
= 0; i
< VIRTIO_PCI_QUEUE_MAX
; i
++) {
766 if (vdev
->vq
[i
].vring
.num
== 0)
772 for (i
= 0; i
< VIRTIO_PCI_QUEUE_MAX
; i
++) {
773 if (vdev
->vq
[i
].vring
.num
== 0)
776 qemu_put_be32(f
, vdev
->vq
[i
].vring
.num
);
777 qemu_put_be64(f
, vdev
->vq
[i
].pa
);
778 qemu_put_be16s(f
, &vdev
->vq
[i
].last_avail_idx
);
779 if (vdev
->binding
->save_queue
)
780 vdev
->binding
->save_queue(vdev
->binding_opaque
, i
, f
);
784 int virtio_set_features(VirtIODevice
*vdev
, uint32_t val
)
786 uint32_t supported_features
=
787 vdev
->binding
->get_features(vdev
->binding_opaque
);
788 bool bad
= (val
& ~supported_features
) != 0;
790 val
&= supported_features
;
791 if (vdev
->set_features
) {
792 vdev
->set_features(vdev
, val
);
794 vdev
->guest_features
= val
;
798 int virtio_load(VirtIODevice
*vdev
, QEMUFile
*f
)
802 uint32_t supported_features
;
804 if (vdev
->binding
->load_config
) {
805 ret
= vdev
->binding
->load_config(vdev
->binding_opaque
, f
);
810 qemu_get_8s(f
, &vdev
->status
);
811 qemu_get_8s(f
, &vdev
->isr
);
812 qemu_get_be16s(f
, &vdev
->queue_sel
);
813 qemu_get_be32s(f
, &features
);
815 if (virtio_set_features(vdev
, features
) < 0) {
816 supported_features
= vdev
->binding
->get_features(vdev
->binding_opaque
);
817 error_report("Features 0x%x unsupported. Allowed features: 0x%x",
818 features
, supported_features
);
821 vdev
->config_len
= qemu_get_be32(f
);
822 qemu_get_buffer(f
, vdev
->config
, vdev
->config_len
);
824 num
= qemu_get_be32(f
);
826 for (i
= 0; i
< num
; i
++) {
827 vdev
->vq
[i
].vring
.num
= qemu_get_be32(f
);
828 vdev
->vq
[i
].pa
= qemu_get_be64(f
);
829 qemu_get_be16s(f
, &vdev
->vq
[i
].last_avail_idx
);
830 vdev
->vq
[i
].signalled_used_valid
= false;
831 vdev
->vq
[i
].notification
= true;
833 if (vdev
->vq
[i
].pa
) {
835 virtqueue_init(&vdev
->vq
[i
]);
836 nheads
= vring_avail_idx(&vdev
->vq
[i
]) - vdev
->vq
[i
].last_avail_idx
;
837 /* Check it isn't doing very strange things with descriptor numbers. */
838 if (nheads
> vdev
->vq
[i
].vring
.num
) {
839 error_report("VQ %d size 0x%x Guest index 0x%x "
840 "inconsistent with Host index 0x%x: delta 0x%x",
841 i
, vdev
->vq
[i
].vring
.num
,
842 vring_avail_idx(&vdev
->vq
[i
]),
843 vdev
->vq
[i
].last_avail_idx
, nheads
);
846 } else if (vdev
->vq
[i
].last_avail_idx
) {
847 error_report("VQ %d address 0x0 "
848 "inconsistent with Host index 0x%x",
849 i
, vdev
->vq
[i
].last_avail_idx
);
852 if (vdev
->binding
->load_queue
) {
853 ret
= vdev
->binding
->load_queue(vdev
->binding_opaque
, i
, f
);
859 virtio_notify_vector(vdev
, VIRTIO_NO_VECTOR
);
863 void virtio_cleanup(VirtIODevice
*vdev
)
865 qemu_del_vm_change_state_handler(vdev
->vmstate
);
866 g_free(vdev
->config
);
871 static void virtio_vmstate_change(void *opaque
, int running
, RunState state
)
873 VirtIODevice
*vdev
= opaque
;
874 bool backend_run
= running
&& (vdev
->status
& VIRTIO_CONFIG_S_DRIVER_OK
);
875 vdev
->vm_running
= running
;
878 virtio_set_status(vdev
, vdev
->status
);
881 if (vdev
->binding
->vmstate_change
) {
882 vdev
->binding
->vmstate_change(vdev
->binding_opaque
, backend_run
);
886 virtio_set_status(vdev
, vdev
->status
);
890 VirtIODevice
*virtio_common_init(const char *name
, uint16_t device_id
,
891 size_t config_size
, size_t struct_size
)
896 vdev
= g_malloc0(struct_size
);
898 vdev
->device_id
= device_id
;
902 vdev
->config_vector
= VIRTIO_NO_VECTOR
;
903 vdev
->vq
= g_malloc0(sizeof(VirtQueue
) * VIRTIO_PCI_QUEUE_MAX
);
904 vdev
->vm_running
= runstate_is_running();
905 for(i
= 0; i
< VIRTIO_PCI_QUEUE_MAX
; i
++) {
906 vdev
->vq
[i
].vector
= VIRTIO_NO_VECTOR
;
907 vdev
->vq
[i
].vdev
= vdev
;
911 vdev
->config_len
= config_size
;
912 if (vdev
->config_len
)
913 vdev
->config
= g_malloc0(config_size
);
917 vdev
->vmstate
= qemu_add_vm_change_state_handler(virtio_vmstate_change
, vdev
);
922 void virtio_bind_device(VirtIODevice
*vdev
, const VirtIOBindings
*binding
,
925 vdev
->binding
= binding
;
926 vdev
->binding_opaque
= opaque
;
929 target_phys_addr_t
virtio_queue_get_desc_addr(VirtIODevice
*vdev
, int n
)
931 return vdev
->vq
[n
].vring
.desc
;
934 target_phys_addr_t
virtio_queue_get_avail_addr(VirtIODevice
*vdev
, int n
)
936 return vdev
->vq
[n
].vring
.avail
;
939 target_phys_addr_t
virtio_queue_get_used_addr(VirtIODevice
*vdev
, int n
)
941 return vdev
->vq
[n
].vring
.used
;
944 target_phys_addr_t
virtio_queue_get_ring_addr(VirtIODevice
*vdev
, int n
)
946 return vdev
->vq
[n
].vring
.desc
;
949 target_phys_addr_t
virtio_queue_get_desc_size(VirtIODevice
*vdev
, int n
)
951 return sizeof(VRingDesc
) * vdev
->vq
[n
].vring
.num
;
954 target_phys_addr_t
virtio_queue_get_avail_size(VirtIODevice
*vdev
, int n
)
956 return offsetof(VRingAvail
, ring
) +
957 sizeof(uint64_t) * vdev
->vq
[n
].vring
.num
;
960 target_phys_addr_t
virtio_queue_get_used_size(VirtIODevice
*vdev
, int n
)
962 return offsetof(VRingUsed
, ring
) +
963 sizeof(VRingUsedElem
) * vdev
->vq
[n
].vring
.num
;
966 target_phys_addr_t
virtio_queue_get_ring_size(VirtIODevice
*vdev
, int n
)
968 return vdev
->vq
[n
].vring
.used
- vdev
->vq
[n
].vring
.desc
+
969 virtio_queue_get_used_size(vdev
, n
);
972 uint16_t virtio_queue_get_last_avail_idx(VirtIODevice
*vdev
, int n
)
974 return vdev
->vq
[n
].last_avail_idx
;
977 void virtio_queue_set_last_avail_idx(VirtIODevice
*vdev
, int n
, uint16_t idx
)
979 vdev
->vq
[n
].last_avail_idx
= idx
;
982 VirtQueue
*virtio_get_queue(VirtIODevice
*vdev
, int n
)
987 static void virtio_queue_guest_notifier_read(EventNotifier
*n
)
989 VirtQueue
*vq
= container_of(n
, VirtQueue
, guest_notifier
);
990 if (event_notifier_test_and_clear(n
)) {
995 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue
*vq
, bool assign
,
998 if (assign
&& !with_irqfd
) {
999 event_notifier_set_handler(&vq
->guest_notifier
,
1000 virtio_queue_guest_notifier_read
);
1002 event_notifier_set_handler(&vq
->guest_notifier
, NULL
);
1005 /* Test and clear notifier before closing it,
1006 * in case poll callback didn't have time to run. */
1007 virtio_queue_guest_notifier_read(&vq
->guest_notifier
);
1011 EventNotifier
*virtio_queue_get_guest_notifier(VirtQueue
*vq
)
1013 return &vq
->guest_notifier
;
1016 static void virtio_queue_host_notifier_read(EventNotifier
*n
)
1018 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
1019 if (event_notifier_test_and_clear(n
)) {
1020 virtio_queue_notify_vq(vq
);
1024 void virtio_queue_set_host_notifier_fd_handler(VirtQueue
*vq
, bool assign
,
1027 if (assign
&& set_handler
) {
1028 event_notifier_set_handler(&vq
->host_notifier
,
1029 virtio_queue_host_notifier_read
);
1031 event_notifier_set_handler(&vq
->host_notifier
, NULL
);
1034 /* Test and clear notifier before after disabling event,
1035 * in case poll callback didn't have time to run. */
1036 virtio_queue_host_notifier_read(&vq
->host_notifier
);
1040 EventNotifier
*virtio_queue_get_host_notifier(VirtQueue
*vq
)
1042 return &vq
->host_notifier
;