4 * Copyright IBM, Corp. 2007
7 * Anthony Liguori <aliguori@us.ibm.com>
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
19 /* The alignment to use between consumer and producer parts of vring.
20 * x86 pagesize again. */
21 #define VIRTIO_PCI_VRING_ALIGN 4096
23 /* QEMU doesn't strictly need write barriers since everything runs in
24 * lock-step. We'll leave the calls to wmb() in though to make it obvious for
25 * KVM or if kqemu gets SMP support.
27 #define wmb() do { } while (0)
29 typedef struct VRingDesc
37 typedef struct VRingAvail
44 typedef struct VRingUsedElem
50 typedef struct VRingUsed
54 VRingUsedElem ring
[0];
60 target_phys_addr_t desc
;
61 target_phys_addr_t avail
;
62 target_phys_addr_t used
;
68 target_phys_addr_t pa
;
69 uint16_t last_avail_idx
;
71 void (*handle_output
)(VirtIODevice
*vdev
, VirtQueue
*vq
);
74 #define VIRTIO_PCI_QUEUE_MAX 16
76 /* virt queue functions */
77 static void virtqueue_init(VirtQueue
*vq
)
79 target_phys_addr_t pa
= vq
->pa
;
82 vq
->vring
.avail
= pa
+ vq
->vring
.num
* sizeof(VRingDesc
);
83 vq
->vring
.used
= vring_align(vq
->vring
.avail
+
84 offsetof(VRingAvail
, ring
[vq
->vring
.num
]),
85 VIRTIO_PCI_VRING_ALIGN
);
88 static inline uint64_t vring_desc_addr(target_phys_addr_t desc_pa
, int i
)
90 target_phys_addr_t pa
;
91 pa
= desc_pa
+ sizeof(VRingDesc
) * i
+ offsetof(VRingDesc
, addr
);
95 static inline uint32_t vring_desc_len(target_phys_addr_t desc_pa
, int i
)
97 target_phys_addr_t pa
;
98 pa
= desc_pa
+ sizeof(VRingDesc
) * i
+ offsetof(VRingDesc
, len
);
102 static inline uint16_t vring_desc_flags(target_phys_addr_t desc_pa
, int i
)
104 target_phys_addr_t pa
;
105 pa
= desc_pa
+ sizeof(VRingDesc
) * i
+ offsetof(VRingDesc
, flags
);
106 return lduw_phys(pa
);
109 static inline uint16_t vring_desc_next(target_phys_addr_t desc_pa
, int i
)
111 target_phys_addr_t pa
;
112 pa
= desc_pa
+ sizeof(VRingDesc
) * i
+ offsetof(VRingDesc
, next
);
113 return lduw_phys(pa
);
116 static inline uint16_t vring_avail_flags(VirtQueue
*vq
)
118 target_phys_addr_t pa
;
119 pa
= vq
->vring
.avail
+ offsetof(VRingAvail
, flags
);
120 return lduw_phys(pa
);
123 static inline uint16_t vring_avail_idx(VirtQueue
*vq
)
125 target_phys_addr_t pa
;
126 pa
= vq
->vring
.avail
+ offsetof(VRingAvail
, idx
);
127 return lduw_phys(pa
);
130 static inline uint16_t vring_avail_ring(VirtQueue
*vq
, int i
)
132 target_phys_addr_t pa
;
133 pa
= vq
->vring
.avail
+ offsetof(VRingAvail
, ring
[i
]);
134 return lduw_phys(pa
);
137 static inline void vring_used_ring_id(VirtQueue
*vq
, int i
, uint32_t val
)
139 target_phys_addr_t pa
;
140 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, ring
[i
].id
);
144 static inline void vring_used_ring_len(VirtQueue
*vq
, int i
, uint32_t val
)
146 target_phys_addr_t pa
;
147 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, ring
[i
].len
);
151 static uint16_t vring_used_idx(VirtQueue
*vq
)
153 target_phys_addr_t pa
;
154 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, idx
);
155 return lduw_phys(pa
);
158 static inline void vring_used_idx_increment(VirtQueue
*vq
, uint16_t val
)
160 target_phys_addr_t pa
;
161 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, idx
);
162 stw_phys(pa
, vring_used_idx(vq
) + val
);
165 static inline void vring_used_flags_set_bit(VirtQueue
*vq
, int mask
)
167 target_phys_addr_t pa
;
168 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, flags
);
169 stw_phys(pa
, lduw_phys(pa
) | mask
);
172 static inline void vring_used_flags_unset_bit(VirtQueue
*vq
, int mask
)
174 target_phys_addr_t pa
;
175 pa
= vq
->vring
.used
+ offsetof(VRingUsed
, flags
);
176 stw_phys(pa
, lduw_phys(pa
) & ~mask
);
179 void virtio_queue_set_notification(VirtQueue
*vq
, int enable
)
182 vring_used_flags_unset_bit(vq
, VRING_USED_F_NO_NOTIFY
);
184 vring_used_flags_set_bit(vq
, VRING_USED_F_NO_NOTIFY
);
187 int virtio_queue_ready(VirtQueue
*vq
)
189 return vq
->vring
.avail
!= 0;
192 int virtio_queue_empty(VirtQueue
*vq
)
194 return vring_avail_idx(vq
) == vq
->last_avail_idx
;
197 void virtqueue_fill(VirtQueue
*vq
, const VirtQueueElement
*elem
,
198 unsigned int len
, unsigned int idx
)
204 for (i
= 0; i
< elem
->in_num
; i
++) {
205 size_t size
= MIN(len
- offset
, elem
->in_sg
[i
].iov_len
);
207 cpu_physical_memory_unmap(elem
->in_sg
[i
].iov_base
,
208 elem
->in_sg
[i
].iov_len
,
211 offset
+= elem
->in_sg
[i
].iov_len
;
214 for (i
= 0; i
< elem
->out_num
; i
++)
215 cpu_physical_memory_unmap(elem
->out_sg
[i
].iov_base
,
216 elem
->out_sg
[i
].iov_len
,
217 0, elem
->out_sg
[i
].iov_len
);
219 idx
= (idx
+ vring_used_idx(vq
)) % vq
->vring
.num
;
221 /* Get a pointer to the next entry in the used ring. */
222 vring_used_ring_id(vq
, idx
, elem
->index
);
223 vring_used_ring_len(vq
, idx
, len
);
226 void virtqueue_flush(VirtQueue
*vq
, unsigned int count
)
228 /* Make sure buffer is written before we update index. */
230 vring_used_idx_increment(vq
, count
);
234 void virtqueue_push(VirtQueue
*vq
, const VirtQueueElement
*elem
,
237 virtqueue_fill(vq
, elem
, len
, 0);
238 virtqueue_flush(vq
, 1);
241 static int virtqueue_num_heads(VirtQueue
*vq
, unsigned int idx
)
243 uint16_t num_heads
= vring_avail_idx(vq
) - idx
;
245 /* Check it isn't doing very strange things with descriptor numbers. */
246 if (num_heads
> vq
->vring
.num
) {
247 fprintf(stderr
, "Guest moved used index from %u to %u",
248 idx
, vring_avail_idx(vq
));
255 static unsigned int virtqueue_get_head(VirtQueue
*vq
, unsigned int idx
)
259 /* Grab the next descriptor number they're advertising, and increment
260 * the index we've seen. */
261 head
= vring_avail_ring(vq
, idx
% vq
->vring
.num
);
263 /* If their number is silly, that's a fatal mistake. */
264 if (head
>= vq
->vring
.num
) {
265 fprintf(stderr
, "Guest says index %u is available", head
);
272 static unsigned virtqueue_next_desc(target_phys_addr_t desc_pa
,
273 unsigned int i
, unsigned int max
)
277 /* If this descriptor says it doesn't chain, we're done. */
278 if (!(vring_desc_flags(desc_pa
, i
) & VRING_DESC_F_NEXT
))
281 /* Check they're not leading us off end of descriptors. */
282 next
= vring_desc_next(desc_pa
, i
);
283 /* Make sure compiler knows to grab that: we don't want it changing! */
287 fprintf(stderr
, "Desc next is %u", next
);
294 int virtqueue_avail_bytes(VirtQueue
*vq
, int in_bytes
, int out_bytes
)
297 int total_bufs
, in_total
, out_total
;
299 idx
= vq
->last_avail_idx
;
301 total_bufs
= in_total
= out_total
= 0;
302 while (virtqueue_num_heads(vq
, idx
)) {
303 unsigned int max
, num_bufs
, indirect
= 0;
304 target_phys_addr_t desc_pa
;
308 num_bufs
= total_bufs
;
309 i
= virtqueue_get_head(vq
, idx
++);
310 desc_pa
= vq
->vring
.desc
;
312 if (vring_desc_flags(desc_pa
, i
) & VRING_DESC_F_INDIRECT
) {
313 if (vring_desc_len(desc_pa
, i
) % sizeof(VRingDesc
)) {
314 fprintf(stderr
, "Invalid size for indirect buffer table\n");
318 /* If we've got too many, that implies a descriptor loop. */
319 if (num_bufs
>= max
) {
320 fprintf(stderr
, "Looped descriptor");
324 /* loop over the indirect descriptor table */
326 max
= vring_desc_len(desc_pa
, i
) / sizeof(VRingDesc
);
328 desc_pa
= vring_desc_addr(desc_pa
, i
);
332 /* If we've got too many, that implies a descriptor loop. */
333 if (++num_bufs
> max
) {
334 fprintf(stderr
, "Looped descriptor");
338 if (vring_desc_flags(desc_pa
, i
) & VRING_DESC_F_WRITE
) {
340 (in_total
+= vring_desc_len(desc_pa
, i
)) >= in_bytes
)
344 (out_total
+= vring_desc_len(desc_pa
, i
)) >= out_bytes
)
347 } while ((i
= virtqueue_next_desc(desc_pa
, i
, max
)) != max
);
350 total_bufs
= num_bufs
;
358 int virtqueue_pop(VirtQueue
*vq
, VirtQueueElement
*elem
)
360 unsigned int i
, head
, max
;
361 target_phys_addr_t desc_pa
= vq
->vring
.desc
;
362 target_phys_addr_t len
;
364 if (!virtqueue_num_heads(vq
, vq
->last_avail_idx
))
367 /* When we start there are none of either input nor output. */
368 elem
->out_num
= elem
->in_num
= 0;
372 i
= head
= virtqueue_get_head(vq
, vq
->last_avail_idx
++);
374 if (vring_desc_flags(desc_pa
, i
) & VRING_DESC_F_INDIRECT
) {
375 if (vring_desc_len(desc_pa
, i
) % sizeof(VRingDesc
)) {
376 fprintf(stderr
, "Invalid size for indirect buffer table\n");
380 /* loop over the indirect descriptor table */
381 max
= vring_desc_len(desc_pa
, i
) / sizeof(VRingDesc
);
382 desc_pa
= vring_desc_addr(desc_pa
, i
);
390 if (vring_desc_flags(desc_pa
, i
) & VRING_DESC_F_WRITE
) {
391 elem
->in_addr
[elem
->in_num
] = vring_desc_addr(desc_pa
, i
);
392 sg
= &elem
->in_sg
[elem
->in_num
++];
395 sg
= &elem
->out_sg
[elem
->out_num
++];
397 /* Grab the first descriptor, and check it's OK. */
398 sg
->iov_len
= vring_desc_len(desc_pa
, i
);
401 sg
->iov_base
= cpu_physical_memory_map(vring_desc_addr(desc_pa
, i
),
404 if (sg
->iov_base
== NULL
|| len
!= sg
->iov_len
) {
405 fprintf(stderr
, "virtio: trying to map MMIO memory\n");
409 /* If we've got too many, that implies a descriptor loop. */
410 if ((elem
->in_num
+ elem
->out_num
) > max
) {
411 fprintf(stderr
, "Looped descriptor");
414 } while ((i
= virtqueue_next_desc(desc_pa
, i
, max
)) != max
);
420 return elem
->in_num
+ elem
->out_num
;
425 void virtio_update_irq(VirtIODevice
*vdev
)
427 if (vdev
->binding
->update_irq
) {
428 vdev
->binding
->update_irq(vdev
->binding_opaque
);
432 void virtio_reset(void *opaque
)
434 VirtIODevice
*vdev
= opaque
;
444 virtio_update_irq(vdev
);
446 for(i
= 0; i
< VIRTIO_PCI_QUEUE_MAX
; i
++) {
447 vdev
->vq
[i
].vring
.desc
= 0;
448 vdev
->vq
[i
].vring
.avail
= 0;
449 vdev
->vq
[i
].vring
.used
= 0;
450 vdev
->vq
[i
].last_avail_idx
= 0;
455 uint32_t virtio_config_readb(VirtIODevice
*vdev
, uint32_t addr
)
459 vdev
->get_config(vdev
, vdev
->config
);
461 if (addr
> (vdev
->config_len
- sizeof(val
)))
464 memcpy(&val
, vdev
->config
+ addr
, sizeof(val
));
468 uint32_t virtio_config_readw(VirtIODevice
*vdev
, uint32_t addr
)
472 vdev
->get_config(vdev
, vdev
->config
);
474 if (addr
> (vdev
->config_len
- sizeof(val
)))
477 memcpy(&val
, vdev
->config
+ addr
, sizeof(val
));
481 uint32_t virtio_config_readl(VirtIODevice
*vdev
, uint32_t addr
)
485 vdev
->get_config(vdev
, vdev
->config
);
487 if (addr
> (vdev
->config_len
- sizeof(val
)))
490 memcpy(&val
, vdev
->config
+ addr
, sizeof(val
));
494 void virtio_config_writeb(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
498 if (addr
> (vdev
->config_len
- sizeof(val
)))
501 memcpy(vdev
->config
+ addr
, &val
, sizeof(val
));
503 if (vdev
->set_config
)
504 vdev
->set_config(vdev
, vdev
->config
);
507 void virtio_config_writew(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
511 if (addr
> (vdev
->config_len
- sizeof(val
)))
514 memcpy(vdev
->config
+ addr
, &val
, sizeof(val
));
516 if (vdev
->set_config
)
517 vdev
->set_config(vdev
, vdev
->config
);
520 void virtio_config_writel(VirtIODevice
*vdev
, uint32_t addr
, uint32_t data
)
524 if (addr
> (vdev
->config_len
- sizeof(val
)))
527 memcpy(vdev
->config
+ addr
, &val
, sizeof(val
));
529 if (vdev
->set_config
)
530 vdev
->set_config(vdev
, vdev
->config
);
533 void virtio_queue_set_addr(VirtIODevice
*vdev
, int n
, target_phys_addr_t addr
)
538 vdev
->vq
[n
].pa
= addr
;
539 virtqueue_init(&vdev
->vq
[n
]);
543 target_phys_addr_t
virtio_queue_get_addr(VirtIODevice
*vdev
, int n
)
545 return vdev
->vq
[n
].pa
;
548 int virtio_queue_get_num(VirtIODevice
*vdev
, int n
)
550 return vdev
->vq
[n
].vring
.num
;
553 void virtio_queue_notify(VirtIODevice
*vdev
, int n
)
555 if (n
< VIRTIO_PCI_QUEUE_MAX
&& vdev
->vq
[n
].vring
.desc
) {
556 vdev
->vq
[n
].handle_output(vdev
, &vdev
->vq
[n
]);
560 VirtQueue
*virtio_add_queue(VirtIODevice
*vdev
, int queue_size
,
561 void (*handle_output
)(VirtIODevice
*, VirtQueue
*))
565 for (i
= 0; i
< VIRTIO_PCI_QUEUE_MAX
; i
++) {
566 if (vdev
->vq
[i
].vring
.num
== 0)
570 if (i
== VIRTIO_PCI_QUEUE_MAX
|| queue_size
> VIRTQUEUE_MAX_SIZE
)
573 vdev
->vq
[i
].vring
.num
= queue_size
;
574 vdev
->vq
[i
].handle_output
= handle_output
;
579 void virtio_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
581 /* Always notify when queue is empty (when feature acknowledge) */
582 if ((vring_avail_flags(vq
) & VRING_AVAIL_F_NO_INTERRUPT
) &&
583 (!(vdev
->features
& (1 << VIRTIO_F_NOTIFY_ON_EMPTY
)) ||
584 (vq
->inuse
|| vring_avail_idx(vq
) != vq
->last_avail_idx
)))
588 virtio_update_irq(vdev
);
591 void virtio_notify_config(VirtIODevice
*vdev
)
593 if (!(vdev
->status
& VIRTIO_CONFIG_S_DRIVER_OK
))
597 virtio_update_irq(vdev
);
600 void virtio_save(VirtIODevice
*vdev
, QEMUFile
*f
)
604 /* FIXME: load/save binding. */
605 //pci_device_save(&vdev->pci_dev, f);
607 qemu_put_8s(f
, &vdev
->status
);
608 qemu_put_8s(f
, &vdev
->isr
);
609 qemu_put_be16s(f
, &vdev
->queue_sel
);
610 qemu_put_be32s(f
, &vdev
->features
);
611 qemu_put_be32(f
, vdev
->config_len
);
612 qemu_put_buffer(f
, vdev
->config
, vdev
->config_len
);
614 for (i
= 0; i
< VIRTIO_PCI_QUEUE_MAX
; i
++) {
615 if (vdev
->vq
[i
].vring
.num
== 0)
621 for (i
= 0; i
< VIRTIO_PCI_QUEUE_MAX
; i
++) {
622 if (vdev
->vq
[i
].vring
.num
== 0)
625 qemu_put_be32(f
, vdev
->vq
[i
].vring
.num
);
626 qemu_put_be64(f
, vdev
->vq
[i
].pa
);
627 qemu_put_be16s(f
, &vdev
->vq
[i
].last_avail_idx
);
631 void virtio_load(VirtIODevice
*vdev
, QEMUFile
*f
)
635 /* FIXME: load/save binding. */
636 //pci_device_load(&vdev->pci_dev, f);
638 qemu_get_8s(f
, &vdev
->status
);
639 qemu_get_8s(f
, &vdev
->isr
);
640 qemu_get_be16s(f
, &vdev
->queue_sel
);
641 qemu_get_be32s(f
, &vdev
->features
);
642 vdev
->config_len
= qemu_get_be32(f
);
643 qemu_get_buffer(f
, vdev
->config
, vdev
->config_len
);
645 num
= qemu_get_be32(f
);
647 for (i
= 0; i
< num
; i
++) {
648 vdev
->vq
[i
].vring
.num
= qemu_get_be32(f
);
649 vdev
->vq
[i
].pa
= qemu_get_be64(f
);
650 qemu_get_be16s(f
, &vdev
->vq
[i
].last_avail_idx
);
652 if (vdev
->vq
[i
].pa
) {
653 virtqueue_init(&vdev
->vq
[i
]);
657 virtio_update_irq(vdev
);
660 void virtio_cleanup(VirtIODevice
*vdev
)
663 qemu_free(vdev
->config
);
667 VirtIODevice
*virtio_common_init(const char *name
, uint16_t device_id
,
668 size_t config_size
, size_t struct_size
)
672 vdev
= qemu_mallocz(struct_size
);
674 vdev
->device_id
= device_id
;
678 vdev
->vq
= qemu_mallocz(sizeof(VirtQueue
) * VIRTIO_PCI_QUEUE_MAX
);
681 vdev
->config_len
= config_size
;
682 if (vdev
->config_len
)
683 vdev
->config
= qemu_mallocz(config_size
);
687 qemu_register_reset(virtio_reset
, 0, vdev
);
692 void virtio_bind_device(VirtIODevice
*vdev
, const VirtIOBindings
*binding
,
695 vdev
->binding
= binding
;
696 vdev
->binding_opaque
= opaque
;