4 * Copyright IBM, Corp. 2007
7 * Anthony Liguori <aliguori@us.ibm.com>
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
14 #include "qemu/osdep.h"
15 #include "qapi/error.h"
16 #include "qapi/qapi-commands-virtio.h"
18 #include "qemu/error-report.h"
20 #include "qemu/main-loop.h"
21 #include "qemu/module.h"
22 #include "qom/object_interfaces.h"
23 #include "hw/core/cpu.h"
24 #include "hw/virtio/virtio.h"
25 #include "hw/virtio/vhost.h"
26 #include "migration/qemu-file-types.h"
27 #include "qemu/atomic.h"
28 #include "hw/virtio/virtio-bus.h"
29 #include "hw/qdev-properties.h"
30 #include "hw/virtio/virtio-access.h"
31 #include "sysemu/dma.h"
32 #include "sysemu/runstate.h"
33 #include "virtio-qmp.h"
35 #include "standard-headers/linux/virtio_ids.h"
36 #include "standard-headers/linux/vhost_types.h"
37 #include "standard-headers/linux/virtio_blk.h"
38 #include "standard-headers/linux/virtio_console.h"
39 #include "standard-headers/linux/virtio_gpu.h"
40 #include "standard-headers/linux/virtio_net.h"
41 #include "standard-headers/linux/virtio_scsi.h"
42 #include "standard-headers/linux/virtio_i2c.h"
43 #include "standard-headers/linux/virtio_balloon.h"
44 #include "standard-headers/linux/virtio_iommu.h"
45 #include "standard-headers/linux/virtio_mem.h"
46 #include "standard-headers/linux/virtio_vsock.h"
48 QmpVirtIODeviceList virtio_list
;
51 * Maximum size of virtio device config space
53 #define VHOST_USER_MAX_CONFIG_SIZE 256
56 * The alignment to use between consumer and producer parts of vring.
57 * x86 pagesize again. This is the default, used by transports like PCI
58 * which don't provide a means for the guest to tell the host the alignment.
60 #define VIRTIO_PCI_VRING_ALIGN 4096
62 typedef struct VRingDesc
70 typedef struct VRingPackedDesc
{
77 typedef struct VRingAvail
84 typedef struct VRingUsedElem
90 typedef struct VRingUsed
97 typedef struct VRingMemoryRegionCaches
{
99 MemoryRegionCache desc
;
100 MemoryRegionCache avail
;
101 MemoryRegionCache used
;
102 } VRingMemoryRegionCaches
;
107 unsigned int num_default
;
112 VRingMemoryRegionCaches
*caches
;
115 typedef struct VRingPackedDescEvent
{
118 } VRingPackedDescEvent
;
123 VirtQueueElement
*used_elems
;
125 /* Next head to pop */
126 uint16_t last_avail_idx
;
127 bool last_avail_wrap_counter
;
129 /* Last avail_idx read from VQ. */
130 uint16_t shadow_avail_idx
;
131 bool shadow_avail_wrap_counter
;
134 bool used_wrap_counter
;
136 /* Last used index value we have signalled on */
137 uint16_t signalled_used
;
139 /* Last used index value we have signalled on */
140 bool signalled_used_valid
;
142 /* Notification enabled? */
145 uint16_t queue_index
;
150 VirtIOHandleOutput handle_output
;
152 EventNotifier guest_notifier
;
153 EventNotifier host_notifier
;
154 bool host_notifier_enabled
;
155 QLIST_ENTRY(VirtQueue
) node
;
158 const char *virtio_device_names
[] = {
159 [VIRTIO_ID_NET
] = "virtio-net",
160 [VIRTIO_ID_BLOCK
] = "virtio-blk",
161 [VIRTIO_ID_CONSOLE
] = "virtio-serial",
162 [VIRTIO_ID_RNG
] = "virtio-rng",
163 [VIRTIO_ID_BALLOON
] = "virtio-balloon",
164 [VIRTIO_ID_IOMEM
] = "virtio-iomem",
165 [VIRTIO_ID_RPMSG
] = "virtio-rpmsg",
166 [VIRTIO_ID_SCSI
] = "virtio-scsi",
167 [VIRTIO_ID_9P
] = "virtio-9p",
168 [VIRTIO_ID_MAC80211_WLAN
] = "virtio-mac-wlan",
169 [VIRTIO_ID_RPROC_SERIAL
] = "virtio-rproc-serial",
170 [VIRTIO_ID_CAIF
] = "virtio-caif",
171 [VIRTIO_ID_MEMORY_BALLOON
] = "virtio-mem-balloon",
172 [VIRTIO_ID_GPU
] = "virtio-gpu",
173 [VIRTIO_ID_CLOCK
] = "virtio-clk",
174 [VIRTIO_ID_INPUT
] = "virtio-input",
175 [VIRTIO_ID_VSOCK
] = "vhost-vsock",
176 [VIRTIO_ID_CRYPTO
] = "virtio-crypto",
177 [VIRTIO_ID_SIGNAL_DIST
] = "virtio-signal",
178 [VIRTIO_ID_PSTORE
] = "virtio-pstore",
179 [VIRTIO_ID_IOMMU
] = "virtio-iommu",
180 [VIRTIO_ID_MEM
] = "virtio-mem",
181 [VIRTIO_ID_SOUND
] = "virtio-sound",
182 [VIRTIO_ID_FS
] = "virtio-user-fs",
183 [VIRTIO_ID_PMEM
] = "virtio-pmem",
184 [VIRTIO_ID_RPMB
] = "virtio-rpmb",
185 [VIRTIO_ID_MAC80211_HWSIM
] = "virtio-mac-hwsim",
186 [VIRTIO_ID_VIDEO_ENCODER
] = "virtio-vid-encoder",
187 [VIRTIO_ID_VIDEO_DECODER
] = "virtio-vid-decoder",
188 [VIRTIO_ID_SCMI
] = "virtio-scmi",
189 [VIRTIO_ID_NITRO_SEC_MOD
] = "virtio-nitro-sec-mod",
190 [VIRTIO_ID_I2C_ADAPTER
] = "vhost-user-i2c",
191 [VIRTIO_ID_WATCHDOG
] = "virtio-watchdog",
192 [VIRTIO_ID_CAN
] = "virtio-can",
193 [VIRTIO_ID_DMABUF
] = "virtio-dmabuf",
194 [VIRTIO_ID_PARAM_SERV
] = "virtio-param-serv",
195 [VIRTIO_ID_AUDIO_POLICY
] = "virtio-audio-pol",
196 [VIRTIO_ID_BT
] = "virtio-bluetooth",
197 [VIRTIO_ID_GPIO
] = "virtio-gpio"
200 static const char *virtio_id_to_name(uint16_t device_id
)
202 assert(device_id
< G_N_ELEMENTS(virtio_device_names
));
203 const char *name
= virtio_device_names
[device_id
];
204 assert(name
!= NULL
);
208 /* Called within call_rcu(). */
209 static void virtio_free_region_cache(VRingMemoryRegionCaches
*caches
)
211 assert(caches
!= NULL
);
212 address_space_cache_destroy(&caches
->desc
);
213 address_space_cache_destroy(&caches
->avail
);
214 address_space_cache_destroy(&caches
->used
);
218 static void virtio_virtqueue_reset_region_cache(struct VirtQueue
*vq
)
220 VRingMemoryRegionCaches
*caches
;
222 caches
= qatomic_read(&vq
->vring
.caches
);
223 qatomic_rcu_set(&vq
->vring
.caches
, NULL
);
225 call_rcu(caches
, virtio_free_region_cache
, rcu
);
229 static void virtio_init_region_cache(VirtIODevice
*vdev
, int n
)
231 VirtQueue
*vq
= &vdev
->vq
[n
];
232 VRingMemoryRegionCaches
*old
= vq
->vring
.caches
;
233 VRingMemoryRegionCaches
*new = NULL
;
239 addr
= vq
->vring
.desc
;
243 new = g_new0(VRingMemoryRegionCaches
, 1);
244 size
= virtio_queue_get_desc_size(vdev
, n
);
245 packed
= virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
) ?
247 len
= address_space_cache_init(&new->desc
, vdev
->dma_as
,
250 virtio_error(vdev
, "Cannot map desc");
254 size
= virtio_queue_get_used_size(vdev
, n
);
255 len
= address_space_cache_init(&new->used
, vdev
->dma_as
,
256 vq
->vring
.used
, size
, true);
258 virtio_error(vdev
, "Cannot map used");
262 size
= virtio_queue_get_avail_size(vdev
, n
);
263 len
= address_space_cache_init(&new->avail
, vdev
->dma_as
,
264 vq
->vring
.avail
, size
, false);
266 virtio_error(vdev
, "Cannot map avail");
270 qatomic_rcu_set(&vq
->vring
.caches
, new);
272 call_rcu(old
, virtio_free_region_cache
, rcu
);
277 address_space_cache_destroy(&new->avail
);
279 address_space_cache_destroy(&new->used
);
281 address_space_cache_destroy(&new->desc
);
284 virtio_virtqueue_reset_region_cache(vq
);
287 /* virt queue functions */
288 void virtio_queue_update_rings(VirtIODevice
*vdev
, int n
)
290 VRing
*vring
= &vdev
->vq
[n
].vring
;
292 if (!vring
->num
|| !vring
->desc
|| !vring
->align
) {
293 /* not yet setup -> nothing to do */
296 vring
->avail
= vring
->desc
+ vring
->num
* sizeof(VRingDesc
);
297 vring
->used
= vring_align(vring
->avail
+
298 offsetof(VRingAvail
, ring
[vring
->num
]),
300 virtio_init_region_cache(vdev
, n
);
303 /* Called within rcu_read_lock(). */
304 static void vring_split_desc_read(VirtIODevice
*vdev
, VRingDesc
*desc
,
305 MemoryRegionCache
*cache
, int i
)
307 address_space_read_cached(cache
, i
* sizeof(VRingDesc
),
308 desc
, sizeof(VRingDesc
));
309 virtio_tswap64s(vdev
, &desc
->addr
);
310 virtio_tswap32s(vdev
, &desc
->len
);
311 virtio_tswap16s(vdev
, &desc
->flags
);
312 virtio_tswap16s(vdev
, &desc
->next
);
315 static void vring_packed_event_read(VirtIODevice
*vdev
,
316 MemoryRegionCache
*cache
,
317 VRingPackedDescEvent
*e
)
319 hwaddr off_off
= offsetof(VRingPackedDescEvent
, off_wrap
);
320 hwaddr off_flags
= offsetof(VRingPackedDescEvent
, flags
);
322 e
->flags
= virtio_lduw_phys_cached(vdev
, cache
, off_flags
);
323 /* Make sure flags is seen before off_wrap */
325 e
->off_wrap
= virtio_lduw_phys_cached(vdev
, cache
, off_off
);
326 virtio_tswap16s(vdev
, &e
->flags
);
329 static void vring_packed_off_wrap_write(VirtIODevice
*vdev
,
330 MemoryRegionCache
*cache
,
333 hwaddr off
= offsetof(VRingPackedDescEvent
, off_wrap
);
335 virtio_stw_phys_cached(vdev
, cache
, off
, off_wrap
);
336 address_space_cache_invalidate(cache
, off
, sizeof(off_wrap
));
339 static void vring_packed_flags_write(VirtIODevice
*vdev
,
340 MemoryRegionCache
*cache
, uint16_t flags
)
342 hwaddr off
= offsetof(VRingPackedDescEvent
, flags
);
344 virtio_stw_phys_cached(vdev
, cache
, off
, flags
);
345 address_space_cache_invalidate(cache
, off
, sizeof(flags
));
348 /* Called within rcu_read_lock(). */
349 static VRingMemoryRegionCaches
*vring_get_region_caches(struct VirtQueue
*vq
)
351 return qatomic_rcu_read(&vq
->vring
.caches
);
354 /* Called within rcu_read_lock(). */
355 static inline uint16_t vring_avail_flags(VirtQueue
*vq
)
357 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
358 hwaddr pa
= offsetof(VRingAvail
, flags
);
364 return virtio_lduw_phys_cached(vq
->vdev
, &caches
->avail
, pa
);
367 /* Called within rcu_read_lock(). */
368 static inline uint16_t vring_avail_idx(VirtQueue
*vq
)
370 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
371 hwaddr pa
= offsetof(VRingAvail
, idx
);
377 vq
->shadow_avail_idx
= virtio_lduw_phys_cached(vq
->vdev
, &caches
->avail
, pa
);
378 return vq
->shadow_avail_idx
;
381 /* Called within rcu_read_lock(). */
382 static inline uint16_t vring_avail_ring(VirtQueue
*vq
, int i
)
384 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
385 hwaddr pa
= offsetof(VRingAvail
, ring
[i
]);
391 return virtio_lduw_phys_cached(vq
->vdev
, &caches
->avail
, pa
);
394 /* Called within rcu_read_lock(). */
395 static inline uint16_t vring_get_used_event(VirtQueue
*vq
)
397 return vring_avail_ring(vq
, vq
->vring
.num
);
400 /* Called within rcu_read_lock(). */
401 static inline void vring_used_write(VirtQueue
*vq
, VRingUsedElem
*uelem
,
404 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
405 hwaddr pa
= offsetof(VRingUsed
, ring
[i
]);
411 virtio_tswap32s(vq
->vdev
, &uelem
->id
);
412 virtio_tswap32s(vq
->vdev
, &uelem
->len
);
413 address_space_write_cached(&caches
->used
, pa
, uelem
, sizeof(VRingUsedElem
));
414 address_space_cache_invalidate(&caches
->used
, pa
, sizeof(VRingUsedElem
));
417 /* Called within rcu_read_lock(). */
418 static inline uint16_t vring_used_flags(VirtQueue
*vq
)
420 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
421 hwaddr pa
= offsetof(VRingUsed
, flags
);
427 return virtio_lduw_phys_cached(vq
->vdev
, &caches
->used
, pa
);
430 /* Called within rcu_read_lock(). */
431 static uint16_t vring_used_idx(VirtQueue
*vq
)
433 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
434 hwaddr pa
= offsetof(VRingUsed
, idx
);
440 return virtio_lduw_phys_cached(vq
->vdev
, &caches
->used
, pa
);
443 /* Called within rcu_read_lock(). */
444 static inline void vring_used_idx_set(VirtQueue
*vq
, uint16_t val
)
446 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
447 hwaddr pa
= offsetof(VRingUsed
, idx
);
450 virtio_stw_phys_cached(vq
->vdev
, &caches
->used
, pa
, val
);
451 address_space_cache_invalidate(&caches
->used
, pa
, sizeof(val
));
457 /* Called within rcu_read_lock(). */
458 static inline void vring_used_flags_set_bit(VirtQueue
*vq
, int mask
)
460 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
461 VirtIODevice
*vdev
= vq
->vdev
;
462 hwaddr pa
= offsetof(VRingUsed
, flags
);
469 flags
= virtio_lduw_phys_cached(vq
->vdev
, &caches
->used
, pa
);
470 virtio_stw_phys_cached(vdev
, &caches
->used
, pa
, flags
| mask
);
471 address_space_cache_invalidate(&caches
->used
, pa
, sizeof(flags
));
474 /* Called within rcu_read_lock(). */
475 static inline void vring_used_flags_unset_bit(VirtQueue
*vq
, int mask
)
477 VRingMemoryRegionCaches
*caches
= vring_get_region_caches(vq
);
478 VirtIODevice
*vdev
= vq
->vdev
;
479 hwaddr pa
= offsetof(VRingUsed
, flags
);
486 flags
= virtio_lduw_phys_cached(vq
->vdev
, &caches
->used
, pa
);
487 virtio_stw_phys_cached(vdev
, &caches
->used
, pa
, flags
& ~mask
);
488 address_space_cache_invalidate(&caches
->used
, pa
, sizeof(flags
));
491 /* Called within rcu_read_lock(). */
492 static inline void vring_set_avail_event(VirtQueue
*vq
, uint16_t val
)
494 VRingMemoryRegionCaches
*caches
;
496 if (!vq
->notification
) {
500 caches
= vring_get_region_caches(vq
);
505 pa
= offsetof(VRingUsed
, ring
[vq
->vring
.num
]);
506 virtio_stw_phys_cached(vq
->vdev
, &caches
->used
, pa
, val
);
507 address_space_cache_invalidate(&caches
->used
, pa
, sizeof(val
));
510 static void virtio_queue_split_set_notification(VirtQueue
*vq
, int enable
)
512 RCU_READ_LOCK_GUARD();
514 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
515 vring_set_avail_event(vq
, vring_avail_idx(vq
));
517 vring_used_flags_unset_bit(vq
, VRING_USED_F_NO_NOTIFY
);
519 vring_used_flags_set_bit(vq
, VRING_USED_F_NO_NOTIFY
);
522 /* Expose avail event/used flags before caller checks the avail idx. */
527 static void virtio_queue_packed_set_notification(VirtQueue
*vq
, int enable
)
530 VRingPackedDescEvent e
;
531 VRingMemoryRegionCaches
*caches
;
533 RCU_READ_LOCK_GUARD();
534 caches
= vring_get_region_caches(vq
);
539 vring_packed_event_read(vq
->vdev
, &caches
->used
, &e
);
542 e
.flags
= VRING_PACKED_EVENT_FLAG_DISABLE
;
543 } else if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
544 off_wrap
= vq
->shadow_avail_idx
| vq
->shadow_avail_wrap_counter
<< 15;
545 vring_packed_off_wrap_write(vq
->vdev
, &caches
->used
, off_wrap
);
546 /* Make sure off_wrap is wrote before flags */
548 e
.flags
= VRING_PACKED_EVENT_FLAG_DESC
;
550 e
.flags
= VRING_PACKED_EVENT_FLAG_ENABLE
;
553 vring_packed_flags_write(vq
->vdev
, &caches
->used
, e
.flags
);
555 /* Expose avail event/used flags before caller checks the avail idx. */
560 bool virtio_queue_get_notification(VirtQueue
*vq
)
562 return vq
->notification
;
565 void virtio_queue_set_notification(VirtQueue
*vq
, int enable
)
567 vq
->notification
= enable
;
569 if (!vq
->vring
.desc
) {
573 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
574 virtio_queue_packed_set_notification(vq
, enable
);
576 virtio_queue_split_set_notification(vq
, enable
);
580 int virtio_queue_ready(VirtQueue
*vq
)
582 return vq
->vring
.avail
!= 0;
585 static void vring_packed_desc_read_flags(VirtIODevice
*vdev
,
587 MemoryRegionCache
*cache
,
590 hwaddr off
= i
* sizeof(VRingPackedDesc
) + offsetof(VRingPackedDesc
, flags
);
592 *flags
= virtio_lduw_phys_cached(vdev
, cache
, off
);
595 static void vring_packed_desc_read(VirtIODevice
*vdev
,
596 VRingPackedDesc
*desc
,
597 MemoryRegionCache
*cache
,
598 int i
, bool strict_order
)
600 hwaddr off
= i
* sizeof(VRingPackedDesc
);
602 vring_packed_desc_read_flags(vdev
, &desc
->flags
, cache
, i
);
605 /* Make sure flags is read before the rest fields. */
609 address_space_read_cached(cache
, off
+ offsetof(VRingPackedDesc
, addr
),
610 &desc
->addr
, sizeof(desc
->addr
));
611 address_space_read_cached(cache
, off
+ offsetof(VRingPackedDesc
, id
),
612 &desc
->id
, sizeof(desc
->id
));
613 address_space_read_cached(cache
, off
+ offsetof(VRingPackedDesc
, len
),
614 &desc
->len
, sizeof(desc
->len
));
615 virtio_tswap64s(vdev
, &desc
->addr
);
616 virtio_tswap16s(vdev
, &desc
->id
);
617 virtio_tswap32s(vdev
, &desc
->len
);
620 static void vring_packed_desc_write_data(VirtIODevice
*vdev
,
621 VRingPackedDesc
*desc
,
622 MemoryRegionCache
*cache
,
625 hwaddr off_id
= i
* sizeof(VRingPackedDesc
) +
626 offsetof(VRingPackedDesc
, id
);
627 hwaddr off_len
= i
* sizeof(VRingPackedDesc
) +
628 offsetof(VRingPackedDesc
, len
);
630 virtio_tswap32s(vdev
, &desc
->len
);
631 virtio_tswap16s(vdev
, &desc
->id
);
632 address_space_write_cached(cache
, off_id
, &desc
->id
, sizeof(desc
->id
));
633 address_space_cache_invalidate(cache
, off_id
, sizeof(desc
->id
));
634 address_space_write_cached(cache
, off_len
, &desc
->len
, sizeof(desc
->len
));
635 address_space_cache_invalidate(cache
, off_len
, sizeof(desc
->len
));
638 static void vring_packed_desc_write_flags(VirtIODevice
*vdev
,
639 VRingPackedDesc
*desc
,
640 MemoryRegionCache
*cache
,
643 hwaddr off
= i
* sizeof(VRingPackedDesc
) + offsetof(VRingPackedDesc
, flags
);
645 virtio_stw_phys_cached(vdev
, cache
, off
, desc
->flags
);
646 address_space_cache_invalidate(cache
, off
, sizeof(desc
->flags
));
649 static void vring_packed_desc_write(VirtIODevice
*vdev
,
650 VRingPackedDesc
*desc
,
651 MemoryRegionCache
*cache
,
652 int i
, bool strict_order
)
654 vring_packed_desc_write_data(vdev
, desc
, cache
, i
);
656 /* Make sure data is wrote before flags. */
659 vring_packed_desc_write_flags(vdev
, desc
, cache
, i
);
662 static inline bool is_desc_avail(uint16_t flags
, bool wrap_counter
)
666 avail
= !!(flags
& (1 << VRING_PACKED_DESC_F_AVAIL
));
667 used
= !!(flags
& (1 << VRING_PACKED_DESC_F_USED
));
668 return (avail
!= used
) && (avail
== wrap_counter
);
671 /* Fetch avail_idx from VQ memory only when we really need to know if
672 * guest has added some buffers.
673 * Called within rcu_read_lock(). */
674 static int virtio_queue_empty_rcu(VirtQueue
*vq
)
676 if (virtio_device_disabled(vq
->vdev
)) {
680 if (unlikely(!vq
->vring
.avail
)) {
684 if (vq
->shadow_avail_idx
!= vq
->last_avail_idx
) {
688 return vring_avail_idx(vq
) == vq
->last_avail_idx
;
691 static int virtio_queue_split_empty(VirtQueue
*vq
)
695 if (virtio_device_disabled(vq
->vdev
)) {
699 if (unlikely(!vq
->vring
.avail
)) {
703 if (vq
->shadow_avail_idx
!= vq
->last_avail_idx
) {
707 RCU_READ_LOCK_GUARD();
708 empty
= vring_avail_idx(vq
) == vq
->last_avail_idx
;
712 /* Called within rcu_read_lock(). */
713 static int virtio_queue_packed_empty_rcu(VirtQueue
*vq
)
715 struct VRingPackedDesc desc
;
716 VRingMemoryRegionCaches
*cache
;
718 if (unlikely(!vq
->vring
.desc
)) {
722 cache
= vring_get_region_caches(vq
);
727 vring_packed_desc_read_flags(vq
->vdev
, &desc
.flags
, &cache
->desc
,
730 return !is_desc_avail(desc
.flags
, vq
->last_avail_wrap_counter
);
733 static int virtio_queue_packed_empty(VirtQueue
*vq
)
735 RCU_READ_LOCK_GUARD();
736 return virtio_queue_packed_empty_rcu(vq
);
739 int virtio_queue_empty(VirtQueue
*vq
)
741 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
742 return virtio_queue_packed_empty(vq
);
744 return virtio_queue_split_empty(vq
);
748 static void virtqueue_unmap_sg(VirtQueue
*vq
, const VirtQueueElement
*elem
,
751 AddressSpace
*dma_as
= vq
->vdev
->dma_as
;
756 for (i
= 0; i
< elem
->in_num
; i
++) {
757 size_t size
= MIN(len
- offset
, elem
->in_sg
[i
].iov_len
);
759 dma_memory_unmap(dma_as
, elem
->in_sg
[i
].iov_base
,
760 elem
->in_sg
[i
].iov_len
,
761 DMA_DIRECTION_FROM_DEVICE
, size
);
766 for (i
= 0; i
< elem
->out_num
; i
++)
767 dma_memory_unmap(dma_as
, elem
->out_sg
[i
].iov_base
,
768 elem
->out_sg
[i
].iov_len
,
769 DMA_DIRECTION_TO_DEVICE
,
770 elem
->out_sg
[i
].iov_len
);
773 /* virtqueue_detach_element:
774 * @vq: The #VirtQueue
775 * @elem: The #VirtQueueElement
776 * @len: number of bytes written
778 * Detach the element from the virtqueue. This function is suitable for device
779 * reset or other situations where a #VirtQueueElement is simply freed and will
780 * not be pushed or discarded.
782 void virtqueue_detach_element(VirtQueue
*vq
, const VirtQueueElement
*elem
,
785 vq
->inuse
-= elem
->ndescs
;
786 virtqueue_unmap_sg(vq
, elem
, len
);
789 static void virtqueue_split_rewind(VirtQueue
*vq
, unsigned int num
)
791 vq
->last_avail_idx
-= num
;
794 static void virtqueue_packed_rewind(VirtQueue
*vq
, unsigned int num
)
796 if (vq
->last_avail_idx
< num
) {
797 vq
->last_avail_idx
= vq
->vring
.num
+ vq
->last_avail_idx
- num
;
798 vq
->last_avail_wrap_counter
^= 1;
800 vq
->last_avail_idx
-= num
;
805 * @vq: The #VirtQueue
806 * @elem: The #VirtQueueElement
807 * @len: number of bytes written
809 * Pretend the most recent element wasn't popped from the virtqueue. The next
810 * call to virtqueue_pop() will refetch the element.
812 void virtqueue_unpop(VirtQueue
*vq
, const VirtQueueElement
*elem
,
816 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
817 virtqueue_packed_rewind(vq
, 1);
819 virtqueue_split_rewind(vq
, 1);
822 virtqueue_detach_element(vq
, elem
, len
);
826 * @vq: The #VirtQueue
827 * @num: Number of elements to push back
829 * Pretend that elements weren't popped from the virtqueue. The next
830 * virtqueue_pop() will refetch the oldest element.
832 * Use virtqueue_unpop() instead if you have a VirtQueueElement.
834 * Returns: true on success, false if @num is greater than the number of in use
837 bool virtqueue_rewind(VirtQueue
*vq
, unsigned int num
)
839 if (num
> vq
->inuse
) {
844 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
845 virtqueue_packed_rewind(vq
, num
);
847 virtqueue_split_rewind(vq
, num
);
852 static void virtqueue_split_fill(VirtQueue
*vq
, const VirtQueueElement
*elem
,
853 unsigned int len
, unsigned int idx
)
857 if (unlikely(!vq
->vring
.used
)) {
861 idx
= (idx
+ vq
->used_idx
) % vq
->vring
.num
;
863 uelem
.id
= elem
->index
;
865 vring_used_write(vq
, &uelem
, idx
);
868 static void virtqueue_packed_fill(VirtQueue
*vq
, const VirtQueueElement
*elem
,
869 unsigned int len
, unsigned int idx
)
871 vq
->used_elems
[idx
].index
= elem
->index
;
872 vq
->used_elems
[idx
].len
= len
;
873 vq
->used_elems
[idx
].ndescs
= elem
->ndescs
;
876 static void virtqueue_packed_fill_desc(VirtQueue
*vq
,
877 const VirtQueueElement
*elem
,
882 VRingMemoryRegionCaches
*caches
;
883 VRingPackedDesc desc
= {
887 bool wrap_counter
= vq
->used_wrap_counter
;
889 if (unlikely(!vq
->vring
.desc
)) {
893 head
= vq
->used_idx
+ idx
;
894 if (head
>= vq
->vring
.num
) {
895 head
-= vq
->vring
.num
;
899 desc
.flags
|= (1 << VRING_PACKED_DESC_F_AVAIL
);
900 desc
.flags
|= (1 << VRING_PACKED_DESC_F_USED
);
902 desc
.flags
&= ~(1 << VRING_PACKED_DESC_F_AVAIL
);
903 desc
.flags
&= ~(1 << VRING_PACKED_DESC_F_USED
);
906 caches
= vring_get_region_caches(vq
);
911 vring_packed_desc_write(vq
->vdev
, &desc
, &caches
->desc
, head
, strict_order
);
914 /* Called within rcu_read_lock(). */
915 void virtqueue_fill(VirtQueue
*vq
, const VirtQueueElement
*elem
,
916 unsigned int len
, unsigned int idx
)
918 trace_virtqueue_fill(vq
, elem
, len
, idx
);
920 virtqueue_unmap_sg(vq
, elem
, len
);
922 if (virtio_device_disabled(vq
->vdev
)) {
926 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
927 virtqueue_packed_fill(vq
, elem
, len
, idx
);
929 virtqueue_split_fill(vq
, elem
, len
, idx
);
933 /* Called within rcu_read_lock(). */
934 static void virtqueue_split_flush(VirtQueue
*vq
, unsigned int count
)
938 if (unlikely(!vq
->vring
.used
)) {
942 /* Make sure buffer is written before we update index. */
944 trace_virtqueue_flush(vq
, count
);
947 vring_used_idx_set(vq
, new);
949 if (unlikely((int16_t)(new - vq
->signalled_used
) < (uint16_t)(new - old
)))
950 vq
->signalled_used_valid
= false;
953 static void virtqueue_packed_flush(VirtQueue
*vq
, unsigned int count
)
955 unsigned int i
, ndescs
= 0;
957 if (unlikely(!vq
->vring
.desc
)) {
961 for (i
= 1; i
< count
; i
++) {
962 virtqueue_packed_fill_desc(vq
, &vq
->used_elems
[i
], i
, false);
963 ndescs
+= vq
->used_elems
[i
].ndescs
;
965 virtqueue_packed_fill_desc(vq
, &vq
->used_elems
[0], 0, true);
966 ndescs
+= vq
->used_elems
[0].ndescs
;
969 vq
->used_idx
+= ndescs
;
970 if (vq
->used_idx
>= vq
->vring
.num
) {
971 vq
->used_idx
-= vq
->vring
.num
;
972 vq
->used_wrap_counter
^= 1;
973 vq
->signalled_used_valid
= false;
977 void virtqueue_flush(VirtQueue
*vq
, unsigned int count
)
979 if (virtio_device_disabled(vq
->vdev
)) {
984 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
985 virtqueue_packed_flush(vq
, count
);
987 virtqueue_split_flush(vq
, count
);
991 void virtqueue_push(VirtQueue
*vq
, const VirtQueueElement
*elem
,
994 RCU_READ_LOCK_GUARD();
995 virtqueue_fill(vq
, elem
, len
, 0);
996 virtqueue_flush(vq
, 1);
999 /* Called within rcu_read_lock(). */
1000 static int virtqueue_num_heads(VirtQueue
*vq
, unsigned int idx
)
1002 uint16_t num_heads
= vring_avail_idx(vq
) - idx
;
1004 /* Check it isn't doing very strange things with descriptor numbers. */
1005 if (num_heads
> vq
->vring
.num
) {
1006 virtio_error(vq
->vdev
, "Guest moved used index from %u to %u",
1007 idx
, vq
->shadow_avail_idx
);
1010 /* On success, callers read a descriptor at vq->last_avail_idx.
1011 * Make sure descriptor read does not bypass avail index read. */
1019 /* Called within rcu_read_lock(). */
1020 static bool virtqueue_get_head(VirtQueue
*vq
, unsigned int idx
,
1023 /* Grab the next descriptor number they're advertising, and increment
1024 * the index we've seen. */
1025 *head
= vring_avail_ring(vq
, idx
% vq
->vring
.num
);
1027 /* If their number is silly, that's a fatal mistake. */
1028 if (*head
>= vq
->vring
.num
) {
1029 virtio_error(vq
->vdev
, "Guest says index %u is available", *head
);
1037 VIRTQUEUE_READ_DESC_ERROR
= -1,
1038 VIRTQUEUE_READ_DESC_DONE
= 0, /* end of chain */
1039 VIRTQUEUE_READ_DESC_MORE
= 1, /* more buffers in chain */
1042 static int virtqueue_split_read_next_desc(VirtIODevice
*vdev
, VRingDesc
*desc
,
1043 MemoryRegionCache
*desc_cache
,
1044 unsigned int max
, unsigned int *next
)
1046 /* If this descriptor says it doesn't chain, we're done. */
1047 if (!(desc
->flags
& VRING_DESC_F_NEXT
)) {
1048 return VIRTQUEUE_READ_DESC_DONE
;
1051 /* Check they're not leading us off end of descriptors. */
1053 /* Make sure compiler knows to grab that: we don't want it changing! */
1057 virtio_error(vdev
, "Desc next is %u", *next
);
1058 return VIRTQUEUE_READ_DESC_ERROR
;
1061 vring_split_desc_read(vdev
, desc
, desc_cache
, *next
);
1062 return VIRTQUEUE_READ_DESC_MORE
;
1065 /* Called within rcu_read_lock(). */
1066 static void virtqueue_split_get_avail_bytes(VirtQueue
*vq
,
1067 unsigned int *in_bytes
, unsigned int *out_bytes
,
1068 unsigned max_in_bytes
, unsigned max_out_bytes
,
1069 VRingMemoryRegionCaches
*caches
)
1071 VirtIODevice
*vdev
= vq
->vdev
;
1072 unsigned int max
, idx
;
1073 unsigned int total_bufs
, in_total
, out_total
;
1074 MemoryRegionCache indirect_desc_cache
= MEMORY_REGION_CACHE_INVALID
;
1078 idx
= vq
->last_avail_idx
;
1079 total_bufs
= in_total
= out_total
= 0;
1081 max
= vq
->vring
.num
;
1083 while ((rc
= virtqueue_num_heads(vq
, idx
)) > 0) {
1084 MemoryRegionCache
*desc_cache
= &caches
->desc
;
1085 unsigned int num_bufs
;
1089 num_bufs
= total_bufs
;
1091 if (!virtqueue_get_head(vq
, idx
++, &i
)) {
1095 vring_split_desc_read(vdev
, &desc
, desc_cache
, i
);
1097 if (desc
.flags
& VRING_DESC_F_INDIRECT
) {
1098 if (!desc
.len
|| (desc
.len
% sizeof(VRingDesc
))) {
1099 virtio_error(vdev
, "Invalid size for indirect buffer table");
1103 /* If we've got too many, that implies a descriptor loop. */
1104 if (num_bufs
>= max
) {
1105 virtio_error(vdev
, "Looped descriptor");
1109 /* loop over the indirect descriptor table */
1110 len
= address_space_cache_init(&indirect_desc_cache
,
1112 desc
.addr
, desc
.len
, false);
1113 desc_cache
= &indirect_desc_cache
;
1114 if (len
< desc
.len
) {
1115 virtio_error(vdev
, "Cannot map indirect buffer");
1119 max
= desc
.len
/ sizeof(VRingDesc
);
1121 vring_split_desc_read(vdev
, &desc
, desc_cache
, i
);
1125 /* If we've got too many, that implies a descriptor loop. */
1126 if (++num_bufs
> max
) {
1127 virtio_error(vdev
, "Looped descriptor");
1131 if (desc
.flags
& VRING_DESC_F_WRITE
) {
1132 in_total
+= desc
.len
;
1134 out_total
+= desc
.len
;
1136 if (in_total
>= max_in_bytes
&& out_total
>= max_out_bytes
) {
1140 rc
= virtqueue_split_read_next_desc(vdev
, &desc
, desc_cache
, max
, &i
);
1141 } while (rc
== VIRTQUEUE_READ_DESC_MORE
);
1143 if (rc
== VIRTQUEUE_READ_DESC_ERROR
) {
1147 if (desc_cache
== &indirect_desc_cache
) {
1148 address_space_cache_destroy(&indirect_desc_cache
);
1151 total_bufs
= num_bufs
;
1160 address_space_cache_destroy(&indirect_desc_cache
);
1162 *in_bytes
= in_total
;
1165 *out_bytes
= out_total
;
1170 in_total
= out_total
= 0;
1174 static int virtqueue_packed_read_next_desc(VirtQueue
*vq
,
1175 VRingPackedDesc
*desc
,
1182 /* If this descriptor says it doesn't chain, we're done. */
1183 if (!indirect
&& !(desc
->flags
& VRING_DESC_F_NEXT
)) {
1184 return VIRTQUEUE_READ_DESC_DONE
;
1190 return VIRTQUEUE_READ_DESC_DONE
;
1192 (*next
) -= vq
->vring
.num
;
1196 vring_packed_desc_read(vq
->vdev
, desc
, desc_cache
, *next
, false);
1197 return VIRTQUEUE_READ_DESC_MORE
;
1200 /* Called within rcu_read_lock(). */
1201 static void virtqueue_packed_get_avail_bytes(VirtQueue
*vq
,
1202 unsigned int *in_bytes
,
1203 unsigned int *out_bytes
,
1204 unsigned max_in_bytes
,
1205 unsigned max_out_bytes
,
1206 VRingMemoryRegionCaches
*caches
)
1208 VirtIODevice
*vdev
= vq
->vdev
;
1209 unsigned int max
, idx
;
1210 unsigned int total_bufs
, in_total
, out_total
;
1211 MemoryRegionCache
*desc_cache
;
1212 MemoryRegionCache indirect_desc_cache
= MEMORY_REGION_CACHE_INVALID
;
1214 VRingPackedDesc desc
;
1217 idx
= vq
->last_avail_idx
;
1218 wrap_counter
= vq
->last_avail_wrap_counter
;
1219 total_bufs
= in_total
= out_total
= 0;
1221 max
= vq
->vring
.num
;
1224 unsigned int num_bufs
= total_bufs
;
1225 unsigned int i
= idx
;
1228 desc_cache
= &caches
->desc
;
1229 vring_packed_desc_read(vdev
, &desc
, desc_cache
, idx
, true);
1230 if (!is_desc_avail(desc
.flags
, wrap_counter
)) {
1234 if (desc
.flags
& VRING_DESC_F_INDIRECT
) {
1235 if (desc
.len
% sizeof(VRingPackedDesc
)) {
1236 virtio_error(vdev
, "Invalid size for indirect buffer table");
1240 /* If we've got too many, that implies a descriptor loop. */
1241 if (num_bufs
>= max
) {
1242 virtio_error(vdev
, "Looped descriptor");
1246 /* loop over the indirect descriptor table */
1247 len
= address_space_cache_init(&indirect_desc_cache
,
1249 desc
.addr
, desc
.len
, false);
1250 desc_cache
= &indirect_desc_cache
;
1251 if (len
< desc
.len
) {
1252 virtio_error(vdev
, "Cannot map indirect buffer");
1256 max
= desc
.len
/ sizeof(VRingPackedDesc
);
1258 vring_packed_desc_read(vdev
, &desc
, desc_cache
, i
, false);
1262 /* If we've got too many, that implies a descriptor loop. */
1263 if (++num_bufs
> max
) {
1264 virtio_error(vdev
, "Looped descriptor");
1268 if (desc
.flags
& VRING_DESC_F_WRITE
) {
1269 in_total
+= desc
.len
;
1271 out_total
+= desc
.len
;
1273 if (in_total
>= max_in_bytes
&& out_total
>= max_out_bytes
) {
1277 rc
= virtqueue_packed_read_next_desc(vq
, &desc
, desc_cache
, max
,
1279 &indirect_desc_cache
);
1280 } while (rc
== VIRTQUEUE_READ_DESC_MORE
);
1282 if (desc_cache
== &indirect_desc_cache
) {
1283 address_space_cache_destroy(&indirect_desc_cache
);
1287 idx
+= num_bufs
- total_bufs
;
1288 total_bufs
= num_bufs
;
1291 if (idx
>= vq
->vring
.num
) {
1292 idx
-= vq
->vring
.num
;
1297 /* Record the index and wrap counter for a kick we want */
1298 vq
->shadow_avail_idx
= idx
;
1299 vq
->shadow_avail_wrap_counter
= wrap_counter
;
1301 address_space_cache_destroy(&indirect_desc_cache
);
1303 *in_bytes
= in_total
;
1306 *out_bytes
= out_total
;
1311 in_total
= out_total
= 0;
1315 void virtqueue_get_avail_bytes(VirtQueue
*vq
, unsigned int *in_bytes
,
1316 unsigned int *out_bytes
,
1317 unsigned max_in_bytes
, unsigned max_out_bytes
)
1320 VRingMemoryRegionCaches
*caches
;
1322 RCU_READ_LOCK_GUARD();
1324 if (unlikely(!vq
->vring
.desc
)) {
1328 caches
= vring_get_region_caches(vq
);
1333 desc_size
= virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
) ?
1334 sizeof(VRingPackedDesc
) : sizeof(VRingDesc
);
1335 if (caches
->desc
.len
< vq
->vring
.num
* desc_size
) {
1336 virtio_error(vq
->vdev
, "Cannot map descriptor ring");
1340 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
1341 virtqueue_packed_get_avail_bytes(vq
, in_bytes
, out_bytes
,
1342 max_in_bytes
, max_out_bytes
,
1345 virtqueue_split_get_avail_bytes(vq
, in_bytes
, out_bytes
,
1346 max_in_bytes
, max_out_bytes
,
1360 int virtqueue_avail_bytes(VirtQueue
*vq
, unsigned int in_bytes
,
1361 unsigned int out_bytes
)
1363 unsigned int in_total
, out_total
;
1365 virtqueue_get_avail_bytes(vq
, &in_total
, &out_total
, in_bytes
, out_bytes
);
1366 return in_bytes
<= in_total
&& out_bytes
<= out_total
;
1369 static bool virtqueue_map_desc(VirtIODevice
*vdev
, unsigned int *p_num_sg
,
1370 hwaddr
*addr
, struct iovec
*iov
,
1371 unsigned int max_num_sg
, bool is_write
,
1372 hwaddr pa
, size_t sz
)
1375 unsigned num_sg
= *p_num_sg
;
1376 assert(num_sg
<= max_num_sg
);
1379 virtio_error(vdev
, "virtio: zero sized buffers are not allowed");
1386 if (num_sg
== max_num_sg
) {
1387 virtio_error(vdev
, "virtio: too many write descriptors in "
1392 iov
[num_sg
].iov_base
= dma_memory_map(vdev
->dma_as
, pa
, &len
,
1394 DMA_DIRECTION_FROM_DEVICE
:
1395 DMA_DIRECTION_TO_DEVICE
,
1396 MEMTXATTRS_UNSPECIFIED
);
1397 if (!iov
[num_sg
].iov_base
) {
1398 virtio_error(vdev
, "virtio: bogus descriptor or out of resources");
1402 iov
[num_sg
].iov_len
= len
;
1416 /* Only used by error code paths before we have a VirtQueueElement (therefore
1417 * virtqueue_unmap_sg() can't be used). Assumes buffers weren't written to
1420 static void virtqueue_undo_map_desc(unsigned int out_num
, unsigned int in_num
,
1425 for (i
= 0; i
< out_num
+ in_num
; i
++) {
1426 int is_write
= i
>= out_num
;
1428 cpu_physical_memory_unmap(iov
->iov_base
, iov
->iov_len
, is_write
, 0);
1433 static void virtqueue_map_iovec(VirtIODevice
*vdev
, struct iovec
*sg
,
1434 hwaddr
*addr
, unsigned int num_sg
,
1440 for (i
= 0; i
< num_sg
; i
++) {
1441 len
= sg
[i
].iov_len
;
1442 sg
[i
].iov_base
= dma_memory_map(vdev
->dma_as
,
1443 addr
[i
], &len
, is_write
?
1444 DMA_DIRECTION_FROM_DEVICE
:
1445 DMA_DIRECTION_TO_DEVICE
,
1446 MEMTXATTRS_UNSPECIFIED
);
1447 if (!sg
[i
].iov_base
) {
1448 error_report("virtio: error trying to map MMIO memory");
1451 if (len
!= sg
[i
].iov_len
) {
1452 error_report("virtio: unexpected memory split");
1458 void virtqueue_map(VirtIODevice
*vdev
, VirtQueueElement
*elem
)
1460 virtqueue_map_iovec(vdev
, elem
->in_sg
, elem
->in_addr
, elem
->in_num
, true);
1461 virtqueue_map_iovec(vdev
, elem
->out_sg
, elem
->out_addr
, elem
->out_num
,
1465 static void *virtqueue_alloc_element(size_t sz
, unsigned out_num
, unsigned in_num
)
1467 VirtQueueElement
*elem
;
1468 size_t in_addr_ofs
= QEMU_ALIGN_UP(sz
, __alignof__(elem
->in_addr
[0]));
1469 size_t out_addr_ofs
= in_addr_ofs
+ in_num
* sizeof(elem
->in_addr
[0]);
1470 size_t out_addr_end
= out_addr_ofs
+ out_num
* sizeof(elem
->out_addr
[0]);
1471 size_t in_sg_ofs
= QEMU_ALIGN_UP(out_addr_end
, __alignof__(elem
->in_sg
[0]));
1472 size_t out_sg_ofs
= in_sg_ofs
+ in_num
* sizeof(elem
->in_sg
[0]);
1473 size_t out_sg_end
= out_sg_ofs
+ out_num
* sizeof(elem
->out_sg
[0]);
1475 assert(sz
>= sizeof(VirtQueueElement
));
1476 elem
= g_malloc(out_sg_end
);
1477 trace_virtqueue_alloc_element(elem
, sz
, in_num
, out_num
);
1478 elem
->out_num
= out_num
;
1479 elem
->in_num
= in_num
;
1480 elem
->in_addr
= (void *)elem
+ in_addr_ofs
;
1481 elem
->out_addr
= (void *)elem
+ out_addr_ofs
;
1482 elem
->in_sg
= (void *)elem
+ in_sg_ofs
;
1483 elem
->out_sg
= (void *)elem
+ out_sg_ofs
;
1487 static void *virtqueue_split_pop(VirtQueue
*vq
, size_t sz
)
1489 unsigned int i
, head
, max
;
1490 VRingMemoryRegionCaches
*caches
;
1491 MemoryRegionCache indirect_desc_cache
= MEMORY_REGION_CACHE_INVALID
;
1492 MemoryRegionCache
*desc_cache
;
1494 VirtIODevice
*vdev
= vq
->vdev
;
1495 VirtQueueElement
*elem
= NULL
;
1496 unsigned out_num
, in_num
, elem_entries
;
1497 hwaddr addr
[VIRTQUEUE_MAX_SIZE
];
1498 struct iovec iov
[VIRTQUEUE_MAX_SIZE
];
1502 RCU_READ_LOCK_GUARD();
1503 if (virtio_queue_empty_rcu(vq
)) {
1506 /* Needed after virtio_queue_empty(), see comment in
1507 * virtqueue_num_heads(). */
1510 /* When we start there are none of either input nor output. */
1511 out_num
= in_num
= elem_entries
= 0;
1513 max
= vq
->vring
.num
;
1515 if (vq
->inuse
>= vq
->vring
.num
) {
1516 virtio_error(vdev
, "Virtqueue size exceeded");
1520 if (!virtqueue_get_head(vq
, vq
->last_avail_idx
++, &head
)) {
1524 if (virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
1525 vring_set_avail_event(vq
, vq
->last_avail_idx
);
1530 caches
= vring_get_region_caches(vq
);
1532 virtio_error(vdev
, "Region caches not initialized");
1536 if (caches
->desc
.len
< max
* sizeof(VRingDesc
)) {
1537 virtio_error(vdev
, "Cannot map descriptor ring");
1541 desc_cache
= &caches
->desc
;
1542 vring_split_desc_read(vdev
, &desc
, desc_cache
, i
);
1543 if (desc
.flags
& VRING_DESC_F_INDIRECT
) {
1544 if (!desc
.len
|| (desc
.len
% sizeof(VRingDesc
))) {
1545 virtio_error(vdev
, "Invalid size for indirect buffer table");
1549 /* loop over the indirect descriptor table */
1550 len
= address_space_cache_init(&indirect_desc_cache
, vdev
->dma_as
,
1551 desc
.addr
, desc
.len
, false);
1552 desc_cache
= &indirect_desc_cache
;
1553 if (len
< desc
.len
) {
1554 virtio_error(vdev
, "Cannot map indirect buffer");
1558 max
= desc
.len
/ sizeof(VRingDesc
);
1560 vring_split_desc_read(vdev
, &desc
, desc_cache
, i
);
1563 /* Collect all the descriptors */
1567 if (desc
.flags
& VRING_DESC_F_WRITE
) {
1568 map_ok
= virtqueue_map_desc(vdev
, &in_num
, addr
+ out_num
,
1570 VIRTQUEUE_MAX_SIZE
- out_num
, true,
1571 desc
.addr
, desc
.len
);
1574 virtio_error(vdev
, "Incorrect order for descriptors");
1577 map_ok
= virtqueue_map_desc(vdev
, &out_num
, addr
, iov
,
1578 VIRTQUEUE_MAX_SIZE
, false,
1579 desc
.addr
, desc
.len
);
1585 /* If we've got too many, that implies a descriptor loop. */
1586 if (++elem_entries
> max
) {
1587 virtio_error(vdev
, "Looped descriptor");
1591 rc
= virtqueue_split_read_next_desc(vdev
, &desc
, desc_cache
, max
, &i
);
1592 } while (rc
== VIRTQUEUE_READ_DESC_MORE
);
1594 if (rc
== VIRTQUEUE_READ_DESC_ERROR
) {
1598 /* Now copy what we have collected and mapped */
1599 elem
= virtqueue_alloc_element(sz
, out_num
, in_num
);
1602 for (i
= 0; i
< out_num
; i
++) {
1603 elem
->out_addr
[i
] = addr
[i
];
1604 elem
->out_sg
[i
] = iov
[i
];
1606 for (i
= 0; i
< in_num
; i
++) {
1607 elem
->in_addr
[i
] = addr
[out_num
+ i
];
1608 elem
->in_sg
[i
] = iov
[out_num
+ i
];
1613 trace_virtqueue_pop(vq
, elem
, elem
->in_num
, elem
->out_num
);
1615 address_space_cache_destroy(&indirect_desc_cache
);
1620 virtqueue_undo_map_desc(out_num
, in_num
, iov
);
1624 static void *virtqueue_packed_pop(VirtQueue
*vq
, size_t sz
)
1626 unsigned int i
, max
;
1627 VRingMemoryRegionCaches
*caches
;
1628 MemoryRegionCache indirect_desc_cache
= MEMORY_REGION_CACHE_INVALID
;
1629 MemoryRegionCache
*desc_cache
;
1631 VirtIODevice
*vdev
= vq
->vdev
;
1632 VirtQueueElement
*elem
= NULL
;
1633 unsigned out_num
, in_num
, elem_entries
;
1634 hwaddr addr
[VIRTQUEUE_MAX_SIZE
];
1635 struct iovec iov
[VIRTQUEUE_MAX_SIZE
];
1636 VRingPackedDesc desc
;
1640 RCU_READ_LOCK_GUARD();
1641 if (virtio_queue_packed_empty_rcu(vq
)) {
1645 /* When we start there are none of either input nor output. */
1646 out_num
= in_num
= elem_entries
= 0;
1648 max
= vq
->vring
.num
;
1650 if (vq
->inuse
>= vq
->vring
.num
) {
1651 virtio_error(vdev
, "Virtqueue size exceeded");
1655 i
= vq
->last_avail_idx
;
1657 caches
= vring_get_region_caches(vq
);
1659 virtio_error(vdev
, "Region caches not initialized");
1663 if (caches
->desc
.len
< max
* sizeof(VRingDesc
)) {
1664 virtio_error(vdev
, "Cannot map descriptor ring");
1668 desc_cache
= &caches
->desc
;
1669 vring_packed_desc_read(vdev
, &desc
, desc_cache
, i
, true);
1671 if (desc
.flags
& VRING_DESC_F_INDIRECT
) {
1672 if (desc
.len
% sizeof(VRingPackedDesc
)) {
1673 virtio_error(vdev
, "Invalid size for indirect buffer table");
1677 /* loop over the indirect descriptor table */
1678 len
= address_space_cache_init(&indirect_desc_cache
, vdev
->dma_as
,
1679 desc
.addr
, desc
.len
, false);
1680 desc_cache
= &indirect_desc_cache
;
1681 if (len
< desc
.len
) {
1682 virtio_error(vdev
, "Cannot map indirect buffer");
1686 max
= desc
.len
/ sizeof(VRingPackedDesc
);
1688 vring_packed_desc_read(vdev
, &desc
, desc_cache
, i
, false);
1691 /* Collect all the descriptors */
1695 if (desc
.flags
& VRING_DESC_F_WRITE
) {
1696 map_ok
= virtqueue_map_desc(vdev
, &in_num
, addr
+ out_num
,
1698 VIRTQUEUE_MAX_SIZE
- out_num
, true,
1699 desc
.addr
, desc
.len
);
1702 virtio_error(vdev
, "Incorrect order for descriptors");
1705 map_ok
= virtqueue_map_desc(vdev
, &out_num
, addr
, iov
,
1706 VIRTQUEUE_MAX_SIZE
, false,
1707 desc
.addr
, desc
.len
);
1713 /* If we've got too many, that implies a descriptor loop. */
1714 if (++elem_entries
> max
) {
1715 virtio_error(vdev
, "Looped descriptor");
1719 rc
= virtqueue_packed_read_next_desc(vq
, &desc
, desc_cache
, max
, &i
,
1721 &indirect_desc_cache
);
1722 } while (rc
== VIRTQUEUE_READ_DESC_MORE
);
1724 /* Now copy what we have collected and mapped */
1725 elem
= virtqueue_alloc_element(sz
, out_num
, in_num
);
1726 for (i
= 0; i
< out_num
; i
++) {
1727 elem
->out_addr
[i
] = addr
[i
];
1728 elem
->out_sg
[i
] = iov
[i
];
1730 for (i
= 0; i
< in_num
; i
++) {
1731 elem
->in_addr
[i
] = addr
[out_num
+ i
];
1732 elem
->in_sg
[i
] = iov
[out_num
+ i
];
1736 elem
->ndescs
= (desc_cache
== &indirect_desc_cache
) ? 1 : elem_entries
;
1737 vq
->last_avail_idx
+= elem
->ndescs
;
1738 vq
->inuse
+= elem
->ndescs
;
1740 if (vq
->last_avail_idx
>= vq
->vring
.num
) {
1741 vq
->last_avail_idx
-= vq
->vring
.num
;
1742 vq
->last_avail_wrap_counter
^= 1;
1745 vq
->shadow_avail_idx
= vq
->last_avail_idx
;
1746 vq
->shadow_avail_wrap_counter
= vq
->last_avail_wrap_counter
;
1748 trace_virtqueue_pop(vq
, elem
, elem
->in_num
, elem
->out_num
);
1750 address_space_cache_destroy(&indirect_desc_cache
);
1755 virtqueue_undo_map_desc(out_num
, in_num
, iov
);
1759 void *virtqueue_pop(VirtQueue
*vq
, size_t sz
)
1761 if (virtio_device_disabled(vq
->vdev
)) {
1765 if (virtio_vdev_has_feature(vq
->vdev
, VIRTIO_F_RING_PACKED
)) {
1766 return virtqueue_packed_pop(vq
, sz
);
1768 return virtqueue_split_pop(vq
, sz
);
1772 static unsigned int virtqueue_packed_drop_all(VirtQueue
*vq
)
1774 VRingMemoryRegionCaches
*caches
;
1775 MemoryRegionCache
*desc_cache
;
1776 unsigned int dropped
= 0;
1777 VirtQueueElement elem
= {};
1778 VirtIODevice
*vdev
= vq
->vdev
;
1779 VRingPackedDesc desc
;
1781 RCU_READ_LOCK_GUARD();
1783 caches
= vring_get_region_caches(vq
);
1788 desc_cache
= &caches
->desc
;
1790 virtio_queue_set_notification(vq
, 0);
1792 while (vq
->inuse
< vq
->vring
.num
) {
1793 unsigned int idx
= vq
->last_avail_idx
;
1795 * works similar to virtqueue_pop but does not map buffers
1796 * and does not allocate any memory.
1798 vring_packed_desc_read(vdev
, &desc
, desc_cache
,
1799 vq
->last_avail_idx
, true);
1800 if (!is_desc_avail(desc
.flags
, vq
->last_avail_wrap_counter
)) {
1803 elem
.index
= desc
.id
;
1805 while (virtqueue_packed_read_next_desc(vq
, &desc
, desc_cache
,
1806 vq
->vring
.num
, &idx
, false)) {
1810 * immediately push the element, nothing to unmap
1811 * as both in_num and out_num are set to 0.
1813 virtqueue_push(vq
, &elem
, 0);
1815 vq
->last_avail_idx
+= elem
.ndescs
;
1816 if (vq
->last_avail_idx
>= vq
->vring
.num
) {
1817 vq
->last_avail_idx
-= vq
->vring
.num
;
1818 vq
->last_avail_wrap_counter
^= 1;
1825 static unsigned int virtqueue_split_drop_all(VirtQueue
*vq
)
1827 unsigned int dropped
= 0;
1828 VirtQueueElement elem
= {};
1829 VirtIODevice
*vdev
= vq
->vdev
;
1830 bool fEventIdx
= virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
);
1832 while (!virtio_queue_empty(vq
) && vq
->inuse
< vq
->vring
.num
) {
1833 /* works similar to virtqueue_pop but does not map buffers
1834 * and does not allocate any memory */
1836 if (!virtqueue_get_head(vq
, vq
->last_avail_idx
, &elem
.index
)) {
1840 vq
->last_avail_idx
++;
1842 vring_set_avail_event(vq
, vq
->last_avail_idx
);
1844 /* immediately push the element, nothing to unmap
1845 * as both in_num and out_num are set to 0 */
1846 virtqueue_push(vq
, &elem
, 0);
1853 /* virtqueue_drop_all:
1854 * @vq: The #VirtQueue
1855 * Drops all queued buffers and indicates them to the guest
1856 * as if they are done. Useful when buffers can not be
1857 * processed but must be returned to the guest.
1859 unsigned int virtqueue_drop_all(VirtQueue
*vq
)
1861 struct VirtIODevice
*vdev
= vq
->vdev
;
1863 if (virtio_device_disabled(vq
->vdev
)) {
1867 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
1868 return virtqueue_packed_drop_all(vq
);
1870 return virtqueue_split_drop_all(vq
);
1874 /* Reading and writing a structure directly to QEMUFile is *awful*, but
1875 * it is what QEMU has always done by mistake. We can change it sooner
1876 * or later by bumping the version number of the affected vm states.
1877 * In the meanwhile, since the in-memory layout of VirtQueueElement
1878 * has changed, we need to marshal to and from the layout that was
1879 * used before the change.
1881 typedef struct VirtQueueElementOld
{
1883 unsigned int out_num
;
1884 unsigned int in_num
;
1885 hwaddr in_addr
[VIRTQUEUE_MAX_SIZE
];
1886 hwaddr out_addr
[VIRTQUEUE_MAX_SIZE
];
1887 struct iovec in_sg
[VIRTQUEUE_MAX_SIZE
];
1888 struct iovec out_sg
[VIRTQUEUE_MAX_SIZE
];
1889 } VirtQueueElementOld
;
1891 void *qemu_get_virtqueue_element(VirtIODevice
*vdev
, QEMUFile
*f
, size_t sz
)
1893 VirtQueueElement
*elem
;
1894 VirtQueueElementOld data
;
1897 qemu_get_buffer(f
, (uint8_t *)&data
, sizeof(VirtQueueElementOld
));
1899 /* TODO: teach all callers that this can fail, and return failure instead
1900 * of asserting here.
1901 * This is just one thing (there are probably more) that must be
1902 * fixed before we can allow NDEBUG compilation.
1904 assert(ARRAY_SIZE(data
.in_addr
) >= data
.in_num
);
1905 assert(ARRAY_SIZE(data
.out_addr
) >= data
.out_num
);
1907 elem
= virtqueue_alloc_element(sz
, data
.out_num
, data
.in_num
);
1908 elem
->index
= data
.index
;
1910 for (i
= 0; i
< elem
->in_num
; i
++) {
1911 elem
->in_addr
[i
] = data
.in_addr
[i
];
1914 for (i
= 0; i
< elem
->out_num
; i
++) {
1915 elem
->out_addr
[i
] = data
.out_addr
[i
];
1918 for (i
= 0; i
< elem
->in_num
; i
++) {
1919 /* Base is overwritten by virtqueue_map. */
1920 elem
->in_sg
[i
].iov_base
= 0;
1921 elem
->in_sg
[i
].iov_len
= data
.in_sg
[i
].iov_len
;
1924 for (i
= 0; i
< elem
->out_num
; i
++) {
1925 /* Base is overwritten by virtqueue_map. */
1926 elem
->out_sg
[i
].iov_base
= 0;
1927 elem
->out_sg
[i
].iov_len
= data
.out_sg
[i
].iov_len
;
1930 if (virtio_host_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
1931 qemu_get_be32s(f
, &elem
->ndescs
);
1934 virtqueue_map(vdev
, elem
);
1938 void qemu_put_virtqueue_element(VirtIODevice
*vdev
, QEMUFile
*f
,
1939 VirtQueueElement
*elem
)
1941 VirtQueueElementOld data
;
1944 memset(&data
, 0, sizeof(data
));
1945 data
.index
= elem
->index
;
1946 data
.in_num
= elem
->in_num
;
1947 data
.out_num
= elem
->out_num
;
1949 for (i
= 0; i
< elem
->in_num
; i
++) {
1950 data
.in_addr
[i
] = elem
->in_addr
[i
];
1953 for (i
= 0; i
< elem
->out_num
; i
++) {
1954 data
.out_addr
[i
] = elem
->out_addr
[i
];
1957 for (i
= 0; i
< elem
->in_num
; i
++) {
1958 /* Base is overwritten by virtqueue_map when loading. Do not
1959 * save it, as it would leak the QEMU address space layout. */
1960 data
.in_sg
[i
].iov_len
= elem
->in_sg
[i
].iov_len
;
1963 for (i
= 0; i
< elem
->out_num
; i
++) {
1964 /* Do not save iov_base as above. */
1965 data
.out_sg
[i
].iov_len
= elem
->out_sg
[i
].iov_len
;
1968 if (virtio_host_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
1969 qemu_put_be32s(f
, &elem
->ndescs
);
1972 qemu_put_buffer(f
, (uint8_t *)&data
, sizeof(VirtQueueElementOld
));
1976 static void virtio_notify_vector(VirtIODevice
*vdev
, uint16_t vector
)
1978 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
1979 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
1981 if (virtio_device_disabled(vdev
)) {
1986 k
->notify(qbus
->parent
, vector
);
1990 void virtio_update_irq(VirtIODevice
*vdev
)
1992 virtio_notify_vector(vdev
, VIRTIO_NO_VECTOR
);
1995 static int virtio_validate_features(VirtIODevice
*vdev
)
1997 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
1999 if (virtio_host_has_feature(vdev
, VIRTIO_F_IOMMU_PLATFORM
) &&
2000 !virtio_vdev_has_feature(vdev
, VIRTIO_F_IOMMU_PLATFORM
)) {
2004 if (k
->validate_features
) {
2005 return k
->validate_features(vdev
);
2011 int virtio_set_status(VirtIODevice
*vdev
, uint8_t val
)
2013 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2014 trace_virtio_set_status(vdev
, val
);
2016 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
2017 if (!(vdev
->status
& VIRTIO_CONFIG_S_FEATURES_OK
) &&
2018 val
& VIRTIO_CONFIG_S_FEATURES_OK
) {
2019 int ret
= virtio_validate_features(vdev
);
2027 if ((vdev
->status
& VIRTIO_CONFIG_S_DRIVER_OK
) !=
2028 (val
& VIRTIO_CONFIG_S_DRIVER_OK
)) {
2029 virtio_set_started(vdev
, val
& VIRTIO_CONFIG_S_DRIVER_OK
);
2032 if (k
->set_status
) {
2033 k
->set_status(vdev
, val
);
2040 static enum virtio_device_endian
virtio_default_endian(void)
2042 if (target_words_bigendian()) {
2043 return VIRTIO_DEVICE_ENDIAN_BIG
;
2045 return VIRTIO_DEVICE_ENDIAN_LITTLE
;
2049 static enum virtio_device_endian
virtio_current_cpu_endian(void)
2051 if (cpu_virtio_is_big_endian(current_cpu
)) {
2052 return VIRTIO_DEVICE_ENDIAN_BIG
;
2054 return VIRTIO_DEVICE_ENDIAN_LITTLE
;
2058 static void __virtio_queue_reset(VirtIODevice
*vdev
, uint32_t i
)
2060 vdev
->vq
[i
].vring
.desc
= 0;
2061 vdev
->vq
[i
].vring
.avail
= 0;
2062 vdev
->vq
[i
].vring
.used
= 0;
2063 vdev
->vq
[i
].last_avail_idx
= 0;
2064 vdev
->vq
[i
].shadow_avail_idx
= 0;
2065 vdev
->vq
[i
].used_idx
= 0;
2066 vdev
->vq
[i
].last_avail_wrap_counter
= true;
2067 vdev
->vq
[i
].shadow_avail_wrap_counter
= true;
2068 vdev
->vq
[i
].used_wrap_counter
= true;
2069 virtio_queue_set_vector(vdev
, i
, VIRTIO_NO_VECTOR
);
2070 vdev
->vq
[i
].signalled_used
= 0;
2071 vdev
->vq
[i
].signalled_used_valid
= false;
2072 vdev
->vq
[i
].notification
= true;
2073 vdev
->vq
[i
].vring
.num
= vdev
->vq
[i
].vring
.num_default
;
2074 vdev
->vq
[i
].inuse
= 0;
2075 virtio_virtqueue_reset_region_cache(&vdev
->vq
[i
]);
2078 void virtio_queue_reset(VirtIODevice
*vdev
, uint32_t queue_index
)
2080 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2082 if (k
->queue_reset
) {
2083 k
->queue_reset(vdev
, queue_index
);
2086 __virtio_queue_reset(vdev
, queue_index
);
2089 void virtio_queue_enable(VirtIODevice
*vdev
, uint32_t queue_index
)
2091 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2094 * TODO: Seabios is currently out of spec and triggering this error.
2095 * So this needs to be fixed in Seabios, then this can
2096 * be re-enabled for new machine types only, and also after
2097 * being converted to LOG_GUEST_ERROR.
2099 if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
2100 error_report("queue_enable is only suppported in devices of virtio "
2105 if (k
->queue_enable
) {
2106 k
->queue_enable(vdev
, queue_index
);
2110 void virtio_reset(void *opaque
)
2112 VirtIODevice
*vdev
= opaque
;
2113 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2116 virtio_set_status(vdev
, 0);
2118 /* Guest initiated reset */
2119 vdev
->device_endian
= virtio_current_cpu_endian();
2122 vdev
->device_endian
= virtio_default_endian();
2129 vdev
->start_on_kick
= false;
2130 vdev
->started
= false;
2131 vdev
->broken
= false;
2132 vdev
->guest_features
= 0;
2133 vdev
->queue_sel
= 0;
2135 vdev
->disabled
= false;
2136 qatomic_set(&vdev
->isr
, 0);
2137 vdev
->config_vector
= VIRTIO_NO_VECTOR
;
2138 virtio_notify_vector(vdev
, vdev
->config_vector
);
2140 for(i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2141 __virtio_queue_reset(vdev
, i
);
2145 void virtio_queue_set_addr(VirtIODevice
*vdev
, int n
, hwaddr addr
)
2147 if (!vdev
->vq
[n
].vring
.num
) {
2150 vdev
->vq
[n
].vring
.desc
= addr
;
2151 virtio_queue_update_rings(vdev
, n
);
2154 hwaddr
virtio_queue_get_addr(VirtIODevice
*vdev
, int n
)
2156 return vdev
->vq
[n
].vring
.desc
;
2159 void virtio_queue_set_rings(VirtIODevice
*vdev
, int n
, hwaddr desc
,
2160 hwaddr avail
, hwaddr used
)
2162 if (!vdev
->vq
[n
].vring
.num
) {
2165 vdev
->vq
[n
].vring
.desc
= desc
;
2166 vdev
->vq
[n
].vring
.avail
= avail
;
2167 vdev
->vq
[n
].vring
.used
= used
;
2168 virtio_init_region_cache(vdev
, n
);
2171 void virtio_queue_set_num(VirtIODevice
*vdev
, int n
, int num
)
2173 /* Don't allow guest to flip queue between existent and
2174 * nonexistent states, or to set it to an invalid size.
2176 if (!!num
!= !!vdev
->vq
[n
].vring
.num
||
2177 num
> VIRTQUEUE_MAX_SIZE
||
2181 vdev
->vq
[n
].vring
.num
= num
;
2184 VirtQueue
*virtio_vector_first_queue(VirtIODevice
*vdev
, uint16_t vector
)
2186 return QLIST_FIRST(&vdev
->vector_queues
[vector
]);
2189 VirtQueue
*virtio_vector_next_queue(VirtQueue
*vq
)
2191 return QLIST_NEXT(vq
, node
);
2194 int virtio_queue_get_num(VirtIODevice
*vdev
, int n
)
2196 return vdev
->vq
[n
].vring
.num
;
2199 int virtio_queue_get_max_num(VirtIODevice
*vdev
, int n
)
2201 return vdev
->vq
[n
].vring
.num_default
;
2204 int virtio_get_num_queues(VirtIODevice
*vdev
)
2208 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2209 if (!virtio_queue_get_num(vdev
, i
)) {
2217 void virtio_queue_set_align(VirtIODevice
*vdev
, int n
, int align
)
2219 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2220 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2222 /* virtio-1 compliant devices cannot change the alignment */
2223 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
2224 error_report("tried to modify queue alignment for virtio-1 device");
2227 /* Check that the transport told us it was going to do this
2228 * (so a buggy transport will immediately assert rather than
2229 * silently failing to migrate this state)
2231 assert(k
->has_variable_vring_alignment
);
2234 vdev
->vq
[n
].vring
.align
= align
;
2235 virtio_queue_update_rings(vdev
, n
);
2239 static void virtio_queue_notify_vq(VirtQueue
*vq
)
2241 if (vq
->vring
.desc
&& vq
->handle_output
) {
2242 VirtIODevice
*vdev
= vq
->vdev
;
2244 if (unlikely(vdev
->broken
)) {
2248 trace_virtio_queue_notify(vdev
, vq
- vdev
->vq
, vq
);
2249 vq
->handle_output(vdev
, vq
);
2251 if (unlikely(vdev
->start_on_kick
)) {
2252 virtio_set_started(vdev
, true);
2257 void virtio_queue_notify(VirtIODevice
*vdev
, int n
)
2259 VirtQueue
*vq
= &vdev
->vq
[n
];
2261 if (unlikely(!vq
->vring
.desc
|| vdev
->broken
)) {
2265 trace_virtio_queue_notify(vdev
, vq
- vdev
->vq
, vq
);
2266 if (vq
->host_notifier_enabled
) {
2267 event_notifier_set(&vq
->host_notifier
);
2268 } else if (vq
->handle_output
) {
2269 vq
->handle_output(vdev
, vq
);
2271 if (unlikely(vdev
->start_on_kick
)) {
2272 virtio_set_started(vdev
, true);
2277 uint16_t virtio_queue_vector(VirtIODevice
*vdev
, int n
)
2279 return n
< VIRTIO_QUEUE_MAX
? vdev
->vq
[n
].vector
:
2283 void virtio_queue_set_vector(VirtIODevice
*vdev
, int n
, uint16_t vector
)
2285 VirtQueue
*vq
= &vdev
->vq
[n
];
2287 if (n
< VIRTIO_QUEUE_MAX
) {
2288 if (vdev
->vector_queues
&&
2289 vdev
->vq
[n
].vector
!= VIRTIO_NO_VECTOR
) {
2290 QLIST_REMOVE(vq
, node
);
2292 vdev
->vq
[n
].vector
= vector
;
2293 if (vdev
->vector_queues
&&
2294 vector
!= VIRTIO_NO_VECTOR
) {
2295 QLIST_INSERT_HEAD(&vdev
->vector_queues
[vector
], vq
, node
);
2300 VirtQueue
*virtio_add_queue(VirtIODevice
*vdev
, int queue_size
,
2301 VirtIOHandleOutput handle_output
)
2305 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2306 if (vdev
->vq
[i
].vring
.num
== 0)
2310 if (i
== VIRTIO_QUEUE_MAX
|| queue_size
> VIRTQUEUE_MAX_SIZE
)
2313 vdev
->vq
[i
].vring
.num
= queue_size
;
2314 vdev
->vq
[i
].vring
.num_default
= queue_size
;
2315 vdev
->vq
[i
].vring
.align
= VIRTIO_PCI_VRING_ALIGN
;
2316 vdev
->vq
[i
].handle_output
= handle_output
;
2317 vdev
->vq
[i
].used_elems
= g_new0(VirtQueueElement
, queue_size
);
2319 return &vdev
->vq
[i
];
2322 void virtio_delete_queue(VirtQueue
*vq
)
2325 vq
->vring
.num_default
= 0;
2326 vq
->handle_output
= NULL
;
2327 g_free(vq
->used_elems
);
2328 vq
->used_elems
= NULL
;
2329 virtio_virtqueue_reset_region_cache(vq
);
2332 void virtio_del_queue(VirtIODevice
*vdev
, int n
)
2334 if (n
< 0 || n
>= VIRTIO_QUEUE_MAX
) {
2338 virtio_delete_queue(&vdev
->vq
[n
]);
2341 static void virtio_set_isr(VirtIODevice
*vdev
, int value
)
2343 uint8_t old
= qatomic_read(&vdev
->isr
);
2345 /* Do not write ISR if it does not change, so that its cacheline remains
2346 * shared in the common case where the guest does not read it.
2348 if ((old
& value
) != value
) {
2349 qatomic_or(&vdev
->isr
, value
);
2353 /* Called within rcu_read_lock(). */
2354 static bool virtio_split_should_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
2358 /* We need to expose used array entries before checking used event. */
2360 /* Always notify when queue is empty (when feature acknowledge) */
2361 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_NOTIFY_ON_EMPTY
) &&
2362 !vq
->inuse
&& virtio_queue_empty(vq
)) {
2366 if (!virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
2367 return !(vring_avail_flags(vq
) & VRING_AVAIL_F_NO_INTERRUPT
);
2370 v
= vq
->signalled_used_valid
;
2371 vq
->signalled_used_valid
= true;
2372 old
= vq
->signalled_used
;
2373 new = vq
->signalled_used
= vq
->used_idx
;
2374 return !v
|| vring_need_event(vring_get_used_event(vq
), new, old
);
2377 static bool vring_packed_need_event(VirtQueue
*vq
, bool wrap
,
2378 uint16_t off_wrap
, uint16_t new,
2381 int off
= off_wrap
& ~(1 << 15);
2383 if (wrap
!= off_wrap
>> 15) {
2384 off
-= vq
->vring
.num
;
2387 return vring_need_event(off
, new, old
);
2390 /* Called within rcu_read_lock(). */
2391 static bool virtio_packed_should_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
2393 VRingPackedDescEvent e
;
2396 VRingMemoryRegionCaches
*caches
;
2398 caches
= vring_get_region_caches(vq
);
2403 vring_packed_event_read(vdev
, &caches
->avail
, &e
);
2405 old
= vq
->signalled_used
;
2406 new = vq
->signalled_used
= vq
->used_idx
;
2407 v
= vq
->signalled_used_valid
;
2408 vq
->signalled_used_valid
= true;
2410 if (e
.flags
== VRING_PACKED_EVENT_FLAG_DISABLE
) {
2412 } else if (e
.flags
== VRING_PACKED_EVENT_FLAG_ENABLE
) {
2416 return !v
|| vring_packed_need_event(vq
, vq
->used_wrap_counter
,
2417 e
.off_wrap
, new, old
);
2420 /* Called within rcu_read_lock(). */
2421 static bool virtio_should_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
2423 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
2424 return virtio_packed_should_notify(vdev
, vq
);
2426 return virtio_split_should_notify(vdev
, vq
);
2430 void virtio_notify_irqfd(VirtIODevice
*vdev
, VirtQueue
*vq
)
2432 WITH_RCU_READ_LOCK_GUARD() {
2433 if (!virtio_should_notify(vdev
, vq
)) {
2438 trace_virtio_notify_irqfd(vdev
, vq
);
2441 * virtio spec 1.0 says ISR bit 0 should be ignored with MSI, but
2442 * windows drivers included in virtio-win 1.8.0 (circa 2015) are
2443 * incorrectly polling this bit during crashdump and hibernation
2444 * in MSI mode, causing a hang if this bit is never updated.
2445 * Recent releases of Windows do not really shut down, but rather
2446 * log out and hibernate to make the next startup faster. Hence,
2447 * this manifested as a more serious hang during shutdown with
2449 * Next driver release from 2016 fixed this problem, so working around it
2450 * is not a must, but it's easy to do so let's do it here.
2452 * Note: it's safe to update ISR from any thread as it was switched
2453 * to an atomic operation.
2455 virtio_set_isr(vq
->vdev
, 0x1);
2456 event_notifier_set(&vq
->guest_notifier
);
2459 static void virtio_irq(VirtQueue
*vq
)
2461 virtio_set_isr(vq
->vdev
, 0x1);
2462 virtio_notify_vector(vq
->vdev
, vq
->vector
);
2465 void virtio_notify(VirtIODevice
*vdev
, VirtQueue
*vq
)
2467 WITH_RCU_READ_LOCK_GUARD() {
2468 if (!virtio_should_notify(vdev
, vq
)) {
2473 trace_virtio_notify(vdev
, vq
);
2477 void virtio_notify_config(VirtIODevice
*vdev
)
2479 if (!(vdev
->status
& VIRTIO_CONFIG_S_DRIVER_OK
))
2482 virtio_set_isr(vdev
, 0x3);
2484 virtio_notify_vector(vdev
, vdev
->config_vector
);
2487 static bool virtio_device_endian_needed(void *opaque
)
2489 VirtIODevice
*vdev
= opaque
;
2491 assert(vdev
->device_endian
!= VIRTIO_DEVICE_ENDIAN_UNKNOWN
);
2492 if (!virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
2493 return vdev
->device_endian
!= virtio_default_endian();
2495 /* Devices conforming to VIRTIO 1.0 or later are always LE. */
2496 return vdev
->device_endian
!= VIRTIO_DEVICE_ENDIAN_LITTLE
;
2499 static bool virtio_64bit_features_needed(void *opaque
)
2501 VirtIODevice
*vdev
= opaque
;
2503 return (vdev
->host_features
>> 32) != 0;
2506 static bool virtio_virtqueue_needed(void *opaque
)
2508 VirtIODevice
*vdev
= opaque
;
2510 return virtio_host_has_feature(vdev
, VIRTIO_F_VERSION_1
);
2513 static bool virtio_packed_virtqueue_needed(void *opaque
)
2515 VirtIODevice
*vdev
= opaque
;
2517 return virtio_host_has_feature(vdev
, VIRTIO_F_RING_PACKED
);
2520 static bool virtio_ringsize_needed(void *opaque
)
2522 VirtIODevice
*vdev
= opaque
;
2525 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2526 if (vdev
->vq
[i
].vring
.num
!= vdev
->vq
[i
].vring
.num_default
) {
2533 static bool virtio_extra_state_needed(void *opaque
)
2535 VirtIODevice
*vdev
= opaque
;
2536 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2537 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2539 return k
->has_extra_state
&&
2540 k
->has_extra_state(qbus
->parent
);
2543 static bool virtio_broken_needed(void *opaque
)
2545 VirtIODevice
*vdev
= opaque
;
2547 return vdev
->broken
;
2550 static bool virtio_started_needed(void *opaque
)
2552 VirtIODevice
*vdev
= opaque
;
2554 return vdev
->started
;
2557 static bool virtio_disabled_needed(void *opaque
)
2559 VirtIODevice
*vdev
= opaque
;
2561 return vdev
->disabled
;
2564 static const VMStateDescription vmstate_virtqueue
= {
2565 .name
= "virtqueue_state",
2567 .minimum_version_id
= 1,
2568 .fields
= (VMStateField
[]) {
2569 VMSTATE_UINT64(vring
.avail
, struct VirtQueue
),
2570 VMSTATE_UINT64(vring
.used
, struct VirtQueue
),
2571 VMSTATE_END_OF_LIST()
2575 static const VMStateDescription vmstate_packed_virtqueue
= {
2576 .name
= "packed_virtqueue_state",
2578 .minimum_version_id
= 1,
2579 .fields
= (VMStateField
[]) {
2580 VMSTATE_UINT16(last_avail_idx
, struct VirtQueue
),
2581 VMSTATE_BOOL(last_avail_wrap_counter
, struct VirtQueue
),
2582 VMSTATE_UINT16(used_idx
, struct VirtQueue
),
2583 VMSTATE_BOOL(used_wrap_counter
, struct VirtQueue
),
2584 VMSTATE_UINT32(inuse
, struct VirtQueue
),
2585 VMSTATE_END_OF_LIST()
2589 static const VMStateDescription vmstate_virtio_virtqueues
= {
2590 .name
= "virtio/virtqueues",
2592 .minimum_version_id
= 1,
2593 .needed
= &virtio_virtqueue_needed
,
2594 .fields
= (VMStateField
[]) {
2595 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq
, struct VirtIODevice
,
2596 VIRTIO_QUEUE_MAX
, 0, vmstate_virtqueue
, VirtQueue
),
2597 VMSTATE_END_OF_LIST()
2601 static const VMStateDescription vmstate_virtio_packed_virtqueues
= {
2602 .name
= "virtio/packed_virtqueues",
2604 .minimum_version_id
= 1,
2605 .needed
= &virtio_packed_virtqueue_needed
,
2606 .fields
= (VMStateField
[]) {
2607 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq
, struct VirtIODevice
,
2608 VIRTIO_QUEUE_MAX
, 0, vmstate_packed_virtqueue
, VirtQueue
),
2609 VMSTATE_END_OF_LIST()
2613 static const VMStateDescription vmstate_ringsize
= {
2614 .name
= "ringsize_state",
2616 .minimum_version_id
= 1,
2617 .fields
= (VMStateField
[]) {
2618 VMSTATE_UINT32(vring
.num_default
, struct VirtQueue
),
2619 VMSTATE_END_OF_LIST()
2623 static const VMStateDescription vmstate_virtio_ringsize
= {
2624 .name
= "virtio/ringsize",
2626 .minimum_version_id
= 1,
2627 .needed
= &virtio_ringsize_needed
,
2628 .fields
= (VMStateField
[]) {
2629 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq
, struct VirtIODevice
,
2630 VIRTIO_QUEUE_MAX
, 0, vmstate_ringsize
, VirtQueue
),
2631 VMSTATE_END_OF_LIST()
2635 static int get_extra_state(QEMUFile
*f
, void *pv
, size_t size
,
2636 const VMStateField
*field
)
2638 VirtIODevice
*vdev
= pv
;
2639 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2640 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2642 if (!k
->load_extra_state
) {
2645 return k
->load_extra_state(qbus
->parent
, f
);
2649 static int put_extra_state(QEMUFile
*f
, void *pv
, size_t size
,
2650 const VMStateField
*field
, JSONWriter
*vmdesc
)
2652 VirtIODevice
*vdev
= pv
;
2653 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2654 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2656 k
->save_extra_state(qbus
->parent
, f
);
2660 static const VMStateInfo vmstate_info_extra_state
= {
2661 .name
= "virtqueue_extra_state",
2662 .get
= get_extra_state
,
2663 .put
= put_extra_state
,
2666 static const VMStateDescription vmstate_virtio_extra_state
= {
2667 .name
= "virtio/extra_state",
2669 .minimum_version_id
= 1,
2670 .needed
= &virtio_extra_state_needed
,
2671 .fields
= (VMStateField
[]) {
2673 .name
= "extra_state",
2675 .field_exists
= NULL
,
2677 .info
= &vmstate_info_extra_state
,
2678 .flags
= VMS_SINGLE
,
2681 VMSTATE_END_OF_LIST()
2685 static const VMStateDescription vmstate_virtio_device_endian
= {
2686 .name
= "virtio/device_endian",
2688 .minimum_version_id
= 1,
2689 .needed
= &virtio_device_endian_needed
,
2690 .fields
= (VMStateField
[]) {
2691 VMSTATE_UINT8(device_endian
, VirtIODevice
),
2692 VMSTATE_END_OF_LIST()
2696 static const VMStateDescription vmstate_virtio_64bit_features
= {
2697 .name
= "virtio/64bit_features",
2699 .minimum_version_id
= 1,
2700 .needed
= &virtio_64bit_features_needed
,
2701 .fields
= (VMStateField
[]) {
2702 VMSTATE_UINT64(guest_features
, VirtIODevice
),
2703 VMSTATE_END_OF_LIST()
2707 static const VMStateDescription vmstate_virtio_broken
= {
2708 .name
= "virtio/broken",
2710 .minimum_version_id
= 1,
2711 .needed
= &virtio_broken_needed
,
2712 .fields
= (VMStateField
[]) {
2713 VMSTATE_BOOL(broken
, VirtIODevice
),
2714 VMSTATE_END_OF_LIST()
2718 static const VMStateDescription vmstate_virtio_started
= {
2719 .name
= "virtio/started",
2721 .minimum_version_id
= 1,
2722 .needed
= &virtio_started_needed
,
2723 .fields
= (VMStateField
[]) {
2724 VMSTATE_BOOL(started
, VirtIODevice
),
2725 VMSTATE_END_OF_LIST()
2729 static const VMStateDescription vmstate_virtio_disabled
= {
2730 .name
= "virtio/disabled",
2732 .minimum_version_id
= 1,
2733 .needed
= &virtio_disabled_needed
,
2734 .fields
= (VMStateField
[]) {
2735 VMSTATE_BOOL(disabled
, VirtIODevice
),
2736 VMSTATE_END_OF_LIST()
2740 static const VMStateDescription vmstate_virtio
= {
2743 .minimum_version_id
= 1,
2744 .fields
= (VMStateField
[]) {
2745 VMSTATE_END_OF_LIST()
2747 .subsections
= (const VMStateDescription
*[]) {
2748 &vmstate_virtio_device_endian
,
2749 &vmstate_virtio_64bit_features
,
2750 &vmstate_virtio_virtqueues
,
2751 &vmstate_virtio_ringsize
,
2752 &vmstate_virtio_broken
,
2753 &vmstate_virtio_extra_state
,
2754 &vmstate_virtio_started
,
2755 &vmstate_virtio_packed_virtqueues
,
2756 &vmstate_virtio_disabled
,
2761 int virtio_save(VirtIODevice
*vdev
, QEMUFile
*f
)
2763 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2764 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2765 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2766 uint32_t guest_features_lo
= (vdev
->guest_features
& 0xffffffff);
2769 if (k
->save_config
) {
2770 k
->save_config(qbus
->parent
, f
);
2773 qemu_put_8s(f
, &vdev
->status
);
2774 qemu_put_8s(f
, &vdev
->isr
);
2775 qemu_put_be16s(f
, &vdev
->queue_sel
);
2776 qemu_put_be32s(f
, &guest_features_lo
);
2777 qemu_put_be32(f
, vdev
->config_len
);
2778 qemu_put_buffer(f
, vdev
->config
, vdev
->config_len
);
2780 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2781 if (vdev
->vq
[i
].vring
.num
== 0)
2785 qemu_put_be32(f
, i
);
2787 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2788 if (vdev
->vq
[i
].vring
.num
== 0)
2791 qemu_put_be32(f
, vdev
->vq
[i
].vring
.num
);
2792 if (k
->has_variable_vring_alignment
) {
2793 qemu_put_be32(f
, vdev
->vq
[i
].vring
.align
);
2796 * Save desc now, the rest of the ring addresses are saved in
2797 * subsections for VIRTIO-1 devices.
2799 qemu_put_be64(f
, vdev
->vq
[i
].vring
.desc
);
2800 qemu_put_be16s(f
, &vdev
->vq
[i
].last_avail_idx
);
2801 if (k
->save_queue
) {
2802 k
->save_queue(qbus
->parent
, i
, f
);
2806 if (vdc
->save
!= NULL
) {
2811 int ret
= vmstate_save_state(f
, vdc
->vmsd
, vdev
, NULL
);
2818 return vmstate_save_state(f
, &vmstate_virtio
, vdev
, NULL
);
2821 /* A wrapper for use as a VMState .put function */
2822 static int virtio_device_put(QEMUFile
*f
, void *opaque
, size_t size
,
2823 const VMStateField
*field
, JSONWriter
*vmdesc
)
2825 return virtio_save(VIRTIO_DEVICE(opaque
), f
);
2828 /* A wrapper for use as a VMState .get function */
2829 static int virtio_device_get(QEMUFile
*f
, void *opaque
, size_t size
,
2830 const VMStateField
*field
)
2832 VirtIODevice
*vdev
= VIRTIO_DEVICE(opaque
);
2833 DeviceClass
*dc
= DEVICE_CLASS(VIRTIO_DEVICE_GET_CLASS(vdev
));
2835 return virtio_load(vdev
, f
, dc
->vmsd
->version_id
);
2838 const VMStateInfo virtio_vmstate_info
= {
2840 .get
= virtio_device_get
,
2841 .put
= virtio_device_put
,
2844 static int virtio_set_features_nocheck(VirtIODevice
*vdev
, uint64_t val
)
2846 VirtioDeviceClass
*k
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2847 bool bad
= (val
& ~(vdev
->host_features
)) != 0;
2849 val
&= vdev
->host_features
;
2850 if (k
->set_features
) {
2851 k
->set_features(vdev
, val
);
2853 vdev
->guest_features
= val
;
2854 return bad
? -1 : 0;
2857 int virtio_set_features(VirtIODevice
*vdev
, uint64_t val
)
2861 * The driver must not attempt to set features after feature negotiation
2864 if (vdev
->status
& VIRTIO_CONFIG_S_FEATURES_OK
) {
2868 if (val
& (1ull << VIRTIO_F_BAD_FEATURE
)) {
2869 qemu_log_mask(LOG_GUEST_ERROR
,
2870 "%s: guest driver for %s has enabled UNUSED(30) feature bit!\n",
2871 __func__
, vdev
->name
);
2874 ret
= virtio_set_features_nocheck(vdev
, val
);
2875 if (virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
)) {
2876 /* VIRTIO_RING_F_EVENT_IDX changes the size of the caches. */
2878 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
2879 if (vdev
->vq
[i
].vring
.num
!= 0) {
2880 virtio_init_region_cache(vdev
, i
);
2885 if (!virtio_device_started(vdev
, vdev
->status
) &&
2886 !virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
2887 vdev
->start_on_kick
= true;
2893 size_t virtio_get_config_size(const VirtIOConfigSizeParams
*params
,
2894 uint64_t host_features
)
2896 size_t config_size
= params
->min_size
;
2897 const VirtIOFeature
*feature_sizes
= params
->feature_sizes
;
2900 for (i
= 0; feature_sizes
[i
].flags
!= 0; i
++) {
2901 if (host_features
& feature_sizes
[i
].flags
) {
2902 config_size
= MAX(feature_sizes
[i
].end
, config_size
);
2906 assert(config_size
<= params
->max_size
);
2910 int virtio_load(VirtIODevice
*vdev
, QEMUFile
*f
, int version_id
)
2916 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
2917 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
2918 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(vdev
);
2921 * We poison the endianness to ensure it does not get used before
2922 * subsections have been loaded.
2924 vdev
->device_endian
= VIRTIO_DEVICE_ENDIAN_UNKNOWN
;
2926 if (k
->load_config
) {
2927 ret
= k
->load_config(qbus
->parent
, f
);
2932 qemu_get_8s(f
, &vdev
->status
);
2933 qemu_get_8s(f
, &vdev
->isr
);
2934 qemu_get_be16s(f
, &vdev
->queue_sel
);
2935 if (vdev
->queue_sel
>= VIRTIO_QUEUE_MAX
) {
2938 qemu_get_be32s(f
, &features
);
2941 * Temporarily set guest_features low bits - needed by
2942 * virtio net load code testing for VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
2943 * VIRTIO_NET_F_GUEST_ANNOUNCE and VIRTIO_NET_F_CTRL_VQ.
2945 * Note: devices should always test host features in future - don't create
2946 * new dependencies like this.
2948 vdev
->guest_features
= features
;
2950 config_len
= qemu_get_be32(f
);
2953 * There are cases where the incoming config can be bigger or smaller
2954 * than what we have; so load what we have space for, and skip
2955 * any excess that's in the stream.
2957 qemu_get_buffer(f
, vdev
->config
, MIN(config_len
, vdev
->config_len
));
2959 while (config_len
> vdev
->config_len
) {
2964 num
= qemu_get_be32(f
);
2966 if (num
> VIRTIO_QUEUE_MAX
) {
2967 error_report("Invalid number of virtqueues: 0x%x", num
);
2971 for (i
= 0; i
< num
; i
++) {
2972 vdev
->vq
[i
].vring
.num
= qemu_get_be32(f
);
2973 if (k
->has_variable_vring_alignment
) {
2974 vdev
->vq
[i
].vring
.align
= qemu_get_be32(f
);
2976 vdev
->vq
[i
].vring
.desc
= qemu_get_be64(f
);
2977 qemu_get_be16s(f
, &vdev
->vq
[i
].last_avail_idx
);
2978 vdev
->vq
[i
].signalled_used_valid
= false;
2979 vdev
->vq
[i
].notification
= true;
2981 if (!vdev
->vq
[i
].vring
.desc
&& vdev
->vq
[i
].last_avail_idx
) {
2982 error_report("VQ %d address 0x0 "
2983 "inconsistent with Host index 0x%x",
2984 i
, vdev
->vq
[i
].last_avail_idx
);
2987 if (k
->load_queue
) {
2988 ret
= k
->load_queue(qbus
->parent
, i
, f
);
2994 virtio_notify_vector(vdev
, VIRTIO_NO_VECTOR
);
2996 if (vdc
->load
!= NULL
) {
2997 ret
= vdc
->load(vdev
, f
, version_id
);
3004 ret
= vmstate_load_state(f
, vdc
->vmsd
, vdev
, version_id
);
3011 ret
= vmstate_load_state(f
, &vmstate_virtio
, vdev
, 1);
3016 if (vdev
->device_endian
== VIRTIO_DEVICE_ENDIAN_UNKNOWN
) {
3017 vdev
->device_endian
= virtio_default_endian();
3020 if (virtio_64bit_features_needed(vdev
)) {
3022 * Subsection load filled vdev->guest_features. Run them
3023 * through virtio_set_features to sanity-check them against
3026 uint64_t features64
= vdev
->guest_features
;
3027 if (virtio_set_features_nocheck(vdev
, features64
) < 0) {
3028 error_report("Features 0x%" PRIx64
" unsupported. "
3029 "Allowed features: 0x%" PRIx64
,
3030 features64
, vdev
->host_features
);
3034 if (virtio_set_features_nocheck(vdev
, features
) < 0) {
3035 error_report("Features 0x%x unsupported. "
3036 "Allowed features: 0x%" PRIx64
,
3037 features
, vdev
->host_features
);
3042 if (!virtio_device_started(vdev
, vdev
->status
) &&
3043 !virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
3044 vdev
->start_on_kick
= true;
3047 RCU_READ_LOCK_GUARD();
3048 for (i
= 0; i
< num
; i
++) {
3049 if (vdev
->vq
[i
].vring
.desc
) {
3053 * VIRTIO-1 devices migrate desc, used, and avail ring addresses so
3054 * only the region cache needs to be set up. Legacy devices need
3055 * to calculate used and avail ring addresses based on the desc
3058 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
3059 virtio_init_region_cache(vdev
, i
);
3061 virtio_queue_update_rings(vdev
, i
);
3064 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3065 vdev
->vq
[i
].shadow_avail_idx
= vdev
->vq
[i
].last_avail_idx
;
3066 vdev
->vq
[i
].shadow_avail_wrap_counter
=
3067 vdev
->vq
[i
].last_avail_wrap_counter
;
3071 nheads
= vring_avail_idx(&vdev
->vq
[i
]) - vdev
->vq
[i
].last_avail_idx
;
3072 /* Check it isn't doing strange things with descriptor numbers. */
3073 if (nheads
> vdev
->vq
[i
].vring
.num
) {
3074 virtio_error(vdev
, "VQ %d size 0x%x Guest index 0x%x "
3075 "inconsistent with Host index 0x%x: delta 0x%x",
3076 i
, vdev
->vq
[i
].vring
.num
,
3077 vring_avail_idx(&vdev
->vq
[i
]),
3078 vdev
->vq
[i
].last_avail_idx
, nheads
);
3079 vdev
->vq
[i
].used_idx
= 0;
3080 vdev
->vq
[i
].shadow_avail_idx
= 0;
3081 vdev
->vq
[i
].inuse
= 0;
3084 vdev
->vq
[i
].used_idx
= vring_used_idx(&vdev
->vq
[i
]);
3085 vdev
->vq
[i
].shadow_avail_idx
= vring_avail_idx(&vdev
->vq
[i
]);
3088 * Some devices migrate VirtQueueElements that have been popped
3089 * from the avail ring but not yet returned to the used ring.
3090 * Since max ring size < UINT16_MAX it's safe to use modulo
3091 * UINT16_MAX + 1 subtraction.
3093 vdev
->vq
[i
].inuse
= (uint16_t)(vdev
->vq
[i
].last_avail_idx
-
3094 vdev
->vq
[i
].used_idx
);
3095 if (vdev
->vq
[i
].inuse
> vdev
->vq
[i
].vring
.num
) {
3096 error_report("VQ %d size 0x%x < last_avail_idx 0x%x - "
3098 i
, vdev
->vq
[i
].vring
.num
,
3099 vdev
->vq
[i
].last_avail_idx
,
3100 vdev
->vq
[i
].used_idx
);
3106 if (vdc
->post_load
) {
3107 ret
= vdc
->post_load(vdev
);
3116 void virtio_cleanup(VirtIODevice
*vdev
)
3118 qemu_del_vm_change_state_handler(vdev
->vmstate
);
3121 static void virtio_vmstate_change(void *opaque
, bool running
, RunState state
)
3123 VirtIODevice
*vdev
= opaque
;
3124 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3125 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
3126 bool backend_run
= running
&& virtio_device_started(vdev
, vdev
->status
);
3127 vdev
->vm_running
= running
;
3130 virtio_set_status(vdev
, vdev
->status
);
3133 if (k
->vmstate_change
) {
3134 k
->vmstate_change(qbus
->parent
, backend_run
);
3138 virtio_set_status(vdev
, vdev
->status
);
3142 void virtio_instance_init_common(Object
*proxy_obj
, void *data
,
3143 size_t vdev_size
, const char *vdev_name
)
3145 DeviceState
*vdev
= data
;
3147 object_initialize_child_with_props(proxy_obj
, "virtio-backend", vdev
,
3148 vdev_size
, vdev_name
, &error_abort
,
3150 qdev_alias_all_properties(vdev
, proxy_obj
);
3153 void virtio_init(VirtIODevice
*vdev
, uint16_t device_id
, size_t config_size
)
3155 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3156 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
3158 int nvectors
= k
->query_nvectors
? k
->query_nvectors(qbus
->parent
) : 0;
3161 vdev
->vector_queues
=
3162 g_malloc0(sizeof(*vdev
->vector_queues
) * nvectors
);
3165 vdev
->start_on_kick
= false;
3166 vdev
->started
= false;
3167 vdev
->vhost_started
= false;
3168 vdev
->device_id
= device_id
;
3170 qatomic_set(&vdev
->isr
, 0);
3171 vdev
->queue_sel
= 0;
3172 vdev
->config_vector
= VIRTIO_NO_VECTOR
;
3173 vdev
->vq
= g_new0(VirtQueue
, VIRTIO_QUEUE_MAX
);
3174 vdev
->vm_running
= runstate_is_running();
3175 vdev
->broken
= false;
3176 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
3177 vdev
->vq
[i
].vector
= VIRTIO_NO_VECTOR
;
3178 vdev
->vq
[i
].vdev
= vdev
;
3179 vdev
->vq
[i
].queue_index
= i
;
3180 vdev
->vq
[i
].host_notifier_enabled
= false;
3183 vdev
->name
= virtio_id_to_name(device_id
);
3184 vdev
->config_len
= config_size
;
3185 if (vdev
->config_len
) {
3186 vdev
->config
= g_malloc0(config_size
);
3188 vdev
->config
= NULL
;
3190 vdev
->vmstate
= qdev_add_vm_change_state_handler(DEVICE(vdev
),
3191 virtio_vmstate_change
, vdev
);
3192 vdev
->device_endian
= virtio_default_endian();
3193 vdev
->use_guest_notifier_mask
= true;
3197 * Only devices that have already been around prior to defining the virtio
3198 * standard support legacy mode; this includes devices not specified in the
3199 * standard. All newer devices conform to the virtio standard only.
3201 bool virtio_legacy_allowed(VirtIODevice
*vdev
)
3203 switch (vdev
->device_id
) {
3205 case VIRTIO_ID_BLOCK
:
3206 case VIRTIO_ID_CONSOLE
:
3208 case VIRTIO_ID_BALLOON
:
3209 case VIRTIO_ID_RPMSG
:
3210 case VIRTIO_ID_SCSI
:
3212 case VIRTIO_ID_RPROC_SERIAL
:
3213 case VIRTIO_ID_CAIF
:
3220 bool virtio_legacy_check_disabled(VirtIODevice
*vdev
)
3222 return vdev
->disable_legacy_check
;
3225 hwaddr
virtio_queue_get_desc_addr(VirtIODevice
*vdev
, int n
)
3227 return vdev
->vq
[n
].vring
.desc
;
3230 bool virtio_queue_enabled_legacy(VirtIODevice
*vdev
, int n
)
3232 return virtio_queue_get_desc_addr(vdev
, n
) != 0;
3235 bool virtio_queue_enabled(VirtIODevice
*vdev
, int n
)
3237 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3238 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
3240 if (k
->queue_enabled
) {
3241 return k
->queue_enabled(qbus
->parent
, n
);
3243 return virtio_queue_enabled_legacy(vdev
, n
);
3246 hwaddr
virtio_queue_get_avail_addr(VirtIODevice
*vdev
, int n
)
3248 return vdev
->vq
[n
].vring
.avail
;
3251 hwaddr
virtio_queue_get_used_addr(VirtIODevice
*vdev
, int n
)
3253 return vdev
->vq
[n
].vring
.used
;
3256 hwaddr
virtio_queue_get_desc_size(VirtIODevice
*vdev
, int n
)
3258 return sizeof(VRingDesc
) * vdev
->vq
[n
].vring
.num
;
3261 hwaddr
virtio_queue_get_avail_size(VirtIODevice
*vdev
, int n
)
3265 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3266 return sizeof(struct VRingPackedDescEvent
);
3269 s
= virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
) ? 2 : 0;
3270 return offsetof(VRingAvail
, ring
) +
3271 sizeof(uint16_t) * vdev
->vq
[n
].vring
.num
+ s
;
3274 hwaddr
virtio_queue_get_used_size(VirtIODevice
*vdev
, int n
)
3278 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3279 return sizeof(struct VRingPackedDescEvent
);
3282 s
= virtio_vdev_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
) ? 2 : 0;
3283 return offsetof(VRingUsed
, ring
) +
3284 sizeof(VRingUsedElem
) * vdev
->vq
[n
].vring
.num
+ s
;
3287 static unsigned int virtio_queue_packed_get_last_avail_idx(VirtIODevice
*vdev
,
3290 unsigned int avail
, used
;
3292 avail
= vdev
->vq
[n
].last_avail_idx
;
3293 avail
|= ((uint16_t)vdev
->vq
[n
].last_avail_wrap_counter
) << 15;
3295 used
= vdev
->vq
[n
].used_idx
;
3296 used
|= ((uint16_t)vdev
->vq
[n
].used_wrap_counter
) << 15;
3298 return avail
| used
<< 16;
3301 static uint16_t virtio_queue_split_get_last_avail_idx(VirtIODevice
*vdev
,
3304 return vdev
->vq
[n
].last_avail_idx
;
3307 unsigned int virtio_queue_get_last_avail_idx(VirtIODevice
*vdev
, int n
)
3309 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3310 return virtio_queue_packed_get_last_avail_idx(vdev
, n
);
3312 return virtio_queue_split_get_last_avail_idx(vdev
, n
);
3316 static void virtio_queue_packed_set_last_avail_idx(VirtIODevice
*vdev
,
3317 int n
, unsigned int idx
)
3319 struct VirtQueue
*vq
= &vdev
->vq
[n
];
3321 vq
->last_avail_idx
= vq
->shadow_avail_idx
= idx
& 0x7fff;
3322 vq
->last_avail_wrap_counter
=
3323 vq
->shadow_avail_wrap_counter
= !!(idx
& 0x8000);
3325 vq
->used_idx
= idx
& 0x7ffff;
3326 vq
->used_wrap_counter
= !!(idx
& 0x8000);
3329 static void virtio_queue_split_set_last_avail_idx(VirtIODevice
*vdev
,
3330 int n
, unsigned int idx
)
3332 vdev
->vq
[n
].last_avail_idx
= idx
;
3333 vdev
->vq
[n
].shadow_avail_idx
= idx
;
3336 void virtio_queue_set_last_avail_idx(VirtIODevice
*vdev
, int n
,
3339 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3340 virtio_queue_packed_set_last_avail_idx(vdev
, n
, idx
);
3342 virtio_queue_split_set_last_avail_idx(vdev
, n
, idx
);
3346 static void virtio_queue_packed_restore_last_avail_idx(VirtIODevice
*vdev
,
3349 /* We don't have a reference like avail idx in shared memory */
3353 static void virtio_queue_split_restore_last_avail_idx(VirtIODevice
*vdev
,
3356 RCU_READ_LOCK_GUARD();
3357 if (vdev
->vq
[n
].vring
.desc
) {
3358 vdev
->vq
[n
].last_avail_idx
= vring_used_idx(&vdev
->vq
[n
]);
3359 vdev
->vq
[n
].shadow_avail_idx
= vdev
->vq
[n
].last_avail_idx
;
3363 void virtio_queue_restore_last_avail_idx(VirtIODevice
*vdev
, int n
)
3365 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3366 virtio_queue_packed_restore_last_avail_idx(vdev
, n
);
3368 virtio_queue_split_restore_last_avail_idx(vdev
, n
);
3372 static void virtio_queue_packed_update_used_idx(VirtIODevice
*vdev
, int n
)
3374 /* used idx was updated through set_last_avail_idx() */
3378 static void virtio_split_packed_update_used_idx(VirtIODevice
*vdev
, int n
)
3380 RCU_READ_LOCK_GUARD();
3381 if (vdev
->vq
[n
].vring
.desc
) {
3382 vdev
->vq
[n
].used_idx
= vring_used_idx(&vdev
->vq
[n
]);
3386 void virtio_queue_update_used_idx(VirtIODevice
*vdev
, int n
)
3388 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3389 return virtio_queue_packed_update_used_idx(vdev
, n
);
3391 return virtio_split_packed_update_used_idx(vdev
, n
);
3395 void virtio_queue_invalidate_signalled_used(VirtIODevice
*vdev
, int n
)
3397 vdev
->vq
[n
].signalled_used_valid
= false;
3400 VirtQueue
*virtio_get_queue(VirtIODevice
*vdev
, int n
)
3402 return vdev
->vq
+ n
;
3405 uint16_t virtio_get_queue_index(VirtQueue
*vq
)
3407 return vq
->queue_index
;
3410 static void virtio_queue_guest_notifier_read(EventNotifier
*n
)
3412 VirtQueue
*vq
= container_of(n
, VirtQueue
, guest_notifier
);
3413 if (event_notifier_test_and_clear(n
)) {
3417 static void virtio_config_guest_notifier_read(EventNotifier
*n
)
3419 VirtIODevice
*vdev
= container_of(n
, VirtIODevice
, config_notifier
);
3421 if (event_notifier_test_and_clear(n
)) {
3422 virtio_notify_config(vdev
);
3425 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue
*vq
, bool assign
,
3428 if (assign
&& !with_irqfd
) {
3429 event_notifier_set_handler(&vq
->guest_notifier
,
3430 virtio_queue_guest_notifier_read
);
3432 event_notifier_set_handler(&vq
->guest_notifier
, NULL
);
3435 /* Test and clear notifier before closing it,
3436 * in case poll callback didn't have time to run. */
3437 virtio_queue_guest_notifier_read(&vq
->guest_notifier
);
3441 void virtio_config_set_guest_notifier_fd_handler(VirtIODevice
*vdev
,
3442 bool assign
, bool with_irqfd
)
3445 n
= &vdev
->config_notifier
;
3446 if (assign
&& !with_irqfd
) {
3447 event_notifier_set_handler(n
, virtio_config_guest_notifier_read
);
3449 event_notifier_set_handler(n
, NULL
);
3452 /* Test and clear notifier before closing it,*/
3453 /* in case poll callback didn't have time to run. */
3454 virtio_config_guest_notifier_read(n
);
3458 EventNotifier
*virtio_queue_get_guest_notifier(VirtQueue
*vq
)
3460 return &vq
->guest_notifier
;
3463 static void virtio_queue_host_notifier_aio_poll_begin(EventNotifier
*n
)
3465 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
3467 virtio_queue_set_notification(vq
, 0);
3470 static bool virtio_queue_host_notifier_aio_poll(void *opaque
)
3472 EventNotifier
*n
= opaque
;
3473 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
3475 return vq
->vring
.desc
&& !virtio_queue_empty(vq
);
3478 static void virtio_queue_host_notifier_aio_poll_ready(EventNotifier
*n
)
3480 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
3482 virtio_queue_notify_vq(vq
);
3485 static void virtio_queue_host_notifier_aio_poll_end(EventNotifier
*n
)
3487 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
3489 /* Caller polls once more after this to catch requests that race with us */
3490 virtio_queue_set_notification(vq
, 1);
3493 void virtio_queue_aio_attach_host_notifier(VirtQueue
*vq
, AioContext
*ctx
)
3495 aio_set_event_notifier(ctx
, &vq
->host_notifier
, true,
3496 virtio_queue_host_notifier_read
,
3497 virtio_queue_host_notifier_aio_poll
,
3498 virtio_queue_host_notifier_aio_poll_ready
);
3499 aio_set_event_notifier_poll(ctx
, &vq
->host_notifier
,
3500 virtio_queue_host_notifier_aio_poll_begin
,
3501 virtio_queue_host_notifier_aio_poll_end
);
3505 * Same as virtio_queue_aio_attach_host_notifier() but without polling. Use
3506 * this for rx virtqueues and similar cases where the virtqueue handler
3507 * function does not pop all elements. When the virtqueue is left non-empty
3508 * polling consumes CPU cycles and should not be used.
3510 void virtio_queue_aio_attach_host_notifier_no_poll(VirtQueue
*vq
, AioContext
*ctx
)
3512 aio_set_event_notifier(ctx
, &vq
->host_notifier
, true,
3513 virtio_queue_host_notifier_read
,
3517 void virtio_queue_aio_detach_host_notifier(VirtQueue
*vq
, AioContext
*ctx
)
3519 aio_set_event_notifier(ctx
, &vq
->host_notifier
, true, NULL
, NULL
, NULL
);
3520 /* Test and clear notifier before after disabling event,
3521 * in case poll callback didn't have time to run. */
3522 virtio_queue_host_notifier_read(&vq
->host_notifier
);
3525 void virtio_queue_host_notifier_read(EventNotifier
*n
)
3527 VirtQueue
*vq
= container_of(n
, VirtQueue
, host_notifier
);
3528 if (event_notifier_test_and_clear(n
)) {
3529 virtio_queue_notify_vq(vq
);
3533 EventNotifier
*virtio_queue_get_host_notifier(VirtQueue
*vq
)
3535 return &vq
->host_notifier
;
3538 EventNotifier
*virtio_config_get_guest_notifier(VirtIODevice
*vdev
)
3540 return &vdev
->config_notifier
;
3543 void virtio_queue_set_host_notifier_enabled(VirtQueue
*vq
, bool enabled
)
3545 vq
->host_notifier_enabled
= enabled
;
3548 int virtio_queue_set_host_notifier_mr(VirtIODevice
*vdev
, int n
,
3549 MemoryRegion
*mr
, bool assign
)
3551 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3552 VirtioBusClass
*k
= VIRTIO_BUS_GET_CLASS(qbus
);
3554 if (k
->set_host_notifier_mr
) {
3555 return k
->set_host_notifier_mr(qbus
->parent
, n
, mr
, assign
);
3561 void virtio_device_set_child_bus_name(VirtIODevice
*vdev
, char *bus_name
)
3563 g_free(vdev
->bus_name
);
3564 vdev
->bus_name
= g_strdup(bus_name
);
3567 void G_GNUC_PRINTF(2, 3) virtio_error(VirtIODevice
*vdev
, const char *fmt
, ...)
3572 error_vreport(fmt
, ap
);
3575 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_VERSION_1
)) {
3576 vdev
->status
= vdev
->status
| VIRTIO_CONFIG_S_NEEDS_RESET
;
3577 virtio_notify_config(vdev
);
3580 vdev
->broken
= true;
3583 static void virtio_memory_listener_commit(MemoryListener
*listener
)
3585 VirtIODevice
*vdev
= container_of(listener
, VirtIODevice
, listener
);
3588 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
3589 if (vdev
->vq
[i
].vring
.num
== 0) {
3592 virtio_init_region_cache(vdev
, i
);
3596 static void virtio_device_realize(DeviceState
*dev
, Error
**errp
)
3598 VirtIODevice
*vdev
= VIRTIO_DEVICE(dev
);
3599 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(dev
);
3602 /* Devices should either use vmsd or the load/save methods */
3603 assert(!vdc
->vmsd
|| !vdc
->load
);
3605 if (vdc
->realize
!= NULL
) {
3606 vdc
->realize(dev
, &err
);
3608 error_propagate(errp
, err
);
3613 virtio_bus_device_plugged(vdev
, &err
);
3615 error_propagate(errp
, err
);
3616 vdc
->unrealize(dev
);
3620 vdev
->listener
.commit
= virtio_memory_listener_commit
;
3621 vdev
->listener
.name
= "virtio";
3622 memory_listener_register(&vdev
->listener
, vdev
->dma_as
);
3623 QTAILQ_INSERT_TAIL(&virtio_list
, vdev
, next
);
3626 static void virtio_device_unrealize(DeviceState
*dev
)
3628 VirtIODevice
*vdev
= VIRTIO_DEVICE(dev
);
3629 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(dev
);
3631 memory_listener_unregister(&vdev
->listener
);
3632 virtio_bus_device_unplugged(vdev
);
3634 if (vdc
->unrealize
!= NULL
) {
3635 vdc
->unrealize(dev
);
3638 QTAILQ_REMOVE(&virtio_list
, vdev
, next
);
3639 g_free(vdev
->bus_name
);
3640 vdev
->bus_name
= NULL
;
3643 static void virtio_device_free_virtqueues(VirtIODevice
*vdev
)
3650 for (i
= 0; i
< VIRTIO_QUEUE_MAX
; i
++) {
3651 if (vdev
->vq
[i
].vring
.num
== 0) {
3654 virtio_virtqueue_reset_region_cache(&vdev
->vq
[i
]);
3659 static void virtio_device_instance_finalize(Object
*obj
)
3661 VirtIODevice
*vdev
= VIRTIO_DEVICE(obj
);
3663 virtio_device_free_virtqueues(vdev
);
3665 g_free(vdev
->config
);
3666 g_free(vdev
->vector_queues
);
3669 static Property virtio_properties
[] = {
3670 DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice
, host_features
),
3671 DEFINE_PROP_BOOL("use-started", VirtIODevice
, use_started
, true),
3672 DEFINE_PROP_BOOL("use-disabled-flag", VirtIODevice
, use_disabled_flag
, true),
3673 DEFINE_PROP_BOOL("x-disable-legacy-check", VirtIODevice
,
3674 disable_legacy_check
, false),
3675 DEFINE_PROP_END_OF_LIST(),
3678 static int virtio_device_start_ioeventfd_impl(VirtIODevice
*vdev
)
3680 VirtioBusState
*qbus
= VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev
)));
3684 * Batch all the host notifiers in a single transaction to avoid
3685 * quadratic time complexity in address_space_update_ioeventfds().
3687 memory_region_transaction_begin();
3688 for (n
= 0; n
< VIRTIO_QUEUE_MAX
; n
++) {
3689 VirtQueue
*vq
= &vdev
->vq
[n
];
3690 if (!virtio_queue_get_num(vdev
, n
)) {
3693 r
= virtio_bus_set_host_notifier(qbus
, n
, true);
3698 event_notifier_set_handler(&vq
->host_notifier
,
3699 virtio_queue_host_notifier_read
);
3702 for (n
= 0; n
< VIRTIO_QUEUE_MAX
; n
++) {
3703 /* Kick right away to begin processing requests already in vring */
3704 VirtQueue
*vq
= &vdev
->vq
[n
];
3705 if (!vq
->vring
.num
) {
3708 event_notifier_set(&vq
->host_notifier
);
3710 memory_region_transaction_commit();
3714 i
= n
; /* save n for a second iteration after transaction is committed. */
3716 VirtQueue
*vq
= &vdev
->vq
[n
];
3717 if (!virtio_queue_get_num(vdev
, n
)) {
3721 event_notifier_set_handler(&vq
->host_notifier
, NULL
);
3722 r
= virtio_bus_set_host_notifier(qbus
, n
, false);
3726 * The transaction expects the ioeventfds to be open when it
3727 * commits. Do it now, before the cleanup loop.
3729 memory_region_transaction_commit();
3732 if (!virtio_queue_get_num(vdev
, i
)) {
3735 virtio_bus_cleanup_host_notifier(qbus
, i
);
3740 int virtio_device_start_ioeventfd(VirtIODevice
*vdev
)
3742 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3743 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
3745 return virtio_bus_start_ioeventfd(vbus
);
3748 static void virtio_device_stop_ioeventfd_impl(VirtIODevice
*vdev
)
3750 VirtioBusState
*qbus
= VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev
)));
3754 * Batch all the host notifiers in a single transaction to avoid
3755 * quadratic time complexity in address_space_update_ioeventfds().
3757 memory_region_transaction_begin();
3758 for (n
= 0; n
< VIRTIO_QUEUE_MAX
; n
++) {
3759 VirtQueue
*vq
= &vdev
->vq
[n
];
3761 if (!virtio_queue_get_num(vdev
, n
)) {
3764 event_notifier_set_handler(&vq
->host_notifier
, NULL
);
3765 r
= virtio_bus_set_host_notifier(qbus
, n
, false);
3769 * The transaction expects the ioeventfds to be open when it
3770 * commits. Do it now, before the cleanup loop.
3772 memory_region_transaction_commit();
3774 for (n
= 0; n
< VIRTIO_QUEUE_MAX
; n
++) {
3775 if (!virtio_queue_get_num(vdev
, n
)) {
3778 virtio_bus_cleanup_host_notifier(qbus
, n
);
3782 int virtio_device_grab_ioeventfd(VirtIODevice
*vdev
)
3784 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3785 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
3787 return virtio_bus_grab_ioeventfd(vbus
);
3790 void virtio_device_release_ioeventfd(VirtIODevice
*vdev
)
3792 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3793 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
3795 virtio_bus_release_ioeventfd(vbus
);
3798 static void virtio_device_class_init(ObjectClass
*klass
, void *data
)
3800 /* Set the default value here. */
3801 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_CLASS(klass
);
3802 DeviceClass
*dc
= DEVICE_CLASS(klass
);
3804 dc
->realize
= virtio_device_realize
;
3805 dc
->unrealize
= virtio_device_unrealize
;
3806 dc
->bus_type
= TYPE_VIRTIO_BUS
;
3807 device_class_set_props(dc
, virtio_properties
);
3808 vdc
->start_ioeventfd
= virtio_device_start_ioeventfd_impl
;
3809 vdc
->stop_ioeventfd
= virtio_device_stop_ioeventfd_impl
;
3811 vdc
->legacy_features
|= VIRTIO_LEGACY_FEATURES
;
3813 QTAILQ_INIT(&virtio_list
);
3816 bool virtio_device_ioeventfd_enabled(VirtIODevice
*vdev
)
3818 BusState
*qbus
= qdev_get_parent_bus(DEVICE(vdev
));
3819 VirtioBusState
*vbus
= VIRTIO_BUS(qbus
);
3821 return virtio_bus_ioeventfd_enabled(vbus
);
3824 VirtQueueStatus
*qmp_x_query_virtio_queue_status(const char *path
,
3829 VirtQueueStatus
*status
;
3831 vdev
= qmp_find_virtio_device(path
);
3833 error_setg(errp
, "Path %s is not a VirtIODevice", path
);
3837 if (queue
>= VIRTIO_QUEUE_MAX
|| !virtio_queue_get_num(vdev
, queue
)) {
3838 error_setg(errp
, "Invalid virtqueue number %d", queue
);
3842 status
= g_new0(VirtQueueStatus
, 1);
3843 status
->name
= g_strdup(vdev
->name
);
3844 status
->queue_index
= vdev
->vq
[queue
].queue_index
;
3845 status
->inuse
= vdev
->vq
[queue
].inuse
;
3846 status
->vring_num
= vdev
->vq
[queue
].vring
.num
;
3847 status
->vring_num_default
= vdev
->vq
[queue
].vring
.num_default
;
3848 status
->vring_align
= vdev
->vq
[queue
].vring
.align
;
3849 status
->vring_desc
= vdev
->vq
[queue
].vring
.desc
;
3850 status
->vring_avail
= vdev
->vq
[queue
].vring
.avail
;
3851 status
->vring_used
= vdev
->vq
[queue
].vring
.used
;
3852 status
->used_idx
= vdev
->vq
[queue
].used_idx
;
3853 status
->signalled_used
= vdev
->vq
[queue
].signalled_used
;
3854 status
->signalled_used_valid
= vdev
->vq
[queue
].signalled_used_valid
;
3856 if (vdev
->vhost_started
) {
3857 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_GET_CLASS(vdev
);
3858 struct vhost_dev
*hdev
= vdc
->get_vhost(vdev
);
3860 /* check if vq index exists for vhost as well */
3861 if (queue
>= hdev
->vq_index
&& queue
< hdev
->vq_index
+ hdev
->nvqs
) {
3862 status
->has_last_avail_idx
= true;
3864 int vhost_vq_index
=
3865 hdev
->vhost_ops
->vhost_get_vq_index(hdev
, queue
);
3866 struct vhost_vring_state state
= {
3867 .index
= vhost_vq_index
,
3870 status
->last_avail_idx
=
3871 hdev
->vhost_ops
->vhost_get_vring_base(hdev
, &state
);
3874 status
->has_shadow_avail_idx
= true;
3875 status
->has_last_avail_idx
= true;
3876 status
->last_avail_idx
= vdev
->vq
[queue
].last_avail_idx
;
3877 status
->shadow_avail_idx
= vdev
->vq
[queue
].shadow_avail_idx
;
3883 static strList
*qmp_decode_vring_desc_flags(uint16_t flags
)
3885 strList
*list
= NULL
;
3893 { VRING_DESC_F_NEXT
, "next" },
3894 { VRING_DESC_F_WRITE
, "write" },
3895 { VRING_DESC_F_INDIRECT
, "indirect" },
3896 { 1 << VRING_PACKED_DESC_F_AVAIL
, "avail" },
3897 { 1 << VRING_PACKED_DESC_F_USED
, "used" },
3901 for (i
= 0; map
[i
].flag
; i
++) {
3902 if ((map
[i
].flag
& flags
) == 0) {
3905 node
= g_malloc0(sizeof(strList
));
3906 node
->value
= g_strdup(map
[i
].value
);
3914 VirtioQueueElement
*qmp_x_query_virtio_queue_element(const char *path
,
3922 VirtioQueueElement
*element
= NULL
;
3924 vdev
= qmp_find_virtio_device(path
);
3926 error_setg(errp
, "Path %s is not a VirtIO device", path
);
3930 if (queue
>= VIRTIO_QUEUE_MAX
|| !virtio_queue_get_num(vdev
, queue
)) {
3931 error_setg(errp
, "Invalid virtqueue number %d", queue
);
3934 vq
= &vdev
->vq
[queue
];
3936 if (virtio_vdev_has_feature(vdev
, VIRTIO_F_RING_PACKED
)) {
3937 error_setg(errp
, "Packed ring not supported");
3940 unsigned int head
, i
, max
;
3941 VRingMemoryRegionCaches
*caches
;
3942 MemoryRegionCache indirect_desc_cache
= MEMORY_REGION_CACHE_INVALID
;
3943 MemoryRegionCache
*desc_cache
;
3945 VirtioRingDescList
*list
= NULL
;
3946 VirtioRingDescList
*node
;
3949 RCU_READ_LOCK_GUARD();
3951 max
= vq
->vring
.num
;
3954 head
= vring_avail_ring(vq
, vq
->last_avail_idx
% vq
->vring
.num
);
3956 head
= vring_avail_ring(vq
, index
% vq
->vring
.num
);
3960 caches
= vring_get_region_caches(vq
);
3962 error_setg(errp
, "Region caches not initialized");
3965 if (caches
->desc
.len
< max
* sizeof(VRingDesc
)) {
3966 error_setg(errp
, "Cannot map descriptor ring");
3970 desc_cache
= &caches
->desc
;
3971 vring_split_desc_read(vdev
, &desc
, desc_cache
, i
);
3972 if (desc
.flags
& VRING_DESC_F_INDIRECT
) {
3974 len
= address_space_cache_init(&indirect_desc_cache
, vdev
->dma_as
,
3975 desc
.addr
, desc
.len
, false);
3976 desc_cache
= &indirect_desc_cache
;
3977 if (len
< desc
.len
) {
3978 error_setg(errp
, "Cannot map indirect buffer");
3982 max
= desc
.len
/ sizeof(VRingDesc
);
3984 vring_split_desc_read(vdev
, &desc
, desc_cache
, i
);
3987 element
= g_new0(VirtioQueueElement
, 1);
3988 element
->avail
= g_new0(VirtioRingAvail
, 1);
3989 element
->used
= g_new0(VirtioRingUsed
, 1);
3990 element
->name
= g_strdup(vdev
->name
);
3991 element
->index
= head
;
3992 element
->avail
->flags
= vring_avail_flags(vq
);
3993 element
->avail
->idx
= vring_avail_idx(vq
);
3994 element
->avail
->ring
= head
;
3995 element
->used
->flags
= vring_used_flags(vq
);
3996 element
->used
->idx
= vring_used_idx(vq
);
4000 /* A buggy driver may produce an infinite loop */
4001 if (ndescs
>= max
) {
4004 node
= g_new0(VirtioRingDescList
, 1);
4005 node
->value
= g_new0(VirtioRingDesc
, 1);
4006 node
->value
->addr
= desc
.addr
;
4007 node
->value
->len
= desc
.len
;
4008 node
->value
->flags
= qmp_decode_vring_desc_flags(desc
.flags
);
4013 rc
= virtqueue_split_read_next_desc(vdev
, &desc
, desc_cache
,
4015 } while (rc
== VIRTQUEUE_READ_DESC_MORE
);
4016 element
->descs
= list
;
4018 address_space_cache_destroy(&indirect_desc_cache
);
4024 static const TypeInfo virtio_device_info
= {
4025 .name
= TYPE_VIRTIO_DEVICE
,
4026 .parent
= TYPE_DEVICE
,
4027 .instance_size
= sizeof(VirtIODevice
),
4028 .class_init
= virtio_device_class_init
,
4029 .instance_finalize
= virtio_device_instance_finalize
,
4031 .class_size
= sizeof(VirtioDeviceClass
),
4034 static void virtio_register_types(void)
4036 type_register_static(&virtio_device_info
);
4039 type_init(virtio_register_types
)