4 * Copyright (c) 2020 Red Hat, Inc.
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2 or later, as published by the Free Software Foundation.
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 * You should have received a copy of the GNU General Public License along with
16 * this program. If not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
23 #include "qemu/range.h"
24 #include "qemu/reserved-region.h"
25 #include "exec/target_page.h"
26 #include "hw/qdev-properties.h"
27 #include "hw/virtio/virtio.h"
28 #include "sysemu/kvm.h"
29 #include "sysemu/reset.h"
30 #include "sysemu/sysemu.h"
31 #include "qemu/reserved-region.h"
32 #include "qapi/error.h"
33 #include "qemu/error-report.h"
36 #include "standard-headers/linux/virtio_ids.h"
38 #include "hw/virtio/virtio-bus.h"
39 #include "hw/virtio/virtio-iommu.h"
40 #include "hw/pci/pci_bus.h"
41 #include "hw/pci/pci.h"
44 #define VIOMMU_DEFAULT_QUEUE_SIZE 256
45 #define VIOMMU_PROBE_SIZE 512
47 typedef struct VirtIOIOMMUDomain
{
51 QLIST_HEAD(, VirtIOIOMMUEndpoint
) endpoint_list
;
54 typedef struct VirtIOIOMMUEndpoint
{
56 VirtIOIOMMUDomain
*domain
;
57 IOMMUMemoryRegion
*iommu_mr
;
58 QLIST_ENTRY(VirtIOIOMMUEndpoint
) next
;
59 } VirtIOIOMMUEndpoint
;
61 typedef struct VirtIOIOMMUInterval
{
64 } VirtIOIOMMUInterval
;
66 typedef struct VirtIOIOMMUMapping
{
71 static inline uint16_t virtio_iommu_get_bdf(IOMMUDevice
*dev
)
73 return PCI_BUILD_BDF(pci_bus_num(dev
->bus
), dev
->devfn
);
76 static bool virtio_iommu_device_bypassed(IOMMUDevice
*sdev
)
80 VirtIOIOMMU
*s
= sdev
->viommu
;
81 VirtIOIOMMUEndpoint
*ep
;
83 sid
= virtio_iommu_get_bdf(sdev
);
85 qemu_rec_mutex_lock(&s
->mutex
);
86 /* need to check bypass before system reset */
88 bypassed
= s
->config
.bypass
;
92 ep
= g_tree_lookup(s
->endpoints
, GUINT_TO_POINTER(sid
));
93 if (!ep
|| !ep
->domain
) {
94 bypassed
= s
->config
.bypass
;
96 bypassed
= ep
->domain
->bypass
;
100 qemu_rec_mutex_unlock(&s
->mutex
);
104 /* Return whether the device is using IOMMU translation. */
105 static bool virtio_iommu_switch_address_space(IOMMUDevice
*sdev
)
111 use_remapping
= !virtio_iommu_device_bypassed(sdev
);
113 trace_virtio_iommu_switch_address_space(pci_bus_num(sdev
->bus
),
114 PCI_SLOT(sdev
->devfn
),
115 PCI_FUNC(sdev
->devfn
),
118 /* Turn off first then on the other */
120 memory_region_set_enabled(&sdev
->bypass_mr
, false);
121 memory_region_set_enabled(MEMORY_REGION(&sdev
->iommu_mr
), true);
123 memory_region_set_enabled(MEMORY_REGION(&sdev
->iommu_mr
), false);
124 memory_region_set_enabled(&sdev
->bypass_mr
, true);
127 return use_remapping
;
130 static void virtio_iommu_switch_address_space_all(VirtIOIOMMU
*s
)
133 IOMMUPciBus
*iommu_pci_bus
;
136 g_hash_table_iter_init(&iter
, s
->as_by_busptr
);
137 while (g_hash_table_iter_next(&iter
, NULL
, (void **)&iommu_pci_bus
)) {
138 for (i
= 0; i
< PCI_DEVFN_MAX
; i
++) {
139 if (!iommu_pci_bus
->pbdev
[i
]) {
142 virtio_iommu_switch_address_space(iommu_pci_bus
->pbdev
[i
]);
148 * The bus number is used for lookup when SID based operations occur.
149 * In that case we lazily populate the IOMMUPciBus array from the bus hash
150 * table. At the time the IOMMUPciBus is created (iommu_find_add_as), the bus
151 * numbers may not be always initialized yet.
153 static IOMMUPciBus
*iommu_find_iommu_pcibus(VirtIOIOMMU
*s
, uint8_t bus_num
)
155 IOMMUPciBus
*iommu_pci_bus
= s
->iommu_pcibus_by_bus_num
[bus_num
];
157 if (!iommu_pci_bus
) {
160 g_hash_table_iter_init(&iter
, s
->as_by_busptr
);
161 while (g_hash_table_iter_next(&iter
, NULL
, (void **)&iommu_pci_bus
)) {
162 if (pci_bus_num(iommu_pci_bus
->bus
) == bus_num
) {
163 s
->iommu_pcibus_by_bus_num
[bus_num
] = iommu_pci_bus
;
164 return iommu_pci_bus
;
169 return iommu_pci_bus
;
172 static IOMMUMemoryRegion
*virtio_iommu_mr(VirtIOIOMMU
*s
, uint32_t sid
)
174 uint8_t bus_n
, devfn
;
175 IOMMUPciBus
*iommu_pci_bus
;
178 bus_n
= PCI_BUS_NUM(sid
);
179 iommu_pci_bus
= iommu_find_iommu_pcibus(s
, bus_n
);
181 devfn
= sid
& (PCI_DEVFN_MAX
- 1);
182 dev
= iommu_pci_bus
->pbdev
[devfn
];
184 return &dev
->iommu_mr
;
190 static gint
interval_cmp(gconstpointer a
, gconstpointer b
, gpointer user_data
)
192 VirtIOIOMMUInterval
*inta
= (VirtIOIOMMUInterval
*)a
;
193 VirtIOIOMMUInterval
*intb
= (VirtIOIOMMUInterval
*)b
;
195 if (inta
->high
< intb
->low
) {
197 } else if (intb
->high
< inta
->low
) {
204 static void virtio_iommu_notify_map_unmap(IOMMUMemoryRegion
*mr
,
205 IOMMUTLBEvent
*event
,
206 hwaddr virt_start
, hwaddr virt_end
)
208 uint64_t delta
= virt_end
- virt_start
;
210 event
->entry
.iova
= virt_start
;
211 event
->entry
.addr_mask
= delta
;
213 if (delta
== UINT64_MAX
) {
214 memory_region_notify_iommu(mr
, 0, *event
);
217 while (virt_start
!= virt_end
+ 1) {
218 uint64_t mask
= dma_aligned_pow2_mask(virt_start
, virt_end
, 64);
220 event
->entry
.addr_mask
= mask
;
221 event
->entry
.iova
= virt_start
;
222 memory_region_notify_iommu(mr
, 0, *event
);
223 virt_start
+= mask
+ 1;
224 if (event
->entry
.perm
!= IOMMU_NONE
) {
225 event
->entry
.translated_addr
+= mask
+ 1;
230 static void virtio_iommu_notify_map(IOMMUMemoryRegion
*mr
, hwaddr virt_start
,
231 hwaddr virt_end
, hwaddr paddr
,
235 IOMMUAccessFlags perm
= IOMMU_ACCESS_FLAG(flags
& VIRTIO_IOMMU_MAP_F_READ
,
236 flags
& VIRTIO_IOMMU_MAP_F_WRITE
);
238 if (!(mr
->iommu_notify_flags
& IOMMU_NOTIFIER_MAP
) ||
239 (flags
& VIRTIO_IOMMU_MAP_F_MMIO
) || !perm
) {
243 trace_virtio_iommu_notify_map(mr
->parent_obj
.name
, virt_start
, virt_end
,
246 event
.type
= IOMMU_NOTIFIER_MAP
;
247 event
.entry
.target_as
= &address_space_memory
;
248 event
.entry
.perm
= perm
;
249 event
.entry
.translated_addr
= paddr
;
251 virtio_iommu_notify_map_unmap(mr
, &event
, virt_start
, virt_end
);
254 static void virtio_iommu_notify_unmap(IOMMUMemoryRegion
*mr
, hwaddr virt_start
,
259 if (!(mr
->iommu_notify_flags
& IOMMU_NOTIFIER_UNMAP
)) {
263 trace_virtio_iommu_notify_unmap(mr
->parent_obj
.name
, virt_start
, virt_end
);
265 event
.type
= IOMMU_NOTIFIER_UNMAP
;
266 event
.entry
.target_as
= &address_space_memory
;
267 event
.entry
.perm
= IOMMU_NONE
;
268 event
.entry
.translated_addr
= 0;
270 virtio_iommu_notify_map_unmap(mr
, &event
, virt_start
, virt_end
);
273 static gboolean
virtio_iommu_notify_unmap_cb(gpointer key
, gpointer value
,
276 VirtIOIOMMUInterval
*interval
= (VirtIOIOMMUInterval
*) key
;
277 IOMMUMemoryRegion
*mr
= (IOMMUMemoryRegion
*) data
;
279 virtio_iommu_notify_unmap(mr
, interval
->low
, interval
->high
);
284 static gboolean
virtio_iommu_notify_map_cb(gpointer key
, gpointer value
,
287 VirtIOIOMMUMapping
*mapping
= (VirtIOIOMMUMapping
*) value
;
288 VirtIOIOMMUInterval
*interval
= (VirtIOIOMMUInterval
*) key
;
289 IOMMUMemoryRegion
*mr
= (IOMMUMemoryRegion
*) data
;
291 virtio_iommu_notify_map(mr
, interval
->low
, interval
->high
,
292 mapping
->phys_addr
, mapping
->flags
);
297 static void virtio_iommu_detach_endpoint_from_domain(VirtIOIOMMUEndpoint
*ep
)
299 VirtIOIOMMUDomain
*domain
= ep
->domain
;
300 IOMMUDevice
*sdev
= container_of(ep
->iommu_mr
, IOMMUDevice
, iommu_mr
);
305 g_tree_foreach(domain
->mappings
, virtio_iommu_notify_unmap_cb
,
307 QLIST_REMOVE(ep
, next
);
309 virtio_iommu_switch_address_space(sdev
);
312 static VirtIOIOMMUEndpoint
*virtio_iommu_get_endpoint(VirtIOIOMMU
*s
,
315 VirtIOIOMMUEndpoint
*ep
;
316 IOMMUMemoryRegion
*mr
;
318 ep
= g_tree_lookup(s
->endpoints
, GUINT_TO_POINTER(ep_id
));
322 mr
= virtio_iommu_mr(s
, ep_id
);
326 ep
= g_malloc0(sizeof(*ep
));
329 trace_virtio_iommu_get_endpoint(ep_id
);
330 g_tree_insert(s
->endpoints
, GUINT_TO_POINTER(ep_id
), ep
);
334 static void virtio_iommu_put_endpoint(gpointer data
)
336 VirtIOIOMMUEndpoint
*ep
= (VirtIOIOMMUEndpoint
*)data
;
339 virtio_iommu_detach_endpoint_from_domain(ep
);
342 trace_virtio_iommu_put_endpoint(ep
->id
);
346 static VirtIOIOMMUDomain
*virtio_iommu_get_domain(VirtIOIOMMU
*s
,
350 VirtIOIOMMUDomain
*domain
;
352 domain
= g_tree_lookup(s
->domains
, GUINT_TO_POINTER(domain_id
));
354 if (domain
->bypass
!= bypass
) {
359 domain
= g_malloc0(sizeof(*domain
));
360 domain
->id
= domain_id
;
361 domain
->mappings
= g_tree_new_full((GCompareDataFunc
)interval_cmp
,
362 NULL
, (GDestroyNotify
)g_free
,
363 (GDestroyNotify
)g_free
);
364 domain
->bypass
= bypass
;
365 g_tree_insert(s
->domains
, GUINT_TO_POINTER(domain_id
), domain
);
366 QLIST_INIT(&domain
->endpoint_list
);
367 trace_virtio_iommu_get_domain(domain_id
);
371 static void virtio_iommu_put_domain(gpointer data
)
373 VirtIOIOMMUDomain
*domain
= (VirtIOIOMMUDomain
*)data
;
374 VirtIOIOMMUEndpoint
*iter
, *tmp
;
376 QLIST_FOREACH_SAFE(iter
, &domain
->endpoint_list
, next
, tmp
) {
377 virtio_iommu_detach_endpoint_from_domain(iter
);
379 g_tree_destroy(domain
->mappings
);
380 trace_virtio_iommu_put_domain(domain
->id
);
384 static void add_prop_resv_regions(IOMMUDevice
*sdev
)
386 VirtIOIOMMU
*s
= sdev
->viommu
;
389 for (i
= 0; i
< s
->nr_prop_resv_regions
; i
++) {
390 ReservedRegion
*reg
= g_new0(ReservedRegion
, 1);
392 *reg
= s
->prop_resv_regions
[i
];
393 sdev
->resv_regions
= resv_region_list_insert(sdev
->resv_regions
, reg
);
397 static AddressSpace
*virtio_iommu_find_add_as(PCIBus
*bus
, void *opaque
,
400 VirtIOIOMMU
*s
= opaque
;
401 IOMMUPciBus
*sbus
= g_hash_table_lookup(s
->as_by_busptr
, bus
);
402 static uint32_t mr_index
;
406 sbus
= g_malloc0(sizeof(IOMMUPciBus
) +
407 sizeof(IOMMUDevice
*) * PCI_DEVFN_MAX
);
409 g_hash_table_insert(s
->as_by_busptr
, bus
, sbus
);
412 sdev
= sbus
->pbdev
[devfn
];
414 char *name
= g_strdup_printf("%s-%d-%d",
415 TYPE_VIRTIO_IOMMU_MEMORY_REGION
,
417 sdev
= sbus
->pbdev
[devfn
] = g_new0(IOMMUDevice
, 1);
423 trace_virtio_iommu_init_iommu_mr(name
);
425 memory_region_init(&sdev
->root
, OBJECT(s
), name
, UINT64_MAX
);
426 address_space_init(&sdev
->as
, &sdev
->root
, TYPE_VIRTIO_IOMMU
);
427 add_prop_resv_regions(sdev
);
430 * Build the IOMMU disabled container with aliases to the
431 * shared MRs. Note that aliasing to a shared memory region
432 * could help the memory API to detect same FlatViews so we
433 * can have devices to share the same FlatView when in bypass
434 * mode. (either by not configuring virtio-iommu driver or with
435 * "iommu=pt"). It will greatly reduce the total number of
436 * FlatViews of the system hence VM runs faster.
438 memory_region_init_alias(&sdev
->bypass_mr
, OBJECT(s
),
439 "system", get_system_memory(), 0,
440 memory_region_size(get_system_memory()));
442 memory_region_init_iommu(&sdev
->iommu_mr
, sizeof(sdev
->iommu_mr
),
443 TYPE_VIRTIO_IOMMU_MEMORY_REGION
,
448 * Hook both the containers under the root container, we
449 * switch between iommu & bypass MRs by enable/disable
450 * corresponding sub-containers
452 memory_region_add_subregion_overlap(&sdev
->root
, 0,
453 MEMORY_REGION(&sdev
->iommu_mr
),
455 memory_region_add_subregion_overlap(&sdev
->root
, 0,
456 &sdev
->bypass_mr
, 0);
458 virtio_iommu_switch_address_space(sdev
);
464 static const PCIIOMMUOps virtio_iommu_ops
= {
465 .get_address_space
= virtio_iommu_find_add_as
,
468 static int virtio_iommu_attach(VirtIOIOMMU
*s
,
469 struct virtio_iommu_req_attach
*req
)
471 uint32_t domain_id
= le32_to_cpu(req
->domain
);
472 uint32_t ep_id
= le32_to_cpu(req
->endpoint
);
473 uint32_t flags
= le32_to_cpu(req
->flags
);
474 VirtIOIOMMUDomain
*domain
;
475 VirtIOIOMMUEndpoint
*ep
;
478 trace_virtio_iommu_attach(domain_id
, ep_id
);
480 if (flags
& ~VIRTIO_IOMMU_ATTACH_F_BYPASS
) {
481 return VIRTIO_IOMMU_S_INVAL
;
484 ep
= virtio_iommu_get_endpoint(s
, ep_id
);
486 return VIRTIO_IOMMU_S_NOENT
;
490 VirtIOIOMMUDomain
*previous_domain
= ep
->domain
;
492 * the device is already attached to a domain,
495 virtio_iommu_detach_endpoint_from_domain(ep
);
496 if (QLIST_EMPTY(&previous_domain
->endpoint_list
)) {
497 g_tree_remove(s
->domains
, GUINT_TO_POINTER(previous_domain
->id
));
501 domain
= virtio_iommu_get_domain(s
, domain_id
,
502 flags
& VIRTIO_IOMMU_ATTACH_F_BYPASS
);
504 /* Incompatible bypass flag */
505 return VIRTIO_IOMMU_S_INVAL
;
507 QLIST_INSERT_HEAD(&domain
->endpoint_list
, ep
, next
);
510 sdev
= container_of(ep
->iommu_mr
, IOMMUDevice
, iommu_mr
);
511 virtio_iommu_switch_address_space(sdev
);
513 /* Replay domain mappings on the associated memory region */
514 g_tree_foreach(domain
->mappings
, virtio_iommu_notify_map_cb
,
517 return VIRTIO_IOMMU_S_OK
;
520 static int virtio_iommu_detach(VirtIOIOMMU
*s
,
521 struct virtio_iommu_req_detach
*req
)
523 uint32_t domain_id
= le32_to_cpu(req
->domain
);
524 uint32_t ep_id
= le32_to_cpu(req
->endpoint
);
525 VirtIOIOMMUDomain
*domain
;
526 VirtIOIOMMUEndpoint
*ep
;
528 trace_virtio_iommu_detach(domain_id
, ep_id
);
530 ep
= g_tree_lookup(s
->endpoints
, GUINT_TO_POINTER(ep_id
));
532 return VIRTIO_IOMMU_S_NOENT
;
537 if (!domain
|| domain
->id
!= domain_id
) {
538 return VIRTIO_IOMMU_S_INVAL
;
541 virtio_iommu_detach_endpoint_from_domain(ep
);
543 if (QLIST_EMPTY(&domain
->endpoint_list
)) {
544 g_tree_remove(s
->domains
, GUINT_TO_POINTER(domain
->id
));
546 return VIRTIO_IOMMU_S_OK
;
549 static int virtio_iommu_map(VirtIOIOMMU
*s
,
550 struct virtio_iommu_req_map
*req
)
552 uint32_t domain_id
= le32_to_cpu(req
->domain
);
553 uint64_t phys_start
= le64_to_cpu(req
->phys_start
);
554 uint64_t virt_start
= le64_to_cpu(req
->virt_start
);
555 uint64_t virt_end
= le64_to_cpu(req
->virt_end
);
556 uint32_t flags
= le32_to_cpu(req
->flags
);
557 VirtIOIOMMUDomain
*domain
;
558 VirtIOIOMMUInterval
*interval
;
559 VirtIOIOMMUMapping
*mapping
;
560 VirtIOIOMMUEndpoint
*ep
;
562 if (flags
& ~VIRTIO_IOMMU_MAP_F_MASK
) {
563 return VIRTIO_IOMMU_S_INVAL
;
566 domain
= g_tree_lookup(s
->domains
, GUINT_TO_POINTER(domain_id
));
568 return VIRTIO_IOMMU_S_NOENT
;
571 if (domain
->bypass
) {
572 return VIRTIO_IOMMU_S_INVAL
;
575 interval
= g_malloc0(sizeof(*interval
));
577 interval
->low
= virt_start
;
578 interval
->high
= virt_end
;
580 mapping
= g_tree_lookup(domain
->mappings
, (gpointer
)interval
);
583 return VIRTIO_IOMMU_S_INVAL
;
586 trace_virtio_iommu_map(domain_id
, virt_start
, virt_end
, phys_start
, flags
);
588 mapping
= g_malloc0(sizeof(*mapping
));
589 mapping
->phys_addr
= phys_start
;
590 mapping
->flags
= flags
;
592 g_tree_insert(domain
->mappings
, interval
, mapping
);
594 QLIST_FOREACH(ep
, &domain
->endpoint_list
, next
) {
595 virtio_iommu_notify_map(ep
->iommu_mr
, virt_start
, virt_end
, phys_start
,
599 return VIRTIO_IOMMU_S_OK
;
602 static int virtio_iommu_unmap(VirtIOIOMMU
*s
,
603 struct virtio_iommu_req_unmap
*req
)
605 uint32_t domain_id
= le32_to_cpu(req
->domain
);
606 uint64_t virt_start
= le64_to_cpu(req
->virt_start
);
607 uint64_t virt_end
= le64_to_cpu(req
->virt_end
);
608 VirtIOIOMMUMapping
*iter_val
;
609 VirtIOIOMMUInterval interval
, *iter_key
;
610 VirtIOIOMMUDomain
*domain
;
611 VirtIOIOMMUEndpoint
*ep
;
612 int ret
= VIRTIO_IOMMU_S_OK
;
614 trace_virtio_iommu_unmap(domain_id
, virt_start
, virt_end
);
616 domain
= g_tree_lookup(s
->domains
, GUINT_TO_POINTER(domain_id
));
618 return VIRTIO_IOMMU_S_NOENT
;
621 if (domain
->bypass
) {
622 return VIRTIO_IOMMU_S_INVAL
;
625 interval
.low
= virt_start
;
626 interval
.high
= virt_end
;
628 while (g_tree_lookup_extended(domain
->mappings
, &interval
,
629 (void **)&iter_key
, (void**)&iter_val
)) {
630 uint64_t current_low
= iter_key
->low
;
631 uint64_t current_high
= iter_key
->high
;
633 if (interval
.low
<= current_low
&& interval
.high
>= current_high
) {
634 QLIST_FOREACH(ep
, &domain
->endpoint_list
, next
) {
635 virtio_iommu_notify_unmap(ep
->iommu_mr
, current_low
,
638 g_tree_remove(domain
->mappings
, iter_key
);
639 trace_virtio_iommu_unmap_done(domain_id
, current_low
, current_high
);
641 ret
= VIRTIO_IOMMU_S_RANGE
;
648 static ssize_t
virtio_iommu_fill_resv_mem_prop(IOMMUDevice
*sdev
, uint32_t ep
,
649 uint8_t *buf
, size_t free
)
651 struct virtio_iommu_probe_resv_mem prop
= {};
652 size_t size
= sizeof(prop
), length
= size
- sizeof(prop
.head
), total
;
655 total
= size
* g_list_length(sdev
->resv_regions
);
660 for (l
= sdev
->resv_regions
; l
; l
= l
->next
) {
661 ReservedRegion
*reg
= l
->data
;
662 unsigned subtype
= reg
->type
;
663 Range
*range
= ®
->range
;
665 assert(subtype
== VIRTIO_IOMMU_RESV_MEM_T_RESERVED
||
666 subtype
== VIRTIO_IOMMU_RESV_MEM_T_MSI
);
667 prop
.head
.type
= cpu_to_le16(VIRTIO_IOMMU_PROBE_T_RESV_MEM
);
668 prop
.head
.length
= cpu_to_le16(length
);
669 prop
.subtype
= subtype
;
670 prop
.start
= cpu_to_le64(range_lob(range
));
671 prop
.end
= cpu_to_le64(range_upb(range
));
673 memcpy(buf
, &prop
, size
);
675 trace_virtio_iommu_fill_resv_property(ep
, prop
.subtype
,
676 prop
.start
, prop
.end
);
683 * virtio_iommu_probe - Fill the probe request buffer with
684 * the properties the device is able to return
686 static int virtio_iommu_probe(VirtIOIOMMU
*s
,
687 struct virtio_iommu_req_probe
*req
,
690 uint32_t ep_id
= le32_to_cpu(req
->endpoint
);
691 IOMMUMemoryRegion
*iommu_mr
= virtio_iommu_mr(s
, ep_id
);
692 size_t free
= VIOMMU_PROBE_SIZE
;
697 return VIRTIO_IOMMU_S_NOENT
;
700 sdev
= container_of(iommu_mr
, IOMMUDevice
, iommu_mr
);
702 count
= virtio_iommu_fill_resv_mem_prop(sdev
, ep_id
, buf
, free
);
704 return VIRTIO_IOMMU_S_INVAL
;
708 sdev
->probe_done
= true;
710 return VIRTIO_IOMMU_S_OK
;
713 static int virtio_iommu_iov_to_req(struct iovec
*iov
,
714 unsigned int iov_cnt
,
715 void *req
, size_t payload_sz
)
717 size_t sz
= iov_to_buf(iov
, iov_cnt
, 0, req
, payload_sz
);
719 if (unlikely(sz
!= payload_sz
)) {
720 return VIRTIO_IOMMU_S_INVAL
;
725 #define virtio_iommu_handle_req(__req) \
726 static int virtio_iommu_handle_ ## __req(VirtIOIOMMU *s, \
728 unsigned int iov_cnt) \
730 struct virtio_iommu_req_ ## __req req; \
731 int ret = virtio_iommu_iov_to_req(iov, iov_cnt, &req, \
732 sizeof(req) - sizeof(struct virtio_iommu_req_tail));\
734 return ret ? ret : virtio_iommu_ ## __req(s, &req); \
737 virtio_iommu_handle_req(attach
)
738 virtio_iommu_handle_req(detach
)
739 virtio_iommu_handle_req(map
)
740 virtio_iommu_handle_req(unmap
)
742 static int virtio_iommu_handle_probe(VirtIOIOMMU
*s
,
744 unsigned int iov_cnt
,
747 struct virtio_iommu_req_probe req
;
748 int ret
= virtio_iommu_iov_to_req(iov
, iov_cnt
, &req
, sizeof(req
));
750 return ret
? ret
: virtio_iommu_probe(s
, &req
, buf
);
753 static void virtio_iommu_handle_command(VirtIODevice
*vdev
, VirtQueue
*vq
)
755 VirtIOIOMMU
*s
= VIRTIO_IOMMU(vdev
);
756 struct virtio_iommu_req_head head
;
757 struct virtio_iommu_req_tail tail
= {};
758 VirtQueueElement
*elem
;
759 unsigned int iov_cnt
;
765 size_t output_size
= sizeof(tail
);
767 elem
= virtqueue_pop(vq
, sizeof(VirtQueueElement
));
772 if (iov_size(elem
->in_sg
, elem
->in_num
) < sizeof(tail
) ||
773 iov_size(elem
->out_sg
, elem
->out_num
) < sizeof(head
)) {
774 virtio_error(vdev
, "virtio-iommu bad head/tail size");
775 virtqueue_detach_element(vq
, elem
, 0);
780 iov_cnt
= elem
->out_num
;
782 sz
= iov_to_buf(iov
, iov_cnt
, 0, &head
, sizeof(head
));
783 if (unlikely(sz
!= sizeof(head
))) {
784 tail
.status
= VIRTIO_IOMMU_S_DEVERR
;
787 qemu_rec_mutex_lock(&s
->mutex
);
789 case VIRTIO_IOMMU_T_ATTACH
:
790 tail
.status
= virtio_iommu_handle_attach(s
, iov
, iov_cnt
);
792 case VIRTIO_IOMMU_T_DETACH
:
793 tail
.status
= virtio_iommu_handle_detach(s
, iov
, iov_cnt
);
795 case VIRTIO_IOMMU_T_MAP
:
796 tail
.status
= virtio_iommu_handle_map(s
, iov
, iov_cnt
);
798 case VIRTIO_IOMMU_T_UNMAP
:
799 tail
.status
= virtio_iommu_handle_unmap(s
, iov
, iov_cnt
);
801 case VIRTIO_IOMMU_T_PROBE
:
803 struct virtio_iommu_req_tail
*ptail
;
805 output_size
= s
->config
.probe_size
+ sizeof(tail
);
806 buf
= g_malloc0(output_size
);
808 ptail
= buf
+ s
->config
.probe_size
;
809 ptail
->status
= virtio_iommu_handle_probe(s
, iov
, iov_cnt
, buf
);
813 tail
.status
= VIRTIO_IOMMU_S_UNSUPP
;
815 qemu_rec_mutex_unlock(&s
->mutex
);
818 sz
= iov_from_buf(elem
->in_sg
, elem
->in_num
, 0,
819 buf
? buf
: &tail
, output_size
);
820 assert(sz
== output_size
);
822 virtqueue_push(vq
, elem
, sz
);
823 virtio_notify(vdev
, vq
);
830 static void virtio_iommu_report_fault(VirtIOIOMMU
*viommu
, uint8_t reason
,
831 int flags
, uint32_t endpoint
,
834 VirtIODevice
*vdev
= &viommu
->parent_obj
;
835 VirtQueue
*vq
= viommu
->event_vq
;
836 struct virtio_iommu_fault fault
;
837 VirtQueueElement
*elem
;
840 memset(&fault
, 0, sizeof(fault
));
841 fault
.reason
= reason
;
842 fault
.flags
= cpu_to_le32(flags
);
843 fault
.endpoint
= cpu_to_le32(endpoint
);
844 fault
.address
= cpu_to_le64(address
);
846 elem
= virtqueue_pop(vq
, sizeof(VirtQueueElement
));
850 "no buffer available in event queue to report event");
854 if (iov_size(elem
->in_sg
, elem
->in_num
) < sizeof(fault
)) {
855 virtio_error(vdev
, "error buffer of wrong size");
856 virtqueue_detach_element(vq
, elem
, 0);
861 sz
= iov_from_buf(elem
->in_sg
, elem
->in_num
, 0,
862 &fault
, sizeof(fault
));
863 assert(sz
== sizeof(fault
));
865 trace_virtio_iommu_report_fault(reason
, flags
, endpoint
, address
);
866 virtqueue_push(vq
, elem
, sz
);
867 virtio_notify(vdev
, vq
);
872 static IOMMUTLBEntry
virtio_iommu_translate(IOMMUMemoryRegion
*mr
, hwaddr addr
,
873 IOMMUAccessFlags flag
,
876 IOMMUDevice
*sdev
= container_of(mr
, IOMMUDevice
, iommu_mr
);
877 VirtIOIOMMUInterval interval
, *mapping_key
;
878 VirtIOIOMMUMapping
*mapping_value
;
879 VirtIOIOMMU
*s
= sdev
->viommu
;
880 bool read_fault
, write_fault
;
881 VirtIOIOMMUEndpoint
*ep
;
889 interval
.high
= addr
+ 1;
890 granule
= ctz64(s
->config
.page_size_mask
);
892 IOMMUTLBEntry entry
= {
893 .target_as
= &address_space_memory
,
895 .translated_addr
= addr
,
896 .addr_mask
= BIT_ULL(granule
) - 1,
900 bypass_allowed
= s
->config
.bypass
;
902 sid
= virtio_iommu_get_bdf(sdev
);
904 trace_virtio_iommu_translate(mr
->parent_obj
.name
, sid
, addr
, flag
);
905 qemu_rec_mutex_lock(&s
->mutex
);
907 ep
= g_tree_lookup(s
->endpoints
, GUINT_TO_POINTER(sid
));
910 assert(ep
&& ep
->domain
&& !ep
->domain
->bypass
);
913 if (!bypass_allowed
) {
914 error_report_once("%s sid=%d is not known!!", __func__
, sid
);
915 virtio_iommu_report_fault(s
, VIRTIO_IOMMU_FAULT_R_UNKNOWN
,
916 VIRTIO_IOMMU_FAULT_F_ADDRESS
,
924 for (l
= sdev
->resv_regions
; l
; l
= l
->next
) {
925 ReservedRegion
*reg
= l
->data
;
927 if (range_contains(®
->range
, addr
)) {
929 case VIRTIO_IOMMU_RESV_MEM_T_MSI
:
932 case VIRTIO_IOMMU_RESV_MEM_T_RESERVED
:
934 virtio_iommu_report_fault(s
, VIRTIO_IOMMU_FAULT_R_MAPPING
,
935 VIRTIO_IOMMU_FAULT_F_ADDRESS
,
944 if (!bypass_allowed
) {
945 error_report_once("%s %02x:%02x.%01x not attached to any domain",
946 __func__
, PCI_BUS_NUM(sid
),
947 PCI_SLOT(sid
), PCI_FUNC(sid
));
948 virtio_iommu_report_fault(s
, VIRTIO_IOMMU_FAULT_R_DOMAIN
,
949 VIRTIO_IOMMU_FAULT_F_ADDRESS
,
955 } else if (ep
->domain
->bypass
) {
960 found
= g_tree_lookup_extended(ep
->domain
->mappings
, (gpointer
)(&interval
),
961 (void **)&mapping_key
,
962 (void **)&mapping_value
);
964 error_report_once("%s no mapping for 0x%"PRIx64
" for sid=%d",
965 __func__
, addr
, sid
);
966 virtio_iommu_report_fault(s
, VIRTIO_IOMMU_FAULT_R_MAPPING
,
967 VIRTIO_IOMMU_FAULT_F_ADDRESS
,
972 read_fault
= (flag
& IOMMU_RO
) &&
973 !(mapping_value
->flags
& VIRTIO_IOMMU_MAP_F_READ
);
974 write_fault
= (flag
& IOMMU_WO
) &&
975 !(mapping_value
->flags
& VIRTIO_IOMMU_MAP_F_WRITE
);
977 flags
= read_fault
? VIRTIO_IOMMU_FAULT_F_READ
: 0;
978 flags
|= write_fault
? VIRTIO_IOMMU_FAULT_F_WRITE
: 0;
980 error_report_once("%s permission error on 0x%"PRIx64
"(%d): allowed=%d",
981 __func__
, addr
, flag
, mapping_value
->flags
);
982 flags
|= VIRTIO_IOMMU_FAULT_F_ADDRESS
;
983 virtio_iommu_report_fault(s
, VIRTIO_IOMMU_FAULT_R_MAPPING
,
984 flags
| VIRTIO_IOMMU_FAULT_F_ADDRESS
,
988 entry
.translated_addr
= addr
- mapping_key
->low
+ mapping_value
->phys_addr
;
990 trace_virtio_iommu_translate_out(addr
, entry
.translated_addr
, sid
);
993 qemu_rec_mutex_unlock(&s
->mutex
);
997 static void virtio_iommu_get_config(VirtIODevice
*vdev
, uint8_t *config_data
)
999 VirtIOIOMMU
*dev
= VIRTIO_IOMMU(vdev
);
1000 struct virtio_iommu_config
*dev_config
= &dev
->config
;
1001 struct virtio_iommu_config
*out_config
= (void *)config_data
;
1003 out_config
->page_size_mask
= cpu_to_le64(dev_config
->page_size_mask
);
1004 out_config
->input_range
.start
= cpu_to_le64(dev_config
->input_range
.start
);
1005 out_config
->input_range
.end
= cpu_to_le64(dev_config
->input_range
.end
);
1006 out_config
->domain_range
.start
= cpu_to_le32(dev_config
->domain_range
.start
);
1007 out_config
->domain_range
.end
= cpu_to_le32(dev_config
->domain_range
.end
);
1008 out_config
->probe_size
= cpu_to_le32(dev_config
->probe_size
);
1009 out_config
->bypass
= dev_config
->bypass
;
1011 trace_virtio_iommu_get_config(dev_config
->page_size_mask
,
1012 dev_config
->input_range
.start
,
1013 dev_config
->input_range
.end
,
1014 dev_config
->domain_range
.start
,
1015 dev_config
->domain_range
.end
,
1016 dev_config
->probe_size
,
1017 dev_config
->bypass
);
1020 static void virtio_iommu_set_config(VirtIODevice
*vdev
,
1021 const uint8_t *config_data
)
1023 VirtIOIOMMU
*dev
= VIRTIO_IOMMU(vdev
);
1024 struct virtio_iommu_config
*dev_config
= &dev
->config
;
1025 const struct virtio_iommu_config
*in_config
= (void *)config_data
;
1027 if (in_config
->bypass
!= dev_config
->bypass
) {
1028 if (!virtio_vdev_has_feature(vdev
, VIRTIO_IOMMU_F_BYPASS_CONFIG
)) {
1029 virtio_error(vdev
, "cannot set config.bypass");
1031 } else if (in_config
->bypass
!= 0 && in_config
->bypass
!= 1) {
1032 virtio_error(vdev
, "invalid config.bypass value '%u'",
1036 dev_config
->bypass
= in_config
->bypass
;
1037 virtio_iommu_switch_address_space_all(dev
);
1040 trace_virtio_iommu_set_config(in_config
->bypass
);
1043 static uint64_t virtio_iommu_get_features(VirtIODevice
*vdev
, uint64_t f
,
1046 VirtIOIOMMU
*dev
= VIRTIO_IOMMU(vdev
);
1049 trace_virtio_iommu_get_features(f
);
1053 static gint
int_cmp(gconstpointer a
, gconstpointer b
, gpointer user_data
)
1055 guint ua
= GPOINTER_TO_UINT(a
);
1056 guint ub
= GPOINTER_TO_UINT(b
);
1057 return (ua
> ub
) - (ua
< ub
);
1060 static gboolean
virtio_iommu_remap(gpointer key
, gpointer value
, gpointer data
)
1062 VirtIOIOMMUMapping
*mapping
= (VirtIOIOMMUMapping
*) value
;
1063 VirtIOIOMMUInterval
*interval
= (VirtIOIOMMUInterval
*) key
;
1064 IOMMUMemoryRegion
*mr
= (IOMMUMemoryRegion
*) data
;
1066 trace_virtio_iommu_remap(mr
->parent_obj
.name
, interval
->low
, interval
->high
,
1067 mapping
->phys_addr
);
1068 virtio_iommu_notify_map(mr
, interval
->low
, interval
->high
,
1069 mapping
->phys_addr
, mapping
->flags
);
1073 static void virtio_iommu_replay(IOMMUMemoryRegion
*mr
, IOMMUNotifier
*n
)
1075 IOMMUDevice
*sdev
= container_of(mr
, IOMMUDevice
, iommu_mr
);
1076 VirtIOIOMMU
*s
= sdev
->viommu
;
1078 VirtIOIOMMUEndpoint
*ep
;
1080 sid
= virtio_iommu_get_bdf(sdev
);
1082 qemu_rec_mutex_lock(&s
->mutex
);
1084 if (!s
->endpoints
) {
1088 ep
= g_tree_lookup(s
->endpoints
, GUINT_TO_POINTER(sid
));
1089 if (!ep
|| !ep
->domain
) {
1093 g_tree_foreach(ep
->domain
->mappings
, virtio_iommu_remap
, mr
);
1096 qemu_rec_mutex_unlock(&s
->mutex
);
1099 static int virtio_iommu_notify_flag_changed(IOMMUMemoryRegion
*iommu_mr
,
1100 IOMMUNotifierFlag old
,
1101 IOMMUNotifierFlag
new,
1104 if (new & IOMMU_NOTIFIER_DEVIOTLB_UNMAP
) {
1105 error_setg(errp
, "Virtio-iommu does not support dev-iotlb yet");
1109 if (old
== IOMMU_NOTIFIER_NONE
) {
1110 trace_virtio_iommu_notify_flag_add(iommu_mr
->parent_obj
.name
);
1111 } else if (new == IOMMU_NOTIFIER_NONE
) {
1112 trace_virtio_iommu_notify_flag_del(iommu_mr
->parent_obj
.name
);
1118 * The default mask (TARGET_PAGE_MASK) is the smallest supported guest granule,
1119 * for example 0xfffffffffffff000. When an assigned device has page size
1120 * restrictions due to the hardware IOMMU configuration, apply this restriction
1123 static int virtio_iommu_set_page_size_mask(IOMMUMemoryRegion
*mr
,
1127 IOMMUDevice
*sdev
= container_of(mr
, IOMMUDevice
, iommu_mr
);
1128 VirtIOIOMMU
*s
= sdev
->viommu
;
1129 uint64_t cur_mask
= s
->config
.page_size_mask
;
1131 trace_virtio_iommu_set_page_size_mask(mr
->parent_obj
.name
, cur_mask
,
1134 if ((cur_mask
& new_mask
) == 0) {
1135 error_setg(errp
, "virtio-iommu %s reports a page size mask 0x%"PRIx64
1136 " incompatible with currently supported mask 0x%"PRIx64
,
1137 mr
->parent_obj
.name
, new_mask
, cur_mask
);
1142 * Once the granule is frozen we can't change the mask anymore. If by
1143 * chance the hotplugged device supports the same granule, we can still
1146 if (s
->granule_frozen
) {
1147 int cur_granule
= ctz64(cur_mask
);
1149 if (!(BIT_ULL(cur_granule
) & new_mask
)) {
1150 error_setg(errp
, "virtio-iommu %s does not support frozen granule 0x%llx",
1151 mr
->parent_obj
.name
, BIT_ULL(cur_granule
));
1157 s
->config
.page_size_mask
&= new_mask
;
1162 * rebuild_resv_regions: rebuild resv regions with both the
1163 * info of host resv ranges and property set resv ranges
1165 static int rebuild_resv_regions(IOMMUDevice
*sdev
)
1170 /* free the existing list and rebuild it from scratch */
1171 g_list_free_full(sdev
->resv_regions
, g_free
);
1172 sdev
->resv_regions
= NULL
;
1174 /* First add host reserved regions if any, all tagged as RESERVED */
1175 for (l
= sdev
->host_resv_ranges
; l
; l
= l
->next
) {
1176 ReservedRegion
*reg
= g_new0(ReservedRegion
, 1);
1177 Range
*r
= (Range
*)l
->data
;
1179 reg
->type
= VIRTIO_IOMMU_RESV_MEM_T_RESERVED
;
1180 range_set_bounds(®
->range
, range_lob(r
), range_upb(r
));
1181 sdev
->resv_regions
= resv_region_list_insert(sdev
->resv_regions
, reg
);
1182 trace_virtio_iommu_host_resv_regions(sdev
->iommu_mr
.parent_obj
.name
, i
,
1183 range_lob(®
->range
),
1184 range_upb(®
->range
));
1188 * then add higher priority reserved regions set by the machine
1189 * through properties
1191 add_prop_resv_regions(sdev
);
1196 * virtio_iommu_set_iova_ranges: Conveys the usable IOVA ranges
1198 * The function turns those into reserved ranges. Once some
1199 * reserved ranges have been set, new reserved regions cannot be
1200 * added outside of the original ones.
1203 * @iova_ranges: list of usable IOVA ranges
1204 * @errp: error handle
1206 static int virtio_iommu_set_iova_ranges(IOMMUMemoryRegion
*mr
,
1210 IOMMUDevice
*sdev
= container_of(mr
, IOMMUDevice
, iommu_mr
);
1211 GList
*current_ranges
= sdev
->host_resv_ranges
;
1212 GList
*l
, *tmp
, *new_ranges
= NULL
;
1215 /* check that each new resv region is included in an existing one */
1216 if (sdev
->host_resv_ranges
) {
1217 range_inverse_array(iova_ranges
,
1221 for (tmp
= new_ranges
; tmp
; tmp
= tmp
->next
) {
1222 Range
*newr
= (Range
*)tmp
->data
;
1223 bool included
= false;
1225 for (l
= current_ranges
; l
; l
= l
->next
) {
1226 Range
* r
= (Range
*)l
->data
;
1228 if (range_contains_range(r
, newr
)) {
1237 /* all new reserved ranges are included in existing ones */
1242 if (sdev
->probe_done
) {
1243 warn_report("%s: Notified about new host reserved regions after probe",
1244 mr
->parent_obj
.name
);
1247 range_inverse_array(iova_ranges
,
1248 &sdev
->host_resv_ranges
,
1250 rebuild_resv_regions(sdev
);
1254 error_setg(errp
, "IOMMU mr=%s Conflicting host reserved ranges set!",
1255 mr
->parent_obj
.name
);
1257 g_list_free_full(new_ranges
, g_free
);
1261 static void virtio_iommu_system_reset(void *opaque
)
1263 VirtIOIOMMU
*s
= opaque
;
1265 trace_virtio_iommu_system_reset();
1268 * config.bypass is sticky across device reset, but should be restored on
1271 s
->config
.bypass
= s
->boot_bypass
;
1272 virtio_iommu_switch_address_space_all(s
);
1276 static void virtio_iommu_freeze_granule(Notifier
*notifier
, void *data
)
1278 VirtIOIOMMU
*s
= container_of(notifier
, VirtIOIOMMU
, machine_done
);
1281 if (likely(s
->config
.bypass
)) {
1283 * Transient IOMMU MR enable to collect page_size_mask requirements
1284 * through memory_region_iommu_set_page_size_mask() called by
1285 * VFIO region_add() callback
1287 s
->config
.bypass
= false;
1288 virtio_iommu_switch_address_space_all(s
);
1289 /* restore default */
1290 s
->config
.bypass
= true;
1291 virtio_iommu_switch_address_space_all(s
);
1293 s
->granule_frozen
= true;
1294 granule
= ctz64(s
->config
.page_size_mask
);
1295 trace_virtio_iommu_freeze_granule(BIT_ULL(granule
));
1298 static void virtio_iommu_device_realize(DeviceState
*dev
, Error
**errp
)
1300 VirtIODevice
*vdev
= VIRTIO_DEVICE(dev
);
1301 VirtIOIOMMU
*s
= VIRTIO_IOMMU(dev
);
1303 virtio_init(vdev
, VIRTIO_ID_IOMMU
, sizeof(struct virtio_iommu_config
));
1305 memset(s
->iommu_pcibus_by_bus_num
, 0, sizeof(s
->iommu_pcibus_by_bus_num
));
1307 s
->req_vq
= virtio_add_queue(vdev
, VIOMMU_DEFAULT_QUEUE_SIZE
,
1308 virtio_iommu_handle_command
);
1309 s
->event_vq
= virtio_add_queue(vdev
, VIOMMU_DEFAULT_QUEUE_SIZE
, NULL
);
1312 * config.bypass is needed to get initial address space early, such as
1315 s
->config
.bypass
= s
->boot_bypass
;
1316 s
->config
.page_size_mask
= qemu_target_page_mask();
1317 s
->config
.input_range
.end
= UINT64_MAX
;
1318 s
->config
.domain_range
.end
= UINT32_MAX
;
1319 s
->config
.probe_size
= VIOMMU_PROBE_SIZE
;
1321 virtio_add_feature(&s
->features
, VIRTIO_RING_F_EVENT_IDX
);
1322 virtio_add_feature(&s
->features
, VIRTIO_RING_F_INDIRECT_DESC
);
1323 virtio_add_feature(&s
->features
, VIRTIO_F_VERSION_1
);
1324 virtio_add_feature(&s
->features
, VIRTIO_IOMMU_F_INPUT_RANGE
);
1325 virtio_add_feature(&s
->features
, VIRTIO_IOMMU_F_DOMAIN_RANGE
);
1326 virtio_add_feature(&s
->features
, VIRTIO_IOMMU_F_MAP_UNMAP
);
1327 virtio_add_feature(&s
->features
, VIRTIO_IOMMU_F_MMIO
);
1328 virtio_add_feature(&s
->features
, VIRTIO_IOMMU_F_PROBE
);
1329 virtio_add_feature(&s
->features
, VIRTIO_IOMMU_F_BYPASS_CONFIG
);
1331 qemu_rec_mutex_init(&s
->mutex
);
1333 s
->as_by_busptr
= g_hash_table_new_full(NULL
, NULL
, NULL
, g_free
);
1335 if (s
->primary_bus
) {
1336 pci_setup_iommu(s
->primary_bus
, &virtio_iommu_ops
, s
);
1338 error_setg(errp
, "VIRTIO-IOMMU is not attached to any PCI bus!");
1341 s
->machine_done
.notify
= virtio_iommu_freeze_granule
;
1342 qemu_add_machine_init_done_notifier(&s
->machine_done
);
1344 qemu_register_reset(virtio_iommu_system_reset
, s
);
1347 static void virtio_iommu_device_unrealize(DeviceState
*dev
)
1349 VirtIODevice
*vdev
= VIRTIO_DEVICE(dev
);
1350 VirtIOIOMMU
*s
= VIRTIO_IOMMU(dev
);
1352 qemu_unregister_reset(virtio_iommu_system_reset
, s
);
1353 qemu_remove_machine_init_done_notifier(&s
->machine_done
);
1355 g_hash_table_destroy(s
->as_by_busptr
);
1357 g_tree_destroy(s
->domains
);
1360 g_tree_destroy(s
->endpoints
);
1363 qemu_rec_mutex_destroy(&s
->mutex
);
1365 virtio_delete_queue(s
->req_vq
);
1366 virtio_delete_queue(s
->event_vq
);
1367 virtio_cleanup(vdev
);
1370 static void virtio_iommu_device_reset(VirtIODevice
*vdev
)
1372 VirtIOIOMMU
*s
= VIRTIO_IOMMU(vdev
);
1374 trace_virtio_iommu_device_reset();
1377 g_tree_destroy(s
->domains
);
1380 g_tree_destroy(s
->endpoints
);
1382 s
->domains
= g_tree_new_full((GCompareDataFunc
)int_cmp
,
1383 NULL
, NULL
, virtio_iommu_put_domain
);
1384 s
->endpoints
= g_tree_new_full((GCompareDataFunc
)int_cmp
,
1385 NULL
, NULL
, virtio_iommu_put_endpoint
);
1388 static void virtio_iommu_set_status(VirtIODevice
*vdev
, uint8_t status
)
1390 trace_virtio_iommu_device_status(status
);
1393 static void virtio_iommu_instance_init(Object
*obj
)
1397 #define VMSTATE_INTERVAL \
1399 .name = "interval", \
1401 .minimum_version_id = 1, \
1402 .fields = (const VMStateField[]) { \
1403 VMSTATE_UINT64(low, VirtIOIOMMUInterval), \
1404 VMSTATE_UINT64(high, VirtIOIOMMUInterval), \
1405 VMSTATE_END_OF_LIST() \
1409 #define VMSTATE_MAPPING \
1411 .name = "mapping", \
1413 .minimum_version_id = 1, \
1414 .fields = (const VMStateField[]) { \
1415 VMSTATE_UINT64(phys_addr, VirtIOIOMMUMapping),\
1416 VMSTATE_UINT32(flags, VirtIOIOMMUMapping), \
1417 VMSTATE_END_OF_LIST() \
1421 static const VMStateDescription vmstate_interval_mapping
[2] = {
1422 VMSTATE_MAPPING
, /* value */
1423 VMSTATE_INTERVAL
/* key */
1426 static int domain_preload(void *opaque
)
1428 VirtIOIOMMUDomain
*domain
= opaque
;
1430 domain
->mappings
= g_tree_new_full((GCompareDataFunc
)interval_cmp
,
1431 NULL
, g_free
, g_free
);
1435 static const VMStateDescription vmstate_endpoint
= {
1438 .minimum_version_id
= 1,
1439 .fields
= (const VMStateField
[]) {
1440 VMSTATE_UINT32(id
, VirtIOIOMMUEndpoint
),
1441 VMSTATE_END_OF_LIST()
1445 static const VMStateDescription vmstate_domain
= {
1448 .minimum_version_id
= 2,
1449 .pre_load
= domain_preload
,
1450 .fields
= (const VMStateField
[]) {
1451 VMSTATE_UINT32(id
, VirtIOIOMMUDomain
),
1452 VMSTATE_GTREE_V(mappings
, VirtIOIOMMUDomain
, 1,
1453 vmstate_interval_mapping
,
1454 VirtIOIOMMUInterval
, VirtIOIOMMUMapping
),
1455 VMSTATE_QLIST_V(endpoint_list
, VirtIOIOMMUDomain
, 1,
1456 vmstate_endpoint
, VirtIOIOMMUEndpoint
, next
),
1457 VMSTATE_BOOL_V(bypass
, VirtIOIOMMUDomain
, 2),
1458 VMSTATE_END_OF_LIST()
1462 static gboolean
reconstruct_endpoints(gpointer key
, gpointer value
,
1465 VirtIOIOMMU
*s
= (VirtIOIOMMU
*)data
;
1466 VirtIOIOMMUDomain
*d
= (VirtIOIOMMUDomain
*)value
;
1467 VirtIOIOMMUEndpoint
*iter
;
1468 IOMMUMemoryRegion
*mr
;
1470 QLIST_FOREACH(iter
, &d
->endpoint_list
, next
) {
1471 mr
= virtio_iommu_mr(s
, iter
->id
);
1475 iter
->iommu_mr
= mr
;
1476 g_tree_insert(s
->endpoints
, GUINT_TO_POINTER(iter
->id
), iter
);
1478 return false; /* continue the domain traversal */
1481 static int iommu_post_load(void *opaque
, int version_id
)
1483 VirtIOIOMMU
*s
= opaque
;
1485 g_tree_foreach(s
->domains
, reconstruct_endpoints
, s
);
1488 * Memory regions are dynamically turned on/off depending on
1489 * 'config.bypass' and attached domain type if there is. After
1490 * migration, we need to make sure the memory regions are
1493 virtio_iommu_switch_address_space_all(s
);
1497 static const VMStateDescription vmstate_virtio_iommu_device
= {
1498 .name
= "virtio-iommu-device",
1499 .minimum_version_id
= 2,
1501 .post_load
= iommu_post_load
,
1502 .fields
= (const VMStateField
[]) {
1503 VMSTATE_GTREE_DIRECT_KEY_V(domains
, VirtIOIOMMU
, 2,
1504 &vmstate_domain
, VirtIOIOMMUDomain
),
1505 VMSTATE_UINT8_V(config
.bypass
, VirtIOIOMMU
, 2),
1506 VMSTATE_END_OF_LIST()
1510 static const VMStateDescription vmstate_virtio_iommu
= {
1511 .name
= "virtio-iommu",
1512 .minimum_version_id
= 2,
1513 .priority
= MIG_PRI_IOMMU
,
1515 .fields
= (const VMStateField
[]) {
1516 VMSTATE_VIRTIO_DEVICE
,
1517 VMSTATE_END_OF_LIST()
1521 static Property virtio_iommu_properties
[] = {
1522 DEFINE_PROP_LINK("primary-bus", VirtIOIOMMU
, primary_bus
,
1523 TYPE_PCI_BUS
, PCIBus
*),
1524 DEFINE_PROP_BOOL("boot-bypass", VirtIOIOMMU
, boot_bypass
, true),
1525 DEFINE_PROP_END_OF_LIST(),
1528 static void virtio_iommu_class_init(ObjectClass
*klass
, void *data
)
1530 DeviceClass
*dc
= DEVICE_CLASS(klass
);
1531 VirtioDeviceClass
*vdc
= VIRTIO_DEVICE_CLASS(klass
);
1533 device_class_set_props(dc
, virtio_iommu_properties
);
1534 dc
->vmsd
= &vmstate_virtio_iommu
;
1536 set_bit(DEVICE_CATEGORY_MISC
, dc
->categories
);
1537 vdc
->realize
= virtio_iommu_device_realize
;
1538 vdc
->unrealize
= virtio_iommu_device_unrealize
;
1539 vdc
->reset
= virtio_iommu_device_reset
;
1540 vdc
->get_config
= virtio_iommu_get_config
;
1541 vdc
->set_config
= virtio_iommu_set_config
;
1542 vdc
->get_features
= virtio_iommu_get_features
;
1543 vdc
->set_status
= virtio_iommu_set_status
;
1544 vdc
->vmsd
= &vmstate_virtio_iommu_device
;
1547 static void virtio_iommu_memory_region_class_init(ObjectClass
*klass
,
1550 IOMMUMemoryRegionClass
*imrc
= IOMMU_MEMORY_REGION_CLASS(klass
);
1552 imrc
->translate
= virtio_iommu_translate
;
1553 imrc
->replay
= virtio_iommu_replay
;
1554 imrc
->notify_flag_changed
= virtio_iommu_notify_flag_changed
;
1555 imrc
->iommu_set_page_size_mask
= virtio_iommu_set_page_size_mask
;
1556 imrc
->iommu_set_iova_ranges
= virtio_iommu_set_iova_ranges
;
1559 static const TypeInfo virtio_iommu_info
= {
1560 .name
= TYPE_VIRTIO_IOMMU
,
1561 .parent
= TYPE_VIRTIO_DEVICE
,
1562 .instance_size
= sizeof(VirtIOIOMMU
),
1563 .instance_init
= virtio_iommu_instance_init
,
1564 .class_init
= virtio_iommu_class_init
,
1567 static const TypeInfo virtio_iommu_memory_region_info
= {
1568 .parent
= TYPE_IOMMU_MEMORY_REGION
,
1569 .name
= TYPE_VIRTIO_IOMMU_MEMORY_REGION
,
1570 .class_init
= virtio_iommu_memory_region_class_init
,
1573 static void virtio_register_types(void)
1575 type_register_static(&virtio_iommu_info
);
1576 type_register_static(&virtio_iommu_memory_region_info
);
1579 type_init(virtio_register_types
)