?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
migration/rdma: Return -1 instead of negative errno code
[qemu/armbru.git] / hw / virtio / virtio-iommu.c
blobbe51635895ce88cb69ec987a5c7d3fa33a703c9e
1 /*
2 * virtio-iommu device
3 *
4 * Copyright (c) 2020 Red Hat, Inc.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2 or later, as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * this program. If not, see <http://www.gnu.org/licenses/>.
17 *
18 */
19
20 #include "qemu/osdep.h"
21 #include "qemu/log.h"
22 #include "qemu/iov.h"
23 #include "exec/target_page.h"
24 #include "hw/qdev-properties.h"
25 #include "hw/virtio/virtio.h"
26 #include "sysemu/kvm.h"
27 #include "sysemu/reset.h"
28 #include "sysemu/sysemu.h"
29 #include "qapi/error.h"
30 #include "qemu/error-report.h"
31 #include "trace.h"
32
33 #include "standard-headers/linux/virtio_ids.h"
34
35 #include "hw/virtio/virtio-bus.h"
36 #include "hw/virtio/virtio-iommu.h"
37 #include "hw/pci/pci_bus.h"
38 #include "hw/pci/pci.h"
39
40 /* Max size */
41 #define VIOMMU_DEFAULT_QUEUE_SIZE 256
42 #define VIOMMU_PROBE_SIZE 512
43
44 typedef struct VirtIOIOMMUDomain {
45 uint32_t id;
46 bool bypass;
47 GTree *mappings;
48 QLIST_HEAD(, VirtIOIOMMUEndpoint) endpoint_list;
49 } VirtIOIOMMUDomain;
50
51 typedef struct VirtIOIOMMUEndpoint {
52 uint32_t id;
53 VirtIOIOMMUDomain *domain;
54 IOMMUMemoryRegion *iommu_mr;
55 QLIST_ENTRY(VirtIOIOMMUEndpoint) next;
56 } VirtIOIOMMUEndpoint;
57
58 typedef struct VirtIOIOMMUInterval {
59 uint64_t low;
60 uint64_t high;
61 } VirtIOIOMMUInterval;
62
63 typedef struct VirtIOIOMMUMapping {
64 uint64_t phys_addr;
65 uint32_t flags;
66 } VirtIOIOMMUMapping;
67
68 static inline uint16_t virtio_iommu_get_bdf(IOMMUDevice *dev)
69 {
70 return PCI_BUILD_BDF(pci_bus_num(dev->bus), dev->devfn);
71 }
72
73 static bool virtio_iommu_device_bypassed(IOMMUDevice *sdev)
74 {
75 uint32_t sid;
76 bool bypassed;
77 VirtIOIOMMU *s = sdev->viommu;
78 VirtIOIOMMUEndpoint *ep;
79
80 sid = virtio_iommu_get_bdf(sdev);
81
82 qemu_rec_mutex_lock(&s->mutex);
83 /* need to check bypass before system reset */
84 if (!s->endpoints) {
85 bypassed = s->config.bypass;
86 goto unlock;
87 }
88
89 ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(sid));
90 if (!ep || !ep->domain) {
91 bypassed = s->config.bypass;
92 } else {
93 bypassed = ep->domain->bypass;
94 }
95
96 unlock:
97 qemu_rec_mutex_unlock(&s->mutex);
98 return bypassed;
99 }
100
101 /* Return whether the device is using IOMMU translation. */
102 static bool virtio_iommu_switch_address_space(IOMMUDevice *sdev)
103 {
104 bool use_remapping;
105
106 assert(sdev);
107
108 use_remapping = !virtio_iommu_device_bypassed(sdev);
109
110 trace_virtio_iommu_switch_address_space(pci_bus_num(sdev->bus),
111 PCI_SLOT(sdev->devfn),
112 PCI_FUNC(sdev->devfn),
113 use_remapping);
114
115 /* Turn off first then on the other */
116 if (use_remapping) {
117 memory_region_set_enabled(&sdev->bypass_mr, false);
118 memory_region_set_enabled(MEMORY_REGION(&sdev->iommu_mr), true);
119 } else {
120 memory_region_set_enabled(MEMORY_REGION(&sdev->iommu_mr), false);
121 memory_region_set_enabled(&sdev->bypass_mr, true);
122 }
123
124 return use_remapping;
125 }
126
127 static void virtio_iommu_switch_address_space_all(VirtIOIOMMU *s)
128 {
129 GHashTableIter iter;
130 IOMMUPciBus *iommu_pci_bus;
131 int i;
132
133 g_hash_table_iter_init(&iter, s->as_by_busptr);
134 while (g_hash_table_iter_next(&iter, NULL, (void **)&iommu_pci_bus)) {
135 for (i = 0; i < PCI_DEVFN_MAX; i++) {
136 if (!iommu_pci_bus->pbdev[i]) {
137 continue;
138 }
139 virtio_iommu_switch_address_space(iommu_pci_bus->pbdev[i]);
140 }
141 }
142 }
143
144 /**
145 * The bus number is used for lookup when SID based operations occur.
146 * In that case we lazily populate the IOMMUPciBus array from the bus hash
147 * table. At the time the IOMMUPciBus is created (iommu_find_add_as), the bus
148 * numbers may not be always initialized yet.
149 */
150 static IOMMUPciBus *iommu_find_iommu_pcibus(VirtIOIOMMU *s, uint8_t bus_num)
151 {
152 IOMMUPciBus *iommu_pci_bus = s->iommu_pcibus_by_bus_num[bus_num];
153
154 if (!iommu_pci_bus) {
155 GHashTableIter iter;
156
157 g_hash_table_iter_init(&iter, s->as_by_busptr);
158 while (g_hash_table_iter_next(&iter, NULL, (void **)&iommu_pci_bus)) {
159 if (pci_bus_num(iommu_pci_bus->bus) == bus_num) {
160 s->iommu_pcibus_by_bus_num[bus_num] = iommu_pci_bus;
161 return iommu_pci_bus;
162 }
163 }
164 return NULL;
165 }
166 return iommu_pci_bus;
167 }
168
169 static IOMMUMemoryRegion *virtio_iommu_mr(VirtIOIOMMU *s, uint32_t sid)
170 {
171 uint8_t bus_n, devfn;
172 IOMMUPciBus *iommu_pci_bus;
173 IOMMUDevice *dev;
174
175 bus_n = PCI_BUS_NUM(sid);
176 iommu_pci_bus = iommu_find_iommu_pcibus(s, bus_n);
177 if (iommu_pci_bus) {
178 devfn = sid & (PCI_DEVFN_MAX - 1);
179 dev = iommu_pci_bus->pbdev[devfn];
180 if (dev) {
181 return &dev->iommu_mr;
182 }
183 }
184 return NULL;
185 }
186
187 static gint interval_cmp(gconstpointer a, gconstpointer b, gpointer user_data)
188 {
189 VirtIOIOMMUInterval *inta = (VirtIOIOMMUInterval *)a;
190 VirtIOIOMMUInterval *intb = (VirtIOIOMMUInterval *)b;
191
192 if (inta->high < intb->low) {
193 return -1;
194 } else if (intb->high < inta->low) {
195 return 1;
196 } else {
197 return 0;
198 }
199 }
200
201 static void virtio_iommu_notify_map_unmap(IOMMUMemoryRegion *mr,
202 IOMMUTLBEvent *event,
203 hwaddr virt_start, hwaddr virt_end)
204 {
205 uint64_t delta = virt_end - virt_start;
206
207 event->entry.iova = virt_start;
208 event->entry.addr_mask = delta;
209
210 if (delta == UINT64_MAX) {
211 memory_region_notify_iommu(mr, 0, *event);
212 }
213
214 while (virt_start != virt_end + 1) {
215 uint64_t mask = dma_aligned_pow2_mask(virt_start, virt_end, 64);
216
217 event->entry.addr_mask = mask;
218 event->entry.iova = virt_start;
219 memory_region_notify_iommu(mr, 0, *event);
220 virt_start += mask + 1;
221 if (event->entry.perm != IOMMU_NONE) {
222 event->entry.translated_addr += mask + 1;
223 }
224 }
225 }
226
227 static void virtio_iommu_notify_map(IOMMUMemoryRegion *mr, hwaddr virt_start,
228 hwaddr virt_end, hwaddr paddr,
229 uint32_t flags)
230 {
231 IOMMUTLBEvent event;
232 IOMMUAccessFlags perm = IOMMU_ACCESS_FLAG(flags & VIRTIO_IOMMU_MAP_F_READ,
233 flags & VIRTIO_IOMMU_MAP_F_WRITE);
234
235 if (!(mr->iommu_notify_flags & IOMMU_NOTIFIER_MAP) ||
236 (flags & VIRTIO_IOMMU_MAP_F_MMIO) || !perm) {
237 return;
238 }
239
240 trace_virtio_iommu_notify_map(mr->parent_obj.name, virt_start, virt_end,
241 paddr, perm);
242
243 event.type = IOMMU_NOTIFIER_MAP;
244 event.entry.target_as = &address_space_memory;
245 event.entry.perm = perm;
246 event.entry.translated_addr = paddr;
247
248 virtio_iommu_notify_map_unmap(mr, &event, virt_start, virt_end);
249 }
250
251 static void virtio_iommu_notify_unmap(IOMMUMemoryRegion *mr, hwaddr virt_start,
252 hwaddr virt_end)
253 {
254 IOMMUTLBEvent event;
255
256 if (!(mr->iommu_notify_flags & IOMMU_NOTIFIER_UNMAP)) {
257 return;
258 }
259
260 trace_virtio_iommu_notify_unmap(mr->parent_obj.name, virt_start, virt_end);
261
262 event.type = IOMMU_NOTIFIER_UNMAP;
263 event.entry.target_as = &address_space_memory;
264 event.entry.perm = IOMMU_NONE;
265 event.entry.translated_addr = 0;
266
267 virtio_iommu_notify_map_unmap(mr, &event, virt_start, virt_end);
268 }
269
270 static gboolean virtio_iommu_notify_unmap_cb(gpointer key, gpointer value,
271 gpointer data)
272 {
273 VirtIOIOMMUInterval *interval = (VirtIOIOMMUInterval *) key;
274 IOMMUMemoryRegion *mr = (IOMMUMemoryRegion *) data;
275
276 virtio_iommu_notify_unmap(mr, interval->low, interval->high);
277
278 return false;
279 }
280
281 static gboolean virtio_iommu_notify_map_cb(gpointer key, gpointer value,
282 gpointer data)
283 {
284 VirtIOIOMMUMapping *mapping = (VirtIOIOMMUMapping *) value;
285 VirtIOIOMMUInterval *interval = (VirtIOIOMMUInterval *) key;
286 IOMMUMemoryRegion *mr = (IOMMUMemoryRegion *) data;
287
288 virtio_iommu_notify_map(mr, interval->low, interval->high,
289 mapping->phys_addr, mapping->flags);
290
291 return false;
292 }
293
294 static void virtio_iommu_detach_endpoint_from_domain(VirtIOIOMMUEndpoint *ep)
295 {
296 VirtIOIOMMUDomain *domain = ep->domain;
297 IOMMUDevice *sdev = container_of(ep->iommu_mr, IOMMUDevice, iommu_mr);
298
299 if (!ep->domain) {
300 return;
301 }
302 g_tree_foreach(domain->mappings, virtio_iommu_notify_unmap_cb,
303 ep->iommu_mr);
304 QLIST_REMOVE(ep, next);
305 ep->domain = NULL;
306 virtio_iommu_switch_address_space(sdev);
307 }
308
309 static VirtIOIOMMUEndpoint *virtio_iommu_get_endpoint(VirtIOIOMMU *s,
310 uint32_t ep_id)
311 {
312 VirtIOIOMMUEndpoint *ep;
313 IOMMUMemoryRegion *mr;
314
315 ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(ep_id));
316 if (ep) {
317 return ep;
318 }
319 mr = virtio_iommu_mr(s, ep_id);
320 if (!mr) {
321 return NULL;
322 }
323 ep = g_malloc0(sizeof(*ep));
324 ep->id = ep_id;
325 ep->iommu_mr = mr;
326 trace_virtio_iommu_get_endpoint(ep_id);
327 g_tree_insert(s->endpoints, GUINT_TO_POINTER(ep_id), ep);
328 return ep;
329 }
330
331 static void virtio_iommu_put_endpoint(gpointer data)
332 {
333 VirtIOIOMMUEndpoint *ep = (VirtIOIOMMUEndpoint *)data;
334
335 if (ep->domain) {
336 virtio_iommu_detach_endpoint_from_domain(ep);
337 }
338
339 trace_virtio_iommu_put_endpoint(ep->id);
340 g_free(ep);
341 }
342
343 static VirtIOIOMMUDomain *virtio_iommu_get_domain(VirtIOIOMMU *s,
344 uint32_t domain_id,
345 bool bypass)
346 {
347 VirtIOIOMMUDomain *domain;
348
349 domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id));
350 if (domain) {
351 if (domain->bypass != bypass) {
352 return NULL;
353 }
354 return domain;
355 }
356 domain = g_malloc0(sizeof(*domain));
357 domain->id = domain_id;
358 domain->mappings = g_tree_new_full((GCompareDataFunc)interval_cmp,
359 NULL, (GDestroyNotify)g_free,
360 (GDestroyNotify)g_free);
361 domain->bypass = bypass;
362 g_tree_insert(s->domains, GUINT_TO_POINTER(domain_id), domain);
363 QLIST_INIT(&domain->endpoint_list);
364 trace_virtio_iommu_get_domain(domain_id);
365 return domain;
366 }
367
368 static void virtio_iommu_put_domain(gpointer data)
369 {
370 VirtIOIOMMUDomain *domain = (VirtIOIOMMUDomain *)data;
371 VirtIOIOMMUEndpoint *iter, *tmp;
372
373 QLIST_FOREACH_SAFE(iter, &domain->endpoint_list, next, tmp) {
374 virtio_iommu_detach_endpoint_from_domain(iter);
375 }
376 g_tree_destroy(domain->mappings);
377 trace_virtio_iommu_put_domain(domain->id);
378 g_free(domain);
379 }
380
381 static AddressSpace *virtio_iommu_find_add_as(PCIBus *bus, void *opaque,
382 int devfn)
383 {
384 VirtIOIOMMU *s = opaque;
385 IOMMUPciBus *sbus = g_hash_table_lookup(s->as_by_busptr, bus);
386 static uint32_t mr_index;
387 IOMMUDevice *sdev;
388
389 if (!sbus) {
390 sbus = g_malloc0(sizeof(IOMMUPciBus) +
391 sizeof(IOMMUDevice *) * PCI_DEVFN_MAX);
392 sbus->bus = bus;
393 g_hash_table_insert(s->as_by_busptr, bus, sbus);
394 }
395
396 sdev = sbus->pbdev[devfn];
397 if (!sdev) {
398 char *name = g_strdup_printf("%s-%d-%d",
399 TYPE_VIRTIO_IOMMU_MEMORY_REGION,
400 mr_index++, devfn);
401 sdev = sbus->pbdev[devfn] = g_new0(IOMMUDevice, 1);
402
403 sdev->viommu = s;
404 sdev->bus = bus;
405 sdev->devfn = devfn;
406
407 trace_virtio_iommu_init_iommu_mr(name);
408
409 memory_region_init(&sdev->root, OBJECT(s), name, UINT64_MAX);
410 address_space_init(&sdev->as, &sdev->root, TYPE_VIRTIO_IOMMU);
411
412 /*
413 * Build the IOMMU disabled container with aliases to the
414 * shared MRs. Note that aliasing to a shared memory region
415 * could help the memory API to detect same FlatViews so we
416 * can have devices to share the same FlatView when in bypass
417 * mode. (either by not configuring virtio-iommu driver or with
418 * "iommu=pt"). It will greatly reduce the total number of
419 * FlatViews of the system hence VM runs faster.
420 */
421 memory_region_init_alias(&sdev->bypass_mr, OBJECT(s),
422 "system", get_system_memory(), 0,
423 memory_region_size(get_system_memory()));
424
425 memory_region_init_iommu(&sdev->iommu_mr, sizeof(sdev->iommu_mr),
426 TYPE_VIRTIO_IOMMU_MEMORY_REGION,
427 OBJECT(s), name,
428 UINT64_MAX);
429
430 /*
431 * Hook both the containers under the root container, we
432 * switch between iommu & bypass MRs by enable/disable
433 * corresponding sub-containers
434 */
435 memory_region_add_subregion_overlap(&sdev->root, 0,
436 MEMORY_REGION(&sdev->iommu_mr),
437 0);
438 memory_region_add_subregion_overlap(&sdev->root, 0,
439 &sdev->bypass_mr, 0);
440
441 virtio_iommu_switch_address_space(sdev);
442 g_free(name);
443 }
444 return &sdev->as;
445 }
446
447 static int virtio_iommu_attach(VirtIOIOMMU *s,
448 struct virtio_iommu_req_attach *req)
449 {
450 uint32_t domain_id = le32_to_cpu(req->domain);
451 uint32_t ep_id = le32_to_cpu(req->endpoint);
452 uint32_t flags = le32_to_cpu(req->flags);
453 VirtIOIOMMUDomain *domain;
454 VirtIOIOMMUEndpoint *ep;
455 IOMMUDevice *sdev;
456
457 trace_virtio_iommu_attach(domain_id, ep_id);
458
459 if (flags & ~VIRTIO_IOMMU_ATTACH_F_BYPASS) {
460 return VIRTIO_IOMMU_S_INVAL;
461 }
462
463 ep = virtio_iommu_get_endpoint(s, ep_id);
464 if (!ep) {
465 return VIRTIO_IOMMU_S_NOENT;
466 }
467
468 if (ep->domain) {
469 VirtIOIOMMUDomain *previous_domain = ep->domain;
470 /*
471 * the device is already attached to a domain,
472 * detach it first
473 */
474 virtio_iommu_detach_endpoint_from_domain(ep);
475 if (QLIST_EMPTY(&previous_domain->endpoint_list)) {
476 g_tree_remove(s->domains, GUINT_TO_POINTER(previous_domain->id));
477 }
478 }
479
480 domain = virtio_iommu_get_domain(s, domain_id,
481 flags & VIRTIO_IOMMU_ATTACH_F_BYPASS);
482 if (!domain) {
483 /* Incompatible bypass flag */
484 return VIRTIO_IOMMU_S_INVAL;
485 }
486 QLIST_INSERT_HEAD(&domain->endpoint_list, ep, next);
487
488 ep->domain = domain;
489 sdev = container_of(ep->iommu_mr, IOMMUDevice, iommu_mr);
490 virtio_iommu_switch_address_space(sdev);
491
492 /* Replay domain mappings on the associated memory region */
493 g_tree_foreach(domain->mappings, virtio_iommu_notify_map_cb,
494 ep->iommu_mr);
495
496 return VIRTIO_IOMMU_S_OK;
497 }
498
499 static int virtio_iommu_detach(VirtIOIOMMU *s,
500 struct virtio_iommu_req_detach *req)
501 {
502 uint32_t domain_id = le32_to_cpu(req->domain);
503 uint32_t ep_id = le32_to_cpu(req->endpoint);
504 VirtIOIOMMUDomain *domain;
505 VirtIOIOMMUEndpoint *ep;
506
507 trace_virtio_iommu_detach(domain_id, ep_id);
508
509 ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(ep_id));
510 if (!ep) {
511 return VIRTIO_IOMMU_S_NOENT;
512 }
513
514 domain = ep->domain;
515
516 if (!domain || domain->id != domain_id) {
517 return VIRTIO_IOMMU_S_INVAL;
518 }
519
520 virtio_iommu_detach_endpoint_from_domain(ep);
521
522 if (QLIST_EMPTY(&domain->endpoint_list)) {
523 g_tree_remove(s->domains, GUINT_TO_POINTER(domain->id));
524 }
525 return VIRTIO_IOMMU_S_OK;
526 }
527
528 static int virtio_iommu_map(VirtIOIOMMU *s,
529 struct virtio_iommu_req_map *req)
530 {
531 uint32_t domain_id = le32_to_cpu(req->domain);
532 uint64_t phys_start = le64_to_cpu(req->phys_start);
533 uint64_t virt_start = le64_to_cpu(req->virt_start);
534 uint64_t virt_end = le64_to_cpu(req->virt_end);
535 uint32_t flags = le32_to_cpu(req->flags);
536 VirtIOIOMMUDomain *domain;
537 VirtIOIOMMUInterval *interval;
538 VirtIOIOMMUMapping *mapping;
539 VirtIOIOMMUEndpoint *ep;
540
541 if (flags & ~VIRTIO_IOMMU_MAP_F_MASK) {
542 return VIRTIO_IOMMU_S_INVAL;
543 }
544
545 domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id));
546 if (!domain) {
547 return VIRTIO_IOMMU_S_NOENT;
548 }
549
550 if (domain->bypass) {
551 return VIRTIO_IOMMU_S_INVAL;
552 }
553
554 interval = g_malloc0(sizeof(*interval));
555
556 interval->low = virt_start;
557 interval->high = virt_end;
558
559 mapping = g_tree_lookup(domain->mappings, (gpointer)interval);
560 if (mapping) {
561 g_free(interval);
562 return VIRTIO_IOMMU_S_INVAL;
563 }
564
565 trace_virtio_iommu_map(domain_id, virt_start, virt_end, phys_start, flags);
566
567 mapping = g_malloc0(sizeof(*mapping));
568 mapping->phys_addr = phys_start;
569 mapping->flags = flags;
570
571 g_tree_insert(domain->mappings, interval, mapping);
572
573 QLIST_FOREACH(ep, &domain->endpoint_list, next) {
574 virtio_iommu_notify_map(ep->iommu_mr, virt_start, virt_end, phys_start,
575 flags);
576 }
577
578 return VIRTIO_IOMMU_S_OK;
579 }
580
581 static int virtio_iommu_unmap(VirtIOIOMMU *s,
582 struct virtio_iommu_req_unmap *req)
583 {
584 uint32_t domain_id = le32_to_cpu(req->domain);
585 uint64_t virt_start = le64_to_cpu(req->virt_start);
586 uint64_t virt_end = le64_to_cpu(req->virt_end);
587 VirtIOIOMMUMapping *iter_val;
588 VirtIOIOMMUInterval interval, *iter_key;
589 VirtIOIOMMUDomain *domain;
590 VirtIOIOMMUEndpoint *ep;
591 int ret = VIRTIO_IOMMU_S_OK;
592
593 trace_virtio_iommu_unmap(domain_id, virt_start, virt_end);
594
595 domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id));
596 if (!domain) {
597 return VIRTIO_IOMMU_S_NOENT;
598 }
599
600 if (domain->bypass) {
601 return VIRTIO_IOMMU_S_INVAL;
602 }
603
604 interval.low = virt_start;
605 interval.high = virt_end;
606
607 while (g_tree_lookup_extended(domain->mappings, &interval,
608 (void **)&iter_key, (void**)&iter_val)) {
609 uint64_t current_low = iter_key->low;
610 uint64_t current_high = iter_key->high;
611
612 if (interval.low <= current_low && interval.high >= current_high) {
613 QLIST_FOREACH(ep, &domain->endpoint_list, next) {
614 virtio_iommu_notify_unmap(ep->iommu_mr, current_low,
615 current_high);
616 }
617 g_tree_remove(domain->mappings, iter_key);
618 trace_virtio_iommu_unmap_done(domain_id, current_low, current_high);
619 } else {
620 ret = VIRTIO_IOMMU_S_RANGE;
621 break;
622 }
623 }
624 return ret;
625 }
626
627 static ssize_t virtio_iommu_fill_resv_mem_prop(VirtIOIOMMU *s, uint32_t ep,
628 uint8_t *buf, size_t free)
629 {
630 struct virtio_iommu_probe_resv_mem prop = {};
631 size_t size = sizeof(prop), length = size - sizeof(prop.head), total;
632 int i;
633
634 total = size * s->nb_reserved_regions;
635
636 if (total > free) {
637 return -ENOSPC;
638 }
639
640 for (i = 0; i < s->nb_reserved_regions; i++) {
641 unsigned subtype = s->reserved_regions[i].type;
642
643 assert(subtype == VIRTIO_IOMMU_RESV_MEM_T_RESERVED ||
644 subtype == VIRTIO_IOMMU_RESV_MEM_T_MSI);
645 prop.head.type = cpu_to_le16(VIRTIO_IOMMU_PROBE_T_RESV_MEM);
646 prop.head.length = cpu_to_le16(length);
647 prop.subtype = subtype;
648 prop.start = cpu_to_le64(s->reserved_regions[i].low);
649 prop.end = cpu_to_le64(s->reserved_regions[i].high);
650
651 memcpy(buf, &prop, size);
652
653 trace_virtio_iommu_fill_resv_property(ep, prop.subtype,
654 prop.start, prop.end);
655 buf += size;
656 }
657 return total;
658 }
659
660 /**
661 * virtio_iommu_probe - Fill the probe request buffer with
662 * the properties the device is able to return
663 */
664 static int virtio_iommu_probe(VirtIOIOMMU *s,
665 struct virtio_iommu_req_probe *req,
666 uint8_t *buf)
667 {
668 uint32_t ep_id = le32_to_cpu(req->endpoint);
669 size_t free = VIOMMU_PROBE_SIZE;
670 ssize_t count;
671
672 if (!virtio_iommu_mr(s, ep_id)) {
673 return VIRTIO_IOMMU_S_NOENT;
674 }
675
676 count = virtio_iommu_fill_resv_mem_prop(s, ep_id, buf, free);
677 if (count < 0) {
678 return VIRTIO_IOMMU_S_INVAL;
679 }
680 buf += count;
681 free -= count;
682
683 return VIRTIO_IOMMU_S_OK;
684 }
685
686 static int virtio_iommu_iov_to_req(struct iovec *iov,
687 unsigned int iov_cnt,
688 void *req, size_t payload_sz)
689 {
690 size_t sz = iov_to_buf(iov, iov_cnt, 0, req, payload_sz);
691
692 if (unlikely(sz != payload_sz)) {
693 return VIRTIO_IOMMU_S_INVAL;
694 }
695 return 0;
696 }
697
698 #define virtio_iommu_handle_req(__req) \
699 static int virtio_iommu_handle_ ## __req(VirtIOIOMMU *s, \
700 struct iovec *iov, \
701 unsigned int iov_cnt) \
702 { \
703 struct virtio_iommu_req_ ## __req req; \
704 int ret = virtio_iommu_iov_to_req(iov, iov_cnt, &req, \
705 sizeof(req) - sizeof(struct virtio_iommu_req_tail));\
706 \
707 return ret ? ret : virtio_iommu_ ## __req(s, &req); \
708 }
709
710 virtio_iommu_handle_req(attach)
711 virtio_iommu_handle_req(detach)
712 virtio_iommu_handle_req(map)
713 virtio_iommu_handle_req(unmap)
714
715 static int virtio_iommu_handle_probe(VirtIOIOMMU *s,
716 struct iovec *iov,
717 unsigned int iov_cnt,
718 uint8_t *buf)
719 {
720 struct virtio_iommu_req_probe req;
721 int ret = virtio_iommu_iov_to_req(iov, iov_cnt, &req, sizeof(req));
722
723 return ret ? ret : virtio_iommu_probe(s, &req, buf);
724 }
725
726 static void virtio_iommu_handle_command(VirtIODevice *vdev, VirtQueue *vq)
727 {
728 VirtIOIOMMU *s = VIRTIO_IOMMU(vdev);
729 struct virtio_iommu_req_head head;
730 struct virtio_iommu_req_tail tail = {};
731 VirtQueueElement *elem;
732 unsigned int iov_cnt;
733 struct iovec *iov;
734 void *buf = NULL;
735 size_t sz;
736
737 for (;;) {
738 size_t output_size = sizeof(tail);
739
740 elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
741 if (!elem) {
742 return;
743 }
744
745 if (iov_size(elem->in_sg, elem->in_num) < sizeof(tail) ||
746 iov_size(elem->out_sg, elem->out_num) < sizeof(head)) {
747 virtio_error(vdev, "virtio-iommu bad head/tail size");
748 virtqueue_detach_element(vq, elem, 0);
749 g_free(elem);
750 break;
751 }
752
753 iov_cnt = elem->out_num;
754 iov = elem->out_sg;
755 sz = iov_to_buf(iov, iov_cnt, 0, &head, sizeof(head));
756 if (unlikely(sz != sizeof(head))) {
757 tail.status = VIRTIO_IOMMU_S_DEVERR;
758 goto out;
759 }
760 qemu_rec_mutex_lock(&s->mutex);
761 switch (head.type) {
762 case VIRTIO_IOMMU_T_ATTACH:
763 tail.status = virtio_iommu_handle_attach(s, iov, iov_cnt);
764 break;
765 case VIRTIO_IOMMU_T_DETACH:
766 tail.status = virtio_iommu_handle_detach(s, iov, iov_cnt);
767 break;
768 case VIRTIO_IOMMU_T_MAP:
769 tail.status = virtio_iommu_handle_map(s, iov, iov_cnt);
770 break;
771 case VIRTIO_IOMMU_T_UNMAP:
772 tail.status = virtio_iommu_handle_unmap(s, iov, iov_cnt);
773 break;
774 case VIRTIO_IOMMU_T_PROBE:
775 {
776 struct virtio_iommu_req_tail *ptail;
777
778 output_size = s->config.probe_size + sizeof(tail);
779 buf = g_malloc0(output_size);
780
781 ptail = buf + s->config.probe_size;
782 ptail->status = virtio_iommu_handle_probe(s, iov, iov_cnt, buf);
783 break;
784 }
785 default:
786 tail.status = VIRTIO_IOMMU_S_UNSUPP;
787 }
788 qemu_rec_mutex_unlock(&s->mutex);
789
790 out:
791 sz = iov_from_buf(elem->in_sg, elem->in_num, 0,
792 buf ? buf : &tail, output_size);
793 assert(sz == output_size);
794
795 virtqueue_push(vq, elem, sz);
796 virtio_notify(vdev, vq);
797 g_free(elem);
798 g_free(buf);
799 buf = NULL;
800 }
801 }
802
803 static void virtio_iommu_report_fault(VirtIOIOMMU *viommu, uint8_t reason,
804 int flags, uint32_t endpoint,
805 uint64_t address)
806 {
807 VirtIODevice *vdev = &viommu->parent_obj;
808 VirtQueue *vq = viommu->event_vq;
809 struct virtio_iommu_fault fault;
810 VirtQueueElement *elem;
811 size_t sz;
812
813 memset(&fault, 0, sizeof(fault));
814 fault.reason = reason;
815 fault.flags = cpu_to_le32(flags);
816 fault.endpoint = cpu_to_le32(endpoint);
817 fault.address = cpu_to_le64(address);
818
819 elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
820
821 if (!elem) {
822 error_report_once(
823 "no buffer available in event queue to report event");
824 return;
825 }
826
827 if (iov_size(elem->in_sg, elem->in_num) < sizeof(fault)) {
828 virtio_error(vdev, "error buffer of wrong size");
829 virtqueue_detach_element(vq, elem, 0);
830 g_free(elem);
831 return;
832 }
833
834 sz = iov_from_buf(elem->in_sg, elem->in_num, 0,
835 &fault, sizeof(fault));
836 assert(sz == sizeof(fault));
837
838 trace_virtio_iommu_report_fault(reason, flags, endpoint, address);
839 virtqueue_push(vq, elem, sz);
840 virtio_notify(vdev, vq);
841 g_free(elem);
842
843 }
844
845 static IOMMUTLBEntry virtio_iommu_translate(IOMMUMemoryRegion *mr, hwaddr addr,
846 IOMMUAccessFlags flag,
847 int iommu_idx)
848 {
849 IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr);
850 VirtIOIOMMUInterval interval, *mapping_key;
851 VirtIOIOMMUMapping *mapping_value;
852 VirtIOIOMMU *s = sdev->viommu;
853 bool read_fault, write_fault;
854 VirtIOIOMMUEndpoint *ep;
855 uint32_t sid, flags;
856 bool bypass_allowed;
857 int granule;
858 bool found;
859 int i;
860
861 interval.low = addr;
862 interval.high = addr + 1;
863 granule = ctz64(s->config.page_size_mask);
864
865 IOMMUTLBEntry entry = {
866 .target_as = &address_space_memory,
867 .iova = addr,
868 .translated_addr = addr,
869 .addr_mask = BIT_ULL(granule) - 1,
870 .perm = IOMMU_NONE,
871 };
872
873 bypass_allowed = s->config.bypass;
874
875 sid = virtio_iommu_get_bdf(sdev);
876
877 trace_virtio_iommu_translate(mr->parent_obj.name, sid, addr, flag);
878 qemu_rec_mutex_lock(&s->mutex);
879
880 ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(sid));
881
882 if (bypass_allowed)
883 assert(ep && ep->domain && !ep->domain->bypass);
884
885 if (!ep) {
886 if (!bypass_allowed) {
887 error_report_once("%s sid=%d is not known!!", __func__, sid);
888 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_UNKNOWN,
889 VIRTIO_IOMMU_FAULT_F_ADDRESS,
890 sid, addr);
891 } else {
892 entry.perm = flag;
893 }
894 goto unlock;
895 }
896
897 for (i = 0; i < s->nb_reserved_regions; i++) {
898 ReservedRegion *reg = &s->reserved_regions[i];
899
900 if (addr >= reg->low && addr <= reg->high) {
901 switch (reg->type) {
902 case VIRTIO_IOMMU_RESV_MEM_T_MSI:
903 entry.perm = flag;
904 break;
905 case VIRTIO_IOMMU_RESV_MEM_T_RESERVED:
906 default:
907 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING,
908 VIRTIO_IOMMU_FAULT_F_ADDRESS,
909 sid, addr);
910 break;
911 }
912 goto unlock;
913 }
914 }
915
916 if (!ep->domain) {
917 if (!bypass_allowed) {
918 error_report_once("%s %02x:%02x.%01x not attached to any domain",
919 __func__, PCI_BUS_NUM(sid),
920 PCI_SLOT(sid), PCI_FUNC(sid));
921 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_DOMAIN,
922 VIRTIO_IOMMU_FAULT_F_ADDRESS,
923 sid, addr);
924 } else {
925 entry.perm = flag;
926 }
927 goto unlock;
928 } else if (ep->domain->bypass) {
929 entry.perm = flag;
930 goto unlock;
931 }
932
933 found = g_tree_lookup_extended(ep->domain->mappings, (gpointer)(&interval),
934 (void **)&mapping_key,
935 (void **)&mapping_value);
936 if (!found) {
937 error_report_once("%s no mapping for 0x%"PRIx64" for sid=%d",
938 __func__, addr, sid);
939 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING,
940 VIRTIO_IOMMU_FAULT_F_ADDRESS,
941 sid, addr);
942 goto unlock;
943 }
944
945 read_fault = (flag & IOMMU_RO) &&
946 !(mapping_value->flags & VIRTIO_IOMMU_MAP_F_READ);
947 write_fault = (flag & IOMMU_WO) &&
948 !(mapping_value->flags & VIRTIO_IOMMU_MAP_F_WRITE);
949
950 flags = read_fault ? VIRTIO_IOMMU_FAULT_F_READ : 0;
951 flags |= write_fault ? VIRTIO_IOMMU_FAULT_F_WRITE : 0;
952 if (flags) {
953 error_report_once("%s permission error on 0x%"PRIx64"(%d): allowed=%d",
954 __func__, addr, flag, mapping_value->flags);
955 flags |= VIRTIO_IOMMU_FAULT_F_ADDRESS;
956 virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING,
957 flags | VIRTIO_IOMMU_FAULT_F_ADDRESS,
958 sid, addr);
959 goto unlock;
960 }
961 entry.translated_addr = addr - mapping_key->low + mapping_value->phys_addr;
962 entry.perm = flag;
963 trace_virtio_iommu_translate_out(addr, entry.translated_addr, sid);
964
965 unlock:
966 qemu_rec_mutex_unlock(&s->mutex);
967 return entry;
968 }
969
970 static void virtio_iommu_get_config(VirtIODevice *vdev, uint8_t *config_data)
971 {
972 VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev);
973 struct virtio_iommu_config *dev_config = &dev->config;
974 struct virtio_iommu_config *out_config = (void *)config_data;
975
976 out_config->page_size_mask = cpu_to_le64(dev_config->page_size_mask);
977 out_config->input_range.start = cpu_to_le64(dev_config->input_range.start);
978 out_config->input_range.end = cpu_to_le64(dev_config->input_range.end);
979 out_config->domain_range.start = cpu_to_le32(dev_config->domain_range.start);
980 out_config->domain_range.end = cpu_to_le32(dev_config->domain_range.end);
981 out_config->probe_size = cpu_to_le32(dev_config->probe_size);
982 out_config->bypass = dev_config->bypass;
983
984 trace_virtio_iommu_get_config(dev_config->page_size_mask,
985 dev_config->input_range.start,
986 dev_config->input_range.end,
987 dev_config->domain_range.start,
988 dev_config->domain_range.end,
989 dev_config->probe_size,
990 dev_config->bypass);
991 }
992
993 static void virtio_iommu_set_config(VirtIODevice *vdev,
994 const uint8_t *config_data)
995 {
996 VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev);
997 struct virtio_iommu_config *dev_config = &dev->config;
998 const struct virtio_iommu_config *in_config = (void *)config_data;
999
1000 if (in_config->bypass != dev_config->bypass) {
1001 if (!virtio_vdev_has_feature(vdev, VIRTIO_IOMMU_F_BYPASS_CONFIG)) {
1002 virtio_error(vdev, "cannot set config.bypass");
1003 return;
1004 } else if (in_config->bypass != 0 && in_config->bypass != 1) {
1005 virtio_error(vdev, "invalid config.bypass value '%u'",
1006 in_config->bypass);
1007 return;
1008 }
1009 dev_config->bypass = in_config->bypass;
1010 virtio_iommu_switch_address_space_all(dev);
1011 }
1012
1013 trace_virtio_iommu_set_config(in_config->bypass);
1014 }
1015
1016 static uint64_t virtio_iommu_get_features(VirtIODevice *vdev, uint64_t f,
1017 Error **errp)
1018 {
1019 VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev);
1020
1021 f |= dev->features;
1022 trace_virtio_iommu_get_features(f);
1023 return f;
1024 }
1025
1026 static gint int_cmp(gconstpointer a, gconstpointer b, gpointer user_data)
1027 {
1028 guint ua = GPOINTER_TO_UINT(a);
1029 guint ub = GPOINTER_TO_UINT(b);
1030 return (ua > ub) - (ua < ub);
1031 }
1032
1033 static gboolean virtio_iommu_remap(gpointer key, gpointer value, gpointer data)
1034 {
1035 VirtIOIOMMUMapping *mapping = (VirtIOIOMMUMapping *) value;
1036 VirtIOIOMMUInterval *interval = (VirtIOIOMMUInterval *) key;
1037 IOMMUMemoryRegion *mr = (IOMMUMemoryRegion *) data;
1038
1039 trace_virtio_iommu_remap(mr->parent_obj.name, interval->low, interval->high,
1040 mapping->phys_addr);
1041 virtio_iommu_notify_map(mr, interval->low, interval->high,
1042 mapping->phys_addr, mapping->flags);
1043 return false;
1044 }
1045
1046 static void virtio_iommu_replay(IOMMUMemoryRegion *mr, IOMMUNotifier *n)
1047 {
1048 IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr);
1049 VirtIOIOMMU *s = sdev->viommu;
1050 uint32_t sid;
1051 VirtIOIOMMUEndpoint *ep;
1052
1053 sid = virtio_iommu_get_bdf(sdev);
1054
1055 qemu_rec_mutex_lock(&s->mutex);
1056
1057 if (!s->endpoints) {
1058 goto unlock;
1059 }
1060
1061 ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(sid));
1062 if (!ep || !ep->domain) {
1063 goto unlock;
1064 }
1065
1066 g_tree_foreach(ep->domain->mappings, virtio_iommu_remap, mr);
1067
1068 unlock:
1069 qemu_rec_mutex_unlock(&s->mutex);
1070 }
1071
1072 static int virtio_iommu_notify_flag_changed(IOMMUMemoryRegion *iommu_mr,
1073 IOMMUNotifierFlag old,
1074 IOMMUNotifierFlag new,
1075 Error **errp)
1076 {
1077 if (new & IOMMU_NOTIFIER_DEVIOTLB_UNMAP) {
1078 error_setg(errp, "Virtio-iommu does not support dev-iotlb yet");
1079 return -EINVAL;
1080 }
1081
1082 if (old == IOMMU_NOTIFIER_NONE) {
1083 trace_virtio_iommu_notify_flag_add(iommu_mr->parent_obj.name);
1084 } else if (new == IOMMU_NOTIFIER_NONE) {
1085 trace_virtio_iommu_notify_flag_del(iommu_mr->parent_obj.name);
1086 }
1087 return 0;
1088 }
1089
1090 /*
1091 * The default mask (TARGET_PAGE_MASK) is the smallest supported guest granule,
1092 * for example 0xfffffffffffff000. When an assigned device has page size
1093 * restrictions due to the hardware IOMMU configuration, apply this restriction
1094 * to the mask.
1095 */
1096 static int virtio_iommu_set_page_size_mask(IOMMUMemoryRegion *mr,
1097 uint64_t new_mask,
1098 Error **errp)
1099 {
1100 IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr);
1101 VirtIOIOMMU *s = sdev->viommu;
1102 uint64_t cur_mask = s->config.page_size_mask;
1103
1104 trace_virtio_iommu_set_page_size_mask(mr->parent_obj.name, cur_mask,
1105 new_mask);
1106
1107 if ((cur_mask & new_mask) == 0) {
1108 error_setg(errp, "virtio-iommu %s reports a page size mask 0x%"PRIx64
1109 " incompatible with currently supported mask 0x%"PRIx64,
1110 mr->parent_obj.name, new_mask, cur_mask);
1111 return -1;
1112 }
1113
1114 /*
1115 * Once the granule is frozen we can't change the mask anymore. If by
1116 * chance the hotplugged device supports the same granule, we can still
1117 * accept it.
1118 */
1119 if (s->granule_frozen) {
1120 int cur_granule = ctz64(cur_mask);
1121
1122 if (!(BIT_ULL(cur_granule) & new_mask)) {
1123 error_setg(errp, "virtio-iommu %s does not support frozen granule 0x%llx",
1124 mr->parent_obj.name, BIT_ULL(cur_granule));
1125 return -1;
1126 }
1127 return 0;
1128 }
1129
1130 s->config.page_size_mask &= new_mask;
1131 return 0;
1132 }
1133
1134 static void virtio_iommu_system_reset(void *opaque)
1135 {
1136 VirtIOIOMMU *s = opaque;
1137
1138 trace_virtio_iommu_system_reset();
1139
1140 /*
1141 * config.bypass is sticky across device reset, but should be restored on
1142 * system reset
1143 */
1144 s->config.bypass = s->boot_bypass;
1145 virtio_iommu_switch_address_space_all(s);
1146
1147 }
1148
1149 static void virtio_iommu_freeze_granule(Notifier *notifier, void *data)
1150 {
1151 VirtIOIOMMU *s = container_of(notifier, VirtIOIOMMU, machine_done);
1152 int granule;
1153
1154 if (likely(s->config.bypass)) {
1155 /*
1156 * Transient IOMMU MR enable to collect page_size_mask requirements
1157 * through memory_region_iommu_set_page_size_mask() called by
1158 * VFIO region_add() callback
1159 */
1160 s->config.bypass = false;
1161 virtio_iommu_switch_address_space_all(s);
1162 /* restore default */
1163 s->config.bypass = true;
1164 virtio_iommu_switch_address_space_all(s);
1165 }
1166 s->granule_frozen = true;
1167 granule = ctz64(s->config.page_size_mask);
1168 trace_virtio_iommu_freeze_granule(BIT_ULL(granule));
1169 }
1170
1171 static void virtio_iommu_device_realize(DeviceState *dev, Error **errp)
1172 {
1173 VirtIODevice *vdev = VIRTIO_DEVICE(dev);
1174 VirtIOIOMMU *s = VIRTIO_IOMMU(dev);
1175
1176 virtio_init(vdev, VIRTIO_ID_IOMMU, sizeof(struct virtio_iommu_config));
1177
1178 memset(s->iommu_pcibus_by_bus_num, 0, sizeof(s->iommu_pcibus_by_bus_num));
1179
1180 s->req_vq = virtio_add_queue(vdev, VIOMMU_DEFAULT_QUEUE_SIZE,
1181 virtio_iommu_handle_command);
1182 s->event_vq = virtio_add_queue(vdev, VIOMMU_DEFAULT_QUEUE_SIZE, NULL);
1183
1184 /*
1185 * config.bypass is needed to get initial address space early, such as
1186 * in vfio realize
1187 */
1188 s->config.bypass = s->boot_bypass;
1189 s->config.page_size_mask = qemu_target_page_mask();
1190 s->config.input_range.end = UINT64_MAX;
1191 s->config.domain_range.end = UINT32_MAX;
1192 s->config.probe_size = VIOMMU_PROBE_SIZE;
1193
1194 virtio_add_feature(&s->features, VIRTIO_RING_F_EVENT_IDX);
1195 virtio_add_feature(&s->features, VIRTIO_RING_F_INDIRECT_DESC);
1196 virtio_add_feature(&s->features, VIRTIO_F_VERSION_1);
1197 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_INPUT_RANGE);
1198 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_DOMAIN_RANGE);
1199 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_MAP_UNMAP);
1200 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_MMIO);
1201 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_PROBE);
1202 virtio_add_feature(&s->features, VIRTIO_IOMMU_F_BYPASS_CONFIG);
1203
1204 qemu_rec_mutex_init(&s->mutex);
1205
1206 s->as_by_busptr = g_hash_table_new_full(NULL, NULL, NULL, g_free);
1207
1208 if (s->primary_bus) {
1209 pci_setup_iommu(s->primary_bus, virtio_iommu_find_add_as, s);
1210 } else {
1211 error_setg(errp, "VIRTIO-IOMMU is not attached to any PCI bus!");
1212 }
1213
1214 s->machine_done.notify = virtio_iommu_freeze_granule;
1215 qemu_add_machine_init_done_notifier(&s->machine_done);
1216
1217 qemu_register_reset(virtio_iommu_system_reset, s);
1218 }
1219
1220 static void virtio_iommu_device_unrealize(DeviceState *dev)
1221 {
1222 VirtIODevice *vdev = VIRTIO_DEVICE(dev);
1223 VirtIOIOMMU *s = VIRTIO_IOMMU(dev);
1224
1225 qemu_unregister_reset(virtio_iommu_system_reset, s);
1226 qemu_remove_machine_init_done_notifier(&s->machine_done);
1227
1228 g_hash_table_destroy(s->as_by_busptr);
1229 if (s->domains) {
1230 g_tree_destroy(s->domains);
1231 }
1232 if (s->endpoints) {
1233 g_tree_destroy(s->endpoints);
1234 }
1235
1236 qemu_rec_mutex_destroy(&s->mutex);
1237
1238 virtio_delete_queue(s->req_vq);
1239 virtio_delete_queue(s->event_vq);
1240 virtio_cleanup(vdev);
1241 }
1242
1243 static void virtio_iommu_device_reset(VirtIODevice *vdev)
1244 {
1245 VirtIOIOMMU *s = VIRTIO_IOMMU(vdev);
1246
1247 trace_virtio_iommu_device_reset();
1248
1249 if (s->domains) {
1250 g_tree_destroy(s->domains);
1251 }
1252 if (s->endpoints) {
1253 g_tree_destroy(s->endpoints);
1254 }
1255 s->domains = g_tree_new_full((GCompareDataFunc)int_cmp,
1256 NULL, NULL, virtio_iommu_put_domain);
1257 s->endpoints = g_tree_new_full((GCompareDataFunc)int_cmp,
1258 NULL, NULL, virtio_iommu_put_endpoint);
1259 }
1260
1261 static void virtio_iommu_set_status(VirtIODevice *vdev, uint8_t status)
1262 {
1263 trace_virtio_iommu_device_status(status);
1264 }
1265
1266 static void virtio_iommu_instance_init(Object *obj)
1267 {
1268 }
1269
1270 #define VMSTATE_INTERVAL \
1271 { \
1272 .name = "interval", \
1273 .version_id = 1, \
1274 .minimum_version_id = 1, \
1275 .fields = (VMStateField[]) { \
1276 VMSTATE_UINT64(low, VirtIOIOMMUInterval), \
1277 VMSTATE_UINT64(high, VirtIOIOMMUInterval), \
1278 VMSTATE_END_OF_LIST() \
1279 } \
1280 }
1281
1282 #define VMSTATE_MAPPING \
1283 { \
1284 .name = "mapping", \
1285 .version_id = 1, \
1286 .minimum_version_id = 1, \
1287 .fields = (VMStateField[]) { \
1288 VMSTATE_UINT64(phys_addr, VirtIOIOMMUMapping),\
1289 VMSTATE_UINT32(flags, VirtIOIOMMUMapping), \
1290 VMSTATE_END_OF_LIST() \
1291 }, \
1292 }
1293
1294 static const VMStateDescription vmstate_interval_mapping[2] = {
1295 VMSTATE_MAPPING, /* value */
1296 VMSTATE_INTERVAL /* key */
1297 };
1298
1299 static int domain_preload(void *opaque)
1300 {
1301 VirtIOIOMMUDomain *domain = opaque;
1302
1303 domain->mappings = g_tree_new_full((GCompareDataFunc)interval_cmp,
1304 NULL, g_free, g_free);
1305 return 0;
1306 }
1307
1308 static const VMStateDescription vmstate_endpoint = {
1309 .name = "endpoint",
1310 .version_id = 1,
1311 .minimum_version_id = 1,
1312 .fields = (VMStateField[]) {
1313 VMSTATE_UINT32(id, VirtIOIOMMUEndpoint),
1314 VMSTATE_END_OF_LIST()
1315 }
1316 };
1317
1318 static const VMStateDescription vmstate_domain = {
1319 .name = "domain",
1320 .version_id = 2,
1321 .minimum_version_id = 2,
1322 .pre_load = domain_preload,
1323 .fields = (VMStateField[]) {
1324 VMSTATE_UINT32(id, VirtIOIOMMUDomain),
1325 VMSTATE_GTREE_V(mappings, VirtIOIOMMUDomain, 1,
1326 vmstate_interval_mapping,
1327 VirtIOIOMMUInterval, VirtIOIOMMUMapping),
1328 VMSTATE_QLIST_V(endpoint_list, VirtIOIOMMUDomain, 1,
1329 vmstate_endpoint, VirtIOIOMMUEndpoint, next),
1330 VMSTATE_BOOL_V(bypass, VirtIOIOMMUDomain, 2),
1331 VMSTATE_END_OF_LIST()
1332 }
1333 };
1334
1335 static gboolean reconstruct_endpoints(gpointer key, gpointer value,
1336 gpointer data)
1337 {
1338 VirtIOIOMMU *s = (VirtIOIOMMU *)data;
1339 VirtIOIOMMUDomain *d = (VirtIOIOMMUDomain *)value;
1340 VirtIOIOMMUEndpoint *iter;
1341 IOMMUMemoryRegion *mr;
1342
1343 QLIST_FOREACH(iter, &d->endpoint_list, next) {
1344 mr = virtio_iommu_mr(s, iter->id);
1345 assert(mr);
1346
1347 iter->domain = d;
1348 iter->iommu_mr = mr;
1349 g_tree_insert(s->endpoints, GUINT_TO_POINTER(iter->id), iter);
1350 }
1351 return false; /* continue the domain traversal */
1352 }
1353
1354 static int iommu_post_load(void *opaque, int version_id)
1355 {
1356 VirtIOIOMMU *s = opaque;
1357
1358 g_tree_foreach(s->domains, reconstruct_endpoints, s);
1359
1360 /*
1361 * Memory regions are dynamically turned on/off depending on
1362 * 'config.bypass' and attached domain type if there is. After
1363 * migration, we need to make sure the memory regions are
1364 * still correct.
1365 */
1366 virtio_iommu_switch_address_space_all(s);
1367 return 0;
1368 }
1369
1370 static const VMStateDescription vmstate_virtio_iommu_device = {
1371 .name = "virtio-iommu-device",
1372 .minimum_version_id = 2,
1373 .version_id = 2,
1374 .post_load = iommu_post_load,
1375 .fields = (VMStateField[]) {
1376 VMSTATE_GTREE_DIRECT_KEY_V(domains, VirtIOIOMMU, 2,
1377 &vmstate_domain, VirtIOIOMMUDomain),
1378 VMSTATE_UINT8_V(config.bypass, VirtIOIOMMU, 2),
1379 VMSTATE_END_OF_LIST()
1380 },
1381 };
1382
1383 static const VMStateDescription vmstate_virtio_iommu = {
1384 .name = "virtio-iommu",
1385 .minimum_version_id = 2,
1386 .priority = MIG_PRI_IOMMU,
1387 .version_id = 2,
1388 .fields = (VMStateField[]) {
1389 VMSTATE_VIRTIO_DEVICE,
1390 VMSTATE_END_OF_LIST()
1391 },
1392 };
1393
1394 static Property virtio_iommu_properties[] = {
1395 DEFINE_PROP_LINK("primary-bus", VirtIOIOMMU, primary_bus,
1396 TYPE_PCI_BUS, PCIBus *),
1397 DEFINE_PROP_BOOL("boot-bypass", VirtIOIOMMU, boot_bypass, true),
1398 DEFINE_PROP_END_OF_LIST(),
1399 };
1400
1401 static void virtio_iommu_class_init(ObjectClass *klass, void *data)
1402 {
1403 DeviceClass *dc = DEVICE_CLASS(klass);
1404 VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
1405
1406 device_class_set_props(dc, virtio_iommu_properties);
1407 dc->vmsd = &vmstate_virtio_iommu;
1408
1409 set_bit(DEVICE_CATEGORY_MISC, dc->categories);
1410 vdc->realize = virtio_iommu_device_realize;
1411 vdc->unrealize = virtio_iommu_device_unrealize;
1412 vdc->reset = virtio_iommu_device_reset;
1413 vdc->get_config = virtio_iommu_get_config;
1414 vdc->set_config = virtio_iommu_set_config;
1415 vdc->get_features = virtio_iommu_get_features;
1416 vdc->set_status = virtio_iommu_set_status;
1417 vdc->vmsd = &vmstate_virtio_iommu_device;
1418 }
1419
1420 static void virtio_iommu_memory_region_class_init(ObjectClass *klass,
1421 void *data)
1422 {
1423 IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass);
1424
1425 imrc->translate = virtio_iommu_translate;
1426 imrc->replay = virtio_iommu_replay;
1427 imrc->notify_flag_changed = virtio_iommu_notify_flag_changed;
1428 imrc->iommu_set_page_size_mask = virtio_iommu_set_page_size_mask;
1429 }
1430
1431 static const TypeInfo virtio_iommu_info = {
1432 .name = TYPE_VIRTIO_IOMMU,
1433 .parent = TYPE_VIRTIO_DEVICE,
1434 .instance_size = sizeof(VirtIOIOMMU),
1435 .instance_init = virtio_iommu_instance_init,
1436 .class_init = virtio_iommu_class_init,
1437 };
1438
1439 static const TypeInfo virtio_iommu_memory_region_info = {
1440 .parent = TYPE_IOMMU_MEMORY_REGION,
1441 .name = TYPE_VIRTIO_IOMMU_MEMORY_REGION,
1442 .class_init = virtio_iommu_memory_region_class_init,
1443 };
1444
1445 static void virtio_register_types(void)
1446 {
1447 type_register_static(&virtio_iommu_info);
1448 type_register_static(&virtio_iommu_memory_region_info);
1449 }
1450
1451 type_init(virtio_register_types)
armbru's QEMU hackery