?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
vfio/migration: Change vIOMMU blocker from global to per device
[qemu/armbru.git] / hw / vfio / common.c
blob9aac21abb76ef7d1abb54428e9a173a33ce16073
1 /*
2 * generic functions used by VFIO devices
3 *
4 * Copyright Red Hat, Inc. 2012
5 *
6 * Authors:
7 * Alex Williamson <alex.williamson@redhat.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
11 *
12 * Based on qemu-kvm device-assignment:
13 * Adapted for KVM by Qumranet.
14 * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com)
15 * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com)
16 * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com)
17 * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
18 * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
19 */
20
21 #include "qemu/osdep.h"
22 #include <sys/ioctl.h>
23 #ifdef CONFIG_KVM
24 #include <linux/kvm.h>
25 #endif
26 #include <linux/vfio.h>
27
28 #include "hw/vfio/vfio-common.h"
29 #include "hw/vfio/vfio.h"
30 #include "exec/address-spaces.h"
31 #include "exec/memory.h"
32 #include "exec/ram_addr.h"
33 #include "hw/hw.h"
34 #include "qemu/error-report.h"
35 #include "qemu/main-loop.h"
36 #include "qemu/range.h"
37 #include "sysemu/kvm.h"
38 #include "sysemu/reset.h"
39 #include "sysemu/runstate.h"
40 #include "trace.h"
41 #include "qapi/error.h"
42 #include "migration/migration.h"
43 #include "migration/misc.h"
44 #include "migration/blocker.h"
45 #include "migration/qemu-file.h"
46 #include "sysemu/tpm.h"
47
48 VFIOGroupList vfio_group_list =
49 QLIST_HEAD_INITIALIZER(vfio_group_list);
50 static QLIST_HEAD(, VFIOAddressSpace) vfio_address_spaces =
51 QLIST_HEAD_INITIALIZER(vfio_address_spaces);
52
53 #ifdef CONFIG_KVM
54 /*
55 * We have a single VFIO pseudo device per KVM VM. Once created it lives
56 * for the life of the VM. Closing the file descriptor only drops our
57 * reference to it and the device's reference to kvm. Therefore once
58 * initialized, this file descriptor is only released on QEMU exit and
59 * we'll re-use it should another vfio device be attached before then.
60 */
61 static int vfio_kvm_device_fd = -1;
62 #endif
63
64 /*
65 * Common VFIO interrupt disable
66 */
67 void vfio_disable_irqindex(VFIODevice *vbasedev, int index)
68 {
69 struct vfio_irq_set irq_set = {
70 .argsz = sizeof(irq_set),
71 .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER,
72 .index = index,
73 .start = 0,
74 .count = 0,
75 };
76
77 ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, &irq_set);
78 }
79
80 void vfio_unmask_single_irqindex(VFIODevice *vbasedev, int index)
81 {
82 struct vfio_irq_set irq_set = {
83 .argsz = sizeof(irq_set),
84 .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_UNMASK,
85 .index = index,
86 .start = 0,
87 .count = 1,
88 };
89
90 ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, &irq_set);
91 }
92
93 void vfio_mask_single_irqindex(VFIODevice *vbasedev, int index)
94 {
95 struct vfio_irq_set irq_set = {
96 .argsz = sizeof(irq_set),
97 .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_MASK,
98 .index = index,
99 .start = 0,
100 .count = 1,
101 };
102
103 ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, &irq_set);
104 }
105
106 static inline const char *action_to_str(int action)
107 {
108 switch (action) {
109 case VFIO_IRQ_SET_ACTION_MASK:
110 return "MASK";
111 case VFIO_IRQ_SET_ACTION_UNMASK:
112 return "UNMASK";
113 case VFIO_IRQ_SET_ACTION_TRIGGER:
114 return "TRIGGER";
115 default:
116 return "UNKNOWN ACTION";
117 }
118 }
119
120 static const char *index_to_str(VFIODevice *vbasedev, int index)
121 {
122 if (vbasedev->type != VFIO_DEVICE_TYPE_PCI) {
123 return NULL;
124 }
125
126 switch (index) {
127 case VFIO_PCI_INTX_IRQ_INDEX:
128 return "INTX";
129 case VFIO_PCI_MSI_IRQ_INDEX:
130 return "MSI";
131 case VFIO_PCI_MSIX_IRQ_INDEX:
132 return "MSIX";
133 case VFIO_PCI_ERR_IRQ_INDEX:
134 return "ERR";
135 case VFIO_PCI_REQ_IRQ_INDEX:
136 return "REQ";
137 default:
138 return NULL;
139 }
140 }
141
142 static int vfio_ram_block_discard_disable(VFIOContainer *container, bool state)
143 {
144 switch (container->iommu_type) {
145 case VFIO_TYPE1v2_IOMMU:
146 case VFIO_TYPE1_IOMMU:
147 /*
148 * We support coordinated discarding of RAM via the RamDiscardManager.
149 */
150 return ram_block_uncoordinated_discard_disable(state);
151 default:
152 /*
153 * VFIO_SPAPR_TCE_IOMMU most probably works just fine with
154 * RamDiscardManager, however, it is completely untested.
155 *
156 * VFIO_SPAPR_TCE_v2_IOMMU with "DMA memory preregistering" does
157 * completely the opposite of managing mapping/pinning dynamically as
158 * required by RamDiscardManager. We would have to special-case sections
159 * with a RamDiscardManager.
160 */
161 return ram_block_discard_disable(state);
162 }
163 }
164
165 int vfio_set_irq_signaling(VFIODevice *vbasedev, int index, int subindex,
166 int action, int fd, Error **errp)
167 {
168 struct vfio_irq_set *irq_set;
169 int argsz, ret = 0;
170 const char *name;
171 int32_t *pfd;
172
173 argsz = sizeof(*irq_set) + sizeof(*pfd);
174
175 irq_set = g_malloc0(argsz);
176 irq_set->argsz = argsz;
177 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | action;
178 irq_set->index = index;
179 irq_set->start = subindex;
180 irq_set->count = 1;
181 pfd = (int32_t *)&irq_set->data;
182 *pfd = fd;
183
184 if (ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, irq_set)) {
185 ret = -errno;
186 }
187 g_free(irq_set);
188
189 if (!ret) {
190 return 0;
191 }
192
193 error_setg_errno(errp, -ret, "VFIO_DEVICE_SET_IRQS failure");
194
195 name = index_to_str(vbasedev, index);
196 if (name) {
197 error_prepend(errp, "%s-%d: ", name, subindex);
198 } else {
199 error_prepend(errp, "index %d-%d: ", index, subindex);
200 }
201 error_prepend(errp,
202 "Failed to %s %s eventfd signaling for interrupt ",
203 fd < 0 ? "tear down" : "set up", action_to_str(action));
204 return ret;
205 }
206
207 /*
208 * IO Port/MMIO - Beware of the endians, VFIO is always little endian
209 */
210 void vfio_region_write(void *opaque, hwaddr addr,
211 uint64_t data, unsigned size)
212 {
213 VFIORegion *region = opaque;
214 VFIODevice *vbasedev = region->vbasedev;
215 union {
216 uint8_t byte;
217 uint16_t word;
218 uint32_t dword;
219 uint64_t qword;
220 } buf;
221
222 switch (size) {
223 case 1:
224 buf.byte = data;
225 break;
226 case 2:
227 buf.word = cpu_to_le16(data);
228 break;
229 case 4:
230 buf.dword = cpu_to_le32(data);
231 break;
232 case 8:
233 buf.qword = cpu_to_le64(data);
234 break;
235 default:
236 hw_error("vfio: unsupported write size, %u bytes", size);
237 break;
238 }
239
240 if (pwrite(vbasedev->fd, &buf, size, region->fd_offset + addr) != size) {
241 error_report("%s(%s:region%d+0x%"HWADDR_PRIx", 0x%"PRIx64
242 ",%d) failed: %m",
243 __func__, vbasedev->name, region->nr,
244 addr, data, size);
245 }
246
247 trace_vfio_region_write(vbasedev->name, region->nr, addr, data, size);
248
249 /*
250 * A read or write to a BAR always signals an INTx EOI. This will
251 * do nothing if not pending (including not in INTx mode). We assume
252 * that a BAR access is in response to an interrupt and that BAR
253 * accesses will service the interrupt. Unfortunately, we don't know
254 * which access will service the interrupt, so we're potentially
255 * getting quite a few host interrupts per guest interrupt.
256 */
257 vbasedev->ops->vfio_eoi(vbasedev);
258 }
259
260 uint64_t vfio_region_read(void *opaque,
261 hwaddr addr, unsigned size)
262 {
263 VFIORegion *region = opaque;
264 VFIODevice *vbasedev = region->vbasedev;
265 union {
266 uint8_t byte;
267 uint16_t word;
268 uint32_t dword;
269 uint64_t qword;
270 } buf;
271 uint64_t data = 0;
272
273 if (pread(vbasedev->fd, &buf, size, region->fd_offset + addr) != size) {
274 error_report("%s(%s:region%d+0x%"HWADDR_PRIx", %d) failed: %m",
275 __func__, vbasedev->name, region->nr,
276 addr, size);
277 return (uint64_t)-1;
278 }
279 switch (size) {
280 case 1:
281 data = buf.byte;
282 break;
283 case 2:
284 data = le16_to_cpu(buf.word);
285 break;
286 case 4:
287 data = le32_to_cpu(buf.dword);
288 break;
289 case 8:
290 data = le64_to_cpu(buf.qword);
291 break;
292 default:
293 hw_error("vfio: unsupported read size, %u bytes", size);
294 break;
295 }
296
297 trace_vfio_region_read(vbasedev->name, region->nr, addr, size, data);
298
299 /* Same as write above */
300 vbasedev->ops->vfio_eoi(vbasedev);
301
302 return data;
303 }
304
305 const MemoryRegionOps vfio_region_ops = {
306 .read = vfio_region_read,
307 .write = vfio_region_write,
308 .endianness = DEVICE_LITTLE_ENDIAN,
309 .valid = {
310 .min_access_size = 1,
311 .max_access_size = 8,
312 },
313 .impl = {
314 .min_access_size = 1,
315 .max_access_size = 8,
316 },
317 };
318
319 /*
320 * Device state interfaces
321 */
322
323 typedef struct {
324 unsigned long *bitmap;
325 hwaddr size;
326 hwaddr pages;
327 } VFIOBitmap;
328
329 static int vfio_bitmap_alloc(VFIOBitmap *vbmap, hwaddr size)
330 {
331 vbmap->pages = REAL_HOST_PAGE_ALIGN(size) / qemu_real_host_page_size();
332 vbmap->size = ROUND_UP(vbmap->pages, sizeof(__u64) * BITS_PER_BYTE) /
333 BITS_PER_BYTE;
334 vbmap->bitmap = g_try_malloc0(vbmap->size);
335 if (!vbmap->bitmap) {
336 return -ENOMEM;
337 }
338
339 return 0;
340 }
341
342 static int vfio_get_dirty_bitmap(VFIOContainer *container, uint64_t iova,
343 uint64_t size, ram_addr_t ram_addr);
344
345 bool vfio_mig_active(void)
346 {
347 VFIOGroup *group;
348 VFIODevice *vbasedev;
349
350 if (QLIST_EMPTY(&vfio_group_list)) {
351 return false;
352 }
353
354 QLIST_FOREACH(group, &vfio_group_list, next) {
355 QLIST_FOREACH(vbasedev, &group->device_list, next) {
356 if (vbasedev->migration_blocker) {
357 return false;
358 }
359 }
360 }
361 return true;
362 }
363
364 static Error *multiple_devices_migration_blocker;
365
366 static unsigned int vfio_migratable_device_num(void)
367 {
368 VFIOGroup *group;
369 VFIODevice *vbasedev;
370 unsigned int device_num = 0;
371
372 QLIST_FOREACH(group, &vfio_group_list, next) {
373 QLIST_FOREACH(vbasedev, &group->device_list, next) {
374 if (vbasedev->migration) {
375 device_num++;
376 }
377 }
378 }
379
380 return device_num;
381 }
382
383 int vfio_block_multiple_devices_migration(VFIODevice *vbasedev, Error **errp)
384 {
385 int ret;
386
387 if (multiple_devices_migration_blocker ||
388 vfio_migratable_device_num() <= 1) {
389 return 0;
390 }
391
392 if (vbasedev->enable_migration == ON_OFF_AUTO_ON) {
393 error_setg(errp, "Migration is currently not supported with multiple "
394 "VFIO devices");
395 return -EINVAL;
396 }
397
398 error_setg(&multiple_devices_migration_blocker,
399 "Migration is currently not supported with multiple "
400 "VFIO devices");
401 ret = migrate_add_blocker(multiple_devices_migration_blocker, errp);
402 if (ret < 0) {
403 error_free(multiple_devices_migration_blocker);
404 multiple_devices_migration_blocker = NULL;
405 }
406
407 return ret;
408 }
409
410 void vfio_unblock_multiple_devices_migration(void)
411 {
412 if (!multiple_devices_migration_blocker ||
413 vfio_migratable_device_num() > 1) {
414 return;
415 }
416
417 migrate_del_blocker(multiple_devices_migration_blocker);
418 error_free(multiple_devices_migration_blocker);
419 multiple_devices_migration_blocker = NULL;
420 }
421
422 bool vfio_viommu_preset(VFIODevice *vbasedev)
423 {
424 return vbasedev->group->container->space->as != &address_space_memory;
425 }
426
427 static void vfio_set_migration_error(int err)
428 {
429 MigrationState *ms = migrate_get_current();
430
431 if (migration_is_setup_or_active(ms->state)) {
432 WITH_QEMU_LOCK_GUARD(&ms->qemu_file_lock) {
433 if (ms->to_dst_file) {
434 qemu_file_set_error(ms->to_dst_file, err);
435 }
436 }
437 }
438 }
439
440 static bool vfio_devices_all_dirty_tracking(VFIOContainer *container)
441 {
442 VFIOGroup *group;
443 VFIODevice *vbasedev;
444 MigrationState *ms = migrate_get_current();
445
446 if (ms->state != MIGRATION_STATUS_ACTIVE &&
447 ms->state != MIGRATION_STATUS_DEVICE) {
448 return false;
449 }
450
451 QLIST_FOREACH(group, &container->group_list, container_next) {
452 QLIST_FOREACH(vbasedev, &group->device_list, next) {
453 VFIOMigration *migration = vbasedev->migration;
454
455 if (!migration) {
456 return false;
457 }
458
459 if (vbasedev->pre_copy_dirty_page_tracking == ON_OFF_AUTO_OFF &&
460 (migration->device_state == VFIO_DEVICE_STATE_RUNNING ||
461 migration->device_state == VFIO_DEVICE_STATE_PRE_COPY)) {
462 return false;
463 }
464 }
465 }
466 return true;
467 }
468
469 static bool vfio_devices_all_device_dirty_tracking(VFIOContainer *container)
470 {
471 VFIOGroup *group;
472 VFIODevice *vbasedev;
473
474 QLIST_FOREACH(group, &container->group_list, container_next) {
475 QLIST_FOREACH(vbasedev, &group->device_list, next) {
476 if (!vbasedev->dirty_pages_supported) {
477 return false;
478 }
479 }
480 }
481
482 return true;
483 }
484
485 /*
486 * Check if all VFIO devices are running and migration is active, which is
487 * essentially equivalent to the migration being in pre-copy phase.
488 */
489 static bool vfio_devices_all_running_and_mig_active(VFIOContainer *container)
490 {
491 VFIOGroup *group;
492 VFIODevice *vbasedev;
493
494 if (!migration_is_active(migrate_get_current())) {
495 return false;
496 }
497
498 QLIST_FOREACH(group, &container->group_list, container_next) {
499 QLIST_FOREACH(vbasedev, &group->device_list, next) {
500 VFIOMigration *migration = vbasedev->migration;
501
502 if (!migration) {
503 return false;
504 }
505
506 if (migration->device_state == VFIO_DEVICE_STATE_RUNNING ||
507 migration->device_state == VFIO_DEVICE_STATE_PRE_COPY) {
508 continue;
509 } else {
510 return false;
511 }
512 }
513 }
514 return true;
515 }
516
517 static int vfio_dma_unmap_bitmap(VFIOContainer *container,
518 hwaddr iova, ram_addr_t size,
519 IOMMUTLBEntry *iotlb)
520 {
521 struct vfio_iommu_type1_dma_unmap *unmap;
522 struct vfio_bitmap *bitmap;
523 VFIOBitmap vbmap;
524 int ret;
525
526 ret = vfio_bitmap_alloc(&vbmap, size);
527 if (ret) {
528 return ret;
529 }
530
531 unmap = g_malloc0(sizeof(*unmap) + sizeof(*bitmap));
532
533 unmap->argsz = sizeof(*unmap) + sizeof(*bitmap);
534 unmap->iova = iova;
535 unmap->size = size;
536 unmap->flags |= VFIO_DMA_UNMAP_FLAG_GET_DIRTY_BITMAP;
537 bitmap = (struct vfio_bitmap *)&unmap->data;
538
539 /*
540 * cpu_physical_memory_set_dirty_lebitmap() supports pages in bitmap of
541 * qemu_real_host_page_size to mark those dirty. Hence set bitmap_pgsize
542 * to qemu_real_host_page_size.
543 */
544 bitmap->pgsize = qemu_real_host_page_size();
545 bitmap->size = vbmap.size;
546 bitmap->data = (__u64 *)vbmap.bitmap;
547
548 if (vbmap.size > container->max_dirty_bitmap_size) {
549 error_report("UNMAP: Size of bitmap too big 0x%"PRIx64, vbmap.size);
550 ret = -E2BIG;
551 goto unmap_exit;
552 }
553
554 ret = ioctl(container->fd, VFIO_IOMMU_UNMAP_DMA, unmap);
555 if (!ret) {
556 cpu_physical_memory_set_dirty_lebitmap(vbmap.bitmap,
557 iotlb->translated_addr, vbmap.pages);
558 } else {
559 error_report("VFIO_UNMAP_DMA with DIRTY_BITMAP : %m");
560 }
561
562 unmap_exit:
563 g_free(unmap);
564 g_free(vbmap.bitmap);
565
566 return ret;
567 }
568
569 /*
570 * DMA - Mapping and unmapping for the "type1" IOMMU interface used on x86
571 */
572 static int vfio_dma_unmap(VFIOContainer *container,
573 hwaddr iova, ram_addr_t size,
574 IOMMUTLBEntry *iotlb)
575 {
576 struct vfio_iommu_type1_dma_unmap unmap = {
577 .argsz = sizeof(unmap),
578 .flags = 0,
579 .iova = iova,
580 .size = size,
581 };
582 bool need_dirty_sync = false;
583 int ret;
584
585 if (iotlb && vfio_devices_all_running_and_mig_active(container)) {
586 if (!vfio_devices_all_device_dirty_tracking(container) &&
587 container->dirty_pages_supported) {
588 return vfio_dma_unmap_bitmap(container, iova, size, iotlb);
589 }
590
591 need_dirty_sync = true;
592 }
593
594 while (ioctl(container->fd, VFIO_IOMMU_UNMAP_DMA, &unmap)) {
595 /*
596 * The type1 backend has an off-by-one bug in the kernel (71a7d3d78e3c
597 * v4.15) where an overflow in its wrap-around check prevents us from
598 * unmapping the last page of the address space. Test for the error
599 * condition and re-try the unmap excluding the last page. The
600 * expectation is that we've never mapped the last page anyway and this
601 * unmap request comes via vIOMMU support which also makes it unlikely
602 * that this page is used. This bug was introduced well after type1 v2
603 * support was introduced, so we shouldn't need to test for v1. A fix
604 * is queued for kernel v5.0 so this workaround can be removed once
605 * affected kernels are sufficiently deprecated.
606 */
607 if (errno == EINVAL && unmap.size && !(unmap.iova + unmap.size) &&
608 container->iommu_type == VFIO_TYPE1v2_IOMMU) {
609 trace_vfio_dma_unmap_overflow_workaround();
610 unmap.size -= 1ULL << ctz64(container->pgsizes);
611 continue;
612 }
613 error_report("VFIO_UNMAP_DMA failed: %s", strerror(errno));
614 return -errno;
615 }
616
617 if (need_dirty_sync) {
618 ret = vfio_get_dirty_bitmap(container, iova, size,
619 iotlb->translated_addr);
620 if (ret) {
621 return ret;
622 }
623 }
624
625 return 0;
626 }
627
628 static int vfio_dma_map(VFIOContainer *container, hwaddr iova,
629 ram_addr_t size, void *vaddr, bool readonly)
630 {
631 struct vfio_iommu_type1_dma_map map = {
632 .argsz = sizeof(map),
633 .flags = VFIO_DMA_MAP_FLAG_READ,
634 .vaddr = (__u64)(uintptr_t)vaddr,
635 .iova = iova,
636 .size = size,
637 };
638
639 if (!readonly) {
640 map.flags |= VFIO_DMA_MAP_FLAG_WRITE;
641 }
642
643 /*
644 * Try the mapping, if it fails with EBUSY, unmap the region and try
645 * again. This shouldn't be necessary, but we sometimes see it in
646 * the VGA ROM space.
647 */
648 if (ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0 ||
649 (errno == EBUSY && vfio_dma_unmap(container, iova, size, NULL) == 0 &&
650 ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0)) {
651 return 0;
652 }
653
654 error_report("VFIO_MAP_DMA failed: %s", strerror(errno));
655 return -errno;
656 }
657
658 static void vfio_host_win_add(VFIOContainer *container,
659 hwaddr min_iova, hwaddr max_iova,
660 uint64_t iova_pgsizes)
661 {
662 VFIOHostDMAWindow *hostwin;
663
664 QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) {
665 if (ranges_overlap(hostwin->min_iova,
666 hostwin->max_iova - hostwin->min_iova + 1,
667 min_iova,
668 max_iova - min_iova + 1)) {
669 hw_error("%s: Overlapped IOMMU are not enabled", __func__);
670 }
671 }
672
673 hostwin = g_malloc0(sizeof(*hostwin));
674
675 hostwin->min_iova = min_iova;
676 hostwin->max_iova = max_iova;
677 hostwin->iova_pgsizes = iova_pgsizes;
678 QLIST_INSERT_HEAD(&container->hostwin_list, hostwin, hostwin_next);
679 }
680
681 static int vfio_host_win_del(VFIOContainer *container, hwaddr min_iova,
682 hwaddr max_iova)
683 {
684 VFIOHostDMAWindow *hostwin;
685
686 QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) {
687 if (hostwin->min_iova == min_iova && hostwin->max_iova == max_iova) {
688 QLIST_REMOVE(hostwin, hostwin_next);
689 g_free(hostwin);
690 return 0;
691 }
692 }
693
694 return -1;
695 }
696
697 static bool vfio_listener_skipped_section(MemoryRegionSection *section)
698 {
699 return (!memory_region_is_ram(section->mr) &&
700 !memory_region_is_iommu(section->mr)) ||
701 memory_region_is_protected(section->mr) ||
702 /*
703 * Sizing an enabled 64-bit BAR can cause spurious mappings to
704 * addresses in the upper part of the 64-bit address space. These
705 * are never accessed by the CPU and beyond the address width of
706 * some IOMMU hardware. TODO: VFIO should tell us the IOMMU width.
707 */
708 section->offset_within_address_space & (1ULL << 63);
709 }
710
711 /* Called with rcu_read_lock held. */
712 static bool vfio_get_xlat_addr(IOMMUTLBEntry *iotlb, void **vaddr,
713 ram_addr_t *ram_addr, bool *read_only)
714 {
715 bool ret, mr_has_discard_manager;
716
717 ret = memory_get_xlat_addr(iotlb, vaddr, ram_addr, read_only,
718 &mr_has_discard_manager);
719 if (ret && mr_has_discard_manager) {
720 /*
721 * Malicious VMs might trigger discarding of IOMMU-mapped memory. The
722 * pages will remain pinned inside vfio until unmapped, resulting in a
723 * higher memory consumption than expected. If memory would get
724 * populated again later, there would be an inconsistency between pages
725 * pinned by vfio and pages seen by QEMU. This is the case until
726 * unmapped from the IOMMU (e.g., during device reset).
727 *
728 * With malicious guests, we really only care about pinning more memory
729 * than expected. RLIMIT_MEMLOCK set for the user/process can never be
730 * exceeded and can be used to mitigate this problem.
731 */
732 warn_report_once("Using vfio with vIOMMUs and coordinated discarding of"
733 " RAM (e.g., virtio-mem) works, however, malicious"
734 " guests can trigger pinning of more memory than"
735 " intended via an IOMMU. It's possible to mitigate "
736 " by setting/adjusting RLIMIT_MEMLOCK.");
737 }
738 return ret;
739 }
740
741 static void vfio_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
742 {
743 VFIOGuestIOMMU *giommu = container_of(n, VFIOGuestIOMMU, n);
744 VFIOContainer *container = giommu->container;
745 hwaddr iova = iotlb->iova + giommu->iommu_offset;
746 void *vaddr;
747 int ret;
748
749 trace_vfio_iommu_map_notify(iotlb->perm == IOMMU_NONE ? "UNMAP" : "MAP",
750 iova, iova + iotlb->addr_mask);
751
752 if (iotlb->target_as != &address_space_memory) {
753 error_report("Wrong target AS \"%s\", only system memory is allowed",
754 iotlb->target_as->name ? iotlb->target_as->name : "none");
755 vfio_set_migration_error(-EINVAL);
756 return;
757 }
758
759 rcu_read_lock();
760
761 if ((iotlb->perm & IOMMU_RW) != IOMMU_NONE) {
762 bool read_only;
763
764 if (!vfio_get_xlat_addr(iotlb, &vaddr, NULL, &read_only)) {
765 goto out;
766 }
767 /*
768 * vaddr is only valid until rcu_read_unlock(). But after
769 * vfio_dma_map has set up the mapping the pages will be
770 * pinned by the kernel. This makes sure that the RAM backend
771 * of vaddr will always be there, even if the memory object is
772 * destroyed and its backing memory munmap-ed.
773 */
774 ret = vfio_dma_map(container, iova,
775 iotlb->addr_mask + 1, vaddr,
776 read_only);
777 if (ret) {
778 error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx", "
779 "0x%"HWADDR_PRIx", %p) = %d (%s)",
780 container, iova,
781 iotlb->addr_mask + 1, vaddr, ret, strerror(-ret));
782 }
783 } else {
784 ret = vfio_dma_unmap(container, iova, iotlb->addr_mask + 1, iotlb);
785 if (ret) {
786 error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", "
787 "0x%"HWADDR_PRIx") = %d (%s)",
788 container, iova,
789 iotlb->addr_mask + 1, ret, strerror(-ret));
790 vfio_set_migration_error(ret);
791 }
792 }
793 out:
794 rcu_read_unlock();
795 }
796
797 static void vfio_ram_discard_notify_discard(RamDiscardListener *rdl,
798 MemoryRegionSection *section)
799 {
800 VFIORamDiscardListener *vrdl = container_of(rdl, VFIORamDiscardListener,
801 listener);
802 const hwaddr size = int128_get64(section->size);
803 const hwaddr iova = section->offset_within_address_space;
804 int ret;
805
806 /* Unmap with a single call. */
807 ret = vfio_dma_unmap(vrdl->container, iova, size , NULL);
808 if (ret) {
809 error_report("%s: vfio_dma_unmap() failed: %s", __func__,
810 strerror(-ret));
811 }
812 }
813
814 static int vfio_ram_discard_notify_populate(RamDiscardListener *rdl,
815 MemoryRegionSection *section)
816 {
817 VFIORamDiscardListener *vrdl = container_of(rdl, VFIORamDiscardListener,
818 listener);
819 const hwaddr end = section->offset_within_region +
820 int128_get64(section->size);
821 hwaddr start, next, iova;
822 void *vaddr;
823 int ret;
824
825 /*
826 * Map in (aligned within memory region) minimum granularity, so we can
827 * unmap in minimum granularity later.
828 */
829 for (start = section->offset_within_region; start < end; start = next) {
830 next = ROUND_UP(start + 1, vrdl->granularity);
831 next = MIN(next, end);
832
833 iova = start - section->offset_within_region +
834 section->offset_within_address_space;
835 vaddr = memory_region_get_ram_ptr(section->mr) + start;
836
837 ret = vfio_dma_map(vrdl->container, iova, next - start,
838 vaddr, section->readonly);
839 if (ret) {
840 /* Rollback */
841 vfio_ram_discard_notify_discard(rdl, section);
842 return ret;
843 }
844 }
845 return 0;
846 }
847
848 static void vfio_register_ram_discard_listener(VFIOContainer *container,
849 MemoryRegionSection *section)
850 {
851 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr);
852 VFIORamDiscardListener *vrdl;
853
854 /* Ignore some corner cases not relevant in practice. */
855 g_assert(QEMU_IS_ALIGNED(section->offset_within_region, TARGET_PAGE_SIZE));
856 g_assert(QEMU_IS_ALIGNED(section->offset_within_address_space,
857 TARGET_PAGE_SIZE));
858 g_assert(QEMU_IS_ALIGNED(int128_get64(section->size), TARGET_PAGE_SIZE));
859
860 vrdl = g_new0(VFIORamDiscardListener, 1);
861 vrdl->container = container;
862 vrdl->mr = section->mr;
863 vrdl->offset_within_address_space = section->offset_within_address_space;
864 vrdl->size = int128_get64(section->size);
865 vrdl->granularity = ram_discard_manager_get_min_granularity(rdm,
866 section->mr);
867
868 g_assert(vrdl->granularity && is_power_of_2(vrdl->granularity));
869 g_assert(container->pgsizes &&
870 vrdl->granularity >= 1ULL << ctz64(container->pgsizes));
871
872 ram_discard_listener_init(&vrdl->listener,
873 vfio_ram_discard_notify_populate,
874 vfio_ram_discard_notify_discard, true);
875 ram_discard_manager_register_listener(rdm, &vrdl->listener, section);
876 QLIST_INSERT_HEAD(&container->vrdl_list, vrdl, next);
877
878 /*
879 * Sanity-check if we have a theoretically problematic setup where we could
880 * exceed the maximum number of possible DMA mappings over time. We assume
881 * that each mapped section in the same address space as a RamDiscardManager
882 * section consumes exactly one DMA mapping, with the exception of
883 * RamDiscardManager sections; i.e., we don't expect to have gIOMMU sections
884 * in the same address space as RamDiscardManager sections.
885 *
886 * We assume that each section in the address space consumes one memslot.
887 * We take the number of KVM memory slots as a best guess for the maximum
888 * number of sections in the address space we could have over time,
889 * also consuming DMA mappings.
890 */
891 if (container->dma_max_mappings) {
892 unsigned int vrdl_count = 0, vrdl_mappings = 0, max_memslots = 512;
893
894 #ifdef CONFIG_KVM
895 if (kvm_enabled()) {
896 max_memslots = kvm_get_max_memslots();
897 }
898 #endif
899
900 QLIST_FOREACH(vrdl, &container->vrdl_list, next) {
901 hwaddr start, end;
902
903 start = QEMU_ALIGN_DOWN(vrdl->offset_within_address_space,
904 vrdl->granularity);
905 end = ROUND_UP(vrdl->offset_within_address_space + vrdl->size,
906 vrdl->granularity);
907 vrdl_mappings += (end - start) / vrdl->granularity;
908 vrdl_count++;
909 }
910
911 if (vrdl_mappings + max_memslots - vrdl_count >
912 container->dma_max_mappings) {
913 warn_report("%s: possibly running out of DMA mappings. E.g., try"
914 " increasing the 'block-size' of virtio-mem devies."
915 " Maximum possible DMA mappings: %d, Maximum possible"
916 " memslots: %d", __func__, container->dma_max_mappings,
917 max_memslots);
918 }
919 }
920 }
921
922 static void vfio_unregister_ram_discard_listener(VFIOContainer *container,
923 MemoryRegionSection *section)
924 {
925 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr);
926 VFIORamDiscardListener *vrdl = NULL;
927
928 QLIST_FOREACH(vrdl, &container->vrdl_list, next) {
929 if (vrdl->mr == section->mr &&
930 vrdl->offset_within_address_space ==
931 section->offset_within_address_space) {
932 break;
933 }
934 }
935
936 if (!vrdl) {
937 hw_error("vfio: Trying to unregister missing RAM discard listener");
938 }
939
940 ram_discard_manager_unregister_listener(rdm, &vrdl->listener);
941 QLIST_REMOVE(vrdl, next);
942 g_free(vrdl);
943 }
944
945 static VFIOHostDMAWindow *vfio_find_hostwin(VFIOContainer *container,
946 hwaddr iova, hwaddr end)
947 {
948 VFIOHostDMAWindow *hostwin;
949 bool hostwin_found = false;
950
951 QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) {
952 if (hostwin->min_iova <= iova && end <= hostwin->max_iova) {
953 hostwin_found = true;
954 break;
955 }
956 }
957
958 return hostwin_found ? hostwin : NULL;
959 }
960
961 static bool vfio_known_safe_misalignment(MemoryRegionSection *section)
962 {
963 MemoryRegion *mr = section->mr;
964
965 if (!TPM_IS_CRB(mr->owner)) {
966 return false;
967 }
968
969 /* this is a known safe misaligned region, just trace for debug purpose */
970 trace_vfio_known_safe_misalignment(memory_region_name(mr),
971 section->offset_within_address_space,
972 section->offset_within_region,
973 qemu_real_host_page_size());
974 return true;
975 }
976
977 static bool vfio_listener_valid_section(MemoryRegionSection *section,
978 const char *name)
979 {
980 if (vfio_listener_skipped_section(section)) {
981 trace_vfio_listener_region_skip(name,
982 section->offset_within_address_space,
983 section->offset_within_address_space +
984 int128_get64(int128_sub(section->size, int128_one())));
985 return false;
986 }
987
988 if (unlikely((section->offset_within_address_space &
989 ~qemu_real_host_page_mask()) !=
990 (section->offset_within_region & ~qemu_real_host_page_mask()))) {
991 if (!vfio_known_safe_misalignment(section)) {
992 error_report("%s received unaligned region %s iova=0x%"PRIx64
993 " offset_within_region=0x%"PRIx64
994 " qemu_real_host_page_size=0x%"PRIxPTR,
995 __func__, memory_region_name(section->mr),
996 section->offset_within_address_space,
997 section->offset_within_region,
998 qemu_real_host_page_size());
999 }
1000 return false;
1001 }
1002
1003 return true;
1004 }
1005
1006 static bool vfio_get_section_iova_range(VFIOContainer *container,
1007 MemoryRegionSection *section,
1008 hwaddr *out_iova, hwaddr *out_end,
1009 Int128 *out_llend)
1010 {
1011 Int128 llend;
1012 hwaddr iova;
1013
1014 iova = REAL_HOST_PAGE_ALIGN(section->offset_within_address_space);
1015 llend = int128_make64(section->offset_within_address_space);
1016 llend = int128_add(llend, section->size);
1017 llend = int128_and(llend, int128_exts64(qemu_real_host_page_mask()));
1018
1019 if (int128_ge(int128_make64(iova), llend)) {
1020 return false;
1021 }
1022
1023 *out_iova = iova;
1024 *out_end = int128_get64(int128_sub(llend, int128_one()));
1025 if (out_llend) {
1026 *out_llend = llend;
1027 }
1028 return true;
1029 }
1030
1031 static void vfio_listener_region_add(MemoryListener *listener,
1032 MemoryRegionSection *section)
1033 {
1034 VFIOContainer *container = container_of(listener, VFIOContainer, listener);
1035 hwaddr iova, end;
1036 Int128 llend, llsize;
1037 void *vaddr;
1038 int ret;
1039 VFIOHostDMAWindow *hostwin;
1040 Error *err = NULL;
1041
1042 if (!vfio_listener_valid_section(section, "region_add")) {
1043 return;
1044 }
1045
1046 if (!vfio_get_section_iova_range(container, section, &iova, &end, &llend)) {
1047 if (memory_region_is_ram_device(section->mr)) {
1048 trace_vfio_listener_region_add_no_dma_map(
1049 memory_region_name(section->mr),
1050 section->offset_within_address_space,
1051 int128_getlo(section->size),
1052 qemu_real_host_page_size());
1053 }
1054 return;
1055 }
1056
1057 if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) {
1058 hwaddr pgsize = 0;
1059
1060 /* For now intersections are not allowed, we may relax this later */
1061 QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) {
1062 if (ranges_overlap(hostwin->min_iova,
1063 hostwin->max_iova - hostwin->min_iova + 1,
1064 section->offset_within_address_space,
1065 int128_get64(section->size))) {
1066 error_setg(&err,
1067 "region [0x%"PRIx64",0x%"PRIx64"] overlaps with existing"
1068 "host DMA window [0x%"PRIx64",0x%"PRIx64"]",
1069 section->offset_within_address_space,
1070 section->offset_within_address_space +
1071 int128_get64(section->size) - 1,
1072 hostwin->min_iova, hostwin->max_iova);
1073 goto fail;
1074 }
1075 }
1076
1077 ret = vfio_spapr_create_window(container, section, &pgsize);
1078 if (ret) {
1079 error_setg_errno(&err, -ret, "Failed to create SPAPR window");
1080 goto fail;
1081 }
1082
1083 vfio_host_win_add(container, section->offset_within_address_space,
1084 section->offset_within_address_space +
1085 int128_get64(section->size) - 1, pgsize);
1086 #ifdef CONFIG_KVM
1087 if (kvm_enabled()) {
1088 VFIOGroup *group;
1089 IOMMUMemoryRegion *iommu_mr = IOMMU_MEMORY_REGION(section->mr);
1090 struct kvm_vfio_spapr_tce param;
1091 struct kvm_device_attr attr = {
1092 .group = KVM_DEV_VFIO_GROUP,
1093 .attr = KVM_DEV_VFIO_GROUP_SET_SPAPR_TCE,
1094 .addr = (uint64_t)(unsigned long)&param,
1095 };
1096
1097 if (!memory_region_iommu_get_attr(iommu_mr, IOMMU_ATTR_SPAPR_TCE_FD,
1098 &param.tablefd)) {
1099 QLIST_FOREACH(group, &container->group_list, container_next) {
1100 param.groupfd = group->fd;
1101 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) {
1102 error_report("vfio: failed to setup fd %d "
1103 "for a group with fd %d: %s",
1104 param.tablefd, param.groupfd,
1105 strerror(errno));
1106 return;
1107 }
1108 trace_vfio_spapr_group_attach(param.groupfd, param.tablefd);
1109 }
1110 }
1111 }
1112 #endif
1113 }
1114
1115 hostwin = vfio_find_hostwin(container, iova, end);
1116 if (!hostwin) {
1117 error_setg(&err, "Container %p can't map guest IOVA region"
1118 " 0x%"HWADDR_PRIx"..0x%"HWADDR_PRIx, container, iova, end);
1119 goto fail;
1120 }
1121
1122 memory_region_ref(section->mr);
1123
1124 if (memory_region_is_iommu(section->mr)) {
1125 VFIOGuestIOMMU *giommu;
1126 IOMMUMemoryRegion *iommu_mr = IOMMU_MEMORY_REGION(section->mr);
1127 int iommu_idx;
1128
1129 trace_vfio_listener_region_add_iommu(iova, end);
1130 /*
1131 * FIXME: For VFIO iommu types which have KVM acceleration to
1132 * avoid bouncing all map/unmaps through qemu this way, this
1133 * would be the right place to wire that up (tell the KVM
1134 * device emulation the VFIO iommu handles to use).
1135 */
1136 giommu = g_malloc0(sizeof(*giommu));
1137 giommu->iommu_mr = iommu_mr;
1138 giommu->iommu_offset = section->offset_within_address_space -
1139 section->offset_within_region;
1140 giommu->container = container;
1141 llend = int128_add(int128_make64(section->offset_within_region),
1142 section->size);
1143 llend = int128_sub(llend, int128_one());
1144 iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr,
1145 MEMTXATTRS_UNSPECIFIED);
1146 iommu_notifier_init(&giommu->n, vfio_iommu_map_notify,
1147 IOMMU_NOTIFIER_IOTLB_EVENTS,
1148 section->offset_within_region,
1149 int128_get64(llend),
1150 iommu_idx);
1151
1152 ret = memory_region_iommu_set_page_size_mask(giommu->iommu_mr,
1153 container->pgsizes,
1154 &err);
1155 if (ret) {
1156 g_free(giommu);
1157 goto fail;
1158 }
1159
1160 ret = memory_region_register_iommu_notifier(section->mr, &giommu->n,
1161 &err);
1162 if (ret) {
1163 g_free(giommu);
1164 goto fail;
1165 }
1166 QLIST_INSERT_HEAD(&container->giommu_list, giommu, giommu_next);
1167 memory_region_iommu_replay(giommu->iommu_mr, &giommu->n);
1168
1169 return;
1170 }
1171
1172 /* Here we assume that memory_region_is_ram(section->mr)==true */
1173
1174 /*
1175 * For RAM memory regions with a RamDiscardManager, we only want to map the
1176 * actually populated parts - and update the mapping whenever we're notified
1177 * about changes.
1178 */
1179 if (memory_region_has_ram_discard_manager(section->mr)) {
1180 vfio_register_ram_discard_listener(container, section);
1181 return;
1182 }
1183
1184 vaddr = memory_region_get_ram_ptr(section->mr) +
1185 section->offset_within_region +
1186 (iova - section->offset_within_address_space);
1187
1188 trace_vfio_listener_region_add_ram(iova, end, vaddr);
1189
1190 llsize = int128_sub(llend, int128_make64(iova));
1191
1192 if (memory_region_is_ram_device(section->mr)) {
1193 hwaddr pgmask = (1ULL << ctz64(hostwin->iova_pgsizes)) - 1;
1194
1195 if ((iova & pgmask) || (int128_get64(llsize) & pgmask)) {
1196 trace_vfio_listener_region_add_no_dma_map(
1197 memory_region_name(section->mr),
1198 section->offset_within_address_space,
1199 int128_getlo(section->size),
1200 pgmask + 1);
1201 return;
1202 }
1203 }
1204
1205 ret = vfio_dma_map(container, iova, int128_get64(llsize),
1206 vaddr, section->readonly);
1207 if (ret) {
1208 error_setg(&err, "vfio_dma_map(%p, 0x%"HWADDR_PRIx", "
1209 "0x%"HWADDR_PRIx", %p) = %d (%s)",
1210 container, iova, int128_get64(llsize), vaddr, ret,
1211 strerror(-ret));
1212 if (memory_region_is_ram_device(section->mr)) {
1213 /* Allow unexpected mappings not to be fatal for RAM devices */
1214 error_report_err(err);
1215 return;
1216 }
1217 goto fail;
1218 }
1219
1220 return;
1221
1222 fail:
1223 if (memory_region_is_ram_device(section->mr)) {
1224 error_report("failed to vfio_dma_map. pci p2p may not work");
1225 return;
1226 }
1227 /*
1228 * On the initfn path, store the first error in the container so we
1229 * can gracefully fail. Runtime, there's not much we can do other
1230 * than throw a hardware error.
1231 */
1232 if (!container->initialized) {
1233 if (!container->error) {
1234 error_propagate_prepend(&container->error, err,
1235 "Region %s: ",
1236 memory_region_name(section->mr));
1237 } else {
1238 error_free(err);
1239 }
1240 } else {
1241 error_report_err(err);
1242 hw_error("vfio: DMA mapping failed, unable to continue");
1243 }
1244 }
1245
1246 static void vfio_listener_region_del(MemoryListener *listener,
1247 MemoryRegionSection *section)
1248 {
1249 VFIOContainer *container = container_of(listener, VFIOContainer, listener);
1250 hwaddr iova, end;
1251 Int128 llend, llsize;
1252 int ret;
1253 bool try_unmap = true;
1254
1255 if (!vfio_listener_valid_section(section, "region_del")) {
1256 return;
1257 }
1258
1259 if (memory_region_is_iommu(section->mr)) {
1260 VFIOGuestIOMMU *giommu;
1261
1262 QLIST_FOREACH(giommu, &container->giommu_list, giommu_next) {
1263 if (MEMORY_REGION(giommu->iommu_mr) == section->mr &&
1264 giommu->n.start == section->offset_within_region) {
1265 memory_region_unregister_iommu_notifier(section->mr,
1266 &giommu->n);
1267 QLIST_REMOVE(giommu, giommu_next);
1268 g_free(giommu);
1269 break;
1270 }
1271 }
1272
1273 /*
1274 * FIXME: We assume the one big unmap below is adequate to
1275 * remove any individual page mappings in the IOMMU which
1276 * might have been copied into VFIO. This works for a page table
1277 * based IOMMU where a big unmap flattens a large range of IO-PTEs.
1278 * That may not be true for all IOMMU types.
1279 */
1280 }
1281
1282 if (!vfio_get_section_iova_range(container, section, &iova, &end, &llend)) {
1283 return;
1284 }
1285
1286 llsize = int128_sub(llend, int128_make64(iova));
1287
1288 trace_vfio_listener_region_del(iova, end);
1289
1290 if (memory_region_is_ram_device(section->mr)) {
1291 hwaddr pgmask;
1292 VFIOHostDMAWindow *hostwin;
1293
1294 hostwin = vfio_find_hostwin(container, iova, end);
1295 assert(hostwin); /* or region_add() would have failed */
1296
1297 pgmask = (1ULL << ctz64(hostwin->iova_pgsizes)) - 1;
1298 try_unmap = !((iova & pgmask) || (int128_get64(llsize) & pgmask));
1299 } else if (memory_region_has_ram_discard_manager(section->mr)) {
1300 vfio_unregister_ram_discard_listener(container, section);
1301 /* Unregistering will trigger an unmap. */
1302 try_unmap = false;
1303 }
1304
1305 if (try_unmap) {
1306 if (int128_eq(llsize, int128_2_64())) {
1307 /* The unmap ioctl doesn't accept a full 64-bit span. */
1308 llsize = int128_rshift(llsize, 1);
1309 ret = vfio_dma_unmap(container, iova, int128_get64(llsize), NULL);
1310 if (ret) {
1311 error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", "
1312 "0x%"HWADDR_PRIx") = %d (%s)",
1313 container, iova, int128_get64(llsize), ret,
1314 strerror(-ret));
1315 }
1316 iova += int128_get64(llsize);
1317 }
1318 ret = vfio_dma_unmap(container, iova, int128_get64(llsize), NULL);
1319 if (ret) {
1320 error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", "
1321 "0x%"HWADDR_PRIx") = %d (%s)",
1322 container, iova, int128_get64(llsize), ret,
1323 strerror(-ret));
1324 }
1325 }
1326
1327 memory_region_unref(section->mr);
1328
1329 if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) {
1330 vfio_spapr_remove_window(container,
1331 section->offset_within_address_space);
1332 if (vfio_host_win_del(container,
1333 section->offset_within_address_space,
1334 section->offset_within_address_space +
1335 int128_get64(section->size) - 1) < 0) {
1336 hw_error("%s: Cannot delete missing window at %"HWADDR_PRIx,
1337 __func__, section->offset_within_address_space);
1338 }
1339 }
1340 }
1341
1342 static int vfio_set_dirty_page_tracking(VFIOContainer *container, bool start)
1343 {
1344 int ret;
1345 struct vfio_iommu_type1_dirty_bitmap dirty = {
1346 .argsz = sizeof(dirty),
1347 };
1348
1349 if (!container->dirty_pages_supported) {
1350 return 0;
1351 }
1352
1353 if (start) {
1354 dirty.flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_START;
1355 } else {
1356 dirty.flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_STOP;
1357 }
1358
1359 ret = ioctl(container->fd, VFIO_IOMMU_DIRTY_PAGES, &dirty);
1360 if (ret) {
1361 ret = -errno;
1362 error_report("Failed to set dirty tracking flag 0x%x errno: %d",
1363 dirty.flags, errno);
1364 }
1365
1366 return ret;
1367 }
1368
1369 typedef struct VFIODirtyRanges {
1370 hwaddr min32;
1371 hwaddr max32;
1372 hwaddr min64;
1373 hwaddr max64;
1374 } VFIODirtyRanges;
1375
1376 typedef struct VFIODirtyRangesListener {
1377 VFIOContainer *container;
1378 VFIODirtyRanges ranges;
1379 MemoryListener listener;
1380 } VFIODirtyRangesListener;
1381
1382 static void vfio_dirty_tracking_update(MemoryListener *listener,
1383 MemoryRegionSection *section)
1384 {
1385 VFIODirtyRangesListener *dirty = container_of(listener,
1386 VFIODirtyRangesListener,
1387 listener);
1388 VFIODirtyRanges *range = &dirty->ranges;
1389 hwaddr iova, end, *min, *max;
1390
1391 if (!vfio_listener_valid_section(section, "tracking_update") ||
1392 !vfio_get_section_iova_range(dirty->container, section,
1393 &iova, &end, NULL)) {
1394 return;
1395 }
1396
1397 /*
1398 * The address space passed to the dirty tracker is reduced to two ranges:
1399 * one for 32-bit DMA ranges, and another one for 64-bit DMA ranges.
1400 * The underlying reports of dirty will query a sub-interval of each of
1401 * these ranges.
1402 *
1403 * The purpose of the dual range handling is to handle known cases of big
1404 * holes in the address space, like the x86 AMD 1T hole. The alternative
1405 * would be an IOVATree but that has a much bigger runtime overhead and
1406 * unnecessary complexity.
1407 */
1408 min = (end <= UINT32_MAX) ? &range->min32 : &range->min64;
1409 max = (end <= UINT32_MAX) ? &range->max32 : &range->max64;
1410
1411 if (*min > iova) {
1412 *min = iova;
1413 }
1414 if (*max < end) {
1415 *max = end;
1416 }
1417
1418 trace_vfio_device_dirty_tracking_update(iova, end, *min, *max);
1419 return;
1420 }
1421
1422 static const MemoryListener vfio_dirty_tracking_listener = {
1423 .name = "vfio-tracking",
1424 .region_add = vfio_dirty_tracking_update,
1425 };
1426
1427 static void vfio_dirty_tracking_init(VFIOContainer *container,
1428 VFIODirtyRanges *ranges)
1429 {
1430 VFIODirtyRangesListener dirty;
1431
1432 memset(&dirty, 0, sizeof(dirty));
1433 dirty.ranges.min32 = UINT32_MAX;
1434 dirty.ranges.min64 = UINT64_MAX;
1435 dirty.listener = vfio_dirty_tracking_listener;
1436 dirty.container = container;
1437
1438 memory_listener_register(&dirty.listener,
1439 container->space->as);
1440
1441 *ranges = dirty.ranges;
1442
1443 /*
1444 * The memory listener is synchronous, and used to calculate the range
1445 * to dirty tracking. Unregister it after we are done as we are not
1446 * interested in any follow-up updates.
1447 */
1448 memory_listener_unregister(&dirty.listener);
1449 }
1450
1451 static void vfio_devices_dma_logging_stop(VFIOContainer *container)
1452 {
1453 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature),
1454 sizeof(uint64_t))] = {};
1455 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf;
1456 VFIODevice *vbasedev;
1457 VFIOGroup *group;
1458
1459 feature->argsz = sizeof(buf);
1460 feature->flags = VFIO_DEVICE_FEATURE_SET |
1461 VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP;
1462
1463 QLIST_FOREACH(group, &container->group_list, container_next) {
1464 QLIST_FOREACH(vbasedev, &group->device_list, next) {
1465 if (!vbasedev->dirty_tracking) {
1466 continue;
1467 }
1468
1469 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) {
1470 warn_report("%s: Failed to stop DMA logging, err %d (%s)",
1471 vbasedev->name, -errno, strerror(errno));
1472 }
1473 vbasedev->dirty_tracking = false;
1474 }
1475 }
1476 }
1477
1478 static struct vfio_device_feature *
1479 vfio_device_feature_dma_logging_start_create(VFIOContainer *container,
1480 VFIODirtyRanges *tracking)
1481 {
1482 struct vfio_device_feature *feature;
1483 size_t feature_size;
1484 struct vfio_device_feature_dma_logging_control *control;
1485 struct vfio_device_feature_dma_logging_range *ranges;
1486
1487 feature_size = sizeof(struct vfio_device_feature) +
1488 sizeof(struct vfio_device_feature_dma_logging_control);
1489 feature = g_try_malloc0(feature_size);
1490 if (!feature) {
1491 errno = ENOMEM;
1492 return NULL;
1493 }
1494 feature->argsz = feature_size;
1495 feature->flags = VFIO_DEVICE_FEATURE_SET |
1496 VFIO_DEVICE_FEATURE_DMA_LOGGING_START;
1497
1498 control = (struct vfio_device_feature_dma_logging_control *)feature->data;
1499 control->page_size = qemu_real_host_page_size();
1500
1501 /*
1502 * DMA logging uAPI guarantees to support at least a number of ranges that
1503 * fits into a single host kernel base page.
1504 */
1505 control->num_ranges = !!tracking->max32 + !!tracking->max64;
1506 ranges = g_try_new0(struct vfio_device_feature_dma_logging_range,
1507 control->num_ranges);
1508 if (!ranges) {
1509 g_free(feature);
1510 errno = ENOMEM;
1511
1512 return NULL;
1513 }
1514
1515 control->ranges = (__u64)(uintptr_t)ranges;
1516 if (tracking->max32) {
1517 ranges->iova = tracking->min32;
1518 ranges->length = (tracking->max32 - tracking->min32) + 1;
1519 ranges++;
1520 }
1521 if (tracking->max64) {
1522 ranges->iova = tracking->min64;
1523 ranges->length = (tracking->max64 - tracking->min64) + 1;
1524 }
1525
1526 trace_vfio_device_dirty_tracking_start(control->num_ranges,
1527 tracking->min32, tracking->max32,
1528 tracking->min64, tracking->max64);
1529
1530 return feature;
1531 }
1532
1533 static void vfio_device_feature_dma_logging_start_destroy(
1534 struct vfio_device_feature *feature)
1535 {
1536 struct vfio_device_feature_dma_logging_control *control =
1537 (struct vfio_device_feature_dma_logging_control *)feature->data;
1538 struct vfio_device_feature_dma_logging_range *ranges =
1539 (struct vfio_device_feature_dma_logging_range *)(uintptr_t)control->ranges;
1540
1541 g_free(ranges);
1542 g_free(feature);
1543 }
1544
1545 static int vfio_devices_dma_logging_start(VFIOContainer *container)
1546 {
1547 struct vfio_device_feature *feature;
1548 VFIODirtyRanges ranges;
1549 VFIODevice *vbasedev;
1550 VFIOGroup *group;
1551 int ret = 0;
1552
1553 vfio_dirty_tracking_init(container, &ranges);
1554 feature = vfio_device_feature_dma_logging_start_create(container,
1555 &ranges);
1556 if (!feature) {
1557 return -errno;
1558 }
1559
1560 QLIST_FOREACH(group, &container->group_list, container_next) {
1561 QLIST_FOREACH(vbasedev, &group->device_list, next) {
1562 if (vbasedev->dirty_tracking) {
1563 continue;
1564 }
1565
1566 ret = ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature);
1567 if (ret) {
1568 ret = -errno;
1569 error_report("%s: Failed to start DMA logging, err %d (%s)",
1570 vbasedev->name, ret, strerror(errno));
1571 goto out;
1572 }
1573 vbasedev->dirty_tracking = true;
1574 }
1575 }
1576
1577 out:
1578 if (ret) {
1579 vfio_devices_dma_logging_stop(container);
1580 }
1581
1582 vfio_device_feature_dma_logging_start_destroy(feature);
1583
1584 return ret;
1585 }
1586
1587 static void vfio_listener_log_global_start(MemoryListener *listener)
1588 {
1589 VFIOContainer *container = container_of(listener, VFIOContainer, listener);
1590 int ret;
1591
1592 if (vfio_devices_all_device_dirty_tracking(container)) {
1593 ret = vfio_devices_dma_logging_start(container);
1594 } else {
1595 ret = vfio_set_dirty_page_tracking(container, true);
1596 }
1597
1598 if (ret) {
1599 error_report("vfio: Could not start dirty page tracking, err: %d (%s)",
1600 ret, strerror(-ret));
1601 vfio_set_migration_error(ret);
1602 }
1603 }
1604
1605 static void vfio_listener_log_global_stop(MemoryListener *listener)
1606 {
1607 VFIOContainer *container = container_of(listener, VFIOContainer, listener);
1608 int ret = 0;
1609
1610 if (vfio_devices_all_device_dirty_tracking(container)) {
1611 vfio_devices_dma_logging_stop(container);
1612 } else {
1613 ret = vfio_set_dirty_page_tracking(container, false);
1614 }
1615
1616 if (ret) {
1617 error_report("vfio: Could not stop dirty page tracking, err: %d (%s)",
1618 ret, strerror(-ret));
1619 vfio_set_migration_error(ret);
1620 }
1621 }
1622
1623 static int vfio_device_dma_logging_report(VFIODevice *vbasedev, hwaddr iova,
1624 hwaddr size, void *bitmap)
1625 {
1626 uint64_t buf[DIV_ROUND_UP(sizeof(struct vfio_device_feature) +
1627 sizeof(struct vfio_device_feature_dma_logging_report),
1628 sizeof(__u64))] = {};
1629 struct vfio_device_feature *feature = (struct vfio_device_feature *)buf;
1630 struct vfio_device_feature_dma_logging_report *report =
1631 (struct vfio_device_feature_dma_logging_report *)feature->data;
1632
1633 report->iova = iova;
1634 report->length = size;
1635 report->page_size = qemu_real_host_page_size();
1636 report->bitmap = (__u64)(uintptr_t)bitmap;
1637
1638 feature->argsz = sizeof(buf);
1639 feature->flags = VFIO_DEVICE_FEATURE_GET |
1640 VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT;
1641
1642 if (ioctl(vbasedev->fd, VFIO_DEVICE_FEATURE, feature)) {
1643 return -errno;
1644 }
1645
1646 return 0;
1647 }
1648
1649 static int vfio_devices_query_dirty_bitmap(VFIOContainer *container,
1650 VFIOBitmap *vbmap, hwaddr iova,
1651 hwaddr size)
1652 {
1653 VFIODevice *vbasedev;
1654 VFIOGroup *group;
1655 int ret;
1656
1657 QLIST_FOREACH(group, &container->group_list, container_next) {
1658 QLIST_FOREACH(vbasedev, &group->device_list, next) {
1659 ret = vfio_device_dma_logging_report(vbasedev, iova, size,
1660 vbmap->bitmap);
1661 if (ret) {
1662 error_report("%s: Failed to get DMA logging report, iova: "
1663 "0x%" HWADDR_PRIx ", size: 0x%" HWADDR_PRIx
1664 ", err: %d (%s)",
1665 vbasedev->name, iova, size, ret, strerror(-ret));
1666
1667 return ret;
1668 }
1669 }
1670 }
1671
1672 return 0;
1673 }
1674
1675 static int vfio_query_dirty_bitmap(VFIOContainer *container, VFIOBitmap *vbmap,
1676 hwaddr iova, hwaddr size)
1677 {
1678 struct vfio_iommu_type1_dirty_bitmap *dbitmap;
1679 struct vfio_iommu_type1_dirty_bitmap_get *range;
1680 int ret;
1681
1682 dbitmap = g_malloc0(sizeof(*dbitmap) + sizeof(*range));
1683
1684 dbitmap->argsz = sizeof(*dbitmap) + sizeof(*range);
1685 dbitmap->flags = VFIO_IOMMU_DIRTY_PAGES_FLAG_GET_BITMAP;
1686 range = (struct vfio_iommu_type1_dirty_bitmap_get *)&dbitmap->data;
1687 range->iova = iova;
1688 range->size = size;
1689
1690 /*
1691 * cpu_physical_memory_set_dirty_lebitmap() supports pages in bitmap of
1692 * qemu_real_host_page_size to mark those dirty. Hence set bitmap's pgsize
1693 * to qemu_real_host_page_size.
1694 */
1695 range->bitmap.pgsize = qemu_real_host_page_size();
1696 range->bitmap.size = vbmap->size;
1697 range->bitmap.data = (__u64 *)vbmap->bitmap;
1698
1699 ret = ioctl(container->fd, VFIO_IOMMU_DIRTY_PAGES, dbitmap);
1700 if (ret) {
1701 ret = -errno;
1702 error_report("Failed to get dirty bitmap for iova: 0x%"PRIx64
1703 " size: 0x%"PRIx64" err: %d", (uint64_t)range->iova,
1704 (uint64_t)range->size, errno);
1705 }
1706
1707 g_free(dbitmap);
1708
1709 return ret;
1710 }
1711
1712 static int vfio_get_dirty_bitmap(VFIOContainer *container, uint64_t iova,
1713 uint64_t size, ram_addr_t ram_addr)
1714 {
1715 bool all_device_dirty_tracking =
1716 vfio_devices_all_device_dirty_tracking(container);
1717 uint64_t dirty_pages;
1718 VFIOBitmap vbmap;
1719 int ret;
1720
1721 if (!container->dirty_pages_supported && !all_device_dirty_tracking) {
1722 cpu_physical_memory_set_dirty_range(ram_addr, size,
1723 tcg_enabled() ? DIRTY_CLIENTS_ALL :
1724 DIRTY_CLIENTS_NOCODE);
1725 return 0;
1726 }
1727
1728 ret = vfio_bitmap_alloc(&vbmap, size);
1729 if (ret) {
1730 return ret;
1731 }
1732
1733 if (all_device_dirty_tracking) {
1734 ret = vfio_devices_query_dirty_bitmap(container, &vbmap, iova, size);
1735 } else {
1736 ret = vfio_query_dirty_bitmap(container, &vbmap, iova, size);
1737 }
1738
1739 if (ret) {
1740 goto out;
1741 }
1742
1743 dirty_pages = cpu_physical_memory_set_dirty_lebitmap(vbmap.bitmap, ram_addr,
1744 vbmap.pages);
1745
1746 trace_vfio_get_dirty_bitmap(container->fd, iova, size, vbmap.size,
1747 ram_addr, dirty_pages);
1748 out:
1749 g_free(vbmap.bitmap);
1750
1751 return ret;
1752 }
1753
1754 typedef struct {
1755 IOMMUNotifier n;
1756 VFIOGuestIOMMU *giommu;
1757 } vfio_giommu_dirty_notifier;
1758
1759 static void vfio_iommu_map_dirty_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
1760 {
1761 vfio_giommu_dirty_notifier *gdn = container_of(n,
1762 vfio_giommu_dirty_notifier, n);
1763 VFIOGuestIOMMU *giommu = gdn->giommu;
1764 VFIOContainer *container = giommu->container;
1765 hwaddr iova = iotlb->iova + giommu->iommu_offset;
1766 ram_addr_t translated_addr;
1767 int ret = -EINVAL;
1768
1769 trace_vfio_iommu_map_dirty_notify(iova, iova + iotlb->addr_mask);
1770
1771 if (iotlb->target_as != &address_space_memory) {
1772 error_report("Wrong target AS \"%s\", only system memory is allowed",
1773 iotlb->target_as->name ? iotlb->target_as->name : "none");
1774 goto out;
1775 }
1776
1777 rcu_read_lock();
1778 if (vfio_get_xlat_addr(iotlb, NULL, &translated_addr, NULL)) {
1779 ret = vfio_get_dirty_bitmap(container, iova, iotlb->addr_mask + 1,
1780 translated_addr);
1781 if (ret) {
1782 error_report("vfio_iommu_map_dirty_notify(%p, 0x%"HWADDR_PRIx", "
1783 "0x%"HWADDR_PRIx") = %d (%s)",
1784 container, iova, iotlb->addr_mask + 1, ret,
1785 strerror(-ret));
1786 }
1787 }
1788 rcu_read_unlock();
1789
1790 out:
1791 if (ret) {
1792 vfio_set_migration_error(ret);
1793 }
1794 }
1795
1796 static int vfio_ram_discard_get_dirty_bitmap(MemoryRegionSection *section,
1797 void *opaque)
1798 {
1799 const hwaddr size = int128_get64(section->size);
1800 const hwaddr iova = section->offset_within_address_space;
1801 const ram_addr_t ram_addr = memory_region_get_ram_addr(section->mr) +
1802 section->offset_within_region;
1803 VFIORamDiscardListener *vrdl = opaque;
1804
1805 /*
1806 * Sync the whole mapped region (spanning multiple individual mappings)
1807 * in one go.
1808 */
1809 return vfio_get_dirty_bitmap(vrdl->container, iova, size, ram_addr);
1810 }
1811
1812 static int vfio_sync_ram_discard_listener_dirty_bitmap(VFIOContainer *container,
1813 MemoryRegionSection *section)
1814 {
1815 RamDiscardManager *rdm = memory_region_get_ram_discard_manager(section->mr);
1816 VFIORamDiscardListener *vrdl = NULL;
1817
1818 QLIST_FOREACH(vrdl, &container->vrdl_list, next) {
1819 if (vrdl->mr == section->mr &&
1820 vrdl->offset_within_address_space ==
1821 section->offset_within_address_space) {
1822 break;
1823 }
1824 }
1825
1826 if (!vrdl) {
1827 hw_error("vfio: Trying to sync missing RAM discard listener");
1828 }
1829
1830 /*
1831 * We only want/can synchronize the bitmap for actually mapped parts -
1832 * which correspond to populated parts. Replay all populated parts.
1833 */
1834 return ram_discard_manager_replay_populated(rdm, section,
1835 vfio_ram_discard_get_dirty_bitmap,
1836 &vrdl);
1837 }
1838
1839 static int vfio_sync_dirty_bitmap(VFIOContainer *container,
1840 MemoryRegionSection *section)
1841 {
1842 ram_addr_t ram_addr;
1843
1844 if (memory_region_is_iommu(section->mr)) {
1845 VFIOGuestIOMMU *giommu;
1846
1847 QLIST_FOREACH(giommu, &container->giommu_list, giommu_next) {
1848 if (MEMORY_REGION(giommu->iommu_mr) == section->mr &&
1849 giommu->n.start == section->offset_within_region) {
1850 Int128 llend;
1851 vfio_giommu_dirty_notifier gdn = { .giommu = giommu };
1852 int idx = memory_region_iommu_attrs_to_index(giommu->iommu_mr,
1853 MEMTXATTRS_UNSPECIFIED);
1854
1855 llend = int128_add(int128_make64(section->offset_within_region),
1856 section->size);
1857 llend = int128_sub(llend, int128_one());
1858
1859 iommu_notifier_init(&gdn.n,
1860 vfio_iommu_map_dirty_notify,
1861 IOMMU_NOTIFIER_MAP,
1862 section->offset_within_region,
1863 int128_get64(llend),
1864 idx);
1865 memory_region_iommu_replay(giommu->iommu_mr, &gdn.n);
1866 break;
1867 }
1868 }
1869 return 0;
1870 } else if (memory_region_has_ram_discard_manager(section->mr)) {
1871 return vfio_sync_ram_discard_listener_dirty_bitmap(container, section);
1872 }
1873
1874 ram_addr = memory_region_get_ram_addr(section->mr) +
1875 section->offset_within_region;
1876
1877 return vfio_get_dirty_bitmap(container,
1878 REAL_HOST_PAGE_ALIGN(section->offset_within_address_space),
1879 int128_get64(section->size), ram_addr);
1880 }
1881
1882 static void vfio_listener_log_sync(MemoryListener *listener,
1883 MemoryRegionSection *section)
1884 {
1885 VFIOContainer *container = container_of(listener, VFIOContainer, listener);
1886 int ret;
1887
1888 if (vfio_listener_skipped_section(section)) {
1889 return;
1890 }
1891
1892 if (vfio_devices_all_dirty_tracking(container)) {
1893 ret = vfio_sync_dirty_bitmap(container, section);
1894 if (ret) {
1895 error_report("vfio: Failed to sync dirty bitmap, err: %d (%s)", ret,
1896 strerror(-ret));
1897 vfio_set_migration_error(ret);
1898 }
1899 }
1900 }
1901
1902 static const MemoryListener vfio_memory_listener = {
1903 .name = "vfio",
1904 .region_add = vfio_listener_region_add,
1905 .region_del = vfio_listener_region_del,
1906 .log_global_start = vfio_listener_log_global_start,
1907 .log_global_stop = vfio_listener_log_global_stop,
1908 .log_sync = vfio_listener_log_sync,
1909 };
1910
1911 static void vfio_listener_release(VFIOContainer *container)
1912 {
1913 memory_listener_unregister(&container->listener);
1914 if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) {
1915 memory_listener_unregister(&container->prereg_listener);
1916 }
1917 }
1918
1919 static struct vfio_info_cap_header *
1920 vfio_get_cap(void *ptr, uint32_t cap_offset, uint16_t id)
1921 {
1922 struct vfio_info_cap_header *hdr;
1923
1924 for (hdr = ptr + cap_offset; hdr != ptr; hdr = ptr + hdr->next) {
1925 if (hdr->id == id) {
1926 return hdr;
1927 }
1928 }
1929
1930 return NULL;
1931 }
1932
1933 struct vfio_info_cap_header *
1934 vfio_get_region_info_cap(struct vfio_region_info *info, uint16_t id)
1935 {
1936 if (!(info->flags & VFIO_REGION_INFO_FLAG_CAPS)) {
1937 return NULL;
1938 }
1939
1940 return vfio_get_cap((void *)info, info->cap_offset, id);
1941 }
1942
1943 static struct vfio_info_cap_header *
1944 vfio_get_iommu_type1_info_cap(struct vfio_iommu_type1_info *info, uint16_t id)
1945 {
1946 if (!(info->flags & VFIO_IOMMU_INFO_CAPS)) {
1947 return NULL;
1948 }
1949
1950 return vfio_get_cap((void *)info, info->cap_offset, id);
1951 }
1952
1953 struct vfio_info_cap_header *
1954 vfio_get_device_info_cap(struct vfio_device_info *info, uint16_t id)
1955 {
1956 if (!(info->flags & VFIO_DEVICE_FLAGS_CAPS)) {
1957 return NULL;
1958 }
1959
1960 return vfio_get_cap((void *)info, info->cap_offset, id);
1961 }
1962
1963 bool vfio_get_info_dma_avail(struct vfio_iommu_type1_info *info,
1964 unsigned int *avail)
1965 {
1966 struct vfio_info_cap_header *hdr;
1967 struct vfio_iommu_type1_info_dma_avail *cap;
1968
1969 /* If the capability cannot be found, assume no DMA limiting */
1970 hdr = vfio_get_iommu_type1_info_cap(info,
1971 VFIO_IOMMU_TYPE1_INFO_DMA_AVAIL);
1972 if (hdr == NULL) {
1973 return false;
1974 }
1975
1976 if (avail != NULL) {
1977 cap = (void *) hdr;
1978 *avail = cap->avail;
1979 }
1980
1981 return true;
1982 }
1983
1984 static int vfio_setup_region_sparse_mmaps(VFIORegion *region,
1985 struct vfio_region_info *info)
1986 {
1987 struct vfio_info_cap_header *hdr;
1988 struct vfio_region_info_cap_sparse_mmap *sparse;
1989 int i, j;
1990
1991 hdr = vfio_get_region_info_cap(info, VFIO_REGION_INFO_CAP_SPARSE_MMAP);
1992 if (!hdr) {
1993 return -ENODEV;
1994 }
1995
1996 sparse = container_of(hdr, struct vfio_region_info_cap_sparse_mmap, header);
1997
1998 trace_vfio_region_sparse_mmap_header(region->vbasedev->name,
1999 region->nr, sparse->nr_areas);
2000
2001 region->mmaps = g_new0(VFIOMmap, sparse->nr_areas);
2002
2003 for (i = 0, j = 0; i < sparse->nr_areas; i++) {
2004 if (sparse->areas[i].size) {
2005 trace_vfio_region_sparse_mmap_entry(i, sparse->areas[i].offset,
2006 sparse->areas[i].offset +
2007 sparse->areas[i].size - 1);
2008 region->mmaps[j].offset = sparse->areas[i].offset;
2009 region->mmaps[j].size = sparse->areas[i].size;
2010 j++;
2011 }
2012 }
2013
2014 region->nr_mmaps = j;
2015 region->mmaps = g_realloc(region->mmaps, j * sizeof(VFIOMmap));
2016
2017 return 0;
2018 }
2019
2020 int vfio_region_setup(Object *obj, VFIODevice *vbasedev, VFIORegion *region,
2021 int index, const char *name)
2022 {
2023 struct vfio_region_info *info;
2024 int ret;
2025
2026 ret = vfio_get_region_info(vbasedev, index, &info);
2027 if (ret) {
2028 return ret;
2029 }
2030
2031 region->vbasedev = vbasedev;
2032 region->flags = info->flags;
2033 region->size = info->size;
2034 region->fd_offset = info->offset;
2035 region->nr = index;
2036
2037 if (region->size) {
2038 region->mem = g_new0(MemoryRegion, 1);
2039 memory_region_init_io(region->mem, obj, &vfio_region_ops,
2040 region, name, region->size);
2041
2042 if (!vbasedev->no_mmap &&
2043 region->flags & VFIO_REGION_INFO_FLAG_MMAP) {
2044
2045 ret = vfio_setup_region_sparse_mmaps(region, info);
2046
2047 if (ret) {
2048 region->nr_mmaps = 1;
2049 region->mmaps = g_new0(VFIOMmap, region->nr_mmaps);
2050 region->mmaps[0].offset = 0;
2051 region->mmaps[0].size = region->size;
2052 }
2053 }
2054 }
2055
2056 g_free(info);
2057
2058 trace_vfio_region_setup(vbasedev->name, index, name,
2059 region->flags, region->fd_offset, region->size);
2060 return 0;
2061 }
2062
2063 static void vfio_subregion_unmap(VFIORegion *region, int index)
2064 {
2065 trace_vfio_region_unmap(memory_region_name(&region->mmaps[index].mem),
2066 region->mmaps[index].offset,
2067 region->mmaps[index].offset +
2068 region->mmaps[index].size - 1);
2069 memory_region_del_subregion(region->mem, &region->mmaps[index].mem);
2070 munmap(region->mmaps[index].mmap, region->mmaps[index].size);
2071 object_unparent(OBJECT(&region->mmaps[index].mem));
2072 region->mmaps[index].mmap = NULL;
2073 }
2074
2075 int vfio_region_mmap(VFIORegion *region)
2076 {
2077 int i, prot = 0;
2078 char *name;
2079
2080 if (!region->mem) {
2081 return 0;
2082 }
2083
2084 prot |= region->flags & VFIO_REGION_INFO_FLAG_READ ? PROT_READ : 0;
2085 prot |= region->flags & VFIO_REGION_INFO_FLAG_WRITE ? PROT_WRITE : 0;
2086
2087 for (i = 0; i < region->nr_mmaps; i++) {
2088 region->mmaps[i].mmap = mmap(NULL, region->mmaps[i].size, prot,
2089 MAP_SHARED, region->vbasedev->fd,
2090 region->fd_offset +
2091 region->mmaps[i].offset);
2092 if (region->mmaps[i].mmap == MAP_FAILED) {
2093 int ret = -errno;
2094
2095 trace_vfio_region_mmap_fault(memory_region_name(region->mem), i,
2096 region->fd_offset +
2097 region->mmaps[i].offset,
2098 region->fd_offset +
2099 region->mmaps[i].offset +
2100 region->mmaps[i].size - 1, ret);
2101
2102 region->mmaps[i].mmap = NULL;
2103
2104 for (i--; i >= 0; i--) {
2105 vfio_subregion_unmap(region, i);
2106 }
2107
2108 return ret;
2109 }
2110
2111 name = g_strdup_printf("%s mmaps[%d]",
2112 memory_region_name(region->mem), i);
2113 memory_region_init_ram_device_ptr(&region->mmaps[i].mem,
2114 memory_region_owner(region->mem),
2115 name, region->mmaps[i].size,
2116 region->mmaps[i].mmap);
2117 g_free(name);
2118 memory_region_add_subregion(region->mem, region->mmaps[i].offset,
2119 &region->mmaps[i].mem);
2120
2121 trace_vfio_region_mmap(memory_region_name(&region->mmaps[i].mem),
2122 region->mmaps[i].offset,
2123 region->mmaps[i].offset +
2124 region->mmaps[i].size - 1);
2125 }
2126
2127 return 0;
2128 }
2129
2130 void vfio_region_unmap(VFIORegion *region)
2131 {
2132 int i;
2133
2134 if (!region->mem) {
2135 return;
2136 }
2137
2138 for (i = 0; i < region->nr_mmaps; i++) {
2139 if (region->mmaps[i].mmap) {
2140 vfio_subregion_unmap(region, i);
2141 }
2142 }
2143 }
2144
2145 void vfio_region_exit(VFIORegion *region)
2146 {
2147 int i;
2148
2149 if (!region->mem) {
2150 return;
2151 }
2152
2153 for (i = 0; i < region->nr_mmaps; i++) {
2154 if (region->mmaps[i].mmap) {
2155 memory_region_del_subregion(region->mem, &region->mmaps[i].mem);
2156 }
2157 }
2158
2159 trace_vfio_region_exit(region->vbasedev->name, region->nr);
2160 }
2161
2162 void vfio_region_finalize(VFIORegion *region)
2163 {
2164 int i;
2165
2166 if (!region->mem) {
2167 return;
2168 }
2169
2170 for (i = 0; i < region->nr_mmaps; i++) {
2171 if (region->mmaps[i].mmap) {
2172 munmap(region->mmaps[i].mmap, region->mmaps[i].size);
2173 object_unparent(OBJECT(&region->mmaps[i].mem));
2174 }
2175 }
2176
2177 object_unparent(OBJECT(region->mem));
2178
2179 g_free(region->mem);
2180 g_free(region->mmaps);
2181
2182 trace_vfio_region_finalize(region->vbasedev->name, region->nr);
2183
2184 region->mem = NULL;
2185 region->mmaps = NULL;
2186 region->nr_mmaps = 0;
2187 region->size = 0;
2188 region->flags = 0;
2189 region->nr = 0;
2190 }
2191
2192 void vfio_region_mmaps_set_enabled(VFIORegion *region, bool enabled)
2193 {
2194 int i;
2195
2196 if (!region->mem) {
2197 return;
2198 }
2199
2200 for (i = 0; i < region->nr_mmaps; i++) {
2201 if (region->mmaps[i].mmap) {
2202 memory_region_set_enabled(&region->mmaps[i].mem, enabled);
2203 }
2204 }
2205
2206 trace_vfio_region_mmaps_set_enabled(memory_region_name(region->mem),
2207 enabled);
2208 }
2209
2210 void vfio_reset_handler(void *opaque)
2211 {
2212 VFIOGroup *group;
2213 VFIODevice *vbasedev;
2214
2215 QLIST_FOREACH(group, &vfio_group_list, next) {
2216 QLIST_FOREACH(vbasedev, &group->device_list, next) {
2217 if (vbasedev->dev->realized) {
2218 vbasedev->ops->vfio_compute_needs_reset(vbasedev);
2219 }
2220 }
2221 }
2222
2223 QLIST_FOREACH(group, &vfio_group_list, next) {
2224 QLIST_FOREACH(vbasedev, &group->device_list, next) {
2225 if (vbasedev->dev->realized && vbasedev->needs_reset) {
2226 vbasedev->ops->vfio_hot_reset_multi(vbasedev);
2227 }
2228 }
2229 }
2230 }
2231
2232 static void vfio_kvm_device_add_group(VFIOGroup *group)
2233 {
2234 #ifdef CONFIG_KVM
2235 struct kvm_device_attr attr = {
2236 .group = KVM_DEV_VFIO_GROUP,
2237 .attr = KVM_DEV_VFIO_GROUP_ADD,
2238 .addr = (uint64_t)(unsigned long)&group->fd,
2239 };
2240
2241 if (!kvm_enabled()) {
2242 return;
2243 }
2244
2245 if (vfio_kvm_device_fd < 0) {
2246 struct kvm_create_device cd = {
2247 .type = KVM_DEV_TYPE_VFIO,
2248 };
2249
2250 if (kvm_vm_ioctl(kvm_state, KVM_CREATE_DEVICE, &cd)) {
2251 error_report("Failed to create KVM VFIO device: %m");
2252 return;
2253 }
2254
2255 vfio_kvm_device_fd = cd.fd;
2256 }
2257
2258 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) {
2259 error_report("Failed to add group %d to KVM VFIO device: %m",
2260 group->groupid);
2261 }
2262 #endif
2263 }
2264
2265 static void vfio_kvm_device_del_group(VFIOGroup *group)
2266 {
2267 #ifdef CONFIG_KVM
2268 struct kvm_device_attr attr = {
2269 .group = KVM_DEV_VFIO_GROUP,
2270 .attr = KVM_DEV_VFIO_GROUP_DEL,
2271 .addr = (uint64_t)(unsigned long)&group->fd,
2272 };
2273
2274 if (vfio_kvm_device_fd < 0) {
2275 return;
2276 }
2277
2278 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) {
2279 error_report("Failed to remove group %d from KVM VFIO device: %m",
2280 group->groupid);
2281 }
2282 #endif
2283 }
2284
2285 static VFIOAddressSpace *vfio_get_address_space(AddressSpace *as)
2286 {
2287 VFIOAddressSpace *space;
2288
2289 QLIST_FOREACH(space, &vfio_address_spaces, list) {
2290 if (space->as == as) {
2291 return space;
2292 }
2293 }
2294
2295 /* No suitable VFIOAddressSpace, create a new one */
2296 space = g_malloc0(sizeof(*space));
2297 space->as = as;
2298 QLIST_INIT(&space->containers);
2299
2300 QLIST_INSERT_HEAD(&vfio_address_spaces, space, list);
2301
2302 return space;
2303 }
2304
2305 static void vfio_put_address_space(VFIOAddressSpace *space)
2306 {
2307 if (QLIST_EMPTY(&space->containers)) {
2308 QLIST_REMOVE(space, list);
2309 g_free(space);
2310 }
2311 }
2312
2313 /*
2314 * vfio_get_iommu_type - selects the richest iommu_type (v2 first)
2315 */
2316 static int vfio_get_iommu_type(VFIOContainer *container,
2317 Error **errp)
2318 {
2319 int iommu_types[] = { VFIO_TYPE1v2_IOMMU, VFIO_TYPE1_IOMMU,
2320 VFIO_SPAPR_TCE_v2_IOMMU, VFIO_SPAPR_TCE_IOMMU };
2321 int i;
2322
2323 for (i = 0; i < ARRAY_SIZE(iommu_types); i++) {
2324 if (ioctl(container->fd, VFIO_CHECK_EXTENSION, iommu_types[i])) {
2325 return iommu_types[i];
2326 }
2327 }
2328 error_setg(errp, "No available IOMMU models");
2329 return -EINVAL;
2330 }
2331
2332 static int vfio_init_container(VFIOContainer *container, int group_fd,
2333 Error **errp)
2334 {
2335 int iommu_type, ret;
2336
2337 iommu_type = vfio_get_iommu_type(container, errp);
2338 if (iommu_type < 0) {
2339 return iommu_type;
2340 }
2341
2342 ret = ioctl(group_fd, VFIO_GROUP_SET_CONTAINER, &container->fd);
2343 if (ret) {
2344 error_setg_errno(errp, errno, "Failed to set group container");
2345 return -errno;
2346 }
2347
2348 while (ioctl(container->fd, VFIO_SET_IOMMU, iommu_type)) {
2349 if (iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) {
2350 /*
2351 * On sPAPR, despite the IOMMU subdriver always advertises v1 and
2352 * v2, the running platform may not support v2 and there is no
2353 * way to guess it until an IOMMU group gets added to the container.
2354 * So in case it fails with v2, try v1 as a fallback.
2355 */
2356 iommu_type = VFIO_SPAPR_TCE_IOMMU;
2357 continue;
2358 }
2359 error_setg_errno(errp, errno, "Failed to set iommu for container");
2360 return -errno;
2361 }
2362
2363 container->iommu_type = iommu_type;
2364 return 0;
2365 }
2366
2367 static int vfio_get_iommu_info(VFIOContainer *container,
2368 struct vfio_iommu_type1_info **info)
2369 {
2370
2371 size_t argsz = sizeof(struct vfio_iommu_type1_info);
2372
2373 *info = g_new0(struct vfio_iommu_type1_info, 1);
2374 again:
2375 (*info)->argsz = argsz;
2376
2377 if (ioctl(container->fd, VFIO_IOMMU_GET_INFO, *info)) {
2378 g_free(*info);
2379 *info = NULL;
2380 return -errno;
2381 }
2382
2383 if (((*info)->argsz > argsz)) {
2384 argsz = (*info)->argsz;
2385 *info = g_realloc(*info, argsz);
2386 goto again;
2387 }
2388
2389 return 0;
2390 }
2391
2392 static struct vfio_info_cap_header *
2393 vfio_get_iommu_info_cap(struct vfio_iommu_type1_info *info, uint16_t id)
2394 {
2395 struct vfio_info_cap_header *hdr;
2396 void *ptr = info;
2397
2398 if (!(info->flags & VFIO_IOMMU_INFO_CAPS)) {
2399 return NULL;
2400 }
2401
2402 for (hdr = ptr + info->cap_offset; hdr != ptr; hdr = ptr + hdr->next) {
2403 if (hdr->id == id) {
2404 return hdr;
2405 }
2406 }
2407
2408 return NULL;
2409 }
2410
2411 static void vfio_get_iommu_info_migration(VFIOContainer *container,
2412 struct vfio_iommu_type1_info *info)
2413 {
2414 struct vfio_info_cap_header *hdr;
2415 struct vfio_iommu_type1_info_cap_migration *cap_mig;
2416
2417 hdr = vfio_get_iommu_info_cap(info, VFIO_IOMMU_TYPE1_INFO_CAP_MIGRATION);
2418 if (!hdr) {
2419 return;
2420 }
2421
2422 cap_mig = container_of(hdr, struct vfio_iommu_type1_info_cap_migration,
2423 header);
2424
2425 /*
2426 * cpu_physical_memory_set_dirty_lebitmap() supports pages in bitmap of
2427 * qemu_real_host_page_size to mark those dirty.
2428 */
2429 if (cap_mig->pgsize_bitmap & qemu_real_host_page_size()) {
2430 container->dirty_pages_supported = true;
2431 container->max_dirty_bitmap_size = cap_mig->max_dirty_bitmap_size;
2432 container->dirty_pgsizes = cap_mig->pgsize_bitmap;
2433 }
2434 }
2435
2436 static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
2437 Error **errp)
2438 {
2439 VFIOContainer *container;
2440 int ret, fd;
2441 VFIOAddressSpace *space;
2442
2443 space = vfio_get_address_space(as);
2444
2445 /*
2446 * VFIO is currently incompatible with discarding of RAM insofar as the
2447 * madvise to purge (zap) the page from QEMU's address space does not
2448 * interact with the memory API and therefore leaves stale virtual to
2449 * physical mappings in the IOMMU if the page was previously pinned. We
2450 * therefore set discarding broken for each group added to a container,
2451 * whether the container is used individually or shared. This provides
2452 * us with options to allow devices within a group to opt-in and allow
2453 * discarding, so long as it is done consistently for a group (for instance
2454 * if the device is an mdev device where it is known that the host vendor
2455 * driver will never pin pages outside of the working set of the guest
2456 * driver, which would thus not be discarding candidates).
2457 *
2458 * The first opportunity to induce pinning occurs here where we attempt to
2459 * attach the group to existing containers within the AddressSpace. If any
2460 * pages are already zapped from the virtual address space, such as from
2461 * previous discards, new pinning will cause valid mappings to be
2462 * re-established. Likewise, when the overall MemoryListener for a new
2463 * container is registered, a replay of mappings within the AddressSpace
2464 * will occur, re-establishing any previously zapped pages as well.
2465 *
2466 * Especially virtio-balloon is currently only prevented from discarding
2467 * new memory, it will not yet set ram_block_discard_set_required() and
2468 * therefore, neither stops us here or deals with the sudden memory
2469 * consumption of inflated memory.
2470 *
2471 * We do support discarding of memory coordinated via the RamDiscardManager
2472 * with some IOMMU types. vfio_ram_block_discard_disable() handles the
2473 * details once we know which type of IOMMU we are using.
2474 */
2475
2476 QLIST_FOREACH(container, &space->containers, next) {
2477 if (!ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &container->fd)) {
2478 ret = vfio_ram_block_discard_disable(container, true);
2479 if (ret) {
2480 error_setg_errno(errp, -ret,
2481 "Cannot set discarding of RAM broken");
2482 if (ioctl(group->fd, VFIO_GROUP_UNSET_CONTAINER,
2483 &container->fd)) {
2484 error_report("vfio: error disconnecting group %d from"
2485 " container", group->groupid);
2486 }
2487 return ret;
2488 }
2489 group->container = container;
2490 QLIST_INSERT_HEAD(&container->group_list, group, container_next);
2491 vfio_kvm_device_add_group(group);
2492 return 0;
2493 }
2494 }
2495
2496 fd = qemu_open_old("/dev/vfio/vfio", O_RDWR);
2497 if (fd < 0) {
2498 error_setg_errno(errp, errno, "failed to open /dev/vfio/vfio");
2499 ret = -errno;
2500 goto put_space_exit;
2501 }
2502
2503 ret = ioctl(fd, VFIO_GET_API_VERSION);
2504 if (ret != VFIO_API_VERSION) {
2505 error_setg(errp, "supported vfio version: %d, "
2506 "reported version: %d", VFIO_API_VERSION, ret);
2507 ret = -EINVAL;
2508 goto close_fd_exit;
2509 }
2510
2511 container = g_malloc0(sizeof(*container));
2512 container->space = space;
2513 container->fd = fd;
2514 container->error = NULL;
2515 container->dirty_pages_supported = false;
2516 container->dma_max_mappings = 0;
2517 QLIST_INIT(&container->giommu_list);
2518 QLIST_INIT(&container->hostwin_list);
2519 QLIST_INIT(&container->vrdl_list);
2520
2521 ret = vfio_init_container(container, group->fd, errp);
2522 if (ret) {
2523 goto free_container_exit;
2524 }
2525
2526 ret = vfio_ram_block_discard_disable(container, true);
2527 if (ret) {
2528 error_setg_errno(errp, -ret, "Cannot set discarding of RAM broken");
2529 goto free_container_exit;
2530 }
2531
2532 switch (container->iommu_type) {
2533 case VFIO_TYPE1v2_IOMMU:
2534 case VFIO_TYPE1_IOMMU:
2535 {
2536 struct vfio_iommu_type1_info *info;
2537
2538 ret = vfio_get_iommu_info(container, &info);
2539 if (ret) {
2540 error_setg_errno(errp, -ret, "Failed to get VFIO IOMMU info");
2541 goto enable_discards_exit;
2542 }
2543
2544 if (info->flags & VFIO_IOMMU_INFO_PGSIZES) {
2545 container->pgsizes = info->iova_pgsizes;
2546 } else {
2547 container->pgsizes = qemu_real_host_page_size();
2548 }
2549
2550 if (!vfio_get_info_dma_avail(info, &container->dma_max_mappings)) {
2551 container->dma_max_mappings = 65535;
2552 }
2553 vfio_get_iommu_info_migration(container, info);
2554 g_free(info);
2555
2556 /*
2557 * FIXME: We should parse VFIO_IOMMU_TYPE1_INFO_CAP_IOVA_RANGE
2558 * information to get the actual window extent rather than assume
2559 * a 64-bit IOVA address space.
2560 */
2561 vfio_host_win_add(container, 0, (hwaddr)-1, container->pgsizes);
2562
2563 break;
2564 }
2565 case VFIO_SPAPR_TCE_v2_IOMMU:
2566 case VFIO_SPAPR_TCE_IOMMU:
2567 {
2568 struct vfio_iommu_spapr_tce_info info;
2569 bool v2 = container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU;
2570
2571 /*
2572 * The host kernel code implementing VFIO_IOMMU_DISABLE is called
2573 * when container fd is closed so we do not call it explicitly
2574 * in this file.
2575 */
2576 if (!v2) {
2577 ret = ioctl(fd, VFIO_IOMMU_ENABLE);
2578 if (ret) {
2579 error_setg_errno(errp, errno, "failed to enable container");
2580 ret = -errno;
2581 goto enable_discards_exit;
2582 }
2583 } else {
2584 container->prereg_listener = vfio_prereg_listener;
2585
2586 memory_listener_register(&container->prereg_listener,
2587 &address_space_memory);
2588 if (container->error) {
2589 memory_listener_unregister(&container->prereg_listener);
2590 ret = -1;
2591 error_propagate_prepend(errp, container->error,
2592 "RAM memory listener initialization failed: ");
2593 goto enable_discards_exit;
2594 }
2595 }
2596
2597 info.argsz = sizeof(info);
2598 ret = ioctl(fd, VFIO_IOMMU_SPAPR_TCE_GET_INFO, &info);
2599 if (ret) {
2600 error_setg_errno(errp, errno,
2601 "VFIO_IOMMU_SPAPR_TCE_GET_INFO failed");
2602 ret = -errno;
2603 if (v2) {
2604 memory_listener_unregister(&container->prereg_listener);
2605 }
2606 goto enable_discards_exit;
2607 }
2608
2609 if (v2) {
2610 container->pgsizes = info.ddw.pgsizes;
2611 /*
2612 * There is a default window in just created container.
2613 * To make region_add/del simpler, we better remove this
2614 * window now and let those iommu_listener callbacks
2615 * create/remove them when needed.
2616 */
2617 ret = vfio_spapr_remove_window(container, info.dma32_window_start);
2618 if (ret) {
2619 error_setg_errno(errp, -ret,
2620 "failed to remove existing window");
2621 goto enable_discards_exit;
2622 }
2623 } else {
2624 /* The default table uses 4K pages */
2625 container->pgsizes = 0x1000;
2626 vfio_host_win_add(container, info.dma32_window_start,
2627 info.dma32_window_start +
2628 info.dma32_window_size - 1,
2629 0x1000);
2630 }
2631 }
2632 }
2633
2634 vfio_kvm_device_add_group(group);
2635
2636 QLIST_INIT(&container->group_list);
2637 QLIST_INSERT_HEAD(&space->containers, container, next);
2638
2639 group->container = container;
2640 QLIST_INSERT_HEAD(&container->group_list, group, container_next);
2641
2642 container->listener = vfio_memory_listener;
2643
2644 memory_listener_register(&container->listener, container->space->as);
2645
2646 if (container->error) {
2647 ret = -1;
2648 error_propagate_prepend(errp, container->error,
2649 "memory listener initialization failed: ");
2650 goto listener_release_exit;
2651 }
2652
2653 container->initialized = true;
2654
2655 return 0;
2656 listener_release_exit:
2657 QLIST_REMOVE(group, container_next);
2658 QLIST_REMOVE(container, next);
2659 vfio_kvm_device_del_group(group);
2660 vfio_listener_release(container);
2661
2662 enable_discards_exit:
2663 vfio_ram_block_discard_disable(container, false);
2664
2665 free_container_exit:
2666 g_free(container);
2667
2668 close_fd_exit:
2669 close(fd);
2670
2671 put_space_exit:
2672 vfio_put_address_space(space);
2673
2674 return ret;
2675 }
2676
2677 static void vfio_disconnect_container(VFIOGroup *group)
2678 {
2679 VFIOContainer *container = group->container;
2680
2681 QLIST_REMOVE(group, container_next);
2682 group->container = NULL;
2683
2684 /*
2685 * Explicitly release the listener first before unset container,
2686 * since unset may destroy the backend container if it's the last
2687 * group.
2688 */
2689 if (QLIST_EMPTY(&container->group_list)) {
2690 vfio_listener_release(container);
2691 }
2692
2693 if (ioctl(group->fd, VFIO_GROUP_UNSET_CONTAINER, &container->fd)) {
2694 error_report("vfio: error disconnecting group %d from container",
2695 group->groupid);
2696 }
2697
2698 if (QLIST_EMPTY(&container->group_list)) {
2699 VFIOAddressSpace *space = container->space;
2700 VFIOGuestIOMMU *giommu, *tmp;
2701 VFIOHostDMAWindow *hostwin, *next;
2702
2703 QLIST_REMOVE(container, next);
2704
2705 QLIST_FOREACH_SAFE(giommu, &container->giommu_list, giommu_next, tmp) {
2706 memory_region_unregister_iommu_notifier(
2707 MEMORY_REGION(giommu->iommu_mr), &giommu->n);
2708 QLIST_REMOVE(giommu, giommu_next);
2709 g_free(giommu);
2710 }
2711
2712 QLIST_FOREACH_SAFE(hostwin, &container->hostwin_list, hostwin_next,
2713 next) {
2714 QLIST_REMOVE(hostwin, hostwin_next);
2715 g_free(hostwin);
2716 }
2717
2718 trace_vfio_disconnect_container(container->fd);
2719 close(container->fd);
2720 g_free(container);
2721
2722 vfio_put_address_space(space);
2723 }
2724 }
2725
2726 VFIOGroup *vfio_get_group(int groupid, AddressSpace *as, Error **errp)
2727 {
2728 VFIOGroup *group;
2729 char path[32];
2730 struct vfio_group_status status = { .argsz = sizeof(status) };
2731
2732 QLIST_FOREACH(group, &vfio_group_list, next) {
2733 if (group->groupid == groupid) {
2734 /* Found it. Now is it already in the right context? */
2735 if (group->container->space->as == as) {
2736 return group;
2737 } else {
2738 error_setg(errp, "group %d used in multiple address spaces",
2739 group->groupid);
2740 return NULL;
2741 }
2742 }
2743 }
2744
2745 group = g_malloc0(sizeof(*group));
2746
2747 snprintf(path, sizeof(path), "/dev/vfio/%d", groupid);
2748 group->fd = qemu_open_old(path, O_RDWR);
2749 if (group->fd < 0) {
2750 error_setg_errno(errp, errno, "failed to open %s", path);
2751 goto free_group_exit;
2752 }
2753
2754 if (ioctl(group->fd, VFIO_GROUP_GET_STATUS, &status)) {
2755 error_setg_errno(errp, errno, "failed to get group %d status", groupid);
2756 goto close_fd_exit;
2757 }
2758
2759 if (!(status.flags & VFIO_GROUP_FLAGS_VIABLE)) {
2760 error_setg(errp, "group %d is not viable", groupid);
2761 error_append_hint(errp,
2762 "Please ensure all devices within the iommu_group "
2763 "are bound to their vfio bus driver.\n");
2764 goto close_fd_exit;
2765 }
2766
2767 group->groupid = groupid;
2768 QLIST_INIT(&group->device_list);
2769
2770 if (vfio_connect_container(group, as, errp)) {
2771 error_prepend(errp, "failed to setup container for group %d: ",
2772 groupid);
2773 goto close_fd_exit;
2774 }
2775
2776 if (QLIST_EMPTY(&vfio_group_list)) {
2777 qemu_register_reset(vfio_reset_handler, NULL);
2778 }
2779
2780 QLIST_INSERT_HEAD(&vfio_group_list, group, next);
2781
2782 return group;
2783
2784 close_fd_exit:
2785 close(group->fd);
2786
2787 free_group_exit:
2788 g_free(group);
2789
2790 return NULL;
2791 }
2792
2793 void vfio_put_group(VFIOGroup *group)
2794 {
2795 if (!group || !QLIST_EMPTY(&group->device_list)) {
2796 return;
2797 }
2798
2799 if (!group->ram_block_discard_allowed) {
2800 vfio_ram_block_discard_disable(group->container, false);
2801 }
2802 vfio_kvm_device_del_group(group);
2803 vfio_disconnect_container(group);
2804 QLIST_REMOVE(group, next);
2805 trace_vfio_put_group(group->fd);
2806 close(group->fd);
2807 g_free(group);
2808
2809 if (QLIST_EMPTY(&vfio_group_list)) {
2810 qemu_unregister_reset(vfio_reset_handler, NULL);
2811 }
2812 }
2813
2814 struct vfio_device_info *vfio_get_device_info(int fd)
2815 {
2816 struct vfio_device_info *info;
2817 uint32_t argsz = sizeof(*info);
2818
2819 info = g_malloc0(argsz);
2820
2821 retry:
2822 info->argsz = argsz;
2823
2824 if (ioctl(fd, VFIO_DEVICE_GET_INFO, info)) {
2825 g_free(info);
2826 return NULL;
2827 }
2828
2829 if (info->argsz > argsz) {
2830 argsz = info->argsz;
2831 info = g_realloc(info, argsz);
2832 goto retry;
2833 }
2834
2835 return info;
2836 }
2837
2838 int vfio_get_device(VFIOGroup *group, const char *name,
2839 VFIODevice *vbasedev, Error **errp)
2840 {
2841 g_autofree struct vfio_device_info *info = NULL;
2842 int fd;
2843
2844 fd = ioctl(group->fd, VFIO_GROUP_GET_DEVICE_FD, name);
2845 if (fd < 0) {
2846 error_setg_errno(errp, errno, "error getting device from group %d",
2847 group->groupid);
2848 error_append_hint(errp,
2849 "Verify all devices in group %d are bound to vfio-<bus> "
2850 "or pci-stub and not already in use\n", group->groupid);
2851 return fd;
2852 }
2853
2854 info = vfio_get_device_info(fd);
2855 if (!info) {
2856 error_setg_errno(errp, errno, "error getting device info");
2857 close(fd);
2858 return -1;
2859 }
2860
2861 /*
2862 * Set discarding of RAM as not broken for this group if the driver knows
2863 * the device operates compatibly with discarding. Setting must be
2864 * consistent per group, but since compatibility is really only possible
2865 * with mdev currently, we expect singleton groups.
2866 */
2867 if (vbasedev->ram_block_discard_allowed !=
2868 group->ram_block_discard_allowed) {
2869 if (!QLIST_EMPTY(&group->device_list)) {
2870 error_setg(errp, "Inconsistent setting of support for discarding "
2871 "RAM (e.g., balloon) within group");
2872 close(fd);
2873 return -1;
2874 }
2875
2876 if (!group->ram_block_discard_allowed) {
2877 group->ram_block_discard_allowed = true;
2878 vfio_ram_block_discard_disable(group->container, false);
2879 }
2880 }
2881
2882 vbasedev->fd = fd;
2883 vbasedev->group = group;
2884 QLIST_INSERT_HEAD(&group->device_list, vbasedev, next);
2885
2886 vbasedev->num_irqs = info->num_irqs;
2887 vbasedev->num_regions = info->num_regions;
2888 vbasedev->flags = info->flags;
2889
2890 trace_vfio_get_device(name, info->flags, info->num_regions, info->num_irqs);
2891
2892 vbasedev->reset_works = !!(info->flags & VFIO_DEVICE_FLAGS_RESET);
2893
2894 return 0;
2895 }
2896
2897 void vfio_put_base_device(VFIODevice *vbasedev)
2898 {
2899 if (!vbasedev->group) {
2900 return;
2901 }
2902 QLIST_REMOVE(vbasedev, next);
2903 vbasedev->group = NULL;
2904 trace_vfio_put_base_device(vbasedev->fd);
2905 close(vbasedev->fd);
2906 }
2907
2908 int vfio_get_region_info(VFIODevice *vbasedev, int index,
2909 struct vfio_region_info **info)
2910 {
2911 size_t argsz = sizeof(struct vfio_region_info);
2912
2913 *info = g_malloc0(argsz);
2914
2915 (*info)->index = index;
2916 retry:
2917 (*info)->argsz = argsz;
2918
2919 if (ioctl(vbasedev->fd, VFIO_DEVICE_GET_REGION_INFO, *info)) {
2920 g_free(*info);
2921 *info = NULL;
2922 return -errno;
2923 }
2924
2925 if ((*info)->argsz > argsz) {
2926 argsz = (*info)->argsz;
2927 *info = g_realloc(*info, argsz);
2928
2929 goto retry;
2930 }
2931
2932 return 0;
2933 }
2934
2935 int vfio_get_dev_region_info(VFIODevice *vbasedev, uint32_t type,
2936 uint32_t subtype, struct vfio_region_info **info)
2937 {
2938 int i;
2939
2940 for (i = 0; i < vbasedev->num_regions; i++) {
2941 struct vfio_info_cap_header *hdr;
2942 struct vfio_region_info_cap_type *cap_type;
2943
2944 if (vfio_get_region_info(vbasedev, i, info)) {
2945 continue;
2946 }
2947
2948 hdr = vfio_get_region_info_cap(*info, VFIO_REGION_INFO_CAP_TYPE);
2949 if (!hdr) {
2950 g_free(*info);
2951 continue;
2952 }
2953
2954 cap_type = container_of(hdr, struct vfio_region_info_cap_type, header);
2955
2956 trace_vfio_get_dev_region(vbasedev->name, i,
2957 cap_type->type, cap_type->subtype);
2958
2959 if (cap_type->type == type && cap_type->subtype == subtype) {
2960 return 0;
2961 }
2962
2963 g_free(*info);
2964 }
2965
2966 *info = NULL;
2967 return -ENODEV;
2968 }
2969
2970 bool vfio_has_region_cap(VFIODevice *vbasedev, int region, uint16_t cap_type)
2971 {
2972 struct vfio_region_info *info = NULL;
2973 bool ret = false;
2974
2975 if (!vfio_get_region_info(vbasedev, region, &info)) {
2976 if (vfio_get_region_info_cap(info, cap_type)) {
2977 ret = true;
2978 }
2979 g_free(info);
2980 }
2981
2982 return ret;
2983 }
2984
2985 /*
2986 * Interfaces for IBM EEH (Enhanced Error Handling)
2987 */
2988 static bool vfio_eeh_container_ok(VFIOContainer *container)
2989 {
2990 /*
2991 * As of 2016-03-04 (linux-4.5) the host kernel EEH/VFIO
2992 * implementation is broken if there are multiple groups in a
2993 * container. The hardware works in units of Partitionable
2994 * Endpoints (== IOMMU groups) and the EEH operations naively
2995 * iterate across all groups in the container, without any logic
2996 * to make sure the groups have their state synchronized. For
2997 * certain operations (ENABLE) that might be ok, until an error
2998 * occurs, but for others (GET_STATE) it's clearly broken.
2999 */
3000
3001 /*
3002 * XXX Once fixed kernels exist, test for them here
3003 */
3004
3005 if (QLIST_EMPTY(&container->group_list)) {
3006 return false;
3007 }
3008
3009 if (QLIST_NEXT(QLIST_FIRST(&container->group_list), container_next)) {
3010 return false;
3011 }
3012
3013 return true;
3014 }
3015
3016 static int vfio_eeh_container_op(VFIOContainer *container, uint32_t op)
3017 {
3018 struct vfio_eeh_pe_op pe_op = {
3019 .argsz = sizeof(pe_op),
3020 .op = op,
3021 };
3022 int ret;
3023
3024 if (!vfio_eeh_container_ok(container)) {
3025 error_report("vfio/eeh: EEH_PE_OP 0x%x: "
3026 "kernel requires a container with exactly one group", op);
3027 return -EPERM;
3028 }
3029
3030 ret = ioctl(container->fd, VFIO_EEH_PE_OP, &pe_op);
3031 if (ret < 0) {
3032 error_report("vfio/eeh: EEH_PE_OP 0x%x failed: %m", op);
3033 return -errno;
3034 }
3035
3036 return ret;
3037 }
3038
3039 static VFIOContainer *vfio_eeh_as_container(AddressSpace *as)
3040 {
3041 VFIOAddressSpace *space = vfio_get_address_space(as);
3042 VFIOContainer *container = NULL;
3043
3044 if (QLIST_EMPTY(&space->containers)) {
3045 /* No containers to act on */
3046 goto out;
3047 }
3048
3049 container = QLIST_FIRST(&space->containers);
3050
3051 if (QLIST_NEXT(container, next)) {
3052 /* We don't yet have logic to synchronize EEH state across
3053 * multiple containers */
3054 container = NULL;
3055 goto out;
3056 }
3057
3058 out:
3059 vfio_put_address_space(space);
3060 return container;
3061 }
3062
3063 bool vfio_eeh_as_ok(AddressSpace *as)
3064 {
3065 VFIOContainer *container = vfio_eeh_as_container(as);
3066
3067 return (container != NULL) && vfio_eeh_container_ok(container);
3068 }
3069
3070 int vfio_eeh_as_op(AddressSpace *as, uint32_t op)
3071 {
3072 VFIOContainer *container = vfio_eeh_as_container(as);
3073
3074 if (!container) {
3075 return -ENODEV;
3076 }
3077 return vfio_eeh_container_op(container, op);
3078 }
armbru's QEMU hackery