2 * vfio based device assignment support
4 * Copyright Red Hat, Inc. 2012
7 * Alex Williamson <alex.williamson@redhat.com>
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
12 * Based on qemu-kvm device-assignment:
13 * Adapted for KVM by Qumranet.
14 * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com)
15 * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com)
16 * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com)
17 * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
18 * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
21 #include "qemu/osdep.h"
22 #include CONFIG_DEVICES /* CONFIG_IOMMUFD */
23 #include <linux/vfio.h>
24 #include <sys/ioctl.h>
27 #include "hw/pci/msi.h"
28 #include "hw/pci/msix.h"
29 #include "hw/pci/pci_bridge.h"
30 #include "hw/qdev-properties.h"
31 #include "hw/qdev-properties-system.h"
32 #include "migration/vmstate.h"
33 #include "qapi/qmp/qdict.h"
34 #include "qemu/error-report.h"
35 #include "qemu/main-loop.h"
36 #include "qemu/module.h"
37 #include "qemu/range.h"
38 #include "qemu/units.h"
39 #include "sysemu/kvm.h"
40 #include "sysemu/runstate.h"
43 #include "qapi/error.h"
44 #include "migration/blocker.h"
45 #include "migration/qemu-file.h"
46 #include "sysemu/iommufd.h"
48 #define TYPE_VFIO_PCI_NOHOTPLUG "vfio-pci-nohotplug"
50 /* Protected by BQL */
51 static KVMRouteChange vfio_route_change
;
53 static void vfio_disable_interrupts(VFIOPCIDevice
*vdev
);
54 static void vfio_mmap_set_enabled(VFIOPCIDevice
*vdev
, bool enabled
);
55 static void vfio_msi_disable_common(VFIOPCIDevice
*vdev
);
58 * Disabling BAR mmaping can be slow, but toggling it around INTx can
59 * also be a huge overhead. We try to get the best of both worlds by
60 * waiting until an interrupt to disable mmaps (subsequent transitions
61 * to the same state are effectively no overhead). If the interrupt has
62 * been serviced and the time gap is long enough, we re-enable mmaps for
63 * performance. This works well for things like graphics cards, which
64 * may not use their interrupt at all and are penalized to an unusable
65 * level by read/write BAR traps. Other devices, like NICs, have more
66 * regular interrupts and see much better latency by staying in non-mmap
67 * mode. We therefore set the default mmap_timeout such that a ping
68 * is just enough to keep the mmap disabled. Users can experiment with
69 * other options with the x-intx-mmap-timeout-ms parameter (a value of
70 * zero disables the timer).
72 static void vfio_intx_mmap_enable(void *opaque
)
74 VFIOPCIDevice
*vdev
= opaque
;
76 if (vdev
->intx
.pending
) {
77 timer_mod(vdev
->intx
.mmap_timer
,
78 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
) + vdev
->intx
.mmap_timeout
);
82 vfio_mmap_set_enabled(vdev
, true);
85 static void vfio_intx_interrupt(void *opaque
)
87 VFIOPCIDevice
*vdev
= opaque
;
89 if (!event_notifier_test_and_clear(&vdev
->intx
.interrupt
)) {
93 trace_vfio_intx_interrupt(vdev
->vbasedev
.name
, 'A' + vdev
->intx
.pin
);
95 vdev
->intx
.pending
= true;
96 pci_irq_assert(&vdev
->pdev
);
97 vfio_mmap_set_enabled(vdev
, false);
98 if (vdev
->intx
.mmap_timeout
) {
99 timer_mod(vdev
->intx
.mmap_timer
,
100 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
) + vdev
->intx
.mmap_timeout
);
104 static void vfio_intx_eoi(VFIODevice
*vbasedev
)
106 VFIOPCIDevice
*vdev
= container_of(vbasedev
, VFIOPCIDevice
, vbasedev
);
108 if (!vdev
->intx
.pending
) {
112 trace_vfio_intx_eoi(vbasedev
->name
);
114 vdev
->intx
.pending
= false;
115 pci_irq_deassert(&vdev
->pdev
);
116 vfio_unmask_single_irqindex(vbasedev
, VFIO_PCI_INTX_IRQ_INDEX
);
119 static void vfio_intx_enable_kvm(VFIOPCIDevice
*vdev
, Error
**errp
)
122 int irq_fd
= event_notifier_get_fd(&vdev
->intx
.interrupt
);
124 if (vdev
->no_kvm_intx
|| !kvm_irqfds_enabled() ||
125 vdev
->intx
.route
.mode
!= PCI_INTX_ENABLED
||
126 !kvm_resamplefds_enabled()) {
130 /* Get to a known interrupt state */
131 qemu_set_fd_handler(irq_fd
, NULL
, NULL
, vdev
);
132 vfio_mask_single_irqindex(&vdev
->vbasedev
, VFIO_PCI_INTX_IRQ_INDEX
);
133 vdev
->intx
.pending
= false;
134 pci_irq_deassert(&vdev
->pdev
);
136 /* Get an eventfd for resample/unmask */
137 if (event_notifier_init(&vdev
->intx
.unmask
, 0)) {
138 error_setg(errp
, "event_notifier_init failed eoi");
142 if (kvm_irqchip_add_irqfd_notifier_gsi(kvm_state
,
143 &vdev
->intx
.interrupt
,
145 vdev
->intx
.route
.irq
)) {
146 error_setg_errno(errp
, errno
, "failed to setup resample irqfd");
150 if (vfio_set_irq_signaling(&vdev
->vbasedev
, VFIO_PCI_INTX_IRQ_INDEX
, 0,
151 VFIO_IRQ_SET_ACTION_UNMASK
,
152 event_notifier_get_fd(&vdev
->intx
.unmask
),
158 vfio_unmask_single_irqindex(&vdev
->vbasedev
, VFIO_PCI_INTX_IRQ_INDEX
);
160 vdev
->intx
.kvm_accel
= true;
162 trace_vfio_intx_enable_kvm(vdev
->vbasedev
.name
);
167 kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state
, &vdev
->intx
.interrupt
,
168 vdev
->intx
.route
.irq
);
170 event_notifier_cleanup(&vdev
->intx
.unmask
);
172 qemu_set_fd_handler(irq_fd
, vfio_intx_interrupt
, NULL
, vdev
);
173 vfio_unmask_single_irqindex(&vdev
->vbasedev
, VFIO_PCI_INTX_IRQ_INDEX
);
177 static void vfio_intx_disable_kvm(VFIOPCIDevice
*vdev
)
180 if (!vdev
->intx
.kvm_accel
) {
185 * Get to a known state, hardware masked, QEMU ready to accept new
186 * interrupts, QEMU IRQ de-asserted.
188 vfio_mask_single_irqindex(&vdev
->vbasedev
, VFIO_PCI_INTX_IRQ_INDEX
);
189 vdev
->intx
.pending
= false;
190 pci_irq_deassert(&vdev
->pdev
);
192 /* Tell KVM to stop listening for an INTx irqfd */
193 if (kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state
, &vdev
->intx
.interrupt
,
194 vdev
->intx
.route
.irq
)) {
195 error_report("vfio: Error: Failed to disable INTx irqfd: %m");
198 /* We only need to close the eventfd for VFIO to cleanup the kernel side */
199 event_notifier_cleanup(&vdev
->intx
.unmask
);
201 /* QEMU starts listening for interrupt events. */
202 qemu_set_fd_handler(event_notifier_get_fd(&vdev
->intx
.interrupt
),
203 vfio_intx_interrupt
, NULL
, vdev
);
205 vdev
->intx
.kvm_accel
= false;
207 /* If we've missed an event, let it re-fire through QEMU */
208 vfio_unmask_single_irqindex(&vdev
->vbasedev
, VFIO_PCI_INTX_IRQ_INDEX
);
210 trace_vfio_intx_disable_kvm(vdev
->vbasedev
.name
);
214 static void vfio_intx_update(VFIOPCIDevice
*vdev
, PCIINTxRoute
*route
)
218 trace_vfio_intx_update(vdev
->vbasedev
.name
,
219 vdev
->intx
.route
.irq
, route
->irq
);
221 vfio_intx_disable_kvm(vdev
);
223 vdev
->intx
.route
= *route
;
225 if (route
->mode
!= PCI_INTX_ENABLED
) {
229 vfio_intx_enable_kvm(vdev
, &err
);
231 warn_reportf_err(err
, VFIO_MSG_PREFIX
, vdev
->vbasedev
.name
);
234 /* Re-enable the interrupt in cased we missed an EOI */
235 vfio_intx_eoi(&vdev
->vbasedev
);
238 static void vfio_intx_routing_notifier(PCIDevice
*pdev
)
240 VFIOPCIDevice
*vdev
= VFIO_PCI(pdev
);
243 if (vdev
->interrupt
!= VFIO_INT_INTx
) {
247 route
= pci_device_route_intx_to_irq(&vdev
->pdev
, vdev
->intx
.pin
);
249 if (pci_intx_route_changed(&vdev
->intx
.route
, &route
)) {
250 vfio_intx_update(vdev
, &route
);
254 static void vfio_irqchip_change(Notifier
*notify
, void *data
)
256 VFIOPCIDevice
*vdev
= container_of(notify
, VFIOPCIDevice
,
257 irqchip_change_notifier
);
259 vfio_intx_update(vdev
, &vdev
->intx
.route
);
262 static int vfio_intx_enable(VFIOPCIDevice
*vdev
, Error
**errp
)
264 uint8_t pin
= vfio_pci_read_config(&vdev
->pdev
, PCI_INTERRUPT_PIN
, 1);
274 vfio_disable_interrupts(vdev
);
276 vdev
->intx
.pin
= pin
- 1; /* Pin A (1) -> irq[0] */
277 pci_config_set_interrupt_pin(vdev
->pdev
.config
, pin
);
281 * Only conditional to avoid generating error messages on platforms
282 * where we won't actually use the result anyway.
284 if (kvm_irqfds_enabled() && kvm_resamplefds_enabled()) {
285 vdev
->intx
.route
= pci_device_route_intx_to_irq(&vdev
->pdev
,
290 ret
= event_notifier_init(&vdev
->intx
.interrupt
, 0);
292 error_setg_errno(errp
, -ret
, "event_notifier_init failed");
295 fd
= event_notifier_get_fd(&vdev
->intx
.interrupt
);
296 qemu_set_fd_handler(fd
, vfio_intx_interrupt
, NULL
, vdev
);
298 if (vfio_set_irq_signaling(&vdev
->vbasedev
, VFIO_PCI_INTX_IRQ_INDEX
, 0,
299 VFIO_IRQ_SET_ACTION_TRIGGER
, fd
, errp
)) {
300 qemu_set_fd_handler(fd
, NULL
, NULL
, vdev
);
301 event_notifier_cleanup(&vdev
->intx
.interrupt
);
305 vfio_intx_enable_kvm(vdev
, &err
);
307 warn_reportf_err(err
, VFIO_MSG_PREFIX
, vdev
->vbasedev
.name
);
310 vdev
->interrupt
= VFIO_INT_INTx
;
312 trace_vfio_intx_enable(vdev
->vbasedev
.name
);
316 static void vfio_intx_disable(VFIOPCIDevice
*vdev
)
320 timer_del(vdev
->intx
.mmap_timer
);
321 vfio_intx_disable_kvm(vdev
);
322 vfio_disable_irqindex(&vdev
->vbasedev
, VFIO_PCI_INTX_IRQ_INDEX
);
323 vdev
->intx
.pending
= false;
324 pci_irq_deassert(&vdev
->pdev
);
325 vfio_mmap_set_enabled(vdev
, true);
327 fd
= event_notifier_get_fd(&vdev
->intx
.interrupt
);
328 qemu_set_fd_handler(fd
, NULL
, NULL
, vdev
);
329 event_notifier_cleanup(&vdev
->intx
.interrupt
);
331 vdev
->interrupt
= VFIO_INT_NONE
;
333 trace_vfio_intx_disable(vdev
->vbasedev
.name
);
339 static void vfio_msi_interrupt(void *opaque
)
341 VFIOMSIVector
*vector
= opaque
;
342 VFIOPCIDevice
*vdev
= vector
->vdev
;
343 MSIMessage (*get_msg
)(PCIDevice
*dev
, unsigned vector
);
344 void (*notify
)(PCIDevice
*dev
, unsigned vector
);
346 int nr
= vector
- vdev
->msi_vectors
;
348 if (!event_notifier_test_and_clear(&vector
->interrupt
)) {
352 if (vdev
->interrupt
== VFIO_INT_MSIX
) {
353 get_msg
= msix_get_message
;
354 notify
= msix_notify
;
356 /* A masked vector firing needs to use the PBA, enable it */
357 if (msix_is_masked(&vdev
->pdev
, nr
)) {
358 set_bit(nr
, vdev
->msix
->pending
);
359 memory_region_set_enabled(&vdev
->pdev
.msix_pba_mmio
, true);
360 trace_vfio_msix_pba_enable(vdev
->vbasedev
.name
);
362 } else if (vdev
->interrupt
== VFIO_INT_MSI
) {
363 get_msg
= msi_get_message
;
369 msg
= get_msg(&vdev
->pdev
, nr
);
370 trace_vfio_msi_interrupt(vdev
->vbasedev
.name
, nr
, msg
.address
, msg
.data
);
371 notify(&vdev
->pdev
, nr
);
375 * Get MSI-X enabled, but no vector enabled, by setting vector 0 with an invalid
378 static int vfio_enable_msix_no_vec(VFIOPCIDevice
*vdev
)
380 g_autofree
struct vfio_irq_set
*irq_set
= NULL
;
384 argsz
= sizeof(*irq_set
) + sizeof(*fd
);
386 irq_set
= g_malloc0(argsz
);
387 irq_set
->argsz
= argsz
;
388 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
|
389 VFIO_IRQ_SET_ACTION_TRIGGER
;
390 irq_set
->index
= VFIO_PCI_MSIX_IRQ_INDEX
;
393 fd
= (int32_t *)&irq_set
->data
;
396 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
401 static int vfio_enable_vectors(VFIOPCIDevice
*vdev
, bool msix
)
403 struct vfio_irq_set
*irq_set
;
404 int ret
= 0, i
, argsz
;
408 * If dynamic MSI-X allocation is supported, the vectors to be allocated
409 * and enabled can be scattered. Before kernel enabling MSI-X, setting
410 * nr_vectors causes all these vectors to be allocated on host.
412 * To keep allocation as needed, use vector 0 with an invalid fd to get
413 * MSI-X enabled first, then set vectors with a potentially sparse set of
414 * eventfds to enable interrupts only when enabled in guest.
416 if (msix
&& !vdev
->msix
->noresize
) {
417 ret
= vfio_enable_msix_no_vec(vdev
);
424 argsz
= sizeof(*irq_set
) + (vdev
->nr_vectors
* sizeof(*fds
));
426 irq_set
= g_malloc0(argsz
);
427 irq_set
->argsz
= argsz
;
428 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
| VFIO_IRQ_SET_ACTION_TRIGGER
;
429 irq_set
->index
= msix
? VFIO_PCI_MSIX_IRQ_INDEX
: VFIO_PCI_MSI_IRQ_INDEX
;
431 irq_set
->count
= vdev
->nr_vectors
;
432 fds
= (int32_t *)&irq_set
->data
;
434 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
438 * MSI vs MSI-X - The guest has direct access to MSI mask and pending
439 * bits, therefore we always use the KVM signaling path when setup.
440 * MSI-X mask and pending bits are emulated, so we want to use the
441 * KVM signaling path only when configured and unmasked.
443 if (vdev
->msi_vectors
[i
].use
) {
444 if (vdev
->msi_vectors
[i
].virq
< 0 ||
445 (msix
&& msix_is_masked(&vdev
->pdev
, i
))) {
446 fd
= event_notifier_get_fd(&vdev
->msi_vectors
[i
].interrupt
);
448 fd
= event_notifier_get_fd(&vdev
->msi_vectors
[i
].kvm_interrupt
);
455 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
462 static void vfio_add_kvm_msi_virq(VFIOPCIDevice
*vdev
, VFIOMSIVector
*vector
,
463 int vector_n
, bool msix
)
465 if ((msix
&& vdev
->no_kvm_msix
) || (!msix
&& vdev
->no_kvm_msi
)) {
469 vector
->virq
= kvm_irqchip_add_msi_route(&vfio_route_change
,
470 vector_n
, &vdev
->pdev
);
473 static void vfio_connect_kvm_msi_virq(VFIOMSIVector
*vector
)
475 if (vector
->virq
< 0) {
479 if (event_notifier_init(&vector
->kvm_interrupt
, 0)) {
483 if (kvm_irqchip_add_irqfd_notifier_gsi(kvm_state
, &vector
->kvm_interrupt
,
484 NULL
, vector
->virq
) < 0) {
491 event_notifier_cleanup(&vector
->kvm_interrupt
);
493 kvm_irqchip_release_virq(kvm_state
, vector
->virq
);
497 static void vfio_remove_kvm_msi_virq(VFIOMSIVector
*vector
)
499 kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state
, &vector
->kvm_interrupt
,
501 kvm_irqchip_release_virq(kvm_state
, vector
->virq
);
503 event_notifier_cleanup(&vector
->kvm_interrupt
);
506 static void vfio_update_kvm_msi_virq(VFIOMSIVector
*vector
, MSIMessage msg
,
509 kvm_irqchip_update_msi_route(kvm_state
, vector
->virq
, msg
, pdev
);
510 kvm_irqchip_commit_routes(kvm_state
);
513 static int vfio_msix_vector_do_use(PCIDevice
*pdev
, unsigned int nr
,
514 MSIMessage
*msg
, IOHandler
*handler
)
516 VFIOPCIDevice
*vdev
= VFIO_PCI(pdev
);
517 VFIOMSIVector
*vector
;
519 bool resizing
= !!(vdev
->nr_vectors
< nr
+ 1);
521 trace_vfio_msix_vector_do_use(vdev
->vbasedev
.name
, nr
);
523 vector
= &vdev
->msi_vectors
[nr
];
528 if (event_notifier_init(&vector
->interrupt
, 0)) {
529 error_report("vfio: Error: event_notifier_init failed");
532 msix_vector_use(pdev
, nr
);
535 qemu_set_fd_handler(event_notifier_get_fd(&vector
->interrupt
),
536 handler
, NULL
, vector
);
539 * Attempt to enable route through KVM irqchip,
540 * default to userspace handling if unavailable.
542 if (vector
->virq
>= 0) {
544 vfio_remove_kvm_msi_virq(vector
);
546 vfio_update_kvm_msi_virq(vector
, *msg
, pdev
);
550 if (vdev
->defer_kvm_irq_routing
) {
551 vfio_add_kvm_msi_virq(vdev
, vector
, nr
, true);
553 vfio_route_change
= kvm_irqchip_begin_route_changes(kvm_state
);
554 vfio_add_kvm_msi_virq(vdev
, vector
, nr
, true);
555 kvm_irqchip_commit_route_changes(&vfio_route_change
);
556 vfio_connect_kvm_msi_virq(vector
);
562 * When dynamic allocation is not supported, we don't want to have the
563 * host allocate all possible MSI vectors for a device if they're not
564 * in use, so we shutdown and incrementally increase them as needed.
565 * nr_vectors represents the total number of vectors allocated.
567 * When dynamic allocation is supported, let the host only allocate
568 * and enable a vector when it is in use in guest. nr_vectors represents
569 * the upper bound of vectors being enabled (but not all of the ranges
570 * is allocated or enabled).
573 vdev
->nr_vectors
= nr
+ 1;
576 if (!vdev
->defer_kvm_irq_routing
) {
577 if (vdev
->msix
->noresize
&& resizing
) {
578 vfio_disable_irqindex(&vdev
->vbasedev
, VFIO_PCI_MSIX_IRQ_INDEX
);
579 ret
= vfio_enable_vectors(vdev
, true);
581 error_report("vfio: failed to enable vectors, %d", ret
);
587 if (vector
->virq
>= 0) {
588 fd
= event_notifier_get_fd(&vector
->kvm_interrupt
);
590 fd
= event_notifier_get_fd(&vector
->interrupt
);
593 if (vfio_set_irq_signaling(&vdev
->vbasedev
,
594 VFIO_PCI_MSIX_IRQ_INDEX
, nr
,
595 VFIO_IRQ_SET_ACTION_TRIGGER
, fd
, &err
)) {
596 error_reportf_err(err
, VFIO_MSG_PREFIX
, vdev
->vbasedev
.name
);
601 /* Disable PBA emulation when nothing more is pending. */
602 clear_bit(nr
, vdev
->msix
->pending
);
603 if (find_first_bit(vdev
->msix
->pending
,
604 vdev
->nr_vectors
) == vdev
->nr_vectors
) {
605 memory_region_set_enabled(&vdev
->pdev
.msix_pba_mmio
, false);
606 trace_vfio_msix_pba_disable(vdev
->vbasedev
.name
);
612 static int vfio_msix_vector_use(PCIDevice
*pdev
,
613 unsigned int nr
, MSIMessage msg
)
615 return vfio_msix_vector_do_use(pdev
, nr
, &msg
, vfio_msi_interrupt
);
618 static void vfio_msix_vector_release(PCIDevice
*pdev
, unsigned int nr
)
620 VFIOPCIDevice
*vdev
= VFIO_PCI(pdev
);
621 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[nr
];
623 trace_vfio_msix_vector_release(vdev
->vbasedev
.name
, nr
);
626 * There are still old guests that mask and unmask vectors on every
627 * interrupt. If we're using QEMU bypass with a KVM irqfd, leave all of
628 * the KVM setup in place, simply switch VFIO to use the non-bypass
629 * eventfd. We'll then fire the interrupt through QEMU and the MSI-X
630 * core will mask the interrupt and set pending bits, allowing it to
631 * be re-asserted on unmask. Nothing to do if already using QEMU mode.
633 if (vector
->virq
>= 0) {
634 int32_t fd
= event_notifier_get_fd(&vector
->interrupt
);
637 if (vfio_set_irq_signaling(&vdev
->vbasedev
, VFIO_PCI_MSIX_IRQ_INDEX
, nr
,
638 VFIO_IRQ_SET_ACTION_TRIGGER
, fd
, &err
)) {
639 error_reportf_err(err
, VFIO_MSG_PREFIX
, vdev
->vbasedev
.name
);
644 static void vfio_prepare_kvm_msi_virq_batch(VFIOPCIDevice
*vdev
)
646 assert(!vdev
->defer_kvm_irq_routing
);
647 vdev
->defer_kvm_irq_routing
= true;
648 vfio_route_change
= kvm_irqchip_begin_route_changes(kvm_state
);
651 static void vfio_commit_kvm_msi_virq_batch(VFIOPCIDevice
*vdev
)
655 assert(vdev
->defer_kvm_irq_routing
);
656 vdev
->defer_kvm_irq_routing
= false;
658 kvm_irqchip_commit_route_changes(&vfio_route_change
);
660 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
661 vfio_connect_kvm_msi_virq(&vdev
->msi_vectors
[i
]);
665 static void vfio_msix_enable(VFIOPCIDevice
*vdev
)
669 vfio_disable_interrupts(vdev
);
671 vdev
->msi_vectors
= g_new0(VFIOMSIVector
, vdev
->msix
->entries
);
673 vdev
->interrupt
= VFIO_INT_MSIX
;
676 * Setting vector notifiers triggers synchronous vector-use
677 * callbacks for each active vector. Deferring to commit the KVM
678 * routes once rather than per vector provides a substantial
679 * performance improvement.
681 vfio_prepare_kvm_msi_virq_batch(vdev
);
683 if (msix_set_vector_notifiers(&vdev
->pdev
, vfio_msix_vector_use
,
684 vfio_msix_vector_release
, NULL
)) {
685 error_report("vfio: msix_set_vector_notifiers failed");
688 vfio_commit_kvm_msi_virq_batch(vdev
);
690 if (vdev
->nr_vectors
) {
691 ret
= vfio_enable_vectors(vdev
, true);
693 error_report("vfio: failed to enable vectors, %d", ret
);
697 * Some communication channels between VF & PF or PF & fw rely on the
698 * physical state of the device and expect that enabling MSI-X from the
699 * guest enables the same on the host. When our guest is Linux, the
700 * guest driver call to pci_enable_msix() sets the enabling bit in the
701 * MSI-X capability, but leaves the vector table masked. We therefore
702 * can't rely on a vector_use callback (from request_irq() in the guest)
703 * to switch the physical device into MSI-X mode because that may come a
704 * long time after pci_enable_msix(). This code sets vector 0 with an
705 * invalid fd to make the physical device MSI-X enabled, but with no
706 * vectors enabled, just like the guest view.
708 ret
= vfio_enable_msix_no_vec(vdev
);
710 error_report("vfio: failed to enable MSI-X, %d", ret
);
714 trace_vfio_msix_enable(vdev
->vbasedev
.name
);
717 static void vfio_msi_enable(VFIOPCIDevice
*vdev
)
721 vfio_disable_interrupts(vdev
);
723 vdev
->nr_vectors
= msi_nr_vectors_allocated(&vdev
->pdev
);
726 * Setting vector notifiers needs to enable route for each vector.
727 * Deferring to commit the KVM routes once rather than per vector
728 * provides a substantial performance improvement.
730 vfio_prepare_kvm_msi_virq_batch(vdev
);
732 vdev
->msi_vectors
= g_new0(VFIOMSIVector
, vdev
->nr_vectors
);
734 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
735 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[i
];
741 if (event_notifier_init(&vector
->interrupt
, 0)) {
742 error_report("vfio: Error: event_notifier_init failed");
745 qemu_set_fd_handler(event_notifier_get_fd(&vector
->interrupt
),
746 vfio_msi_interrupt
, NULL
, vector
);
749 * Attempt to enable route through KVM irqchip,
750 * default to userspace handling if unavailable.
752 vfio_add_kvm_msi_virq(vdev
, vector
, i
, false);
755 vfio_commit_kvm_msi_virq_batch(vdev
);
757 /* Set interrupt type prior to possible interrupts */
758 vdev
->interrupt
= VFIO_INT_MSI
;
760 ret
= vfio_enable_vectors(vdev
, false);
763 error_report("vfio: Error: Failed to setup MSI fds: %m");
765 error_report("vfio: Error: Failed to enable %d "
766 "MSI vectors, retry with %d", vdev
->nr_vectors
, ret
);
769 vfio_msi_disable_common(vdev
);
772 vdev
->nr_vectors
= ret
;
777 * Failing to setup MSI doesn't really fall within any specification.
778 * Let's try leaving interrupts disabled and hope the guest figures
779 * out to fall back to INTx for this device.
781 error_report("vfio: Error: Failed to enable MSI");
786 trace_vfio_msi_enable(vdev
->vbasedev
.name
, vdev
->nr_vectors
);
789 static void vfio_msi_disable_common(VFIOPCIDevice
*vdev
)
793 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
794 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[i
];
795 if (vdev
->msi_vectors
[i
].use
) {
796 if (vector
->virq
>= 0) {
797 vfio_remove_kvm_msi_virq(vector
);
799 qemu_set_fd_handler(event_notifier_get_fd(&vector
->interrupt
),
801 event_notifier_cleanup(&vector
->interrupt
);
805 g_free(vdev
->msi_vectors
);
806 vdev
->msi_vectors
= NULL
;
807 vdev
->nr_vectors
= 0;
808 vdev
->interrupt
= VFIO_INT_NONE
;
811 static void vfio_msix_disable(VFIOPCIDevice
*vdev
)
816 msix_unset_vector_notifiers(&vdev
->pdev
);
819 * MSI-X will only release vectors if MSI-X is still enabled on the
820 * device, check through the rest and release it ourselves if necessary.
822 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
823 if (vdev
->msi_vectors
[i
].use
) {
824 vfio_msix_vector_release(&vdev
->pdev
, i
);
825 msix_vector_unuse(&vdev
->pdev
, i
);
829 if (vdev
->nr_vectors
) {
830 vfio_disable_irqindex(&vdev
->vbasedev
, VFIO_PCI_MSIX_IRQ_INDEX
);
833 vfio_msi_disable_common(vdev
);
834 vfio_intx_enable(vdev
, &err
);
836 error_reportf_err(err
, VFIO_MSG_PREFIX
, vdev
->vbasedev
.name
);
839 memset(vdev
->msix
->pending
, 0,
840 BITS_TO_LONGS(vdev
->msix
->entries
) * sizeof(unsigned long));
842 trace_vfio_msix_disable(vdev
->vbasedev
.name
);
845 static void vfio_msi_disable(VFIOPCIDevice
*vdev
)
849 vfio_disable_irqindex(&vdev
->vbasedev
, VFIO_PCI_MSI_IRQ_INDEX
);
850 vfio_msi_disable_common(vdev
);
851 vfio_intx_enable(vdev
, &err
);
853 error_reportf_err(err
, VFIO_MSG_PREFIX
, vdev
->vbasedev
.name
);
856 trace_vfio_msi_disable(vdev
->vbasedev
.name
);
859 static void vfio_update_msi(VFIOPCIDevice
*vdev
)
863 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
864 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[i
];
867 if (!vector
->use
|| vector
->virq
< 0) {
871 msg
= msi_get_message(&vdev
->pdev
, i
);
872 vfio_update_kvm_msi_virq(vector
, msg
, &vdev
->pdev
);
876 static void vfio_pci_load_rom(VFIOPCIDevice
*vdev
)
878 struct vfio_region_info
*reg_info
;
883 if (vfio_get_region_info(&vdev
->vbasedev
,
884 VFIO_PCI_ROM_REGION_INDEX
, ®_info
)) {
885 error_report("vfio: Error getting ROM info: %m");
889 trace_vfio_pci_load_rom(vdev
->vbasedev
.name
, (unsigned long)reg_info
->size
,
890 (unsigned long)reg_info
->offset
,
891 (unsigned long)reg_info
->flags
);
893 vdev
->rom_size
= size
= reg_info
->size
;
894 vdev
->rom_offset
= reg_info
->offset
;
898 if (!vdev
->rom_size
) {
899 vdev
->rom_read_failed
= true;
900 error_report("vfio-pci: Cannot read device rom at "
901 "%s", vdev
->vbasedev
.name
);
902 error_printf("Device option ROM contents are probably invalid "
903 "(check dmesg).\nSkip option ROM probe with rombar=0, "
904 "or load from file with romfile=\n");
908 vdev
->rom
= g_malloc(size
);
909 memset(vdev
->rom
, 0xff, size
);
912 bytes
= pread(vdev
->vbasedev
.fd
, vdev
->rom
+ off
,
913 size
, vdev
->rom_offset
+ off
);
916 } else if (bytes
> 0) {
920 if (errno
== EINTR
|| errno
== EAGAIN
) {
923 error_report("vfio: Error reading device ROM: %m");
929 * Test the ROM signature against our device, if the vendor is correct
930 * but the device ID doesn't match, store the correct device ID and
931 * recompute the checksum. Intel IGD devices need this and are known
932 * to have bogus checksums so we can't simply adjust the checksum.
934 if (pci_get_word(vdev
->rom
) == 0xaa55 &&
935 pci_get_word(vdev
->rom
+ 0x18) + 8 < vdev
->rom_size
&&
936 !memcmp(vdev
->rom
+ pci_get_word(vdev
->rom
+ 0x18), "PCIR", 4)) {
939 vid
= pci_get_word(vdev
->rom
+ pci_get_word(vdev
->rom
+ 0x18) + 4);
940 did
= pci_get_word(vdev
->rom
+ pci_get_word(vdev
->rom
+ 0x18) + 6);
942 if (vid
== vdev
->vendor_id
&& did
!= vdev
->device_id
) {
944 uint8_t csum
, *data
= vdev
->rom
;
946 pci_set_word(vdev
->rom
+ pci_get_word(vdev
->rom
+ 0x18) + 6,
950 for (csum
= 0, i
= 0; i
< vdev
->rom_size
; i
++) {
959 static uint64_t vfio_rom_read(void *opaque
, hwaddr addr
, unsigned size
)
961 VFIOPCIDevice
*vdev
= opaque
;
970 /* Load the ROM lazily when the guest tries to read it */
971 if (unlikely(!vdev
->rom
&& !vdev
->rom_read_failed
)) {
972 vfio_pci_load_rom(vdev
);
975 memcpy(&val
, vdev
->rom
+ addr
,
976 (addr
< vdev
->rom_size
) ? MIN(size
, vdev
->rom_size
- addr
) : 0);
983 data
= le16_to_cpu(val
.word
);
986 data
= le32_to_cpu(val
.dword
);
989 hw_error("vfio: unsupported read size, %d bytes\n", size
);
993 trace_vfio_rom_read(vdev
->vbasedev
.name
, addr
, size
, data
);
998 static void vfio_rom_write(void *opaque
, hwaddr addr
,
999 uint64_t data
, unsigned size
)
1003 static const MemoryRegionOps vfio_rom_ops
= {
1004 .read
= vfio_rom_read
,
1005 .write
= vfio_rom_write
,
1006 .endianness
= DEVICE_LITTLE_ENDIAN
,
1009 static void vfio_pci_size_rom(VFIOPCIDevice
*vdev
)
1011 uint32_t orig
, size
= cpu_to_le32((uint32_t)PCI_ROM_ADDRESS_MASK
);
1012 off_t offset
= vdev
->config_offset
+ PCI_ROM_ADDRESS
;
1013 DeviceState
*dev
= DEVICE(vdev
);
1015 int fd
= vdev
->vbasedev
.fd
;
1017 if (vdev
->pdev
.romfile
|| !vdev
->pdev
.rom_bar
) {
1018 /* Since pci handles romfile, just print a message and return */
1019 if (vfio_opt_rom_in_denylist(vdev
) && vdev
->pdev
.romfile
) {
1020 warn_report("Device at %s is known to cause system instability"
1021 " issues during option rom execution",
1022 vdev
->vbasedev
.name
);
1023 error_printf("Proceeding anyway since user specified romfile\n");
1029 * Use the same size ROM BAR as the physical device. The contents
1030 * will get filled in later when the guest tries to read it.
1032 if (pread(fd
, &orig
, 4, offset
) != 4 ||
1033 pwrite(fd
, &size
, 4, offset
) != 4 ||
1034 pread(fd
, &size
, 4, offset
) != 4 ||
1035 pwrite(fd
, &orig
, 4, offset
) != 4) {
1036 error_report("%s(%s) failed: %m", __func__
, vdev
->vbasedev
.name
);
1040 size
= ~(le32_to_cpu(size
) & PCI_ROM_ADDRESS_MASK
) + 1;
1046 if (vfio_opt_rom_in_denylist(vdev
)) {
1047 if (dev
->opts
&& qdict_haskey(dev
->opts
, "rombar")) {
1048 warn_report("Device at %s is known to cause system instability"
1049 " issues during option rom execution",
1050 vdev
->vbasedev
.name
);
1051 error_printf("Proceeding anyway since user specified"
1052 " non zero value for rombar\n");
1054 warn_report("Rom loading for device at %s has been disabled"
1055 " due to system instability issues",
1056 vdev
->vbasedev
.name
);
1057 error_printf("Specify rombar=1 or romfile to force\n");
1062 trace_vfio_pci_size_rom(vdev
->vbasedev
.name
, size
);
1064 name
= g_strdup_printf("vfio[%s].rom", vdev
->vbasedev
.name
);
1066 memory_region_init_io(&vdev
->pdev
.rom
, OBJECT(vdev
),
1067 &vfio_rom_ops
, vdev
, name
, size
);
1070 pci_register_bar(&vdev
->pdev
, PCI_ROM_SLOT
,
1071 PCI_BASE_ADDRESS_SPACE_MEMORY
, &vdev
->pdev
.rom
);
1073 vdev
->rom_read_failed
= false;
1076 void vfio_vga_write(void *opaque
, hwaddr addr
,
1077 uint64_t data
, unsigned size
)
1079 VFIOVGARegion
*region
= opaque
;
1080 VFIOVGA
*vga
= container_of(region
, VFIOVGA
, region
[region
->nr
]);
1087 off_t offset
= vga
->fd_offset
+ region
->offset
+ addr
;
1094 buf
.word
= cpu_to_le16(data
);
1097 buf
.dword
= cpu_to_le32(data
);
1100 hw_error("vfio: unsupported write size, %d bytes", size
);
1104 if (pwrite(vga
->fd
, &buf
, size
, offset
) != size
) {
1105 error_report("%s(,0x%"HWADDR_PRIx
", 0x%"PRIx64
", %d) failed: %m",
1106 __func__
, region
->offset
+ addr
, data
, size
);
1109 trace_vfio_vga_write(region
->offset
+ addr
, data
, size
);
1112 uint64_t vfio_vga_read(void *opaque
, hwaddr addr
, unsigned size
)
1114 VFIOVGARegion
*region
= opaque
;
1115 VFIOVGA
*vga
= container_of(region
, VFIOVGA
, region
[region
->nr
]);
1123 off_t offset
= vga
->fd_offset
+ region
->offset
+ addr
;
1125 if (pread(vga
->fd
, &buf
, size
, offset
) != size
) {
1126 error_report("%s(,0x%"HWADDR_PRIx
", %d) failed: %m",
1127 __func__
, region
->offset
+ addr
, size
);
1128 return (uint64_t)-1;
1136 data
= le16_to_cpu(buf
.word
);
1139 data
= le32_to_cpu(buf
.dword
);
1142 hw_error("vfio: unsupported read size, %d bytes", size
);
1146 trace_vfio_vga_read(region
->offset
+ addr
, size
, data
);
1151 static const MemoryRegionOps vfio_vga_ops
= {
1152 .read
= vfio_vga_read
,
1153 .write
= vfio_vga_write
,
1154 .endianness
= DEVICE_LITTLE_ENDIAN
,
1158 * Expand memory region of sub-page(size < PAGE_SIZE) MMIO BAR to page
1159 * size if the BAR is in an exclusive page in host so that we could map
1160 * this BAR to guest. But this sub-page BAR may not occupy an exclusive
1161 * page in guest. So we should set the priority of the expanded memory
1162 * region to zero in case of overlap with BARs which share the same page
1163 * with the sub-page BAR in guest. Besides, we should also recover the
1164 * size of this sub-page BAR when its base address is changed in guest
1165 * and not page aligned any more.
1167 static void vfio_sub_page_bar_update_mapping(PCIDevice
*pdev
, int bar
)
1169 VFIOPCIDevice
*vdev
= VFIO_PCI(pdev
);
1170 VFIORegion
*region
= &vdev
->bars
[bar
].region
;
1171 MemoryRegion
*mmap_mr
, *region_mr
, *base_mr
;
1174 uint64_t size
= region
->size
;
1176 /* Make sure that the whole region is allowed to be mmapped */
1177 if (region
->nr_mmaps
!= 1 || !region
->mmaps
[0].mmap
||
1178 region
->mmaps
[0].size
!= region
->size
) {
1182 r
= &pdev
->io_regions
[bar
];
1184 base_mr
= vdev
->bars
[bar
].mr
;
1185 region_mr
= region
->mem
;
1186 mmap_mr
= ®ion
->mmaps
[0].mem
;
1188 /* If BAR is mapped and page aligned, update to fill PAGE_SIZE */
1189 if (bar_addr
!= PCI_BAR_UNMAPPED
&&
1190 !(bar_addr
& ~qemu_real_host_page_mask())) {
1191 size
= qemu_real_host_page_size();
1194 memory_region_transaction_begin();
1196 if (vdev
->bars
[bar
].size
< size
) {
1197 memory_region_set_size(base_mr
, size
);
1199 memory_region_set_size(region_mr
, size
);
1200 memory_region_set_size(mmap_mr
, size
);
1201 if (size
!= vdev
->bars
[bar
].size
&& memory_region_is_mapped(base_mr
)) {
1202 memory_region_del_subregion(r
->address_space
, base_mr
);
1203 memory_region_add_subregion_overlap(r
->address_space
,
1204 bar_addr
, base_mr
, 0);
1207 memory_region_transaction_commit();
1213 uint32_t vfio_pci_read_config(PCIDevice
*pdev
, uint32_t addr
, int len
)
1215 VFIOPCIDevice
*vdev
= VFIO_PCI(pdev
);
1216 uint32_t emu_bits
= 0, emu_val
= 0, phys_val
= 0, val
;
1218 memcpy(&emu_bits
, vdev
->emulated_config_bits
+ addr
, len
);
1219 emu_bits
= le32_to_cpu(emu_bits
);
1222 emu_val
= pci_default_read_config(pdev
, addr
, len
);
1225 if (~emu_bits
& (0xffffffffU
>> (32 - len
* 8))) {
1228 ret
= pread(vdev
->vbasedev
.fd
, &phys_val
, len
,
1229 vdev
->config_offset
+ addr
);
1231 error_report("%s(%s, 0x%x, 0x%x) failed: %m",
1232 __func__
, vdev
->vbasedev
.name
, addr
, len
);
1235 phys_val
= le32_to_cpu(phys_val
);
1238 val
= (emu_val
& emu_bits
) | (phys_val
& ~emu_bits
);
1240 trace_vfio_pci_read_config(vdev
->vbasedev
.name
, addr
, len
, val
);
1245 void vfio_pci_write_config(PCIDevice
*pdev
,
1246 uint32_t addr
, uint32_t val
, int len
)
1248 VFIOPCIDevice
*vdev
= VFIO_PCI(pdev
);
1249 uint32_t val_le
= cpu_to_le32(val
);
1251 trace_vfio_pci_write_config(vdev
->vbasedev
.name
, addr
, val
, len
);
1253 /* Write everything to VFIO, let it filter out what we can't write */
1254 if (pwrite(vdev
->vbasedev
.fd
, &val_le
, len
, vdev
->config_offset
+ addr
)
1256 error_report("%s(%s, 0x%x, 0x%x, 0x%x) failed: %m",
1257 __func__
, vdev
->vbasedev
.name
, addr
, val
, len
);
1260 /* MSI/MSI-X Enabling/Disabling */
1261 if (pdev
->cap_present
& QEMU_PCI_CAP_MSI
&&
1262 ranges_overlap(addr
, len
, pdev
->msi_cap
, vdev
->msi_cap_size
)) {
1263 int is_enabled
, was_enabled
= msi_enabled(pdev
);
1265 pci_default_write_config(pdev
, addr
, val
, len
);
1267 is_enabled
= msi_enabled(pdev
);
1271 vfio_msi_enable(vdev
);
1275 vfio_msi_disable(vdev
);
1277 vfio_update_msi(vdev
);
1280 } else if (pdev
->cap_present
& QEMU_PCI_CAP_MSIX
&&
1281 ranges_overlap(addr
, len
, pdev
->msix_cap
, MSIX_CAP_LENGTH
)) {
1282 int is_enabled
, was_enabled
= msix_enabled(pdev
);
1284 pci_default_write_config(pdev
, addr
, val
, len
);
1286 is_enabled
= msix_enabled(pdev
);
1288 if (!was_enabled
&& is_enabled
) {
1289 vfio_msix_enable(vdev
);
1290 } else if (was_enabled
&& !is_enabled
) {
1291 vfio_msix_disable(vdev
);
1293 } else if (ranges_overlap(addr
, len
, PCI_BASE_ADDRESS_0
, 24) ||
1294 range_covers_byte(addr
, len
, PCI_COMMAND
)) {
1295 pcibus_t old_addr
[PCI_NUM_REGIONS
- 1];
1298 for (bar
= 0; bar
< PCI_ROM_SLOT
; bar
++) {
1299 old_addr
[bar
] = pdev
->io_regions
[bar
].addr
;
1302 pci_default_write_config(pdev
, addr
, val
, len
);
1304 for (bar
= 0; bar
< PCI_ROM_SLOT
; bar
++) {
1305 if (old_addr
[bar
] != pdev
->io_regions
[bar
].addr
&&
1306 vdev
->bars
[bar
].region
.size
> 0 &&
1307 vdev
->bars
[bar
].region
.size
< qemu_real_host_page_size()) {
1308 vfio_sub_page_bar_update_mapping(pdev
, bar
);
1312 /* Write everything to QEMU to keep emulated bits correct */
1313 pci_default_write_config(pdev
, addr
, val
, len
);
1320 static void vfio_disable_interrupts(VFIOPCIDevice
*vdev
)
1323 * More complicated than it looks. Disabling MSI/X transitions the
1324 * device to INTx mode (if supported). Therefore we need to first
1325 * disable MSI/X and then cleanup by disabling INTx.
1327 if (vdev
->interrupt
== VFIO_INT_MSIX
) {
1328 vfio_msix_disable(vdev
);
1329 } else if (vdev
->interrupt
== VFIO_INT_MSI
) {
1330 vfio_msi_disable(vdev
);
1333 if (vdev
->interrupt
== VFIO_INT_INTx
) {
1334 vfio_intx_disable(vdev
);
1338 static int vfio_msi_setup(VFIOPCIDevice
*vdev
, int pos
, Error
**errp
)
1341 bool msi_64bit
, msi_maskbit
;
1345 if (pread(vdev
->vbasedev
.fd
, &ctrl
, sizeof(ctrl
),
1346 vdev
->config_offset
+ pos
+ PCI_CAP_FLAGS
) != sizeof(ctrl
)) {
1347 error_setg_errno(errp
, errno
, "failed reading MSI PCI_CAP_FLAGS");
1350 ctrl
= le16_to_cpu(ctrl
);
1352 msi_64bit
= !!(ctrl
& PCI_MSI_FLAGS_64BIT
);
1353 msi_maskbit
= !!(ctrl
& PCI_MSI_FLAGS_MASKBIT
);
1354 entries
= 1 << ((ctrl
& PCI_MSI_FLAGS_QMASK
) >> 1);
1356 trace_vfio_msi_setup(vdev
->vbasedev
.name
, pos
);
1358 ret
= msi_init(&vdev
->pdev
, pos
, entries
, msi_64bit
, msi_maskbit
, &err
);
1360 if (ret
== -ENOTSUP
) {
1363 error_propagate_prepend(errp
, err
, "msi_init failed: ");
1366 vdev
->msi_cap_size
= 0xa + (msi_maskbit
? 0xa : 0) + (msi_64bit
? 0x4 : 0);
1371 static void vfio_pci_fixup_msix_region(VFIOPCIDevice
*vdev
)
1374 VFIORegion
*region
= &vdev
->bars
[vdev
->msix
->table_bar
].region
;
1377 * If the host driver allows mapping of a MSIX data, we are going to
1378 * do map the entire BAR and emulate MSIX table on top of that.
1380 if (vfio_has_region_cap(&vdev
->vbasedev
, region
->nr
,
1381 VFIO_REGION_INFO_CAP_MSIX_MAPPABLE
)) {
1386 * We expect to find a single mmap covering the whole BAR, anything else
1387 * means it's either unsupported or already setup.
1389 if (region
->nr_mmaps
!= 1 || region
->mmaps
[0].offset
||
1390 region
->size
!= region
->mmaps
[0].size
) {
1394 /* MSI-X table start and end aligned to host page size */
1395 start
= vdev
->msix
->table_offset
& qemu_real_host_page_mask();
1396 end
= REAL_HOST_PAGE_ALIGN((uint64_t)vdev
->msix
->table_offset
+
1397 (vdev
->msix
->entries
* PCI_MSIX_ENTRY_SIZE
));
1400 * Does the MSI-X table cover the beginning of the BAR? The whole BAR?
1401 * NB - Host page size is necessarily a power of two and so is the PCI
1402 * BAR (not counting EA yet), therefore if we have host page aligned
1403 * @start and @end, then any remainder of the BAR before or after those
1404 * must be at least host page sized and therefore mmap'able.
1407 if (end
>= region
->size
) {
1408 region
->nr_mmaps
= 0;
1409 g_free(region
->mmaps
);
1410 region
->mmaps
= NULL
;
1411 trace_vfio_msix_fixup(vdev
->vbasedev
.name
,
1412 vdev
->msix
->table_bar
, 0, 0);
1414 region
->mmaps
[0].offset
= end
;
1415 region
->mmaps
[0].size
= region
->size
- end
;
1416 trace_vfio_msix_fixup(vdev
->vbasedev
.name
,
1417 vdev
->msix
->table_bar
, region
->mmaps
[0].offset
,
1418 region
->mmaps
[0].offset
+ region
->mmaps
[0].size
);
1421 /* Maybe it's aligned at the end of the BAR */
1422 } else if (end
>= region
->size
) {
1423 region
->mmaps
[0].size
= start
;
1424 trace_vfio_msix_fixup(vdev
->vbasedev
.name
,
1425 vdev
->msix
->table_bar
, region
->mmaps
[0].offset
,
1426 region
->mmaps
[0].offset
+ region
->mmaps
[0].size
);
1428 /* Otherwise it must split the BAR */
1430 region
->nr_mmaps
= 2;
1431 region
->mmaps
= g_renew(VFIOMmap
, region
->mmaps
, 2);
1433 memcpy(®ion
->mmaps
[1], ®ion
->mmaps
[0], sizeof(VFIOMmap
));
1435 region
->mmaps
[0].size
= start
;
1436 trace_vfio_msix_fixup(vdev
->vbasedev
.name
,
1437 vdev
->msix
->table_bar
, region
->mmaps
[0].offset
,
1438 region
->mmaps
[0].offset
+ region
->mmaps
[0].size
);
1440 region
->mmaps
[1].offset
= end
;
1441 region
->mmaps
[1].size
= region
->size
- end
;
1442 trace_vfio_msix_fixup(vdev
->vbasedev
.name
,
1443 vdev
->msix
->table_bar
, region
->mmaps
[1].offset
,
1444 region
->mmaps
[1].offset
+ region
->mmaps
[1].size
);
1448 static void vfio_pci_relocate_msix(VFIOPCIDevice
*vdev
, Error
**errp
)
1450 int target_bar
= -1;
1453 if (!vdev
->msix
|| vdev
->msix_relo
== OFF_AUTOPCIBAR_OFF
) {
1457 /* The actual minimum size of MSI-X structures */
1458 msix_sz
= (vdev
->msix
->entries
* PCI_MSIX_ENTRY_SIZE
) +
1459 (QEMU_ALIGN_UP(vdev
->msix
->entries
, 64) / 8);
1460 /* Round up to host pages, we don't want to share a page */
1461 msix_sz
= REAL_HOST_PAGE_ALIGN(msix_sz
);
1462 /* PCI BARs must be a power of 2 */
1463 msix_sz
= pow2ceil(msix_sz
);
1465 if (vdev
->msix_relo
== OFF_AUTOPCIBAR_AUTO
) {
1467 * TODO: Lookup table for known devices.
1469 * Logically we might use an algorithm here to select the BAR adding
1470 * the least additional MMIO space, but we cannot programmatically
1471 * predict the driver dependency on BAR ordering or sizing, therefore
1472 * 'auto' becomes a lookup for combinations reported to work.
1474 if (target_bar
< 0) {
1475 error_setg(errp
, "No automatic MSI-X relocation available for "
1476 "device %04x:%04x", vdev
->vendor_id
, vdev
->device_id
);
1480 target_bar
= (int)(vdev
->msix_relo
- OFF_AUTOPCIBAR_BAR0
);
1483 /* I/O port BARs cannot host MSI-X structures */
1484 if (vdev
->bars
[target_bar
].ioport
) {
1485 error_setg(errp
, "Invalid MSI-X relocation BAR %d, "
1486 "I/O port BAR", target_bar
);
1490 /* Cannot use a BAR in the "shadow" of a 64-bit BAR */
1491 if (!vdev
->bars
[target_bar
].size
&&
1492 target_bar
> 0 && vdev
->bars
[target_bar
- 1].mem64
) {
1493 error_setg(errp
, "Invalid MSI-X relocation BAR %d, "
1494 "consumed by 64-bit BAR %d", target_bar
, target_bar
- 1);
1498 /* 2GB max size for 32-bit BARs, cannot double if already > 1G */
1499 if (vdev
->bars
[target_bar
].size
> 1 * GiB
&&
1500 !vdev
->bars
[target_bar
].mem64
) {
1501 error_setg(errp
, "Invalid MSI-X relocation BAR %d, "
1502 "no space to extend 32-bit BAR", target_bar
);
1507 * If adding a new BAR, test if we can make it 64bit. We make it
1508 * prefetchable since QEMU MSI-X emulation has no read side effects
1509 * and doing so makes mapping more flexible.
1511 if (!vdev
->bars
[target_bar
].size
) {
1512 if (target_bar
< (PCI_ROM_SLOT
- 1) &&
1513 !vdev
->bars
[target_bar
+ 1].size
) {
1514 vdev
->bars
[target_bar
].mem64
= true;
1515 vdev
->bars
[target_bar
].type
= PCI_BASE_ADDRESS_MEM_TYPE_64
;
1517 vdev
->bars
[target_bar
].type
|= PCI_BASE_ADDRESS_MEM_PREFETCH
;
1518 vdev
->bars
[target_bar
].size
= msix_sz
;
1519 vdev
->msix
->table_offset
= 0;
1521 vdev
->bars
[target_bar
].size
= MAX(vdev
->bars
[target_bar
].size
* 2,
1524 * Due to above size calc, MSI-X always starts halfway into the BAR,
1525 * which will always be a separate host page.
1527 vdev
->msix
->table_offset
= vdev
->bars
[target_bar
].size
/ 2;
1530 vdev
->msix
->table_bar
= target_bar
;
1531 vdev
->msix
->pba_bar
= target_bar
;
1532 /* Requires 8-byte alignment, but PCI_MSIX_ENTRY_SIZE guarantees that */
1533 vdev
->msix
->pba_offset
= vdev
->msix
->table_offset
+
1534 (vdev
->msix
->entries
* PCI_MSIX_ENTRY_SIZE
);
1536 trace_vfio_msix_relo(vdev
->vbasedev
.name
,
1537 vdev
->msix
->table_bar
, vdev
->msix
->table_offset
);
1541 * We don't have any control over how pci_add_capability() inserts
1542 * capabilities into the chain. In order to setup MSI-X we need a
1543 * MemoryRegion for the BAR. In order to setup the BAR and not
1544 * attempt to mmap the MSI-X table area, which VFIO won't allow, we
1545 * need to first look for where the MSI-X table lives. So we
1546 * unfortunately split MSI-X setup across two functions.
1548 static void vfio_msix_early_setup(VFIOPCIDevice
*vdev
, Error
**errp
)
1552 uint32_t table
, pba
;
1553 int ret
, fd
= vdev
->vbasedev
.fd
;
1554 struct vfio_irq_info irq_info
= { .argsz
= sizeof(irq_info
),
1555 .index
= VFIO_PCI_MSIX_IRQ_INDEX
};
1558 pos
= pci_find_capability(&vdev
->pdev
, PCI_CAP_ID_MSIX
);
1563 if (pread(fd
, &ctrl
, sizeof(ctrl
),
1564 vdev
->config_offset
+ pos
+ PCI_MSIX_FLAGS
) != sizeof(ctrl
)) {
1565 error_setg_errno(errp
, errno
, "failed to read PCI MSIX FLAGS");
1569 if (pread(fd
, &table
, sizeof(table
),
1570 vdev
->config_offset
+ pos
+ PCI_MSIX_TABLE
) != sizeof(table
)) {
1571 error_setg_errno(errp
, errno
, "failed to read PCI MSIX TABLE");
1575 if (pread(fd
, &pba
, sizeof(pba
),
1576 vdev
->config_offset
+ pos
+ PCI_MSIX_PBA
) != sizeof(pba
)) {
1577 error_setg_errno(errp
, errno
, "failed to read PCI MSIX PBA");
1581 ctrl
= le16_to_cpu(ctrl
);
1582 table
= le32_to_cpu(table
);
1583 pba
= le32_to_cpu(pba
);
1585 msix
= g_malloc0(sizeof(*msix
));
1586 msix
->table_bar
= table
& PCI_MSIX_FLAGS_BIRMASK
;
1587 msix
->table_offset
= table
& ~PCI_MSIX_FLAGS_BIRMASK
;
1588 msix
->pba_bar
= pba
& PCI_MSIX_FLAGS_BIRMASK
;
1589 msix
->pba_offset
= pba
& ~PCI_MSIX_FLAGS_BIRMASK
;
1590 msix
->entries
= (ctrl
& PCI_MSIX_FLAGS_QSIZE
) + 1;
1592 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_GET_IRQ_INFO
, &irq_info
);
1594 error_setg_errno(errp
, -ret
, "failed to get MSI-X irq info");
1599 msix
->noresize
= !!(irq_info
.flags
& VFIO_IRQ_INFO_NORESIZE
);
1602 * Test the size of the pba_offset variable and catch if it extends outside
1603 * of the specified BAR. If it is the case, we need to apply a hardware
1604 * specific quirk if the device is known or we have a broken configuration.
1606 if (msix
->pba_offset
>= vdev
->bars
[msix
->pba_bar
].region
.size
) {
1608 * Chelsio T5 Virtual Function devices are encoded as 0x58xx for T5
1609 * adapters. The T5 hardware returns an incorrect value of 0x8000 for
1610 * the VF PBA offset while the BAR itself is only 8k. The correct value
1611 * is 0x1000, so we hard code that here.
1613 if (vdev
->vendor_id
== PCI_VENDOR_ID_CHELSIO
&&
1614 (vdev
->device_id
& 0xff00) == 0x5800) {
1615 msix
->pba_offset
= 0x1000;
1617 * BAIDU KUNLUN Virtual Function devices for KUNLUN AI processor
1618 * return an incorrect value of 0x460000 for the VF PBA offset while
1619 * the BAR itself is only 0x10000. The correct value is 0xb400.
1621 } else if (vfio_pci_is(vdev
, PCI_VENDOR_ID_BAIDU
,
1622 PCI_DEVICE_ID_KUNLUN_VF
)) {
1623 msix
->pba_offset
= 0xb400;
1624 } else if (vdev
->msix_relo
== OFF_AUTOPCIBAR_OFF
) {
1625 error_setg(errp
, "hardware reports invalid configuration, "
1626 "MSIX PBA outside of specified BAR");
1632 trace_vfio_msix_early_setup(vdev
->vbasedev
.name
, pos
, msix
->table_bar
,
1633 msix
->table_offset
, msix
->entries
,
1637 vfio_pci_fixup_msix_region(vdev
);
1639 vfio_pci_relocate_msix(vdev
, errp
);
1642 static int vfio_msix_setup(VFIOPCIDevice
*vdev
, int pos
, Error
**errp
)
1647 vdev
->msix
->pending
= g_new0(unsigned long,
1648 BITS_TO_LONGS(vdev
->msix
->entries
));
1649 ret
= msix_init(&vdev
->pdev
, vdev
->msix
->entries
,
1650 vdev
->bars
[vdev
->msix
->table_bar
].mr
,
1651 vdev
->msix
->table_bar
, vdev
->msix
->table_offset
,
1652 vdev
->bars
[vdev
->msix
->pba_bar
].mr
,
1653 vdev
->msix
->pba_bar
, vdev
->msix
->pba_offset
, pos
,
1656 if (ret
== -ENOTSUP
) {
1657 warn_report_err(err
);
1661 error_propagate(errp
, err
);
1666 * The PCI spec suggests that devices provide additional alignment for
1667 * MSI-X structures and avoid overlapping non-MSI-X related registers.
1668 * For an assigned device, this hopefully means that emulation of MSI-X
1669 * structures does not affect the performance of the device. If devices
1670 * fail to provide that alignment, a significant performance penalty may
1671 * result, for instance Mellanox MT27500 VFs:
1672 * http://www.spinics.net/lists/kvm/msg125881.html
1674 * The PBA is simply not that important for such a serious regression and
1675 * most drivers do not appear to look at it. The solution for this is to
1676 * disable the PBA MemoryRegion unless it's being used. We disable it
1677 * here and only enable it if a masked vector fires through QEMU. As the
1678 * vector-use notifier is called, which occurs on unmask, we test whether
1679 * PBA emulation is needed and again disable if not.
1681 memory_region_set_enabled(&vdev
->pdev
.msix_pba_mmio
, false);
1684 * The emulated machine may provide a paravirt interface for MSIX setup
1685 * so it is not strictly necessary to emulate MSIX here. This becomes
1686 * helpful when frequently accessed MMIO registers are located in
1687 * subpages adjacent to the MSIX table but the MSIX data containing page
1688 * cannot be mapped because of a host page size bigger than the MSIX table
1691 if (object_property_get_bool(OBJECT(qdev_get_machine()),
1692 "vfio-no-msix-emulation", NULL
)) {
1693 memory_region_set_enabled(&vdev
->pdev
.msix_table_mmio
, false);
1699 static void vfio_teardown_msi(VFIOPCIDevice
*vdev
)
1701 msi_uninit(&vdev
->pdev
);
1704 msix_uninit(&vdev
->pdev
,
1705 vdev
->bars
[vdev
->msix
->table_bar
].mr
,
1706 vdev
->bars
[vdev
->msix
->pba_bar
].mr
);
1707 g_free(vdev
->msix
->pending
);
1714 static void vfio_mmap_set_enabled(VFIOPCIDevice
*vdev
, bool enabled
)
1718 for (i
= 0; i
< PCI_ROM_SLOT
; i
++) {
1719 vfio_region_mmaps_set_enabled(&vdev
->bars
[i
].region
, enabled
);
1723 static void vfio_bar_prepare(VFIOPCIDevice
*vdev
, int nr
)
1725 VFIOBAR
*bar
= &vdev
->bars
[nr
];
1730 /* Skip both unimplemented BARs and the upper half of 64bit BARS. */
1731 if (!bar
->region
.size
) {
1735 /* Determine what type of BAR this is for registration */
1736 ret
= pread(vdev
->vbasedev
.fd
, &pci_bar
, sizeof(pci_bar
),
1737 vdev
->config_offset
+ PCI_BASE_ADDRESS_0
+ (4 * nr
));
1738 if (ret
!= sizeof(pci_bar
)) {
1739 error_report("vfio: Failed to read BAR %d (%m)", nr
);
1743 pci_bar
= le32_to_cpu(pci_bar
);
1744 bar
->ioport
= (pci_bar
& PCI_BASE_ADDRESS_SPACE_IO
);
1745 bar
->mem64
= bar
->ioport
? 0 : (pci_bar
& PCI_BASE_ADDRESS_MEM_TYPE_64
);
1746 bar
->type
= pci_bar
& (bar
->ioport
? ~PCI_BASE_ADDRESS_IO_MASK
:
1747 ~PCI_BASE_ADDRESS_MEM_MASK
);
1748 bar
->size
= bar
->region
.size
;
1751 static void vfio_bars_prepare(VFIOPCIDevice
*vdev
)
1755 for (i
= 0; i
< PCI_ROM_SLOT
; i
++) {
1756 vfio_bar_prepare(vdev
, i
);
1760 static void vfio_bar_register(VFIOPCIDevice
*vdev
, int nr
)
1762 VFIOBAR
*bar
= &vdev
->bars
[nr
];
1769 bar
->mr
= g_new0(MemoryRegion
, 1);
1770 name
= g_strdup_printf("%s base BAR %d", vdev
->vbasedev
.name
, nr
);
1771 memory_region_init_io(bar
->mr
, OBJECT(vdev
), NULL
, NULL
, name
, bar
->size
);
1774 if (bar
->region
.size
) {
1775 memory_region_add_subregion(bar
->mr
, 0, bar
->region
.mem
);
1777 if (vfio_region_mmap(&bar
->region
)) {
1778 error_report("Failed to mmap %s BAR %d. Performance may be slow",
1779 vdev
->vbasedev
.name
, nr
);
1783 pci_register_bar(&vdev
->pdev
, nr
, bar
->type
, bar
->mr
);
1786 static void vfio_bars_register(VFIOPCIDevice
*vdev
)
1790 for (i
= 0; i
< PCI_ROM_SLOT
; i
++) {
1791 vfio_bar_register(vdev
, i
);
1795 static void vfio_bars_exit(VFIOPCIDevice
*vdev
)
1799 for (i
= 0; i
< PCI_ROM_SLOT
; i
++) {
1800 VFIOBAR
*bar
= &vdev
->bars
[i
];
1802 vfio_bar_quirk_exit(vdev
, i
);
1803 vfio_region_exit(&bar
->region
);
1804 if (bar
->region
.size
) {
1805 memory_region_del_subregion(bar
->mr
, bar
->region
.mem
);
1810 pci_unregister_vga(&vdev
->pdev
);
1811 vfio_vga_quirk_exit(vdev
);
1815 static void vfio_bars_finalize(VFIOPCIDevice
*vdev
)
1819 for (i
= 0; i
< PCI_ROM_SLOT
; i
++) {
1820 VFIOBAR
*bar
= &vdev
->bars
[i
];
1822 vfio_bar_quirk_finalize(vdev
, i
);
1823 vfio_region_finalize(&bar
->region
);
1826 object_unparent(OBJECT(bar
->mr
));
1833 vfio_vga_quirk_finalize(vdev
);
1834 for (i
= 0; i
< ARRAY_SIZE(vdev
->vga
->region
); i
++) {
1835 object_unparent(OBJECT(&vdev
->vga
->region
[i
].mem
));
1844 static uint8_t vfio_std_cap_max_size(PCIDevice
*pdev
, uint8_t pos
)
1847 uint16_t next
= PCI_CONFIG_SPACE_SIZE
;
1849 for (tmp
= pdev
->config
[PCI_CAPABILITY_LIST
]; tmp
;
1850 tmp
= pdev
->config
[tmp
+ PCI_CAP_LIST_NEXT
]) {
1851 if (tmp
> pos
&& tmp
< next
) {
1860 static uint16_t vfio_ext_cap_max_size(const uint8_t *config
, uint16_t pos
)
1862 uint16_t tmp
, next
= PCIE_CONFIG_SPACE_SIZE
;
1864 for (tmp
= PCI_CONFIG_SPACE_SIZE
; tmp
;
1865 tmp
= PCI_EXT_CAP_NEXT(pci_get_long(config
+ tmp
))) {
1866 if (tmp
> pos
&& tmp
< next
) {
1874 static void vfio_set_word_bits(uint8_t *buf
, uint16_t val
, uint16_t mask
)
1876 pci_set_word(buf
, (pci_get_word(buf
) & ~mask
) | val
);
1879 static void vfio_add_emulated_word(VFIOPCIDevice
*vdev
, int pos
,
1880 uint16_t val
, uint16_t mask
)
1882 vfio_set_word_bits(vdev
->pdev
.config
+ pos
, val
, mask
);
1883 vfio_set_word_bits(vdev
->pdev
.wmask
+ pos
, ~mask
, mask
);
1884 vfio_set_word_bits(vdev
->emulated_config_bits
+ pos
, mask
, mask
);
1887 static void vfio_set_long_bits(uint8_t *buf
, uint32_t val
, uint32_t mask
)
1889 pci_set_long(buf
, (pci_get_long(buf
) & ~mask
) | val
);
1892 static void vfio_add_emulated_long(VFIOPCIDevice
*vdev
, int pos
,
1893 uint32_t val
, uint32_t mask
)
1895 vfio_set_long_bits(vdev
->pdev
.config
+ pos
, val
, mask
);
1896 vfio_set_long_bits(vdev
->pdev
.wmask
+ pos
, ~mask
, mask
);
1897 vfio_set_long_bits(vdev
->emulated_config_bits
+ pos
, mask
, mask
);
1900 static void vfio_pci_enable_rp_atomics(VFIOPCIDevice
*vdev
)
1902 struct vfio_device_info_cap_pci_atomic_comp
*cap
;
1903 g_autofree
struct vfio_device_info
*info
= NULL
;
1904 PCIBus
*bus
= pci_get_bus(&vdev
->pdev
);
1905 PCIDevice
*parent
= bus
->parent_dev
;
1906 struct vfio_info_cap_header
*hdr
;
1911 * PCIe Atomic Ops completer support is only added automatically for single
1912 * function devices downstream of a root port supporting DEVCAP2. Support
1913 * is added during realize and, if added, removed during device exit. The
1914 * single function requirement avoids conflicting requirements should a
1915 * slot be composed of multiple devices with differing capabilities.
1917 if (pci_bus_is_root(bus
) || !parent
|| !parent
->exp
.exp_cap
||
1918 pcie_cap_get_type(parent
) != PCI_EXP_TYPE_ROOT_PORT
||
1919 pcie_cap_get_version(parent
) != PCI_EXP_FLAGS_VER2
||
1921 vdev
->pdev
.cap_present
& QEMU_PCI_CAP_MULTIFUNCTION
) {
1925 pos
= parent
->config
+ parent
->exp
.exp_cap
+ PCI_EXP_DEVCAP2
;
1927 /* Abort if there'a already an Atomic Ops configuration on the root port */
1928 if (pci_get_long(pos
) & (PCI_EXP_DEVCAP2_ATOMIC_COMP32
|
1929 PCI_EXP_DEVCAP2_ATOMIC_COMP64
|
1930 PCI_EXP_DEVCAP2_ATOMIC_COMP128
)) {
1934 info
= vfio_get_device_info(vdev
->vbasedev
.fd
);
1939 hdr
= vfio_get_device_info_cap(info
, VFIO_DEVICE_INFO_CAP_PCI_ATOMIC_COMP
);
1945 if (cap
->flags
& VFIO_PCI_ATOMIC_COMP32
) {
1946 mask
|= PCI_EXP_DEVCAP2_ATOMIC_COMP32
;
1948 if (cap
->flags
& VFIO_PCI_ATOMIC_COMP64
) {
1949 mask
|= PCI_EXP_DEVCAP2_ATOMIC_COMP64
;
1951 if (cap
->flags
& VFIO_PCI_ATOMIC_COMP128
) {
1952 mask
|= PCI_EXP_DEVCAP2_ATOMIC_COMP128
;
1959 pci_long_test_and_set_mask(pos
, mask
);
1960 vdev
->clear_parent_atomics_on_exit
= true;
1963 static void vfio_pci_disable_rp_atomics(VFIOPCIDevice
*vdev
)
1965 if (vdev
->clear_parent_atomics_on_exit
) {
1966 PCIDevice
*parent
= pci_get_bus(&vdev
->pdev
)->parent_dev
;
1967 uint8_t *pos
= parent
->config
+ parent
->exp
.exp_cap
+ PCI_EXP_DEVCAP2
;
1969 pci_long_test_and_clear_mask(pos
, PCI_EXP_DEVCAP2_ATOMIC_COMP32
|
1970 PCI_EXP_DEVCAP2_ATOMIC_COMP64
|
1971 PCI_EXP_DEVCAP2_ATOMIC_COMP128
);
1975 static int vfio_setup_pcie_cap(VFIOPCIDevice
*vdev
, int pos
, uint8_t size
,
1981 flags
= pci_get_word(vdev
->pdev
.config
+ pos
+ PCI_CAP_FLAGS
);
1982 type
= (flags
& PCI_EXP_FLAGS_TYPE
) >> 4;
1984 if (type
!= PCI_EXP_TYPE_ENDPOINT
&&
1985 type
!= PCI_EXP_TYPE_LEG_END
&&
1986 type
!= PCI_EXP_TYPE_RC_END
) {
1988 error_setg(errp
, "assignment of PCIe type 0x%x "
1989 "devices is not currently supported", type
);
1993 if (!pci_bus_is_express(pci_get_bus(&vdev
->pdev
))) {
1994 PCIBus
*bus
= pci_get_bus(&vdev
->pdev
);
1998 * Traditionally PCI device assignment exposes the PCIe capability
1999 * as-is on non-express buses. The reason being that some drivers
2000 * simply assume that it's there, for example tg3. However when
2001 * we're running on a native PCIe machine type, like Q35, we need
2002 * to hide the PCIe capability. The reason for this is twofold;
2003 * first Windows guests get a Code 10 error when the PCIe capability
2004 * is exposed in this configuration. Therefore express devices won't
2005 * work at all unless they're attached to express buses in the VM.
2006 * Second, a native PCIe machine introduces the possibility of fine
2007 * granularity IOMMUs supporting both translation and isolation.
2008 * Guest code to discover the IOMMU visibility of a device, such as
2009 * IOMMU grouping code on Linux, is very aware of device types and
2010 * valid transitions between bus types. An express device on a non-
2011 * express bus is not a valid combination on bare metal systems.
2013 * Drivers that require a PCIe capability to make the device
2014 * functional are simply going to need to have their devices placed
2015 * on a PCIe bus in the VM.
2017 while (!pci_bus_is_root(bus
)) {
2018 bridge
= pci_bridge_get_device(bus
);
2019 bus
= pci_get_bus(bridge
);
2022 if (pci_bus_is_express(bus
)) {
2026 } else if (pci_bus_is_root(pci_get_bus(&vdev
->pdev
))) {
2028 * On a Root Complex bus Endpoints become Root Complex Integrated
2029 * Endpoints, which changes the type and clears the LNK & LNK2 fields.
2031 if (type
== PCI_EXP_TYPE_ENDPOINT
) {
2032 vfio_add_emulated_word(vdev
, pos
+ PCI_CAP_FLAGS
,
2033 PCI_EXP_TYPE_RC_END
<< 4,
2034 PCI_EXP_FLAGS_TYPE
);
2036 /* Link Capabilities, Status, and Control goes away */
2037 if (size
> PCI_EXP_LNKCTL
) {
2038 vfio_add_emulated_long(vdev
, pos
+ PCI_EXP_LNKCAP
, 0, ~0);
2039 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKCTL
, 0, ~0);
2040 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKSTA
, 0, ~0);
2042 #ifndef PCI_EXP_LNKCAP2
2043 #define PCI_EXP_LNKCAP2 44
2045 #ifndef PCI_EXP_LNKSTA2
2046 #define PCI_EXP_LNKSTA2 50
2048 /* Link 2 Capabilities, Status, and Control goes away */
2049 if (size
> PCI_EXP_LNKCAP2
) {
2050 vfio_add_emulated_long(vdev
, pos
+ PCI_EXP_LNKCAP2
, 0, ~0);
2051 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKCTL2
, 0, ~0);
2052 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKSTA2
, 0, ~0);
2056 } else if (type
== PCI_EXP_TYPE_LEG_END
) {
2058 * Legacy endpoints don't belong on the root complex. Windows
2059 * seems to be happier with devices if we skip the capability.
2066 * Convert Root Complex Integrated Endpoints to regular endpoints.
2067 * These devices don't support LNK/LNK2 capabilities, so make them up.
2069 if (type
== PCI_EXP_TYPE_RC_END
) {
2070 vfio_add_emulated_word(vdev
, pos
+ PCI_CAP_FLAGS
,
2071 PCI_EXP_TYPE_ENDPOINT
<< 4,
2072 PCI_EXP_FLAGS_TYPE
);
2073 vfio_add_emulated_long(vdev
, pos
+ PCI_EXP_LNKCAP
,
2074 QEMU_PCI_EXP_LNKCAP_MLW(QEMU_PCI_EXP_LNK_X1
) |
2075 QEMU_PCI_EXP_LNKCAP_MLS(QEMU_PCI_EXP_LNK_2_5GT
), ~0);
2076 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKCTL
, 0, ~0);
2079 vfio_pci_enable_rp_atomics(vdev
);
2083 * Intel 82599 SR-IOV VFs report an invalid PCIe capability version 0
2084 * (Niantic errate #35) causing Windows to error with a Code 10 for the
2085 * device on Q35. Fixup any such devices to report version 1. If we
2086 * were to remove the capability entirely the guest would lose extended
2089 if ((flags
& PCI_EXP_FLAGS_VERS
) == 0) {
2090 vfio_add_emulated_word(vdev
, pos
+ PCI_CAP_FLAGS
,
2091 1, PCI_EXP_FLAGS_VERS
);
2094 pos
= pci_add_capability(&vdev
->pdev
, PCI_CAP_ID_EXP
, pos
, size
,
2100 vdev
->pdev
.exp
.exp_cap
= pos
;
2105 static void vfio_check_pcie_flr(VFIOPCIDevice
*vdev
, uint8_t pos
)
2107 uint32_t cap
= pci_get_long(vdev
->pdev
.config
+ pos
+ PCI_EXP_DEVCAP
);
2109 if (cap
& PCI_EXP_DEVCAP_FLR
) {
2110 trace_vfio_check_pcie_flr(vdev
->vbasedev
.name
);
2111 vdev
->has_flr
= true;
2115 static void vfio_check_pm_reset(VFIOPCIDevice
*vdev
, uint8_t pos
)
2117 uint16_t csr
= pci_get_word(vdev
->pdev
.config
+ pos
+ PCI_PM_CTRL
);
2119 if (!(csr
& PCI_PM_CTRL_NO_SOFT_RESET
)) {
2120 trace_vfio_check_pm_reset(vdev
->vbasedev
.name
);
2121 vdev
->has_pm_reset
= true;
2125 static void vfio_check_af_flr(VFIOPCIDevice
*vdev
, uint8_t pos
)
2127 uint8_t cap
= pci_get_byte(vdev
->pdev
.config
+ pos
+ PCI_AF_CAP
);
2129 if ((cap
& PCI_AF_CAP_TP
) && (cap
& PCI_AF_CAP_FLR
)) {
2130 trace_vfio_check_af_flr(vdev
->vbasedev
.name
);
2131 vdev
->has_flr
= true;
2135 static int vfio_add_std_cap(VFIOPCIDevice
*vdev
, uint8_t pos
, Error
**errp
)
2137 PCIDevice
*pdev
= &vdev
->pdev
;
2138 uint8_t cap_id
, next
, size
;
2141 cap_id
= pdev
->config
[pos
];
2142 next
= pdev
->config
[pos
+ PCI_CAP_LIST_NEXT
];
2145 * If it becomes important to configure capabilities to their actual
2146 * size, use this as the default when it's something we don't recognize.
2147 * Since QEMU doesn't actually handle many of the config accesses,
2148 * exact size doesn't seem worthwhile.
2150 size
= vfio_std_cap_max_size(pdev
, pos
);
2153 * pci_add_capability always inserts the new capability at the head
2154 * of the chain. Therefore to end up with a chain that matches the
2155 * physical device, we insert from the end by making this recursive.
2156 * This is also why we pre-calculate size above as cached config space
2157 * will be changed as we unwind the stack.
2160 ret
= vfio_add_std_cap(vdev
, next
, errp
);
2165 /* Begin the rebuild, use QEMU emulated list bits */
2166 pdev
->config
[PCI_CAPABILITY_LIST
] = 0;
2167 vdev
->emulated_config_bits
[PCI_CAPABILITY_LIST
] = 0xff;
2168 vdev
->emulated_config_bits
[PCI_STATUS
] |= PCI_STATUS_CAP_LIST
;
2170 ret
= vfio_add_virt_caps(vdev
, errp
);
2176 /* Scale down size, esp in case virt caps were added above */
2177 size
= MIN(size
, vfio_std_cap_max_size(pdev
, pos
));
2179 /* Use emulated next pointer to allow dropping caps */
2180 pci_set_byte(vdev
->emulated_config_bits
+ pos
+ PCI_CAP_LIST_NEXT
, 0xff);
2183 case PCI_CAP_ID_MSI
:
2184 ret
= vfio_msi_setup(vdev
, pos
, errp
);
2186 case PCI_CAP_ID_EXP
:
2187 vfio_check_pcie_flr(vdev
, pos
);
2188 ret
= vfio_setup_pcie_cap(vdev
, pos
, size
, errp
);
2190 case PCI_CAP_ID_MSIX
:
2191 ret
= vfio_msix_setup(vdev
, pos
, errp
);
2194 vfio_check_pm_reset(vdev
, pos
);
2196 ret
= pci_add_capability(pdev
, cap_id
, pos
, size
, errp
);
2199 vfio_check_af_flr(vdev
, pos
);
2200 ret
= pci_add_capability(pdev
, cap_id
, pos
, size
, errp
);
2203 ret
= pci_add_capability(pdev
, cap_id
, pos
, size
, errp
);
2209 "failed to add PCI capability 0x%x[0x%x]@0x%x: ",
2217 static int vfio_setup_rebar_ecap(VFIOPCIDevice
*vdev
, uint16_t pos
)
2222 ctrl
= pci_get_long(vdev
->pdev
.config
+ pos
+ PCI_REBAR_CTRL
);
2223 nbar
= (ctrl
& PCI_REBAR_CTRL_NBAR_MASK
) >> PCI_REBAR_CTRL_NBAR_SHIFT
;
2225 for (i
= 0; i
< nbar
; i
++) {
2229 ctrl
= pci_get_long(vdev
->pdev
.config
+ pos
+ PCI_REBAR_CTRL
+ (i
* 8));
2230 size
= (ctrl
& PCI_REBAR_CTRL_BAR_SIZE
) >> PCI_REBAR_CTRL_BAR_SHIFT
;
2232 /* The cap register reports sizes 1MB to 128TB, with 4 reserved bits */
2233 cap
= size
<= 27 ? 1U << (size
+ 4) : 0;
2236 * The PCIe spec (v6.0.1, 7.8.6) requires HW to support at least one
2237 * size in the range 1MB to 512GB. We intend to mask all sizes except
2238 * the one currently enabled in the size field, therefore if it's
2239 * outside the range, hide the whole capability as this virtualization
2240 * trick won't work. If >512GB resizable BARs start to appear, we
2241 * might need an opt-in or reservation scheme in the kernel.
2243 if (!(cap
& PCI_REBAR_CAP_SIZES
)) {
2247 /* Hide all sizes reported in the ctrl reg per above requirement. */
2248 ctrl
&= (PCI_REBAR_CTRL_BAR_SIZE
|
2249 PCI_REBAR_CTRL_NBAR_MASK
|
2250 PCI_REBAR_CTRL_BAR_IDX
);
2253 * The BAR size field is RW, however we've mangled the capability
2254 * register such that we only report a single size, ie. the current
2255 * BAR size. A write of an unsupported value is undefined, therefore
2256 * the register field is essentially RO.
2258 vfio_add_emulated_long(vdev
, pos
+ PCI_REBAR_CAP
+ (i
* 8), cap
, ~0);
2259 vfio_add_emulated_long(vdev
, pos
+ PCI_REBAR_CTRL
+ (i
* 8), ctrl
, ~0);
2265 static void vfio_add_ext_cap(VFIOPCIDevice
*vdev
)
2267 PCIDevice
*pdev
= &vdev
->pdev
;
2269 uint16_t cap_id
, next
, size
;
2273 /* Only add extended caps if we have them and the guest can see them */
2274 if (!pci_is_express(pdev
) || !pci_bus_is_express(pci_get_bus(pdev
)) ||
2275 !pci_get_long(pdev
->config
+ PCI_CONFIG_SPACE_SIZE
)) {
2280 * pcie_add_capability always inserts the new capability at the tail
2281 * of the chain. Therefore to end up with a chain that matches the
2282 * physical device, we cache the config space to avoid overwriting
2283 * the original config space when we parse the extended capabilities.
2285 config
= g_memdup(pdev
->config
, vdev
->config_size
);
2288 * Extended capabilities are chained with each pointing to the next, so we
2289 * can drop anything other than the head of the chain simply by modifying
2290 * the previous next pointer. Seed the head of the chain here such that
2291 * we can simply skip any capabilities we want to drop below, regardless
2292 * of their position in the chain. If this stub capability still exists
2293 * after we add the capabilities we want to expose, update the capability
2294 * ID to zero. Note that we cannot seed with the capability header being
2295 * zero as this conflicts with definition of an absent capability chain
2296 * and prevents capabilities beyond the head of the list from being added.
2297 * By replacing the dummy capability ID with zero after walking the device
2298 * chain, we also transparently mark extended capabilities as absent if
2299 * no capabilities were added. Note that the PCIe spec defines an absence
2300 * of extended capabilities to be determined by a value of zero for the
2301 * capability ID, version, AND next pointer. A non-zero next pointer
2302 * should be sufficient to indicate additional capabilities are present,
2303 * which will occur if we call pcie_add_capability() below. The entire
2304 * first dword is emulated to support this.
2306 * NB. The kernel side does similar masking, so be prepared that our
2307 * view of the device may also contain a capability ID zero in the head
2308 * of the chain. Skip it for the same reason that we cannot seed the
2309 * chain with a zero capability.
2311 pci_set_long(pdev
->config
+ PCI_CONFIG_SPACE_SIZE
,
2312 PCI_EXT_CAP(0xFFFF, 0, 0));
2313 pci_set_long(pdev
->wmask
+ PCI_CONFIG_SPACE_SIZE
, 0);
2314 pci_set_long(vdev
->emulated_config_bits
+ PCI_CONFIG_SPACE_SIZE
, ~0);
2316 for (next
= PCI_CONFIG_SPACE_SIZE
; next
;
2317 next
= PCI_EXT_CAP_NEXT(pci_get_long(config
+ next
))) {
2318 header
= pci_get_long(config
+ next
);
2319 cap_id
= PCI_EXT_CAP_ID(header
);
2320 cap_ver
= PCI_EXT_CAP_VER(header
);
2323 * If it becomes important to configure extended capabilities to their
2324 * actual size, use this as the default when it's something we don't
2325 * recognize. Since QEMU doesn't actually handle many of the config
2326 * accesses, exact size doesn't seem worthwhile.
2328 size
= vfio_ext_cap_max_size(config
, next
);
2330 /* Use emulated next pointer to allow dropping extended caps */
2331 pci_long_test_and_set_mask(vdev
->emulated_config_bits
+ next
,
2332 PCI_EXT_CAP_NEXT_MASK
);
2335 case 0: /* kernel masked capability */
2336 case PCI_EXT_CAP_ID_SRIOV
: /* Read-only VF BARs confuse OVMF */
2337 case PCI_EXT_CAP_ID_ARI
: /* XXX Needs next function virtualization */
2338 trace_vfio_add_ext_cap_dropped(vdev
->vbasedev
.name
, cap_id
, next
);
2340 case PCI_EXT_CAP_ID_REBAR
:
2341 if (!vfio_setup_rebar_ecap(vdev
, next
)) {
2342 pcie_add_capability(pdev
, cap_id
, cap_ver
, next
, size
);
2346 pcie_add_capability(pdev
, cap_id
, cap_ver
, next
, size
);
2351 /* Cleanup chain head ID if necessary */
2352 if (pci_get_word(pdev
->config
+ PCI_CONFIG_SPACE_SIZE
) == 0xFFFF) {
2353 pci_set_word(pdev
->config
+ PCI_CONFIG_SPACE_SIZE
, 0);
2360 static int vfio_add_capabilities(VFIOPCIDevice
*vdev
, Error
**errp
)
2362 PCIDevice
*pdev
= &vdev
->pdev
;
2365 if (!(pdev
->config
[PCI_STATUS
] & PCI_STATUS_CAP_LIST
) ||
2366 !pdev
->config
[PCI_CAPABILITY_LIST
]) {
2367 return 0; /* Nothing to add */
2370 ret
= vfio_add_std_cap(vdev
, pdev
->config
[PCI_CAPABILITY_LIST
], errp
);
2375 vfio_add_ext_cap(vdev
);
2379 void vfio_pci_pre_reset(VFIOPCIDevice
*vdev
)
2381 PCIDevice
*pdev
= &vdev
->pdev
;
2384 vfio_disable_interrupts(vdev
);
2386 /* Make sure the device is in D0 */
2391 pmcsr
= vfio_pci_read_config(pdev
, vdev
->pm_cap
+ PCI_PM_CTRL
, 2);
2392 state
= pmcsr
& PCI_PM_CTRL_STATE_MASK
;
2394 pmcsr
&= ~PCI_PM_CTRL_STATE_MASK
;
2395 vfio_pci_write_config(pdev
, vdev
->pm_cap
+ PCI_PM_CTRL
, pmcsr
, 2);
2396 /* vfio handles the necessary delay here */
2397 pmcsr
= vfio_pci_read_config(pdev
, vdev
->pm_cap
+ PCI_PM_CTRL
, 2);
2398 state
= pmcsr
& PCI_PM_CTRL_STATE_MASK
;
2400 error_report("vfio: Unable to power on device, stuck in D%d",
2407 * Stop any ongoing DMA by disconnecting I/O, MMIO, and bus master.
2408 * Also put INTx Disable in known state.
2410 cmd
= vfio_pci_read_config(pdev
, PCI_COMMAND
, 2);
2411 cmd
&= ~(PCI_COMMAND_IO
| PCI_COMMAND_MEMORY
| PCI_COMMAND_MASTER
|
2412 PCI_COMMAND_INTX_DISABLE
);
2413 vfio_pci_write_config(pdev
, PCI_COMMAND
, cmd
, 2);
2416 void vfio_pci_post_reset(VFIOPCIDevice
*vdev
)
2421 vfio_intx_enable(vdev
, &err
);
2423 error_reportf_err(err
, VFIO_MSG_PREFIX
, vdev
->vbasedev
.name
);
2426 for (nr
= 0; nr
< PCI_NUM_REGIONS
- 1; ++nr
) {
2427 off_t addr
= vdev
->config_offset
+ PCI_BASE_ADDRESS_0
+ (4 * nr
);
2429 uint32_t len
= sizeof(val
);
2431 if (pwrite(vdev
->vbasedev
.fd
, &val
, len
, addr
) != len
) {
2432 error_report("%s(%s) reset bar %d failed: %m", __func__
,
2433 vdev
->vbasedev
.name
, nr
);
2437 vfio_quirk_reset(vdev
);
2440 bool vfio_pci_host_match(PCIHostDeviceAddress
*addr
, const char *name
)
2444 sprintf(tmp
, "%04x:%02x:%02x.%1x", addr
->domain
,
2445 addr
->bus
, addr
->slot
, addr
->function
);
2447 return (strcmp(tmp
, name
) == 0);
2450 int vfio_pci_get_pci_hot_reset_info(VFIOPCIDevice
*vdev
,
2451 struct vfio_pci_hot_reset_info
**info_p
)
2453 struct vfio_pci_hot_reset_info
*info
;
2456 assert(info_p
&& !*info_p
);
2458 info
= g_malloc0(sizeof(*info
));
2459 info
->argsz
= sizeof(*info
);
2461 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO
, info
);
2462 if (ret
&& errno
!= ENOSPC
) {
2465 if (!vdev
->has_pm_reset
) {
2466 error_report("vfio: Cannot reset device %s, "
2467 "no available reset mechanism.", vdev
->vbasedev
.name
);
2472 count
= info
->count
;
2473 info
= g_realloc(info
, sizeof(*info
) + (count
* sizeof(info
->devices
[0])));
2474 info
->argsz
= sizeof(*info
) + (count
* sizeof(info
->devices
[0]));
2476 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO
, info
);
2480 error_report("vfio: hot reset info failed: %m");
2488 static int vfio_pci_hot_reset(VFIOPCIDevice
*vdev
, bool single
)
2490 VFIODevice
*vbasedev
= &vdev
->vbasedev
;
2491 const VFIOIOMMUOps
*ops
= vbasedev
->bcontainer
->ops
;
2493 return ops
->pci_hot_reset(vbasedev
, single
);
2497 * We want to differentiate hot reset of multiple in-use devices vs hot reset
2498 * of a single in-use device. VFIO_DEVICE_RESET will already handle the case
2499 * of doing hot resets when there is only a single device per bus. The in-use
2500 * here refers to how many VFIODevices are affected. A hot reset that affects
2501 * multiple devices, but only a single in-use device, means that we can call
2502 * it from our bus ->reset() callback since the extent is effectively a single
2503 * device. This allows us to make use of it in the hotplug path. When there
2504 * are multiple in-use devices, we can only trigger the hot reset during a
2505 * system reset and thus from our reset handler. We separate _one vs _multi
2506 * here so that we don't overlap and do a double reset on the system reset
2507 * path where both our reset handler and ->reset() callback are used. Calling
2508 * _one() will only do a hot reset for the one in-use devices case, calling
2509 * _multi() will do nothing if a _one() would have been sufficient.
2511 static int vfio_pci_hot_reset_one(VFIOPCIDevice
*vdev
)
2513 return vfio_pci_hot_reset(vdev
, true);
2516 static int vfio_pci_hot_reset_multi(VFIODevice
*vbasedev
)
2518 VFIOPCIDevice
*vdev
= container_of(vbasedev
, VFIOPCIDevice
, vbasedev
);
2519 return vfio_pci_hot_reset(vdev
, false);
2522 static void vfio_pci_compute_needs_reset(VFIODevice
*vbasedev
)
2524 VFIOPCIDevice
*vdev
= container_of(vbasedev
, VFIOPCIDevice
, vbasedev
);
2525 if (!vbasedev
->reset_works
|| (!vdev
->has_flr
&& vdev
->has_pm_reset
)) {
2526 vbasedev
->needs_reset
= true;
2530 static Object
*vfio_pci_get_object(VFIODevice
*vbasedev
)
2532 VFIOPCIDevice
*vdev
= container_of(vbasedev
, VFIOPCIDevice
, vbasedev
);
2534 return OBJECT(vdev
);
2537 static bool vfio_msix_present(void *opaque
, int version_id
)
2539 PCIDevice
*pdev
= opaque
;
2541 return msix_present(pdev
);
2544 static bool vfio_display_migration_needed(void *opaque
)
2546 VFIOPCIDevice
*vdev
= opaque
;
2549 * We need to migrate the VFIODisplay object if ramfb *migration* was
2550 * explicitly requested (in which case we enforced both ramfb=on and
2551 * display=on), or ramfb migration was left at the default "auto"
2552 * setting, and *ramfb* was explicitly requested (in which case we
2553 * enforced display=on).
2555 return vdev
->ramfb_migrate
== ON_OFF_AUTO_ON
||
2556 (vdev
->ramfb_migrate
== ON_OFF_AUTO_AUTO
&& vdev
->enable_ramfb
);
2559 const VMStateDescription vmstate_vfio_display
= {
2560 .name
= "VFIOPCIDevice/VFIODisplay",
2562 .minimum_version_id
= 1,
2563 .needed
= vfio_display_migration_needed
,
2564 .fields
= (VMStateField
[]){
2565 VMSTATE_STRUCT_POINTER(dpy
, VFIOPCIDevice
, vfio_display_vmstate
,
2567 VMSTATE_END_OF_LIST()
2571 const VMStateDescription vmstate_vfio_pci_config
= {
2572 .name
= "VFIOPCIDevice",
2574 .minimum_version_id
= 1,
2575 .fields
= (VMStateField
[]) {
2576 VMSTATE_PCI_DEVICE(pdev
, VFIOPCIDevice
),
2577 VMSTATE_MSIX_TEST(pdev
, VFIOPCIDevice
, vfio_msix_present
),
2578 VMSTATE_END_OF_LIST()
2580 .subsections
= (const VMStateDescription
* []) {
2581 &vmstate_vfio_display
,
2586 static void vfio_pci_save_config(VFIODevice
*vbasedev
, QEMUFile
*f
)
2588 VFIOPCIDevice
*vdev
= container_of(vbasedev
, VFIOPCIDevice
, vbasedev
);
2590 vmstate_save_state(f
, &vmstate_vfio_pci_config
, vdev
, NULL
);
2593 static int vfio_pci_load_config(VFIODevice
*vbasedev
, QEMUFile
*f
)
2595 VFIOPCIDevice
*vdev
= container_of(vbasedev
, VFIOPCIDevice
, vbasedev
);
2596 PCIDevice
*pdev
= &vdev
->pdev
;
2597 pcibus_t old_addr
[PCI_NUM_REGIONS
- 1];
2600 for (bar
= 0; bar
< PCI_ROM_SLOT
; bar
++) {
2601 old_addr
[bar
] = pdev
->io_regions
[bar
].addr
;
2604 ret
= vmstate_load_state(f
, &vmstate_vfio_pci_config
, vdev
, 1);
2609 vfio_pci_write_config(pdev
, PCI_COMMAND
,
2610 pci_get_word(pdev
->config
+ PCI_COMMAND
), 2);
2612 for (bar
= 0; bar
< PCI_ROM_SLOT
; bar
++) {
2614 * The address may not be changed in some scenarios
2615 * (e.g. the VF driver isn't loaded in VM).
2617 if (old_addr
[bar
] != pdev
->io_regions
[bar
].addr
&&
2618 vdev
->bars
[bar
].region
.size
> 0 &&
2619 vdev
->bars
[bar
].region
.size
< qemu_real_host_page_size()) {
2620 vfio_sub_page_bar_update_mapping(pdev
, bar
);
2624 if (msi_enabled(pdev
)) {
2625 vfio_msi_enable(vdev
);
2626 } else if (msix_enabled(pdev
)) {
2627 vfio_msix_enable(vdev
);
2633 static VFIODeviceOps vfio_pci_ops
= {
2634 .vfio_compute_needs_reset
= vfio_pci_compute_needs_reset
,
2635 .vfio_hot_reset_multi
= vfio_pci_hot_reset_multi
,
2636 .vfio_eoi
= vfio_intx_eoi
,
2637 .vfio_get_object
= vfio_pci_get_object
,
2638 .vfio_save_config
= vfio_pci_save_config
,
2639 .vfio_load_config
= vfio_pci_load_config
,
2642 int vfio_populate_vga(VFIOPCIDevice
*vdev
, Error
**errp
)
2644 VFIODevice
*vbasedev
= &vdev
->vbasedev
;
2645 struct vfio_region_info
*reg_info
;
2648 ret
= vfio_get_region_info(vbasedev
, VFIO_PCI_VGA_REGION_INDEX
, ®_info
);
2650 error_setg_errno(errp
, -ret
,
2651 "failed getting region info for VGA region index %d",
2652 VFIO_PCI_VGA_REGION_INDEX
);
2656 if (!(reg_info
->flags
& VFIO_REGION_INFO_FLAG_READ
) ||
2657 !(reg_info
->flags
& VFIO_REGION_INFO_FLAG_WRITE
) ||
2658 reg_info
->size
< 0xbffff + 1) {
2659 error_setg(errp
, "unexpected VGA info, flags 0x%lx, size 0x%lx",
2660 (unsigned long)reg_info
->flags
,
2661 (unsigned long)reg_info
->size
);
2666 vdev
->vga
= g_new0(VFIOVGA
, 1);
2668 vdev
->vga
->fd_offset
= reg_info
->offset
;
2669 vdev
->vga
->fd
= vdev
->vbasedev
.fd
;
2673 vdev
->vga
->region
[QEMU_PCI_VGA_MEM
].offset
= QEMU_PCI_VGA_MEM_BASE
;
2674 vdev
->vga
->region
[QEMU_PCI_VGA_MEM
].nr
= QEMU_PCI_VGA_MEM
;
2675 QLIST_INIT(&vdev
->vga
->region
[QEMU_PCI_VGA_MEM
].quirks
);
2677 memory_region_init_io(&vdev
->vga
->region
[QEMU_PCI_VGA_MEM
].mem
,
2678 OBJECT(vdev
), &vfio_vga_ops
,
2679 &vdev
->vga
->region
[QEMU_PCI_VGA_MEM
],
2680 "vfio-vga-mmio@0xa0000",
2681 QEMU_PCI_VGA_MEM_SIZE
);
2683 vdev
->vga
->region
[QEMU_PCI_VGA_IO_LO
].offset
= QEMU_PCI_VGA_IO_LO_BASE
;
2684 vdev
->vga
->region
[QEMU_PCI_VGA_IO_LO
].nr
= QEMU_PCI_VGA_IO_LO
;
2685 QLIST_INIT(&vdev
->vga
->region
[QEMU_PCI_VGA_IO_LO
].quirks
);
2687 memory_region_init_io(&vdev
->vga
->region
[QEMU_PCI_VGA_IO_LO
].mem
,
2688 OBJECT(vdev
), &vfio_vga_ops
,
2689 &vdev
->vga
->region
[QEMU_PCI_VGA_IO_LO
],
2690 "vfio-vga-io@0x3b0",
2691 QEMU_PCI_VGA_IO_LO_SIZE
);
2693 vdev
->vga
->region
[QEMU_PCI_VGA_IO_HI
].offset
= QEMU_PCI_VGA_IO_HI_BASE
;
2694 vdev
->vga
->region
[QEMU_PCI_VGA_IO_HI
].nr
= QEMU_PCI_VGA_IO_HI
;
2695 QLIST_INIT(&vdev
->vga
->region
[QEMU_PCI_VGA_IO_HI
].quirks
);
2697 memory_region_init_io(&vdev
->vga
->region
[QEMU_PCI_VGA_IO_HI
].mem
,
2698 OBJECT(vdev
), &vfio_vga_ops
,
2699 &vdev
->vga
->region
[QEMU_PCI_VGA_IO_HI
],
2700 "vfio-vga-io@0x3c0",
2701 QEMU_PCI_VGA_IO_HI_SIZE
);
2703 pci_register_vga(&vdev
->pdev
, &vdev
->vga
->region
[QEMU_PCI_VGA_MEM
].mem
,
2704 &vdev
->vga
->region
[QEMU_PCI_VGA_IO_LO
].mem
,
2705 &vdev
->vga
->region
[QEMU_PCI_VGA_IO_HI
].mem
);
2710 static void vfio_populate_device(VFIOPCIDevice
*vdev
, Error
**errp
)
2712 VFIODevice
*vbasedev
= &vdev
->vbasedev
;
2713 struct vfio_region_info
*reg_info
;
2714 struct vfio_irq_info irq_info
= { .argsz
= sizeof(irq_info
) };
2717 /* Sanity check device */
2718 if (!(vbasedev
->flags
& VFIO_DEVICE_FLAGS_PCI
)) {
2719 error_setg(errp
, "this isn't a PCI device");
2723 if (vbasedev
->num_regions
< VFIO_PCI_CONFIG_REGION_INDEX
+ 1) {
2724 error_setg(errp
, "unexpected number of io regions %u",
2725 vbasedev
->num_regions
);
2729 if (vbasedev
->num_irqs
< VFIO_PCI_MSIX_IRQ_INDEX
+ 1) {
2730 error_setg(errp
, "unexpected number of irqs %u", vbasedev
->num_irqs
);
2734 for (i
= VFIO_PCI_BAR0_REGION_INDEX
; i
< VFIO_PCI_ROM_REGION_INDEX
; i
++) {
2735 char *name
= g_strdup_printf("%s BAR %d", vbasedev
->name
, i
);
2737 ret
= vfio_region_setup(OBJECT(vdev
), vbasedev
,
2738 &vdev
->bars
[i
].region
, i
, name
);
2742 error_setg_errno(errp
, -ret
, "failed to get region %d info", i
);
2746 QLIST_INIT(&vdev
->bars
[i
].quirks
);
2749 ret
= vfio_get_region_info(vbasedev
,
2750 VFIO_PCI_CONFIG_REGION_INDEX
, ®_info
);
2752 error_setg_errno(errp
, -ret
, "failed to get config info");
2756 trace_vfio_populate_device_config(vdev
->vbasedev
.name
,
2757 (unsigned long)reg_info
->size
,
2758 (unsigned long)reg_info
->offset
,
2759 (unsigned long)reg_info
->flags
);
2761 vdev
->config_size
= reg_info
->size
;
2762 if (vdev
->config_size
== PCI_CONFIG_SPACE_SIZE
) {
2763 vdev
->pdev
.cap_present
&= ~QEMU_PCI_CAP_EXPRESS
;
2765 vdev
->config_offset
= reg_info
->offset
;
2769 if (vdev
->features
& VFIO_FEATURE_ENABLE_VGA
) {
2770 ret
= vfio_populate_vga(vdev
, errp
);
2772 error_append_hint(errp
, "device does not support "
2773 "requested feature x-vga\n");
2778 irq_info
.index
= VFIO_PCI_ERR_IRQ_INDEX
;
2780 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_GET_IRQ_INFO
, &irq_info
);
2782 /* This can fail for an old kernel or legacy PCI dev */
2783 trace_vfio_populate_device_get_irq_info_failure(strerror(errno
));
2784 } else if (irq_info
.count
== 1) {
2785 vdev
->pci_aer
= true;
2787 warn_report(VFIO_MSG_PREFIX
2788 "Could not enable error recovery for the device",
2793 static void vfio_pci_put_device(VFIOPCIDevice
*vdev
)
2795 vfio_detach_device(&vdev
->vbasedev
);
2797 g_free(vdev
->vbasedev
.name
);
2801 static void vfio_err_notifier_handler(void *opaque
)
2803 VFIOPCIDevice
*vdev
= opaque
;
2805 if (!event_notifier_test_and_clear(&vdev
->err_notifier
)) {
2810 * TBD. Retrieve the error details and decide what action
2811 * needs to be taken. One of the actions could be to pass
2812 * the error to the guest and have the guest driver recover
2813 * from the error. This requires that PCIe capabilities be
2814 * exposed to the guest. For now, we just terminate the
2815 * guest to contain the error.
2818 error_report("%s(%s) Unrecoverable error detected. Please collect any data possible and then kill the guest", __func__
, vdev
->vbasedev
.name
);
2820 vm_stop(RUN_STATE_INTERNAL_ERROR
);
2824 * Registers error notifier for devices supporting error recovery.
2825 * If we encounter a failure in this function, we report an error
2826 * and continue after disabling error recovery support for the
2829 static void vfio_register_err_notifier(VFIOPCIDevice
*vdev
)
2834 if (!vdev
->pci_aer
) {
2838 if (event_notifier_init(&vdev
->err_notifier
, 0)) {
2839 error_report("vfio: Unable to init event notifier for error detection");
2840 vdev
->pci_aer
= false;
2844 fd
= event_notifier_get_fd(&vdev
->err_notifier
);
2845 qemu_set_fd_handler(fd
, vfio_err_notifier_handler
, NULL
, vdev
);
2847 if (vfio_set_irq_signaling(&vdev
->vbasedev
, VFIO_PCI_ERR_IRQ_INDEX
, 0,
2848 VFIO_IRQ_SET_ACTION_TRIGGER
, fd
, &err
)) {
2849 error_reportf_err(err
, VFIO_MSG_PREFIX
, vdev
->vbasedev
.name
);
2850 qemu_set_fd_handler(fd
, NULL
, NULL
, vdev
);
2851 event_notifier_cleanup(&vdev
->err_notifier
);
2852 vdev
->pci_aer
= false;
2856 static void vfio_unregister_err_notifier(VFIOPCIDevice
*vdev
)
2860 if (!vdev
->pci_aer
) {
2864 if (vfio_set_irq_signaling(&vdev
->vbasedev
, VFIO_PCI_ERR_IRQ_INDEX
, 0,
2865 VFIO_IRQ_SET_ACTION_TRIGGER
, -1, &err
)) {
2866 error_reportf_err(err
, VFIO_MSG_PREFIX
, vdev
->vbasedev
.name
);
2868 qemu_set_fd_handler(event_notifier_get_fd(&vdev
->err_notifier
),
2870 event_notifier_cleanup(&vdev
->err_notifier
);
2873 static void vfio_req_notifier_handler(void *opaque
)
2875 VFIOPCIDevice
*vdev
= opaque
;
2878 if (!event_notifier_test_and_clear(&vdev
->req_notifier
)) {
2882 qdev_unplug(DEVICE(vdev
), &err
);
2884 warn_reportf_err(err
, VFIO_MSG_PREFIX
, vdev
->vbasedev
.name
);
2888 static void vfio_register_req_notifier(VFIOPCIDevice
*vdev
)
2890 struct vfio_irq_info irq_info
= { .argsz
= sizeof(irq_info
),
2891 .index
= VFIO_PCI_REQ_IRQ_INDEX
};
2895 if (!(vdev
->features
& VFIO_FEATURE_ENABLE_REQ
)) {
2899 if (ioctl(vdev
->vbasedev
.fd
,
2900 VFIO_DEVICE_GET_IRQ_INFO
, &irq_info
) < 0 || irq_info
.count
< 1) {
2904 if (event_notifier_init(&vdev
->req_notifier
, 0)) {
2905 error_report("vfio: Unable to init event notifier for device request");
2909 fd
= event_notifier_get_fd(&vdev
->req_notifier
);
2910 qemu_set_fd_handler(fd
, vfio_req_notifier_handler
, NULL
, vdev
);
2912 if (vfio_set_irq_signaling(&vdev
->vbasedev
, VFIO_PCI_REQ_IRQ_INDEX
, 0,
2913 VFIO_IRQ_SET_ACTION_TRIGGER
, fd
, &err
)) {
2914 error_reportf_err(err
, VFIO_MSG_PREFIX
, vdev
->vbasedev
.name
);
2915 qemu_set_fd_handler(fd
, NULL
, NULL
, vdev
);
2916 event_notifier_cleanup(&vdev
->req_notifier
);
2918 vdev
->req_enabled
= true;
2922 static void vfio_unregister_req_notifier(VFIOPCIDevice
*vdev
)
2926 if (!vdev
->req_enabled
) {
2930 if (vfio_set_irq_signaling(&vdev
->vbasedev
, VFIO_PCI_REQ_IRQ_INDEX
, 0,
2931 VFIO_IRQ_SET_ACTION_TRIGGER
, -1, &err
)) {
2932 error_reportf_err(err
, VFIO_MSG_PREFIX
, vdev
->vbasedev
.name
);
2934 qemu_set_fd_handler(event_notifier_get_fd(&vdev
->req_notifier
),
2936 event_notifier_cleanup(&vdev
->req_notifier
);
2938 vdev
->req_enabled
= false;
2941 static void vfio_realize(PCIDevice
*pdev
, Error
**errp
)
2943 VFIOPCIDevice
*vdev
= VFIO_PCI(pdev
);
2944 VFIODevice
*vbasedev
= &vdev
->vbasedev
;
2949 char uuid
[UUID_STR_LEN
];
2952 if (vbasedev
->fd
< 0 && !vbasedev
->sysfsdev
) {
2953 if (!(~vdev
->host
.domain
|| ~vdev
->host
.bus
||
2954 ~vdev
->host
.slot
|| ~vdev
->host
.function
)) {
2955 error_setg(errp
, "No provided host device");
2956 error_append_hint(errp
, "Use -device vfio-pci,host=DDDD:BB:DD.F "
2957 #ifdef CONFIG_IOMMUFD
2958 "or -device vfio-pci,fd=DEVICE_FD "
2960 "or -device vfio-pci,sysfsdev=PATH_TO_DEVICE\n");
2963 vbasedev
->sysfsdev
=
2964 g_strdup_printf("/sys/bus/pci/devices/%04x:%02x:%02x.%01x",
2965 vdev
->host
.domain
, vdev
->host
.bus
,
2966 vdev
->host
.slot
, vdev
->host
.function
);
2969 if (vfio_device_get_name(vbasedev
, errp
) < 0) {
2972 vbasedev
->ops
= &vfio_pci_ops
;
2973 vbasedev
->type
= VFIO_DEVICE_TYPE_PCI
;
2974 vbasedev
->dev
= DEVICE(vdev
);
2977 * Mediated devices *might* operate compatibly with discarding of RAM, but
2978 * we cannot know for certain, it depends on whether the mdev vendor driver
2979 * stays in sync with the active working set of the guest driver. Prevent
2980 * the x-balloon-allowed option unless this is minimally an mdev device.
2982 tmp
= g_strdup_printf("%s/subsystem", vbasedev
->sysfsdev
);
2983 subsys
= realpath(tmp
, NULL
);
2985 is_mdev
= subsys
&& (strcmp(subsys
, "/sys/bus/mdev") == 0);
2988 trace_vfio_mdev(vbasedev
->name
, is_mdev
);
2990 if (vbasedev
->ram_block_discard_allowed
&& !is_mdev
) {
2991 error_setg(errp
, "x-balloon-allowed only potentially compatible "
2992 "with mdev devices");
2996 if (!qemu_uuid_is_null(&vdev
->vf_token
)) {
2997 qemu_uuid_unparse(&vdev
->vf_token
, uuid
);
2998 name
= g_strdup_printf("%s vf_token=%s", vbasedev
->name
, uuid
);
3000 name
= g_strdup(vbasedev
->name
);
3003 ret
= vfio_attach_device(name
, vbasedev
,
3004 pci_device_iommu_address_space(pdev
), errp
);
3010 vfio_populate_device(vdev
, &err
);
3012 error_propagate(errp
, err
);
3016 /* Get a copy of config space */
3017 ret
= pread(vbasedev
->fd
, vdev
->pdev
.config
,
3018 MIN(pci_config_size(&vdev
->pdev
), vdev
->config_size
),
3019 vdev
->config_offset
);
3020 if (ret
< (int)MIN(pci_config_size(&vdev
->pdev
), vdev
->config_size
)) {
3021 ret
= ret
< 0 ? -errno
: -EFAULT
;
3022 error_setg_errno(errp
, -ret
, "failed to read device config space");
3026 /* vfio emulates a lot for us, but some bits need extra love */
3027 vdev
->emulated_config_bits
= g_malloc0(vdev
->config_size
);
3029 /* QEMU can choose to expose the ROM or not */
3030 memset(vdev
->emulated_config_bits
+ PCI_ROM_ADDRESS
, 0xff, 4);
3031 /* QEMU can also add or extend BARs */
3032 memset(vdev
->emulated_config_bits
+ PCI_BASE_ADDRESS_0
, 0xff, 6 * 4);
3035 * The PCI spec reserves vendor ID 0xffff as an invalid value. The
3036 * device ID is managed by the vendor and need only be a 16-bit value.
3037 * Allow any 16-bit value for subsystem so they can be hidden or changed.
3039 if (vdev
->vendor_id
!= PCI_ANY_ID
) {
3040 if (vdev
->vendor_id
>= 0xffff) {
3041 error_setg(errp
, "invalid PCI vendor ID provided");
3044 vfio_add_emulated_word(vdev
, PCI_VENDOR_ID
, vdev
->vendor_id
, ~0);
3045 trace_vfio_pci_emulated_vendor_id(vbasedev
->name
, vdev
->vendor_id
);
3047 vdev
->vendor_id
= pci_get_word(pdev
->config
+ PCI_VENDOR_ID
);
3050 if (vdev
->device_id
!= PCI_ANY_ID
) {
3051 if (vdev
->device_id
> 0xffff) {
3052 error_setg(errp
, "invalid PCI device ID provided");
3055 vfio_add_emulated_word(vdev
, PCI_DEVICE_ID
, vdev
->device_id
, ~0);
3056 trace_vfio_pci_emulated_device_id(vbasedev
->name
, vdev
->device_id
);
3058 vdev
->device_id
= pci_get_word(pdev
->config
+ PCI_DEVICE_ID
);
3061 if (vdev
->sub_vendor_id
!= PCI_ANY_ID
) {
3062 if (vdev
->sub_vendor_id
> 0xffff) {
3063 error_setg(errp
, "invalid PCI subsystem vendor ID provided");
3066 vfio_add_emulated_word(vdev
, PCI_SUBSYSTEM_VENDOR_ID
,
3067 vdev
->sub_vendor_id
, ~0);
3068 trace_vfio_pci_emulated_sub_vendor_id(vbasedev
->name
,
3069 vdev
->sub_vendor_id
);
3072 if (vdev
->sub_device_id
!= PCI_ANY_ID
) {
3073 if (vdev
->sub_device_id
> 0xffff) {
3074 error_setg(errp
, "invalid PCI subsystem device ID provided");
3077 vfio_add_emulated_word(vdev
, PCI_SUBSYSTEM_ID
, vdev
->sub_device_id
, ~0);
3078 trace_vfio_pci_emulated_sub_device_id(vbasedev
->name
,
3079 vdev
->sub_device_id
);
3082 /* QEMU can change multi-function devices to single function, or reverse */
3083 vdev
->emulated_config_bits
[PCI_HEADER_TYPE
] =
3084 PCI_HEADER_TYPE_MULTI_FUNCTION
;
3086 /* Restore or clear multifunction, this is always controlled by QEMU */
3087 if (vdev
->pdev
.cap_present
& QEMU_PCI_CAP_MULTIFUNCTION
) {
3088 vdev
->pdev
.config
[PCI_HEADER_TYPE
] |= PCI_HEADER_TYPE_MULTI_FUNCTION
;
3090 vdev
->pdev
.config
[PCI_HEADER_TYPE
] &= ~PCI_HEADER_TYPE_MULTI_FUNCTION
;
3094 * Clear host resource mapping info. If we choose not to register a
3095 * BAR, such as might be the case with the option ROM, we can get
3096 * confusing, unwritable, residual addresses from the host here.
3098 memset(&vdev
->pdev
.config
[PCI_BASE_ADDRESS_0
], 0, 24);
3099 memset(&vdev
->pdev
.config
[PCI_ROM_ADDRESS
], 0, 4);
3101 vfio_pci_size_rom(vdev
);
3103 vfio_bars_prepare(vdev
);
3105 vfio_msix_early_setup(vdev
, &err
);
3107 error_propagate(errp
, err
);
3111 vfio_bars_register(vdev
);
3113 ret
= vfio_add_capabilities(vdev
, errp
);
3119 vfio_vga_quirk_setup(vdev
);
3122 for (i
= 0; i
< PCI_ROM_SLOT
; i
++) {
3123 vfio_bar_quirk_setup(vdev
, i
);
3126 if (!vdev
->igd_opregion
&&
3127 vdev
->features
& VFIO_FEATURE_ENABLE_IGD_OPREGION
) {
3128 struct vfio_region_info
*opregion
;
3130 if (vdev
->pdev
.qdev
.hotplugged
) {
3132 "cannot support IGD OpRegion feature on hotplugged "
3137 ret
= vfio_get_dev_region_info(vbasedev
,
3138 VFIO_REGION_TYPE_PCI_VENDOR_TYPE
| PCI_VENDOR_ID_INTEL
,
3139 VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION
, &opregion
);
3141 error_setg_errno(errp
, -ret
,
3142 "does not support requested IGD OpRegion feature");
3146 ret
= vfio_pci_igd_opregion_init(vdev
, opregion
, errp
);
3153 /* QEMU emulates all of MSI & MSIX */
3154 if (pdev
->cap_present
& QEMU_PCI_CAP_MSIX
) {
3155 memset(vdev
->emulated_config_bits
+ pdev
->msix_cap
, 0xff,
3159 if (pdev
->cap_present
& QEMU_PCI_CAP_MSI
) {
3160 memset(vdev
->emulated_config_bits
+ pdev
->msi_cap
, 0xff,
3161 vdev
->msi_cap_size
);
3164 if (vfio_pci_read_config(&vdev
->pdev
, PCI_INTERRUPT_PIN
, 1)) {
3165 vdev
->intx
.mmap_timer
= timer_new_ms(QEMU_CLOCK_VIRTUAL
,
3166 vfio_intx_mmap_enable
, vdev
);
3167 pci_device_set_intx_routing_notifier(&vdev
->pdev
,
3168 vfio_intx_routing_notifier
);
3169 vdev
->irqchip_change_notifier
.notify
= vfio_irqchip_change
;
3170 kvm_irqchip_add_change_notifier(&vdev
->irqchip_change_notifier
);
3171 ret
= vfio_intx_enable(vdev
, errp
);
3173 goto out_deregister
;
3177 if (vdev
->display
!= ON_OFF_AUTO_OFF
) {
3178 ret
= vfio_display_probe(vdev
, errp
);
3180 goto out_deregister
;
3183 if (vdev
->enable_ramfb
&& vdev
->dpy
== NULL
) {
3184 error_setg(errp
, "ramfb=on requires display=on");
3185 goto out_deregister
;
3187 if (vdev
->display_xres
|| vdev
->display_yres
) {
3188 if (vdev
->dpy
== NULL
) {
3189 error_setg(errp
, "xres and yres properties require display=on");
3190 goto out_deregister
;
3192 if (vdev
->dpy
->edid_regs
== NULL
) {
3193 error_setg(errp
, "xres and yres properties need edid support");
3194 goto out_deregister
;
3198 if (vdev
->ramfb_migrate
== ON_OFF_AUTO_ON
&& !vdev
->enable_ramfb
) {
3199 warn_report("x-ramfb-migrate=on but ramfb=off. "
3200 "Forcing x-ramfb-migrate to off.");
3201 vdev
->ramfb_migrate
= ON_OFF_AUTO_OFF
;
3203 if (vbasedev
->enable_migration
== ON_OFF_AUTO_OFF
) {
3204 if (vdev
->ramfb_migrate
== ON_OFF_AUTO_AUTO
) {
3205 vdev
->ramfb_migrate
= ON_OFF_AUTO_OFF
;
3206 } else if (vdev
->ramfb_migrate
== ON_OFF_AUTO_ON
) {
3207 error_setg(errp
, "x-ramfb-migrate requires enable-migration");
3208 goto out_deregister
;
3212 if (!pdev
->failover_pair_id
) {
3213 if (!vfio_migration_realize(vbasedev
, errp
)) {
3214 goto out_deregister
;
3218 vfio_register_err_notifier(vdev
);
3219 vfio_register_req_notifier(vdev
);
3220 vfio_setup_resetfn_quirk(vdev
);
3225 if (vdev
->interrupt
== VFIO_INT_INTx
) {
3226 vfio_intx_disable(vdev
);
3228 pci_device_set_intx_routing_notifier(&vdev
->pdev
, NULL
);
3229 if (vdev
->irqchip_change_notifier
.notify
) {
3230 kvm_irqchip_remove_change_notifier(&vdev
->irqchip_change_notifier
);
3232 if (vdev
->intx
.mmap_timer
) {
3233 timer_free(vdev
->intx
.mmap_timer
);
3236 vfio_teardown_msi(vdev
);
3237 vfio_bars_exit(vdev
);
3239 error_prepend(errp
, VFIO_MSG_PREFIX
, vbasedev
->name
);
3242 static void vfio_instance_finalize(Object
*obj
)
3244 VFIOPCIDevice
*vdev
= VFIO_PCI(obj
);
3246 vfio_display_finalize(vdev
);
3247 vfio_bars_finalize(vdev
);
3248 g_free(vdev
->emulated_config_bits
);
3251 * XXX Leaking igd_opregion is not an oversight, we can't remove the
3252 * fw_cfg entry therefore leaking this allocation seems like the safest
3255 * g_free(vdev->igd_opregion);
3257 vfio_pci_put_device(vdev
);
3260 static void vfio_exitfn(PCIDevice
*pdev
)
3262 VFIOPCIDevice
*vdev
= VFIO_PCI(pdev
);
3264 vfio_unregister_req_notifier(vdev
);
3265 vfio_unregister_err_notifier(vdev
);
3266 pci_device_set_intx_routing_notifier(&vdev
->pdev
, NULL
);
3267 if (vdev
->irqchip_change_notifier
.notify
) {
3268 kvm_irqchip_remove_change_notifier(&vdev
->irqchip_change_notifier
);
3270 vfio_disable_interrupts(vdev
);
3271 if (vdev
->intx
.mmap_timer
) {
3272 timer_free(vdev
->intx
.mmap_timer
);
3274 vfio_teardown_msi(vdev
);
3275 vfio_pci_disable_rp_atomics(vdev
);
3276 vfio_bars_exit(vdev
);
3277 vfio_migration_exit(&vdev
->vbasedev
);
3280 static void vfio_pci_reset(DeviceState
*dev
)
3282 VFIOPCIDevice
*vdev
= VFIO_PCI(dev
);
3284 trace_vfio_pci_reset(vdev
->vbasedev
.name
);
3286 vfio_pci_pre_reset(vdev
);
3288 if (vdev
->display
!= ON_OFF_AUTO_OFF
) {
3289 vfio_display_reset(vdev
);
3292 if (vdev
->resetfn
&& !vdev
->resetfn(vdev
)) {
3296 if (vdev
->vbasedev
.reset_works
&&
3297 (vdev
->has_flr
|| !vdev
->has_pm_reset
) &&
3298 !ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_RESET
)) {
3299 trace_vfio_pci_reset_flr(vdev
->vbasedev
.name
);
3303 /* See if we can do our own bus reset */
3304 if (!vfio_pci_hot_reset_one(vdev
)) {
3308 /* If nothing else works and the device supports PM reset, use it */
3309 if (vdev
->vbasedev
.reset_works
&& vdev
->has_pm_reset
&&
3310 !ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_RESET
)) {
3311 trace_vfio_pci_reset_pm(vdev
->vbasedev
.name
);
3316 vfio_pci_post_reset(vdev
);
3319 static void vfio_instance_init(Object
*obj
)
3321 PCIDevice
*pci_dev
= PCI_DEVICE(obj
);
3322 VFIOPCIDevice
*vdev
= VFIO_PCI(obj
);
3324 device_add_bootindex_property(obj
, &vdev
->bootindex
,
3327 vdev
->host
.domain
= ~0U;
3328 vdev
->host
.bus
= ~0U;
3329 vdev
->host
.slot
= ~0U;
3330 vdev
->host
.function
= ~0U;
3331 vdev
->vbasedev
.fd
= -1;
3333 vdev
->nv_gpudirect_clique
= 0xFF;
3335 /* QEMU_PCI_CAP_EXPRESS initialization does not depend on QEMU command
3336 * line, therefore, no need to wait to realize like other devices */
3337 pci_dev
->cap_present
|= QEMU_PCI_CAP_EXPRESS
;
3340 static Property vfio_pci_dev_properties
[] = {
3341 DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIOPCIDevice
, host
),
3342 DEFINE_PROP_UUID_NODEFAULT("vf-token", VFIOPCIDevice
, vf_token
),
3343 DEFINE_PROP_STRING("sysfsdev", VFIOPCIDevice
, vbasedev
.sysfsdev
),
3344 DEFINE_PROP_ON_OFF_AUTO("x-pre-copy-dirty-page-tracking", VFIOPCIDevice
,
3345 vbasedev
.pre_copy_dirty_page_tracking
,
3347 DEFINE_PROP_ON_OFF_AUTO("display", VFIOPCIDevice
,
3348 display
, ON_OFF_AUTO_OFF
),
3349 DEFINE_PROP_UINT32("xres", VFIOPCIDevice
, display_xres
, 0),
3350 DEFINE_PROP_UINT32("yres", VFIOPCIDevice
, display_yres
, 0),
3351 DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIOPCIDevice
,
3352 intx
.mmap_timeout
, 1100),
3353 DEFINE_PROP_BIT("x-vga", VFIOPCIDevice
, features
,
3354 VFIO_FEATURE_ENABLE_VGA_BIT
, false),
3355 DEFINE_PROP_BIT("x-req", VFIOPCIDevice
, features
,
3356 VFIO_FEATURE_ENABLE_REQ_BIT
, true),
3357 DEFINE_PROP_BIT("x-igd-opregion", VFIOPCIDevice
, features
,
3358 VFIO_FEATURE_ENABLE_IGD_OPREGION_BIT
, false),
3359 DEFINE_PROP_ON_OFF_AUTO("enable-migration", VFIOPCIDevice
,
3360 vbasedev
.enable_migration
, ON_OFF_AUTO_AUTO
),
3361 DEFINE_PROP_BOOL("x-no-mmap", VFIOPCIDevice
, vbasedev
.no_mmap
, false),
3362 DEFINE_PROP_BOOL("x-balloon-allowed", VFIOPCIDevice
,
3363 vbasedev
.ram_block_discard_allowed
, false),
3364 DEFINE_PROP_BOOL("x-no-kvm-intx", VFIOPCIDevice
, no_kvm_intx
, false),
3365 DEFINE_PROP_BOOL("x-no-kvm-msi", VFIOPCIDevice
, no_kvm_msi
, false),
3366 DEFINE_PROP_BOOL("x-no-kvm-msix", VFIOPCIDevice
, no_kvm_msix
, false),
3367 DEFINE_PROP_BOOL("x-no-geforce-quirks", VFIOPCIDevice
,
3368 no_geforce_quirks
, false),
3369 DEFINE_PROP_BOOL("x-no-kvm-ioeventfd", VFIOPCIDevice
, no_kvm_ioeventfd
,
3371 DEFINE_PROP_BOOL("x-no-vfio-ioeventfd", VFIOPCIDevice
, no_vfio_ioeventfd
,
3373 DEFINE_PROP_UINT32("x-pci-vendor-id", VFIOPCIDevice
, vendor_id
, PCI_ANY_ID
),
3374 DEFINE_PROP_UINT32("x-pci-device-id", VFIOPCIDevice
, device_id
, PCI_ANY_ID
),
3375 DEFINE_PROP_UINT32("x-pci-sub-vendor-id", VFIOPCIDevice
,
3376 sub_vendor_id
, PCI_ANY_ID
),
3377 DEFINE_PROP_UINT32("x-pci-sub-device-id", VFIOPCIDevice
,
3378 sub_device_id
, PCI_ANY_ID
),
3379 DEFINE_PROP_UINT32("x-igd-gms", VFIOPCIDevice
, igd_gms
, 0),
3380 DEFINE_PROP_UNSIGNED_NODEFAULT("x-nv-gpudirect-clique", VFIOPCIDevice
,
3381 nv_gpudirect_clique
,
3382 qdev_prop_nv_gpudirect_clique
, uint8_t),
3383 DEFINE_PROP_OFF_AUTO_PCIBAR("x-msix-relocation", VFIOPCIDevice
, msix_relo
,
3384 OFF_AUTOPCIBAR_OFF
),
3385 #ifdef CONFIG_IOMMUFD
3386 DEFINE_PROP_LINK("iommufd", VFIOPCIDevice
, vbasedev
.iommufd
,
3387 TYPE_IOMMUFD_BACKEND
, IOMMUFDBackend
*),
3389 DEFINE_PROP_END_OF_LIST(),
3392 #ifdef CONFIG_IOMMUFD
3393 static void vfio_pci_set_fd(Object
*obj
, const char *str
, Error
**errp
)
3395 vfio_device_set_fd(&VFIO_PCI(obj
)->vbasedev
, str
, errp
);
3399 static void vfio_pci_dev_class_init(ObjectClass
*klass
, void *data
)
3401 DeviceClass
*dc
= DEVICE_CLASS(klass
);
3402 PCIDeviceClass
*pdc
= PCI_DEVICE_CLASS(klass
);
3404 dc
->reset
= vfio_pci_reset
;
3405 device_class_set_props(dc
, vfio_pci_dev_properties
);
3406 #ifdef CONFIG_IOMMUFD
3407 object_class_property_add_str(klass
, "fd", NULL
, vfio_pci_set_fd
);
3409 dc
->desc
= "VFIO-based PCI device assignment";
3410 set_bit(DEVICE_CATEGORY_MISC
, dc
->categories
);
3411 pdc
->realize
= vfio_realize
;
3412 pdc
->exit
= vfio_exitfn
;
3413 pdc
->config_read
= vfio_pci_read_config
;
3414 pdc
->config_write
= vfio_pci_write_config
;
3417 static const TypeInfo vfio_pci_dev_info
= {
3418 .name
= TYPE_VFIO_PCI
,
3419 .parent
= TYPE_PCI_DEVICE
,
3420 .instance_size
= sizeof(VFIOPCIDevice
),
3421 .class_init
= vfio_pci_dev_class_init
,
3422 .instance_init
= vfio_instance_init
,
3423 .instance_finalize
= vfio_instance_finalize
,
3424 .interfaces
= (InterfaceInfo
[]) {
3425 { INTERFACE_PCIE_DEVICE
},
3426 { INTERFACE_CONVENTIONAL_PCI_DEVICE
},
3431 static Property vfio_pci_dev_nohotplug_properties
[] = {
3432 DEFINE_PROP_BOOL("ramfb", VFIOPCIDevice
, enable_ramfb
, false),
3433 DEFINE_PROP_ON_OFF_AUTO("x-ramfb-migrate", VFIOPCIDevice
, ramfb_migrate
,
3435 DEFINE_PROP_END_OF_LIST(),
3438 static void vfio_pci_nohotplug_dev_class_init(ObjectClass
*klass
, void *data
)
3440 DeviceClass
*dc
= DEVICE_CLASS(klass
);
3442 device_class_set_props(dc
, vfio_pci_dev_nohotplug_properties
);
3443 dc
->hotpluggable
= false;
3446 static const TypeInfo vfio_pci_nohotplug_dev_info
= {
3447 .name
= TYPE_VFIO_PCI_NOHOTPLUG
,
3448 .parent
= TYPE_VFIO_PCI
,
3449 .instance_size
= sizeof(VFIOPCIDevice
),
3450 .class_init
= vfio_pci_nohotplug_dev_class_init
,
3453 static void register_vfio_pci_dev_type(void)
3455 type_register_static(&vfio_pci_dev_info
);
3456 type_register_static(&vfio_pci_nohotplug_dev_info
);
3459 type_init(register_vfio_pci_dev_type
)