2 * vfio based device assignment support
4 * Copyright Red Hat, Inc. 2012
7 * Alex Williamson <alex.williamson@redhat.com>
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
12 * Based on qemu-kvm device-assignment:
13 * Adapted for KVM by Qumranet.
14 * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com)
15 * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com)
16 * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com)
17 * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
18 * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
21 #include "qemu/osdep.h"
22 #include <linux/vfio.h>
23 #include <sys/ioctl.h>
25 #include "hw/pci/msi.h"
26 #include "hw/pci/msix.h"
27 #include "hw/pci/pci_bridge.h"
28 #include "qemu/error-report.h"
29 #include "qemu/range.h"
30 #include "sysemu/kvm.h"
31 #include "sysemu/sysemu.h"
34 #include "qapi/error.h"
36 #define MSIX_CAP_LENGTH 12
38 static void vfio_disable_interrupts(VFIOPCIDevice
*vdev
);
39 static void vfio_mmap_set_enabled(VFIOPCIDevice
*vdev
, bool enabled
);
42 * Disabling BAR mmaping can be slow, but toggling it around INTx can
43 * also be a huge overhead. We try to get the best of both worlds by
44 * waiting until an interrupt to disable mmaps (subsequent transitions
45 * to the same state are effectively no overhead). If the interrupt has
46 * been serviced and the time gap is long enough, we re-enable mmaps for
47 * performance. This works well for things like graphics cards, which
48 * may not use their interrupt at all and are penalized to an unusable
49 * level by read/write BAR traps. Other devices, like NICs, have more
50 * regular interrupts and see much better latency by staying in non-mmap
51 * mode. We therefore set the default mmap_timeout such that a ping
52 * is just enough to keep the mmap disabled. Users can experiment with
53 * other options with the x-intx-mmap-timeout-ms parameter (a value of
54 * zero disables the timer).
56 static void vfio_intx_mmap_enable(void *opaque
)
58 VFIOPCIDevice
*vdev
= opaque
;
60 if (vdev
->intx
.pending
) {
61 timer_mod(vdev
->intx
.mmap_timer
,
62 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
) + vdev
->intx
.mmap_timeout
);
66 vfio_mmap_set_enabled(vdev
, true);
69 static void vfio_intx_interrupt(void *opaque
)
71 VFIOPCIDevice
*vdev
= opaque
;
73 if (!event_notifier_test_and_clear(&vdev
->intx
.interrupt
)) {
77 trace_vfio_intx_interrupt(vdev
->vbasedev
.name
, 'A' + vdev
->intx
.pin
);
79 vdev
->intx
.pending
= true;
80 pci_irq_assert(&vdev
->pdev
);
81 vfio_mmap_set_enabled(vdev
, false);
82 if (vdev
->intx
.mmap_timeout
) {
83 timer_mod(vdev
->intx
.mmap_timer
,
84 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
) + vdev
->intx
.mmap_timeout
);
88 static void vfio_intx_eoi(VFIODevice
*vbasedev
)
90 VFIOPCIDevice
*vdev
= container_of(vbasedev
, VFIOPCIDevice
, vbasedev
);
92 if (!vdev
->intx
.pending
) {
96 trace_vfio_intx_eoi(vbasedev
->name
);
98 vdev
->intx
.pending
= false;
99 pci_irq_deassert(&vdev
->pdev
);
100 vfio_unmask_single_irqindex(vbasedev
, VFIO_PCI_INTX_IRQ_INDEX
);
103 static void vfio_intx_enable_kvm(VFIOPCIDevice
*vdev
, Error
**errp
)
106 struct kvm_irqfd irqfd
= {
107 .fd
= event_notifier_get_fd(&vdev
->intx
.interrupt
),
108 .gsi
= vdev
->intx
.route
.irq
,
109 .flags
= KVM_IRQFD_FLAG_RESAMPLE
,
111 struct vfio_irq_set
*irq_set
;
115 if (vdev
->no_kvm_intx
|| !kvm_irqfds_enabled() ||
116 vdev
->intx
.route
.mode
!= PCI_INTX_ENABLED
||
117 !kvm_resamplefds_enabled()) {
121 /* Get to a known interrupt state */
122 qemu_set_fd_handler(irqfd
.fd
, NULL
, NULL
, vdev
);
123 vfio_mask_single_irqindex(&vdev
->vbasedev
, VFIO_PCI_INTX_IRQ_INDEX
);
124 vdev
->intx
.pending
= false;
125 pci_irq_deassert(&vdev
->pdev
);
127 /* Get an eventfd for resample/unmask */
128 if (event_notifier_init(&vdev
->intx
.unmask
, 0)) {
129 error_setg(errp
, "event_notifier_init failed eoi");
133 /* KVM triggers it, VFIO listens for it */
134 irqfd
.resamplefd
= event_notifier_get_fd(&vdev
->intx
.unmask
);
136 if (kvm_vm_ioctl(kvm_state
, KVM_IRQFD
, &irqfd
)) {
137 error_setg_errno(errp
, errno
, "failed to setup resample irqfd");
141 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
143 irq_set
= g_malloc0(argsz
);
144 irq_set
->argsz
= argsz
;
145 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
| VFIO_IRQ_SET_ACTION_UNMASK
;
146 irq_set
->index
= VFIO_PCI_INTX_IRQ_INDEX
;
149 pfd
= (int32_t *)&irq_set
->data
;
151 *pfd
= irqfd
.resamplefd
;
153 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
156 error_setg_errno(errp
, -ret
, "failed to setup INTx unmask fd");
161 vfio_unmask_single_irqindex(&vdev
->vbasedev
, VFIO_PCI_INTX_IRQ_INDEX
);
163 vdev
->intx
.kvm_accel
= true;
165 trace_vfio_intx_enable_kvm(vdev
->vbasedev
.name
);
170 irqfd
.flags
= KVM_IRQFD_FLAG_DEASSIGN
;
171 kvm_vm_ioctl(kvm_state
, KVM_IRQFD
, &irqfd
);
173 event_notifier_cleanup(&vdev
->intx
.unmask
);
175 qemu_set_fd_handler(irqfd
.fd
, vfio_intx_interrupt
, NULL
, vdev
);
176 vfio_unmask_single_irqindex(&vdev
->vbasedev
, VFIO_PCI_INTX_IRQ_INDEX
);
180 static void vfio_intx_disable_kvm(VFIOPCIDevice
*vdev
)
183 struct kvm_irqfd irqfd
= {
184 .fd
= event_notifier_get_fd(&vdev
->intx
.interrupt
),
185 .gsi
= vdev
->intx
.route
.irq
,
186 .flags
= KVM_IRQFD_FLAG_DEASSIGN
,
189 if (!vdev
->intx
.kvm_accel
) {
194 * Get to a known state, hardware masked, QEMU ready to accept new
195 * interrupts, QEMU IRQ de-asserted.
197 vfio_mask_single_irqindex(&vdev
->vbasedev
, VFIO_PCI_INTX_IRQ_INDEX
);
198 vdev
->intx
.pending
= false;
199 pci_irq_deassert(&vdev
->pdev
);
201 /* Tell KVM to stop listening for an INTx irqfd */
202 if (kvm_vm_ioctl(kvm_state
, KVM_IRQFD
, &irqfd
)) {
203 error_report("vfio: Error: Failed to disable INTx irqfd: %m");
206 /* We only need to close the eventfd for VFIO to cleanup the kernel side */
207 event_notifier_cleanup(&vdev
->intx
.unmask
);
209 /* QEMU starts listening for interrupt events. */
210 qemu_set_fd_handler(irqfd
.fd
, vfio_intx_interrupt
, NULL
, vdev
);
212 vdev
->intx
.kvm_accel
= false;
214 /* If we've missed an event, let it re-fire through QEMU */
215 vfio_unmask_single_irqindex(&vdev
->vbasedev
, VFIO_PCI_INTX_IRQ_INDEX
);
217 trace_vfio_intx_disable_kvm(vdev
->vbasedev
.name
);
221 static void vfio_intx_update(PCIDevice
*pdev
)
223 VFIOPCIDevice
*vdev
= DO_UPCAST(VFIOPCIDevice
, pdev
, pdev
);
227 if (vdev
->interrupt
!= VFIO_INT_INTx
) {
231 route
= pci_device_route_intx_to_irq(&vdev
->pdev
, vdev
->intx
.pin
);
233 if (!pci_intx_route_changed(&vdev
->intx
.route
, &route
)) {
234 return; /* Nothing changed */
237 trace_vfio_intx_update(vdev
->vbasedev
.name
,
238 vdev
->intx
.route
.irq
, route
.irq
);
240 vfio_intx_disable_kvm(vdev
);
242 vdev
->intx
.route
= route
;
244 if (route
.mode
!= PCI_INTX_ENABLED
) {
248 vfio_intx_enable_kvm(vdev
, &err
);
250 error_reportf_err(err
, WARN_PREFIX
, vdev
->vbasedev
.name
);
253 /* Re-enable the interrupt in cased we missed an EOI */
254 vfio_intx_eoi(&vdev
->vbasedev
);
257 static int vfio_intx_enable(VFIOPCIDevice
*vdev
, Error
**errp
)
259 uint8_t pin
= vfio_pci_read_config(&vdev
->pdev
, PCI_INTERRUPT_PIN
, 1);
260 int ret
, argsz
, retval
= 0;
261 struct vfio_irq_set
*irq_set
;
269 vfio_disable_interrupts(vdev
);
271 vdev
->intx
.pin
= pin
- 1; /* Pin A (1) -> irq[0] */
272 pci_config_set_interrupt_pin(vdev
->pdev
.config
, pin
);
276 * Only conditional to avoid generating error messages on platforms
277 * where we won't actually use the result anyway.
279 if (kvm_irqfds_enabled() && kvm_resamplefds_enabled()) {
280 vdev
->intx
.route
= pci_device_route_intx_to_irq(&vdev
->pdev
,
285 ret
= event_notifier_init(&vdev
->intx
.interrupt
, 0);
287 error_setg_errno(errp
, -ret
, "event_notifier_init failed");
291 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
293 irq_set
= g_malloc0(argsz
);
294 irq_set
->argsz
= argsz
;
295 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
| VFIO_IRQ_SET_ACTION_TRIGGER
;
296 irq_set
->index
= VFIO_PCI_INTX_IRQ_INDEX
;
299 pfd
= (int32_t *)&irq_set
->data
;
301 *pfd
= event_notifier_get_fd(&vdev
->intx
.interrupt
);
302 qemu_set_fd_handler(*pfd
, vfio_intx_interrupt
, NULL
, vdev
);
304 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
306 error_setg_errno(errp
, -ret
, "failed to setup INTx fd");
307 qemu_set_fd_handler(*pfd
, NULL
, NULL
, vdev
);
308 event_notifier_cleanup(&vdev
->intx
.interrupt
);
313 vfio_intx_enable_kvm(vdev
, &err
);
315 error_reportf_err(err
, WARN_PREFIX
, vdev
->vbasedev
.name
);
318 vdev
->interrupt
= VFIO_INT_INTx
;
320 trace_vfio_intx_enable(vdev
->vbasedev
.name
);
328 static void vfio_intx_disable(VFIOPCIDevice
*vdev
)
332 timer_del(vdev
->intx
.mmap_timer
);
333 vfio_intx_disable_kvm(vdev
);
334 vfio_disable_irqindex(&vdev
->vbasedev
, VFIO_PCI_INTX_IRQ_INDEX
);
335 vdev
->intx
.pending
= false;
336 pci_irq_deassert(&vdev
->pdev
);
337 vfio_mmap_set_enabled(vdev
, true);
339 fd
= event_notifier_get_fd(&vdev
->intx
.interrupt
);
340 qemu_set_fd_handler(fd
, NULL
, NULL
, vdev
);
341 event_notifier_cleanup(&vdev
->intx
.interrupt
);
343 vdev
->interrupt
= VFIO_INT_NONE
;
345 trace_vfio_intx_disable(vdev
->vbasedev
.name
);
351 static void vfio_msi_interrupt(void *opaque
)
353 VFIOMSIVector
*vector
= opaque
;
354 VFIOPCIDevice
*vdev
= vector
->vdev
;
355 MSIMessage (*get_msg
)(PCIDevice
*dev
, unsigned vector
);
356 void (*notify
)(PCIDevice
*dev
, unsigned vector
);
358 int nr
= vector
- vdev
->msi_vectors
;
360 if (!event_notifier_test_and_clear(&vector
->interrupt
)) {
364 if (vdev
->interrupt
== VFIO_INT_MSIX
) {
365 get_msg
= msix_get_message
;
366 notify
= msix_notify
;
368 /* A masked vector firing needs to use the PBA, enable it */
369 if (msix_is_masked(&vdev
->pdev
, nr
)) {
370 set_bit(nr
, vdev
->msix
->pending
);
371 memory_region_set_enabled(&vdev
->pdev
.msix_pba_mmio
, true);
372 trace_vfio_msix_pba_enable(vdev
->vbasedev
.name
);
374 } else if (vdev
->interrupt
== VFIO_INT_MSI
) {
375 get_msg
= msi_get_message
;
381 msg
= get_msg(&vdev
->pdev
, nr
);
382 trace_vfio_msi_interrupt(vdev
->vbasedev
.name
, nr
, msg
.address
, msg
.data
);
383 notify(&vdev
->pdev
, nr
);
386 static int vfio_enable_vectors(VFIOPCIDevice
*vdev
, bool msix
)
388 struct vfio_irq_set
*irq_set
;
389 int ret
= 0, i
, argsz
;
392 argsz
= sizeof(*irq_set
) + (vdev
->nr_vectors
* sizeof(*fds
));
394 irq_set
= g_malloc0(argsz
);
395 irq_set
->argsz
= argsz
;
396 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
| VFIO_IRQ_SET_ACTION_TRIGGER
;
397 irq_set
->index
= msix
? VFIO_PCI_MSIX_IRQ_INDEX
: VFIO_PCI_MSI_IRQ_INDEX
;
399 irq_set
->count
= vdev
->nr_vectors
;
400 fds
= (int32_t *)&irq_set
->data
;
402 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
406 * MSI vs MSI-X - The guest has direct access to MSI mask and pending
407 * bits, therefore we always use the KVM signaling path when setup.
408 * MSI-X mask and pending bits are emulated, so we want to use the
409 * KVM signaling path only when configured and unmasked.
411 if (vdev
->msi_vectors
[i
].use
) {
412 if (vdev
->msi_vectors
[i
].virq
< 0 ||
413 (msix
&& msix_is_masked(&vdev
->pdev
, i
))) {
414 fd
= event_notifier_get_fd(&vdev
->msi_vectors
[i
].interrupt
);
416 fd
= event_notifier_get_fd(&vdev
->msi_vectors
[i
].kvm_interrupt
);
423 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
430 static void vfio_add_kvm_msi_virq(VFIOPCIDevice
*vdev
, VFIOMSIVector
*vector
,
431 int vector_n
, bool msix
)
435 if ((msix
&& vdev
->no_kvm_msix
) || (!msix
&& vdev
->no_kvm_msi
)) {
439 if (event_notifier_init(&vector
->kvm_interrupt
, 0)) {
443 virq
= kvm_irqchip_add_msi_route(kvm_state
, vector_n
, &vdev
->pdev
);
445 event_notifier_cleanup(&vector
->kvm_interrupt
);
449 if (kvm_irqchip_add_irqfd_notifier_gsi(kvm_state
, &vector
->kvm_interrupt
,
451 kvm_irqchip_release_virq(kvm_state
, virq
);
452 event_notifier_cleanup(&vector
->kvm_interrupt
);
459 static void vfio_remove_kvm_msi_virq(VFIOMSIVector
*vector
)
461 kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state
, &vector
->kvm_interrupt
,
463 kvm_irqchip_release_virq(kvm_state
, vector
->virq
);
465 event_notifier_cleanup(&vector
->kvm_interrupt
);
468 static void vfio_update_kvm_msi_virq(VFIOMSIVector
*vector
, MSIMessage msg
,
471 kvm_irqchip_update_msi_route(kvm_state
, vector
->virq
, msg
, pdev
);
472 kvm_irqchip_commit_routes(kvm_state
);
475 static int vfio_msix_vector_do_use(PCIDevice
*pdev
, unsigned int nr
,
476 MSIMessage
*msg
, IOHandler
*handler
)
478 VFIOPCIDevice
*vdev
= DO_UPCAST(VFIOPCIDevice
, pdev
, pdev
);
479 VFIOMSIVector
*vector
;
482 trace_vfio_msix_vector_do_use(vdev
->vbasedev
.name
, nr
);
484 vector
= &vdev
->msi_vectors
[nr
];
489 if (event_notifier_init(&vector
->interrupt
, 0)) {
490 error_report("vfio: Error: event_notifier_init failed");
493 msix_vector_use(pdev
, nr
);
496 qemu_set_fd_handler(event_notifier_get_fd(&vector
->interrupt
),
497 handler
, NULL
, vector
);
500 * Attempt to enable route through KVM irqchip,
501 * default to userspace handling if unavailable.
503 if (vector
->virq
>= 0) {
505 vfio_remove_kvm_msi_virq(vector
);
507 vfio_update_kvm_msi_virq(vector
, *msg
, pdev
);
511 vfio_add_kvm_msi_virq(vdev
, vector
, nr
, true);
516 * We don't want to have the host allocate all possible MSI vectors
517 * for a device if they're not in use, so we shutdown and incrementally
518 * increase them as needed.
520 if (vdev
->nr_vectors
< nr
+ 1) {
521 vfio_disable_irqindex(&vdev
->vbasedev
, VFIO_PCI_MSIX_IRQ_INDEX
);
522 vdev
->nr_vectors
= nr
+ 1;
523 ret
= vfio_enable_vectors(vdev
, true);
525 error_report("vfio: failed to enable vectors, %d", ret
);
529 struct vfio_irq_set
*irq_set
;
532 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
534 irq_set
= g_malloc0(argsz
);
535 irq_set
->argsz
= argsz
;
536 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
|
537 VFIO_IRQ_SET_ACTION_TRIGGER
;
538 irq_set
->index
= VFIO_PCI_MSIX_IRQ_INDEX
;
541 pfd
= (int32_t *)&irq_set
->data
;
543 if (vector
->virq
>= 0) {
544 *pfd
= event_notifier_get_fd(&vector
->kvm_interrupt
);
546 *pfd
= event_notifier_get_fd(&vector
->interrupt
);
549 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
552 error_report("vfio: failed to modify vector, %d", ret
);
556 /* Disable PBA emulation when nothing more is pending. */
557 clear_bit(nr
, vdev
->msix
->pending
);
558 if (find_first_bit(vdev
->msix
->pending
,
559 vdev
->nr_vectors
) == vdev
->nr_vectors
) {
560 memory_region_set_enabled(&vdev
->pdev
.msix_pba_mmio
, false);
561 trace_vfio_msix_pba_disable(vdev
->vbasedev
.name
);
567 static int vfio_msix_vector_use(PCIDevice
*pdev
,
568 unsigned int nr
, MSIMessage msg
)
570 return vfio_msix_vector_do_use(pdev
, nr
, &msg
, vfio_msi_interrupt
);
573 static void vfio_msix_vector_release(PCIDevice
*pdev
, unsigned int nr
)
575 VFIOPCIDevice
*vdev
= DO_UPCAST(VFIOPCIDevice
, pdev
, pdev
);
576 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[nr
];
578 trace_vfio_msix_vector_release(vdev
->vbasedev
.name
, nr
);
581 * There are still old guests that mask and unmask vectors on every
582 * interrupt. If we're using QEMU bypass with a KVM irqfd, leave all of
583 * the KVM setup in place, simply switch VFIO to use the non-bypass
584 * eventfd. We'll then fire the interrupt through QEMU and the MSI-X
585 * core will mask the interrupt and set pending bits, allowing it to
586 * be re-asserted on unmask. Nothing to do if already using QEMU mode.
588 if (vector
->virq
>= 0) {
590 struct vfio_irq_set
*irq_set
;
593 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
595 irq_set
= g_malloc0(argsz
);
596 irq_set
->argsz
= argsz
;
597 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
|
598 VFIO_IRQ_SET_ACTION_TRIGGER
;
599 irq_set
->index
= VFIO_PCI_MSIX_IRQ_INDEX
;
602 pfd
= (int32_t *)&irq_set
->data
;
604 *pfd
= event_notifier_get_fd(&vector
->interrupt
);
606 ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
612 static void vfio_msix_enable(VFIOPCIDevice
*vdev
)
614 vfio_disable_interrupts(vdev
);
616 vdev
->msi_vectors
= g_new0(VFIOMSIVector
, vdev
->msix
->entries
);
618 vdev
->interrupt
= VFIO_INT_MSIX
;
621 * Some communication channels between VF & PF or PF & fw rely on the
622 * physical state of the device and expect that enabling MSI-X from the
623 * guest enables the same on the host. When our guest is Linux, the
624 * guest driver call to pci_enable_msix() sets the enabling bit in the
625 * MSI-X capability, but leaves the vector table masked. We therefore
626 * can't rely on a vector_use callback (from request_irq() in the guest)
627 * to switch the physical device into MSI-X mode because that may come a
628 * long time after pci_enable_msix(). This code enables vector 0 with
629 * triggering to userspace, then immediately release the vector, leaving
630 * the physical device with no vectors enabled, but MSI-X enabled, just
631 * like the guest view.
633 vfio_msix_vector_do_use(&vdev
->pdev
, 0, NULL
, NULL
);
634 vfio_msix_vector_release(&vdev
->pdev
, 0);
636 if (msix_set_vector_notifiers(&vdev
->pdev
, vfio_msix_vector_use
,
637 vfio_msix_vector_release
, NULL
)) {
638 error_report("vfio: msix_set_vector_notifiers failed");
641 trace_vfio_msix_enable(vdev
->vbasedev
.name
);
644 static void vfio_msi_enable(VFIOPCIDevice
*vdev
)
648 vfio_disable_interrupts(vdev
);
650 vdev
->nr_vectors
= msi_nr_vectors_allocated(&vdev
->pdev
);
652 vdev
->msi_vectors
= g_new0(VFIOMSIVector
, vdev
->nr_vectors
);
654 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
655 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[i
];
661 if (event_notifier_init(&vector
->interrupt
, 0)) {
662 error_report("vfio: Error: event_notifier_init failed");
665 qemu_set_fd_handler(event_notifier_get_fd(&vector
->interrupt
),
666 vfio_msi_interrupt
, NULL
, vector
);
669 * Attempt to enable route through KVM irqchip,
670 * default to userspace handling if unavailable.
672 vfio_add_kvm_msi_virq(vdev
, vector
, i
, false);
675 /* Set interrupt type prior to possible interrupts */
676 vdev
->interrupt
= VFIO_INT_MSI
;
678 ret
= vfio_enable_vectors(vdev
, false);
681 error_report("vfio: Error: Failed to setup MSI fds: %m");
682 } else if (ret
!= vdev
->nr_vectors
) {
683 error_report("vfio: Error: Failed to enable %d "
684 "MSI vectors, retry with %d", vdev
->nr_vectors
, ret
);
687 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
688 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[i
];
689 if (vector
->virq
>= 0) {
690 vfio_remove_kvm_msi_virq(vector
);
692 qemu_set_fd_handler(event_notifier_get_fd(&vector
->interrupt
),
694 event_notifier_cleanup(&vector
->interrupt
);
697 g_free(vdev
->msi_vectors
);
699 if (ret
> 0 && ret
!= vdev
->nr_vectors
) {
700 vdev
->nr_vectors
= ret
;
703 vdev
->nr_vectors
= 0;
706 * Failing to setup MSI doesn't really fall within any specification.
707 * Let's try leaving interrupts disabled and hope the guest figures
708 * out to fall back to INTx for this device.
710 error_report("vfio: Error: Failed to enable MSI");
711 vdev
->interrupt
= VFIO_INT_NONE
;
716 trace_vfio_msi_enable(vdev
->vbasedev
.name
, vdev
->nr_vectors
);
719 static void vfio_msi_disable_common(VFIOPCIDevice
*vdev
)
724 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
725 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[i
];
726 if (vdev
->msi_vectors
[i
].use
) {
727 if (vector
->virq
>= 0) {
728 vfio_remove_kvm_msi_virq(vector
);
730 qemu_set_fd_handler(event_notifier_get_fd(&vector
->interrupt
),
732 event_notifier_cleanup(&vector
->interrupt
);
736 g_free(vdev
->msi_vectors
);
737 vdev
->msi_vectors
= NULL
;
738 vdev
->nr_vectors
= 0;
739 vdev
->interrupt
= VFIO_INT_NONE
;
741 vfio_intx_enable(vdev
, &err
);
743 error_reportf_err(err
, ERR_PREFIX
, vdev
->vbasedev
.name
);
747 static void vfio_msix_disable(VFIOPCIDevice
*vdev
)
751 msix_unset_vector_notifiers(&vdev
->pdev
);
754 * MSI-X will only release vectors if MSI-X is still enabled on the
755 * device, check through the rest and release it ourselves if necessary.
757 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
758 if (vdev
->msi_vectors
[i
].use
) {
759 vfio_msix_vector_release(&vdev
->pdev
, i
);
760 msix_vector_unuse(&vdev
->pdev
, i
);
764 if (vdev
->nr_vectors
) {
765 vfio_disable_irqindex(&vdev
->vbasedev
, VFIO_PCI_MSIX_IRQ_INDEX
);
768 vfio_msi_disable_common(vdev
);
770 memset(vdev
->msix
->pending
, 0,
771 BITS_TO_LONGS(vdev
->msix
->entries
) * sizeof(unsigned long));
773 trace_vfio_msix_disable(vdev
->vbasedev
.name
);
776 static void vfio_msi_disable(VFIOPCIDevice
*vdev
)
778 vfio_disable_irqindex(&vdev
->vbasedev
, VFIO_PCI_MSI_IRQ_INDEX
);
779 vfio_msi_disable_common(vdev
);
781 trace_vfio_msi_disable(vdev
->vbasedev
.name
);
784 static void vfio_update_msi(VFIOPCIDevice
*vdev
)
788 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
789 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[i
];
792 if (!vector
->use
|| vector
->virq
< 0) {
796 msg
= msi_get_message(&vdev
->pdev
, i
);
797 vfio_update_kvm_msi_virq(vector
, msg
, &vdev
->pdev
);
801 static void vfio_pci_load_rom(VFIOPCIDevice
*vdev
)
803 struct vfio_region_info
*reg_info
;
808 if (vfio_get_region_info(&vdev
->vbasedev
,
809 VFIO_PCI_ROM_REGION_INDEX
, ®_info
)) {
810 error_report("vfio: Error getting ROM info: %m");
814 trace_vfio_pci_load_rom(vdev
->vbasedev
.name
, (unsigned long)reg_info
->size
,
815 (unsigned long)reg_info
->offset
,
816 (unsigned long)reg_info
->flags
);
818 vdev
->rom_size
= size
= reg_info
->size
;
819 vdev
->rom_offset
= reg_info
->offset
;
823 if (!vdev
->rom_size
) {
824 vdev
->rom_read_failed
= true;
825 error_report("vfio-pci: Cannot read device rom at "
826 "%s", vdev
->vbasedev
.name
);
827 error_printf("Device option ROM contents are probably invalid "
828 "(check dmesg).\nSkip option ROM probe with rombar=0, "
829 "or load from file with romfile=\n");
833 vdev
->rom
= g_malloc(size
);
834 memset(vdev
->rom
, 0xff, size
);
837 bytes
= pread(vdev
->vbasedev
.fd
, vdev
->rom
+ off
,
838 size
, vdev
->rom_offset
+ off
);
841 } else if (bytes
> 0) {
845 if (errno
== EINTR
|| errno
== EAGAIN
) {
848 error_report("vfio: Error reading device ROM: %m");
854 * Test the ROM signature against our device, if the vendor is correct
855 * but the device ID doesn't match, store the correct device ID and
856 * recompute the checksum. Intel IGD devices need this and are known
857 * to have bogus checksums so we can't simply adjust the checksum.
859 if (pci_get_word(vdev
->rom
) == 0xaa55 &&
860 pci_get_word(vdev
->rom
+ 0x18) + 8 < vdev
->rom_size
&&
861 !memcmp(vdev
->rom
+ pci_get_word(vdev
->rom
+ 0x18), "PCIR", 4)) {
864 vid
= pci_get_word(vdev
->rom
+ pci_get_word(vdev
->rom
+ 0x18) + 4);
865 did
= pci_get_word(vdev
->rom
+ pci_get_word(vdev
->rom
+ 0x18) + 6);
867 if (vid
== vdev
->vendor_id
&& did
!= vdev
->device_id
) {
869 uint8_t csum
, *data
= vdev
->rom
;
871 pci_set_word(vdev
->rom
+ pci_get_word(vdev
->rom
+ 0x18) + 6,
875 for (csum
= 0, i
= 0; i
< vdev
->rom_size
; i
++) {
884 static uint64_t vfio_rom_read(void *opaque
, hwaddr addr
, unsigned size
)
886 VFIOPCIDevice
*vdev
= opaque
;
895 /* Load the ROM lazily when the guest tries to read it */
896 if (unlikely(!vdev
->rom
&& !vdev
->rom_read_failed
)) {
897 vfio_pci_load_rom(vdev
);
900 memcpy(&val
, vdev
->rom
+ addr
,
901 (addr
< vdev
->rom_size
) ? MIN(size
, vdev
->rom_size
- addr
) : 0);
908 data
= le16_to_cpu(val
.word
);
911 data
= le32_to_cpu(val
.dword
);
914 hw_error("vfio: unsupported read size, %d bytes\n", size
);
918 trace_vfio_rom_read(vdev
->vbasedev
.name
, addr
, size
, data
);
923 static void vfio_rom_write(void *opaque
, hwaddr addr
,
924 uint64_t data
, unsigned size
)
928 static const MemoryRegionOps vfio_rom_ops
= {
929 .read
= vfio_rom_read
,
930 .write
= vfio_rom_write
,
931 .endianness
= DEVICE_LITTLE_ENDIAN
,
934 static void vfio_pci_size_rom(VFIOPCIDevice
*vdev
)
936 uint32_t orig
, size
= cpu_to_le32((uint32_t)PCI_ROM_ADDRESS_MASK
);
937 off_t offset
= vdev
->config_offset
+ PCI_ROM_ADDRESS
;
938 DeviceState
*dev
= DEVICE(vdev
);
940 int fd
= vdev
->vbasedev
.fd
;
942 if (vdev
->pdev
.romfile
|| !vdev
->pdev
.rom_bar
) {
943 /* Since pci handles romfile, just print a message and return */
944 if (vfio_blacklist_opt_rom(vdev
) && vdev
->pdev
.romfile
) {
945 error_printf("Warning : Device at %s is known to cause system instability issues during option rom execution. Proceeding anyway since user specified romfile\n",
946 vdev
->vbasedev
.name
);
952 * Use the same size ROM BAR as the physical device. The contents
953 * will get filled in later when the guest tries to read it.
955 if (pread(fd
, &orig
, 4, offset
) != 4 ||
956 pwrite(fd
, &size
, 4, offset
) != 4 ||
957 pread(fd
, &size
, 4, offset
) != 4 ||
958 pwrite(fd
, &orig
, 4, offset
) != 4) {
959 error_report("%s(%s) failed: %m", __func__
, vdev
->vbasedev
.name
);
963 size
= ~(le32_to_cpu(size
) & PCI_ROM_ADDRESS_MASK
) + 1;
969 if (vfio_blacklist_opt_rom(vdev
)) {
970 if (dev
->opts
&& qemu_opt_get(dev
->opts
, "rombar")) {
971 error_printf("Warning : Device at %s is known to cause system instability issues during option rom execution. Proceeding anyway since user specified non zero value for rombar\n",
972 vdev
->vbasedev
.name
);
974 error_printf("Warning : Rom loading for device at %s has been disabled due to system instability issues. Specify rombar=1 or romfile to force\n",
975 vdev
->vbasedev
.name
);
980 trace_vfio_pci_size_rom(vdev
->vbasedev
.name
, size
);
982 name
= g_strdup_printf("vfio[%s].rom", vdev
->vbasedev
.name
);
984 memory_region_init_io(&vdev
->pdev
.rom
, OBJECT(vdev
),
985 &vfio_rom_ops
, vdev
, name
, size
);
988 pci_register_bar(&vdev
->pdev
, PCI_ROM_SLOT
,
989 PCI_BASE_ADDRESS_SPACE_MEMORY
, &vdev
->pdev
.rom
);
991 vdev
->pdev
.has_rom
= true;
992 vdev
->rom_read_failed
= false;
995 void vfio_vga_write(void *opaque
, hwaddr addr
,
996 uint64_t data
, unsigned size
)
998 VFIOVGARegion
*region
= opaque
;
999 VFIOVGA
*vga
= container_of(region
, VFIOVGA
, region
[region
->nr
]);
1006 off_t offset
= vga
->fd_offset
+ region
->offset
+ addr
;
1013 buf
.word
= cpu_to_le16(data
);
1016 buf
.dword
= cpu_to_le32(data
);
1019 hw_error("vfio: unsupported write size, %d bytes", size
);
1023 if (pwrite(vga
->fd
, &buf
, size
, offset
) != size
) {
1024 error_report("%s(,0x%"HWADDR_PRIx
", 0x%"PRIx64
", %d) failed: %m",
1025 __func__
, region
->offset
+ addr
, data
, size
);
1028 trace_vfio_vga_write(region
->offset
+ addr
, data
, size
);
1031 uint64_t vfio_vga_read(void *opaque
, hwaddr addr
, unsigned size
)
1033 VFIOVGARegion
*region
= opaque
;
1034 VFIOVGA
*vga
= container_of(region
, VFIOVGA
, region
[region
->nr
]);
1042 off_t offset
= vga
->fd_offset
+ region
->offset
+ addr
;
1044 if (pread(vga
->fd
, &buf
, size
, offset
) != size
) {
1045 error_report("%s(,0x%"HWADDR_PRIx
", %d) failed: %m",
1046 __func__
, region
->offset
+ addr
, size
);
1047 return (uint64_t)-1;
1055 data
= le16_to_cpu(buf
.word
);
1058 data
= le32_to_cpu(buf
.dword
);
1061 hw_error("vfio: unsupported read size, %d bytes", size
);
1065 trace_vfio_vga_read(region
->offset
+ addr
, size
, data
);
1070 static const MemoryRegionOps vfio_vga_ops
= {
1071 .read
= vfio_vga_read
,
1072 .write
= vfio_vga_write
,
1073 .endianness
= DEVICE_LITTLE_ENDIAN
,
1077 * Expand memory region of sub-page(size < PAGE_SIZE) MMIO BAR to page
1078 * size if the BAR is in an exclusive page in host so that we could map
1079 * this BAR to guest. But this sub-page BAR may not occupy an exclusive
1080 * page in guest. So we should set the priority of the expanded memory
1081 * region to zero in case of overlap with BARs which share the same page
1082 * with the sub-page BAR in guest. Besides, we should also recover the
1083 * size of this sub-page BAR when its base address is changed in guest
1084 * and not page aligned any more.
1086 static void vfio_sub_page_bar_update_mapping(PCIDevice
*pdev
, int bar
)
1088 VFIOPCIDevice
*vdev
= DO_UPCAST(VFIOPCIDevice
, pdev
, pdev
);
1089 VFIORegion
*region
= &vdev
->bars
[bar
].region
;
1090 MemoryRegion
*mmap_mr
, *mr
;
1093 uint64_t size
= region
->size
;
1095 /* Make sure that the whole region is allowed to be mmapped */
1096 if (region
->nr_mmaps
!= 1 || !region
->mmaps
[0].mmap
||
1097 region
->mmaps
[0].size
!= region
->size
) {
1101 r
= &pdev
->io_regions
[bar
];
1104 mmap_mr
= ®ion
->mmaps
[0].mem
;
1106 /* If BAR is mapped and page aligned, update to fill PAGE_SIZE */
1107 if (bar_addr
!= PCI_BAR_UNMAPPED
&&
1108 !(bar_addr
& ~qemu_real_host_page_mask
)) {
1109 size
= qemu_real_host_page_size
;
1112 memory_region_transaction_begin();
1114 memory_region_set_size(mr
, size
);
1115 memory_region_set_size(mmap_mr
, size
);
1116 if (size
!= region
->size
&& memory_region_is_mapped(mr
)) {
1117 memory_region_del_subregion(r
->address_space
, mr
);
1118 memory_region_add_subregion_overlap(r
->address_space
,
1122 memory_region_transaction_commit();
1128 uint32_t vfio_pci_read_config(PCIDevice
*pdev
, uint32_t addr
, int len
)
1130 VFIOPCIDevice
*vdev
= DO_UPCAST(VFIOPCIDevice
, pdev
, pdev
);
1131 uint32_t emu_bits
= 0, emu_val
= 0, phys_val
= 0, val
;
1133 memcpy(&emu_bits
, vdev
->emulated_config_bits
+ addr
, len
);
1134 emu_bits
= le32_to_cpu(emu_bits
);
1137 emu_val
= pci_default_read_config(pdev
, addr
, len
);
1140 if (~emu_bits
& (0xffffffffU
>> (32 - len
* 8))) {
1143 ret
= pread(vdev
->vbasedev
.fd
, &phys_val
, len
,
1144 vdev
->config_offset
+ addr
);
1146 error_report("%s(%s, 0x%x, 0x%x) failed: %m",
1147 __func__
, vdev
->vbasedev
.name
, addr
, len
);
1150 phys_val
= le32_to_cpu(phys_val
);
1153 val
= (emu_val
& emu_bits
) | (phys_val
& ~emu_bits
);
1155 trace_vfio_pci_read_config(vdev
->vbasedev
.name
, addr
, len
, val
);
1160 void vfio_pci_write_config(PCIDevice
*pdev
,
1161 uint32_t addr
, uint32_t val
, int len
)
1163 VFIOPCIDevice
*vdev
= DO_UPCAST(VFIOPCIDevice
, pdev
, pdev
);
1164 uint32_t val_le
= cpu_to_le32(val
);
1166 trace_vfio_pci_write_config(vdev
->vbasedev
.name
, addr
, val
, len
);
1168 /* Write everything to VFIO, let it filter out what we can't write */
1169 if (pwrite(vdev
->vbasedev
.fd
, &val_le
, len
, vdev
->config_offset
+ addr
)
1171 error_report("%s(%s, 0x%x, 0x%x, 0x%x) failed: %m",
1172 __func__
, vdev
->vbasedev
.name
, addr
, val
, len
);
1175 /* MSI/MSI-X Enabling/Disabling */
1176 if (pdev
->cap_present
& QEMU_PCI_CAP_MSI
&&
1177 ranges_overlap(addr
, len
, pdev
->msi_cap
, vdev
->msi_cap_size
)) {
1178 int is_enabled
, was_enabled
= msi_enabled(pdev
);
1180 pci_default_write_config(pdev
, addr
, val
, len
);
1182 is_enabled
= msi_enabled(pdev
);
1186 vfio_msi_enable(vdev
);
1190 vfio_msi_disable(vdev
);
1192 vfio_update_msi(vdev
);
1195 } else if (pdev
->cap_present
& QEMU_PCI_CAP_MSIX
&&
1196 ranges_overlap(addr
, len
, pdev
->msix_cap
, MSIX_CAP_LENGTH
)) {
1197 int is_enabled
, was_enabled
= msix_enabled(pdev
);
1199 pci_default_write_config(pdev
, addr
, val
, len
);
1201 is_enabled
= msix_enabled(pdev
);
1203 if (!was_enabled
&& is_enabled
) {
1204 vfio_msix_enable(vdev
);
1205 } else if (was_enabled
&& !is_enabled
) {
1206 vfio_msix_disable(vdev
);
1208 } else if (ranges_overlap(addr
, len
, PCI_BASE_ADDRESS_0
, 24) ||
1209 range_covers_byte(addr
, len
, PCI_COMMAND
)) {
1210 pcibus_t old_addr
[PCI_NUM_REGIONS
- 1];
1213 for (bar
= 0; bar
< PCI_ROM_SLOT
; bar
++) {
1214 old_addr
[bar
] = pdev
->io_regions
[bar
].addr
;
1217 pci_default_write_config(pdev
, addr
, val
, len
);
1219 for (bar
= 0; bar
< PCI_ROM_SLOT
; bar
++) {
1220 if (old_addr
[bar
] != pdev
->io_regions
[bar
].addr
&&
1221 pdev
->io_regions
[bar
].size
> 0 &&
1222 pdev
->io_regions
[bar
].size
< qemu_real_host_page_size
) {
1223 vfio_sub_page_bar_update_mapping(pdev
, bar
);
1227 /* Write everything to QEMU to keep emulated bits correct */
1228 pci_default_write_config(pdev
, addr
, val
, len
);
1235 static void vfio_disable_interrupts(VFIOPCIDevice
*vdev
)
1238 * More complicated than it looks. Disabling MSI/X transitions the
1239 * device to INTx mode (if supported). Therefore we need to first
1240 * disable MSI/X and then cleanup by disabling INTx.
1242 if (vdev
->interrupt
== VFIO_INT_MSIX
) {
1243 vfio_msix_disable(vdev
);
1244 } else if (vdev
->interrupt
== VFIO_INT_MSI
) {
1245 vfio_msi_disable(vdev
);
1248 if (vdev
->interrupt
== VFIO_INT_INTx
) {
1249 vfio_intx_disable(vdev
);
1253 static int vfio_msi_setup(VFIOPCIDevice
*vdev
, int pos
, Error
**errp
)
1256 bool msi_64bit
, msi_maskbit
;
1260 if (pread(vdev
->vbasedev
.fd
, &ctrl
, sizeof(ctrl
),
1261 vdev
->config_offset
+ pos
+ PCI_CAP_FLAGS
) != sizeof(ctrl
)) {
1262 error_setg_errno(errp
, errno
, "failed reading MSI PCI_CAP_FLAGS");
1265 ctrl
= le16_to_cpu(ctrl
);
1267 msi_64bit
= !!(ctrl
& PCI_MSI_FLAGS_64BIT
);
1268 msi_maskbit
= !!(ctrl
& PCI_MSI_FLAGS_MASKBIT
);
1269 entries
= 1 << ((ctrl
& PCI_MSI_FLAGS_QMASK
) >> 1);
1271 trace_vfio_msi_setup(vdev
->vbasedev
.name
, pos
);
1273 ret
= msi_init(&vdev
->pdev
, pos
, entries
, msi_64bit
, msi_maskbit
, &err
);
1275 if (ret
== -ENOTSUP
) {
1278 error_prepend(&err
, "msi_init failed: ");
1279 error_propagate(errp
, err
);
1282 vdev
->msi_cap_size
= 0xa + (msi_maskbit
? 0xa : 0) + (msi_64bit
? 0x4 : 0);
1287 static void vfio_pci_fixup_msix_region(VFIOPCIDevice
*vdev
)
1290 VFIORegion
*region
= &vdev
->bars
[vdev
->msix
->table_bar
].region
;
1293 * We expect to find a single mmap covering the whole BAR, anything else
1294 * means it's either unsupported or already setup.
1296 if (region
->nr_mmaps
!= 1 || region
->mmaps
[0].offset
||
1297 region
->size
!= region
->mmaps
[0].size
) {
1301 /* MSI-X table start and end aligned to host page size */
1302 start
= vdev
->msix
->table_offset
& qemu_real_host_page_mask
;
1303 end
= REAL_HOST_PAGE_ALIGN((uint64_t)vdev
->msix
->table_offset
+
1304 (vdev
->msix
->entries
* PCI_MSIX_ENTRY_SIZE
));
1307 * Does the MSI-X table cover the beginning of the BAR? The whole BAR?
1308 * NB - Host page size is necessarily a power of two and so is the PCI
1309 * BAR (not counting EA yet), therefore if we have host page aligned
1310 * @start and @end, then any remainder of the BAR before or after those
1311 * must be at least host page sized and therefore mmap'able.
1314 if (end
>= region
->size
) {
1315 region
->nr_mmaps
= 0;
1316 g_free(region
->mmaps
);
1317 region
->mmaps
= NULL
;
1318 trace_vfio_msix_fixup(vdev
->vbasedev
.name
,
1319 vdev
->msix
->table_bar
, 0, 0);
1321 region
->mmaps
[0].offset
= end
;
1322 region
->mmaps
[0].size
= region
->size
- end
;
1323 trace_vfio_msix_fixup(vdev
->vbasedev
.name
,
1324 vdev
->msix
->table_bar
, region
->mmaps
[0].offset
,
1325 region
->mmaps
[0].offset
+ region
->mmaps
[0].size
);
1328 /* Maybe it's aligned at the end of the BAR */
1329 } else if (end
>= region
->size
) {
1330 region
->mmaps
[0].size
= start
;
1331 trace_vfio_msix_fixup(vdev
->vbasedev
.name
,
1332 vdev
->msix
->table_bar
, region
->mmaps
[0].offset
,
1333 region
->mmaps
[0].offset
+ region
->mmaps
[0].size
);
1335 /* Otherwise it must split the BAR */
1337 region
->nr_mmaps
= 2;
1338 region
->mmaps
= g_renew(VFIOMmap
, region
->mmaps
, 2);
1340 memcpy(®ion
->mmaps
[1], ®ion
->mmaps
[0], sizeof(VFIOMmap
));
1342 region
->mmaps
[0].size
= start
;
1343 trace_vfio_msix_fixup(vdev
->vbasedev
.name
,
1344 vdev
->msix
->table_bar
, region
->mmaps
[0].offset
,
1345 region
->mmaps
[0].offset
+ region
->mmaps
[0].size
);
1347 region
->mmaps
[1].offset
= end
;
1348 region
->mmaps
[1].size
= region
->size
- end
;
1349 trace_vfio_msix_fixup(vdev
->vbasedev
.name
,
1350 vdev
->msix
->table_bar
, region
->mmaps
[1].offset
,
1351 region
->mmaps
[1].offset
+ region
->mmaps
[1].size
);
1356 * We don't have any control over how pci_add_capability() inserts
1357 * capabilities into the chain. In order to setup MSI-X we need a
1358 * MemoryRegion for the BAR. In order to setup the BAR and not
1359 * attempt to mmap the MSI-X table area, which VFIO won't allow, we
1360 * need to first look for where the MSI-X table lives. So we
1361 * unfortunately split MSI-X setup across two functions.
1363 static void vfio_msix_early_setup(VFIOPCIDevice
*vdev
, Error
**errp
)
1367 uint32_t table
, pba
;
1368 int fd
= vdev
->vbasedev
.fd
;
1371 pos
= pci_find_capability(&vdev
->pdev
, PCI_CAP_ID_MSIX
);
1376 if (pread(fd
, &ctrl
, sizeof(ctrl
),
1377 vdev
->config_offset
+ pos
+ PCI_MSIX_FLAGS
) != sizeof(ctrl
)) {
1378 error_setg_errno(errp
, errno
, "failed to read PCI MSIX FLAGS");
1382 if (pread(fd
, &table
, sizeof(table
),
1383 vdev
->config_offset
+ pos
+ PCI_MSIX_TABLE
) != sizeof(table
)) {
1384 error_setg_errno(errp
, errno
, "failed to read PCI MSIX TABLE");
1388 if (pread(fd
, &pba
, sizeof(pba
),
1389 vdev
->config_offset
+ pos
+ PCI_MSIX_PBA
) != sizeof(pba
)) {
1390 error_setg_errno(errp
, errno
, "failed to read PCI MSIX PBA");
1394 ctrl
= le16_to_cpu(ctrl
);
1395 table
= le32_to_cpu(table
);
1396 pba
= le32_to_cpu(pba
);
1398 msix
= g_malloc0(sizeof(*msix
));
1399 msix
->table_bar
= table
& PCI_MSIX_FLAGS_BIRMASK
;
1400 msix
->table_offset
= table
& ~PCI_MSIX_FLAGS_BIRMASK
;
1401 msix
->pba_bar
= pba
& PCI_MSIX_FLAGS_BIRMASK
;
1402 msix
->pba_offset
= pba
& ~PCI_MSIX_FLAGS_BIRMASK
;
1403 msix
->entries
= (ctrl
& PCI_MSIX_FLAGS_QSIZE
) + 1;
1406 * Test the size of the pba_offset variable and catch if it extends outside
1407 * of the specified BAR. If it is the case, we need to apply a hardware
1408 * specific quirk if the device is known or we have a broken configuration.
1410 if (msix
->pba_offset
>= vdev
->bars
[msix
->pba_bar
].region
.size
) {
1412 * Chelsio T5 Virtual Function devices are encoded as 0x58xx for T5
1413 * adapters. The T5 hardware returns an incorrect value of 0x8000 for
1414 * the VF PBA offset while the BAR itself is only 8k. The correct value
1415 * is 0x1000, so we hard code that here.
1417 if (vdev
->vendor_id
== PCI_VENDOR_ID_CHELSIO
&&
1418 (vdev
->device_id
& 0xff00) == 0x5800) {
1419 msix
->pba_offset
= 0x1000;
1421 error_setg(errp
, "hardware reports invalid configuration, "
1422 "MSIX PBA outside of specified BAR");
1428 trace_vfio_msix_early_setup(vdev
->vbasedev
.name
, pos
, msix
->table_bar
,
1429 msix
->table_offset
, msix
->entries
);
1432 vfio_pci_fixup_msix_region(vdev
);
1435 static int vfio_msix_setup(VFIOPCIDevice
*vdev
, int pos
, Error
**errp
)
1440 vdev
->msix
->pending
= g_malloc0(BITS_TO_LONGS(vdev
->msix
->entries
) *
1441 sizeof(unsigned long));
1442 ret
= msix_init(&vdev
->pdev
, vdev
->msix
->entries
,
1443 vdev
->bars
[vdev
->msix
->table_bar
].region
.mem
,
1444 vdev
->msix
->table_bar
, vdev
->msix
->table_offset
,
1445 vdev
->bars
[vdev
->msix
->pba_bar
].region
.mem
,
1446 vdev
->msix
->pba_bar
, vdev
->msix
->pba_offset
, pos
,
1449 if (ret
== -ENOTSUP
) {
1450 error_report_err(err
);
1454 error_propagate(errp
, err
);
1459 * The PCI spec suggests that devices provide additional alignment for
1460 * MSI-X structures and avoid overlapping non-MSI-X related registers.
1461 * For an assigned device, this hopefully means that emulation of MSI-X
1462 * structures does not affect the performance of the device. If devices
1463 * fail to provide that alignment, a significant performance penalty may
1464 * result, for instance Mellanox MT27500 VFs:
1465 * http://www.spinics.net/lists/kvm/msg125881.html
1467 * The PBA is simply not that important for such a serious regression and
1468 * most drivers do not appear to look at it. The solution for this is to
1469 * disable the PBA MemoryRegion unless it's being used. We disable it
1470 * here and only enable it if a masked vector fires through QEMU. As the
1471 * vector-use notifier is called, which occurs on unmask, we test whether
1472 * PBA emulation is needed and again disable if not.
1474 memory_region_set_enabled(&vdev
->pdev
.msix_pba_mmio
, false);
1479 static void vfio_teardown_msi(VFIOPCIDevice
*vdev
)
1481 msi_uninit(&vdev
->pdev
);
1484 msix_uninit(&vdev
->pdev
,
1485 vdev
->bars
[vdev
->msix
->table_bar
].region
.mem
,
1486 vdev
->bars
[vdev
->msix
->pba_bar
].region
.mem
);
1487 g_free(vdev
->msix
->pending
);
1494 static void vfio_mmap_set_enabled(VFIOPCIDevice
*vdev
, bool enabled
)
1498 for (i
= 0; i
< PCI_ROM_SLOT
; i
++) {
1499 vfio_region_mmaps_set_enabled(&vdev
->bars
[i
].region
, enabled
);
1503 static void vfio_bar_setup(VFIOPCIDevice
*vdev
, int nr
)
1505 VFIOBAR
*bar
= &vdev
->bars
[nr
];
1511 /* Skip both unimplemented BARs and the upper half of 64bit BARS. */
1512 if (!bar
->region
.size
) {
1516 /* Determine what type of BAR this is for registration */
1517 ret
= pread(vdev
->vbasedev
.fd
, &pci_bar
, sizeof(pci_bar
),
1518 vdev
->config_offset
+ PCI_BASE_ADDRESS_0
+ (4 * nr
));
1519 if (ret
!= sizeof(pci_bar
)) {
1520 error_report("vfio: Failed to read BAR %d (%m)", nr
);
1524 pci_bar
= le32_to_cpu(pci_bar
);
1525 bar
->ioport
= (pci_bar
& PCI_BASE_ADDRESS_SPACE_IO
);
1526 bar
->mem64
= bar
->ioport
? 0 : (pci_bar
& PCI_BASE_ADDRESS_MEM_TYPE_64
);
1527 type
= pci_bar
& (bar
->ioport
? ~PCI_BASE_ADDRESS_IO_MASK
:
1528 ~PCI_BASE_ADDRESS_MEM_MASK
);
1530 if (vfio_region_mmap(&bar
->region
)) {
1531 error_report("Failed to mmap %s BAR %d. Performance may be slow",
1532 vdev
->vbasedev
.name
, nr
);
1535 pci_register_bar(&vdev
->pdev
, nr
, type
, bar
->region
.mem
);
1538 static void vfio_bars_setup(VFIOPCIDevice
*vdev
)
1542 for (i
= 0; i
< PCI_ROM_SLOT
; i
++) {
1543 vfio_bar_setup(vdev
, i
);
1547 static void vfio_bars_exit(VFIOPCIDevice
*vdev
)
1551 for (i
= 0; i
< PCI_ROM_SLOT
; i
++) {
1552 vfio_bar_quirk_exit(vdev
, i
);
1553 vfio_region_exit(&vdev
->bars
[i
].region
);
1557 pci_unregister_vga(&vdev
->pdev
);
1558 vfio_vga_quirk_exit(vdev
);
1562 static void vfio_bars_finalize(VFIOPCIDevice
*vdev
)
1566 for (i
= 0; i
< PCI_ROM_SLOT
; i
++) {
1567 vfio_bar_quirk_finalize(vdev
, i
);
1568 vfio_region_finalize(&vdev
->bars
[i
].region
);
1572 vfio_vga_quirk_finalize(vdev
);
1573 for (i
= 0; i
< ARRAY_SIZE(vdev
->vga
->region
); i
++) {
1574 object_unparent(OBJECT(&vdev
->vga
->region
[i
].mem
));
1583 static uint8_t vfio_std_cap_max_size(PCIDevice
*pdev
, uint8_t pos
)
1586 uint16_t next
= PCI_CONFIG_SPACE_SIZE
;
1588 for (tmp
= pdev
->config
[PCI_CAPABILITY_LIST
]; tmp
;
1589 tmp
= pdev
->config
[tmp
+ PCI_CAP_LIST_NEXT
]) {
1590 if (tmp
> pos
&& tmp
< next
) {
1599 static uint16_t vfio_ext_cap_max_size(const uint8_t *config
, uint16_t pos
)
1601 uint16_t tmp
, next
= PCIE_CONFIG_SPACE_SIZE
;
1603 for (tmp
= PCI_CONFIG_SPACE_SIZE
; tmp
;
1604 tmp
= PCI_EXT_CAP_NEXT(pci_get_long(config
+ tmp
))) {
1605 if (tmp
> pos
&& tmp
< next
) {
1613 static void vfio_set_word_bits(uint8_t *buf
, uint16_t val
, uint16_t mask
)
1615 pci_set_word(buf
, (pci_get_word(buf
) & ~mask
) | val
);
1618 static void vfio_add_emulated_word(VFIOPCIDevice
*vdev
, int pos
,
1619 uint16_t val
, uint16_t mask
)
1621 vfio_set_word_bits(vdev
->pdev
.config
+ pos
, val
, mask
);
1622 vfio_set_word_bits(vdev
->pdev
.wmask
+ pos
, ~mask
, mask
);
1623 vfio_set_word_bits(vdev
->emulated_config_bits
+ pos
, mask
, mask
);
1626 static void vfio_set_long_bits(uint8_t *buf
, uint32_t val
, uint32_t mask
)
1628 pci_set_long(buf
, (pci_get_long(buf
) & ~mask
) | val
);
1631 static void vfio_add_emulated_long(VFIOPCIDevice
*vdev
, int pos
,
1632 uint32_t val
, uint32_t mask
)
1634 vfio_set_long_bits(vdev
->pdev
.config
+ pos
, val
, mask
);
1635 vfio_set_long_bits(vdev
->pdev
.wmask
+ pos
, ~mask
, mask
);
1636 vfio_set_long_bits(vdev
->emulated_config_bits
+ pos
, mask
, mask
);
1639 static int vfio_setup_pcie_cap(VFIOPCIDevice
*vdev
, int pos
, uint8_t size
,
1645 flags
= pci_get_word(vdev
->pdev
.config
+ pos
+ PCI_CAP_FLAGS
);
1646 type
= (flags
& PCI_EXP_FLAGS_TYPE
) >> 4;
1648 if (type
!= PCI_EXP_TYPE_ENDPOINT
&&
1649 type
!= PCI_EXP_TYPE_LEG_END
&&
1650 type
!= PCI_EXP_TYPE_RC_END
) {
1652 error_setg(errp
, "assignment of PCIe type 0x%x "
1653 "devices is not currently supported", type
);
1657 if (!pci_bus_is_express(vdev
->pdev
.bus
)) {
1658 PCIBus
*bus
= vdev
->pdev
.bus
;
1662 * Traditionally PCI device assignment exposes the PCIe capability
1663 * as-is on non-express buses. The reason being that some drivers
1664 * simply assume that it's there, for example tg3. However when
1665 * we're running on a native PCIe machine type, like Q35, we need
1666 * to hide the PCIe capability. The reason for this is twofold;
1667 * first Windows guests get a Code 10 error when the PCIe capability
1668 * is exposed in this configuration. Therefore express devices won't
1669 * work at all unless they're attached to express buses in the VM.
1670 * Second, a native PCIe machine introduces the possibility of fine
1671 * granularity IOMMUs supporting both translation and isolation.
1672 * Guest code to discover the IOMMU visibility of a device, such as
1673 * IOMMU grouping code on Linux, is very aware of device types and
1674 * valid transitions between bus types. An express device on a non-
1675 * express bus is not a valid combination on bare metal systems.
1677 * Drivers that require a PCIe capability to make the device
1678 * functional are simply going to need to have their devices placed
1679 * on a PCIe bus in the VM.
1681 while (!pci_bus_is_root(bus
)) {
1682 bridge
= pci_bridge_get_device(bus
);
1686 if (pci_bus_is_express(bus
)) {
1690 } else if (pci_bus_is_root(vdev
->pdev
.bus
)) {
1692 * On a Root Complex bus Endpoints become Root Complex Integrated
1693 * Endpoints, which changes the type and clears the LNK & LNK2 fields.
1695 if (type
== PCI_EXP_TYPE_ENDPOINT
) {
1696 vfio_add_emulated_word(vdev
, pos
+ PCI_CAP_FLAGS
,
1697 PCI_EXP_TYPE_RC_END
<< 4,
1698 PCI_EXP_FLAGS_TYPE
);
1700 /* Link Capabilities, Status, and Control goes away */
1701 if (size
> PCI_EXP_LNKCTL
) {
1702 vfio_add_emulated_long(vdev
, pos
+ PCI_EXP_LNKCAP
, 0, ~0);
1703 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKCTL
, 0, ~0);
1704 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKSTA
, 0, ~0);
1706 #ifndef PCI_EXP_LNKCAP2
1707 #define PCI_EXP_LNKCAP2 44
1709 #ifndef PCI_EXP_LNKSTA2
1710 #define PCI_EXP_LNKSTA2 50
1712 /* Link 2 Capabilities, Status, and Control goes away */
1713 if (size
> PCI_EXP_LNKCAP2
) {
1714 vfio_add_emulated_long(vdev
, pos
+ PCI_EXP_LNKCAP2
, 0, ~0);
1715 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKCTL2
, 0, ~0);
1716 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKSTA2
, 0, ~0);
1720 } else if (type
== PCI_EXP_TYPE_LEG_END
) {
1722 * Legacy endpoints don't belong on the root complex. Windows
1723 * seems to be happier with devices if we skip the capability.
1730 * Convert Root Complex Integrated Endpoints to regular endpoints.
1731 * These devices don't support LNK/LNK2 capabilities, so make them up.
1733 if (type
== PCI_EXP_TYPE_RC_END
) {
1734 vfio_add_emulated_word(vdev
, pos
+ PCI_CAP_FLAGS
,
1735 PCI_EXP_TYPE_ENDPOINT
<< 4,
1736 PCI_EXP_FLAGS_TYPE
);
1737 vfio_add_emulated_long(vdev
, pos
+ PCI_EXP_LNKCAP
,
1738 PCI_EXP_LNK_MLW_1
| PCI_EXP_LNK_LS_25
, ~0);
1739 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKCTL
, 0, ~0);
1742 /* Mark the Link Status bits as emulated to allow virtual negotiation */
1743 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKSTA
,
1744 pci_get_word(vdev
->pdev
.config
+ pos
+
1746 PCI_EXP_LNKCAP_MLW
| PCI_EXP_LNKCAP_SLS
);
1750 * Intel 82599 SR-IOV VFs report an invalid PCIe capability version 0
1751 * (Niantic errate #35) causing Windows to error with a Code 10 for the
1752 * device on Q35. Fixup any such devices to report version 1. If we
1753 * were to remove the capability entirely the guest would lose extended
1756 if ((flags
& PCI_EXP_FLAGS_VERS
) == 0) {
1757 vfio_add_emulated_word(vdev
, pos
+ PCI_CAP_FLAGS
,
1758 1, PCI_EXP_FLAGS_VERS
);
1761 pos
= pci_add_capability(&vdev
->pdev
, PCI_CAP_ID_EXP
, pos
, size
,
1767 vdev
->pdev
.exp
.exp_cap
= pos
;
1772 static void vfio_check_pcie_flr(VFIOPCIDevice
*vdev
, uint8_t pos
)
1774 uint32_t cap
= pci_get_long(vdev
->pdev
.config
+ pos
+ PCI_EXP_DEVCAP
);
1776 if (cap
& PCI_EXP_DEVCAP_FLR
) {
1777 trace_vfio_check_pcie_flr(vdev
->vbasedev
.name
);
1778 vdev
->has_flr
= true;
1782 static void vfio_check_pm_reset(VFIOPCIDevice
*vdev
, uint8_t pos
)
1784 uint16_t csr
= pci_get_word(vdev
->pdev
.config
+ pos
+ PCI_PM_CTRL
);
1786 if (!(csr
& PCI_PM_CTRL_NO_SOFT_RESET
)) {
1787 trace_vfio_check_pm_reset(vdev
->vbasedev
.name
);
1788 vdev
->has_pm_reset
= true;
1792 static void vfio_check_af_flr(VFIOPCIDevice
*vdev
, uint8_t pos
)
1794 uint8_t cap
= pci_get_byte(vdev
->pdev
.config
+ pos
+ PCI_AF_CAP
);
1796 if ((cap
& PCI_AF_CAP_TP
) && (cap
& PCI_AF_CAP_FLR
)) {
1797 trace_vfio_check_af_flr(vdev
->vbasedev
.name
);
1798 vdev
->has_flr
= true;
1802 static int vfio_add_std_cap(VFIOPCIDevice
*vdev
, uint8_t pos
, Error
**errp
)
1804 PCIDevice
*pdev
= &vdev
->pdev
;
1805 uint8_t cap_id
, next
, size
;
1808 cap_id
= pdev
->config
[pos
];
1809 next
= pdev
->config
[pos
+ PCI_CAP_LIST_NEXT
];
1812 * If it becomes important to configure capabilities to their actual
1813 * size, use this as the default when it's something we don't recognize.
1814 * Since QEMU doesn't actually handle many of the config accesses,
1815 * exact size doesn't seem worthwhile.
1817 size
= vfio_std_cap_max_size(pdev
, pos
);
1820 * pci_add_capability always inserts the new capability at the head
1821 * of the chain. Therefore to end up with a chain that matches the
1822 * physical device, we insert from the end by making this recursive.
1823 * This is also why we pre-calculate size above as cached config space
1824 * will be changed as we unwind the stack.
1827 ret
= vfio_add_std_cap(vdev
, next
, errp
);
1832 /* Begin the rebuild, use QEMU emulated list bits */
1833 pdev
->config
[PCI_CAPABILITY_LIST
] = 0;
1834 vdev
->emulated_config_bits
[PCI_CAPABILITY_LIST
] = 0xff;
1835 vdev
->emulated_config_bits
[PCI_STATUS
] |= PCI_STATUS_CAP_LIST
;
1837 ret
= vfio_add_virt_caps(vdev
, errp
);
1843 /* Scale down size, esp in case virt caps were added above */
1844 size
= MIN(size
, vfio_std_cap_max_size(pdev
, pos
));
1846 /* Use emulated next pointer to allow dropping caps */
1847 pci_set_byte(vdev
->emulated_config_bits
+ pos
+ PCI_CAP_LIST_NEXT
, 0xff);
1850 case PCI_CAP_ID_MSI
:
1851 ret
= vfio_msi_setup(vdev
, pos
, errp
);
1853 case PCI_CAP_ID_EXP
:
1854 vfio_check_pcie_flr(vdev
, pos
);
1855 ret
= vfio_setup_pcie_cap(vdev
, pos
, size
, errp
);
1857 case PCI_CAP_ID_MSIX
:
1858 ret
= vfio_msix_setup(vdev
, pos
, errp
);
1861 vfio_check_pm_reset(vdev
, pos
);
1863 ret
= pci_add_capability(pdev
, cap_id
, pos
, size
, errp
);
1866 vfio_check_af_flr(vdev
, pos
);
1867 ret
= pci_add_capability(pdev
, cap_id
, pos
, size
, errp
);
1870 ret
= pci_add_capability(pdev
, cap_id
, pos
, size
, errp
);
1876 "failed to add PCI capability 0x%x[0x%x]@0x%x: ",
1884 static void vfio_add_ext_cap(VFIOPCIDevice
*vdev
)
1886 PCIDevice
*pdev
= &vdev
->pdev
;
1888 uint16_t cap_id
, next
, size
;
1892 /* Only add extended caps if we have them and the guest can see them */
1893 if (!pci_is_express(pdev
) || !pci_bus_is_express(pdev
->bus
) ||
1894 !pci_get_long(pdev
->config
+ PCI_CONFIG_SPACE_SIZE
)) {
1899 * pcie_add_capability always inserts the new capability at the tail
1900 * of the chain. Therefore to end up with a chain that matches the
1901 * physical device, we cache the config space to avoid overwriting
1902 * the original config space when we parse the extended capabilities.
1904 config
= g_memdup(pdev
->config
, vdev
->config_size
);
1907 * Extended capabilities are chained with each pointing to the next, so we
1908 * can drop anything other than the head of the chain simply by modifying
1909 * the previous next pointer. Seed the head of the chain here such that
1910 * we can simply skip any capabilities we want to drop below, regardless
1911 * of their position in the chain. If this stub capability still exists
1912 * after we add the capabilities we want to expose, update the capability
1913 * ID to zero. Note that we cannot seed with the capability header being
1914 * zero as this conflicts with definition of an absent capability chain
1915 * and prevents capabilities beyond the head of the list from being added.
1916 * By replacing the dummy capability ID with zero after walking the device
1917 * chain, we also transparently mark extended capabilities as absent if
1918 * no capabilities were added. Note that the PCIe spec defines an absence
1919 * of extended capabilities to be determined by a value of zero for the
1920 * capability ID, version, AND next pointer. A non-zero next pointer
1921 * should be sufficient to indicate additional capabilities are present,
1922 * which will occur if we call pcie_add_capability() below. The entire
1923 * first dword is emulated to support this.
1925 * NB. The kernel side does similar masking, so be prepared that our
1926 * view of the device may also contain a capability ID zero in the head
1927 * of the chain. Skip it for the same reason that we cannot seed the
1928 * chain with a zero capability.
1930 pci_set_long(pdev
->config
+ PCI_CONFIG_SPACE_SIZE
,
1931 PCI_EXT_CAP(0xFFFF, 0, 0));
1932 pci_set_long(pdev
->wmask
+ PCI_CONFIG_SPACE_SIZE
, 0);
1933 pci_set_long(vdev
->emulated_config_bits
+ PCI_CONFIG_SPACE_SIZE
, ~0);
1935 for (next
= PCI_CONFIG_SPACE_SIZE
; next
;
1936 next
= PCI_EXT_CAP_NEXT(pci_get_long(config
+ next
))) {
1937 header
= pci_get_long(config
+ next
);
1938 cap_id
= PCI_EXT_CAP_ID(header
);
1939 cap_ver
= PCI_EXT_CAP_VER(header
);
1942 * If it becomes important to configure extended capabilities to their
1943 * actual size, use this as the default when it's something we don't
1944 * recognize. Since QEMU doesn't actually handle many of the config
1945 * accesses, exact size doesn't seem worthwhile.
1947 size
= vfio_ext_cap_max_size(config
, next
);
1949 /* Use emulated next pointer to allow dropping extended caps */
1950 pci_long_test_and_set_mask(vdev
->emulated_config_bits
+ next
,
1951 PCI_EXT_CAP_NEXT_MASK
);
1954 case 0: /* kernel masked capability */
1955 case PCI_EXT_CAP_ID_SRIOV
: /* Read-only VF BARs confuse OVMF */
1956 case PCI_EXT_CAP_ID_ARI
: /* XXX Needs next function virtualization */
1957 trace_vfio_add_ext_cap_dropped(vdev
->vbasedev
.name
, cap_id
, next
);
1960 pcie_add_capability(pdev
, cap_id
, cap_ver
, next
, size
);
1965 /* Cleanup chain head ID if necessary */
1966 if (pci_get_word(pdev
->config
+ PCI_CONFIG_SPACE_SIZE
) == 0xFFFF) {
1967 pci_set_word(pdev
->config
+ PCI_CONFIG_SPACE_SIZE
, 0);
1974 static int vfio_add_capabilities(VFIOPCIDevice
*vdev
, Error
**errp
)
1976 PCIDevice
*pdev
= &vdev
->pdev
;
1979 if (!(pdev
->config
[PCI_STATUS
] & PCI_STATUS_CAP_LIST
) ||
1980 !pdev
->config
[PCI_CAPABILITY_LIST
]) {
1981 return 0; /* Nothing to add */
1984 ret
= vfio_add_std_cap(vdev
, pdev
->config
[PCI_CAPABILITY_LIST
], errp
);
1989 vfio_add_ext_cap(vdev
);
1993 static void vfio_pci_pre_reset(VFIOPCIDevice
*vdev
)
1995 PCIDevice
*pdev
= &vdev
->pdev
;
1998 vfio_disable_interrupts(vdev
);
2000 /* Make sure the device is in D0 */
2005 pmcsr
= vfio_pci_read_config(pdev
, vdev
->pm_cap
+ PCI_PM_CTRL
, 2);
2006 state
= pmcsr
& PCI_PM_CTRL_STATE_MASK
;
2008 pmcsr
&= ~PCI_PM_CTRL_STATE_MASK
;
2009 vfio_pci_write_config(pdev
, vdev
->pm_cap
+ PCI_PM_CTRL
, pmcsr
, 2);
2010 /* vfio handles the necessary delay here */
2011 pmcsr
= vfio_pci_read_config(pdev
, vdev
->pm_cap
+ PCI_PM_CTRL
, 2);
2012 state
= pmcsr
& PCI_PM_CTRL_STATE_MASK
;
2014 error_report("vfio: Unable to power on device, stuck in D%d",
2021 * Stop any ongoing DMA by disconecting I/O, MMIO, and bus master.
2022 * Also put INTx Disable in known state.
2024 cmd
= vfio_pci_read_config(pdev
, PCI_COMMAND
, 2);
2025 cmd
&= ~(PCI_COMMAND_IO
| PCI_COMMAND_MEMORY
| PCI_COMMAND_MASTER
|
2026 PCI_COMMAND_INTX_DISABLE
);
2027 vfio_pci_write_config(pdev
, PCI_COMMAND
, cmd
, 2);
2030 static void vfio_pci_post_reset(VFIOPCIDevice
*vdev
)
2035 vfio_intx_enable(vdev
, &err
);
2037 error_reportf_err(err
, ERR_PREFIX
, vdev
->vbasedev
.name
);
2040 for (nr
= 0; nr
< PCI_NUM_REGIONS
- 1; ++nr
) {
2041 off_t addr
= vdev
->config_offset
+ PCI_BASE_ADDRESS_0
+ (4 * nr
);
2043 uint32_t len
= sizeof(val
);
2045 if (pwrite(vdev
->vbasedev
.fd
, &val
, len
, addr
) != len
) {
2046 error_report("%s(%s) reset bar %d failed: %m", __func__
,
2047 vdev
->vbasedev
.name
, nr
);
2052 static bool vfio_pci_host_match(PCIHostDeviceAddress
*addr
, const char *name
)
2056 sprintf(tmp
, "%04x:%02x:%02x.%1x", addr
->domain
,
2057 addr
->bus
, addr
->slot
, addr
->function
);
2059 return (strcmp(tmp
, name
) == 0);
2062 static int vfio_pci_hot_reset(VFIOPCIDevice
*vdev
, bool single
)
2065 struct vfio_pci_hot_reset_info
*info
;
2066 struct vfio_pci_dependent_device
*devices
;
2067 struct vfio_pci_hot_reset
*reset
;
2072 trace_vfio_pci_hot_reset(vdev
->vbasedev
.name
, single
? "one" : "multi");
2075 vfio_pci_pre_reset(vdev
);
2077 vdev
->vbasedev
.needs_reset
= false;
2079 info
= g_malloc0(sizeof(*info
));
2080 info
->argsz
= sizeof(*info
);
2082 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO
, info
);
2083 if (ret
&& errno
!= ENOSPC
) {
2085 if (!vdev
->has_pm_reset
) {
2086 error_report("vfio: Cannot reset device %s, "
2087 "no available reset mechanism.", vdev
->vbasedev
.name
);
2092 count
= info
->count
;
2093 info
= g_realloc(info
, sizeof(*info
) + (count
* sizeof(*devices
)));
2094 info
->argsz
= sizeof(*info
) + (count
* sizeof(*devices
));
2095 devices
= &info
->devices
[0];
2097 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO
, info
);
2100 error_report("vfio: hot reset info failed: %m");
2104 trace_vfio_pci_hot_reset_has_dep_devices(vdev
->vbasedev
.name
);
2106 /* Verify that we have all the groups required */
2107 for (i
= 0; i
< info
->count
; i
++) {
2108 PCIHostDeviceAddress host
;
2110 VFIODevice
*vbasedev_iter
;
2112 host
.domain
= devices
[i
].segment
;
2113 host
.bus
= devices
[i
].bus
;
2114 host
.slot
= PCI_SLOT(devices
[i
].devfn
);
2115 host
.function
= PCI_FUNC(devices
[i
].devfn
);
2117 trace_vfio_pci_hot_reset_dep_devices(host
.domain
,
2118 host
.bus
, host
.slot
, host
.function
, devices
[i
].group_id
);
2120 if (vfio_pci_host_match(&host
, vdev
->vbasedev
.name
)) {
2124 QLIST_FOREACH(group
, &vfio_group_list
, next
) {
2125 if (group
->groupid
== devices
[i
].group_id
) {
2131 if (!vdev
->has_pm_reset
) {
2132 error_report("vfio: Cannot reset device %s, "
2133 "depends on group %d which is not owned.",
2134 vdev
->vbasedev
.name
, devices
[i
].group_id
);
2140 /* Prep dependent devices for reset and clear our marker. */
2141 QLIST_FOREACH(vbasedev_iter
, &group
->device_list
, next
) {
2142 if (!vbasedev_iter
->dev
->realized
||
2143 vbasedev_iter
->type
!= VFIO_DEVICE_TYPE_PCI
) {
2146 tmp
= container_of(vbasedev_iter
, VFIOPCIDevice
, vbasedev
);
2147 if (vfio_pci_host_match(&host
, tmp
->vbasedev
.name
)) {
2152 vfio_pci_pre_reset(tmp
);
2153 tmp
->vbasedev
.needs_reset
= false;
2160 if (!single
&& !multi
) {
2165 /* Determine how many group fds need to be passed */
2167 QLIST_FOREACH(group
, &vfio_group_list
, next
) {
2168 for (i
= 0; i
< info
->count
; i
++) {
2169 if (group
->groupid
== devices
[i
].group_id
) {
2176 reset
= g_malloc0(sizeof(*reset
) + (count
* sizeof(*fds
)));
2177 reset
->argsz
= sizeof(*reset
) + (count
* sizeof(*fds
));
2178 fds
= &reset
->group_fds
[0];
2180 /* Fill in group fds */
2181 QLIST_FOREACH(group
, &vfio_group_list
, next
) {
2182 for (i
= 0; i
< info
->count
; i
++) {
2183 if (group
->groupid
== devices
[i
].group_id
) {
2184 fds
[reset
->count
++] = group
->fd
;
2191 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_PCI_HOT_RESET
, reset
);
2194 trace_vfio_pci_hot_reset_result(vdev
->vbasedev
.name
,
2195 ret
? "%m" : "Success");
2198 /* Re-enable INTx on affected devices */
2199 for (i
= 0; i
< info
->count
; i
++) {
2200 PCIHostDeviceAddress host
;
2202 VFIODevice
*vbasedev_iter
;
2204 host
.domain
= devices
[i
].segment
;
2205 host
.bus
= devices
[i
].bus
;
2206 host
.slot
= PCI_SLOT(devices
[i
].devfn
);
2207 host
.function
= PCI_FUNC(devices
[i
].devfn
);
2209 if (vfio_pci_host_match(&host
, vdev
->vbasedev
.name
)) {
2213 QLIST_FOREACH(group
, &vfio_group_list
, next
) {
2214 if (group
->groupid
== devices
[i
].group_id
) {
2223 QLIST_FOREACH(vbasedev_iter
, &group
->device_list
, next
) {
2224 if (!vbasedev_iter
->dev
->realized
||
2225 vbasedev_iter
->type
!= VFIO_DEVICE_TYPE_PCI
) {
2228 tmp
= container_of(vbasedev_iter
, VFIOPCIDevice
, vbasedev
);
2229 if (vfio_pci_host_match(&host
, tmp
->vbasedev
.name
)) {
2230 vfio_pci_post_reset(tmp
);
2237 vfio_pci_post_reset(vdev
);
2245 * We want to differentiate hot reset of mulitple in-use devices vs hot reset
2246 * of a single in-use device. VFIO_DEVICE_RESET will already handle the case
2247 * of doing hot resets when there is only a single device per bus. The in-use
2248 * here refers to how many VFIODevices are affected. A hot reset that affects
2249 * multiple devices, but only a single in-use device, means that we can call
2250 * it from our bus ->reset() callback since the extent is effectively a single
2251 * device. This allows us to make use of it in the hotplug path. When there
2252 * are multiple in-use devices, we can only trigger the hot reset during a
2253 * system reset and thus from our reset handler. We separate _one vs _multi
2254 * here so that we don't overlap and do a double reset on the system reset
2255 * path where both our reset handler and ->reset() callback are used. Calling
2256 * _one() will only do a hot reset for the one in-use devices case, calling
2257 * _multi() will do nothing if a _one() would have been sufficient.
2259 static int vfio_pci_hot_reset_one(VFIOPCIDevice
*vdev
)
2261 return vfio_pci_hot_reset(vdev
, true);
2264 static int vfio_pci_hot_reset_multi(VFIODevice
*vbasedev
)
2266 VFIOPCIDevice
*vdev
= container_of(vbasedev
, VFIOPCIDevice
, vbasedev
);
2267 return vfio_pci_hot_reset(vdev
, false);
2270 static void vfio_pci_compute_needs_reset(VFIODevice
*vbasedev
)
2272 VFIOPCIDevice
*vdev
= container_of(vbasedev
, VFIOPCIDevice
, vbasedev
);
2273 if (!vbasedev
->reset_works
|| (!vdev
->has_flr
&& vdev
->has_pm_reset
)) {
2274 vbasedev
->needs_reset
= true;
2278 static VFIODeviceOps vfio_pci_ops
= {
2279 .vfio_compute_needs_reset
= vfio_pci_compute_needs_reset
,
2280 .vfio_hot_reset_multi
= vfio_pci_hot_reset_multi
,
2281 .vfio_eoi
= vfio_intx_eoi
,
2284 int vfio_populate_vga(VFIOPCIDevice
*vdev
, Error
**errp
)
2286 VFIODevice
*vbasedev
= &vdev
->vbasedev
;
2287 struct vfio_region_info
*reg_info
;
2290 ret
= vfio_get_region_info(vbasedev
, VFIO_PCI_VGA_REGION_INDEX
, ®_info
);
2292 error_setg_errno(errp
, -ret
,
2293 "failed getting region info for VGA region index %d",
2294 VFIO_PCI_VGA_REGION_INDEX
);
2298 if (!(reg_info
->flags
& VFIO_REGION_INFO_FLAG_READ
) ||
2299 !(reg_info
->flags
& VFIO_REGION_INFO_FLAG_WRITE
) ||
2300 reg_info
->size
< 0xbffff + 1) {
2301 error_setg(errp
, "unexpected VGA info, flags 0x%lx, size 0x%lx",
2302 (unsigned long)reg_info
->flags
,
2303 (unsigned long)reg_info
->size
);
2308 vdev
->vga
= g_new0(VFIOVGA
, 1);
2310 vdev
->vga
->fd_offset
= reg_info
->offset
;
2311 vdev
->vga
->fd
= vdev
->vbasedev
.fd
;
2315 vdev
->vga
->region
[QEMU_PCI_VGA_MEM
].offset
= QEMU_PCI_VGA_MEM_BASE
;
2316 vdev
->vga
->region
[QEMU_PCI_VGA_MEM
].nr
= QEMU_PCI_VGA_MEM
;
2317 QLIST_INIT(&vdev
->vga
->region
[QEMU_PCI_VGA_MEM
].quirks
);
2319 memory_region_init_io(&vdev
->vga
->region
[QEMU_PCI_VGA_MEM
].mem
,
2320 OBJECT(vdev
), &vfio_vga_ops
,
2321 &vdev
->vga
->region
[QEMU_PCI_VGA_MEM
],
2322 "vfio-vga-mmio@0xa0000",
2323 QEMU_PCI_VGA_MEM_SIZE
);
2325 vdev
->vga
->region
[QEMU_PCI_VGA_IO_LO
].offset
= QEMU_PCI_VGA_IO_LO_BASE
;
2326 vdev
->vga
->region
[QEMU_PCI_VGA_IO_LO
].nr
= QEMU_PCI_VGA_IO_LO
;
2327 QLIST_INIT(&vdev
->vga
->region
[QEMU_PCI_VGA_IO_LO
].quirks
);
2329 memory_region_init_io(&vdev
->vga
->region
[QEMU_PCI_VGA_IO_LO
].mem
,
2330 OBJECT(vdev
), &vfio_vga_ops
,
2331 &vdev
->vga
->region
[QEMU_PCI_VGA_IO_LO
],
2332 "vfio-vga-io@0x3b0",
2333 QEMU_PCI_VGA_IO_LO_SIZE
);
2335 vdev
->vga
->region
[QEMU_PCI_VGA_IO_HI
].offset
= QEMU_PCI_VGA_IO_HI_BASE
;
2336 vdev
->vga
->region
[QEMU_PCI_VGA_IO_HI
].nr
= QEMU_PCI_VGA_IO_HI
;
2337 QLIST_INIT(&vdev
->vga
->region
[QEMU_PCI_VGA_IO_HI
].quirks
);
2339 memory_region_init_io(&vdev
->vga
->region
[QEMU_PCI_VGA_IO_HI
].mem
,
2340 OBJECT(vdev
), &vfio_vga_ops
,
2341 &vdev
->vga
->region
[QEMU_PCI_VGA_IO_HI
],
2342 "vfio-vga-io@0x3c0",
2343 QEMU_PCI_VGA_IO_HI_SIZE
);
2345 pci_register_vga(&vdev
->pdev
, &vdev
->vga
->region
[QEMU_PCI_VGA_MEM
].mem
,
2346 &vdev
->vga
->region
[QEMU_PCI_VGA_IO_LO
].mem
,
2347 &vdev
->vga
->region
[QEMU_PCI_VGA_IO_HI
].mem
);
2352 static void vfio_populate_device(VFIOPCIDevice
*vdev
, Error
**errp
)
2354 VFIODevice
*vbasedev
= &vdev
->vbasedev
;
2355 struct vfio_region_info
*reg_info
;
2356 struct vfio_irq_info irq_info
= { .argsz
= sizeof(irq_info
) };
2359 /* Sanity check device */
2360 if (!(vbasedev
->flags
& VFIO_DEVICE_FLAGS_PCI
)) {
2361 error_setg(errp
, "this isn't a PCI device");
2365 if (vbasedev
->num_regions
< VFIO_PCI_CONFIG_REGION_INDEX
+ 1) {
2366 error_setg(errp
, "unexpected number of io regions %u",
2367 vbasedev
->num_regions
);
2371 if (vbasedev
->num_irqs
< VFIO_PCI_MSIX_IRQ_INDEX
+ 1) {
2372 error_setg(errp
, "unexpected number of irqs %u", vbasedev
->num_irqs
);
2376 for (i
= VFIO_PCI_BAR0_REGION_INDEX
; i
< VFIO_PCI_ROM_REGION_INDEX
; i
++) {
2377 char *name
= g_strdup_printf("%s BAR %d", vbasedev
->name
, i
);
2379 ret
= vfio_region_setup(OBJECT(vdev
), vbasedev
,
2380 &vdev
->bars
[i
].region
, i
, name
);
2384 error_setg_errno(errp
, -ret
, "failed to get region %d info", i
);
2388 QLIST_INIT(&vdev
->bars
[i
].quirks
);
2391 ret
= vfio_get_region_info(vbasedev
,
2392 VFIO_PCI_CONFIG_REGION_INDEX
, ®_info
);
2394 error_setg_errno(errp
, -ret
, "failed to get config info");
2398 trace_vfio_populate_device_config(vdev
->vbasedev
.name
,
2399 (unsigned long)reg_info
->size
,
2400 (unsigned long)reg_info
->offset
,
2401 (unsigned long)reg_info
->flags
);
2403 vdev
->config_size
= reg_info
->size
;
2404 if (vdev
->config_size
== PCI_CONFIG_SPACE_SIZE
) {
2405 vdev
->pdev
.cap_present
&= ~QEMU_PCI_CAP_EXPRESS
;
2407 vdev
->config_offset
= reg_info
->offset
;
2411 if (vdev
->features
& VFIO_FEATURE_ENABLE_VGA
) {
2412 ret
= vfio_populate_vga(vdev
, errp
);
2414 error_append_hint(errp
, "device does not support "
2415 "requested feature x-vga\n");
2420 irq_info
.index
= VFIO_PCI_ERR_IRQ_INDEX
;
2422 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_GET_IRQ_INFO
, &irq_info
);
2424 /* This can fail for an old kernel or legacy PCI dev */
2425 trace_vfio_populate_device_get_irq_info_failure();
2426 } else if (irq_info
.count
== 1) {
2427 vdev
->pci_aer
= true;
2429 error_report(WARN_PREFIX
2430 "Could not enable error recovery for the device",
2435 static void vfio_put_device(VFIOPCIDevice
*vdev
)
2437 g_free(vdev
->vbasedev
.name
);
2440 vfio_put_base_device(&vdev
->vbasedev
);
2443 static void vfio_err_notifier_handler(void *opaque
)
2445 VFIOPCIDevice
*vdev
= opaque
;
2447 if (!event_notifier_test_and_clear(&vdev
->err_notifier
)) {
2452 * TBD. Retrieve the error details and decide what action
2453 * needs to be taken. One of the actions could be to pass
2454 * the error to the guest and have the guest driver recover
2455 * from the error. This requires that PCIe capabilities be
2456 * exposed to the guest. For now, we just terminate the
2457 * guest to contain the error.
2460 error_report("%s(%s) Unrecoverable error detected. Please collect any data possible and then kill the guest", __func__
, vdev
->vbasedev
.name
);
2462 vm_stop(RUN_STATE_INTERNAL_ERROR
);
2466 * Registers error notifier for devices supporting error recovery.
2467 * If we encounter a failure in this function, we report an error
2468 * and continue after disabling error recovery support for the
2471 static void vfio_register_err_notifier(VFIOPCIDevice
*vdev
)
2475 struct vfio_irq_set
*irq_set
;
2478 if (!vdev
->pci_aer
) {
2482 if (event_notifier_init(&vdev
->err_notifier
, 0)) {
2483 error_report("vfio: Unable to init event notifier for error detection");
2484 vdev
->pci_aer
= false;
2488 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
2490 irq_set
= g_malloc0(argsz
);
2491 irq_set
->argsz
= argsz
;
2492 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
|
2493 VFIO_IRQ_SET_ACTION_TRIGGER
;
2494 irq_set
->index
= VFIO_PCI_ERR_IRQ_INDEX
;
2497 pfd
= (int32_t *)&irq_set
->data
;
2499 *pfd
= event_notifier_get_fd(&vdev
->err_notifier
);
2500 qemu_set_fd_handler(*pfd
, vfio_err_notifier_handler
, NULL
, vdev
);
2502 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
2504 error_report("vfio: Failed to set up error notification");
2505 qemu_set_fd_handler(*pfd
, NULL
, NULL
, vdev
);
2506 event_notifier_cleanup(&vdev
->err_notifier
);
2507 vdev
->pci_aer
= false;
2512 static void vfio_unregister_err_notifier(VFIOPCIDevice
*vdev
)
2515 struct vfio_irq_set
*irq_set
;
2519 if (!vdev
->pci_aer
) {
2523 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
2525 irq_set
= g_malloc0(argsz
);
2526 irq_set
->argsz
= argsz
;
2527 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
|
2528 VFIO_IRQ_SET_ACTION_TRIGGER
;
2529 irq_set
->index
= VFIO_PCI_ERR_IRQ_INDEX
;
2532 pfd
= (int32_t *)&irq_set
->data
;
2535 ret
= ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
2537 error_report("vfio: Failed to de-assign error fd: %m");
2540 qemu_set_fd_handler(event_notifier_get_fd(&vdev
->err_notifier
),
2542 event_notifier_cleanup(&vdev
->err_notifier
);
2545 static void vfio_req_notifier_handler(void *opaque
)
2547 VFIOPCIDevice
*vdev
= opaque
;
2550 if (!event_notifier_test_and_clear(&vdev
->req_notifier
)) {
2554 qdev_unplug(&vdev
->pdev
.qdev
, &err
);
2556 error_reportf_err(err
, WARN_PREFIX
, vdev
->vbasedev
.name
);
2560 static void vfio_register_req_notifier(VFIOPCIDevice
*vdev
)
2562 struct vfio_irq_info irq_info
= { .argsz
= sizeof(irq_info
),
2563 .index
= VFIO_PCI_REQ_IRQ_INDEX
};
2565 struct vfio_irq_set
*irq_set
;
2568 if (!(vdev
->features
& VFIO_FEATURE_ENABLE_REQ
)) {
2572 if (ioctl(vdev
->vbasedev
.fd
,
2573 VFIO_DEVICE_GET_IRQ_INFO
, &irq_info
) < 0 || irq_info
.count
< 1) {
2577 if (event_notifier_init(&vdev
->req_notifier
, 0)) {
2578 error_report("vfio: Unable to init event notifier for device request");
2582 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
2584 irq_set
= g_malloc0(argsz
);
2585 irq_set
->argsz
= argsz
;
2586 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
|
2587 VFIO_IRQ_SET_ACTION_TRIGGER
;
2588 irq_set
->index
= VFIO_PCI_REQ_IRQ_INDEX
;
2591 pfd
= (int32_t *)&irq_set
->data
;
2593 *pfd
= event_notifier_get_fd(&vdev
->req_notifier
);
2594 qemu_set_fd_handler(*pfd
, vfio_req_notifier_handler
, NULL
, vdev
);
2596 if (ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_SET_IRQS
, irq_set
)) {
2597 error_report("vfio: Failed to set up device request notification");
2598 qemu_set_fd_handler(*pfd
, NULL
, NULL
, vdev
);
2599 event_notifier_cleanup(&vdev
->req_notifier
);
2601 vdev
->req_enabled
= true;
2607 static void vfio_unregister_req_notifier(VFIOPCIDevice
*vdev
)
2610 struct vfio_irq_set
*irq_set
;
2613 if (!vdev
->req_enabled
) {
2617 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
2619 irq_set
= g_malloc0(argsz
);
2620 irq_set
->argsz
= argsz
;
2621 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
|
2622 VFIO_IRQ_SET_ACTION_TRIGGER
;
2623 irq_set
->index
= VFIO_PCI_REQ_IRQ_INDEX
;
2626 pfd
= (int32_t *)&irq_set
->data
;
2629 if (ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_SET_IRQS
, irq_set
)) {
2630 error_report("vfio: Failed to de-assign device request fd: %m");
2633 qemu_set_fd_handler(event_notifier_get_fd(&vdev
->req_notifier
),
2635 event_notifier_cleanup(&vdev
->req_notifier
);
2637 vdev
->req_enabled
= false;
2640 static void vfio_realize(PCIDevice
*pdev
, Error
**errp
)
2642 VFIOPCIDevice
*vdev
= DO_UPCAST(VFIOPCIDevice
, pdev
, pdev
);
2643 VFIODevice
*vbasedev_iter
;
2645 char *tmp
, group_path
[PATH_MAX
], *group_name
;
2652 if (!vdev
->vbasedev
.sysfsdev
) {
2653 if (!(~vdev
->host
.domain
|| ~vdev
->host
.bus
||
2654 ~vdev
->host
.slot
|| ~vdev
->host
.function
)) {
2655 error_setg(errp
, "No provided host device");
2656 error_append_hint(errp
, "Use -device vfio-pci,host=DDDD:BB:DD.F "
2657 "or -device vfio-pci,sysfsdev=PATH_TO_DEVICE\n");
2660 vdev
->vbasedev
.sysfsdev
=
2661 g_strdup_printf("/sys/bus/pci/devices/%04x:%02x:%02x.%01x",
2662 vdev
->host
.domain
, vdev
->host
.bus
,
2663 vdev
->host
.slot
, vdev
->host
.function
);
2666 if (stat(vdev
->vbasedev
.sysfsdev
, &st
) < 0) {
2667 error_setg_errno(errp
, errno
, "no such host device");
2668 error_prepend(errp
, ERR_PREFIX
, vdev
->vbasedev
.sysfsdev
);
2672 vdev
->vbasedev
.name
= g_strdup(basename(vdev
->vbasedev
.sysfsdev
));
2673 vdev
->vbasedev
.ops
= &vfio_pci_ops
;
2674 vdev
->vbasedev
.type
= VFIO_DEVICE_TYPE_PCI
;
2675 vdev
->vbasedev
.dev
= &vdev
->pdev
.qdev
;
2677 tmp
= g_strdup_printf("%s/iommu_group", vdev
->vbasedev
.sysfsdev
);
2678 len
= readlink(tmp
, group_path
, sizeof(group_path
));
2681 if (len
<= 0 || len
>= sizeof(group_path
)) {
2682 error_setg_errno(errp
, len
< 0 ? errno
: ENAMETOOLONG
,
2683 "no iommu_group found");
2687 group_path
[len
] = 0;
2689 group_name
= basename(group_path
);
2690 if (sscanf(group_name
, "%d", &groupid
) != 1) {
2691 error_setg_errno(errp
, errno
, "failed to read %s", group_path
);
2695 trace_vfio_realize(vdev
->vbasedev
.name
, groupid
);
2697 group
= vfio_get_group(groupid
, pci_device_iommu_address_space(pdev
), errp
);
2702 QLIST_FOREACH(vbasedev_iter
, &group
->device_list
, next
) {
2703 if (strcmp(vbasedev_iter
->name
, vdev
->vbasedev
.name
) == 0) {
2704 error_setg(errp
, "device is already attached");
2705 vfio_put_group(group
);
2710 ret
= vfio_get_device(group
, vdev
->vbasedev
.name
, &vdev
->vbasedev
, errp
);
2712 vfio_put_group(group
);
2716 vfio_populate_device(vdev
, &err
);
2718 error_propagate(errp
, err
);
2722 /* Get a copy of config space */
2723 ret
= pread(vdev
->vbasedev
.fd
, vdev
->pdev
.config
,
2724 MIN(pci_config_size(&vdev
->pdev
), vdev
->config_size
),
2725 vdev
->config_offset
);
2726 if (ret
< (int)MIN(pci_config_size(&vdev
->pdev
), vdev
->config_size
)) {
2727 ret
= ret
< 0 ? -errno
: -EFAULT
;
2728 error_setg_errno(errp
, -ret
, "failed to read device config space");
2732 /* vfio emulates a lot for us, but some bits need extra love */
2733 vdev
->emulated_config_bits
= g_malloc0(vdev
->config_size
);
2735 /* QEMU can choose to expose the ROM or not */
2736 memset(vdev
->emulated_config_bits
+ PCI_ROM_ADDRESS
, 0xff, 4);
2739 * The PCI spec reserves vendor ID 0xffff as an invalid value. The
2740 * device ID is managed by the vendor and need only be a 16-bit value.
2741 * Allow any 16-bit value for subsystem so they can be hidden or changed.
2743 if (vdev
->vendor_id
!= PCI_ANY_ID
) {
2744 if (vdev
->vendor_id
>= 0xffff) {
2745 error_setg(errp
, "invalid PCI vendor ID provided");
2748 vfio_add_emulated_word(vdev
, PCI_VENDOR_ID
, vdev
->vendor_id
, ~0);
2749 trace_vfio_pci_emulated_vendor_id(vdev
->vbasedev
.name
, vdev
->vendor_id
);
2751 vdev
->vendor_id
= pci_get_word(pdev
->config
+ PCI_VENDOR_ID
);
2754 if (vdev
->device_id
!= PCI_ANY_ID
) {
2755 if (vdev
->device_id
> 0xffff) {
2756 error_setg(errp
, "invalid PCI device ID provided");
2759 vfio_add_emulated_word(vdev
, PCI_DEVICE_ID
, vdev
->device_id
, ~0);
2760 trace_vfio_pci_emulated_device_id(vdev
->vbasedev
.name
, vdev
->device_id
);
2762 vdev
->device_id
= pci_get_word(pdev
->config
+ PCI_DEVICE_ID
);
2765 if (vdev
->sub_vendor_id
!= PCI_ANY_ID
) {
2766 if (vdev
->sub_vendor_id
> 0xffff) {
2767 error_setg(errp
, "invalid PCI subsystem vendor ID provided");
2770 vfio_add_emulated_word(vdev
, PCI_SUBSYSTEM_VENDOR_ID
,
2771 vdev
->sub_vendor_id
, ~0);
2772 trace_vfio_pci_emulated_sub_vendor_id(vdev
->vbasedev
.name
,
2773 vdev
->sub_vendor_id
);
2776 if (vdev
->sub_device_id
!= PCI_ANY_ID
) {
2777 if (vdev
->sub_device_id
> 0xffff) {
2778 error_setg(errp
, "invalid PCI subsystem device ID provided");
2781 vfio_add_emulated_word(vdev
, PCI_SUBSYSTEM_ID
, vdev
->sub_device_id
, ~0);
2782 trace_vfio_pci_emulated_sub_device_id(vdev
->vbasedev
.name
,
2783 vdev
->sub_device_id
);
2786 /* QEMU can change multi-function devices to single function, or reverse */
2787 vdev
->emulated_config_bits
[PCI_HEADER_TYPE
] =
2788 PCI_HEADER_TYPE_MULTI_FUNCTION
;
2790 /* Restore or clear multifunction, this is always controlled by QEMU */
2791 if (vdev
->pdev
.cap_present
& QEMU_PCI_CAP_MULTIFUNCTION
) {
2792 vdev
->pdev
.config
[PCI_HEADER_TYPE
] |= PCI_HEADER_TYPE_MULTI_FUNCTION
;
2794 vdev
->pdev
.config
[PCI_HEADER_TYPE
] &= ~PCI_HEADER_TYPE_MULTI_FUNCTION
;
2798 * Clear host resource mapping info. If we choose not to register a
2799 * BAR, such as might be the case with the option ROM, we can get
2800 * confusing, unwritable, residual addresses from the host here.
2802 memset(&vdev
->pdev
.config
[PCI_BASE_ADDRESS_0
], 0, 24);
2803 memset(&vdev
->pdev
.config
[PCI_ROM_ADDRESS
], 0, 4);
2805 vfio_pci_size_rom(vdev
);
2807 vfio_msix_early_setup(vdev
, &err
);
2809 error_propagate(errp
, err
);
2813 vfio_bars_setup(vdev
);
2815 ret
= vfio_add_capabilities(vdev
, errp
);
2821 vfio_vga_quirk_setup(vdev
);
2824 for (i
= 0; i
< PCI_ROM_SLOT
; i
++) {
2825 vfio_bar_quirk_setup(vdev
, i
);
2828 if (!vdev
->igd_opregion
&&
2829 vdev
->features
& VFIO_FEATURE_ENABLE_IGD_OPREGION
) {
2830 struct vfio_region_info
*opregion
;
2832 if (vdev
->pdev
.qdev
.hotplugged
) {
2834 "cannot support IGD OpRegion feature on hotplugged "
2839 ret
= vfio_get_dev_region_info(&vdev
->vbasedev
,
2840 VFIO_REGION_TYPE_PCI_VENDOR_TYPE
| PCI_VENDOR_ID_INTEL
,
2841 VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION
, &opregion
);
2843 error_setg_errno(errp
, -ret
,
2844 "does not support requested IGD OpRegion feature");
2848 ret
= vfio_pci_igd_opregion_init(vdev
, opregion
, errp
);
2855 /* QEMU emulates all of MSI & MSIX */
2856 if (pdev
->cap_present
& QEMU_PCI_CAP_MSIX
) {
2857 memset(vdev
->emulated_config_bits
+ pdev
->msix_cap
, 0xff,
2861 if (pdev
->cap_present
& QEMU_PCI_CAP_MSI
) {
2862 memset(vdev
->emulated_config_bits
+ pdev
->msi_cap
, 0xff,
2863 vdev
->msi_cap_size
);
2866 if (vfio_pci_read_config(&vdev
->pdev
, PCI_INTERRUPT_PIN
, 1)) {
2867 vdev
->intx
.mmap_timer
= timer_new_ms(QEMU_CLOCK_VIRTUAL
,
2868 vfio_intx_mmap_enable
, vdev
);
2869 pci_device_set_intx_routing_notifier(&vdev
->pdev
, vfio_intx_update
);
2870 ret
= vfio_intx_enable(vdev
, errp
);
2876 vfio_register_err_notifier(vdev
);
2877 vfio_register_req_notifier(vdev
);
2878 vfio_setup_resetfn_quirk(vdev
);
2883 pci_device_set_intx_routing_notifier(&vdev
->pdev
, NULL
);
2884 vfio_teardown_msi(vdev
);
2885 vfio_bars_exit(vdev
);
2887 error_prepend(errp
, ERR_PREFIX
, vdev
->vbasedev
.name
);
2890 static void vfio_instance_finalize(Object
*obj
)
2892 PCIDevice
*pci_dev
= PCI_DEVICE(obj
);
2893 VFIOPCIDevice
*vdev
= DO_UPCAST(VFIOPCIDevice
, pdev
, pci_dev
);
2894 VFIOGroup
*group
= vdev
->vbasedev
.group
;
2896 vfio_bars_finalize(vdev
);
2897 g_free(vdev
->emulated_config_bits
);
2900 * XXX Leaking igd_opregion is not an oversight, we can't remove the
2901 * fw_cfg entry therefore leaking this allocation seems like the safest
2904 * g_free(vdev->igd_opregion);
2906 vfio_put_device(vdev
);
2907 vfio_put_group(group
);
2910 static void vfio_exitfn(PCIDevice
*pdev
)
2912 VFIOPCIDevice
*vdev
= DO_UPCAST(VFIOPCIDevice
, pdev
, pdev
);
2914 vfio_unregister_req_notifier(vdev
);
2915 vfio_unregister_err_notifier(vdev
);
2916 pci_device_set_intx_routing_notifier(&vdev
->pdev
, NULL
);
2917 vfio_disable_interrupts(vdev
);
2918 if (vdev
->intx
.mmap_timer
) {
2919 timer_free(vdev
->intx
.mmap_timer
);
2921 vfio_teardown_msi(vdev
);
2922 vfio_bars_exit(vdev
);
2925 static void vfio_pci_reset(DeviceState
*dev
)
2927 PCIDevice
*pdev
= DO_UPCAST(PCIDevice
, qdev
, dev
);
2928 VFIOPCIDevice
*vdev
= DO_UPCAST(VFIOPCIDevice
, pdev
, pdev
);
2930 trace_vfio_pci_reset(vdev
->vbasedev
.name
);
2932 vfio_pci_pre_reset(vdev
);
2934 if (vdev
->resetfn
&& !vdev
->resetfn(vdev
)) {
2938 if (vdev
->vbasedev
.reset_works
&&
2939 (vdev
->has_flr
|| !vdev
->has_pm_reset
) &&
2940 !ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_RESET
)) {
2941 trace_vfio_pci_reset_flr(vdev
->vbasedev
.name
);
2945 /* See if we can do our own bus reset */
2946 if (!vfio_pci_hot_reset_one(vdev
)) {
2950 /* If nothing else works and the device supports PM reset, use it */
2951 if (vdev
->vbasedev
.reset_works
&& vdev
->has_pm_reset
&&
2952 !ioctl(vdev
->vbasedev
.fd
, VFIO_DEVICE_RESET
)) {
2953 trace_vfio_pci_reset_pm(vdev
->vbasedev
.name
);
2958 vfio_pci_post_reset(vdev
);
2961 static void vfio_instance_init(Object
*obj
)
2963 PCIDevice
*pci_dev
= PCI_DEVICE(obj
);
2964 VFIOPCIDevice
*vdev
= DO_UPCAST(VFIOPCIDevice
, pdev
, PCI_DEVICE(obj
));
2966 device_add_bootindex_property(obj
, &vdev
->bootindex
,
2968 &pci_dev
->qdev
, NULL
);
2969 vdev
->host
.domain
= ~0U;
2970 vdev
->host
.bus
= ~0U;
2971 vdev
->host
.slot
= ~0U;
2972 vdev
->host
.function
= ~0U;
2974 vdev
->nv_gpudirect_clique
= 0xFF;
2977 static Property vfio_pci_dev_properties
[] = {
2978 DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIOPCIDevice
, host
),
2979 DEFINE_PROP_STRING("sysfsdev", VFIOPCIDevice
, vbasedev
.sysfsdev
),
2980 DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIOPCIDevice
,
2981 intx
.mmap_timeout
, 1100),
2982 DEFINE_PROP_BIT("x-vga", VFIOPCIDevice
, features
,
2983 VFIO_FEATURE_ENABLE_VGA_BIT
, false),
2984 DEFINE_PROP_BIT("x-req", VFIOPCIDevice
, features
,
2985 VFIO_FEATURE_ENABLE_REQ_BIT
, true),
2986 DEFINE_PROP_BIT("x-igd-opregion", VFIOPCIDevice
, features
,
2987 VFIO_FEATURE_ENABLE_IGD_OPREGION_BIT
, false),
2988 DEFINE_PROP_BOOL("x-no-mmap", VFIOPCIDevice
, vbasedev
.no_mmap
, false),
2989 DEFINE_PROP_BOOL("x-no-kvm-intx", VFIOPCIDevice
, no_kvm_intx
, false),
2990 DEFINE_PROP_BOOL("x-no-kvm-msi", VFIOPCIDevice
, no_kvm_msi
, false),
2991 DEFINE_PROP_BOOL("x-no-kvm-msix", VFIOPCIDevice
, no_kvm_msix
, false),
2992 DEFINE_PROP_UINT32("x-pci-vendor-id", VFIOPCIDevice
, vendor_id
, PCI_ANY_ID
),
2993 DEFINE_PROP_UINT32("x-pci-device-id", VFIOPCIDevice
, device_id
, PCI_ANY_ID
),
2994 DEFINE_PROP_UINT32("x-pci-sub-vendor-id", VFIOPCIDevice
,
2995 sub_vendor_id
, PCI_ANY_ID
),
2996 DEFINE_PROP_UINT32("x-pci-sub-device-id", VFIOPCIDevice
,
2997 sub_device_id
, PCI_ANY_ID
),
2998 DEFINE_PROP_UINT32("x-igd-gms", VFIOPCIDevice
, igd_gms
, 0),
2999 DEFINE_PROP_UNSIGNED_NODEFAULT("x-nv-gpudirect-clique", VFIOPCIDevice
,
3000 nv_gpudirect_clique
,
3001 qdev_prop_nv_gpudirect_clique
, uint8_t),
3003 * TODO - support passed fds... is this necessary?
3004 * DEFINE_PROP_STRING("vfiofd", VFIOPCIDevice, vfiofd_name),
3005 * DEFINE_PROP_STRING("vfiogroupfd, VFIOPCIDevice, vfiogroupfd_name),
3007 DEFINE_PROP_END_OF_LIST(),
3010 static const VMStateDescription vfio_pci_vmstate
= {
3015 static void vfio_pci_dev_class_init(ObjectClass
*klass
, void *data
)
3017 DeviceClass
*dc
= DEVICE_CLASS(klass
);
3018 PCIDeviceClass
*pdc
= PCI_DEVICE_CLASS(klass
);
3020 dc
->reset
= vfio_pci_reset
;
3021 dc
->props
= vfio_pci_dev_properties
;
3022 dc
->vmsd
= &vfio_pci_vmstate
;
3023 dc
->desc
= "VFIO-based PCI device assignment";
3024 set_bit(DEVICE_CATEGORY_MISC
, dc
->categories
);
3025 pdc
->realize
= vfio_realize
;
3026 pdc
->exit
= vfio_exitfn
;
3027 pdc
->config_read
= vfio_pci_read_config
;
3028 pdc
->config_write
= vfio_pci_write_config
;
3029 pdc
->is_express
= 1; /* We might be */
3032 static const TypeInfo vfio_pci_dev_info
= {
3034 .parent
= TYPE_PCI_DEVICE
,
3035 .instance_size
= sizeof(VFIOPCIDevice
),
3036 .class_init
= vfio_pci_dev_class_init
,
3037 .instance_init
= vfio_instance_init
,
3038 .instance_finalize
= vfio_instance_finalize
,
3039 .interfaces
= (InterfaceInfo
[]) {
3040 { INTERFACE_PCIE_DEVICE
},
3041 { INTERFACE_CONVENTIONAL_PCI_DEVICE
},
3046 static void register_vfio_pci_dev_type(void)
3048 type_register_static(&vfio_pci_dev_info
);
3051 type_init(register_vfio_pci_dev_type
)