2 * vfio based device assignment support
4 * Copyright Red Hat, Inc. 2012
7 * Alex Williamson <alex.williamson@redhat.com>
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
12 * Based on qemu-kvm device-assignment:
13 * Adapted for KVM by Qumranet.
14 * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com)
15 * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com)
16 * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com)
17 * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
18 * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
22 #include <linux/vfio.h>
23 #include <sys/ioctl.h>
26 #include <sys/types.h>
30 #include "exec/address-spaces.h"
31 #include "exec/memory.h"
32 #include "hw/pci/msi.h"
33 #include "hw/pci/msix.h"
34 #include "hw/pci/pci.h"
35 #include "qemu-common.h"
36 #include "qemu/error-report.h"
37 #include "qemu/event_notifier.h"
38 #include "qemu/queue.h"
39 #include "qemu/range.h"
40 #include "sysemu/kvm.h"
41 #include "sysemu/sysemu.h"
42 #include "hw/misc/vfio.h"
44 /* #define DEBUG_VFIO */
46 #define DPRINTF(fmt, ...) \
47 do { fprintf(stderr, "vfio: " fmt, ## __VA_ARGS__); } while (0)
49 #define DPRINTF(fmt, ...) \
53 /* Extra debugging, trap acceleration paths for more logging */
54 #define VFIO_ALLOW_MMAP 1
55 #define VFIO_ALLOW_KVM_INTX 1
56 #define VFIO_ALLOW_KVM_MSI 1
57 #define VFIO_ALLOW_KVM_MSIX 1
61 typedef struct VFIOQuirk
{
63 struct VFIODevice
*vdev
;
64 QLIST_ENTRY(VFIOQuirk
) next
;
66 uint32_t base_offset
:TARGET_PAGE_BITS
;
67 uint32_t address_offset
:TARGET_PAGE_BITS
;
68 uint32_t address_size
:3;
71 uint32_t address_match
;
72 uint32_t address_mask
;
74 uint32_t address_val
:TARGET_PAGE_BITS
;
75 uint32_t data_offset
:TARGET_PAGE_BITS
;
84 typedef struct VFIOBAR
{
85 off_t fd_offset
; /* offset of BAR within device fd */
86 int fd
; /* device fd, allows us to pass VFIOBAR as opaque data */
87 MemoryRegion mem
; /* slow, read/write access */
88 MemoryRegion mmap_mem
; /* direct mapped access */
91 uint32_t flags
; /* VFIO region flags (rd/wr/mmap) */
92 uint8_t nr
; /* cache the BAR number for debug */
95 QLIST_HEAD(, VFIOQuirk
) quirks
;
98 typedef struct VFIOVGARegion
{
102 QLIST_HEAD(, VFIOQuirk
) quirks
;
105 typedef struct VFIOVGA
{
108 VFIOVGARegion region
[QEMU_PCI_VGA_NUM_REGIONS
];
111 typedef struct VFIOINTx
{
112 bool pending
; /* interrupt pending */
113 bool kvm_accel
; /* set when QEMU bypass through KVM enabled */
114 uint8_t pin
; /* which pin to pull for qemu_set_irq */
115 EventNotifier interrupt
; /* eventfd triggered on interrupt */
116 EventNotifier unmask
; /* eventfd for unmask on QEMU bypass */
117 PCIINTxRoute route
; /* routing info for QEMU bypass */
118 uint32_t mmap_timeout
; /* delay to re-enable mmaps after interrupt */
119 QEMUTimer
*mmap_timer
; /* enable mmaps after periods w/o interrupts */
122 typedef struct VFIOMSIVector
{
123 EventNotifier interrupt
; /* eventfd triggered on interrupt */
124 EventNotifier kvm_interrupt
; /* eventfd triggered for KVM irqfd bypass */
125 struct VFIODevice
*vdev
; /* back pointer to device */
126 MSIMessage msg
; /* cache the MSI message so we know when it changes */
127 int virq
; /* KVM irqchip route for QEMU bypass */
138 typedef struct VFIOAddressSpace
{
140 QLIST_HEAD(, VFIOContainer
) containers
;
141 QLIST_ENTRY(VFIOAddressSpace
) list
;
144 static QLIST_HEAD(, VFIOAddressSpace
) vfio_address_spaces
=
145 QLIST_HEAD_INITIALIZER(vfio_address_spaces
);
149 typedef struct VFIOType1
{
150 MemoryListener listener
;
155 typedef struct VFIOContainer
{
156 VFIOAddressSpace
*space
;
157 int fd
; /* /dev/vfio/vfio, empowered by the attached groups */
159 /* enable abstraction to support various iommu backends */
163 void (*release
)(struct VFIOContainer
*);
165 QLIST_HEAD(, VFIOGuestIOMMU
) giommu_list
;
166 QLIST_HEAD(, VFIOGroup
) group_list
;
167 QLIST_ENTRY(VFIOContainer
) next
;
170 typedef struct VFIOGuestIOMMU
{
171 VFIOContainer
*container
;
174 QLIST_ENTRY(VFIOGuestIOMMU
) giommu_next
;
177 /* Cache of MSI-X setup plus extra mmap and memory region for split BAR map */
178 typedef struct VFIOMSIXInfo
{
182 uint32_t table_offset
;
184 MemoryRegion mmap_mem
;
188 typedef struct VFIODevice
{
192 unsigned int config_size
;
193 uint8_t *emulated_config_bits
; /* QEMU emulated bits, little-endian */
194 off_t config_offset
; /* Offset of config space region within device fd */
195 unsigned int rom_size
;
196 off_t rom_offset
; /* Offset of ROM region within device fd */
199 VFIOMSIVector
*msi_vectors
;
201 int nr_vectors
; /* Number of MSI/MSIX vectors currently in use */
202 int interrupt
; /* Current interrupt type */
203 VFIOBAR bars
[PCI_NUM_REGIONS
- 1]; /* No ROM */
204 VFIOVGA vga
; /* 0xa0000, 0x3b0, 0x3c0 */
205 PCIHostDeviceAddress host
;
206 QLIST_ENTRY(VFIODevice
) next
;
207 struct VFIOGroup
*group
;
208 EventNotifier err_notifier
;
210 #define VFIO_FEATURE_ENABLE_VGA_BIT 0
211 #define VFIO_FEATURE_ENABLE_VGA (1 << VFIO_FEATURE_ENABLE_VGA_BIT)
220 bool rom_read_failed
;
223 typedef struct VFIOGroup
{
226 VFIOContainer
*container
;
227 QLIST_HEAD(, VFIODevice
) device_list
;
228 QLIST_ENTRY(VFIOGroup
) next
;
229 QLIST_ENTRY(VFIOGroup
) container_next
;
232 typedef struct VFIORomBlacklistEntry
{
235 } VFIORomBlacklistEntry
;
238 * List of device ids/vendor ids for which to disable
239 * option rom loading. This avoids the guest hangs during rom
240 * execution as noticed with the BCM 57810 card for lack of a
241 * more better way to handle such issues.
242 * The user can still override by specifying a romfile or
244 * Please see https://bugs.launchpad.net/qemu/+bug/1284874
245 * for an analysis of the 57810 card hang. When adding
246 * a new vendor id/device id combination below, please also add
247 * your card/environment details and information that could
248 * help in debugging to the bug tracking this issue
250 static const VFIORomBlacklistEntry romblacklist
[] = {
251 /* Broadcom BCM 57810 */
255 #define MSIX_CAP_LENGTH 12
257 static QLIST_HEAD(, VFIOGroup
)
258 group_list
= QLIST_HEAD_INITIALIZER(group_list
);
262 * We have a single VFIO pseudo device per KVM VM. Once created it lives
263 * for the life of the VM. Closing the file descriptor only drops our
264 * reference to it and the device's reference to kvm. Therefore once
265 * initialized, this file descriptor is only released on QEMU exit and
266 * we'll re-use it should another vfio device be attached before then.
268 static int vfio_kvm_device_fd
= -1;
271 static void vfio_disable_interrupts(VFIODevice
*vdev
);
272 static uint32_t vfio_pci_read_config(PCIDevice
*pdev
, uint32_t addr
, int len
);
273 static void vfio_pci_write_config(PCIDevice
*pdev
, uint32_t addr
,
274 uint32_t val
, int len
);
275 static void vfio_mmap_set_enabled(VFIODevice
*vdev
, bool enabled
);
278 * Common VFIO interrupt disable
280 static void vfio_disable_irqindex(VFIODevice
*vdev
, int index
)
282 struct vfio_irq_set irq_set
= {
283 .argsz
= sizeof(irq_set
),
284 .flags
= VFIO_IRQ_SET_DATA_NONE
| VFIO_IRQ_SET_ACTION_TRIGGER
,
290 ioctl(vdev
->fd
, VFIO_DEVICE_SET_IRQS
, &irq_set
);
296 static void vfio_unmask_intx(VFIODevice
*vdev
)
298 struct vfio_irq_set irq_set
= {
299 .argsz
= sizeof(irq_set
),
300 .flags
= VFIO_IRQ_SET_DATA_NONE
| VFIO_IRQ_SET_ACTION_UNMASK
,
301 .index
= VFIO_PCI_INTX_IRQ_INDEX
,
306 ioctl(vdev
->fd
, VFIO_DEVICE_SET_IRQS
, &irq_set
);
309 #ifdef CONFIG_KVM /* Unused outside of CONFIG_KVM code */
310 static void vfio_mask_intx(VFIODevice
*vdev
)
312 struct vfio_irq_set irq_set
= {
313 .argsz
= sizeof(irq_set
),
314 .flags
= VFIO_IRQ_SET_DATA_NONE
| VFIO_IRQ_SET_ACTION_MASK
,
315 .index
= VFIO_PCI_INTX_IRQ_INDEX
,
320 ioctl(vdev
->fd
, VFIO_DEVICE_SET_IRQS
, &irq_set
);
325 * Disabling BAR mmaping can be slow, but toggling it around INTx can
326 * also be a huge overhead. We try to get the best of both worlds by
327 * waiting until an interrupt to disable mmaps (subsequent transitions
328 * to the same state are effectively no overhead). If the interrupt has
329 * been serviced and the time gap is long enough, we re-enable mmaps for
330 * performance. This works well for things like graphics cards, which
331 * may not use their interrupt at all and are penalized to an unusable
332 * level by read/write BAR traps. Other devices, like NICs, have more
333 * regular interrupts and see much better latency by staying in non-mmap
334 * mode. We therefore set the default mmap_timeout such that a ping
335 * is just enough to keep the mmap disabled. Users can experiment with
336 * other options with the x-intx-mmap-timeout-ms parameter (a value of
337 * zero disables the timer).
339 static void vfio_intx_mmap_enable(void *opaque
)
341 VFIODevice
*vdev
= opaque
;
343 if (vdev
->intx
.pending
) {
344 timer_mod(vdev
->intx
.mmap_timer
,
345 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
) + vdev
->intx
.mmap_timeout
);
349 vfio_mmap_set_enabled(vdev
, true);
352 static void vfio_intx_interrupt(void *opaque
)
354 VFIODevice
*vdev
= opaque
;
356 if (!event_notifier_test_and_clear(&vdev
->intx
.interrupt
)) {
360 DPRINTF("%s(%04x:%02x:%02x.%x) Pin %c\n", __func__
, vdev
->host
.domain
,
361 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
,
362 'A' + vdev
->intx
.pin
);
364 vdev
->intx
.pending
= true;
365 pci_irq_assert(&vdev
->pdev
);
366 vfio_mmap_set_enabled(vdev
, false);
367 if (vdev
->intx
.mmap_timeout
) {
368 timer_mod(vdev
->intx
.mmap_timer
,
369 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL
) + vdev
->intx
.mmap_timeout
);
373 static void vfio_eoi(VFIODevice
*vdev
)
375 if (!vdev
->intx
.pending
) {
379 DPRINTF("%s(%04x:%02x:%02x.%x) EOI\n", __func__
, vdev
->host
.domain
,
380 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
382 vdev
->intx
.pending
= false;
383 pci_irq_deassert(&vdev
->pdev
);
384 vfio_unmask_intx(vdev
);
387 static void vfio_enable_intx_kvm(VFIODevice
*vdev
)
390 struct kvm_irqfd irqfd
= {
391 .fd
= event_notifier_get_fd(&vdev
->intx
.interrupt
),
392 .gsi
= vdev
->intx
.route
.irq
,
393 .flags
= KVM_IRQFD_FLAG_RESAMPLE
,
395 struct vfio_irq_set
*irq_set
;
399 if (!VFIO_ALLOW_KVM_INTX
|| !kvm_irqfds_enabled() ||
400 vdev
->intx
.route
.mode
!= PCI_INTX_ENABLED
||
401 !kvm_check_extension(kvm_state
, KVM_CAP_IRQFD_RESAMPLE
)) {
405 /* Get to a known interrupt state */
406 qemu_set_fd_handler(irqfd
.fd
, NULL
, NULL
, vdev
);
407 vfio_mask_intx(vdev
);
408 vdev
->intx
.pending
= false;
409 pci_irq_deassert(&vdev
->pdev
);
411 /* Get an eventfd for resample/unmask */
412 if (event_notifier_init(&vdev
->intx
.unmask
, 0)) {
413 error_report("vfio: Error: event_notifier_init failed eoi");
417 /* KVM triggers it, VFIO listens for it */
418 irqfd
.resamplefd
= event_notifier_get_fd(&vdev
->intx
.unmask
);
420 if (kvm_vm_ioctl(kvm_state
, KVM_IRQFD
, &irqfd
)) {
421 error_report("vfio: Error: Failed to setup resample irqfd: %m");
425 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
427 irq_set
= g_malloc0(argsz
);
428 irq_set
->argsz
= argsz
;
429 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
| VFIO_IRQ_SET_ACTION_UNMASK
;
430 irq_set
->index
= VFIO_PCI_INTX_IRQ_INDEX
;
433 pfd
= (int32_t *)&irq_set
->data
;
435 *pfd
= irqfd
.resamplefd
;
437 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
440 error_report("vfio: Error: Failed to setup INTx unmask fd: %m");
445 vfio_unmask_intx(vdev
);
447 vdev
->intx
.kvm_accel
= true;
449 DPRINTF("%s(%04x:%02x:%02x.%x) KVM INTx accel enabled\n",
450 __func__
, vdev
->host
.domain
, vdev
->host
.bus
,
451 vdev
->host
.slot
, vdev
->host
.function
);
456 irqfd
.flags
= KVM_IRQFD_FLAG_DEASSIGN
;
457 kvm_vm_ioctl(kvm_state
, KVM_IRQFD
, &irqfd
);
459 event_notifier_cleanup(&vdev
->intx
.unmask
);
461 qemu_set_fd_handler(irqfd
.fd
, vfio_intx_interrupt
, NULL
, vdev
);
462 vfio_unmask_intx(vdev
);
466 static void vfio_disable_intx_kvm(VFIODevice
*vdev
)
469 struct kvm_irqfd irqfd
= {
470 .fd
= event_notifier_get_fd(&vdev
->intx
.interrupt
),
471 .gsi
= vdev
->intx
.route
.irq
,
472 .flags
= KVM_IRQFD_FLAG_DEASSIGN
,
475 if (!vdev
->intx
.kvm_accel
) {
480 * Get to a known state, hardware masked, QEMU ready to accept new
481 * interrupts, QEMU IRQ de-asserted.
483 vfio_mask_intx(vdev
);
484 vdev
->intx
.pending
= false;
485 pci_irq_deassert(&vdev
->pdev
);
487 /* Tell KVM to stop listening for an INTx irqfd */
488 if (kvm_vm_ioctl(kvm_state
, KVM_IRQFD
, &irqfd
)) {
489 error_report("vfio: Error: Failed to disable INTx irqfd: %m");
492 /* We only need to close the eventfd for VFIO to cleanup the kernel side */
493 event_notifier_cleanup(&vdev
->intx
.unmask
);
495 /* QEMU starts listening for interrupt events. */
496 qemu_set_fd_handler(irqfd
.fd
, vfio_intx_interrupt
, NULL
, vdev
);
498 vdev
->intx
.kvm_accel
= false;
500 /* If we've missed an event, let it re-fire through QEMU */
501 vfio_unmask_intx(vdev
);
503 DPRINTF("%s(%04x:%02x:%02x.%x) KVM INTx accel disabled\n",
504 __func__
, vdev
->host
.domain
, vdev
->host
.bus
,
505 vdev
->host
.slot
, vdev
->host
.function
);
509 static void vfio_update_irq(PCIDevice
*pdev
)
511 VFIODevice
*vdev
= DO_UPCAST(VFIODevice
, pdev
, pdev
);
514 if (vdev
->interrupt
!= VFIO_INT_INTx
) {
518 route
= pci_device_route_intx_to_irq(&vdev
->pdev
, vdev
->intx
.pin
);
520 if (!pci_intx_route_changed(&vdev
->intx
.route
, &route
)) {
521 return; /* Nothing changed */
524 DPRINTF("%s(%04x:%02x:%02x.%x) IRQ moved %d -> %d\n", __func__
,
525 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
526 vdev
->host
.function
, vdev
->intx
.route
.irq
, route
.irq
);
528 vfio_disable_intx_kvm(vdev
);
530 vdev
->intx
.route
= route
;
532 if (route
.mode
!= PCI_INTX_ENABLED
) {
536 vfio_enable_intx_kvm(vdev
);
538 /* Re-enable the interrupt in cased we missed an EOI */
542 static int vfio_enable_intx(VFIODevice
*vdev
)
544 uint8_t pin
= vfio_pci_read_config(&vdev
->pdev
, PCI_INTERRUPT_PIN
, 1);
546 struct vfio_irq_set
*irq_set
;
553 vfio_disable_interrupts(vdev
);
555 vdev
->intx
.pin
= pin
- 1; /* Pin A (1) -> irq[0] */
556 pci_config_set_interrupt_pin(vdev
->pdev
.config
, pin
);
560 * Only conditional to avoid generating error messages on platforms
561 * where we won't actually use the result anyway.
563 if (kvm_irqfds_enabled() &&
564 kvm_check_extension(kvm_state
, KVM_CAP_IRQFD_RESAMPLE
)) {
565 vdev
->intx
.route
= pci_device_route_intx_to_irq(&vdev
->pdev
,
570 ret
= event_notifier_init(&vdev
->intx
.interrupt
, 0);
572 error_report("vfio: Error: event_notifier_init failed");
576 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
578 irq_set
= g_malloc0(argsz
);
579 irq_set
->argsz
= argsz
;
580 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
| VFIO_IRQ_SET_ACTION_TRIGGER
;
581 irq_set
->index
= VFIO_PCI_INTX_IRQ_INDEX
;
584 pfd
= (int32_t *)&irq_set
->data
;
586 *pfd
= event_notifier_get_fd(&vdev
->intx
.interrupt
);
587 qemu_set_fd_handler(*pfd
, vfio_intx_interrupt
, NULL
, vdev
);
589 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
592 error_report("vfio: Error: Failed to setup INTx fd: %m");
593 qemu_set_fd_handler(*pfd
, NULL
, NULL
, vdev
);
594 event_notifier_cleanup(&vdev
->intx
.interrupt
);
598 vfio_enable_intx_kvm(vdev
);
600 vdev
->interrupt
= VFIO_INT_INTx
;
602 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__
, vdev
->host
.domain
,
603 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
608 static void vfio_disable_intx(VFIODevice
*vdev
)
612 timer_del(vdev
->intx
.mmap_timer
);
613 vfio_disable_intx_kvm(vdev
);
614 vfio_disable_irqindex(vdev
, VFIO_PCI_INTX_IRQ_INDEX
);
615 vdev
->intx
.pending
= false;
616 pci_irq_deassert(&vdev
->pdev
);
617 vfio_mmap_set_enabled(vdev
, true);
619 fd
= event_notifier_get_fd(&vdev
->intx
.interrupt
);
620 qemu_set_fd_handler(fd
, NULL
, NULL
, vdev
);
621 event_notifier_cleanup(&vdev
->intx
.interrupt
);
623 vdev
->interrupt
= VFIO_INT_NONE
;
625 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__
, vdev
->host
.domain
,
626 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
632 static void vfio_msi_interrupt(void *opaque
)
634 VFIOMSIVector
*vector
= opaque
;
635 VFIODevice
*vdev
= vector
->vdev
;
636 int nr
= vector
- vdev
->msi_vectors
;
638 if (!event_notifier_test_and_clear(&vector
->interrupt
)) {
645 if (vdev
->interrupt
== VFIO_INT_MSIX
) {
646 msg
= msix_get_message(&vdev
->pdev
, nr
);
647 } else if (vdev
->interrupt
== VFIO_INT_MSI
) {
648 msg
= msi_get_message(&vdev
->pdev
, nr
);
653 DPRINTF("%s(%04x:%02x:%02x.%x) vector %d 0x%"PRIx64
"/0x%x\n", __func__
,
654 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
655 vdev
->host
.function
, nr
, msg
.address
, msg
.data
);
658 if (vdev
->interrupt
== VFIO_INT_MSIX
) {
659 msix_notify(&vdev
->pdev
, nr
);
660 } else if (vdev
->interrupt
== VFIO_INT_MSI
) {
661 msi_notify(&vdev
->pdev
, nr
);
663 error_report("vfio: MSI interrupt receieved, but not enabled?");
667 static int vfio_enable_vectors(VFIODevice
*vdev
, bool msix
)
669 struct vfio_irq_set
*irq_set
;
670 int ret
= 0, i
, argsz
;
673 argsz
= sizeof(*irq_set
) + (vdev
->nr_vectors
* sizeof(*fds
));
675 irq_set
= g_malloc0(argsz
);
676 irq_set
->argsz
= argsz
;
677 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
| VFIO_IRQ_SET_ACTION_TRIGGER
;
678 irq_set
->index
= msix
? VFIO_PCI_MSIX_IRQ_INDEX
: VFIO_PCI_MSI_IRQ_INDEX
;
680 irq_set
->count
= vdev
->nr_vectors
;
681 fds
= (int32_t *)&irq_set
->data
;
683 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
684 if (!vdev
->msi_vectors
[i
].use
) {
686 } else if (vdev
->msi_vectors
[i
].virq
>= 0) {
687 fds
[i
] = event_notifier_get_fd(&vdev
->msi_vectors
[i
].kvm_interrupt
);
689 fds
[i
] = event_notifier_get_fd(&vdev
->msi_vectors
[i
].interrupt
);
693 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
700 static void vfio_add_kvm_msi_virq(VFIOMSIVector
*vector
, MSIMessage
*msg
,
705 if ((msix
&& !VFIO_ALLOW_KVM_MSIX
) ||
706 (!msix
&& !VFIO_ALLOW_KVM_MSI
) || !msg
) {
710 if (event_notifier_init(&vector
->kvm_interrupt
, 0)) {
714 virq
= kvm_irqchip_add_msi_route(kvm_state
, *msg
);
716 event_notifier_cleanup(&vector
->kvm_interrupt
);
720 if (kvm_irqchip_add_irqfd_notifier(kvm_state
, &vector
->kvm_interrupt
,
722 kvm_irqchip_release_virq(kvm_state
, virq
);
723 event_notifier_cleanup(&vector
->kvm_interrupt
);
731 static void vfio_remove_kvm_msi_virq(VFIOMSIVector
*vector
)
733 kvm_irqchip_remove_irqfd_notifier(kvm_state
, &vector
->kvm_interrupt
,
735 kvm_irqchip_release_virq(kvm_state
, vector
->virq
);
737 event_notifier_cleanup(&vector
->kvm_interrupt
);
740 static void vfio_update_kvm_msi_virq(VFIOMSIVector
*vector
, MSIMessage msg
)
742 kvm_irqchip_update_msi_route(kvm_state
, vector
->virq
, msg
);
746 static int vfio_msix_vector_do_use(PCIDevice
*pdev
, unsigned int nr
,
747 MSIMessage
*msg
, IOHandler
*handler
)
749 VFIODevice
*vdev
= DO_UPCAST(VFIODevice
, pdev
, pdev
);
750 VFIOMSIVector
*vector
;
753 DPRINTF("%s(%04x:%02x:%02x.%x) vector %d used\n", __func__
,
754 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
755 vdev
->host
.function
, nr
);
757 vector
= &vdev
->msi_vectors
[nr
];
762 if (event_notifier_init(&vector
->interrupt
, 0)) {
763 error_report("vfio: Error: event_notifier_init failed");
766 msix_vector_use(pdev
, nr
);
769 qemu_set_fd_handler(event_notifier_get_fd(&vector
->interrupt
),
770 handler
, NULL
, vector
);
773 * Attempt to enable route through KVM irqchip,
774 * default to userspace handling if unavailable.
776 if (vector
->virq
>= 0) {
778 vfio_remove_kvm_msi_virq(vector
);
780 vfio_update_kvm_msi_virq(vector
, *msg
);
783 vfio_add_kvm_msi_virq(vector
, msg
, true);
787 * We don't want to have the host allocate all possible MSI vectors
788 * for a device if they're not in use, so we shutdown and incrementally
789 * increase them as needed.
791 if (vdev
->nr_vectors
< nr
+ 1) {
792 vfio_disable_irqindex(vdev
, VFIO_PCI_MSIX_IRQ_INDEX
);
793 vdev
->nr_vectors
= nr
+ 1;
794 ret
= vfio_enable_vectors(vdev
, true);
796 error_report("vfio: failed to enable vectors, %d", ret
);
800 struct vfio_irq_set
*irq_set
;
803 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
805 irq_set
= g_malloc0(argsz
);
806 irq_set
->argsz
= argsz
;
807 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
|
808 VFIO_IRQ_SET_ACTION_TRIGGER
;
809 irq_set
->index
= VFIO_PCI_MSIX_IRQ_INDEX
;
812 pfd
= (int32_t *)&irq_set
->data
;
814 if (vector
->virq
>= 0) {
815 *pfd
= event_notifier_get_fd(&vector
->kvm_interrupt
);
817 *pfd
= event_notifier_get_fd(&vector
->interrupt
);
820 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
823 error_report("vfio: failed to modify vector, %d", ret
);
830 static int vfio_msix_vector_use(PCIDevice
*pdev
,
831 unsigned int nr
, MSIMessage msg
)
833 return vfio_msix_vector_do_use(pdev
, nr
, &msg
, vfio_msi_interrupt
);
836 static void vfio_msix_vector_release(PCIDevice
*pdev
, unsigned int nr
)
838 VFIODevice
*vdev
= DO_UPCAST(VFIODevice
, pdev
, pdev
);
839 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[nr
];
841 DPRINTF("%s(%04x:%02x:%02x.%x) vector %d released\n", __func__
,
842 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
843 vdev
->host
.function
, nr
);
846 * There are still old guests that mask and unmask vectors on every
847 * interrupt. If we're using QEMU bypass with a KVM irqfd, leave all of
848 * the KVM setup in place, simply switch VFIO to use the non-bypass
849 * eventfd. We'll then fire the interrupt through QEMU and the MSI-X
850 * core will mask the interrupt and set pending bits, allowing it to
851 * be re-asserted on unmask. Nothing to do if already using QEMU mode.
853 if (vector
->virq
>= 0) {
855 struct vfio_irq_set
*irq_set
;
858 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
860 irq_set
= g_malloc0(argsz
);
861 irq_set
->argsz
= argsz
;
862 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
|
863 VFIO_IRQ_SET_ACTION_TRIGGER
;
864 irq_set
->index
= VFIO_PCI_MSIX_IRQ_INDEX
;
867 pfd
= (int32_t *)&irq_set
->data
;
869 *pfd
= event_notifier_get_fd(&vector
->interrupt
);
871 ioctl(vdev
->fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
877 static void vfio_enable_msix(VFIODevice
*vdev
)
879 vfio_disable_interrupts(vdev
);
881 vdev
->msi_vectors
= g_malloc0(vdev
->msix
->entries
* sizeof(VFIOMSIVector
));
883 vdev
->interrupt
= VFIO_INT_MSIX
;
886 * Some communication channels between VF & PF or PF & fw rely on the
887 * physical state of the device and expect that enabling MSI-X from the
888 * guest enables the same on the host. When our guest is Linux, the
889 * guest driver call to pci_enable_msix() sets the enabling bit in the
890 * MSI-X capability, but leaves the vector table masked. We therefore
891 * can't rely on a vector_use callback (from request_irq() in the guest)
892 * to switch the physical device into MSI-X mode because that may come a
893 * long time after pci_enable_msix(). This code enables vector 0 with
894 * triggering to userspace, then immediately release the vector, leaving
895 * the physical device with no vectors enabled, but MSI-X enabled, just
896 * like the guest view.
898 vfio_msix_vector_do_use(&vdev
->pdev
, 0, NULL
, NULL
);
899 vfio_msix_vector_release(&vdev
->pdev
, 0);
901 if (msix_set_vector_notifiers(&vdev
->pdev
, vfio_msix_vector_use
,
902 vfio_msix_vector_release
, NULL
)) {
903 error_report("vfio: msix_set_vector_notifiers failed");
906 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__
, vdev
->host
.domain
,
907 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
910 static void vfio_enable_msi(VFIODevice
*vdev
)
914 vfio_disable_interrupts(vdev
);
916 vdev
->nr_vectors
= msi_nr_vectors_allocated(&vdev
->pdev
);
918 vdev
->msi_vectors
= g_malloc0(vdev
->nr_vectors
* sizeof(VFIOMSIVector
));
920 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
921 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[i
];
927 if (event_notifier_init(&vector
->interrupt
, 0)) {
928 error_report("vfio: Error: event_notifier_init failed");
931 qemu_set_fd_handler(event_notifier_get_fd(&vector
->interrupt
),
932 vfio_msi_interrupt
, NULL
, vector
);
934 vector
->msg
= msi_get_message(&vdev
->pdev
, i
);
937 * Attempt to enable route through KVM irqchip,
938 * default to userspace handling if unavailable.
940 vfio_add_kvm_msi_virq(vector
, &vector
->msg
, false);
943 /* Set interrupt type prior to possible interrupts */
944 vdev
->interrupt
= VFIO_INT_MSI
;
946 ret
= vfio_enable_vectors(vdev
, false);
949 error_report("vfio: Error: Failed to setup MSI fds: %m");
950 } else if (ret
!= vdev
->nr_vectors
) {
951 error_report("vfio: Error: Failed to enable %d "
952 "MSI vectors, retry with %d", vdev
->nr_vectors
, ret
);
955 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
956 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[i
];
957 if (vector
->virq
>= 0) {
958 vfio_remove_kvm_msi_virq(vector
);
960 qemu_set_fd_handler(event_notifier_get_fd(&vector
->interrupt
),
962 event_notifier_cleanup(&vector
->interrupt
);
965 g_free(vdev
->msi_vectors
);
967 if (ret
> 0 && ret
!= vdev
->nr_vectors
) {
968 vdev
->nr_vectors
= ret
;
971 vdev
->nr_vectors
= 0;
974 * Failing to setup MSI doesn't really fall within any specification.
975 * Let's try leaving interrupts disabled and hope the guest figures
976 * out to fall back to INTx for this device.
978 error_report("vfio: Error: Failed to enable MSI");
979 vdev
->interrupt
= VFIO_INT_NONE
;
984 DPRINTF("%s(%04x:%02x:%02x.%x) Enabled %d MSI vectors\n", __func__
,
985 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
986 vdev
->host
.function
, vdev
->nr_vectors
);
989 static void vfio_disable_msi_common(VFIODevice
*vdev
)
993 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
994 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[i
];
995 if (vdev
->msi_vectors
[i
].use
) {
996 if (vector
->virq
>= 0) {
997 vfio_remove_kvm_msi_virq(vector
);
999 qemu_set_fd_handler(event_notifier_get_fd(&vector
->interrupt
),
1001 event_notifier_cleanup(&vector
->interrupt
);
1005 g_free(vdev
->msi_vectors
);
1006 vdev
->msi_vectors
= NULL
;
1007 vdev
->nr_vectors
= 0;
1008 vdev
->interrupt
= VFIO_INT_NONE
;
1010 vfio_enable_intx(vdev
);
1013 static void vfio_disable_msix(VFIODevice
*vdev
)
1017 msix_unset_vector_notifiers(&vdev
->pdev
);
1020 * MSI-X will only release vectors if MSI-X is still enabled on the
1021 * device, check through the rest and release it ourselves if necessary.
1023 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
1024 if (vdev
->msi_vectors
[i
].use
) {
1025 vfio_msix_vector_release(&vdev
->pdev
, i
);
1026 msix_vector_unuse(&vdev
->pdev
, i
);
1030 if (vdev
->nr_vectors
) {
1031 vfio_disable_irqindex(vdev
, VFIO_PCI_MSIX_IRQ_INDEX
);
1034 vfio_disable_msi_common(vdev
);
1036 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__
, vdev
->host
.domain
,
1037 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
1040 static void vfio_disable_msi(VFIODevice
*vdev
)
1042 vfio_disable_irqindex(vdev
, VFIO_PCI_MSI_IRQ_INDEX
);
1043 vfio_disable_msi_common(vdev
);
1045 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__
, vdev
->host
.domain
,
1046 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
1049 static void vfio_update_msi(VFIODevice
*vdev
)
1053 for (i
= 0; i
< vdev
->nr_vectors
; i
++) {
1054 VFIOMSIVector
*vector
= &vdev
->msi_vectors
[i
];
1057 if (!vector
->use
|| vector
->virq
< 0) {
1061 msg
= msi_get_message(&vdev
->pdev
, i
);
1062 vfio_update_kvm_msi_virq(vector
, msg
);
1067 * IO Port/MMIO - Beware of the endians, VFIO is always little endian
1069 static void vfio_bar_write(void *opaque
, hwaddr addr
,
1070 uint64_t data
, unsigned size
)
1072 VFIOBAR
*bar
= opaque
;
1085 buf
.word
= cpu_to_le16(data
);
1088 buf
.dword
= cpu_to_le32(data
);
1091 hw_error("vfio: unsupported write size, %d bytes", size
);
1095 if (pwrite(bar
->fd
, &buf
, size
, bar
->fd_offset
+ addr
) != size
) {
1096 error_report("%s(,0x%"HWADDR_PRIx
", 0x%"PRIx64
", %d) failed: %m",
1097 __func__
, addr
, data
, size
);
1102 VFIODevice
*vdev
= container_of(bar
, VFIODevice
, bars
[bar
->nr
]);
1104 DPRINTF("%s(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx
", 0x%"PRIx64
1105 ", %d)\n", __func__
, vdev
->host
.domain
, vdev
->host
.bus
,
1106 vdev
->host
.slot
, vdev
->host
.function
, bar
->nr
, addr
,
1112 * A read or write to a BAR always signals an INTx EOI. This will
1113 * do nothing if not pending (including not in INTx mode). We assume
1114 * that a BAR access is in response to an interrupt and that BAR
1115 * accesses will service the interrupt. Unfortunately, we don't know
1116 * which access will service the interrupt, so we're potentially
1117 * getting quite a few host interrupts per guest interrupt.
1119 vfio_eoi(container_of(bar
, VFIODevice
, bars
[bar
->nr
]));
1122 static uint64_t vfio_bar_read(void *opaque
,
1123 hwaddr addr
, unsigned size
)
1125 VFIOBAR
*bar
= opaque
;
1134 if (pread(bar
->fd
, &buf
, size
, bar
->fd_offset
+ addr
) != size
) {
1135 error_report("%s(,0x%"HWADDR_PRIx
", %d) failed: %m",
1136 __func__
, addr
, size
);
1137 return (uint64_t)-1;
1145 data
= le16_to_cpu(buf
.word
);
1148 data
= le32_to_cpu(buf
.dword
);
1151 hw_error("vfio: unsupported read size, %d bytes", size
);
1157 VFIODevice
*vdev
= container_of(bar
, VFIODevice
, bars
[bar
->nr
]);
1159 DPRINTF("%s(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx
1160 ", %d) = 0x%"PRIx64
"\n", __func__
, vdev
->host
.domain
,
1161 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
,
1162 bar
->nr
, addr
, size
, data
);
1166 /* Same as write above */
1167 vfio_eoi(container_of(bar
, VFIODevice
, bars
[bar
->nr
]));
1172 static const MemoryRegionOps vfio_bar_ops
= {
1173 .read
= vfio_bar_read
,
1174 .write
= vfio_bar_write
,
1175 .endianness
= DEVICE_LITTLE_ENDIAN
,
1178 static void vfio_pci_load_rom(VFIODevice
*vdev
)
1180 struct vfio_region_info reg_info
= {
1181 .argsz
= sizeof(reg_info
),
1182 .index
= VFIO_PCI_ROM_REGION_INDEX
1188 if (ioctl(vdev
->fd
, VFIO_DEVICE_GET_REGION_INFO
, ®_info
)) {
1189 error_report("vfio: Error getting ROM info: %m");
1193 DPRINTF("Device %04x:%02x:%02x.%x ROM:\n", vdev
->host
.domain
,
1194 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
1195 DPRINTF(" size: 0x%lx, offset: 0x%lx, flags: 0x%lx\n",
1196 (unsigned long)reg_info
.size
, (unsigned long)reg_info
.offset
,
1197 (unsigned long)reg_info
.flags
);
1199 vdev
->rom_size
= size
= reg_info
.size
;
1200 vdev
->rom_offset
= reg_info
.offset
;
1202 if (!vdev
->rom_size
) {
1203 vdev
->rom_read_failed
= true;
1204 error_report("vfio-pci: Cannot read device rom at "
1205 "%04x:%02x:%02x.%x",
1206 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1207 vdev
->host
.function
);
1208 error_printf("Device option ROM contents are probably invalid "
1209 "(check dmesg).\nSkip option ROM probe with rombar=0, "
1210 "or load from file with romfile=\n");
1214 vdev
->rom
= g_malloc(size
);
1215 memset(vdev
->rom
, 0xff, size
);
1218 bytes
= pread(vdev
->fd
, vdev
->rom
+ off
, size
, vdev
->rom_offset
+ off
);
1221 } else if (bytes
> 0) {
1225 if (errno
== EINTR
|| errno
== EAGAIN
) {
1228 error_report("vfio: Error reading device ROM: %m");
1234 static uint64_t vfio_rom_read(void *opaque
, hwaddr addr
, unsigned size
)
1236 VFIODevice
*vdev
= opaque
;
1237 uint64_t val
= ((uint64_t)1 << (size
* 8)) - 1;
1239 /* Load the ROM lazily when the guest tries to read it */
1240 if (unlikely(!vdev
->rom
&& !vdev
->rom_read_failed
)) {
1241 vfio_pci_load_rom(vdev
);
1244 memcpy(&val
, vdev
->rom
+ addr
,
1245 (addr
< vdev
->rom_size
) ? MIN(size
, vdev
->rom_size
- addr
) : 0);
1247 DPRINTF("%s(%04x:%02x:%02x.%x, 0x%"HWADDR_PRIx
", 0x%x) = 0x%"PRIx64
"\n",
1248 __func__
, vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1249 vdev
->host
.function
, addr
, size
, val
);
1254 static void vfio_rom_write(void *opaque
, hwaddr addr
,
1255 uint64_t data
, unsigned size
)
1259 static const MemoryRegionOps vfio_rom_ops
= {
1260 .read
= vfio_rom_read
,
1261 .write
= vfio_rom_write
,
1262 .endianness
= DEVICE_LITTLE_ENDIAN
,
1265 static bool vfio_blacklist_opt_rom(VFIODevice
*vdev
)
1267 PCIDevice
*pdev
= &vdev
->pdev
;
1268 uint16_t vendor_id
, device_id
;
1271 vendor_id
= pci_get_word(pdev
->config
+ PCI_VENDOR_ID
);
1272 device_id
= pci_get_word(pdev
->config
+ PCI_DEVICE_ID
);
1274 while (count
< ARRAY_SIZE(romblacklist
)) {
1275 if (romblacklist
[count
].vendor_id
== vendor_id
&&
1276 romblacklist
[count
].device_id
== device_id
) {
1285 static void vfio_pci_size_rom(VFIODevice
*vdev
)
1287 uint32_t orig
, size
= cpu_to_le32((uint32_t)PCI_ROM_ADDRESS_MASK
);
1288 off_t offset
= vdev
->config_offset
+ PCI_ROM_ADDRESS
;
1289 DeviceState
*dev
= DEVICE(vdev
);
1292 if (vdev
->pdev
.romfile
|| !vdev
->pdev
.rom_bar
) {
1293 /* Since pci handles romfile, just print a message and return */
1294 if (vfio_blacklist_opt_rom(vdev
) && vdev
->pdev
.romfile
) {
1295 error_printf("Warning : Device at %04x:%02x:%02x.%x "
1296 "is known to cause system instability issues during "
1297 "option rom execution. "
1298 "Proceeding anyway since user specified romfile\n",
1299 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1300 vdev
->host
.function
);
1306 * Use the same size ROM BAR as the physical device. The contents
1307 * will get filled in later when the guest tries to read it.
1309 if (pread(vdev
->fd
, &orig
, 4, offset
) != 4 ||
1310 pwrite(vdev
->fd
, &size
, 4, offset
) != 4 ||
1311 pread(vdev
->fd
, &size
, 4, offset
) != 4 ||
1312 pwrite(vdev
->fd
, &orig
, 4, offset
) != 4) {
1313 error_report("%s(%04x:%02x:%02x.%x) failed: %m",
1314 __func__
, vdev
->host
.domain
, vdev
->host
.bus
,
1315 vdev
->host
.slot
, vdev
->host
.function
);
1319 size
= ~(le32_to_cpu(size
) & PCI_ROM_ADDRESS_MASK
) + 1;
1325 if (vfio_blacklist_opt_rom(vdev
)) {
1326 if (dev
->opts
&& qemu_opt_get(dev
->opts
, "rombar")) {
1327 error_printf("Warning : Device at %04x:%02x:%02x.%x "
1328 "is known to cause system instability issues during "
1329 "option rom execution. "
1330 "Proceeding anyway since user specified non zero value for "
1332 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1333 vdev
->host
.function
);
1335 error_printf("Warning : Rom loading for device at "
1336 "%04x:%02x:%02x.%x has been disabled due to "
1337 "system instability issues. "
1338 "Specify rombar=1 or romfile to force\n",
1339 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1340 vdev
->host
.function
);
1345 DPRINTF("%04x:%02x:%02x.%x ROM size 0x%x\n", vdev
->host
.domain
,
1346 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
, size
);
1348 snprintf(name
, sizeof(name
), "vfio[%04x:%02x:%02x.%x].rom",
1349 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1350 vdev
->host
.function
);
1352 memory_region_init_io(&vdev
->pdev
.rom
, OBJECT(vdev
),
1353 &vfio_rom_ops
, vdev
, name
, size
);
1355 pci_register_bar(&vdev
->pdev
, PCI_ROM_SLOT
,
1356 PCI_BASE_ADDRESS_SPACE_MEMORY
, &vdev
->pdev
.rom
);
1358 vdev
->pdev
.has_rom
= true;
1359 vdev
->rom_read_failed
= false;
1362 static void vfio_vga_write(void *opaque
, hwaddr addr
,
1363 uint64_t data
, unsigned size
)
1365 VFIOVGARegion
*region
= opaque
;
1366 VFIOVGA
*vga
= container_of(region
, VFIOVGA
, region
[region
->nr
]);
1373 off_t offset
= vga
->fd_offset
+ region
->offset
+ addr
;
1380 buf
.word
= cpu_to_le16(data
);
1383 buf
.dword
= cpu_to_le32(data
);
1386 hw_error("vfio: unsupported write size, %d bytes", size
);
1390 if (pwrite(vga
->fd
, &buf
, size
, offset
) != size
) {
1391 error_report("%s(,0x%"HWADDR_PRIx
", 0x%"PRIx64
", %d) failed: %m",
1392 __func__
, region
->offset
+ addr
, data
, size
);
1395 DPRINTF("%s(0x%"HWADDR_PRIx
", 0x%"PRIx64
", %d)\n",
1396 __func__
, region
->offset
+ addr
, data
, size
);
1399 static uint64_t vfio_vga_read(void *opaque
, hwaddr addr
, unsigned size
)
1401 VFIOVGARegion
*region
= opaque
;
1402 VFIOVGA
*vga
= container_of(region
, VFIOVGA
, region
[region
->nr
]);
1410 off_t offset
= vga
->fd_offset
+ region
->offset
+ addr
;
1412 if (pread(vga
->fd
, &buf
, size
, offset
) != size
) {
1413 error_report("%s(,0x%"HWADDR_PRIx
", %d) failed: %m",
1414 __func__
, region
->offset
+ addr
, size
);
1415 return (uint64_t)-1;
1423 data
= le16_to_cpu(buf
.word
);
1426 data
= le32_to_cpu(buf
.dword
);
1429 hw_error("vfio: unsupported read size, %d bytes", size
);
1433 DPRINTF("%s(0x%"HWADDR_PRIx
", %d) = 0x%"PRIx64
"\n",
1434 __func__
, region
->offset
+ addr
, size
, data
);
1439 static const MemoryRegionOps vfio_vga_ops
= {
1440 .read
= vfio_vga_read
,
1441 .write
= vfio_vga_write
,
1442 .endianness
= DEVICE_LITTLE_ENDIAN
,
1446 * Device specific quirks
1449 /* Is range1 fully contained within range2? */
1450 static bool vfio_range_contained(uint64_t first1
, uint64_t len1
,
1451 uint64_t first2
, uint64_t len2
) {
1452 return (first1
>= first2
&& first1
+ len1
<= first2
+ len2
);
1455 static bool vfio_flags_enabled(uint8_t flags
, uint8_t mask
)
1457 return (mask
&& (flags
& mask
) == mask
);
1460 static uint64_t vfio_generic_window_quirk_read(void *opaque
,
1461 hwaddr addr
, unsigned size
)
1463 VFIOQuirk
*quirk
= opaque
;
1464 VFIODevice
*vdev
= quirk
->vdev
;
1467 if (vfio_flags_enabled(quirk
->data
.flags
, quirk
->data
.read_flags
) &&
1468 ranges_overlap(addr
, size
,
1469 quirk
->data
.data_offset
, quirk
->data
.data_size
)) {
1470 hwaddr offset
= addr
- quirk
->data
.data_offset
;
1472 if (!vfio_range_contained(addr
, size
, quirk
->data
.data_offset
,
1473 quirk
->data
.data_size
)) {
1474 hw_error("%s: window data read not fully contained: %s",
1475 __func__
, memory_region_name(&quirk
->mem
));
1478 data
= vfio_pci_read_config(&vdev
->pdev
,
1479 quirk
->data
.address_val
+ offset
, size
);
1481 DPRINTF("%s read(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx
", %d) = 0x%"
1482 PRIx64
"\n", memory_region_name(&quirk
->mem
), vdev
->host
.domain
,
1483 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
,
1484 quirk
->data
.bar
, addr
, size
, data
);
1486 data
= vfio_bar_read(&vdev
->bars
[quirk
->data
.bar
],
1487 addr
+ quirk
->data
.base_offset
, size
);
1493 static void vfio_generic_window_quirk_write(void *opaque
, hwaddr addr
,
1494 uint64_t data
, unsigned size
)
1496 VFIOQuirk
*quirk
= opaque
;
1497 VFIODevice
*vdev
= quirk
->vdev
;
1499 if (ranges_overlap(addr
, size
,
1500 quirk
->data
.address_offset
, quirk
->data
.address_size
)) {
1502 if (addr
!= quirk
->data
.address_offset
) {
1503 hw_error("%s: offset write into address window: %s",
1504 __func__
, memory_region_name(&quirk
->mem
));
1507 if ((data
& ~quirk
->data
.address_mask
) == quirk
->data
.address_match
) {
1508 quirk
->data
.flags
|= quirk
->data
.write_flags
|
1509 quirk
->data
.read_flags
;
1510 quirk
->data
.address_val
= data
& quirk
->data
.address_mask
;
1512 quirk
->data
.flags
&= ~(quirk
->data
.write_flags
|
1513 quirk
->data
.read_flags
);
1517 if (vfio_flags_enabled(quirk
->data
.flags
, quirk
->data
.write_flags
) &&
1518 ranges_overlap(addr
, size
,
1519 quirk
->data
.data_offset
, quirk
->data
.data_size
)) {
1520 hwaddr offset
= addr
- quirk
->data
.data_offset
;
1522 if (!vfio_range_contained(addr
, size
, quirk
->data
.data_offset
,
1523 quirk
->data
.data_size
)) {
1524 hw_error("%s: window data write not fully contained: %s",
1525 __func__
, memory_region_name(&quirk
->mem
));
1528 vfio_pci_write_config(&vdev
->pdev
,
1529 quirk
->data
.address_val
+ offset
, data
, size
);
1530 DPRINTF("%s write(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx
", 0x%"
1531 PRIx64
", %d)\n", memory_region_name(&quirk
->mem
),
1532 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1533 vdev
->host
.function
, quirk
->data
.bar
, addr
, data
, size
);
1537 vfio_bar_write(&vdev
->bars
[quirk
->data
.bar
],
1538 addr
+ quirk
->data
.base_offset
, data
, size
);
1541 static const MemoryRegionOps vfio_generic_window_quirk
= {
1542 .read
= vfio_generic_window_quirk_read
,
1543 .write
= vfio_generic_window_quirk_write
,
1544 .endianness
= DEVICE_LITTLE_ENDIAN
,
1547 static uint64_t vfio_generic_quirk_read(void *opaque
,
1548 hwaddr addr
, unsigned size
)
1550 VFIOQuirk
*quirk
= opaque
;
1551 VFIODevice
*vdev
= quirk
->vdev
;
1552 hwaddr base
= quirk
->data
.address_match
& TARGET_PAGE_MASK
;
1553 hwaddr offset
= quirk
->data
.address_match
& ~TARGET_PAGE_MASK
;
1556 if (vfio_flags_enabled(quirk
->data
.flags
, quirk
->data
.read_flags
) &&
1557 ranges_overlap(addr
, size
, offset
, quirk
->data
.address_mask
+ 1)) {
1558 if (!vfio_range_contained(addr
, size
, offset
,
1559 quirk
->data
.address_mask
+ 1)) {
1560 hw_error("%s: read not fully contained: %s",
1561 __func__
, memory_region_name(&quirk
->mem
));
1564 data
= vfio_pci_read_config(&vdev
->pdev
, addr
- offset
, size
);
1566 DPRINTF("%s read(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx
", %d) = 0x%"
1567 PRIx64
"\n", memory_region_name(&quirk
->mem
), vdev
->host
.domain
,
1568 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
,
1569 quirk
->data
.bar
, addr
+ base
, size
, data
);
1571 data
= vfio_bar_read(&vdev
->bars
[quirk
->data
.bar
], addr
+ base
, size
);
1577 static void vfio_generic_quirk_write(void *opaque
, hwaddr addr
,
1578 uint64_t data
, unsigned size
)
1580 VFIOQuirk
*quirk
= opaque
;
1581 VFIODevice
*vdev
= quirk
->vdev
;
1582 hwaddr base
= quirk
->data
.address_match
& TARGET_PAGE_MASK
;
1583 hwaddr offset
= quirk
->data
.address_match
& ~TARGET_PAGE_MASK
;
1585 if (vfio_flags_enabled(quirk
->data
.flags
, quirk
->data
.write_flags
) &&
1586 ranges_overlap(addr
, size
, offset
, quirk
->data
.address_mask
+ 1)) {
1587 if (!vfio_range_contained(addr
, size
, offset
,
1588 quirk
->data
.address_mask
+ 1)) {
1589 hw_error("%s: write not fully contained: %s",
1590 __func__
, memory_region_name(&quirk
->mem
));
1593 vfio_pci_write_config(&vdev
->pdev
, addr
- offset
, data
, size
);
1595 DPRINTF("%s write(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx
", 0x%"
1596 PRIx64
", %d)\n", memory_region_name(&quirk
->mem
),
1597 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1598 vdev
->host
.function
, quirk
->data
.bar
, addr
+ base
, data
, size
);
1600 vfio_bar_write(&vdev
->bars
[quirk
->data
.bar
], addr
+ base
, data
, size
);
1604 static const MemoryRegionOps vfio_generic_quirk
= {
1605 .read
= vfio_generic_quirk_read
,
1606 .write
= vfio_generic_quirk_write
,
1607 .endianness
= DEVICE_LITTLE_ENDIAN
,
1610 #define PCI_VENDOR_ID_ATI 0x1002
1613 * Radeon HD cards (HD5450 & HD7850) report the upper byte of the I/O port BAR
1614 * through VGA register 0x3c3. On newer cards, the I/O port BAR is always
1615 * BAR4 (older cards like the X550 used BAR1, but we don't care to support
1616 * those). Note that on bare metal, a read of 0x3c3 doesn't always return the
1617 * I/O port BAR address. Originally this was coded to return the virtual BAR
1618 * address only if the physical register read returns the actual BAR address,
1619 * but users have reported greater success if we return the virtual address
1622 static uint64_t vfio_ati_3c3_quirk_read(void *opaque
,
1623 hwaddr addr
, unsigned size
)
1625 VFIOQuirk
*quirk
= opaque
;
1626 VFIODevice
*vdev
= quirk
->vdev
;
1627 uint64_t data
= vfio_pci_read_config(&vdev
->pdev
,
1628 PCI_BASE_ADDRESS_0
+ (4 * 4) + 1,
1630 DPRINTF("%s(0x3c3, 1) = 0x%"PRIx64
"\n", __func__
, data
);
1635 static const MemoryRegionOps vfio_ati_3c3_quirk
= {
1636 .read
= vfio_ati_3c3_quirk_read
,
1637 .endianness
= DEVICE_LITTLE_ENDIAN
,
1640 static void vfio_vga_probe_ati_3c3_quirk(VFIODevice
*vdev
)
1642 PCIDevice
*pdev
= &vdev
->pdev
;
1645 if (pci_get_word(pdev
->config
+ PCI_VENDOR_ID
) != PCI_VENDOR_ID_ATI
) {
1650 * As long as the BAR is >= 256 bytes it will be aligned such that the
1651 * lower byte is always zero. Filter out anything else, if it exists.
1653 if (!vdev
->bars
[4].ioport
|| vdev
->bars
[4].size
< 256) {
1657 quirk
= g_malloc0(sizeof(*quirk
));
1660 memory_region_init_io(&quirk
->mem
, OBJECT(vdev
), &vfio_ati_3c3_quirk
, quirk
,
1661 "vfio-ati-3c3-quirk", 1);
1662 memory_region_add_subregion(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].mem
,
1663 3 /* offset 3 bytes from 0x3c0 */, &quirk
->mem
);
1665 QLIST_INSERT_HEAD(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].quirks
,
1668 DPRINTF("Enabled ATI/AMD quirk 0x3c3 BAR4for device %04x:%02x:%02x.%x\n",
1669 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1670 vdev
->host
.function
);
1674 * Newer ATI/AMD devices, including HD5450 and HD7850, have a window to PCI
1675 * config space through MMIO BAR2 at offset 0x4000. Nothing seems to access
1676 * the MMIO space directly, but a window to this space is provided through
1677 * I/O port BAR4. Offset 0x0 is the address register and offset 0x4 is the
1678 * data register. When the address is programmed to a range of 0x4000-0x4fff
1679 * PCI configuration space is available. Experimentation seems to indicate
1680 * that only read-only access is provided, but we drop writes when the window
1681 * is enabled to config space nonetheless.
1683 static void vfio_probe_ati_bar4_window_quirk(VFIODevice
*vdev
, int nr
)
1685 PCIDevice
*pdev
= &vdev
->pdev
;
1688 if (!vdev
->has_vga
|| nr
!= 4 ||
1689 pci_get_word(pdev
->config
+ PCI_VENDOR_ID
) != PCI_VENDOR_ID_ATI
) {
1693 quirk
= g_malloc0(sizeof(*quirk
));
1695 quirk
->data
.address_size
= 4;
1696 quirk
->data
.data_offset
= 4;
1697 quirk
->data
.data_size
= 4;
1698 quirk
->data
.address_match
= 0x4000;
1699 quirk
->data
.address_mask
= PCIE_CONFIG_SPACE_SIZE
- 1;
1700 quirk
->data
.bar
= nr
;
1701 quirk
->data
.read_flags
= quirk
->data
.write_flags
= 1;
1703 memory_region_init_io(&quirk
->mem
, OBJECT(vdev
),
1704 &vfio_generic_window_quirk
, quirk
,
1705 "vfio-ati-bar4-window-quirk", 8);
1706 memory_region_add_subregion_overlap(&vdev
->bars
[nr
].mem
,
1707 quirk
->data
.base_offset
, &quirk
->mem
, 1);
1709 QLIST_INSERT_HEAD(&vdev
->bars
[nr
].quirks
, quirk
, next
);
1711 DPRINTF("Enabled ATI/AMD BAR4 window quirk for device %04x:%02x:%02x.%x\n",
1712 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1713 vdev
->host
.function
);
1716 #define PCI_VENDOR_ID_REALTEK 0x10ec
1719 * RTL8168 devices have a backdoor that can access the MSI-X table. At BAR2
1720 * offset 0x70 there is a dword data register, offset 0x74 is a dword address
1721 * register. According to the Linux r8169 driver, the MSI-X table is addressed
1722 * when the "type" portion of the address register is set to 0x1. This appears
1723 * to be bits 16:30. Bit 31 is both a write indicator and some sort of
1724 * "address latched" indicator. Bits 12:15 are a mask field, which we can
1725 * ignore because the MSI-X table should always be accessed as a dword (full
1726 * mask). Bits 0:11 is offset within the type.
1730 * Read from MSI-X table offset 0
1731 * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x74, 0x1f000, 4) // store read addr
1732 * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x74, 4) = 0x8001f000 // latch
1733 * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x70, 4) = 0xfee00398 // read data
1735 * Write 0xfee00000 to MSI-X table offset 0
1736 * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x70, 0xfee00000, 4) // write data
1737 * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x74, 0x8001f000, 4) // do write
1738 * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x74, 4) = 0x1f000 // complete
1741 static uint64_t vfio_rtl8168_window_quirk_read(void *opaque
,
1742 hwaddr addr
, unsigned size
)
1744 VFIOQuirk
*quirk
= opaque
;
1745 VFIODevice
*vdev
= quirk
->vdev
;
1748 case 4: /* address */
1749 if (quirk
->data
.flags
) {
1750 DPRINTF("%s fake read(%04x:%02x:%02x.%d)\n",
1751 memory_region_name(&quirk
->mem
), vdev
->host
.domain
,
1752 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
1754 return quirk
->data
.address_match
^ 0x10000000U
;
1758 if (quirk
->data
.flags
) {
1761 DPRINTF("%s MSI-X table read(%04x:%02x:%02x.%d)\n",
1762 memory_region_name(&quirk
->mem
), vdev
->host
.domain
,
1763 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
1765 if (!(vdev
->pdev
.cap_present
& QEMU_PCI_CAP_MSIX
)) {
1769 io_mem_read(&vdev
->pdev
.msix_table_mmio
,
1770 (hwaddr
)(quirk
->data
.address_match
& 0xfff),
1776 DPRINTF("%s direct read(%04x:%02x:%02x.%d)\n",
1777 memory_region_name(&quirk
->mem
), vdev
->host
.domain
,
1778 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
1780 return vfio_bar_read(&vdev
->bars
[quirk
->data
.bar
], addr
+ 0x70, size
);
1783 static void vfio_rtl8168_window_quirk_write(void *opaque
, hwaddr addr
,
1784 uint64_t data
, unsigned size
)
1786 VFIOQuirk
*quirk
= opaque
;
1787 VFIODevice
*vdev
= quirk
->vdev
;
1790 case 4: /* address */
1791 if ((data
& 0x7fff0000) == 0x10000) {
1792 if (data
& 0x10000000U
&&
1793 vdev
->pdev
.cap_present
& QEMU_PCI_CAP_MSIX
) {
1795 DPRINTF("%s MSI-X table write(%04x:%02x:%02x.%d)\n",
1796 memory_region_name(&quirk
->mem
), vdev
->host
.domain
,
1797 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
1799 io_mem_write(&vdev
->pdev
.msix_table_mmio
,
1800 (hwaddr
)(quirk
->data
.address_match
& 0xfff),
1804 quirk
->data
.flags
= 1;
1805 quirk
->data
.address_match
= data
;
1809 quirk
->data
.flags
= 0;
1812 quirk
->data
.address_mask
= data
;
1816 DPRINTF("%s direct write(%04x:%02x:%02x.%d)\n",
1817 memory_region_name(&quirk
->mem
), vdev
->host
.domain
,
1818 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
1820 vfio_bar_write(&vdev
->bars
[quirk
->data
.bar
], addr
+ 0x70, data
, size
);
1823 static const MemoryRegionOps vfio_rtl8168_window_quirk
= {
1824 .read
= vfio_rtl8168_window_quirk_read
,
1825 .write
= vfio_rtl8168_window_quirk_write
,
1827 .min_access_size
= 4,
1828 .max_access_size
= 4,
1831 .endianness
= DEVICE_LITTLE_ENDIAN
,
1834 static void vfio_probe_rtl8168_bar2_window_quirk(VFIODevice
*vdev
, int nr
)
1836 PCIDevice
*pdev
= &vdev
->pdev
;
1839 if (pci_get_word(pdev
->config
+ PCI_VENDOR_ID
) != PCI_VENDOR_ID_REALTEK
||
1840 pci_get_word(pdev
->config
+ PCI_DEVICE_ID
) != 0x8168 || nr
!= 2) {
1844 quirk
= g_malloc0(sizeof(*quirk
));
1846 quirk
->data
.bar
= nr
;
1848 memory_region_init_io(&quirk
->mem
, OBJECT(vdev
), &vfio_rtl8168_window_quirk
,
1849 quirk
, "vfio-rtl8168-window-quirk", 8);
1850 memory_region_add_subregion_overlap(&vdev
->bars
[nr
].mem
,
1851 0x70, &quirk
->mem
, 1);
1853 QLIST_INSERT_HEAD(&vdev
->bars
[nr
].quirks
, quirk
, next
);
1855 DPRINTF("Enabled RTL8168 BAR2 window quirk for device %04x:%02x:%02x.%x\n",
1856 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1857 vdev
->host
.function
);
1860 * Trap the BAR2 MMIO window to config space as well.
1862 static void vfio_probe_ati_bar2_4000_quirk(VFIODevice
*vdev
, int nr
)
1864 PCIDevice
*pdev
= &vdev
->pdev
;
1867 /* Only enable on newer devices where BAR2 is 64bit */
1868 if (!vdev
->has_vga
|| nr
!= 2 || !vdev
->bars
[2].mem64
||
1869 pci_get_word(pdev
->config
+ PCI_VENDOR_ID
) != PCI_VENDOR_ID_ATI
) {
1873 quirk
= g_malloc0(sizeof(*quirk
));
1875 quirk
->data
.flags
= quirk
->data
.read_flags
= quirk
->data
.write_flags
= 1;
1876 quirk
->data
.address_match
= 0x4000;
1877 quirk
->data
.address_mask
= PCIE_CONFIG_SPACE_SIZE
- 1;
1878 quirk
->data
.bar
= nr
;
1880 memory_region_init_io(&quirk
->mem
, OBJECT(vdev
), &vfio_generic_quirk
, quirk
,
1881 "vfio-ati-bar2-4000-quirk",
1882 TARGET_PAGE_ALIGN(quirk
->data
.address_mask
+ 1));
1883 memory_region_add_subregion_overlap(&vdev
->bars
[nr
].mem
,
1884 quirk
->data
.address_match
& TARGET_PAGE_MASK
,
1887 QLIST_INSERT_HEAD(&vdev
->bars
[nr
].quirks
, quirk
, next
);
1889 DPRINTF("Enabled ATI/AMD BAR2 0x4000 quirk for device %04x:%02x:%02x.%x\n",
1890 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
1891 vdev
->host
.function
);
1895 * Older ATI/AMD cards like the X550 have a similar window to that above.
1896 * I/O port BAR1 provides a window to a mirror of PCI config space located
1897 * in BAR2 at offset 0xf00. We don't care to support such older cards, but
1898 * note it for future reference.
1901 #define PCI_VENDOR_ID_NVIDIA 0x10de
1904 * Nvidia has several different methods to get to config space, the
1905 * nouveu project has several of these documented here:
1906 * https://github.com/pathscale/envytools/tree/master/hwdocs
1908 * The first quirk is actually not documented in envytools and is found
1909 * on 10de:01d1 (NVIDIA Corporation G72 [GeForce 7300 LE]). This is an
1910 * NV46 chipset. The backdoor uses the legacy VGA I/O ports to access
1911 * the mirror of PCI config space found at BAR0 offset 0x1800. The access
1912 * sequence first writes 0x338 to I/O port 0x3d4. The target offset is
1913 * then written to 0x3d0. Finally 0x538 is written for a read and 0x738
1914 * is written for a write to 0x3d4. The BAR0 offset is then accessible
1915 * through 0x3d0. This quirk doesn't seem to be necessary on newer cards
1916 * that use the I/O port BAR5 window but it doesn't hurt to leave it.
1926 static uint64_t vfio_nvidia_3d0_quirk_read(void *opaque
,
1927 hwaddr addr
, unsigned size
)
1929 VFIOQuirk
*quirk
= opaque
;
1930 VFIODevice
*vdev
= quirk
->vdev
;
1931 PCIDevice
*pdev
= &vdev
->pdev
;
1932 uint64_t data
= vfio_vga_read(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
],
1933 addr
+ quirk
->data
.base_offset
, size
);
1935 if (quirk
->data
.flags
== NV_3D0_READ
&& addr
== quirk
->data
.data_offset
) {
1936 data
= vfio_pci_read_config(pdev
, quirk
->data
.address_val
, size
);
1937 DPRINTF("%s(0x3d0, %d) = 0x%"PRIx64
"\n", __func__
, size
, data
);
1940 quirk
->data
.flags
= NV_3D0_NONE
;
1945 static void vfio_nvidia_3d0_quirk_write(void *opaque
, hwaddr addr
,
1946 uint64_t data
, unsigned size
)
1948 VFIOQuirk
*quirk
= opaque
;
1949 VFIODevice
*vdev
= quirk
->vdev
;
1950 PCIDevice
*pdev
= &vdev
->pdev
;
1952 switch (quirk
->data
.flags
) {
1954 if (addr
== quirk
->data
.address_offset
&& data
== 0x338) {
1955 quirk
->data
.flags
= NV_3D0_SELECT
;
1959 quirk
->data
.flags
= NV_3D0_NONE
;
1960 if (addr
== quirk
->data
.data_offset
&&
1961 (data
& ~quirk
->data
.address_mask
) == quirk
->data
.address_match
) {
1962 quirk
->data
.flags
= NV_3D0_WINDOW
;
1963 quirk
->data
.address_val
= data
& quirk
->data
.address_mask
;
1967 quirk
->data
.flags
= NV_3D0_NONE
;
1968 if (addr
== quirk
->data
.address_offset
) {
1969 if (data
== 0x538) {
1970 quirk
->data
.flags
= NV_3D0_READ
;
1971 } else if (data
== 0x738) {
1972 quirk
->data
.flags
= NV_3D0_WRITE
;
1977 quirk
->data
.flags
= NV_3D0_NONE
;
1978 if (addr
== quirk
->data
.data_offset
) {
1979 vfio_pci_write_config(pdev
, quirk
->data
.address_val
, data
, size
);
1980 DPRINTF("%s(0x3d0, 0x%"PRIx64
", %d)\n", __func__
, data
, size
);
1986 vfio_vga_write(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
],
1987 addr
+ quirk
->data
.base_offset
, data
, size
);
1990 static const MemoryRegionOps vfio_nvidia_3d0_quirk
= {
1991 .read
= vfio_nvidia_3d0_quirk_read
,
1992 .write
= vfio_nvidia_3d0_quirk_write
,
1993 .endianness
= DEVICE_LITTLE_ENDIAN
,
1996 static void vfio_vga_probe_nvidia_3d0_quirk(VFIODevice
*vdev
)
1998 PCIDevice
*pdev
= &vdev
->pdev
;
2001 if (pci_get_word(pdev
->config
+ PCI_VENDOR_ID
) != PCI_VENDOR_ID_NVIDIA
||
2002 !vdev
->bars
[1].size
) {
2006 quirk
= g_malloc0(sizeof(*quirk
));
2008 quirk
->data
.base_offset
= 0x10;
2009 quirk
->data
.address_offset
= 4;
2010 quirk
->data
.address_size
= 2;
2011 quirk
->data
.address_match
= 0x1800;
2012 quirk
->data
.address_mask
= PCI_CONFIG_SPACE_SIZE
- 1;
2013 quirk
->data
.data_offset
= 0;
2014 quirk
->data
.data_size
= 4;
2016 memory_region_init_io(&quirk
->mem
, OBJECT(vdev
), &vfio_nvidia_3d0_quirk
,
2017 quirk
, "vfio-nvidia-3d0-quirk", 6);
2018 memory_region_add_subregion(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].mem
,
2019 quirk
->data
.base_offset
, &quirk
->mem
);
2021 QLIST_INSERT_HEAD(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].quirks
,
2024 DPRINTF("Enabled NVIDIA VGA 0x3d0 quirk for device %04x:%02x:%02x.%x\n",
2025 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
2026 vdev
->host
.function
);
2030 * The second quirk is documented in envytools. The I/O port BAR5 is just
2031 * a set of address/data ports to the MMIO BARs. The BAR we care about is
2032 * again BAR0. This backdoor is apparently a bit newer than the one above
2033 * so we need to not only trap 256 bytes @0x1800, but all of PCI config
2034 * space, including extended space is available at the 4k @0x88000.
2037 NV_BAR5_ADDRESS
= 0x1,
2038 NV_BAR5_ENABLE
= 0x2,
2039 NV_BAR5_MASTER
= 0x4,
2040 NV_BAR5_VALID
= 0x7,
2043 static void vfio_nvidia_bar5_window_quirk_write(void *opaque
, hwaddr addr
,
2044 uint64_t data
, unsigned size
)
2046 VFIOQuirk
*quirk
= opaque
;
2051 quirk
->data
.flags
|= NV_BAR5_MASTER
;
2053 quirk
->data
.flags
&= ~NV_BAR5_MASTER
;
2058 quirk
->data
.flags
|= NV_BAR5_ENABLE
;
2060 quirk
->data
.flags
&= ~NV_BAR5_ENABLE
;
2064 if (quirk
->data
.flags
& NV_BAR5_MASTER
) {
2065 if ((data
& ~0xfff) == 0x88000) {
2066 quirk
->data
.flags
|= NV_BAR5_ADDRESS
;
2067 quirk
->data
.address_val
= data
& 0xfff;
2068 } else if ((data
& ~0xff) == 0x1800) {
2069 quirk
->data
.flags
|= NV_BAR5_ADDRESS
;
2070 quirk
->data
.address_val
= data
& 0xff;
2072 quirk
->data
.flags
&= ~NV_BAR5_ADDRESS
;
2078 vfio_generic_window_quirk_write(opaque
, addr
, data
, size
);
2081 static const MemoryRegionOps vfio_nvidia_bar5_window_quirk
= {
2082 .read
= vfio_generic_window_quirk_read
,
2083 .write
= vfio_nvidia_bar5_window_quirk_write
,
2084 .valid
.min_access_size
= 4,
2085 .endianness
= DEVICE_LITTLE_ENDIAN
,
2088 static void vfio_probe_nvidia_bar5_window_quirk(VFIODevice
*vdev
, int nr
)
2090 PCIDevice
*pdev
= &vdev
->pdev
;
2093 if (!vdev
->has_vga
|| nr
!= 5 ||
2094 pci_get_word(pdev
->config
+ PCI_VENDOR_ID
) != PCI_VENDOR_ID_NVIDIA
) {
2098 quirk
= g_malloc0(sizeof(*quirk
));
2100 quirk
->data
.read_flags
= quirk
->data
.write_flags
= NV_BAR5_VALID
;
2101 quirk
->data
.address_offset
= 0x8;
2102 quirk
->data
.address_size
= 0; /* actually 4, but avoids generic code */
2103 quirk
->data
.data_offset
= 0xc;
2104 quirk
->data
.data_size
= 4;
2105 quirk
->data
.bar
= nr
;
2107 memory_region_init_io(&quirk
->mem
, OBJECT(vdev
),
2108 &vfio_nvidia_bar5_window_quirk
, quirk
,
2109 "vfio-nvidia-bar5-window-quirk", 16);
2110 memory_region_add_subregion_overlap(&vdev
->bars
[nr
].mem
, 0, &quirk
->mem
, 1);
2112 QLIST_INSERT_HEAD(&vdev
->bars
[nr
].quirks
, quirk
, next
);
2114 DPRINTF("Enabled NVIDIA BAR5 window quirk for device %04x:%02x:%02x.%x\n",
2115 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
2116 vdev
->host
.function
);
2119 static void vfio_nvidia_88000_quirk_write(void *opaque
, hwaddr addr
,
2120 uint64_t data
, unsigned size
)
2122 VFIOQuirk
*quirk
= opaque
;
2123 VFIODevice
*vdev
= quirk
->vdev
;
2124 PCIDevice
*pdev
= &vdev
->pdev
;
2125 hwaddr base
= quirk
->data
.address_match
& TARGET_PAGE_MASK
;
2127 vfio_generic_quirk_write(opaque
, addr
, data
, size
);
2130 * Nvidia seems to acknowledge MSI interrupts by writing 0xff to the
2131 * MSI capability ID register. Both the ID and next register are
2132 * read-only, so we allow writes covering either of those to real hw.
2133 * NB - only fixed for the 0x88000 MMIO window.
2135 if ((pdev
->cap_present
& QEMU_PCI_CAP_MSI
) &&
2136 vfio_range_contained(addr
, size
, pdev
->msi_cap
, PCI_MSI_FLAGS
)) {
2137 vfio_bar_write(&vdev
->bars
[quirk
->data
.bar
], addr
+ base
, data
, size
);
2141 static const MemoryRegionOps vfio_nvidia_88000_quirk
= {
2142 .read
= vfio_generic_quirk_read
,
2143 .write
= vfio_nvidia_88000_quirk_write
,
2144 .endianness
= DEVICE_LITTLE_ENDIAN
,
2148 * Finally, BAR0 itself. We want to redirect any accesses to either
2149 * 0x1800 or 0x88000 through the PCI config space access functions.
2151 * NB - quirk at a page granularity or else they don't seem to work when
2154 * Here's offset 0x88000...
2156 static void vfio_probe_nvidia_bar0_88000_quirk(VFIODevice
*vdev
, int nr
)
2158 PCIDevice
*pdev
= &vdev
->pdev
;
2161 if (!vdev
->has_vga
|| nr
!= 0 ||
2162 pci_get_word(pdev
->config
+ PCI_VENDOR_ID
) != PCI_VENDOR_ID_NVIDIA
) {
2166 quirk
= g_malloc0(sizeof(*quirk
));
2168 quirk
->data
.flags
= quirk
->data
.read_flags
= quirk
->data
.write_flags
= 1;
2169 quirk
->data
.address_match
= 0x88000;
2170 quirk
->data
.address_mask
= PCIE_CONFIG_SPACE_SIZE
- 1;
2171 quirk
->data
.bar
= nr
;
2173 memory_region_init_io(&quirk
->mem
, OBJECT(vdev
), &vfio_nvidia_88000_quirk
,
2174 quirk
, "vfio-nvidia-bar0-88000-quirk",
2175 TARGET_PAGE_ALIGN(quirk
->data
.address_mask
+ 1));
2176 memory_region_add_subregion_overlap(&vdev
->bars
[nr
].mem
,
2177 quirk
->data
.address_match
& TARGET_PAGE_MASK
,
2180 QLIST_INSERT_HEAD(&vdev
->bars
[nr
].quirks
, quirk
, next
);
2182 DPRINTF("Enabled NVIDIA BAR0 0x88000 quirk for device %04x:%02x:%02x.%x\n",
2183 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
2184 vdev
->host
.function
);
2188 * And here's the same for BAR0 offset 0x1800...
2190 static void vfio_probe_nvidia_bar0_1800_quirk(VFIODevice
*vdev
, int nr
)
2192 PCIDevice
*pdev
= &vdev
->pdev
;
2195 if (!vdev
->has_vga
|| nr
!= 0 ||
2196 pci_get_word(pdev
->config
+ PCI_VENDOR_ID
) != PCI_VENDOR_ID_NVIDIA
) {
2200 /* Log the chipset ID */
2201 DPRINTF("Nvidia NV%02x\n",
2202 (unsigned int)(vfio_bar_read(&vdev
->bars
[0], 0, 4) >> 20) & 0xff);
2204 quirk
= g_malloc0(sizeof(*quirk
));
2206 quirk
->data
.flags
= quirk
->data
.read_flags
= quirk
->data
.write_flags
= 1;
2207 quirk
->data
.address_match
= 0x1800;
2208 quirk
->data
.address_mask
= PCI_CONFIG_SPACE_SIZE
- 1;
2209 quirk
->data
.bar
= nr
;
2211 memory_region_init_io(&quirk
->mem
, OBJECT(vdev
), &vfio_generic_quirk
, quirk
,
2212 "vfio-nvidia-bar0-1800-quirk",
2213 TARGET_PAGE_ALIGN(quirk
->data
.address_mask
+ 1));
2214 memory_region_add_subregion_overlap(&vdev
->bars
[nr
].mem
,
2215 quirk
->data
.address_match
& TARGET_PAGE_MASK
,
2218 QLIST_INSERT_HEAD(&vdev
->bars
[nr
].quirks
, quirk
, next
);
2220 DPRINTF("Enabled NVIDIA BAR0 0x1800 quirk for device %04x:%02x:%02x.%x\n",
2221 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
2222 vdev
->host
.function
);
2226 * TODO - Some Nvidia devices provide config access to their companion HDA
2227 * device and even to their parent bridge via these config space mirrors.
2228 * Add quirks for those regions.
2232 * Common quirk probe entry points.
2234 static void vfio_vga_quirk_setup(VFIODevice
*vdev
)
2236 vfio_vga_probe_ati_3c3_quirk(vdev
);
2237 vfio_vga_probe_nvidia_3d0_quirk(vdev
);
2240 static void vfio_vga_quirk_teardown(VFIODevice
*vdev
)
2244 for (i
= 0; i
< ARRAY_SIZE(vdev
->vga
.region
); i
++) {
2245 while (!QLIST_EMPTY(&vdev
->vga
.region
[i
].quirks
)) {
2246 VFIOQuirk
*quirk
= QLIST_FIRST(&vdev
->vga
.region
[i
].quirks
);
2247 memory_region_del_subregion(&vdev
->vga
.region
[i
].mem
, &quirk
->mem
);
2248 memory_region_destroy(&quirk
->mem
);
2249 QLIST_REMOVE(quirk
, next
);
2255 static void vfio_bar_quirk_setup(VFIODevice
*vdev
, int nr
)
2257 vfio_probe_ati_bar4_window_quirk(vdev
, nr
);
2258 vfio_probe_ati_bar2_4000_quirk(vdev
, nr
);
2259 vfio_probe_nvidia_bar5_window_quirk(vdev
, nr
);
2260 vfio_probe_nvidia_bar0_88000_quirk(vdev
, nr
);
2261 vfio_probe_nvidia_bar0_1800_quirk(vdev
, nr
);
2262 vfio_probe_rtl8168_bar2_window_quirk(vdev
, nr
);
2265 static void vfio_bar_quirk_teardown(VFIODevice
*vdev
, int nr
)
2267 VFIOBAR
*bar
= &vdev
->bars
[nr
];
2269 while (!QLIST_EMPTY(&bar
->quirks
)) {
2270 VFIOQuirk
*quirk
= QLIST_FIRST(&bar
->quirks
);
2271 memory_region_del_subregion(&bar
->mem
, &quirk
->mem
);
2272 memory_region_destroy(&quirk
->mem
);
2273 QLIST_REMOVE(quirk
, next
);
2281 static uint32_t vfio_pci_read_config(PCIDevice
*pdev
, uint32_t addr
, int len
)
2283 VFIODevice
*vdev
= DO_UPCAST(VFIODevice
, pdev
, pdev
);
2284 uint32_t emu_bits
= 0, emu_val
= 0, phys_val
= 0, val
;
2286 memcpy(&emu_bits
, vdev
->emulated_config_bits
+ addr
, len
);
2287 emu_bits
= le32_to_cpu(emu_bits
);
2290 emu_val
= pci_default_read_config(pdev
, addr
, len
);
2293 if (~emu_bits
& (0xffffffffU
>> (32 - len
* 8))) {
2296 ret
= pread(vdev
->fd
, &phys_val
, len
, vdev
->config_offset
+ addr
);
2298 error_report("%s(%04x:%02x:%02x.%x, 0x%x, 0x%x) failed: %m",
2299 __func__
, vdev
->host
.domain
, vdev
->host
.bus
,
2300 vdev
->host
.slot
, vdev
->host
.function
, addr
, len
);
2303 phys_val
= le32_to_cpu(phys_val
);
2306 val
= (emu_val
& emu_bits
) | (phys_val
& ~emu_bits
);
2308 DPRINTF("%s(%04x:%02x:%02x.%x, @0x%x, len=0x%x) %x\n", __func__
,
2309 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
2310 vdev
->host
.function
, addr
, len
, val
);
2315 static void vfio_pci_write_config(PCIDevice
*pdev
, uint32_t addr
,
2316 uint32_t val
, int len
)
2318 VFIODevice
*vdev
= DO_UPCAST(VFIODevice
, pdev
, pdev
);
2319 uint32_t val_le
= cpu_to_le32(val
);
2321 DPRINTF("%s(%04x:%02x:%02x.%x, @0x%x, 0x%x, len=0x%x)\n", __func__
,
2322 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
2323 vdev
->host
.function
, addr
, val
, len
);
2325 /* Write everything to VFIO, let it filter out what we can't write */
2326 if (pwrite(vdev
->fd
, &val_le
, len
, vdev
->config_offset
+ addr
) != len
) {
2327 error_report("%s(%04x:%02x:%02x.%x, 0x%x, 0x%x, 0x%x) failed: %m",
2328 __func__
, vdev
->host
.domain
, vdev
->host
.bus
,
2329 vdev
->host
.slot
, vdev
->host
.function
, addr
, val
, len
);
2332 /* MSI/MSI-X Enabling/Disabling */
2333 if (pdev
->cap_present
& QEMU_PCI_CAP_MSI
&&
2334 ranges_overlap(addr
, len
, pdev
->msi_cap
, vdev
->msi_cap_size
)) {
2335 int is_enabled
, was_enabled
= msi_enabled(pdev
);
2337 pci_default_write_config(pdev
, addr
, val
, len
);
2339 is_enabled
= msi_enabled(pdev
);
2343 vfio_enable_msi(vdev
);
2347 vfio_disable_msi(vdev
);
2349 vfio_update_msi(vdev
);
2352 } else if (pdev
->cap_present
& QEMU_PCI_CAP_MSIX
&&
2353 ranges_overlap(addr
, len
, pdev
->msix_cap
, MSIX_CAP_LENGTH
)) {
2354 int is_enabled
, was_enabled
= msix_enabled(pdev
);
2356 pci_default_write_config(pdev
, addr
, val
, len
);
2358 is_enabled
= msix_enabled(pdev
);
2360 if (!was_enabled
&& is_enabled
) {
2361 vfio_enable_msix(vdev
);
2362 } else if (was_enabled
&& !is_enabled
) {
2363 vfio_disable_msix(vdev
);
2366 /* Write everything to QEMU to keep emulated bits correct */
2367 pci_default_write_config(pdev
, addr
, val
, len
);
2372 * DMA - Mapping and unmapping for the "type1" IOMMU interface used on x86
2374 static int vfio_dma_unmap(VFIOContainer
*container
,
2375 hwaddr iova
, ram_addr_t size
)
2377 struct vfio_iommu_type1_dma_unmap unmap
= {
2378 .argsz
= sizeof(unmap
),
2384 if (ioctl(container
->fd
, VFIO_IOMMU_UNMAP_DMA
, &unmap
)) {
2385 DPRINTF("VFIO_UNMAP_DMA: %d\n", -errno
);
2392 static int vfio_dma_map(VFIOContainer
*container
, hwaddr iova
,
2393 ram_addr_t size
, void *vaddr
, bool readonly
)
2395 struct vfio_iommu_type1_dma_map map
= {
2396 .argsz
= sizeof(map
),
2397 .flags
= VFIO_DMA_MAP_FLAG_READ
,
2398 .vaddr
= (__u64
)(uintptr_t)vaddr
,
2404 map
.flags
|= VFIO_DMA_MAP_FLAG_WRITE
;
2408 * Try the mapping, if it fails with EBUSY, unmap the region and try
2409 * again. This shouldn't be necessary, but we sometimes see it in
2410 * the the VGA ROM space.
2412 if (ioctl(container
->fd
, VFIO_IOMMU_MAP_DMA
, &map
) == 0 ||
2413 (errno
== EBUSY
&& vfio_dma_unmap(container
, iova
, size
) == 0 &&
2414 ioctl(container
->fd
, VFIO_IOMMU_MAP_DMA
, &map
) == 0)) {
2418 DPRINTF("VFIO_MAP_DMA: %d\n", -errno
);
2422 static bool vfio_listener_skipped_section(MemoryRegionSection
*section
)
2424 return (!memory_region_is_ram(section
->mr
) &&
2425 !memory_region_is_iommu(section
->mr
)) ||
2427 * Sizing an enabled 64-bit BAR can cause spurious mappings to
2428 * addresses in the upper part of the 64-bit address space. These
2429 * are never accessed by the CPU and beyond the address width of
2430 * some IOMMU hardware. TODO: VFIO should tell us the IOMMU width.
2432 section
->offset_within_address_space
& (1ULL << 63);
2435 static void vfio_iommu_map_notify(Notifier
*n
, void *data
)
2437 VFIOGuestIOMMU
*giommu
= container_of(n
, VFIOGuestIOMMU
, n
);
2438 VFIOContainer
*container
= giommu
->container
;
2439 IOMMUTLBEntry
*iotlb
= data
;
2442 hwaddr len
= iotlb
->addr_mask
+ 1;
2446 DPRINTF("iommu map @ %"HWADDR_PRIx
" - %"HWADDR_PRIx
"\n",
2447 iotlb
->iova
, iotlb
->iova
+ iotlb
->addr_mask
);
2450 * The IOMMU TLB entry we have just covers translation through
2451 * this IOMMU to its immediate target. We need to translate
2452 * it the rest of the way through to memory.
2454 mr
= address_space_translate(&address_space_memory
,
2455 iotlb
->translated_addr
,
2456 &xlat
, &len
, iotlb
->perm
& IOMMU_WO
);
2457 if (!memory_region_is_ram(mr
)) {
2458 DPRINTF("iommu map to non memory area %"HWADDR_PRIx
"\n",
2463 * Translation truncates length to the IOMMU page size,
2464 * check that it did not truncate too much.
2466 if (len
& iotlb
->addr_mask
) {
2467 DPRINTF("iommu has granularity incompatible with target AS\n");
2471 if (iotlb
->perm
!= IOMMU_NONE
) {
2472 vaddr
= memory_region_get_ram_ptr(mr
) + xlat
;
2474 ret
= vfio_dma_map(container
, iotlb
->iova
,
2475 iotlb
->addr_mask
+ 1, vaddr
,
2476 !(iotlb
->perm
& IOMMU_WO
) || mr
->readonly
);
2478 error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx
", "
2479 "0x%"HWADDR_PRIx
", %p) = %d (%m)",
2480 container
, iotlb
->iova
,
2481 iotlb
->addr_mask
+ 1, vaddr
, ret
);
2484 ret
= vfio_dma_unmap(container
, iotlb
->iova
, iotlb
->addr_mask
+ 1);
2486 error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx
", "
2487 "0x%"HWADDR_PRIx
") = %d (%m)",
2488 container
, iotlb
->iova
,
2489 iotlb
->addr_mask
+ 1, ret
);
2494 static void vfio_listener_region_add(MemoryListener
*listener
,
2495 MemoryRegionSection
*section
)
2497 VFIOContainer
*container
= container_of(listener
, VFIOContainer
,
2498 iommu_data
.type1
.listener
);
2504 if (vfio_listener_skipped_section(section
)) {
2505 DPRINTF("SKIPPING region_add %"HWADDR_PRIx
" - %"PRIx64
"\n",
2506 section
->offset_within_address_space
,
2507 section
->offset_within_address_space
+
2508 int128_get64(int128_sub(section
->size
, int128_one())));
2512 if (unlikely((section
->offset_within_address_space
& ~TARGET_PAGE_MASK
) !=
2513 (section
->offset_within_region
& ~TARGET_PAGE_MASK
))) {
2514 error_report("%s received unaligned region", __func__
);
2518 iova
= TARGET_PAGE_ALIGN(section
->offset_within_address_space
);
2519 llend
= int128_make64(section
->offset_within_address_space
);
2520 llend
= int128_add(llend
, section
->size
);
2521 llend
= int128_and(llend
, int128_exts64(TARGET_PAGE_MASK
));
2523 if (int128_ge(int128_make64(iova
), llend
)) {
2527 memory_region_ref(section
->mr
);
2529 if (memory_region_is_iommu(section
->mr
)) {
2530 VFIOGuestIOMMU
*giommu
;
2532 DPRINTF("region_add [iommu] %"HWADDR_PRIx
" - %"HWADDR_PRIx
"\n",
2533 iova
, int128_get64(int128_sub(llend
, int128_one())));
2535 * FIXME: We should do some checking to see if the
2536 * capabilities of the host VFIO IOMMU are adequate to model
2539 * FIXME: For VFIO iommu types which have KVM acceleration to
2540 * avoid bouncing all map/unmaps through qemu this way, this
2541 * would be the right place to wire that up (tell the KVM
2542 * device emulation the VFIO iommu handles to use).
2545 * This assumes that the guest IOMMU is empty of
2546 * mappings at this point.
2548 * One way of doing this is:
2549 * 1. Avoid sharing IOMMUs between emulated devices or different
2551 * 2. Implement VFIO_IOMMU_ENABLE in the host kernel to fail if
2552 * there are some mappings in IOMMU.
2554 * VFIO on SPAPR does that. Other IOMMU models may do that different,
2555 * they must make sure there are no existing mappings or
2556 * loop through existing mappings to map them into VFIO.
2558 giommu
= g_malloc0(sizeof(*giommu
));
2559 giommu
->iommu
= section
->mr
;
2560 giommu
->container
= container
;
2561 giommu
->n
.notify
= vfio_iommu_map_notify
;
2562 QLIST_INSERT_HEAD(&container
->giommu_list
, giommu
, giommu_next
);
2563 memory_region_register_iommu_notifier(giommu
->iommu
, &giommu
->n
);
2568 /* Here we assume that memory_region_is_ram(section->mr)==true */
2570 end
= int128_get64(llend
);
2571 vaddr
= memory_region_get_ram_ptr(section
->mr
) +
2572 section
->offset_within_region
+
2573 (iova
- section
->offset_within_address_space
);
2575 DPRINTF("region_add [ram] %"HWADDR_PRIx
" - %"HWADDR_PRIx
" [%p]\n",
2576 iova
, end
- 1, vaddr
);
2578 ret
= vfio_dma_map(container
, iova
, end
- iova
, vaddr
, section
->readonly
);
2580 error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx
", "
2581 "0x%"HWADDR_PRIx
", %p) = %d (%m)",
2582 container
, iova
, end
- iova
, vaddr
, ret
);
2585 * On the initfn path, store the first error in the container so we
2586 * can gracefully fail. Runtime, there's not much we can do other
2587 * than throw a hardware error.
2589 if (!container
->iommu_data
.type1
.initialized
) {
2590 if (!container
->iommu_data
.type1
.error
) {
2591 container
->iommu_data
.type1
.error
= ret
;
2594 hw_error("vfio: DMA mapping failed, unable to continue");
2599 static void vfio_listener_region_del(MemoryListener
*listener
,
2600 MemoryRegionSection
*section
)
2602 VFIOContainer
*container
= container_of(listener
, VFIOContainer
,
2603 iommu_data
.type1
.listener
);
2607 if (vfio_listener_skipped_section(section
)) {
2608 DPRINTF("SKIPPING region_del %"HWADDR_PRIx
" - %"PRIx64
"\n",
2609 section
->offset_within_address_space
,
2610 section
->offset_within_address_space
+
2611 int128_get64(int128_sub(section
->size
, int128_one())));
2615 if (unlikely((section
->offset_within_address_space
& ~TARGET_PAGE_MASK
) !=
2616 (section
->offset_within_region
& ~TARGET_PAGE_MASK
))) {
2617 error_report("%s received unaligned region", __func__
);
2621 if (memory_region_is_iommu(section
->mr
)) {
2622 VFIOGuestIOMMU
*giommu
;
2624 QLIST_FOREACH(giommu
, &container
->giommu_list
, giommu_next
) {
2625 if (giommu
->iommu
== section
->mr
) {
2626 memory_region_unregister_iommu_notifier(&giommu
->n
);
2627 QLIST_REMOVE(giommu
, giommu_next
);
2634 * FIXME: We assume the one big unmap below is adequate to
2635 * remove any individual page mappings in the IOMMU which
2636 * might have been copied into VFIO. This works for a page table
2637 * based IOMMU where a big unmap flattens a large range of IO-PTEs.
2638 * That may not be true for all IOMMU types.
2642 iova
= TARGET_PAGE_ALIGN(section
->offset_within_address_space
);
2643 end
= (section
->offset_within_address_space
+ int128_get64(section
->size
)) &
2650 DPRINTF("region_del %"HWADDR_PRIx
" - %"HWADDR_PRIx
"\n",
2653 ret
= vfio_dma_unmap(container
, iova
, end
- iova
);
2654 memory_region_unref(section
->mr
);
2656 error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx
", "
2657 "0x%"HWADDR_PRIx
") = %d (%m)",
2658 container
, iova
, end
- iova
, ret
);
2662 static MemoryListener vfio_memory_listener
= {
2663 .region_add
= vfio_listener_region_add
,
2664 .region_del
= vfio_listener_region_del
,
2667 static void vfio_listener_release(VFIOContainer
*container
)
2669 memory_listener_unregister(&container
->iommu_data
.type1
.listener
);
2675 static void vfio_disable_interrupts(VFIODevice
*vdev
)
2677 switch (vdev
->interrupt
) {
2679 vfio_disable_intx(vdev
);
2682 vfio_disable_msi(vdev
);
2685 vfio_disable_msix(vdev
);
2690 static int vfio_setup_msi(VFIODevice
*vdev
, int pos
)
2693 bool msi_64bit
, msi_maskbit
;
2696 if (pread(vdev
->fd
, &ctrl
, sizeof(ctrl
),
2697 vdev
->config_offset
+ pos
+ PCI_CAP_FLAGS
) != sizeof(ctrl
)) {
2700 ctrl
= le16_to_cpu(ctrl
);
2702 msi_64bit
= !!(ctrl
& PCI_MSI_FLAGS_64BIT
);
2703 msi_maskbit
= !!(ctrl
& PCI_MSI_FLAGS_MASKBIT
);
2704 entries
= 1 << ((ctrl
& PCI_MSI_FLAGS_QMASK
) >> 1);
2706 DPRINTF("%04x:%02x:%02x.%x PCI MSI CAP @0x%x\n", vdev
->host
.domain
,
2707 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
, pos
);
2709 ret
= msi_init(&vdev
->pdev
, pos
, entries
, msi_64bit
, msi_maskbit
);
2711 if (ret
== -ENOTSUP
) {
2714 error_report("vfio: msi_init failed");
2717 vdev
->msi_cap_size
= 0xa + (msi_maskbit
? 0xa : 0) + (msi_64bit
? 0x4 : 0);
2723 * We don't have any control over how pci_add_capability() inserts
2724 * capabilities into the chain. In order to setup MSI-X we need a
2725 * MemoryRegion for the BAR. In order to setup the BAR and not
2726 * attempt to mmap the MSI-X table area, which VFIO won't allow, we
2727 * need to first look for where the MSI-X table lives. So we
2728 * unfortunately split MSI-X setup across two functions.
2730 static int vfio_early_setup_msix(VFIODevice
*vdev
)
2734 uint32_t table
, pba
;
2736 pos
= pci_find_capability(&vdev
->pdev
, PCI_CAP_ID_MSIX
);
2741 if (pread(vdev
->fd
, &ctrl
, sizeof(ctrl
),
2742 vdev
->config_offset
+ pos
+ PCI_CAP_FLAGS
) != sizeof(ctrl
)) {
2746 if (pread(vdev
->fd
, &table
, sizeof(table
),
2747 vdev
->config_offset
+ pos
+ PCI_MSIX_TABLE
) != sizeof(table
)) {
2751 if (pread(vdev
->fd
, &pba
, sizeof(pba
),
2752 vdev
->config_offset
+ pos
+ PCI_MSIX_PBA
) != sizeof(pba
)) {
2756 ctrl
= le16_to_cpu(ctrl
);
2757 table
= le32_to_cpu(table
);
2758 pba
= le32_to_cpu(pba
);
2760 vdev
->msix
= g_malloc0(sizeof(*(vdev
->msix
)));
2761 vdev
->msix
->table_bar
= table
& PCI_MSIX_FLAGS_BIRMASK
;
2762 vdev
->msix
->table_offset
= table
& ~PCI_MSIX_FLAGS_BIRMASK
;
2763 vdev
->msix
->pba_bar
= pba
& PCI_MSIX_FLAGS_BIRMASK
;
2764 vdev
->msix
->pba_offset
= pba
& ~PCI_MSIX_FLAGS_BIRMASK
;
2765 vdev
->msix
->entries
= (ctrl
& PCI_MSIX_FLAGS_QSIZE
) + 1;
2767 DPRINTF("%04x:%02x:%02x.%x "
2768 "PCI MSI-X CAP @0x%x, BAR %d, offset 0x%x, entries %d\n",
2769 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
2770 vdev
->host
.function
, pos
, vdev
->msix
->table_bar
,
2771 vdev
->msix
->table_offset
, vdev
->msix
->entries
);
2776 static int vfio_setup_msix(VFIODevice
*vdev
, int pos
)
2780 ret
= msix_init(&vdev
->pdev
, vdev
->msix
->entries
,
2781 &vdev
->bars
[vdev
->msix
->table_bar
].mem
,
2782 vdev
->msix
->table_bar
, vdev
->msix
->table_offset
,
2783 &vdev
->bars
[vdev
->msix
->pba_bar
].mem
,
2784 vdev
->msix
->pba_bar
, vdev
->msix
->pba_offset
, pos
);
2786 if (ret
== -ENOTSUP
) {
2789 error_report("vfio: msix_init failed");
2796 static void vfio_teardown_msi(VFIODevice
*vdev
)
2798 msi_uninit(&vdev
->pdev
);
2801 msix_uninit(&vdev
->pdev
, &vdev
->bars
[vdev
->msix
->table_bar
].mem
,
2802 &vdev
->bars
[vdev
->msix
->pba_bar
].mem
);
2809 static void vfio_mmap_set_enabled(VFIODevice
*vdev
, bool enabled
)
2813 for (i
= 0; i
< PCI_ROM_SLOT
; i
++) {
2814 VFIOBAR
*bar
= &vdev
->bars
[i
];
2820 memory_region_set_enabled(&bar
->mmap_mem
, enabled
);
2821 if (vdev
->msix
&& vdev
->msix
->table_bar
== i
) {
2822 memory_region_set_enabled(&vdev
->msix
->mmap_mem
, enabled
);
2827 static void vfio_unmap_bar(VFIODevice
*vdev
, int nr
)
2829 VFIOBAR
*bar
= &vdev
->bars
[nr
];
2835 vfio_bar_quirk_teardown(vdev
, nr
);
2837 memory_region_del_subregion(&bar
->mem
, &bar
->mmap_mem
);
2838 munmap(bar
->mmap
, memory_region_size(&bar
->mmap_mem
));
2839 memory_region_destroy(&bar
->mmap_mem
);
2841 if (vdev
->msix
&& vdev
->msix
->table_bar
== nr
) {
2842 memory_region_del_subregion(&bar
->mem
, &vdev
->msix
->mmap_mem
);
2843 munmap(vdev
->msix
->mmap
, memory_region_size(&vdev
->msix
->mmap_mem
));
2844 memory_region_destroy(&vdev
->msix
->mmap_mem
);
2847 memory_region_destroy(&bar
->mem
);
2850 static int vfio_mmap_bar(VFIODevice
*vdev
, VFIOBAR
*bar
,
2851 MemoryRegion
*mem
, MemoryRegion
*submem
,
2852 void **map
, size_t size
, off_t offset
,
2857 if (VFIO_ALLOW_MMAP
&& size
&& bar
->flags
& VFIO_REGION_INFO_FLAG_MMAP
) {
2860 if (bar
->flags
& VFIO_REGION_INFO_FLAG_READ
) {
2864 if (bar
->flags
& VFIO_REGION_INFO_FLAG_WRITE
) {
2868 *map
= mmap(NULL
, size
, prot
, MAP_SHARED
,
2869 bar
->fd
, bar
->fd_offset
+ offset
);
2870 if (*map
== MAP_FAILED
) {
2876 memory_region_init_ram_ptr(submem
, OBJECT(vdev
), name
, size
, *map
);
2879 /* Create a zero sized sub-region to make cleanup easy. */
2880 memory_region_init(submem
, OBJECT(vdev
), name
, 0);
2883 memory_region_add_subregion(mem
, offset
, submem
);
2888 static void vfio_map_bar(VFIODevice
*vdev
, int nr
)
2890 VFIOBAR
*bar
= &vdev
->bars
[nr
];
2891 unsigned size
= bar
->size
;
2897 /* Skip both unimplemented BARs and the upper half of 64bit BARS. */
2902 snprintf(name
, sizeof(name
), "VFIO %04x:%02x:%02x.%x BAR %d",
2903 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
2904 vdev
->host
.function
, nr
);
2906 /* Determine what type of BAR this is for registration */
2907 ret
= pread(vdev
->fd
, &pci_bar
, sizeof(pci_bar
),
2908 vdev
->config_offset
+ PCI_BASE_ADDRESS_0
+ (4 * nr
));
2909 if (ret
!= sizeof(pci_bar
)) {
2910 error_report("vfio: Failed to read BAR %d (%m)", nr
);
2914 pci_bar
= le32_to_cpu(pci_bar
);
2915 bar
->ioport
= (pci_bar
& PCI_BASE_ADDRESS_SPACE_IO
);
2916 bar
->mem64
= bar
->ioport
? 0 : (pci_bar
& PCI_BASE_ADDRESS_MEM_TYPE_64
);
2917 type
= pci_bar
& (bar
->ioport
? ~PCI_BASE_ADDRESS_IO_MASK
:
2918 ~PCI_BASE_ADDRESS_MEM_MASK
);
2920 /* A "slow" read/write mapping underlies all BARs */
2921 memory_region_init_io(&bar
->mem
, OBJECT(vdev
), &vfio_bar_ops
,
2923 pci_register_bar(&vdev
->pdev
, nr
, type
, &bar
->mem
);
2926 * We can't mmap areas overlapping the MSIX vector table, so we
2927 * potentially insert a direct-mapped subregion before and after it.
2929 if (vdev
->msix
&& vdev
->msix
->table_bar
== nr
) {
2930 size
= vdev
->msix
->table_offset
& qemu_host_page_mask
;
2933 strncat(name
, " mmap", sizeof(name
) - strlen(name
) - 1);
2934 if (vfio_mmap_bar(vdev
, bar
, &bar
->mem
,
2935 &bar
->mmap_mem
, &bar
->mmap
, size
, 0, name
)) {
2936 error_report("%s unsupported. Performance may be slow", name
);
2939 if (vdev
->msix
&& vdev
->msix
->table_bar
== nr
) {
2942 start
= HOST_PAGE_ALIGN(vdev
->msix
->table_offset
+
2943 (vdev
->msix
->entries
* PCI_MSIX_ENTRY_SIZE
));
2945 size
= start
< bar
->size
? bar
->size
- start
: 0;
2946 strncat(name
, " msix-hi", sizeof(name
) - strlen(name
) - 1);
2947 /* VFIOMSIXInfo contains another MemoryRegion for this mapping */
2948 if (vfio_mmap_bar(vdev
, bar
, &bar
->mem
, &vdev
->msix
->mmap_mem
,
2949 &vdev
->msix
->mmap
, size
, start
, name
)) {
2950 error_report("%s unsupported. Performance may be slow", name
);
2954 vfio_bar_quirk_setup(vdev
, nr
);
2957 static void vfio_map_bars(VFIODevice
*vdev
)
2961 for (i
= 0; i
< PCI_ROM_SLOT
; i
++) {
2962 vfio_map_bar(vdev
, i
);
2965 if (vdev
->has_vga
) {
2966 memory_region_init_io(&vdev
->vga
.region
[QEMU_PCI_VGA_MEM
].mem
,
2967 OBJECT(vdev
), &vfio_vga_ops
,
2968 &vdev
->vga
.region
[QEMU_PCI_VGA_MEM
],
2969 "vfio-vga-mmio@0xa0000",
2970 QEMU_PCI_VGA_MEM_SIZE
);
2971 memory_region_init_io(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_LO
].mem
,
2972 OBJECT(vdev
), &vfio_vga_ops
,
2973 &vdev
->vga
.region
[QEMU_PCI_VGA_IO_LO
],
2974 "vfio-vga-io@0x3b0",
2975 QEMU_PCI_VGA_IO_LO_SIZE
);
2976 memory_region_init_io(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].mem
,
2977 OBJECT(vdev
), &vfio_vga_ops
,
2978 &vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
],
2979 "vfio-vga-io@0x3c0",
2980 QEMU_PCI_VGA_IO_HI_SIZE
);
2982 pci_register_vga(&vdev
->pdev
, &vdev
->vga
.region
[QEMU_PCI_VGA_MEM
].mem
,
2983 &vdev
->vga
.region
[QEMU_PCI_VGA_IO_LO
].mem
,
2984 &vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].mem
);
2985 vfio_vga_quirk_setup(vdev
);
2989 static void vfio_unmap_bars(VFIODevice
*vdev
)
2993 for (i
= 0; i
< PCI_ROM_SLOT
; i
++) {
2994 vfio_unmap_bar(vdev
, i
);
2997 if (vdev
->has_vga
) {
2998 vfio_vga_quirk_teardown(vdev
);
2999 pci_unregister_vga(&vdev
->pdev
);
3000 memory_region_destroy(&vdev
->vga
.region
[QEMU_PCI_VGA_MEM
].mem
);
3001 memory_region_destroy(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_LO
].mem
);
3002 memory_region_destroy(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].mem
);
3009 static uint8_t vfio_std_cap_max_size(PCIDevice
*pdev
, uint8_t pos
)
3011 uint8_t tmp
, next
= 0xff;
3013 for (tmp
= pdev
->config
[PCI_CAPABILITY_LIST
]; tmp
;
3014 tmp
= pdev
->config
[tmp
+ 1]) {
3015 if (tmp
> pos
&& tmp
< next
) {
3023 static void vfio_set_word_bits(uint8_t *buf
, uint16_t val
, uint16_t mask
)
3025 pci_set_word(buf
, (pci_get_word(buf
) & ~mask
) | val
);
3028 static void vfio_add_emulated_word(VFIODevice
*vdev
, int pos
,
3029 uint16_t val
, uint16_t mask
)
3031 vfio_set_word_bits(vdev
->pdev
.config
+ pos
, val
, mask
);
3032 vfio_set_word_bits(vdev
->pdev
.wmask
+ pos
, ~mask
, mask
);
3033 vfio_set_word_bits(vdev
->emulated_config_bits
+ pos
, mask
, mask
);
3036 static void vfio_set_long_bits(uint8_t *buf
, uint32_t val
, uint32_t mask
)
3038 pci_set_long(buf
, (pci_get_long(buf
) & ~mask
) | val
);
3041 static void vfio_add_emulated_long(VFIODevice
*vdev
, int pos
,
3042 uint32_t val
, uint32_t mask
)
3044 vfio_set_long_bits(vdev
->pdev
.config
+ pos
, val
, mask
);
3045 vfio_set_long_bits(vdev
->pdev
.wmask
+ pos
, ~mask
, mask
);
3046 vfio_set_long_bits(vdev
->emulated_config_bits
+ pos
, mask
, mask
);
3049 static int vfio_setup_pcie_cap(VFIODevice
*vdev
, int pos
, uint8_t size
)
3054 flags
= pci_get_word(vdev
->pdev
.config
+ pos
+ PCI_CAP_FLAGS
);
3055 type
= (flags
& PCI_EXP_FLAGS_TYPE
) >> 4;
3057 if (type
!= PCI_EXP_TYPE_ENDPOINT
&&
3058 type
!= PCI_EXP_TYPE_LEG_END
&&
3059 type
!= PCI_EXP_TYPE_RC_END
) {
3061 error_report("vfio: Assignment of PCIe type 0x%x "
3062 "devices is not currently supported", type
);
3066 if (!pci_bus_is_express(vdev
->pdev
.bus
)) {
3068 * Use express capability as-is on PCI bus. It doesn't make much
3069 * sense to even expose, but some drivers (ex. tg3) depend on it
3070 * and guests don't seem to be particular about it. We'll need
3071 * to revist this or force express devices to express buses if we
3072 * ever expose an IOMMU to the guest.
3074 } else if (pci_bus_is_root(vdev
->pdev
.bus
)) {
3076 * On a Root Complex bus Endpoints become Root Complex Integrated
3077 * Endpoints, which changes the type and clears the LNK & LNK2 fields.
3079 if (type
== PCI_EXP_TYPE_ENDPOINT
) {
3080 vfio_add_emulated_word(vdev
, pos
+ PCI_CAP_FLAGS
,
3081 PCI_EXP_TYPE_RC_END
<< 4,
3082 PCI_EXP_FLAGS_TYPE
);
3084 /* Link Capabilities, Status, and Control goes away */
3085 if (size
> PCI_EXP_LNKCTL
) {
3086 vfio_add_emulated_long(vdev
, pos
+ PCI_EXP_LNKCAP
, 0, ~0);
3087 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKCTL
, 0, ~0);
3088 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKSTA
, 0, ~0);
3090 #ifndef PCI_EXP_LNKCAP2
3091 #define PCI_EXP_LNKCAP2 44
3093 #ifndef PCI_EXP_LNKSTA2
3094 #define PCI_EXP_LNKSTA2 50
3096 /* Link 2 Capabilities, Status, and Control goes away */
3097 if (size
> PCI_EXP_LNKCAP2
) {
3098 vfio_add_emulated_long(vdev
, pos
+ PCI_EXP_LNKCAP2
, 0, ~0);
3099 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKCTL2
, 0, ~0);
3100 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKSTA2
, 0, ~0);
3104 } else if (type
== PCI_EXP_TYPE_LEG_END
) {
3106 * Legacy endpoints don't belong on the root complex. Windows
3107 * seems to be happier with devices if we skip the capability.
3114 * Convert Root Complex Integrated Endpoints to regular endpoints.
3115 * These devices don't support LNK/LNK2 capabilities, so make them up.
3117 if (type
== PCI_EXP_TYPE_RC_END
) {
3118 vfio_add_emulated_word(vdev
, pos
+ PCI_CAP_FLAGS
,
3119 PCI_EXP_TYPE_ENDPOINT
<< 4,
3120 PCI_EXP_FLAGS_TYPE
);
3121 vfio_add_emulated_long(vdev
, pos
+ PCI_EXP_LNKCAP
,
3122 PCI_EXP_LNK_MLW_1
| PCI_EXP_LNK_LS_25
, ~0);
3123 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKCTL
, 0, ~0);
3126 /* Mark the Link Status bits as emulated to allow virtual negotiation */
3127 vfio_add_emulated_word(vdev
, pos
+ PCI_EXP_LNKSTA
,
3128 pci_get_word(vdev
->pdev
.config
+ pos
+
3130 PCI_EXP_LNKCAP_MLW
| PCI_EXP_LNKCAP_SLS
);
3133 pos
= pci_add_capability(&vdev
->pdev
, PCI_CAP_ID_EXP
, pos
, size
);
3135 vdev
->pdev
.exp
.exp_cap
= pos
;
3141 static void vfio_check_pcie_flr(VFIODevice
*vdev
, uint8_t pos
)
3143 uint32_t cap
= pci_get_long(vdev
->pdev
.config
+ pos
+ PCI_EXP_DEVCAP
);
3145 if (cap
& PCI_EXP_DEVCAP_FLR
) {
3146 DPRINTF("%04x:%02x:%02x.%x Supports FLR via PCIe cap\n",
3147 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
3148 vdev
->host
.function
);
3149 vdev
->has_flr
= true;
3153 static void vfio_check_pm_reset(VFIODevice
*vdev
, uint8_t pos
)
3155 uint16_t csr
= pci_get_word(vdev
->pdev
.config
+ pos
+ PCI_PM_CTRL
);
3157 if (!(csr
& PCI_PM_CTRL_NO_SOFT_RESET
)) {
3158 DPRINTF("%04x:%02x:%02x.%x Supports PM reset\n",
3159 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
3160 vdev
->host
.function
);
3161 vdev
->has_pm_reset
= true;
3165 static void vfio_check_af_flr(VFIODevice
*vdev
, uint8_t pos
)
3167 uint8_t cap
= pci_get_byte(vdev
->pdev
.config
+ pos
+ PCI_AF_CAP
);
3169 if ((cap
& PCI_AF_CAP_TP
) && (cap
& PCI_AF_CAP_FLR
)) {
3170 DPRINTF("%04x:%02x:%02x.%x Supports FLR via AF cap\n",
3171 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
3172 vdev
->host
.function
);
3173 vdev
->has_flr
= true;
3177 static int vfio_add_std_cap(VFIODevice
*vdev
, uint8_t pos
)
3179 PCIDevice
*pdev
= &vdev
->pdev
;
3180 uint8_t cap_id
, next
, size
;
3183 cap_id
= pdev
->config
[pos
];
3184 next
= pdev
->config
[pos
+ 1];
3187 * If it becomes important to configure capabilities to their actual
3188 * size, use this as the default when it's something we don't recognize.
3189 * Since QEMU doesn't actually handle many of the config accesses,
3190 * exact size doesn't seem worthwhile.
3192 size
= vfio_std_cap_max_size(pdev
, pos
);
3195 * pci_add_capability always inserts the new capability at the head
3196 * of the chain. Therefore to end up with a chain that matches the
3197 * physical device, we insert from the end by making this recursive.
3198 * This is also why we pre-caclulate size above as cached config space
3199 * will be changed as we unwind the stack.
3202 ret
= vfio_add_std_cap(vdev
, next
);
3207 /* Begin the rebuild, use QEMU emulated list bits */
3208 pdev
->config
[PCI_CAPABILITY_LIST
] = 0;
3209 vdev
->emulated_config_bits
[PCI_CAPABILITY_LIST
] = 0xff;
3210 vdev
->emulated_config_bits
[PCI_STATUS
] |= PCI_STATUS_CAP_LIST
;
3213 /* Use emulated next pointer to allow dropping caps */
3214 pci_set_byte(vdev
->emulated_config_bits
+ pos
+ 1, 0xff);
3217 case PCI_CAP_ID_MSI
:
3218 ret
= vfio_setup_msi(vdev
, pos
);
3220 case PCI_CAP_ID_EXP
:
3221 vfio_check_pcie_flr(vdev
, pos
);
3222 ret
= vfio_setup_pcie_cap(vdev
, pos
, size
);
3224 case PCI_CAP_ID_MSIX
:
3225 ret
= vfio_setup_msix(vdev
, pos
);
3228 vfio_check_pm_reset(vdev
, pos
);
3230 ret
= pci_add_capability(pdev
, cap_id
, pos
, size
);
3233 vfio_check_af_flr(vdev
, pos
);
3234 ret
= pci_add_capability(pdev
, cap_id
, pos
, size
);
3237 ret
= pci_add_capability(pdev
, cap_id
, pos
, size
);
3242 error_report("vfio: %04x:%02x:%02x.%x Error adding PCI capability "
3243 "0x%x[0x%x]@0x%x: %d", vdev
->host
.domain
,
3244 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
,
3245 cap_id
, size
, pos
, ret
);
3252 static int vfio_add_capabilities(VFIODevice
*vdev
)
3254 PCIDevice
*pdev
= &vdev
->pdev
;
3256 if (!(pdev
->config
[PCI_STATUS
] & PCI_STATUS_CAP_LIST
) ||
3257 !pdev
->config
[PCI_CAPABILITY_LIST
]) {
3258 return 0; /* Nothing to add */
3261 return vfio_add_std_cap(vdev
, pdev
->config
[PCI_CAPABILITY_LIST
]);
3264 static void vfio_pci_pre_reset(VFIODevice
*vdev
)
3266 PCIDevice
*pdev
= &vdev
->pdev
;
3269 vfio_disable_interrupts(vdev
);
3271 /* Make sure the device is in D0 */
3276 pmcsr
= vfio_pci_read_config(pdev
, vdev
->pm_cap
+ PCI_PM_CTRL
, 2);
3277 state
= pmcsr
& PCI_PM_CTRL_STATE_MASK
;
3279 pmcsr
&= ~PCI_PM_CTRL_STATE_MASK
;
3280 vfio_pci_write_config(pdev
, vdev
->pm_cap
+ PCI_PM_CTRL
, pmcsr
, 2);
3281 /* vfio handles the necessary delay here */
3282 pmcsr
= vfio_pci_read_config(pdev
, vdev
->pm_cap
+ PCI_PM_CTRL
, 2);
3283 state
= pmcsr
& PCI_PM_CTRL_STATE_MASK
;
3285 error_report("vfio: Unable to power on device, stuck in D%d",
3292 * Stop any ongoing DMA by disconecting I/O, MMIO, and bus master.
3293 * Also put INTx Disable in known state.
3295 cmd
= vfio_pci_read_config(pdev
, PCI_COMMAND
, 2);
3296 cmd
&= ~(PCI_COMMAND_IO
| PCI_COMMAND_MEMORY
| PCI_COMMAND_MASTER
|
3297 PCI_COMMAND_INTX_DISABLE
);
3298 vfio_pci_write_config(pdev
, PCI_COMMAND
, cmd
, 2);
3301 static void vfio_pci_post_reset(VFIODevice
*vdev
)
3303 vfio_enable_intx(vdev
);
3306 static bool vfio_pci_host_match(PCIHostDeviceAddress
*host1
,
3307 PCIHostDeviceAddress
*host2
)
3309 return (host1
->domain
== host2
->domain
&& host1
->bus
== host2
->bus
&&
3310 host1
->slot
== host2
->slot
&& host1
->function
== host2
->function
);
3313 static int vfio_pci_hot_reset(VFIODevice
*vdev
, bool single
)
3316 struct vfio_pci_hot_reset_info
*info
;
3317 struct vfio_pci_dependent_device
*devices
;
3318 struct vfio_pci_hot_reset
*reset
;
3323 DPRINTF("%s(%04x:%02x:%02x.%x) %s\n", __func__
, vdev
->host
.domain
,
3324 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
,
3325 single
? "one" : "multi");
3327 vfio_pci_pre_reset(vdev
);
3328 vdev
->needs_reset
= false;
3330 info
= g_malloc0(sizeof(*info
));
3331 info
->argsz
= sizeof(*info
);
3333 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO
, info
);
3334 if (ret
&& errno
!= ENOSPC
) {
3336 if (!vdev
->has_pm_reset
) {
3337 error_report("vfio: Cannot reset device %04x:%02x:%02x.%x, "
3338 "no available reset mechanism.", vdev
->host
.domain
,
3339 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
3344 count
= info
->count
;
3345 info
= g_realloc(info
, sizeof(*info
) + (count
* sizeof(*devices
)));
3346 info
->argsz
= sizeof(*info
) + (count
* sizeof(*devices
));
3347 devices
= &info
->devices
[0];
3349 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO
, info
);
3352 error_report("vfio: hot reset info failed: %m");
3356 DPRINTF("%04x:%02x:%02x.%x: hot reset dependent devices:\n",
3357 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
3358 vdev
->host
.function
);
3360 /* Verify that we have all the groups required */
3361 for (i
= 0; i
< info
->count
; i
++) {
3362 PCIHostDeviceAddress host
;
3365 host
.domain
= devices
[i
].segment
;
3366 host
.bus
= devices
[i
].bus
;
3367 host
.slot
= PCI_SLOT(devices
[i
].devfn
);
3368 host
.function
= PCI_FUNC(devices
[i
].devfn
);
3370 DPRINTF("\t%04x:%02x:%02x.%x group %d\n", host
.domain
,
3371 host
.bus
, host
.slot
, host
.function
, devices
[i
].group_id
);
3373 if (vfio_pci_host_match(&host
, &vdev
->host
)) {
3377 QLIST_FOREACH(group
, &group_list
, next
) {
3378 if (group
->groupid
== devices
[i
].group_id
) {
3384 if (!vdev
->has_pm_reset
) {
3385 error_report("vfio: Cannot reset device %04x:%02x:%02x.%x, "
3386 "depends on group %d which is not owned.",
3387 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
3388 vdev
->host
.function
, devices
[i
].group_id
);
3394 /* Prep dependent devices for reset and clear our marker. */
3395 QLIST_FOREACH(tmp
, &group
->device_list
, next
) {
3396 if (vfio_pci_host_match(&host
, &tmp
->host
)) {
3398 DPRINTF("vfio: found another in-use device "
3399 "%04x:%02x:%02x.%x\n", host
.domain
, host
.bus
,
3400 host
.slot
, host
.function
);
3404 vfio_pci_pre_reset(tmp
);
3405 tmp
->needs_reset
= false;
3412 if (!single
&& !multi
) {
3413 DPRINTF("vfio: No other in-use devices for multi hot reset\n");
3418 /* Determine how many group fds need to be passed */
3420 QLIST_FOREACH(group
, &group_list
, next
) {
3421 for (i
= 0; i
< info
->count
; i
++) {
3422 if (group
->groupid
== devices
[i
].group_id
) {
3429 reset
= g_malloc0(sizeof(*reset
) + (count
* sizeof(*fds
)));
3430 reset
->argsz
= sizeof(*reset
) + (count
* sizeof(*fds
));
3431 fds
= &reset
->group_fds
[0];
3433 /* Fill in group fds */
3434 QLIST_FOREACH(group
, &group_list
, next
) {
3435 for (i
= 0; i
< info
->count
; i
++) {
3436 if (group
->groupid
== devices
[i
].group_id
) {
3437 fds
[reset
->count
++] = group
->fd
;
3444 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_PCI_HOT_RESET
, reset
);
3447 DPRINTF("%04x:%02x:%02x.%x hot reset: %s\n", vdev
->host
.domain
,
3448 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
,
3449 ret
? "%m" : "Success");
3452 /* Re-enable INTx on affected devices */
3453 for (i
= 0; i
< info
->count
; i
++) {
3454 PCIHostDeviceAddress host
;
3457 host
.domain
= devices
[i
].segment
;
3458 host
.bus
= devices
[i
].bus
;
3459 host
.slot
= PCI_SLOT(devices
[i
].devfn
);
3460 host
.function
= PCI_FUNC(devices
[i
].devfn
);
3462 if (vfio_pci_host_match(&host
, &vdev
->host
)) {
3466 QLIST_FOREACH(group
, &group_list
, next
) {
3467 if (group
->groupid
== devices
[i
].group_id
) {
3476 QLIST_FOREACH(tmp
, &group
->device_list
, next
) {
3477 if (vfio_pci_host_match(&host
, &tmp
->host
)) {
3478 vfio_pci_post_reset(tmp
);
3484 vfio_pci_post_reset(vdev
);
3491 * We want to differentiate hot reset of mulitple in-use devices vs hot reset
3492 * of a single in-use device. VFIO_DEVICE_RESET will already handle the case
3493 * of doing hot resets when there is only a single device per bus. The in-use
3494 * here refers to how many VFIODevices are affected. A hot reset that affects
3495 * multiple devices, but only a single in-use device, means that we can call
3496 * it from our bus ->reset() callback since the extent is effectively a single
3497 * device. This allows us to make use of it in the hotplug path. When there
3498 * are multiple in-use devices, we can only trigger the hot reset during a
3499 * system reset and thus from our reset handler. We separate _one vs _multi
3500 * here so that we don't overlap and do a double reset on the system reset
3501 * path where both our reset handler and ->reset() callback are used. Calling
3502 * _one() will only do a hot reset for the one in-use devices case, calling
3503 * _multi() will do nothing if a _one() would have been sufficient.
3505 static int vfio_pci_hot_reset_one(VFIODevice
*vdev
)
3507 return vfio_pci_hot_reset(vdev
, true);
3510 static int vfio_pci_hot_reset_multi(VFIODevice
*vdev
)
3512 return vfio_pci_hot_reset(vdev
, false);
3515 static void vfio_pci_reset_handler(void *opaque
)
3520 QLIST_FOREACH(group
, &group_list
, next
) {
3521 QLIST_FOREACH(vdev
, &group
->device_list
, next
) {
3522 if (!vdev
->reset_works
|| (!vdev
->has_flr
&& vdev
->has_pm_reset
)) {
3523 vdev
->needs_reset
= true;
3528 QLIST_FOREACH(group
, &group_list
, next
) {
3529 QLIST_FOREACH(vdev
, &group
->device_list
, next
) {
3530 if (vdev
->needs_reset
) {
3531 vfio_pci_hot_reset_multi(vdev
);
3537 static void vfio_kvm_device_add_group(VFIOGroup
*group
)
3540 struct kvm_device_attr attr
= {
3541 .group
= KVM_DEV_VFIO_GROUP
,
3542 .attr
= KVM_DEV_VFIO_GROUP_ADD
,
3543 .addr
= (uint64_t)(unsigned long)&group
->fd
,
3546 if (!kvm_enabled()) {
3550 if (vfio_kvm_device_fd
< 0) {
3551 struct kvm_create_device cd
= {
3552 .type
= KVM_DEV_TYPE_VFIO
,
3555 if (kvm_vm_ioctl(kvm_state
, KVM_CREATE_DEVICE
, &cd
)) {
3556 DPRINTF("KVM_CREATE_DEVICE: %m\n");
3560 vfio_kvm_device_fd
= cd
.fd
;
3563 if (ioctl(vfio_kvm_device_fd
, KVM_SET_DEVICE_ATTR
, &attr
)) {
3564 error_report("Failed to add group %d to KVM VFIO device: %m",
3570 static void vfio_kvm_device_del_group(VFIOGroup
*group
)
3573 struct kvm_device_attr attr
= {
3574 .group
= KVM_DEV_VFIO_GROUP
,
3575 .attr
= KVM_DEV_VFIO_GROUP_DEL
,
3576 .addr
= (uint64_t)(unsigned long)&group
->fd
,
3579 if (vfio_kvm_device_fd
< 0) {
3583 if (ioctl(vfio_kvm_device_fd
, KVM_SET_DEVICE_ATTR
, &attr
)) {
3584 error_report("Failed to remove group %d from KVM VFIO device: %m",
3590 static VFIOAddressSpace
*vfio_get_address_space(AddressSpace
*as
)
3592 VFIOAddressSpace
*space
;
3594 QLIST_FOREACH(space
, &vfio_address_spaces
, list
) {
3595 if (space
->as
== as
) {
3600 /* No suitable VFIOAddressSpace, create a new one */
3601 space
= g_malloc0(sizeof(*space
));
3603 QLIST_INIT(&space
->containers
);
3605 QLIST_INSERT_HEAD(&vfio_address_spaces
, space
, list
);
3610 static void vfio_put_address_space(VFIOAddressSpace
*space
)
3612 if (QLIST_EMPTY(&space
->containers
)) {
3613 QLIST_REMOVE(space
, list
);
3618 static int vfio_connect_container(VFIOGroup
*group
, AddressSpace
*as
)
3620 VFIOContainer
*container
;
3622 VFIOAddressSpace
*space
;
3624 space
= vfio_get_address_space(as
);
3626 QLIST_FOREACH(container
, &space
->containers
, next
) {
3627 if (!ioctl(group
->fd
, VFIO_GROUP_SET_CONTAINER
, &container
->fd
)) {
3628 group
->container
= container
;
3629 QLIST_INSERT_HEAD(&container
->group_list
, group
, container_next
);
3634 fd
= qemu_open("/dev/vfio/vfio", O_RDWR
);
3636 error_report("vfio: failed to open /dev/vfio/vfio: %m");
3638 goto put_space_exit
;
3641 ret
= ioctl(fd
, VFIO_GET_API_VERSION
);
3642 if (ret
!= VFIO_API_VERSION
) {
3643 error_report("vfio: supported vfio version: %d, "
3644 "reported version: %d", VFIO_API_VERSION
, ret
);
3649 container
= g_malloc0(sizeof(*container
));
3650 container
->space
= space
;
3653 if (ioctl(fd
, VFIO_CHECK_EXTENSION
, VFIO_TYPE1_IOMMU
)) {
3654 ret
= ioctl(group
->fd
, VFIO_GROUP_SET_CONTAINER
, &fd
);
3656 error_report("vfio: failed to set group container: %m");
3658 goto free_container_exit
;
3661 ret
= ioctl(fd
, VFIO_SET_IOMMU
, VFIO_TYPE1_IOMMU
);
3663 error_report("vfio: failed to set iommu for container: %m");
3665 goto free_container_exit
;
3668 container
->iommu_data
.type1
.listener
= vfio_memory_listener
;
3669 container
->iommu_data
.release
= vfio_listener_release
;
3671 memory_listener_register(&container
->iommu_data
.type1
.listener
,
3672 &address_space_memory
);
3674 if (container
->iommu_data
.type1
.error
) {
3675 ret
= container
->iommu_data
.type1
.error
;
3676 error_report("vfio: memory listener initialization failed for container");
3677 goto listener_release_exit
;
3680 container
->iommu_data
.type1
.initialized
= true;
3682 } else if (ioctl(fd
, VFIO_CHECK_EXTENSION
, VFIO_SPAPR_TCE_IOMMU
)) {
3683 ret
= ioctl(group
->fd
, VFIO_GROUP_SET_CONTAINER
, &fd
);
3685 error_report("vfio: failed to set group container: %m");
3687 goto free_container_exit
;
3690 ret
= ioctl(fd
, VFIO_SET_IOMMU
, VFIO_SPAPR_TCE_IOMMU
);
3692 error_report("vfio: failed to set iommu for container: %m");
3694 goto free_container_exit
;
3698 * The host kernel code implementing VFIO_IOMMU_DISABLE is called
3699 * when container fd is closed so we do not call it explicitly
3702 ret
= ioctl(fd
, VFIO_IOMMU_ENABLE
);
3704 error_report("vfio: failed to enable container: %m");
3706 goto free_container_exit
;
3709 container
->iommu_data
.type1
.listener
= vfio_memory_listener
;
3710 container
->iommu_data
.release
= vfio_listener_release
;
3712 memory_listener_register(&container
->iommu_data
.type1
.listener
,
3713 container
->space
->as
);
3716 error_report("vfio: No available IOMMU models");
3718 goto free_container_exit
;
3721 QLIST_INIT(&container
->group_list
);
3722 QLIST_INSERT_HEAD(&space
->containers
, container
, next
);
3724 group
->container
= container
;
3725 QLIST_INSERT_HEAD(&container
->group_list
, group
, container_next
);
3729 listener_release_exit
:
3730 vfio_listener_release(container
);
3732 free_container_exit
:
3739 vfio_put_address_space(space
);
3744 static void vfio_disconnect_container(VFIOGroup
*group
)
3746 VFIOContainer
*container
= group
->container
;
3748 if (ioctl(group
->fd
, VFIO_GROUP_UNSET_CONTAINER
, &container
->fd
)) {
3749 error_report("vfio: error disconnecting group %d from container",
3753 QLIST_REMOVE(group
, container_next
);
3754 group
->container
= NULL
;
3756 if (QLIST_EMPTY(&container
->group_list
)) {
3757 VFIOAddressSpace
*space
= container
->space
;
3759 if (container
->iommu_data
.release
) {
3760 container
->iommu_data
.release(container
);
3762 QLIST_REMOVE(container
, next
);
3763 DPRINTF("vfio_disconnect_container: close container->fd\n");
3764 close(container
->fd
);
3767 vfio_put_address_space(space
);
3771 static VFIOGroup
*vfio_get_group(int groupid
, AddressSpace
*as
)
3775 struct vfio_group_status status
= { .argsz
= sizeof(status
) };
3777 QLIST_FOREACH(group
, &group_list
, next
) {
3778 if (group
->groupid
== groupid
) {
3779 /* Found it. Now is it already in the right context? */
3780 if (group
->container
->space
->as
== as
) {
3783 error_report("vfio: group %d used in multiple address spaces",
3790 group
= g_malloc0(sizeof(*group
));
3792 snprintf(path
, sizeof(path
), "/dev/vfio/%d", groupid
);
3793 group
->fd
= qemu_open(path
, O_RDWR
);
3794 if (group
->fd
< 0) {
3795 error_report("vfio: error opening %s: %m", path
);
3796 goto free_group_exit
;
3799 if (ioctl(group
->fd
, VFIO_GROUP_GET_STATUS
, &status
)) {
3800 error_report("vfio: error getting group status: %m");
3804 if (!(status
.flags
& VFIO_GROUP_FLAGS_VIABLE
)) {
3805 error_report("vfio: error, group %d is not viable, please ensure "
3806 "all devices within the iommu_group are bound to their "
3807 "vfio bus driver.", groupid
);
3811 group
->groupid
= groupid
;
3812 QLIST_INIT(&group
->device_list
);
3814 if (vfio_connect_container(group
, as
)) {
3815 error_report("vfio: failed to setup container for group %d", groupid
);
3819 if (QLIST_EMPTY(&group_list
)) {
3820 qemu_register_reset(vfio_pci_reset_handler
, NULL
);
3823 QLIST_INSERT_HEAD(&group_list
, group
, next
);
3825 vfio_kvm_device_add_group(group
);
3838 static void vfio_put_group(VFIOGroup
*group
)
3840 if (!QLIST_EMPTY(&group
->device_list
)) {
3844 vfio_kvm_device_del_group(group
);
3845 vfio_disconnect_container(group
);
3846 QLIST_REMOVE(group
, next
);
3847 DPRINTF("vfio_put_group: close group->fd\n");
3851 if (QLIST_EMPTY(&group_list
)) {
3852 qemu_unregister_reset(vfio_pci_reset_handler
, NULL
);
3856 static int vfio_get_device(VFIOGroup
*group
, const char *name
, VFIODevice
*vdev
)
3858 struct vfio_device_info dev_info
= { .argsz
= sizeof(dev_info
) };
3859 struct vfio_region_info reg_info
= { .argsz
= sizeof(reg_info
) };
3860 struct vfio_irq_info irq_info
= { .argsz
= sizeof(irq_info
) };
3863 ret
= ioctl(group
->fd
, VFIO_GROUP_GET_DEVICE_FD
, name
);
3865 error_report("vfio: error getting device %s from group %d: %m",
3866 name
, group
->groupid
);
3867 error_printf("Verify all devices in group %d are bound to vfio-pci "
3868 "or pci-stub and not already in use\n", group
->groupid
);
3873 vdev
->group
= group
;
3874 QLIST_INSERT_HEAD(&group
->device_list
, vdev
, next
);
3876 /* Sanity check device */
3877 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_GET_INFO
, &dev_info
);
3879 error_report("vfio: error getting device info: %m");
3883 DPRINTF("Device %s flags: %u, regions: %u, irgs: %u\n", name
,
3884 dev_info
.flags
, dev_info
.num_regions
, dev_info
.num_irqs
);
3886 if (!(dev_info
.flags
& VFIO_DEVICE_FLAGS_PCI
)) {
3887 error_report("vfio: Um, this isn't a PCI device");
3891 vdev
->reset_works
= !!(dev_info
.flags
& VFIO_DEVICE_FLAGS_RESET
);
3893 if (dev_info
.num_regions
< VFIO_PCI_CONFIG_REGION_INDEX
+ 1) {
3894 error_report("vfio: unexpected number of io regions %u",
3895 dev_info
.num_regions
);
3899 if (dev_info
.num_irqs
< VFIO_PCI_MSIX_IRQ_INDEX
+ 1) {
3900 error_report("vfio: unexpected number of irqs %u", dev_info
.num_irqs
);
3904 for (i
= VFIO_PCI_BAR0_REGION_INDEX
; i
< VFIO_PCI_ROM_REGION_INDEX
; i
++) {
3907 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_GET_REGION_INFO
, ®_info
);
3909 error_report("vfio: Error getting region %d info: %m", i
);
3913 DPRINTF("Device %s region %d:\n", name
, i
);
3914 DPRINTF(" size: 0x%lx, offset: 0x%lx, flags: 0x%lx\n",
3915 (unsigned long)reg_info
.size
, (unsigned long)reg_info
.offset
,
3916 (unsigned long)reg_info
.flags
);
3918 vdev
->bars
[i
].flags
= reg_info
.flags
;
3919 vdev
->bars
[i
].size
= reg_info
.size
;
3920 vdev
->bars
[i
].fd_offset
= reg_info
.offset
;
3921 vdev
->bars
[i
].fd
= vdev
->fd
;
3922 vdev
->bars
[i
].nr
= i
;
3923 QLIST_INIT(&vdev
->bars
[i
].quirks
);
3926 reg_info
.index
= VFIO_PCI_CONFIG_REGION_INDEX
;
3928 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_GET_REGION_INFO
, ®_info
);
3930 error_report("vfio: Error getting config info: %m");
3934 DPRINTF("Device %s config:\n", name
);
3935 DPRINTF(" size: 0x%lx, offset: 0x%lx, flags: 0x%lx\n",
3936 (unsigned long)reg_info
.size
, (unsigned long)reg_info
.offset
,
3937 (unsigned long)reg_info
.flags
);
3939 vdev
->config_size
= reg_info
.size
;
3940 if (vdev
->config_size
== PCI_CONFIG_SPACE_SIZE
) {
3941 vdev
->pdev
.cap_present
&= ~QEMU_PCI_CAP_EXPRESS
;
3943 vdev
->config_offset
= reg_info
.offset
;
3945 if ((vdev
->features
& VFIO_FEATURE_ENABLE_VGA
) &&
3946 dev_info
.num_regions
> VFIO_PCI_VGA_REGION_INDEX
) {
3947 struct vfio_region_info vga_info
= {
3948 .argsz
= sizeof(vga_info
),
3949 .index
= VFIO_PCI_VGA_REGION_INDEX
,
3952 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_GET_REGION_INFO
, &vga_info
);
3955 "vfio: Device does not support requested feature x-vga");
3959 if (!(vga_info
.flags
& VFIO_REGION_INFO_FLAG_READ
) ||
3960 !(vga_info
.flags
& VFIO_REGION_INFO_FLAG_WRITE
) ||
3961 vga_info
.size
< 0xbffff + 1) {
3962 error_report("vfio: Unexpected VGA info, flags 0x%lx, size 0x%lx",
3963 (unsigned long)vga_info
.flags
,
3964 (unsigned long)vga_info
.size
);
3968 vdev
->vga
.fd_offset
= vga_info
.offset
;
3969 vdev
->vga
.fd
= vdev
->fd
;
3971 vdev
->vga
.region
[QEMU_PCI_VGA_MEM
].offset
= QEMU_PCI_VGA_MEM_BASE
;
3972 vdev
->vga
.region
[QEMU_PCI_VGA_MEM
].nr
= QEMU_PCI_VGA_MEM
;
3973 QLIST_INIT(&vdev
->vga
.region
[QEMU_PCI_VGA_MEM
].quirks
);
3975 vdev
->vga
.region
[QEMU_PCI_VGA_IO_LO
].offset
= QEMU_PCI_VGA_IO_LO_BASE
;
3976 vdev
->vga
.region
[QEMU_PCI_VGA_IO_LO
].nr
= QEMU_PCI_VGA_IO_LO
;
3977 QLIST_INIT(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_LO
].quirks
);
3979 vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].offset
= QEMU_PCI_VGA_IO_HI_BASE
;
3980 vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].nr
= QEMU_PCI_VGA_IO_HI
;
3981 QLIST_INIT(&vdev
->vga
.region
[QEMU_PCI_VGA_IO_HI
].quirks
);
3983 vdev
->has_vga
= true;
3985 irq_info
.index
= VFIO_PCI_ERR_IRQ_INDEX
;
3987 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_GET_IRQ_INFO
, &irq_info
);
3989 /* This can fail for an old kernel or legacy PCI dev */
3990 DPRINTF("VFIO_DEVICE_GET_IRQ_INFO failure: %m\n");
3992 } else if (irq_info
.count
== 1) {
3993 vdev
->pci_aer
= true;
3995 error_report("vfio: %04x:%02x:%02x.%x "
3996 "Could not enable error recovery for the device",
3997 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
3998 vdev
->host
.function
);
4003 QLIST_REMOVE(vdev
, next
);
4010 static void vfio_put_device(VFIODevice
*vdev
)
4012 QLIST_REMOVE(vdev
, next
);
4014 DPRINTF("vfio_put_device: close vdev->fd\n");
4022 static void vfio_err_notifier_handler(void *opaque
)
4024 VFIODevice
*vdev
= opaque
;
4026 if (!event_notifier_test_and_clear(&vdev
->err_notifier
)) {
4031 * TBD. Retrieve the error details and decide what action
4032 * needs to be taken. One of the actions could be to pass
4033 * the error to the guest and have the guest driver recover
4034 * from the error. This requires that PCIe capabilities be
4035 * exposed to the guest. For now, we just terminate the
4036 * guest to contain the error.
4039 error_report("%s(%04x:%02x:%02x.%x) Unrecoverable error detected. "
4040 "Please collect any data possible and then kill the guest",
4041 __func__
, vdev
->host
.domain
, vdev
->host
.bus
,
4042 vdev
->host
.slot
, vdev
->host
.function
);
4044 vm_stop(RUN_STATE_IO_ERROR
);
4048 * Registers error notifier for devices supporting error recovery.
4049 * If we encounter a failure in this function, we report an error
4050 * and continue after disabling error recovery support for the
4053 static void vfio_register_err_notifier(VFIODevice
*vdev
)
4057 struct vfio_irq_set
*irq_set
;
4060 if (!vdev
->pci_aer
) {
4064 if (event_notifier_init(&vdev
->err_notifier
, 0)) {
4065 error_report("vfio: Unable to init event notifier for error detection");
4066 vdev
->pci_aer
= false;
4070 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
4072 irq_set
= g_malloc0(argsz
);
4073 irq_set
->argsz
= argsz
;
4074 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
|
4075 VFIO_IRQ_SET_ACTION_TRIGGER
;
4076 irq_set
->index
= VFIO_PCI_ERR_IRQ_INDEX
;
4079 pfd
= (int32_t *)&irq_set
->data
;
4081 *pfd
= event_notifier_get_fd(&vdev
->err_notifier
);
4082 qemu_set_fd_handler(*pfd
, vfio_err_notifier_handler
, NULL
, vdev
);
4084 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
4086 error_report("vfio: Failed to set up error notification");
4087 qemu_set_fd_handler(*pfd
, NULL
, NULL
, vdev
);
4088 event_notifier_cleanup(&vdev
->err_notifier
);
4089 vdev
->pci_aer
= false;
4094 static void vfio_unregister_err_notifier(VFIODevice
*vdev
)
4097 struct vfio_irq_set
*irq_set
;
4101 if (!vdev
->pci_aer
) {
4105 argsz
= sizeof(*irq_set
) + sizeof(*pfd
);
4107 irq_set
= g_malloc0(argsz
);
4108 irq_set
->argsz
= argsz
;
4109 irq_set
->flags
= VFIO_IRQ_SET_DATA_EVENTFD
|
4110 VFIO_IRQ_SET_ACTION_TRIGGER
;
4111 irq_set
->index
= VFIO_PCI_ERR_IRQ_INDEX
;
4114 pfd
= (int32_t *)&irq_set
->data
;
4117 ret
= ioctl(vdev
->fd
, VFIO_DEVICE_SET_IRQS
, irq_set
);
4119 error_report("vfio: Failed to de-assign error fd: %m");
4122 qemu_set_fd_handler(event_notifier_get_fd(&vdev
->err_notifier
),
4124 event_notifier_cleanup(&vdev
->err_notifier
);
4127 static int vfio_initfn(PCIDevice
*pdev
)
4129 VFIODevice
*pvdev
, *vdev
= DO_UPCAST(VFIODevice
, pdev
, pdev
);
4131 char path
[PATH_MAX
], iommu_group_path
[PATH_MAX
], *group_name
;
4137 /* Check that the host device exists */
4138 snprintf(path
, sizeof(path
),
4139 "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/",
4140 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
4141 vdev
->host
.function
);
4142 if (stat(path
, &st
) < 0) {
4143 error_report("vfio: error: no such host device: %s", path
);
4147 strncat(path
, "iommu_group", sizeof(path
) - strlen(path
) - 1);
4149 len
= readlink(path
, iommu_group_path
, sizeof(path
));
4150 if (len
<= 0 || len
>= sizeof(path
)) {
4151 error_report("vfio: error no iommu_group for device");
4152 return len
< 0 ? -errno
: ENAMETOOLONG
;
4155 iommu_group_path
[len
] = 0;
4156 group_name
= basename(iommu_group_path
);
4158 if (sscanf(group_name
, "%d", &groupid
) != 1) {
4159 error_report("vfio: error reading %s: %m", path
);
4163 DPRINTF("%s(%04x:%02x:%02x.%x) group %d\n", __func__
, vdev
->host
.domain
,
4164 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
, groupid
);
4166 group
= vfio_get_group(groupid
, pci_device_iommu_address_space(pdev
));
4168 error_report("vfio: failed to get group %d", groupid
);
4172 snprintf(path
, sizeof(path
), "%04x:%02x:%02x.%01x",
4173 vdev
->host
.domain
, vdev
->host
.bus
, vdev
->host
.slot
,
4174 vdev
->host
.function
);
4176 QLIST_FOREACH(pvdev
, &group
->device_list
, next
) {
4177 if (pvdev
->host
.domain
== vdev
->host
.domain
&&
4178 pvdev
->host
.bus
== vdev
->host
.bus
&&
4179 pvdev
->host
.slot
== vdev
->host
.slot
&&
4180 pvdev
->host
.function
== vdev
->host
.function
) {
4182 error_report("vfio: error: device %s is already attached", path
);
4183 vfio_put_group(group
);
4188 ret
= vfio_get_device(group
, path
, vdev
);
4190 error_report("vfio: failed to get device %s", path
);
4191 vfio_put_group(group
);
4195 /* Get a copy of config space */
4196 ret
= pread(vdev
->fd
, vdev
->pdev
.config
,
4197 MIN(pci_config_size(&vdev
->pdev
), vdev
->config_size
),
4198 vdev
->config_offset
);
4199 if (ret
< (int)MIN(pci_config_size(&vdev
->pdev
), vdev
->config_size
)) {
4200 ret
= ret
< 0 ? -errno
: -EFAULT
;
4201 error_report("vfio: Failed to read device config space");
4205 /* vfio emulates a lot for us, but some bits need extra love */
4206 vdev
->emulated_config_bits
= g_malloc0(vdev
->config_size
);
4208 /* QEMU can choose to expose the ROM or not */
4209 memset(vdev
->emulated_config_bits
+ PCI_ROM_ADDRESS
, 0xff, 4);
4211 /* QEMU can change multi-function devices to single function, or reverse */
4212 vdev
->emulated_config_bits
[PCI_HEADER_TYPE
] =
4213 PCI_HEADER_TYPE_MULTI_FUNCTION
;
4215 /* Restore or clear multifunction, this is always controlled by QEMU */
4216 if (vdev
->pdev
.cap_present
& QEMU_PCI_CAP_MULTIFUNCTION
) {
4217 vdev
->pdev
.config
[PCI_HEADER_TYPE
] |= PCI_HEADER_TYPE_MULTI_FUNCTION
;
4219 vdev
->pdev
.config
[PCI_HEADER_TYPE
] &= ~PCI_HEADER_TYPE_MULTI_FUNCTION
;
4223 * Clear host resource mapping info. If we choose not to register a
4224 * BAR, such as might be the case with the option ROM, we can get
4225 * confusing, unwritable, residual addresses from the host here.
4227 memset(&vdev
->pdev
.config
[PCI_BASE_ADDRESS_0
], 0, 24);
4228 memset(&vdev
->pdev
.config
[PCI_ROM_ADDRESS
], 0, 4);
4230 vfio_pci_size_rom(vdev
);
4232 ret
= vfio_early_setup_msix(vdev
);
4237 vfio_map_bars(vdev
);
4239 ret
= vfio_add_capabilities(vdev
);
4244 /* QEMU emulates all of MSI & MSIX */
4245 if (pdev
->cap_present
& QEMU_PCI_CAP_MSIX
) {
4246 memset(vdev
->emulated_config_bits
+ pdev
->msix_cap
, 0xff,
4250 if (pdev
->cap_present
& QEMU_PCI_CAP_MSI
) {
4251 memset(vdev
->emulated_config_bits
+ pdev
->msi_cap
, 0xff,
4252 vdev
->msi_cap_size
);
4255 if (vfio_pci_read_config(&vdev
->pdev
, PCI_INTERRUPT_PIN
, 1)) {
4256 vdev
->intx
.mmap_timer
= timer_new_ms(QEMU_CLOCK_VIRTUAL
,
4257 vfio_intx_mmap_enable
, vdev
);
4258 pci_device_set_intx_routing_notifier(&vdev
->pdev
, vfio_update_irq
);
4259 ret
= vfio_enable_intx(vdev
);
4265 add_boot_device_path(vdev
->bootindex
, &pdev
->qdev
, NULL
);
4266 vfio_register_err_notifier(vdev
);
4271 pci_device_set_intx_routing_notifier(&vdev
->pdev
, NULL
);
4272 vfio_teardown_msi(vdev
);
4273 vfio_unmap_bars(vdev
);
4275 g_free(vdev
->emulated_config_bits
);
4276 vfio_put_device(vdev
);
4277 vfio_put_group(group
);
4281 static void vfio_exitfn(PCIDevice
*pdev
)
4283 VFIODevice
*vdev
= DO_UPCAST(VFIODevice
, pdev
, pdev
);
4284 VFIOGroup
*group
= vdev
->group
;
4286 vfio_unregister_err_notifier(vdev
);
4287 pci_device_set_intx_routing_notifier(&vdev
->pdev
, NULL
);
4288 vfio_disable_interrupts(vdev
);
4289 if (vdev
->intx
.mmap_timer
) {
4290 timer_free(vdev
->intx
.mmap_timer
);
4292 vfio_teardown_msi(vdev
);
4293 vfio_unmap_bars(vdev
);
4294 g_free(vdev
->emulated_config_bits
);
4296 vfio_put_device(vdev
);
4297 vfio_put_group(group
);
4300 static void vfio_pci_reset(DeviceState
*dev
)
4302 PCIDevice
*pdev
= DO_UPCAST(PCIDevice
, qdev
, dev
);
4303 VFIODevice
*vdev
= DO_UPCAST(VFIODevice
, pdev
, pdev
);
4305 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__
, vdev
->host
.domain
,
4306 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
4308 vfio_pci_pre_reset(vdev
);
4310 if (vdev
->reset_works
&& (vdev
->has_flr
|| !vdev
->has_pm_reset
) &&
4311 !ioctl(vdev
->fd
, VFIO_DEVICE_RESET
)) {
4312 DPRINTF("%04x:%02x:%02x.%x FLR/VFIO_DEVICE_RESET\n", vdev
->host
.domain
,
4313 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
4317 /* See if we can do our own bus reset */
4318 if (!vfio_pci_hot_reset_one(vdev
)) {
4322 /* If nothing else works and the device supports PM reset, use it */
4323 if (vdev
->reset_works
&& vdev
->has_pm_reset
&&
4324 !ioctl(vdev
->fd
, VFIO_DEVICE_RESET
)) {
4325 DPRINTF("%04x:%02x:%02x.%x PCI PM Reset\n", vdev
->host
.domain
,
4326 vdev
->host
.bus
, vdev
->host
.slot
, vdev
->host
.function
);
4331 vfio_pci_post_reset(vdev
);
4334 static Property vfio_pci_dev_properties
[] = {
4335 DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIODevice
, host
),
4336 DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIODevice
,
4337 intx
.mmap_timeout
, 1100),
4338 DEFINE_PROP_BIT("x-vga", VFIODevice
, features
,
4339 VFIO_FEATURE_ENABLE_VGA_BIT
, false),
4340 DEFINE_PROP_INT32("bootindex", VFIODevice
, bootindex
, -1),
4342 * TODO - support passed fds... is this necessary?
4343 * DEFINE_PROP_STRING("vfiofd", VFIODevice, vfiofd_name),
4344 * DEFINE_PROP_STRING("vfiogroupfd, VFIODevice, vfiogroupfd_name),
4346 DEFINE_PROP_END_OF_LIST(),
4349 static const VMStateDescription vfio_pci_vmstate
= {
4354 static void vfio_pci_dev_class_init(ObjectClass
*klass
, void *data
)
4356 DeviceClass
*dc
= DEVICE_CLASS(klass
);
4357 PCIDeviceClass
*pdc
= PCI_DEVICE_CLASS(klass
);
4359 dc
->reset
= vfio_pci_reset
;
4360 dc
->props
= vfio_pci_dev_properties
;
4361 dc
->vmsd
= &vfio_pci_vmstate
;
4362 dc
->desc
= "VFIO-based PCI device assignment";
4363 set_bit(DEVICE_CATEGORY_MISC
, dc
->categories
);
4364 pdc
->init
= vfio_initfn
;
4365 pdc
->exit
= vfio_exitfn
;
4366 pdc
->config_read
= vfio_pci_read_config
;
4367 pdc
->config_write
= vfio_pci_write_config
;
4368 pdc
->is_express
= 1; /* We might be */
4371 static const TypeInfo vfio_pci_dev_info
= {
4373 .parent
= TYPE_PCI_DEVICE
,
4374 .instance_size
= sizeof(VFIODevice
),
4375 .class_init
= vfio_pci_dev_class_init
,
4378 static void register_vfio_pci_dev_type(void)
4380 type_register_static(&vfio_pci_dev_info
);
4383 type_init(register_vfio_pci_dev_type
)
4385 static int vfio_container_do_ioctl(AddressSpace
*as
, int32_t groupid
,
4386 int req
, void *param
)
4389 VFIOContainer
*container
;
4392 group
= vfio_get_group(groupid
, as
);
4394 error_report("vfio: group %d not registered", groupid
);
4398 container
= group
->container
;
4399 if (group
->container
) {
4400 ret
= ioctl(container
->fd
, req
, param
);
4402 error_report("vfio: failed to ioctl container: ret=%d, %s",
4403 ret
, strerror(errno
));
4407 vfio_put_group(group
);
4412 int vfio_container_ioctl(AddressSpace
*as
, int32_t groupid
,
4413 int req
, void *param
)
4415 /* We allow only certain ioctls to the container */
4417 case VFIO_CHECK_EXTENSION
:
4418 case VFIO_IOMMU_SPAPR_TCE_GET_INFO
:
4421 /* Return an error on unknown requests */
4422 error_report("vfio: unsupported ioctl %X", req
);
4426 return vfio_container_do_ioctl(as
, groupid
, req
, param
);