?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
qga: Don't require 'time' argument in guest-set-time command
[qemu.git] / hw / misc / vfio.c
blob8db182fa3d79b9f36d77d684163766d9bc041f4f
1 /*
2 * vfio based device assignment support
3 *
4 * Copyright Red Hat, Inc. 2012
5 *
6 * Authors:
7 * Alex Williamson <alex.williamson@redhat.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
11 *
12 * Based on qemu-kvm device-assignment:
13 * Adapted for KVM by Qumranet.
14 * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com)
15 * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com)
16 * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com)
17 * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
18 * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
19 */
20
21 #include <dirent.h>
22 #include <linux/vfio.h>
23 #include <sys/ioctl.h>
24 #include <sys/mman.h>
25 #include <sys/stat.h>
26 #include <sys/types.h>
27 #include <unistd.h>
28
29 #include "config.h"
30 #include "exec/address-spaces.h"
31 #include "exec/memory.h"
32 #include "hw/pci/msi.h"
33 #include "hw/pci/msix.h"
34 #include "hw/pci/pci.h"
35 #include "qemu-common.h"
36 #include "qemu/error-report.h"
37 #include "qemu/event_notifier.h"
38 #include "qemu/queue.h"
39 #include "qemu/range.h"
40 #include "sysemu/kvm.h"
41 #include "sysemu/sysemu.h"
42
43 /* #define DEBUG_VFIO */
44 #ifdef DEBUG_VFIO
45 #define DPRINTF(fmt, ...) \
46 do { fprintf(stderr, "vfio: " fmt, ## __VA_ARGS__); } while (0)
47 #else
48 #define DPRINTF(fmt, ...) \
49 do { } while (0)
50 #endif
51
52 /* Extra debugging, trap acceleration paths for more logging */
53 #define VFIO_ALLOW_MMAP 1
54 #define VFIO_ALLOW_KVM_INTX 1
55 #define VFIO_ALLOW_KVM_MSI 1
56 #define VFIO_ALLOW_KVM_MSIX 1
57
58 struct VFIODevice;
59
60 typedef struct VFIOQuirk {
61 MemoryRegion mem;
62 struct VFIODevice *vdev;
63 QLIST_ENTRY(VFIOQuirk) next;
64 struct {
65 uint32_t base_offset:TARGET_PAGE_BITS;
66 uint32_t address_offset:TARGET_PAGE_BITS;
67 uint32_t address_size:3;
68 uint32_t bar:3;
69
70 uint32_t address_match;
71 uint32_t address_mask;
72
73 uint32_t address_val:TARGET_PAGE_BITS;
74 uint32_t data_offset:TARGET_PAGE_BITS;
75 uint32_t data_size:3;
76
77 uint8_t flags;
78 uint8_t read_flags;
79 uint8_t write_flags;
80 } data;
81 } VFIOQuirk;
82
83 typedef struct VFIOBAR {
84 off_t fd_offset; /* offset of BAR within device fd */
85 int fd; /* device fd, allows us to pass VFIOBAR as opaque data */
86 MemoryRegion mem; /* slow, read/write access */
87 MemoryRegion mmap_mem; /* direct mapped access */
88 void *mmap;
89 size_t size;
90 uint32_t flags; /* VFIO region flags (rd/wr/mmap) */
91 uint8_t nr; /* cache the BAR number for debug */
92 bool ioport;
93 bool mem64;
94 QLIST_HEAD(, VFIOQuirk) quirks;
95 } VFIOBAR;
96
97 typedef struct VFIOVGARegion {
98 MemoryRegion mem;
99 off_t offset;
100 int nr;
101 QLIST_HEAD(, VFIOQuirk) quirks;
102 } VFIOVGARegion;
103
104 typedef struct VFIOVGA {
105 off_t fd_offset;
106 int fd;
107 VFIOVGARegion region[QEMU_PCI_VGA_NUM_REGIONS];
108 } VFIOVGA;
109
110 typedef struct VFIOINTx {
111 bool pending; /* interrupt pending */
112 bool kvm_accel; /* set when QEMU bypass through KVM enabled */
113 uint8_t pin; /* which pin to pull for qemu_set_irq */
114 EventNotifier interrupt; /* eventfd triggered on interrupt */
115 EventNotifier unmask; /* eventfd for unmask on QEMU bypass */
116 PCIINTxRoute route; /* routing info for QEMU bypass */
117 uint32_t mmap_timeout; /* delay to re-enable mmaps after interrupt */
118 QEMUTimer *mmap_timer; /* enable mmaps after periods w/o interrupts */
119 } VFIOINTx;
120
121 typedef struct VFIOMSIVector {
122 EventNotifier interrupt; /* eventfd triggered on interrupt */
123 struct VFIODevice *vdev; /* back pointer to device */
124 MSIMessage msg; /* cache the MSI message so we know when it changes */
125 int virq; /* KVM irqchip route for QEMU bypass */
126 bool use;
127 } VFIOMSIVector;
128
129 enum {
130 VFIO_INT_NONE = 0,
131 VFIO_INT_INTx = 1,
132 VFIO_INT_MSI = 2,
133 VFIO_INT_MSIX = 3,
134 };
135
136 struct VFIOGroup;
137
138 typedef struct VFIOType1 {
139 MemoryListener listener;
140 int error;
141 bool initialized;
142 } VFIOType1;
143
144 typedef struct VFIOContainer {
145 int fd; /* /dev/vfio/vfio, empowered by the attached groups */
146 struct {
147 /* enable abstraction to support various iommu backends */
148 union {
149 VFIOType1 type1;
150 };
151 void (*release)(struct VFIOContainer *);
152 } iommu_data;
153 QLIST_HEAD(, VFIOGroup) group_list;
154 QLIST_ENTRY(VFIOContainer) next;
155 } VFIOContainer;
156
157 /* Cache of MSI-X setup plus extra mmap and memory region for split BAR map */
158 typedef struct VFIOMSIXInfo {
159 uint8_t table_bar;
160 uint8_t pba_bar;
161 uint16_t entries;
162 uint32_t table_offset;
163 uint32_t pba_offset;
164 MemoryRegion mmap_mem;
165 void *mmap;
166 } VFIOMSIXInfo;
167
168 typedef struct VFIODevice {
169 PCIDevice pdev;
170 int fd;
171 VFIOINTx intx;
172 unsigned int config_size;
173 uint8_t *emulated_config_bits; /* QEMU emulated bits, little-endian */
174 off_t config_offset; /* Offset of config space region within device fd */
175 unsigned int rom_size;
176 off_t rom_offset; /* Offset of ROM region within device fd */
177 void *rom;
178 int msi_cap_size;
179 VFIOMSIVector *msi_vectors;
180 VFIOMSIXInfo *msix;
181 int nr_vectors; /* Number of MSI/MSIX vectors currently in use */
182 int interrupt; /* Current interrupt type */
183 VFIOBAR bars[PCI_NUM_REGIONS - 1]; /* No ROM */
184 VFIOVGA vga; /* 0xa0000, 0x3b0, 0x3c0 */
185 PCIHostDeviceAddress host;
186 QLIST_ENTRY(VFIODevice) next;
187 struct VFIOGroup *group;
188 EventNotifier err_notifier;
189 uint32_t features;
190 #define VFIO_FEATURE_ENABLE_VGA_BIT 0
191 #define VFIO_FEATURE_ENABLE_VGA (1 << VFIO_FEATURE_ENABLE_VGA_BIT)
192 int32_t bootindex;
193 uint8_t pm_cap;
194 bool reset_works;
195 bool has_vga;
196 bool pci_aer;
197 bool has_flr;
198 bool has_pm_reset;
199 bool needs_reset;
200 bool rom_read_failed;
201 } VFIODevice;
202
203 typedef struct VFIOGroup {
204 int fd;
205 int groupid;
206 VFIOContainer *container;
207 QLIST_HEAD(, VFIODevice) device_list;
208 QLIST_ENTRY(VFIOGroup) next;
209 QLIST_ENTRY(VFIOGroup) container_next;
210 } VFIOGroup;
211
212 #define MSIX_CAP_LENGTH 12
213
214 static QLIST_HEAD(, VFIOContainer)
215 container_list = QLIST_HEAD_INITIALIZER(container_list);
216
217 static QLIST_HEAD(, VFIOGroup)
218 group_list = QLIST_HEAD_INITIALIZER(group_list);
219
220 #ifdef CONFIG_KVM
221 /*
222 * We have a single VFIO pseudo device per KVM VM. Once created it lives
223 * for the life of the VM. Closing the file descriptor only drops our
224 * reference to it and the device's reference to kvm. Therefore once
225 * initialized, this file descriptor is only released on QEMU exit and
226 * we'll re-use it should another vfio device be attached before then.
227 */
228 static int vfio_kvm_device_fd = -1;
229 #endif
230
231 static void vfio_disable_interrupts(VFIODevice *vdev);
232 static uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len);
233 static void vfio_pci_write_config(PCIDevice *pdev, uint32_t addr,
234 uint32_t val, int len);
235 static void vfio_mmap_set_enabled(VFIODevice *vdev, bool enabled);
236
237 /*
238 * Common VFIO interrupt disable
239 */
240 static void vfio_disable_irqindex(VFIODevice *vdev, int index)
241 {
242 struct vfio_irq_set irq_set = {
243 .argsz = sizeof(irq_set),
244 .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER,
245 .index = index,
246 .start = 0,
247 .count = 0,
248 };
249
250 ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &irq_set);
251 }
252
253 /*
254 * INTx
255 */
256 static void vfio_unmask_intx(VFIODevice *vdev)
257 {
258 struct vfio_irq_set irq_set = {
259 .argsz = sizeof(irq_set),
260 .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_UNMASK,
261 .index = VFIO_PCI_INTX_IRQ_INDEX,
262 .start = 0,
263 .count = 1,
264 };
265
266 ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &irq_set);
267 }
268
269 #ifdef CONFIG_KVM /* Unused outside of CONFIG_KVM code */
270 static void vfio_mask_intx(VFIODevice *vdev)
271 {
272 struct vfio_irq_set irq_set = {
273 .argsz = sizeof(irq_set),
274 .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_MASK,
275 .index = VFIO_PCI_INTX_IRQ_INDEX,
276 .start = 0,
277 .count = 1,
278 };
279
280 ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &irq_set);
281 }
282 #endif
283
284 /*
285 * Disabling BAR mmaping can be slow, but toggling it around INTx can
286 * also be a huge overhead. We try to get the best of both worlds by
287 * waiting until an interrupt to disable mmaps (subsequent transitions
288 * to the same state are effectively no overhead). If the interrupt has
289 * been serviced and the time gap is long enough, we re-enable mmaps for
290 * performance. This works well for things like graphics cards, which
291 * may not use their interrupt at all and are penalized to an unusable
292 * level by read/write BAR traps. Other devices, like NICs, have more
293 * regular interrupts and see much better latency by staying in non-mmap
294 * mode. We therefore set the default mmap_timeout such that a ping
295 * is just enough to keep the mmap disabled. Users can experiment with
296 * other options with the x-intx-mmap-timeout-ms parameter (a value of
297 * zero disables the timer).
298 */
299 static void vfio_intx_mmap_enable(void *opaque)
300 {
301 VFIODevice *vdev = opaque;
302
303 if (vdev->intx.pending) {
304 timer_mod(vdev->intx.mmap_timer,
305 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout);
306 return;
307 }
308
309 vfio_mmap_set_enabled(vdev, true);
310 }
311
312 static void vfio_intx_interrupt(void *opaque)
313 {
314 VFIODevice *vdev = opaque;
315
316 if (!event_notifier_test_and_clear(&vdev->intx.interrupt)) {
317 return;
318 }
319
320 DPRINTF("%s(%04x:%02x:%02x.%x) Pin %c\n", __func__, vdev->host.domain,
321 vdev->host.bus, vdev->host.slot, vdev->host.function,
322 'A' + vdev->intx.pin);
323
324 vdev->intx.pending = true;
325 pci_irq_assert(&vdev->pdev);
326 vfio_mmap_set_enabled(vdev, false);
327 if (vdev->intx.mmap_timeout) {
328 timer_mod(vdev->intx.mmap_timer,
329 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout);
330 }
331 }
332
333 static void vfio_eoi(VFIODevice *vdev)
334 {
335 if (!vdev->intx.pending) {
336 return;
337 }
338
339 DPRINTF("%s(%04x:%02x:%02x.%x) EOI\n", __func__, vdev->host.domain,
340 vdev->host.bus, vdev->host.slot, vdev->host.function);
341
342 vdev->intx.pending = false;
343 pci_irq_deassert(&vdev->pdev);
344 vfio_unmask_intx(vdev);
345 }
346
347 static void vfio_enable_intx_kvm(VFIODevice *vdev)
348 {
349 #ifdef CONFIG_KVM
350 struct kvm_irqfd irqfd = {
351 .fd = event_notifier_get_fd(&vdev->intx.interrupt),
352 .gsi = vdev->intx.route.irq,
353 .flags = KVM_IRQFD_FLAG_RESAMPLE,
354 };
355 struct vfio_irq_set *irq_set;
356 int ret, argsz;
357 int32_t *pfd;
358
359 if (!VFIO_ALLOW_KVM_INTX || !kvm_irqfds_enabled() ||
360 vdev->intx.route.mode != PCI_INTX_ENABLED ||
361 !kvm_check_extension(kvm_state, KVM_CAP_IRQFD_RESAMPLE)) {
362 return;
363 }
364
365 /* Get to a known interrupt state */
366 qemu_set_fd_handler(irqfd.fd, NULL, NULL, vdev);
367 vfio_mask_intx(vdev);
368 vdev->intx.pending = false;
369 pci_irq_deassert(&vdev->pdev);
370
371 /* Get an eventfd for resample/unmask */
372 if (event_notifier_init(&vdev->intx.unmask, 0)) {
373 error_report("vfio: Error: event_notifier_init failed eoi");
374 goto fail;
375 }
376
377 /* KVM triggers it, VFIO listens for it */
378 irqfd.resamplefd = event_notifier_get_fd(&vdev->intx.unmask);
379
380 if (kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd)) {
381 error_report("vfio: Error: Failed to setup resample irqfd: %m");
382 goto fail_irqfd;
383 }
384
385 argsz = sizeof(*irq_set) + sizeof(*pfd);
386
387 irq_set = g_malloc0(argsz);
388 irq_set->argsz = argsz;
389 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_UNMASK;
390 irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
391 irq_set->start = 0;
392 irq_set->count = 1;
393 pfd = (int32_t *)&irq_set->data;
394
395 *pfd = irqfd.resamplefd;
396
397 ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
398 g_free(irq_set);
399 if (ret) {
400 error_report("vfio: Error: Failed to setup INTx unmask fd: %m");
401 goto fail_vfio;
402 }
403
404 /* Let'em rip */
405 vfio_unmask_intx(vdev);
406
407 vdev->intx.kvm_accel = true;
408
409 DPRINTF("%s(%04x:%02x:%02x.%x) KVM INTx accel enabled\n",
410 __func__, vdev->host.domain, vdev->host.bus,
411 vdev->host.slot, vdev->host.function);
412
413 return;
414
415 fail_vfio:
416 irqfd.flags = KVM_IRQFD_FLAG_DEASSIGN;
417 kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd);
418 fail_irqfd:
419 event_notifier_cleanup(&vdev->intx.unmask);
420 fail:
421 qemu_set_fd_handler(irqfd.fd, vfio_intx_interrupt, NULL, vdev);
422 vfio_unmask_intx(vdev);
423 #endif
424 }
425
426 static void vfio_disable_intx_kvm(VFIODevice *vdev)
427 {
428 #ifdef CONFIG_KVM
429 struct kvm_irqfd irqfd = {
430 .fd = event_notifier_get_fd(&vdev->intx.interrupt),
431 .gsi = vdev->intx.route.irq,
432 .flags = KVM_IRQFD_FLAG_DEASSIGN,
433 };
434
435 if (!vdev->intx.kvm_accel) {
436 return;
437 }
438
439 /*
440 * Get to a known state, hardware masked, QEMU ready to accept new
441 * interrupts, QEMU IRQ de-asserted.
442 */
443 vfio_mask_intx(vdev);
444 vdev->intx.pending = false;
445 pci_irq_deassert(&vdev->pdev);
446
447 /* Tell KVM to stop listening for an INTx irqfd */
448 if (kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd)) {
449 error_report("vfio: Error: Failed to disable INTx irqfd: %m");
450 }
451
452 /* We only need to close the eventfd for VFIO to cleanup the kernel side */
453 event_notifier_cleanup(&vdev->intx.unmask);
454
455 /* QEMU starts listening for interrupt events. */
456 qemu_set_fd_handler(irqfd.fd, vfio_intx_interrupt, NULL, vdev);
457
458 vdev->intx.kvm_accel = false;
459
460 /* If we've missed an event, let it re-fire through QEMU */
461 vfio_unmask_intx(vdev);
462
463 DPRINTF("%s(%04x:%02x:%02x.%x) KVM INTx accel disabled\n",
464 __func__, vdev->host.domain, vdev->host.bus,
465 vdev->host.slot, vdev->host.function);
466 #endif
467 }
468
469 static void vfio_update_irq(PCIDevice *pdev)
470 {
471 VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
472 PCIINTxRoute route;
473
474 if (vdev->interrupt != VFIO_INT_INTx) {
475 return;
476 }
477
478 route = pci_device_route_intx_to_irq(&vdev->pdev, vdev->intx.pin);
479
480 if (!pci_intx_route_changed(&vdev->intx.route, &route)) {
481 return; /* Nothing changed */
482 }
483
484 DPRINTF("%s(%04x:%02x:%02x.%x) IRQ moved %d -> %d\n", __func__,
485 vdev->host.domain, vdev->host.bus, vdev->host.slot,
486 vdev->host.function, vdev->intx.route.irq, route.irq);
487
488 vfio_disable_intx_kvm(vdev);
489
490 vdev->intx.route = route;
491
492 if (route.mode != PCI_INTX_ENABLED) {
493 return;
494 }
495
496 vfio_enable_intx_kvm(vdev);
497
498 /* Re-enable the interrupt in cased we missed an EOI */
499 vfio_eoi(vdev);
500 }
501
502 static int vfio_enable_intx(VFIODevice *vdev)
503 {
504 uint8_t pin = vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1);
505 int ret, argsz;
506 struct vfio_irq_set *irq_set;
507 int32_t *pfd;
508
509 if (!pin) {
510 return 0;
511 }
512
513 vfio_disable_interrupts(vdev);
514
515 vdev->intx.pin = pin - 1; /* Pin A (1) -> irq[0] */
516 pci_config_set_interrupt_pin(vdev->pdev.config, pin);
517
518 #ifdef CONFIG_KVM
519 /*
520 * Only conditional to avoid generating error messages on platforms
521 * where we won't actually use the result anyway.
522 */
523 if (kvm_irqfds_enabled() &&
524 kvm_check_extension(kvm_state, KVM_CAP_IRQFD_RESAMPLE)) {
525 vdev->intx.route = pci_device_route_intx_to_irq(&vdev->pdev,
526 vdev->intx.pin);
527 }
528 #endif
529
530 ret = event_notifier_init(&vdev->intx.interrupt, 0);
531 if (ret) {
532 error_report("vfio: Error: event_notifier_init failed");
533 return ret;
534 }
535
536 argsz = sizeof(*irq_set) + sizeof(*pfd);
537
538 irq_set = g_malloc0(argsz);
539 irq_set->argsz = argsz;
540 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
541 irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
542 irq_set->start = 0;
543 irq_set->count = 1;
544 pfd = (int32_t *)&irq_set->data;
545
546 *pfd = event_notifier_get_fd(&vdev->intx.interrupt);
547 qemu_set_fd_handler(*pfd, vfio_intx_interrupt, NULL, vdev);
548
549 ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
550 g_free(irq_set);
551 if (ret) {
552 error_report("vfio: Error: Failed to setup INTx fd: %m");
553 qemu_set_fd_handler(*pfd, NULL, NULL, vdev);
554 event_notifier_cleanup(&vdev->intx.interrupt);
555 return -errno;
556 }
557
558 vfio_enable_intx_kvm(vdev);
559
560 vdev->interrupt = VFIO_INT_INTx;
561
562 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
563 vdev->host.bus, vdev->host.slot, vdev->host.function);
564
565 return 0;
566 }
567
568 static void vfio_disable_intx(VFIODevice *vdev)
569 {
570 int fd;
571
572 timer_del(vdev->intx.mmap_timer);
573 vfio_disable_intx_kvm(vdev);
574 vfio_disable_irqindex(vdev, VFIO_PCI_INTX_IRQ_INDEX);
575 vdev->intx.pending = false;
576 pci_irq_deassert(&vdev->pdev);
577 vfio_mmap_set_enabled(vdev, true);
578
579 fd = event_notifier_get_fd(&vdev->intx.interrupt);
580 qemu_set_fd_handler(fd, NULL, NULL, vdev);
581 event_notifier_cleanup(&vdev->intx.interrupt);
582
583 vdev->interrupt = VFIO_INT_NONE;
584
585 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
586 vdev->host.bus, vdev->host.slot, vdev->host.function);
587 }
588
589 /*
590 * MSI/X
591 */
592 static void vfio_msi_interrupt(void *opaque)
593 {
594 VFIOMSIVector *vector = opaque;
595 VFIODevice *vdev = vector->vdev;
596 int nr = vector - vdev->msi_vectors;
597
598 if (!event_notifier_test_and_clear(&vector->interrupt)) {
599 return;
600 }
601
602 #ifdef DEBUG_VFIO
603 MSIMessage msg;
604
605 if (vdev->interrupt == VFIO_INT_MSIX) {
606 msg = msi_get_message(&vdev->pdev, nr);
607 } else if (vdev->interrupt == VFIO_INT_MSI) {
608 msg = msix_get_message(&vdev->pdev, nr);
609 } else {
610 abort();
611 }
612
613 DPRINTF("%s(%04x:%02x:%02x.%x) vector %d 0x%"PRIx64"/0x%x\n", __func__,
614 vdev->host.domain, vdev->host.bus, vdev->host.slot,
615 vdev->host.function, nr, msg.address, msg.data);
616 #endif
617
618 if (vdev->interrupt == VFIO_INT_MSIX) {
619 msix_notify(&vdev->pdev, nr);
620 } else if (vdev->interrupt == VFIO_INT_MSI) {
621 msi_notify(&vdev->pdev, nr);
622 } else {
623 error_report("vfio: MSI interrupt receieved, but not enabled?");
624 }
625 }
626
627 static int vfio_enable_vectors(VFIODevice *vdev, bool msix)
628 {
629 struct vfio_irq_set *irq_set;
630 int ret = 0, i, argsz;
631 int32_t *fds;
632
633 argsz = sizeof(*irq_set) + (vdev->nr_vectors * sizeof(*fds));
634
635 irq_set = g_malloc0(argsz);
636 irq_set->argsz = argsz;
637 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
638 irq_set->index = msix ? VFIO_PCI_MSIX_IRQ_INDEX : VFIO_PCI_MSI_IRQ_INDEX;
639 irq_set->start = 0;
640 irq_set->count = vdev->nr_vectors;
641 fds = (int32_t *)&irq_set->data;
642
643 for (i = 0; i < vdev->nr_vectors; i++) {
644 if (!vdev->msi_vectors[i].use) {
645 fds[i] = -1;
646 continue;
647 }
648
649 fds[i] = event_notifier_get_fd(&vdev->msi_vectors[i].interrupt);
650 }
651
652 ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
653
654 g_free(irq_set);
655
656 return ret;
657 }
658
659 static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
660 MSIMessage *msg, IOHandler *handler)
661 {
662 VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
663 VFIOMSIVector *vector;
664 int ret;
665
666 DPRINTF("%s(%04x:%02x:%02x.%x) vector %d used\n", __func__,
667 vdev->host.domain, vdev->host.bus, vdev->host.slot,
668 vdev->host.function, nr);
669
670 vector = &vdev->msi_vectors[nr];
671 vector->vdev = vdev;
672 vector->use = true;
673
674 msix_vector_use(pdev, nr);
675
676 if (event_notifier_init(&vector->interrupt, 0)) {
677 error_report("vfio: Error: event_notifier_init failed");
678 }
679
680 /*
681 * Attempt to enable route through KVM irqchip,
682 * default to userspace handling if unavailable.
683 */
684 vector->virq = msg && VFIO_ALLOW_KVM_MSIX ?
685 kvm_irqchip_add_msi_route(kvm_state, *msg) : -1;
686 if (vector->virq < 0 ||
687 kvm_irqchip_add_irqfd_notifier(kvm_state, &vector->interrupt,
688 NULL, vector->virq) < 0) {
689 if (vector->virq >= 0) {
690 kvm_irqchip_release_virq(kvm_state, vector->virq);
691 vector->virq = -1;
692 }
693 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
694 handler, NULL, vector);
695 }
696
697 /*
698 * We don't want to have the host allocate all possible MSI vectors
699 * for a device if they're not in use, so we shutdown and incrementally
700 * increase them as needed.
701 */
702 if (vdev->nr_vectors < nr + 1) {
703 vfio_disable_irqindex(vdev, VFIO_PCI_MSIX_IRQ_INDEX);
704 vdev->nr_vectors = nr + 1;
705 ret = vfio_enable_vectors(vdev, true);
706 if (ret) {
707 error_report("vfio: failed to enable vectors, %d", ret);
708 }
709 } else {
710 int argsz;
711 struct vfio_irq_set *irq_set;
712 int32_t *pfd;
713
714 argsz = sizeof(*irq_set) + sizeof(*pfd);
715
716 irq_set = g_malloc0(argsz);
717 irq_set->argsz = argsz;
718 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
719 VFIO_IRQ_SET_ACTION_TRIGGER;
720 irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
721 irq_set->start = nr;
722 irq_set->count = 1;
723 pfd = (int32_t *)&irq_set->data;
724
725 *pfd = event_notifier_get_fd(&vector->interrupt);
726
727 ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
728 g_free(irq_set);
729 if (ret) {
730 error_report("vfio: failed to modify vector, %d", ret);
731 }
732 }
733
734 return 0;
735 }
736
737 static int vfio_msix_vector_use(PCIDevice *pdev,
738 unsigned int nr, MSIMessage msg)
739 {
740 return vfio_msix_vector_do_use(pdev, nr, &msg, vfio_msi_interrupt);
741 }
742
743 static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr)
744 {
745 VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
746 VFIOMSIVector *vector = &vdev->msi_vectors[nr];
747 int argsz;
748 struct vfio_irq_set *irq_set;
749 int32_t *pfd;
750
751 DPRINTF("%s(%04x:%02x:%02x.%x) vector %d released\n", __func__,
752 vdev->host.domain, vdev->host.bus, vdev->host.slot,
753 vdev->host.function, nr);
754
755 /*
756 * XXX What's the right thing to do here? This turns off the interrupt
757 * completely, but do we really just want to switch the interrupt to
758 * bouncing through userspace and let msix.c drop it? Not sure.
759 */
760 msix_vector_unuse(pdev, nr);
761
762 argsz = sizeof(*irq_set) + sizeof(*pfd);
763
764 irq_set = g_malloc0(argsz);
765 irq_set->argsz = argsz;
766 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
767 VFIO_IRQ_SET_ACTION_TRIGGER;
768 irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
769 irq_set->start = nr;
770 irq_set->count = 1;
771 pfd = (int32_t *)&irq_set->data;
772
773 *pfd = -1;
774
775 ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
776
777 g_free(irq_set);
778
779 if (vector->virq < 0) {
780 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
781 NULL, NULL, NULL);
782 } else {
783 kvm_irqchip_remove_irqfd_notifier(kvm_state, &vector->interrupt,
784 vector->virq);
785 kvm_irqchip_release_virq(kvm_state, vector->virq);
786 vector->virq = -1;
787 }
788
789 event_notifier_cleanup(&vector->interrupt);
790 vector->use = false;
791 }
792
793 static void vfio_enable_msix(VFIODevice *vdev)
794 {
795 vfio_disable_interrupts(vdev);
796
797 vdev->msi_vectors = g_malloc0(vdev->msix->entries * sizeof(VFIOMSIVector));
798
799 vdev->interrupt = VFIO_INT_MSIX;
800
801 /*
802 * Some communication channels between VF & PF or PF & fw rely on the
803 * physical state of the device and expect that enabling MSI-X from the
804 * guest enables the same on the host. When our guest is Linux, the
805 * guest driver call to pci_enable_msix() sets the enabling bit in the
806 * MSI-X capability, but leaves the vector table masked. We therefore
807 * can't rely on a vector_use callback (from request_irq() in the guest)
808 * to switch the physical device into MSI-X mode because that may come a
809 * long time after pci_enable_msix(). This code enables vector 0 with
810 * triggering to userspace, then immediately release the vector, leaving
811 * the physical device with no vectors enabled, but MSI-X enabled, just
812 * like the guest view.
813 */
814 vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL);
815 vfio_msix_vector_release(&vdev->pdev, 0);
816
817 if (msix_set_vector_notifiers(&vdev->pdev, vfio_msix_vector_use,
818 vfio_msix_vector_release, NULL)) {
819 error_report("vfio: msix_set_vector_notifiers failed");
820 }
821
822 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
823 vdev->host.bus, vdev->host.slot, vdev->host.function);
824 }
825
826 static void vfio_enable_msi(VFIODevice *vdev)
827 {
828 int ret, i;
829
830 vfio_disable_interrupts(vdev);
831
832 vdev->nr_vectors = msi_nr_vectors_allocated(&vdev->pdev);
833 retry:
834 vdev->msi_vectors = g_malloc0(vdev->nr_vectors * sizeof(VFIOMSIVector));
835
836 for (i = 0; i < vdev->nr_vectors; i++) {
837 VFIOMSIVector *vector = &vdev->msi_vectors[i];
838
839 vector->vdev = vdev;
840 vector->use = true;
841
842 if (event_notifier_init(&vector->interrupt, 0)) {
843 error_report("vfio: Error: event_notifier_init failed");
844 }
845
846 vector->msg = msi_get_message(&vdev->pdev, i);
847
848 /*
849 * Attempt to enable route through KVM irqchip,
850 * default to userspace handling if unavailable.
851 */
852 vector->virq = VFIO_ALLOW_KVM_MSI ?
853 kvm_irqchip_add_msi_route(kvm_state, vector->msg) : -1;
854 if (vector->virq < 0 ||
855 kvm_irqchip_add_irqfd_notifier(kvm_state, &vector->interrupt,
856 NULL, vector->virq) < 0) {
857 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
858 vfio_msi_interrupt, NULL, vector);
859 }
860 }
861
862 ret = vfio_enable_vectors(vdev, false);
863 if (ret) {
864 if (ret < 0) {
865 error_report("vfio: Error: Failed to setup MSI fds: %m");
866 } else if (ret != vdev->nr_vectors) {
867 error_report("vfio: Error: Failed to enable %d "
868 "MSI vectors, retry with %d", vdev->nr_vectors, ret);
869 }
870
871 for (i = 0; i < vdev->nr_vectors; i++) {
872 VFIOMSIVector *vector = &vdev->msi_vectors[i];
873 if (vector->virq >= 0) {
874 kvm_irqchip_remove_irqfd_notifier(kvm_state, &vector->interrupt,
875 vector->virq);
876 kvm_irqchip_release_virq(kvm_state, vector->virq);
877 vector->virq = -1;
878 } else {
879 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
880 NULL, NULL, NULL);
881 }
882 event_notifier_cleanup(&vector->interrupt);
883 }
884
885 g_free(vdev->msi_vectors);
886
887 if (ret > 0 && ret != vdev->nr_vectors) {
888 vdev->nr_vectors = ret;
889 goto retry;
890 }
891 vdev->nr_vectors = 0;
892
893 return;
894 }
895
896 vdev->interrupt = VFIO_INT_MSI;
897
898 DPRINTF("%s(%04x:%02x:%02x.%x) Enabled %d MSI vectors\n", __func__,
899 vdev->host.domain, vdev->host.bus, vdev->host.slot,
900 vdev->host.function, vdev->nr_vectors);
901 }
902
903 static void vfio_disable_msi_common(VFIODevice *vdev)
904 {
905 g_free(vdev->msi_vectors);
906 vdev->msi_vectors = NULL;
907 vdev->nr_vectors = 0;
908 vdev->interrupt = VFIO_INT_NONE;
909
910 vfio_enable_intx(vdev);
911 }
912
913 static void vfio_disable_msix(VFIODevice *vdev)
914 {
915 int i;
916
917 msix_unset_vector_notifiers(&vdev->pdev);
918
919 /*
920 * MSI-X will only release vectors if MSI-X is still enabled on the
921 * device, check through the rest and release it ourselves if necessary.
922 */
923 for (i = 0; i < vdev->nr_vectors; i++) {
924 if (vdev->msi_vectors[i].use) {
925 vfio_msix_vector_release(&vdev->pdev, i);
926 }
927 }
928
929 if (vdev->nr_vectors) {
930 vfio_disable_irqindex(vdev, VFIO_PCI_MSIX_IRQ_INDEX);
931 }
932
933 vfio_disable_msi_common(vdev);
934
935 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
936 vdev->host.bus, vdev->host.slot, vdev->host.function);
937 }
938
939 static void vfio_disable_msi(VFIODevice *vdev)
940 {
941 int i;
942
943 vfio_disable_irqindex(vdev, VFIO_PCI_MSI_IRQ_INDEX);
944
945 for (i = 0; i < vdev->nr_vectors; i++) {
946 VFIOMSIVector *vector = &vdev->msi_vectors[i];
947
948 if (!vector->use) {
949 continue;
950 }
951
952 if (vector->virq >= 0) {
953 kvm_irqchip_remove_irqfd_notifier(kvm_state,
954 &vector->interrupt, vector->virq);
955 kvm_irqchip_release_virq(kvm_state, vector->virq);
956 vector->virq = -1;
957 } else {
958 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
959 NULL, NULL, NULL);
960 }
961
962 event_notifier_cleanup(&vector->interrupt);
963 }
964
965 vfio_disable_msi_common(vdev);
966
967 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
968 vdev->host.bus, vdev->host.slot, vdev->host.function);
969 }
970
971 static void vfio_update_msi(VFIODevice *vdev)
972 {
973 int i;
974
975 for (i = 0; i < vdev->nr_vectors; i++) {
976 VFIOMSIVector *vector = &vdev->msi_vectors[i];
977 MSIMessage msg;
978
979 if (!vector->use || vector->virq < 0) {
980 continue;
981 }
982
983 msg = msi_get_message(&vdev->pdev, i);
984
985 if (msg.address != vector->msg.address ||
986 msg.data != vector->msg.data) {
987
988 DPRINTF("%s(%04x:%02x:%02x.%x) MSI vector %d changed\n",
989 __func__, vdev->host.domain, vdev->host.bus,
990 vdev->host.slot, vdev->host.function, i);
991
992 kvm_irqchip_update_msi_route(kvm_state, vector->virq, msg);
993 vector->msg = msg;
994 }
995 }
996 }
997
998 /*
999 * IO Port/MMIO - Beware of the endians, VFIO is always little endian
1000 */
1001 static void vfio_bar_write(void *opaque, hwaddr addr,
1002 uint64_t data, unsigned size)
1003 {
1004 VFIOBAR *bar = opaque;
1005 union {
1006 uint8_t byte;
1007 uint16_t word;
1008 uint32_t dword;
1009 uint64_t qword;
1010 } buf;
1011
1012 switch (size) {
1013 case 1:
1014 buf.byte = data;
1015 break;
1016 case 2:
1017 buf.word = cpu_to_le16(data);
1018 break;
1019 case 4:
1020 buf.dword = cpu_to_le32(data);
1021 break;
1022 default:
1023 hw_error("vfio: unsupported write size, %d bytes\n", size);
1024 break;
1025 }
1026
1027 if (pwrite(bar->fd, &buf, size, bar->fd_offset + addr) != size) {
1028 error_report("%s(,0x%"HWADDR_PRIx", 0x%"PRIx64", %d) failed: %m",
1029 __func__, addr, data, size);
1030 }
1031
1032 #ifdef DEBUG_VFIO
1033 {
1034 VFIODevice *vdev = container_of(bar, VFIODevice, bars[bar->nr]);
1035
1036 DPRINTF("%s(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx", 0x%"PRIx64
1037 ", %d)\n", __func__, vdev->host.domain, vdev->host.bus,
1038 vdev->host.slot, vdev->host.function, bar->nr, addr,
1039 data, size);
1040 }
1041 #endif
1042
1043 /*
1044 * A read or write to a BAR always signals an INTx EOI. This will
1045 * do nothing if not pending (including not in INTx mode). We assume
1046 * that a BAR access is in response to an interrupt and that BAR
1047 * accesses will service the interrupt. Unfortunately, we don't know
1048 * which access will service the interrupt, so we're potentially
1049 * getting quite a few host interrupts per guest interrupt.
1050 */
1051 vfio_eoi(container_of(bar, VFIODevice, bars[bar->nr]));
1052 }
1053
1054 static uint64_t vfio_bar_read(void *opaque,
1055 hwaddr addr, unsigned size)
1056 {
1057 VFIOBAR *bar = opaque;
1058 union {
1059 uint8_t byte;
1060 uint16_t word;
1061 uint32_t dword;
1062 uint64_t qword;
1063 } buf;
1064 uint64_t data = 0;
1065
1066 if (pread(bar->fd, &buf, size, bar->fd_offset + addr) != size) {
1067 error_report("%s(,0x%"HWADDR_PRIx", %d) failed: %m",
1068 __func__, addr, size);
1069 return (uint64_t)-1;
1070 }
1071
1072 switch (size) {
1073 case 1:
1074 data = buf.byte;
1075 break;
1076 case 2:
1077 data = le16_to_cpu(buf.word);
1078 break;
1079 case 4:
1080 data = le32_to_cpu(buf.dword);
1081 break;
1082 default:
1083 hw_error("vfio: unsupported read size, %d bytes\n", size);
1084 break;
1085 }
1086
1087 #ifdef DEBUG_VFIO
1088 {
1089 VFIODevice *vdev = container_of(bar, VFIODevice, bars[bar->nr]);
1090
1091 DPRINTF("%s(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx
1092 ", %d) = 0x%"PRIx64"\n", __func__, vdev->host.domain,
1093 vdev->host.bus, vdev->host.slot, vdev->host.function,
1094 bar->nr, addr, size, data);
1095 }
1096 #endif
1097
1098 /* Same as write above */
1099 vfio_eoi(container_of(bar, VFIODevice, bars[bar->nr]));
1100
1101 return data;
1102 }
1103
1104 static const MemoryRegionOps vfio_bar_ops = {
1105 .read = vfio_bar_read,
1106 .write = vfio_bar_write,
1107 .endianness = DEVICE_LITTLE_ENDIAN,
1108 };
1109
1110 static void vfio_pci_load_rom(VFIODevice *vdev)
1111 {
1112 struct vfio_region_info reg_info = {
1113 .argsz = sizeof(reg_info),
1114 .index = VFIO_PCI_ROM_REGION_INDEX
1115 };
1116 uint64_t size;
1117 off_t off = 0;
1118 size_t bytes;
1119
1120 if (ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info)) {
1121 error_report("vfio: Error getting ROM info: %m");
1122 return;
1123 }
1124
1125 DPRINTF("Device %04x:%02x:%02x.%x ROM:\n", vdev->host.domain,
1126 vdev->host.bus, vdev->host.slot, vdev->host.function);
1127 DPRINTF(" size: 0x%lx, offset: 0x%lx, flags: 0x%lx\n",
1128 (unsigned long)reg_info.size, (unsigned long)reg_info.offset,
1129 (unsigned long)reg_info.flags);
1130
1131 vdev->rom_size = size = reg_info.size;
1132 vdev->rom_offset = reg_info.offset;
1133
1134 if (!vdev->rom_size) {
1135 vdev->rom_read_failed = true;
1136 error_report("vfio-pci: Cannot read device rom at "
1137 "%04x:%02x:%02x.%x\n",
1138 vdev->host.domain, vdev->host.bus, vdev->host.slot,
1139 vdev->host.function);
1140 error_printf("Device option ROM contents are probably invalid "
1141 "(check dmesg).\nSkip option ROM probe with rombar=0, "
1142 "or load from file with romfile=\n");
1143 return;
1144 }
1145
1146 vdev->rom = g_malloc(size);
1147 memset(vdev->rom, 0xff, size);
1148
1149 while (size) {
1150 bytes = pread(vdev->fd, vdev->rom + off, size, vdev->rom_offset + off);
1151 if (bytes == 0) {
1152 break;
1153 } else if (bytes > 0) {
1154 off += bytes;
1155 size -= bytes;
1156 } else {
1157 if (errno == EINTR || errno == EAGAIN) {
1158 continue;
1159 }
1160 error_report("vfio: Error reading device ROM: %m");
1161 break;
1162 }
1163 }
1164 }
1165
1166 static uint64_t vfio_rom_read(void *opaque, hwaddr addr, unsigned size)
1167 {
1168 VFIODevice *vdev = opaque;
1169 uint64_t val = ((uint64_t)1 << (size * 8)) - 1;
1170
1171 /* Load the ROM lazily when the guest tries to read it */
1172 if (unlikely(!vdev->rom)) {
1173 vfio_pci_load_rom(vdev);
1174 if (unlikely(!vdev->rom && !vdev->rom_read_failed)) {
1175 vfio_pci_load_rom(vdev);
1176 }
1177 }
1178
1179 memcpy(&val, vdev->rom + addr,
1180 (addr < vdev->rom_size) ? MIN(size, vdev->rom_size - addr) : 0);
1181
1182 DPRINTF("%s(%04x:%02x:%02x.%x, 0x%"HWADDR_PRIx", 0x%x) = 0x%"PRIx64"\n",
1183 __func__, vdev->host.domain, vdev->host.bus, vdev->host.slot,
1184 vdev->host.function, addr, size, val);
1185
1186 return val;
1187 }
1188
1189 static void vfio_rom_write(void *opaque, hwaddr addr,
1190 uint64_t data, unsigned size)
1191 {
1192 }
1193
1194 static const MemoryRegionOps vfio_rom_ops = {
1195 .read = vfio_rom_read,
1196 .write = vfio_rom_write,
1197 .endianness = DEVICE_LITTLE_ENDIAN,
1198 };
1199
1200 static void vfio_pci_size_rom(VFIODevice *vdev)
1201 {
1202 uint32_t orig, size = cpu_to_le32((uint32_t)PCI_ROM_ADDRESS_MASK);
1203 off_t offset = vdev->config_offset + PCI_ROM_ADDRESS;
1204 char name[32];
1205
1206 if (vdev->pdev.romfile || !vdev->pdev.rom_bar) {
1207 return;
1208 }
1209
1210 /*
1211 * Use the same size ROM BAR as the physical device. The contents
1212 * will get filled in later when the guest tries to read it.
1213 */
1214 if (pread(vdev->fd, &orig, 4, offset) != 4 ||
1215 pwrite(vdev->fd, &size, 4, offset) != 4 ||
1216 pread(vdev->fd, &size, 4, offset) != 4 ||
1217 pwrite(vdev->fd, &orig, 4, offset) != 4) {
1218 error_report("%s(%04x:%02x:%02x.%x) failed: %m",
1219 __func__, vdev->host.domain, vdev->host.bus,
1220 vdev->host.slot, vdev->host.function);
1221 return;
1222 }
1223
1224 size = ~(le32_to_cpu(size) & PCI_ROM_ADDRESS_MASK) + 1;
1225
1226 if (!size) {
1227 return;
1228 }
1229
1230 DPRINTF("%04x:%02x:%02x.%x ROM size 0x%x\n", vdev->host.domain,
1231 vdev->host.bus, vdev->host.slot, vdev->host.function, size);
1232
1233 snprintf(name, sizeof(name), "vfio[%04x:%02x:%02x.%x].rom",
1234 vdev->host.domain, vdev->host.bus, vdev->host.slot,
1235 vdev->host.function);
1236
1237 memory_region_init_io(&vdev->pdev.rom, OBJECT(vdev),
1238 &vfio_rom_ops, vdev, name, size);
1239
1240 pci_register_bar(&vdev->pdev, PCI_ROM_SLOT,
1241 PCI_BASE_ADDRESS_SPACE_MEMORY, &vdev->pdev.rom);
1242
1243 vdev->pdev.has_rom = true;
1244 vdev->rom_read_failed = false;
1245 }
1246
1247 static void vfio_vga_write(void *opaque, hwaddr addr,
1248 uint64_t data, unsigned size)
1249 {
1250 VFIOVGARegion *region = opaque;
1251 VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]);
1252 union {
1253 uint8_t byte;
1254 uint16_t word;
1255 uint32_t dword;
1256 uint64_t qword;
1257 } buf;
1258 off_t offset = vga->fd_offset + region->offset + addr;
1259
1260 switch (size) {
1261 case 1:
1262 buf.byte = data;
1263 break;
1264 case 2:
1265 buf.word = cpu_to_le16(data);
1266 break;
1267 case 4:
1268 buf.dword = cpu_to_le32(data);
1269 break;
1270 default:
1271 hw_error("vfio: unsupported write size, %d bytes\n", size);
1272 break;
1273 }
1274
1275 if (pwrite(vga->fd, &buf, size, offset) != size) {
1276 error_report("%s(,0x%"HWADDR_PRIx", 0x%"PRIx64", %d) failed: %m",
1277 __func__, region->offset + addr, data, size);
1278 }
1279
1280 DPRINTF("%s(0x%"HWADDR_PRIx", 0x%"PRIx64", %d)\n",
1281 __func__, region->offset + addr, data, size);
1282 }
1283
1284 static uint64_t vfio_vga_read(void *opaque, hwaddr addr, unsigned size)
1285 {
1286 VFIOVGARegion *region = opaque;
1287 VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]);
1288 union {
1289 uint8_t byte;
1290 uint16_t word;
1291 uint32_t dword;
1292 uint64_t qword;
1293 } buf;
1294 uint64_t data = 0;
1295 off_t offset = vga->fd_offset + region->offset + addr;
1296
1297 if (pread(vga->fd, &buf, size, offset) != size) {
1298 error_report("%s(,0x%"HWADDR_PRIx", %d) failed: %m",
1299 __func__, region->offset + addr, size);
1300 return (uint64_t)-1;
1301 }
1302
1303 switch (size) {
1304 case 1:
1305 data = buf.byte;
1306 break;
1307 case 2:
1308 data = le16_to_cpu(buf.word);
1309 break;
1310 case 4:
1311 data = le32_to_cpu(buf.dword);
1312 break;
1313 default:
1314 hw_error("vfio: unsupported read size, %d bytes\n", size);
1315 break;
1316 }
1317
1318 DPRINTF("%s(0x%"HWADDR_PRIx", %d) = 0x%"PRIx64"\n",
1319 __func__, region->offset + addr, size, data);
1320
1321 return data;
1322 }
1323
1324 static const MemoryRegionOps vfio_vga_ops = {
1325 .read = vfio_vga_read,
1326 .write = vfio_vga_write,
1327 .endianness = DEVICE_LITTLE_ENDIAN,
1328 };
1329
1330 /*
1331 * Device specific quirks
1332 */
1333
1334 /* Is range1 fully contained within range2? */
1335 static bool vfio_range_contained(uint64_t first1, uint64_t len1,
1336 uint64_t first2, uint64_t len2) {
1337 return (first1 >= first2 && first1 + len1 <= first2 + len2);
1338 }
1339
1340 static bool vfio_flags_enabled(uint8_t flags, uint8_t mask)
1341 {
1342 return (mask && (flags & mask) == mask);
1343 }
1344
1345 static uint64_t vfio_generic_window_quirk_read(void *opaque,
1346 hwaddr addr, unsigned size)
1347 {
1348 VFIOQuirk *quirk = opaque;
1349 VFIODevice *vdev = quirk->vdev;
1350 uint64_t data;
1351
1352 if (vfio_flags_enabled(quirk->data.flags, quirk->data.read_flags) &&
1353 ranges_overlap(addr, size,
1354 quirk->data.data_offset, quirk->data.data_size)) {
1355 hwaddr offset = addr - quirk->data.data_offset;
1356
1357 if (!vfio_range_contained(addr, size, quirk->data.data_offset,
1358 quirk->data.data_size)) {
1359 hw_error("%s: window data read not fully contained: %s\n",
1360 __func__, memory_region_name(&quirk->mem));
1361 }
1362
1363 data = vfio_pci_read_config(&vdev->pdev,
1364 quirk->data.address_val + offset, size);
1365
1366 DPRINTF("%s read(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx", %d) = 0x%"
1367 PRIx64"\n", memory_region_name(&quirk->mem), vdev->host.domain,
1368 vdev->host.bus, vdev->host.slot, vdev->host.function,
1369 quirk->data.bar, addr, size, data);
1370 } else {
1371 data = vfio_bar_read(&vdev->bars[quirk->data.bar],
1372 addr + quirk->data.base_offset, size);
1373 }
1374
1375 return data;
1376 }
1377
1378 static void vfio_generic_window_quirk_write(void *opaque, hwaddr addr,
1379 uint64_t data, unsigned size)
1380 {
1381 VFIOQuirk *quirk = opaque;
1382 VFIODevice *vdev = quirk->vdev;
1383
1384 if (ranges_overlap(addr, size,
1385 quirk->data.address_offset, quirk->data.address_size)) {
1386
1387 if (addr != quirk->data.address_offset) {
1388 hw_error("%s: offset write into address window: %s\n",
1389 __func__, memory_region_name(&quirk->mem));
1390 }
1391
1392 if ((data & ~quirk->data.address_mask) == quirk->data.address_match) {
1393 quirk->data.flags |= quirk->data.write_flags |
1394 quirk->data.read_flags;
1395 quirk->data.address_val = data & quirk->data.address_mask;
1396 } else {
1397 quirk->data.flags &= ~(quirk->data.write_flags |
1398 quirk->data.read_flags);
1399 }
1400 }
1401
1402 if (vfio_flags_enabled(quirk->data.flags, quirk->data.write_flags) &&
1403 ranges_overlap(addr, size,
1404 quirk->data.data_offset, quirk->data.data_size)) {
1405 hwaddr offset = addr - quirk->data.data_offset;
1406
1407 if (!vfio_range_contained(addr, size, quirk->data.data_offset,
1408 quirk->data.data_size)) {
1409 hw_error("%s: window data write not fully contained: %s\n",
1410 __func__, memory_region_name(&quirk->mem));
1411 }
1412
1413 vfio_pci_write_config(&vdev->pdev,
1414 quirk->data.address_val + offset, data, size);
1415 DPRINTF("%s write(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx", 0x%"
1416 PRIx64", %d)\n", memory_region_name(&quirk->mem),
1417 vdev->host.domain, vdev->host.bus, vdev->host.slot,
1418 vdev->host.function, quirk->data.bar, addr, data, size);
1419 return;
1420 }
1421
1422 vfio_bar_write(&vdev->bars[quirk->data.bar],
1423 addr + quirk->data.base_offset, data, size);
1424 }
1425
1426 static const MemoryRegionOps vfio_generic_window_quirk = {
1427 .read = vfio_generic_window_quirk_read,
1428 .write = vfio_generic_window_quirk_write,
1429 .endianness = DEVICE_LITTLE_ENDIAN,
1430 };
1431
1432 static uint64_t vfio_generic_quirk_read(void *opaque,
1433 hwaddr addr, unsigned size)
1434 {
1435 VFIOQuirk *quirk = opaque;
1436 VFIODevice *vdev = quirk->vdev;
1437 hwaddr base = quirk->data.address_match & TARGET_PAGE_MASK;
1438 hwaddr offset = quirk->data.address_match & ~TARGET_PAGE_MASK;
1439 uint64_t data;
1440
1441 if (vfio_flags_enabled(quirk->data.flags, quirk->data.read_flags) &&
1442 ranges_overlap(addr, size, offset, quirk->data.address_mask + 1)) {
1443 if (!vfio_range_contained(addr, size, offset,
1444 quirk->data.address_mask + 1)) {
1445 hw_error("%s: read not fully contained: %s\n",
1446 __func__, memory_region_name(&quirk->mem));
1447 }
1448
1449 data = vfio_pci_read_config(&vdev->pdev, addr - offset, size);
1450
1451 DPRINTF("%s read(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx", %d) = 0x%"
1452 PRIx64"\n", memory_region_name(&quirk->mem), vdev->host.domain,
1453 vdev->host.bus, vdev->host.slot, vdev->host.function,
1454 quirk->data.bar, addr + base, size, data);
1455 } else {
1456 data = vfio_bar_read(&vdev->bars[quirk->data.bar], addr + base, size);
1457 }
1458
1459 return data;
1460 }
1461
1462 static void vfio_generic_quirk_write(void *opaque, hwaddr addr,
1463 uint64_t data, unsigned size)
1464 {
1465 VFIOQuirk *quirk = opaque;
1466 VFIODevice *vdev = quirk->vdev;
1467 hwaddr base = quirk->data.address_match & TARGET_PAGE_MASK;
1468 hwaddr offset = quirk->data.address_match & ~TARGET_PAGE_MASK;
1469
1470 if (vfio_flags_enabled(quirk->data.flags, quirk->data.write_flags) &&
1471 ranges_overlap(addr, size, offset, quirk->data.address_mask + 1)) {
1472 if (!vfio_range_contained(addr, size, offset,
1473 quirk->data.address_mask + 1)) {
1474 hw_error("%s: write not fully contained: %s\n",
1475 __func__, memory_region_name(&quirk->mem));
1476 }
1477
1478 vfio_pci_write_config(&vdev->pdev, addr - offset, data, size);
1479
1480 DPRINTF("%s write(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx", 0x%"
1481 PRIx64", %d)\n", memory_region_name(&quirk->mem),
1482 vdev->host.domain, vdev->host.bus, vdev->host.slot,
1483 vdev->host.function, quirk->data.bar, addr + base, data, size);
1484 } else {
1485 vfio_bar_write(&vdev->bars[quirk->data.bar], addr + base, data, size);
1486 }
1487 }
1488
1489 static const MemoryRegionOps vfio_generic_quirk = {
1490 .read = vfio_generic_quirk_read,
1491 .write = vfio_generic_quirk_write,
1492 .endianness = DEVICE_LITTLE_ENDIAN,
1493 };
1494
1495 #define PCI_VENDOR_ID_ATI 0x1002
1496
1497 /*
1498 * Radeon HD cards (HD5450 & HD7850) report the upper byte of the I/O port BAR
1499 * through VGA register 0x3c3. On newer cards, the I/O port BAR is always
1500 * BAR4 (older cards like the X550 used BAR1, but we don't care to support
1501 * those). Note that on bare metal, a read of 0x3c3 doesn't always return the
1502 * I/O port BAR address. Originally this was coded to return the virtual BAR
1503 * address only if the physical register read returns the actual BAR address,
1504 * but users have reported greater success if we return the virtual address
1505 * unconditionally.
1506 */
1507 static uint64_t vfio_ati_3c3_quirk_read(void *opaque,
1508 hwaddr addr, unsigned size)
1509 {
1510 VFIOQuirk *quirk = opaque;
1511 VFIODevice *vdev = quirk->vdev;
1512 uint64_t data = vfio_pci_read_config(&vdev->pdev,
1513 PCI_BASE_ADDRESS_0 + (4 * 4) + 1,
1514 size);
1515 DPRINTF("%s(0x3c3, 1) = 0x%"PRIx64"\n", __func__, data);
1516
1517 return data;
1518 }
1519
1520 static const MemoryRegionOps vfio_ati_3c3_quirk = {
1521 .read = vfio_ati_3c3_quirk_read,
1522 .endianness = DEVICE_LITTLE_ENDIAN,
1523 };
1524
1525 static void vfio_vga_probe_ati_3c3_quirk(VFIODevice *vdev)
1526 {
1527 PCIDevice *pdev = &vdev->pdev;
1528 VFIOQuirk *quirk;
1529
1530 if (pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_ATI) {
1531 return;
1532 }
1533
1534 /*
1535 * As long as the BAR is >= 256 bytes it will be aligned such that the
1536 * lower byte is always zero. Filter out anything else, if it exists.
1537 */
1538 if (!vdev->bars[4].ioport || vdev->bars[4].size < 256) {
1539 return;
1540 }
1541
1542 quirk = g_malloc0(sizeof(*quirk));
1543 quirk->vdev = vdev;
1544
1545 memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_ati_3c3_quirk, quirk,
1546 "vfio-ati-3c3-quirk", 1);
1547 memory_region_add_subregion(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem,
1548 3 /* offset 3 bytes from 0x3c0 */, &quirk->mem);
1549
1550 QLIST_INSERT_HEAD(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].quirks,
1551 quirk, next);
1552
1553 DPRINTF("Enabled ATI/AMD quirk 0x3c3 BAR4for device %04x:%02x:%02x.%x\n",
1554 vdev->host.domain, vdev->host.bus, vdev->host.slot,
1555 vdev->host.function);
1556 }
1557
1558 /*
1559 * Newer ATI/AMD devices, including HD5450 and HD7850, have a window to PCI
1560 * config space through MMIO BAR2 at offset 0x4000. Nothing seems to access
1561 * the MMIO space directly, but a window to this space is provided through
1562 * I/O port BAR4. Offset 0x0 is the address register and offset 0x4 is the
1563 * data register. When the address is programmed to a range of 0x4000-0x4fff
1564 * PCI configuration space is available. Experimentation seems to indicate
1565 * that only read-only access is provided, but we drop writes when the window
1566 * is enabled to config space nonetheless.
1567 */
1568 static void vfio_probe_ati_bar4_window_quirk(VFIODevice *vdev, int nr)
1569 {
1570 PCIDevice *pdev = &vdev->pdev;
1571 VFIOQuirk *quirk;
1572
1573 if (!vdev->has_vga || nr != 4 ||
1574 pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_ATI) {
1575 return;
1576 }
1577
1578 quirk = g_malloc0(sizeof(*quirk));
1579 quirk->vdev = vdev;
1580 quirk->data.address_size = 4;
1581 quirk->data.data_offset = 4;
1582 quirk->data.data_size = 4;
1583 quirk->data.address_match = 0x4000;
1584 quirk->data.address_mask = PCIE_CONFIG_SPACE_SIZE - 1;
1585 quirk->data.bar = nr;
1586 quirk->data.read_flags = quirk->data.write_flags = 1;
1587
1588 memory_region_init_io(&quirk->mem, OBJECT(vdev),
1589 &vfio_generic_window_quirk, quirk,
1590 "vfio-ati-bar4-window-quirk", 8);
1591 memory_region_add_subregion_overlap(&vdev->bars[nr].mem,
1592 quirk->data.base_offset, &quirk->mem, 1);
1593
1594 QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
1595
1596 DPRINTF("Enabled ATI/AMD BAR4 window quirk for device %04x:%02x:%02x.%x\n",
1597 vdev->host.domain, vdev->host.bus, vdev->host.slot,
1598 vdev->host.function);
1599 }
1600
1601 /*
1602 * Trap the BAR2 MMIO window to config space as well.
1603 */
1604 static void vfio_probe_ati_bar2_4000_quirk(VFIODevice *vdev, int nr)
1605 {
1606 PCIDevice *pdev = &vdev->pdev;
1607 VFIOQuirk *quirk;
1608
1609 /* Only enable on newer devices where BAR2 is 64bit */
1610 if (!vdev->has_vga || nr != 2 || !vdev->bars[2].mem64 ||
1611 pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_ATI) {
1612 return;
1613 }
1614
1615 quirk = g_malloc0(sizeof(*quirk));
1616 quirk->vdev = vdev;
1617 quirk->data.flags = quirk->data.read_flags = quirk->data.write_flags = 1;
1618 quirk->data.address_match = 0x4000;
1619 quirk->data.address_mask = PCIE_CONFIG_SPACE_SIZE - 1;
1620 quirk->data.bar = nr;
1621
1622 memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_generic_quirk, quirk,
1623 "vfio-ati-bar2-4000-quirk",
1624 TARGET_PAGE_ALIGN(quirk->data.address_mask + 1));
1625 memory_region_add_subregion_overlap(&vdev->bars[nr].mem,
1626 quirk->data.address_match & TARGET_PAGE_MASK,
1627 &quirk->mem, 1);
1628
1629 QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
1630
1631 DPRINTF("Enabled ATI/AMD BAR2 0x4000 quirk for device %04x:%02x:%02x.%x\n",
1632 vdev->host.domain, vdev->host.bus, vdev->host.slot,
1633 vdev->host.function);
1634 }
1635
1636 /*
1637 * Older ATI/AMD cards like the X550 have a similar window to that above.
1638 * I/O port BAR1 provides a window to a mirror of PCI config space located
1639 * in BAR2 at offset 0xf00. We don't care to support such older cards, but
1640 * note it for future reference.
1641 */
1642
1643 #define PCI_VENDOR_ID_NVIDIA 0x10de
1644
1645 /*
1646 * Nvidia has several different methods to get to config space, the
1647 * nouveu project has several of these documented here:
1648 * https://github.com/pathscale/envytools/tree/master/hwdocs
1649 *
1650 * The first quirk is actually not documented in envytools and is found
1651 * on 10de:01d1 (NVIDIA Corporation G72 [GeForce 7300 LE]). This is an
1652 * NV46 chipset. The backdoor uses the legacy VGA I/O ports to access
1653 * the mirror of PCI config space found at BAR0 offset 0x1800. The access
1654 * sequence first writes 0x338 to I/O port 0x3d4. The target offset is
1655 * then written to 0x3d0. Finally 0x538 is written for a read and 0x738
1656 * is written for a write to 0x3d4. The BAR0 offset is then accessible
1657 * through 0x3d0. This quirk doesn't seem to be necessary on newer cards
1658 * that use the I/O port BAR5 window but it doesn't hurt to leave it.
1659 */
1660 enum {
1661 NV_3D0_NONE = 0,
1662 NV_3D0_SELECT,
1663 NV_3D0_WINDOW,
1664 NV_3D0_READ,
1665 NV_3D0_WRITE,
1666 };
1667
1668 static uint64_t vfio_nvidia_3d0_quirk_read(void *opaque,
1669 hwaddr addr, unsigned size)
1670 {
1671 VFIOQuirk *quirk = opaque;
1672 VFIODevice *vdev = quirk->vdev;
1673 PCIDevice *pdev = &vdev->pdev;
1674 uint64_t data = vfio_vga_read(&vdev->vga.region[QEMU_PCI_VGA_IO_HI],
1675 addr + quirk->data.base_offset, size);
1676
1677 if (quirk->data.flags == NV_3D0_READ && addr == quirk->data.data_offset) {
1678 data = vfio_pci_read_config(pdev, quirk->data.address_val, size);
1679 DPRINTF("%s(0x3d0, %d) = 0x%"PRIx64"\n", __func__, size, data);
1680 }
1681
1682 quirk->data.flags = NV_3D0_NONE;
1683
1684 return data;
1685 }
1686
1687 static void vfio_nvidia_3d0_quirk_write(void *opaque, hwaddr addr,
1688 uint64_t data, unsigned size)
1689 {
1690 VFIOQuirk *quirk = opaque;
1691 VFIODevice *vdev = quirk->vdev;
1692 PCIDevice *pdev = &vdev->pdev;
1693
1694 switch (quirk->data.flags) {
1695 case NV_3D0_NONE:
1696 if (addr == quirk->data.address_offset && data == 0x338) {
1697 quirk->data.flags = NV_3D0_SELECT;
1698 }
1699 break;
1700 case NV_3D0_SELECT:
1701 quirk->data.flags = NV_3D0_NONE;
1702 if (addr == quirk->data.data_offset &&
1703 (data & ~quirk->data.address_mask) == quirk->data.address_match) {
1704 quirk->data.flags = NV_3D0_WINDOW;
1705 quirk->data.address_val = data & quirk->data.address_mask;
1706 }
1707 break;
1708 case NV_3D0_WINDOW:
1709 quirk->data.flags = NV_3D0_NONE;
1710 if (addr == quirk->data.address_offset) {
1711 if (data == 0x538) {
1712 quirk->data.flags = NV_3D0_READ;
1713 } else if (data == 0x738) {
1714 quirk->data.flags = NV_3D0_WRITE;
1715 }
1716 }
1717 break;
1718 case NV_3D0_WRITE:
1719 quirk->data.flags = NV_3D0_NONE;
1720 if (addr == quirk->data.data_offset) {
1721 vfio_pci_write_config(pdev, quirk->data.address_val, data, size);
1722 DPRINTF("%s(0x3d0, 0x%"PRIx64", %d)\n", __func__, data, size);
1723 return;
1724 }
1725 break;
1726 }
1727
1728 vfio_vga_write(&vdev->vga.region[QEMU_PCI_VGA_IO_HI],
1729 addr + quirk->data.base_offset, data, size);
1730 }
1731
1732 static const MemoryRegionOps vfio_nvidia_3d0_quirk = {
1733 .read = vfio_nvidia_3d0_quirk_read,
1734 .write = vfio_nvidia_3d0_quirk_write,
1735 .endianness = DEVICE_LITTLE_ENDIAN,
1736 };
1737
1738 static void vfio_vga_probe_nvidia_3d0_quirk(VFIODevice *vdev)
1739 {
1740 PCIDevice *pdev = &vdev->pdev;
1741 VFIOQuirk *quirk;
1742
1743 if (pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_NVIDIA ||
1744 !vdev->bars[1].size) {
1745 return;
1746 }
1747
1748 quirk = g_malloc0(sizeof(*quirk));
1749 quirk->vdev = vdev;
1750 quirk->data.base_offset = 0x10;
1751 quirk->data.address_offset = 4;
1752 quirk->data.address_size = 2;
1753 quirk->data.address_match = 0x1800;
1754 quirk->data.address_mask = PCI_CONFIG_SPACE_SIZE - 1;
1755 quirk->data.data_offset = 0;
1756 quirk->data.data_size = 4;
1757
1758 memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_nvidia_3d0_quirk,
1759 quirk, "vfio-nvidia-3d0-quirk", 6);
1760 memory_region_add_subregion(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem,
1761 quirk->data.base_offset, &quirk->mem);
1762
1763 QLIST_INSERT_HEAD(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].quirks,
1764 quirk, next);
1765
1766 DPRINTF("Enabled NVIDIA VGA 0x3d0 quirk for device %04x:%02x:%02x.%x\n",
1767 vdev->host.domain, vdev->host.bus, vdev->host.slot,
1768 vdev->host.function);
1769 }
1770
1771 /*
1772 * The second quirk is documented in envytools. The I/O port BAR5 is just
1773 * a set of address/data ports to the MMIO BARs. The BAR we care about is
1774 * again BAR0. This backdoor is apparently a bit newer than the one above
1775 * so we need to not only trap 256 bytes @0x1800, but all of PCI config
1776 * space, including extended space is available at the 4k @0x88000.
1777 */
1778 enum {
1779 NV_BAR5_ADDRESS = 0x1,
1780 NV_BAR5_ENABLE = 0x2,
1781 NV_BAR5_MASTER = 0x4,
1782 NV_BAR5_VALID = 0x7,
1783 };
1784
1785 static void vfio_nvidia_bar5_window_quirk_write(void *opaque, hwaddr addr,
1786 uint64_t data, unsigned size)
1787 {
1788 VFIOQuirk *quirk = opaque;
1789
1790 switch (addr) {
1791 case 0x0:
1792 if (data & 0x1) {
1793 quirk->data.flags |= NV_BAR5_MASTER;
1794 } else {
1795 quirk->data.flags &= ~NV_BAR5_MASTER;
1796 }
1797 break;
1798 case 0x4:
1799 if (data & 0x1) {
1800 quirk->data.flags |= NV_BAR5_ENABLE;
1801 } else {
1802 quirk->data.flags &= ~NV_BAR5_ENABLE;
1803 }
1804 break;
1805 case 0x8:
1806 if (quirk->data.flags & NV_BAR5_MASTER) {
1807 if ((data & ~0xfff) == 0x88000) {
1808 quirk->data.flags |= NV_BAR5_ADDRESS;
1809 quirk->data.address_val = data & 0xfff;
1810 } else if ((data & ~0xff) == 0x1800) {
1811 quirk->data.flags |= NV_BAR5_ADDRESS;
1812 quirk->data.address_val = data & 0xff;
1813 } else {
1814 quirk->data.flags &= ~NV_BAR5_ADDRESS;
1815 }
1816 }
1817 break;
1818 }
1819
1820 vfio_generic_window_quirk_write(opaque, addr, data, size);
1821 }
1822
1823 static const MemoryRegionOps vfio_nvidia_bar5_window_quirk = {
1824 .read = vfio_generic_window_quirk_read,
1825 .write = vfio_nvidia_bar5_window_quirk_write,
1826 .valid.min_access_size = 4,
1827 .endianness = DEVICE_LITTLE_ENDIAN,
1828 };
1829
1830 static void vfio_probe_nvidia_bar5_window_quirk(VFIODevice *vdev, int nr)
1831 {
1832 PCIDevice *pdev = &vdev->pdev;
1833 VFIOQuirk *quirk;
1834
1835 if (!vdev->has_vga || nr != 5 ||
1836 pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_NVIDIA) {
1837 return;
1838 }
1839
1840 quirk = g_malloc0(sizeof(*quirk));
1841 quirk->vdev = vdev;
1842 quirk->data.read_flags = quirk->data.write_flags = NV_BAR5_VALID;
1843 quirk->data.address_offset = 0x8;
1844 quirk->data.address_size = 0; /* actually 4, but avoids generic code */
1845 quirk->data.data_offset = 0xc;
1846 quirk->data.data_size = 4;
1847 quirk->data.bar = nr;
1848
1849 memory_region_init_io(&quirk->mem, OBJECT(vdev),
1850 &vfio_nvidia_bar5_window_quirk, quirk,
1851 "vfio-nvidia-bar5-window-quirk", 16);
1852 memory_region_add_subregion_overlap(&vdev->bars[nr].mem, 0, &quirk->mem, 1);
1853
1854 QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
1855
1856 DPRINTF("Enabled NVIDIA BAR5 window quirk for device %04x:%02x:%02x.%x\n",
1857 vdev->host.domain, vdev->host.bus, vdev->host.slot,
1858 vdev->host.function);
1859 }
1860
1861 static void vfio_nvidia_88000_quirk_write(void *opaque, hwaddr addr,
1862 uint64_t data, unsigned size)
1863 {
1864 VFIOQuirk *quirk = opaque;
1865 VFIODevice *vdev = quirk->vdev;
1866 PCIDevice *pdev = &vdev->pdev;
1867 hwaddr base = quirk->data.address_match & TARGET_PAGE_MASK;
1868
1869 vfio_generic_quirk_write(opaque, addr, data, size);
1870
1871 /*
1872 * Nvidia seems to acknowledge MSI interrupts by writing 0xff to the
1873 * MSI capability ID register. Both the ID and next register are
1874 * read-only, so we allow writes covering either of those to real hw.
1875 * NB - only fixed for the 0x88000 MMIO window.
1876 */
1877 if ((pdev->cap_present & QEMU_PCI_CAP_MSI) &&
1878 vfio_range_contained(addr, size, pdev->msi_cap, PCI_MSI_FLAGS)) {
1879 vfio_bar_write(&vdev->bars[quirk->data.bar], addr + base, data, size);
1880 }
1881 }
1882
1883 static const MemoryRegionOps vfio_nvidia_88000_quirk = {
1884 .read = vfio_generic_quirk_read,
1885 .write = vfio_nvidia_88000_quirk_write,
1886 .endianness = DEVICE_LITTLE_ENDIAN,
1887 };
1888
1889 /*
1890 * Finally, BAR0 itself. We want to redirect any accesses to either
1891 * 0x1800 or 0x88000 through the PCI config space access functions.
1892 *
1893 * NB - quirk at a page granularity or else they don't seem to work when
1894 * BARs are mmap'd
1895 *
1896 * Here's offset 0x88000...
1897 */
1898 static void vfio_probe_nvidia_bar0_88000_quirk(VFIODevice *vdev, int nr)
1899 {
1900 PCIDevice *pdev = &vdev->pdev;
1901 VFIOQuirk *quirk;
1902
1903 if (!vdev->has_vga || nr != 0 ||
1904 pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_NVIDIA) {
1905 return;
1906 }
1907
1908 quirk = g_malloc0(sizeof(*quirk));
1909 quirk->vdev = vdev;
1910 quirk->data.flags = quirk->data.read_flags = quirk->data.write_flags = 1;
1911 quirk->data.address_match = 0x88000;
1912 quirk->data.address_mask = PCIE_CONFIG_SPACE_SIZE - 1;
1913 quirk->data.bar = nr;
1914
1915 memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_nvidia_88000_quirk,
1916 quirk, "vfio-nvidia-bar0-88000-quirk",
1917 TARGET_PAGE_ALIGN(quirk->data.address_mask + 1));
1918 memory_region_add_subregion_overlap(&vdev->bars[nr].mem,
1919 quirk->data.address_match & TARGET_PAGE_MASK,
1920 &quirk->mem, 1);
1921
1922 QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
1923
1924 DPRINTF("Enabled NVIDIA BAR0 0x88000 quirk for device %04x:%02x:%02x.%x\n",
1925 vdev->host.domain, vdev->host.bus, vdev->host.slot,
1926 vdev->host.function);
1927 }
1928
1929 /*
1930 * And here's the same for BAR0 offset 0x1800...
1931 */
1932 static void vfio_probe_nvidia_bar0_1800_quirk(VFIODevice *vdev, int nr)
1933 {
1934 PCIDevice *pdev = &vdev->pdev;
1935 VFIOQuirk *quirk;
1936
1937 if (!vdev->has_vga || nr != 0 ||
1938 pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_NVIDIA) {
1939 return;
1940 }
1941
1942 /* Log the chipset ID */
1943 DPRINTF("Nvidia NV%02x\n",
1944 (unsigned int)(vfio_bar_read(&vdev->bars[0], 0, 4) >> 20) & 0xff);
1945
1946 quirk = g_malloc0(sizeof(*quirk));
1947 quirk->vdev = vdev;
1948 quirk->data.flags = quirk->data.read_flags = quirk->data.write_flags = 1;
1949 quirk->data.address_match = 0x1800;
1950 quirk->data.address_mask = PCI_CONFIG_SPACE_SIZE - 1;
1951 quirk->data.bar = nr;
1952
1953 memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_generic_quirk, quirk,
1954 "vfio-nvidia-bar0-1800-quirk",
1955 TARGET_PAGE_ALIGN(quirk->data.address_mask + 1));
1956 memory_region_add_subregion_overlap(&vdev->bars[nr].mem,
1957 quirk->data.address_match & TARGET_PAGE_MASK,
1958 &quirk->mem, 1);
1959
1960 QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
1961
1962 DPRINTF("Enabled NVIDIA BAR0 0x1800 quirk for device %04x:%02x:%02x.%x\n",
1963 vdev->host.domain, vdev->host.bus, vdev->host.slot,
1964 vdev->host.function);
1965 }
1966
1967 /*
1968 * TODO - Some Nvidia devices provide config access to their companion HDA
1969 * device and even to their parent bridge via these config space mirrors.
1970 * Add quirks for those regions.
1971 */
1972
1973 /*
1974 * Common quirk probe entry points.
1975 */
1976 static void vfio_vga_quirk_setup(VFIODevice *vdev)
1977 {
1978 vfio_vga_probe_ati_3c3_quirk(vdev);
1979 vfio_vga_probe_nvidia_3d0_quirk(vdev);
1980 }
1981
1982 static void vfio_vga_quirk_teardown(VFIODevice *vdev)
1983 {
1984 int i;
1985
1986 for (i = 0; i < ARRAY_SIZE(vdev->vga.region); i++) {
1987 while (!QLIST_EMPTY(&vdev->vga.region[i].quirks)) {
1988 VFIOQuirk *quirk = QLIST_FIRST(&vdev->vga.region[i].quirks);
1989 memory_region_del_subregion(&vdev->vga.region[i].mem, &quirk->mem);
1990 memory_region_destroy(&quirk->mem);
1991 QLIST_REMOVE(quirk, next);
1992 g_free(quirk);
1993 }
1994 }
1995 }
1996
1997 static void vfio_bar_quirk_setup(VFIODevice *vdev, int nr)
1998 {
1999 vfio_probe_ati_bar4_window_quirk(vdev, nr);
2000 vfio_probe_ati_bar2_4000_quirk(vdev, nr);
2001 vfio_probe_nvidia_bar5_window_quirk(vdev, nr);
2002 vfio_probe_nvidia_bar0_88000_quirk(vdev, nr);
2003 vfio_probe_nvidia_bar0_1800_quirk(vdev, nr);
2004 }
2005
2006 static void vfio_bar_quirk_teardown(VFIODevice *vdev, int nr)
2007 {
2008 VFIOBAR *bar = &vdev->bars[nr];
2009
2010 while (!QLIST_EMPTY(&bar->quirks)) {
2011 VFIOQuirk *quirk = QLIST_FIRST(&bar->quirks);
2012 memory_region_del_subregion(&bar->mem, &quirk->mem);
2013 memory_region_destroy(&quirk->mem);
2014 QLIST_REMOVE(quirk, next);
2015 g_free(quirk);
2016 }
2017 }
2018
2019 /*
2020 * PCI config space
2021 */
2022 static uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len)
2023 {
2024 VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
2025 uint32_t emu_bits = 0, emu_val = 0, phys_val = 0, val;
2026
2027 memcpy(&emu_bits, vdev->emulated_config_bits + addr, len);
2028 emu_bits = le32_to_cpu(emu_bits);
2029
2030 if (emu_bits) {
2031 emu_val = pci_default_read_config(pdev, addr, len);
2032 }
2033
2034 if (~emu_bits & (0xffffffffU >> (32 - len * 8))) {
2035 ssize_t ret;
2036
2037 ret = pread(vdev->fd, &phys_val, len, vdev->config_offset + addr);
2038 if (ret != len) {
2039 error_report("%s(%04x:%02x:%02x.%x, 0x%x, 0x%x) failed: %m",
2040 __func__, vdev->host.domain, vdev->host.bus,
2041 vdev->host.slot, vdev->host.function, addr, len);
2042 return -errno;
2043 }
2044 phys_val = le32_to_cpu(phys_val);
2045 }
2046
2047 val = (emu_val & emu_bits) | (phys_val & ~emu_bits);
2048
2049 DPRINTF("%s(%04x:%02x:%02x.%x, @0x%x, len=0x%x) %x\n", __func__,
2050 vdev->host.domain, vdev->host.bus, vdev->host.slot,
2051 vdev->host.function, addr, len, val);
2052
2053 return val;
2054 }
2055
2056 static void vfio_pci_write_config(PCIDevice *pdev, uint32_t addr,
2057 uint32_t val, int len)
2058 {
2059 VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
2060 uint32_t val_le = cpu_to_le32(val);
2061
2062 DPRINTF("%s(%04x:%02x:%02x.%x, @0x%x, 0x%x, len=0x%x)\n", __func__,
2063 vdev->host.domain, vdev->host.bus, vdev->host.slot,
2064 vdev->host.function, addr, val, len);
2065
2066 /* Write everything to VFIO, let it filter out what we can't write */
2067 if (pwrite(vdev->fd, &val_le, len, vdev->config_offset + addr) != len) {
2068 error_report("%s(%04x:%02x:%02x.%x, 0x%x, 0x%x, 0x%x) failed: %m",
2069 __func__, vdev->host.domain, vdev->host.bus,
2070 vdev->host.slot, vdev->host.function, addr, val, len);
2071 }
2072
2073 /* MSI/MSI-X Enabling/Disabling */
2074 if (pdev->cap_present & QEMU_PCI_CAP_MSI &&
2075 ranges_overlap(addr, len, pdev->msi_cap, vdev->msi_cap_size)) {
2076 int is_enabled, was_enabled = msi_enabled(pdev);
2077
2078 pci_default_write_config(pdev, addr, val, len);
2079
2080 is_enabled = msi_enabled(pdev);
2081
2082 if (!was_enabled) {
2083 if (is_enabled) {
2084 vfio_enable_msi(vdev);
2085 }
2086 } else {
2087 if (!is_enabled) {
2088 vfio_disable_msi(vdev);
2089 } else {
2090 vfio_update_msi(vdev);
2091 }
2092 }
2093 } else if (pdev->cap_present & QEMU_PCI_CAP_MSIX &&
2094 ranges_overlap(addr, len, pdev->msix_cap, MSIX_CAP_LENGTH)) {
2095 int is_enabled, was_enabled = msix_enabled(pdev);
2096
2097 pci_default_write_config(pdev, addr, val, len);
2098
2099 is_enabled = msix_enabled(pdev);
2100
2101 if (!was_enabled && is_enabled) {
2102 vfio_enable_msix(vdev);
2103 } else if (was_enabled && !is_enabled) {
2104 vfio_disable_msix(vdev);
2105 }
2106 } else {
2107 /* Write everything to QEMU to keep emulated bits correct */
2108 pci_default_write_config(pdev, addr, val, len);
2109 }
2110 }
2111
2112 /*
2113 * DMA - Mapping and unmapping for the "type1" IOMMU interface used on x86
2114 */
2115 static int vfio_dma_unmap(VFIOContainer *container,
2116 hwaddr iova, ram_addr_t size)
2117 {
2118 struct vfio_iommu_type1_dma_unmap unmap = {
2119 .argsz = sizeof(unmap),
2120 .flags = 0,
2121 .iova = iova,
2122 .size = size,
2123 };
2124
2125 if (ioctl(container->fd, VFIO_IOMMU_UNMAP_DMA, &unmap)) {
2126 DPRINTF("VFIO_UNMAP_DMA: %d\n", -errno);
2127 return -errno;
2128 }
2129
2130 return 0;
2131 }
2132
2133 static int vfio_dma_map(VFIOContainer *container, hwaddr iova,
2134 ram_addr_t size, void *vaddr, bool readonly)
2135 {
2136 struct vfio_iommu_type1_dma_map map = {
2137 .argsz = sizeof(map),
2138 .flags = VFIO_DMA_MAP_FLAG_READ,
2139 .vaddr = (__u64)(uintptr_t)vaddr,
2140 .iova = iova,
2141 .size = size,
2142 };
2143
2144 if (!readonly) {
2145 map.flags |= VFIO_DMA_MAP_FLAG_WRITE;
2146 }
2147
2148 /*
2149 * Try the mapping, if it fails with EBUSY, unmap the region and try
2150 * again. This shouldn't be necessary, but we sometimes see it in
2151 * the the VGA ROM space.
2152 */
2153 if (ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0 ||
2154 (errno == EBUSY && vfio_dma_unmap(container, iova, size) == 0 &&
2155 ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0)) {
2156 return 0;
2157 }
2158
2159 DPRINTF("VFIO_MAP_DMA: %d\n", -errno);
2160 return -errno;
2161 }
2162
2163 static bool vfio_listener_skipped_section(MemoryRegionSection *section)
2164 {
2165 return !memory_region_is_ram(section->mr) ||
2166 /*
2167 * Sizing an enabled 64-bit BAR can cause spurious mappings to
2168 * addresses in the upper part of the 64-bit address space. These
2169 * are never accessed by the CPU and beyond the address width of
2170 * some IOMMU hardware. TODO: VFIO should tell us the IOMMU width.
2171 */
2172 section->offset_within_address_space & (1ULL << 63);
2173 }
2174
2175 static void vfio_listener_region_add(MemoryListener *listener,
2176 MemoryRegionSection *section)
2177 {
2178 VFIOContainer *container = container_of(listener, VFIOContainer,
2179 iommu_data.type1.listener);
2180 hwaddr iova, end;
2181 void *vaddr;
2182 int ret;
2183
2184 assert(!memory_region_is_iommu(section->mr));
2185
2186 if (vfio_listener_skipped_section(section)) {
2187 DPRINTF("SKIPPING region_add %"HWADDR_PRIx" - %"PRIx64"\n",
2188 section->offset_within_address_space,
2189 section->offset_within_address_space +
2190 int128_get64(int128_sub(section->size, int128_one())));
2191 return;
2192 }
2193
2194 if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) !=
2195 (section->offset_within_region & ~TARGET_PAGE_MASK))) {
2196 error_report("%s received unaligned region", __func__);
2197 return;
2198 }
2199
2200 iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
2201 end = (section->offset_within_address_space + int128_get64(section->size)) &
2202 TARGET_PAGE_MASK;
2203
2204 if (iova >= end) {
2205 return;
2206 }
2207
2208 vaddr = memory_region_get_ram_ptr(section->mr) +
2209 section->offset_within_region +
2210 (iova - section->offset_within_address_space);
2211
2212 DPRINTF("region_add %"HWADDR_PRIx" - %"HWADDR_PRIx" [%p]\n",
2213 iova, end - 1, vaddr);
2214
2215 memory_region_ref(section->mr);
2216 ret = vfio_dma_map(container, iova, end - iova, vaddr, section->readonly);
2217 if (ret) {
2218 error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx", "
2219 "0x%"HWADDR_PRIx", %p) = %d (%m)",
2220 container, iova, end - iova, vaddr, ret);
2221
2222 /*
2223 * On the initfn path, store the first error in the container so we
2224 * can gracefully fail. Runtime, there's not much we can do other
2225 * than throw a hardware error.
2226 */
2227 if (!container->iommu_data.type1.initialized) {
2228 if (!container->iommu_data.type1.error) {
2229 container->iommu_data.type1.error = ret;
2230 }
2231 } else {
2232 hw_error("vfio: DMA mapping failed, unable to continue\n");
2233 }
2234 }
2235 }
2236
2237 static void vfio_listener_region_del(MemoryListener *listener,
2238 MemoryRegionSection *section)
2239 {
2240 VFIOContainer *container = container_of(listener, VFIOContainer,
2241 iommu_data.type1.listener);
2242 hwaddr iova, end;
2243 int ret;
2244
2245 if (vfio_listener_skipped_section(section)) {
2246 DPRINTF("SKIPPING region_del %"HWADDR_PRIx" - %"PRIx64"\n",
2247 section->offset_within_address_space,
2248 section->offset_within_address_space +
2249 int128_get64(int128_sub(section->size, int128_one())));
2250 return;
2251 }
2252
2253 if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) !=
2254 (section->offset_within_region & ~TARGET_PAGE_MASK))) {
2255 error_report("%s received unaligned region", __func__);
2256 return;
2257 }
2258
2259 iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
2260 end = (section->offset_within_address_space + int128_get64(section->size)) &
2261 TARGET_PAGE_MASK;
2262
2263 if (iova >= end) {
2264 return;
2265 }
2266
2267 DPRINTF("region_del %"HWADDR_PRIx" - %"HWADDR_PRIx"\n",
2268 iova, end - 1);
2269
2270 ret = vfio_dma_unmap(container, iova, end - iova);
2271 memory_region_unref(section->mr);
2272 if (ret) {
2273 error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", "
2274 "0x%"HWADDR_PRIx") = %d (%m)",
2275 container, iova, end - iova, ret);
2276 }
2277 }
2278
2279 static MemoryListener vfio_memory_listener = {
2280 .region_add = vfio_listener_region_add,
2281 .region_del = vfio_listener_region_del,
2282 };
2283
2284 static void vfio_listener_release(VFIOContainer *container)
2285 {
2286 memory_listener_unregister(&container->iommu_data.type1.listener);
2287 }
2288
2289 /*
2290 * Interrupt setup
2291 */
2292 static void vfio_disable_interrupts(VFIODevice *vdev)
2293 {
2294 switch (vdev->interrupt) {
2295 case VFIO_INT_INTx:
2296 vfio_disable_intx(vdev);
2297 break;
2298 case VFIO_INT_MSI:
2299 vfio_disable_msi(vdev);
2300 break;
2301 case VFIO_INT_MSIX:
2302 vfio_disable_msix(vdev);
2303 break;
2304 }
2305 }
2306
2307 static int vfio_setup_msi(VFIODevice *vdev, int pos)
2308 {
2309 uint16_t ctrl;
2310 bool msi_64bit, msi_maskbit;
2311 int ret, entries;
2312
2313 if (pread(vdev->fd, &ctrl, sizeof(ctrl),
2314 vdev->config_offset + pos + PCI_CAP_FLAGS) != sizeof(ctrl)) {
2315 return -errno;
2316 }
2317 ctrl = le16_to_cpu(ctrl);
2318
2319 msi_64bit = !!(ctrl & PCI_MSI_FLAGS_64BIT);
2320 msi_maskbit = !!(ctrl & PCI_MSI_FLAGS_MASKBIT);
2321 entries = 1 << ((ctrl & PCI_MSI_FLAGS_QMASK) >> 1);
2322
2323 DPRINTF("%04x:%02x:%02x.%x PCI MSI CAP @0x%x\n", vdev->host.domain,
2324 vdev->host.bus, vdev->host.slot, vdev->host.function, pos);
2325
2326 ret = msi_init(&vdev->pdev, pos, entries, msi_64bit, msi_maskbit);
2327 if (ret < 0) {
2328 if (ret == -ENOTSUP) {
2329 return 0;
2330 }
2331 error_report("vfio: msi_init failed");
2332 return ret;
2333 }
2334 vdev->msi_cap_size = 0xa + (msi_maskbit ? 0xa : 0) + (msi_64bit ? 0x4 : 0);
2335
2336 return 0;
2337 }
2338
2339 /*
2340 * We don't have any control over how pci_add_capability() inserts
2341 * capabilities into the chain. In order to setup MSI-X we need a
2342 * MemoryRegion for the BAR. In order to setup the BAR and not
2343 * attempt to mmap the MSI-X table area, which VFIO won't allow, we
2344 * need to first look for where the MSI-X table lives. So we
2345 * unfortunately split MSI-X setup across two functions.
2346 */
2347 static int vfio_early_setup_msix(VFIODevice *vdev)
2348 {
2349 uint8_t pos;
2350 uint16_t ctrl;
2351 uint32_t table, pba;
2352
2353 pos = pci_find_capability(&vdev->pdev, PCI_CAP_ID_MSIX);
2354 if (!pos) {
2355 return 0;
2356 }
2357
2358 if (pread(vdev->fd, &ctrl, sizeof(ctrl),
2359 vdev->config_offset + pos + PCI_CAP_FLAGS) != sizeof(ctrl)) {
2360 return -errno;
2361 }
2362
2363 if (pread(vdev->fd, &table, sizeof(table),
2364 vdev->config_offset + pos + PCI_MSIX_TABLE) != sizeof(table)) {
2365 return -errno;
2366 }
2367
2368 if (pread(vdev->fd, &pba, sizeof(pba),
2369 vdev->config_offset + pos + PCI_MSIX_PBA) != sizeof(pba)) {
2370 return -errno;
2371 }
2372
2373 ctrl = le16_to_cpu(ctrl);
2374 table = le32_to_cpu(table);
2375 pba = le32_to_cpu(pba);
2376
2377 vdev->msix = g_malloc0(sizeof(*(vdev->msix)));
2378 vdev->msix->table_bar = table & PCI_MSIX_FLAGS_BIRMASK;
2379 vdev->msix->table_offset = table & ~PCI_MSIX_FLAGS_BIRMASK;
2380 vdev->msix->pba_bar = pba & PCI_MSIX_FLAGS_BIRMASK;
2381 vdev->msix->pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK;
2382 vdev->msix->entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
2383
2384 DPRINTF("%04x:%02x:%02x.%x "
2385 "PCI MSI-X CAP @0x%x, BAR %d, offset 0x%x, entries %d\n",
2386 vdev->host.domain, vdev->host.bus, vdev->host.slot,
2387 vdev->host.function, pos, vdev->msix->table_bar,
2388 vdev->msix->table_offset, vdev->msix->entries);
2389
2390 return 0;
2391 }
2392
2393 static int vfio_setup_msix(VFIODevice *vdev, int pos)
2394 {
2395 int ret;
2396
2397 ret = msix_init(&vdev->pdev, vdev->msix->entries,
2398 &vdev->bars[vdev->msix->table_bar].mem,
2399 vdev->msix->table_bar, vdev->msix->table_offset,
2400 &vdev->bars[vdev->msix->pba_bar].mem,
2401 vdev->msix->pba_bar, vdev->msix->pba_offset, pos);
2402 if (ret < 0) {
2403 if (ret == -ENOTSUP) {
2404 return 0;
2405 }
2406 error_report("vfio: msix_init failed");
2407 return ret;
2408 }
2409
2410 return 0;
2411 }
2412
2413 static void vfio_teardown_msi(VFIODevice *vdev)
2414 {
2415 msi_uninit(&vdev->pdev);
2416
2417 if (vdev->msix) {
2418 msix_uninit(&vdev->pdev, &vdev->bars[vdev->msix->table_bar].mem,
2419 &vdev->bars[vdev->msix->pba_bar].mem);
2420 }
2421 }
2422
2423 /*
2424 * Resource setup
2425 */
2426 static void vfio_mmap_set_enabled(VFIODevice *vdev, bool enabled)
2427 {
2428 int i;
2429
2430 for (i = 0; i < PCI_ROM_SLOT; i++) {
2431 VFIOBAR *bar = &vdev->bars[i];
2432
2433 if (!bar->size) {
2434 continue;
2435 }
2436
2437 memory_region_set_enabled(&bar->mmap_mem, enabled);
2438 if (vdev->msix && vdev->msix->table_bar == i) {
2439 memory_region_set_enabled(&vdev->msix->mmap_mem, enabled);
2440 }
2441 }
2442 }
2443
2444 static void vfio_unmap_bar(VFIODevice *vdev, int nr)
2445 {
2446 VFIOBAR *bar = &vdev->bars[nr];
2447
2448 if (!bar->size) {
2449 return;
2450 }
2451
2452 vfio_bar_quirk_teardown(vdev, nr);
2453
2454 memory_region_del_subregion(&bar->mem, &bar->mmap_mem);
2455 munmap(bar->mmap, memory_region_size(&bar->mmap_mem));
2456 memory_region_destroy(&bar->mmap_mem);
2457
2458 if (vdev->msix && vdev->msix->table_bar == nr) {
2459 memory_region_del_subregion(&bar->mem, &vdev->msix->mmap_mem);
2460 munmap(vdev->msix->mmap, memory_region_size(&vdev->msix->mmap_mem));
2461 memory_region_destroy(&vdev->msix->mmap_mem);
2462 }
2463
2464 memory_region_destroy(&bar->mem);
2465 }
2466
2467 static int vfio_mmap_bar(VFIODevice *vdev, VFIOBAR *bar,
2468 MemoryRegion *mem, MemoryRegion *submem,
2469 void **map, size_t size, off_t offset,
2470 const char *name)
2471 {
2472 int ret = 0;
2473
2474 if (VFIO_ALLOW_MMAP && size && bar->flags & VFIO_REGION_INFO_FLAG_MMAP) {
2475 int prot = 0;
2476
2477 if (bar->flags & VFIO_REGION_INFO_FLAG_READ) {
2478 prot |= PROT_READ;
2479 }
2480
2481 if (bar->flags & VFIO_REGION_INFO_FLAG_WRITE) {
2482 prot |= PROT_WRITE;
2483 }
2484
2485 *map = mmap(NULL, size, prot, MAP_SHARED,
2486 bar->fd, bar->fd_offset + offset);
2487 if (*map == MAP_FAILED) {
2488 *map = NULL;
2489 ret = -errno;
2490 goto empty_region;
2491 }
2492
2493 memory_region_init_ram_ptr(submem, OBJECT(vdev), name, size, *map);
2494 } else {
2495 empty_region:
2496 /* Create a zero sized sub-region to make cleanup easy. */
2497 memory_region_init(submem, OBJECT(vdev), name, 0);
2498 }
2499
2500 memory_region_add_subregion(mem, offset, submem);
2501
2502 return ret;
2503 }
2504
2505 static void vfio_map_bar(VFIODevice *vdev, int nr)
2506 {
2507 VFIOBAR *bar = &vdev->bars[nr];
2508 unsigned size = bar->size;
2509 char name[64];
2510 uint32_t pci_bar;
2511 uint8_t type;
2512 int ret;
2513
2514 /* Skip both unimplemented BARs and the upper half of 64bit BARS. */
2515 if (!size) {
2516 return;
2517 }
2518
2519 snprintf(name, sizeof(name), "VFIO %04x:%02x:%02x.%x BAR %d",
2520 vdev->host.domain, vdev->host.bus, vdev->host.slot,
2521 vdev->host.function, nr);
2522
2523 /* Determine what type of BAR this is for registration */
2524 ret = pread(vdev->fd, &pci_bar, sizeof(pci_bar),
2525 vdev->config_offset + PCI_BASE_ADDRESS_0 + (4 * nr));
2526 if (ret != sizeof(pci_bar)) {
2527 error_report("vfio: Failed to read BAR %d (%m)", nr);
2528 return;
2529 }
2530
2531 pci_bar = le32_to_cpu(pci_bar);
2532 bar->ioport = (pci_bar & PCI_BASE_ADDRESS_SPACE_IO);
2533 bar->mem64 = bar->ioport ? 0 : (pci_bar & PCI_BASE_ADDRESS_MEM_TYPE_64);
2534 type = pci_bar & (bar->ioport ? ~PCI_BASE_ADDRESS_IO_MASK :
2535 ~PCI_BASE_ADDRESS_MEM_MASK);
2536
2537 /* A "slow" read/write mapping underlies all BARs */
2538 memory_region_init_io(&bar->mem, OBJECT(vdev), &vfio_bar_ops,
2539 bar, name, size);
2540 pci_register_bar(&vdev->pdev, nr, type, &bar->mem);
2541
2542 /*
2543 * We can't mmap areas overlapping the MSIX vector table, so we
2544 * potentially insert a direct-mapped subregion before and after it.
2545 */
2546 if (vdev->msix && vdev->msix->table_bar == nr) {
2547 size = vdev->msix->table_offset & qemu_host_page_mask;
2548 }
2549
2550 strncat(name, " mmap", sizeof(name) - strlen(name) - 1);
2551 if (vfio_mmap_bar(vdev, bar, &bar->mem,
2552 &bar->mmap_mem, &bar->mmap, size, 0, name)) {
2553 error_report("%s unsupported. Performance may be slow", name);
2554 }
2555
2556 if (vdev->msix && vdev->msix->table_bar == nr) {
2557 unsigned start;
2558
2559 start = HOST_PAGE_ALIGN(vdev->msix->table_offset +
2560 (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE));
2561
2562 size = start < bar->size ? bar->size - start : 0;
2563 strncat(name, " msix-hi", sizeof(name) - strlen(name) - 1);
2564 /* VFIOMSIXInfo contains another MemoryRegion for this mapping */
2565 if (vfio_mmap_bar(vdev, bar, &bar->mem, &vdev->msix->mmap_mem,
2566 &vdev->msix->mmap, size, start, name)) {
2567 error_report("%s unsupported. Performance may be slow", name);
2568 }
2569 }
2570
2571 vfio_bar_quirk_setup(vdev, nr);
2572 }
2573
2574 static void vfio_map_bars(VFIODevice *vdev)
2575 {
2576 int i;
2577
2578 for (i = 0; i < PCI_ROM_SLOT; i++) {
2579 vfio_map_bar(vdev, i);
2580 }
2581
2582 if (vdev->has_vga) {
2583 memory_region_init_io(&vdev->vga.region[QEMU_PCI_VGA_MEM].mem,
2584 OBJECT(vdev), &vfio_vga_ops,
2585 &vdev->vga.region[QEMU_PCI_VGA_MEM],
2586 "vfio-vga-mmio@0xa0000",
2587 QEMU_PCI_VGA_MEM_SIZE);
2588 memory_region_init_io(&vdev->vga.region[QEMU_PCI_VGA_IO_LO].mem,
2589 OBJECT(vdev), &vfio_vga_ops,
2590 &vdev->vga.region[QEMU_PCI_VGA_IO_LO],
2591 "vfio-vga-io@0x3b0",
2592 QEMU_PCI_VGA_IO_LO_SIZE);
2593 memory_region_init_io(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem,
2594 OBJECT(vdev), &vfio_vga_ops,
2595 &vdev->vga.region[QEMU_PCI_VGA_IO_HI],
2596 "vfio-vga-io@0x3c0",
2597 QEMU_PCI_VGA_IO_HI_SIZE);
2598
2599 pci_register_vga(&vdev->pdev, &vdev->vga.region[QEMU_PCI_VGA_MEM].mem,
2600 &vdev->vga.region[QEMU_PCI_VGA_IO_LO].mem,
2601 &vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem);
2602 vfio_vga_quirk_setup(vdev);
2603 }
2604 }
2605
2606 static void vfio_unmap_bars(VFIODevice *vdev)
2607 {
2608 int i;
2609
2610 for (i = 0; i < PCI_ROM_SLOT; i++) {
2611 vfio_unmap_bar(vdev, i);
2612 }
2613
2614 if (vdev->has_vga) {
2615 vfio_vga_quirk_teardown(vdev);
2616 pci_unregister_vga(&vdev->pdev);
2617 memory_region_destroy(&vdev->vga.region[QEMU_PCI_VGA_MEM].mem);
2618 memory_region_destroy(&vdev->vga.region[QEMU_PCI_VGA_IO_LO].mem);
2619 memory_region_destroy(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem);
2620 }
2621 }
2622
2623 /*
2624 * General setup
2625 */
2626 static uint8_t vfio_std_cap_max_size(PCIDevice *pdev, uint8_t pos)
2627 {
2628 uint8_t tmp, next = 0xff;
2629
2630 for (tmp = pdev->config[PCI_CAPABILITY_LIST]; tmp;
2631 tmp = pdev->config[tmp + 1]) {
2632 if (tmp > pos && tmp < next) {
2633 next = tmp;
2634 }
2635 }
2636
2637 return next - pos;
2638 }
2639
2640 static void vfio_set_word_bits(uint8_t *buf, uint16_t val, uint16_t mask)
2641 {
2642 pci_set_word(buf, (pci_get_word(buf) & ~mask) | val);
2643 }
2644
2645 static void vfio_add_emulated_word(VFIODevice *vdev, int pos,
2646 uint16_t val, uint16_t mask)
2647 {
2648 vfio_set_word_bits(vdev->pdev.config + pos, val, mask);
2649 vfio_set_word_bits(vdev->pdev.wmask + pos, ~mask, mask);
2650 vfio_set_word_bits(vdev->emulated_config_bits + pos, mask, mask);
2651 }
2652
2653 static void vfio_set_long_bits(uint8_t *buf, uint32_t val, uint32_t mask)
2654 {
2655 pci_set_long(buf, (pci_get_long(buf) & ~mask) | val);
2656 }
2657
2658 static void vfio_add_emulated_long(VFIODevice *vdev, int pos,
2659 uint32_t val, uint32_t mask)
2660 {
2661 vfio_set_long_bits(vdev->pdev.config + pos, val, mask);
2662 vfio_set_long_bits(vdev->pdev.wmask + pos, ~mask, mask);
2663 vfio_set_long_bits(vdev->emulated_config_bits + pos, mask, mask);
2664 }
2665
2666 static int vfio_setup_pcie_cap(VFIODevice *vdev, int pos, uint8_t size)
2667 {
2668 uint16_t flags;
2669 uint8_t type;
2670
2671 flags = pci_get_word(vdev->pdev.config + pos + PCI_CAP_FLAGS);
2672 type = (flags & PCI_EXP_FLAGS_TYPE) >> 4;
2673
2674 if (type != PCI_EXP_TYPE_ENDPOINT &&
2675 type != PCI_EXP_TYPE_LEG_END &&
2676 type != PCI_EXP_TYPE_RC_END) {
2677
2678 error_report("vfio: Assignment of PCIe type 0x%x "
2679 "devices is not currently supported", type);
2680 return -EINVAL;
2681 }
2682
2683 if (!pci_bus_is_express(vdev->pdev.bus)) {
2684 /*
2685 * Use express capability as-is on PCI bus. It doesn't make much
2686 * sense to even expose, but some drivers (ex. tg3) depend on it
2687 * and guests don't seem to be particular about it. We'll need
2688 * to revist this or force express devices to express buses if we
2689 * ever expose an IOMMU to the guest.
2690 */
2691 } else if (pci_bus_is_root(vdev->pdev.bus)) {
2692 /*
2693 * On a Root Complex bus Endpoints become Root Complex Integrated
2694 * Endpoints, which changes the type and clears the LNK & LNK2 fields.
2695 */
2696 if (type == PCI_EXP_TYPE_ENDPOINT) {
2697 vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
2698 PCI_EXP_TYPE_RC_END << 4,
2699 PCI_EXP_FLAGS_TYPE);
2700
2701 /* Link Capabilities, Status, and Control goes away */
2702 if (size > PCI_EXP_LNKCTL) {
2703 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP, 0, ~0);
2704 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0);
2705 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA, 0, ~0);
2706
2707 #ifndef PCI_EXP_LNKCAP2
2708 #define PCI_EXP_LNKCAP2 44
2709 #endif
2710 #ifndef PCI_EXP_LNKSTA2
2711 #define PCI_EXP_LNKSTA2 50
2712 #endif
2713 /* Link 2 Capabilities, Status, and Control goes away */
2714 if (size > PCI_EXP_LNKCAP2) {
2715 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP2, 0, ~0);
2716 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL2, 0, ~0);
2717 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA2, 0, ~0);
2718 }
2719 }
2720
2721 } else if (type == PCI_EXP_TYPE_LEG_END) {
2722 /*
2723 * Legacy endpoints don't belong on the root complex. Windows
2724 * seems to be happier with devices if we skip the capability.
2725 */
2726 return 0;
2727 }
2728
2729 } else {
2730 /*
2731 * Convert Root Complex Integrated Endpoints to regular endpoints.
2732 * These devices don't support LNK/LNK2 capabilities, so make them up.
2733 */
2734 if (type == PCI_EXP_TYPE_RC_END) {
2735 vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
2736 PCI_EXP_TYPE_ENDPOINT << 4,
2737 PCI_EXP_FLAGS_TYPE);
2738 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP,
2739 PCI_EXP_LNK_MLW_1 | PCI_EXP_LNK_LS_25, ~0);
2740 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0);
2741 }
2742
2743 /* Mark the Link Status bits as emulated to allow virtual negotiation */
2744 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA,
2745 pci_get_word(vdev->pdev.config + pos +
2746 PCI_EXP_LNKSTA),
2747 PCI_EXP_LNKCAP_MLW | PCI_EXP_LNKCAP_SLS);
2748 }
2749
2750 pos = pci_add_capability(&vdev->pdev, PCI_CAP_ID_EXP, pos, size);
2751 if (pos >= 0) {
2752 vdev->pdev.exp.exp_cap = pos;
2753 }
2754
2755 return pos;
2756 }
2757
2758 static void vfio_check_pcie_flr(VFIODevice *vdev, uint8_t pos)
2759 {
2760 uint32_t cap = pci_get_long(vdev->pdev.config + pos + PCI_EXP_DEVCAP);
2761
2762 if (cap & PCI_EXP_DEVCAP_FLR) {
2763 DPRINTF("%04x:%02x:%02x.%x Supports FLR via PCIe cap\n",
2764 vdev->host.domain, vdev->host.bus, vdev->host.slot,
2765 vdev->host.function);
2766 vdev->has_flr = true;
2767 }
2768 }
2769
2770 static void vfio_check_pm_reset(VFIODevice *vdev, uint8_t pos)
2771 {
2772 uint16_t csr = pci_get_word(vdev->pdev.config + pos + PCI_PM_CTRL);
2773
2774 if (!(csr & PCI_PM_CTRL_NO_SOFT_RESET)) {
2775 DPRINTF("%04x:%02x:%02x.%x Supports PM reset\n",
2776 vdev->host.domain, vdev->host.bus, vdev->host.slot,
2777 vdev->host.function);
2778 vdev->has_pm_reset = true;
2779 }
2780 }
2781
2782 static void vfio_check_af_flr(VFIODevice *vdev, uint8_t pos)
2783 {
2784 uint8_t cap = pci_get_byte(vdev->pdev.config + pos + PCI_AF_CAP);
2785
2786 if ((cap & PCI_AF_CAP_TP) && (cap & PCI_AF_CAP_FLR)) {
2787 DPRINTF("%04x:%02x:%02x.%x Supports FLR via AF cap\n",
2788 vdev->host.domain, vdev->host.bus, vdev->host.slot,
2789 vdev->host.function);
2790 vdev->has_flr = true;
2791 }
2792 }
2793
2794 static int vfio_add_std_cap(VFIODevice *vdev, uint8_t pos)
2795 {
2796 PCIDevice *pdev = &vdev->pdev;
2797 uint8_t cap_id, next, size;
2798 int ret;
2799
2800 cap_id = pdev->config[pos];
2801 next = pdev->config[pos + 1];
2802
2803 /*
2804 * If it becomes important to configure capabilities to their actual
2805 * size, use this as the default when it's something we don't recognize.
2806 * Since QEMU doesn't actually handle many of the config accesses,
2807 * exact size doesn't seem worthwhile.
2808 */
2809 size = vfio_std_cap_max_size(pdev, pos);
2810
2811 /*
2812 * pci_add_capability always inserts the new capability at the head
2813 * of the chain. Therefore to end up with a chain that matches the
2814 * physical device, we insert from the end by making this recursive.
2815 * This is also why we pre-caclulate size above as cached config space
2816 * will be changed as we unwind the stack.
2817 */
2818 if (next) {
2819 ret = vfio_add_std_cap(vdev, next);
2820 if (ret) {
2821 return ret;
2822 }
2823 } else {
2824 /* Begin the rebuild, use QEMU emulated list bits */
2825 pdev->config[PCI_CAPABILITY_LIST] = 0;
2826 vdev->emulated_config_bits[PCI_CAPABILITY_LIST] = 0xff;
2827 vdev->emulated_config_bits[PCI_STATUS] |= PCI_STATUS_CAP_LIST;
2828 }
2829
2830 /* Use emulated next pointer to allow dropping caps */
2831 pci_set_byte(vdev->emulated_config_bits + pos + 1, 0xff);
2832
2833 switch (cap_id) {
2834 case PCI_CAP_ID_MSI:
2835 ret = vfio_setup_msi(vdev, pos);
2836 break;
2837 case PCI_CAP_ID_EXP:
2838 vfio_check_pcie_flr(vdev, pos);
2839 ret = vfio_setup_pcie_cap(vdev, pos, size);
2840 break;
2841 case PCI_CAP_ID_MSIX:
2842 ret = vfio_setup_msix(vdev, pos);
2843 break;
2844 case PCI_CAP_ID_PM:
2845 vfio_check_pm_reset(vdev, pos);
2846 vdev->pm_cap = pos;
2847 ret = pci_add_capability(pdev, cap_id, pos, size);
2848 break;
2849 case PCI_CAP_ID_AF:
2850 vfio_check_af_flr(vdev, pos);
2851 ret = pci_add_capability(pdev, cap_id, pos, size);
2852 break;
2853 default:
2854 ret = pci_add_capability(pdev, cap_id, pos, size);
2855 break;
2856 }
2857
2858 if (ret < 0) {
2859 error_report("vfio: %04x:%02x:%02x.%x Error adding PCI capability "
2860 "0x%x[0x%x]@0x%x: %d", vdev->host.domain,
2861 vdev->host.bus, vdev->host.slot, vdev->host.function,
2862 cap_id, size, pos, ret);
2863 return ret;
2864 }
2865
2866 return 0;
2867 }
2868
2869 static int vfio_add_capabilities(VFIODevice *vdev)
2870 {
2871 PCIDevice *pdev = &vdev->pdev;
2872
2873 if (!(pdev->config[PCI_STATUS] & PCI_STATUS_CAP_LIST) ||
2874 !pdev->config[PCI_CAPABILITY_LIST]) {
2875 return 0; /* Nothing to add */
2876 }
2877
2878 return vfio_add_std_cap(vdev, pdev->config[PCI_CAPABILITY_LIST]);
2879 }
2880
2881 static void vfio_pci_pre_reset(VFIODevice *vdev)
2882 {
2883 PCIDevice *pdev = &vdev->pdev;
2884 uint16_t cmd;
2885
2886 vfio_disable_interrupts(vdev);
2887
2888 /* Make sure the device is in D0 */
2889 if (vdev->pm_cap) {
2890 uint16_t pmcsr;
2891 uint8_t state;
2892
2893 pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2);
2894 state = pmcsr & PCI_PM_CTRL_STATE_MASK;
2895 if (state) {
2896 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
2897 vfio_pci_write_config(pdev, vdev->pm_cap + PCI_PM_CTRL, pmcsr, 2);
2898 /* vfio handles the necessary delay here */
2899 pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2);
2900 state = pmcsr & PCI_PM_CTRL_STATE_MASK;
2901 if (state) {
2902 error_report("vfio: Unable to power on device, stuck in D%d\n",
2903 state);
2904 }
2905 }
2906 }
2907
2908 /*
2909 * Stop any ongoing DMA by disconecting I/O, MMIO, and bus master.
2910 * Also put INTx Disable in known state.
2911 */
2912 cmd = vfio_pci_read_config(pdev, PCI_COMMAND, 2);
2913 cmd &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
2914 PCI_COMMAND_INTX_DISABLE);
2915 vfio_pci_write_config(pdev, PCI_COMMAND, cmd, 2);
2916 }
2917
2918 static void vfio_pci_post_reset(VFIODevice *vdev)
2919 {
2920 vfio_enable_intx(vdev);
2921 }
2922
2923 static bool vfio_pci_host_match(PCIHostDeviceAddress *host1,
2924 PCIHostDeviceAddress *host2)
2925 {
2926 return (host1->domain == host2->domain && host1->bus == host2->bus &&
2927 host1->slot == host2->slot && host1->function == host2->function);
2928 }
2929
2930 static int vfio_pci_hot_reset(VFIODevice *vdev, bool single)
2931 {
2932 VFIOGroup *group;
2933 struct vfio_pci_hot_reset_info *info;
2934 struct vfio_pci_dependent_device *devices;
2935 struct vfio_pci_hot_reset *reset;
2936 int32_t *fds;
2937 int ret, i, count;
2938 bool multi = false;
2939
2940 DPRINTF("%s(%04x:%02x:%02x.%x) %s\n", __func__, vdev->host.domain,
2941 vdev->host.bus, vdev->host.slot, vdev->host.function,
2942 single ? "one" : "multi");
2943
2944 vfio_pci_pre_reset(vdev);
2945 vdev->needs_reset = false;
2946
2947 info = g_malloc0(sizeof(*info));
2948 info->argsz = sizeof(*info);
2949
2950 ret = ioctl(vdev->fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
2951 if (ret && errno != ENOSPC) {
2952 ret = -errno;
2953 if (!vdev->has_pm_reset) {
2954 error_report("vfio: Cannot reset device %04x:%02x:%02x.%x, "
2955 "no available reset mechanism.", vdev->host.domain,
2956 vdev->host.bus, vdev->host.slot, vdev->host.function);
2957 }
2958 goto out_single;
2959 }
2960
2961 count = info->count;
2962 info = g_realloc(info, sizeof(*info) + (count * sizeof(*devices)));
2963 info->argsz = sizeof(*info) + (count * sizeof(*devices));
2964 devices = &info->devices[0];
2965
2966 ret = ioctl(vdev->fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
2967 if (ret) {
2968 ret = -errno;
2969 error_report("vfio: hot reset info failed: %m");
2970 goto out_single;
2971 }
2972
2973 DPRINTF("%04x:%02x:%02x.%x: hot reset dependent devices:\n",
2974 vdev->host.domain, vdev->host.bus, vdev->host.slot,
2975 vdev->host.function);
2976
2977 /* Verify that we have all the groups required */
2978 for (i = 0; i < info->count; i++) {
2979 PCIHostDeviceAddress host;
2980 VFIODevice *tmp;
2981
2982 host.domain = devices[i].segment;
2983 host.bus = devices[i].bus;
2984 host.slot = PCI_SLOT(devices[i].devfn);
2985 host.function = PCI_FUNC(devices[i].devfn);
2986
2987 DPRINTF("\t%04x:%02x:%02x.%x group %d\n", host.domain,
2988 host.bus, host.slot, host.function, devices[i].group_id);
2989
2990 if (vfio_pci_host_match(&host, &vdev->host)) {
2991 continue;
2992 }
2993
2994 QLIST_FOREACH(group, &group_list, next) {
2995 if (group->groupid == devices[i].group_id) {
2996 break;
2997 }
2998 }
2999
3000 if (!group) {
3001 if (!vdev->has_pm_reset) {
3002 error_report("vfio: Cannot reset device %04x:%02x:%02x.%x, "
3003 "depends on group %d which is not owned.",
3004 vdev->host.domain, vdev->host.bus, vdev->host.slot,
3005 vdev->host.function, devices[i].group_id);
3006 }
3007 ret = -EPERM;
3008 goto out;
3009 }
3010
3011 /* Prep dependent devices for reset and clear our marker. */
3012 QLIST_FOREACH(tmp, &group->device_list, next) {
3013 if (vfio_pci_host_match(&host, &tmp->host)) {
3014 if (single) {
3015 DPRINTF("vfio: found another in-use device "
3016 "%04x:%02x:%02x.%x\n", host.domain, host.bus,
3017 host.slot, host.function);
3018 ret = -EINVAL;
3019 goto out_single;
3020 }
3021 vfio_pci_pre_reset(tmp);
3022 tmp->needs_reset = false;
3023 multi = true;
3024 break;
3025 }
3026 }
3027 }
3028
3029 if (!single && !multi) {
3030 DPRINTF("vfio: No other in-use devices for multi hot reset\n");
3031 ret = -EINVAL;
3032 goto out_single;
3033 }
3034
3035 /* Determine how many group fds need to be passed */
3036 count = 0;
3037 QLIST_FOREACH(group, &group_list, next) {
3038 for (i = 0; i < info->count; i++) {
3039 if (group->groupid == devices[i].group_id) {
3040 count++;
3041 break;
3042 }
3043 }
3044 }
3045
3046 reset = g_malloc0(sizeof(*reset) + (count * sizeof(*fds)));
3047 reset->argsz = sizeof(*reset) + (count * sizeof(*fds));
3048 fds = &reset->group_fds[0];
3049
3050 /* Fill in group fds */
3051 QLIST_FOREACH(group, &group_list, next) {
3052 for (i = 0; i < info->count; i++) {
3053 if (group->groupid == devices[i].group_id) {
3054 fds[reset->count++] = group->fd;
3055 break;
3056 }
3057 }
3058 }
3059
3060 /* Bus reset! */
3061 ret = ioctl(vdev->fd, VFIO_DEVICE_PCI_HOT_RESET, reset);
3062 g_free(reset);
3063
3064 DPRINTF("%04x:%02x:%02x.%x hot reset: %s\n", vdev->host.domain,
3065 vdev->host.bus, vdev->host.slot, vdev->host.function,
3066 ret ? "%m" : "Success");
3067
3068 out:
3069 /* Re-enable INTx on affected devices */
3070 for (i = 0; i < info->count; i++) {
3071 PCIHostDeviceAddress host;
3072 VFIODevice *tmp;
3073
3074 host.domain = devices[i].segment;
3075 host.bus = devices[i].bus;
3076 host.slot = PCI_SLOT(devices[i].devfn);
3077 host.function = PCI_FUNC(devices[i].devfn);
3078
3079 if (vfio_pci_host_match(&host, &vdev->host)) {
3080 continue;
3081 }
3082
3083 QLIST_FOREACH(group, &group_list, next) {
3084 if (group->groupid == devices[i].group_id) {
3085 break;
3086 }
3087 }
3088
3089 if (!group) {
3090 break;
3091 }
3092
3093 QLIST_FOREACH(tmp, &group->device_list, next) {
3094 if (vfio_pci_host_match(&host, &tmp->host)) {
3095 vfio_pci_post_reset(tmp);
3096 break;
3097 }
3098 }
3099 }
3100 out_single:
3101 vfio_pci_post_reset(vdev);
3102 g_free(info);
3103
3104 return ret;
3105 }
3106
3107 /*
3108 * We want to differentiate hot reset of mulitple in-use devices vs hot reset
3109 * of a single in-use device. VFIO_DEVICE_RESET will already handle the case
3110 * of doing hot resets when there is only a single device per bus. The in-use
3111 * here refers to how many VFIODevices are affected. A hot reset that affects
3112 * multiple devices, but only a single in-use device, means that we can call
3113 * it from our bus ->reset() callback since the extent is effectively a single
3114 * device. This allows us to make use of it in the hotplug path. When there
3115 * are multiple in-use devices, we can only trigger the hot reset during a
3116 * system reset and thus from our reset handler. We separate _one vs _multi
3117 * here so that we don't overlap and do a double reset on the system reset
3118 * path where both our reset handler and ->reset() callback are used. Calling
3119 * _one() will only do a hot reset for the one in-use devices case, calling
3120 * _multi() will do nothing if a _one() would have been sufficient.
3121 */
3122 static int vfio_pci_hot_reset_one(VFIODevice *vdev)
3123 {
3124 return vfio_pci_hot_reset(vdev, true);
3125 }
3126
3127 static int vfio_pci_hot_reset_multi(VFIODevice *vdev)
3128 {
3129 return vfio_pci_hot_reset(vdev, false);
3130 }
3131
3132 static void vfio_pci_reset_handler(void *opaque)
3133 {
3134 VFIOGroup *group;
3135 VFIODevice *vdev;
3136
3137 QLIST_FOREACH(group, &group_list, next) {
3138 QLIST_FOREACH(vdev, &group->device_list, next) {
3139 if (!vdev->reset_works || (!vdev->has_flr && vdev->has_pm_reset)) {
3140 vdev->needs_reset = true;
3141 }
3142 }
3143 }
3144
3145 QLIST_FOREACH(group, &group_list, next) {
3146 QLIST_FOREACH(vdev, &group->device_list, next) {
3147 if (vdev->needs_reset) {
3148 vfio_pci_hot_reset_multi(vdev);
3149 }
3150 }
3151 }
3152 }
3153
3154 static void vfio_kvm_device_add_group(VFIOGroup *group)
3155 {
3156 #ifdef CONFIG_KVM
3157 struct kvm_device_attr attr = {
3158 .group = KVM_DEV_VFIO_GROUP,
3159 .attr = KVM_DEV_VFIO_GROUP_ADD,
3160 .addr = (uint64_t)(unsigned long)&group->fd,
3161 };
3162
3163 if (!kvm_enabled()) {
3164 return;
3165 }
3166
3167 if (vfio_kvm_device_fd < 0) {
3168 struct kvm_create_device cd = {
3169 .type = KVM_DEV_TYPE_VFIO,
3170 };
3171
3172 if (kvm_vm_ioctl(kvm_state, KVM_CREATE_DEVICE, &cd)) {
3173 DPRINTF("KVM_CREATE_DEVICE: %m\n");
3174 return;
3175 }
3176
3177 vfio_kvm_device_fd = cd.fd;
3178 }
3179
3180 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) {
3181 error_report("Failed to add group %d to KVM VFIO device: %m",
3182 group->groupid);
3183 }
3184 #endif
3185 }
3186
3187 static void vfio_kvm_device_del_group(VFIOGroup *group)
3188 {
3189 #ifdef CONFIG_KVM
3190 struct kvm_device_attr attr = {
3191 .group = KVM_DEV_VFIO_GROUP,
3192 .attr = KVM_DEV_VFIO_GROUP_DEL,
3193 .addr = (uint64_t)(unsigned long)&group->fd,
3194 };
3195
3196 if (vfio_kvm_device_fd < 0) {
3197 return;
3198 }
3199
3200 if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) {
3201 error_report("Failed to remove group %d to KVM VFIO device: %m",
3202 group->groupid);
3203 }
3204 #endif
3205 }
3206
3207 static int vfio_connect_container(VFIOGroup *group)
3208 {
3209 VFIOContainer *container;
3210 int ret, fd;
3211
3212 if (group->container) {
3213 return 0;
3214 }
3215
3216 QLIST_FOREACH(container, &container_list, next) {
3217 if (!ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &container->fd)) {
3218 group->container = container;
3219 QLIST_INSERT_HEAD(&container->group_list, group, container_next);
3220 return 0;
3221 }
3222 }
3223
3224 fd = qemu_open("/dev/vfio/vfio", O_RDWR);
3225 if (fd < 0) {
3226 error_report("vfio: failed to open /dev/vfio/vfio: %m");
3227 return -errno;
3228 }
3229
3230 ret = ioctl(fd, VFIO_GET_API_VERSION);
3231 if (ret != VFIO_API_VERSION) {
3232 error_report("vfio: supported vfio version: %d, "
3233 "reported version: %d", VFIO_API_VERSION, ret);
3234 close(fd);
3235 return -EINVAL;
3236 }
3237
3238 container = g_malloc0(sizeof(*container));
3239 container->fd = fd;
3240
3241 if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU)) {
3242 ret = ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &fd);
3243 if (ret) {
3244 error_report("vfio: failed to set group container: %m");
3245 g_free(container);
3246 close(fd);
3247 return -errno;
3248 }
3249
3250 ret = ioctl(fd, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU);
3251 if (ret) {
3252 error_report("vfio: failed to set iommu for container: %m");
3253 g_free(container);
3254 close(fd);
3255 return -errno;
3256 }
3257
3258 container->iommu_data.type1.listener = vfio_memory_listener;
3259 container->iommu_data.release = vfio_listener_release;
3260
3261 memory_listener_register(&container->iommu_data.type1.listener,
3262 &address_space_memory);
3263
3264 if (container->iommu_data.type1.error) {
3265 ret = container->iommu_data.type1.error;
3266 vfio_listener_release(container);
3267 g_free(container);
3268 close(fd);
3269 error_report("vfio: memory listener initialization failed for container\n");
3270 return ret;
3271 }
3272
3273 container->iommu_data.type1.initialized = true;
3274
3275 } else {
3276 error_report("vfio: No available IOMMU models");
3277 g_free(container);
3278 close(fd);
3279 return -EINVAL;
3280 }
3281
3282 QLIST_INIT(&container->group_list);
3283 QLIST_INSERT_HEAD(&container_list, container, next);
3284
3285 group->container = container;
3286 QLIST_INSERT_HEAD(&container->group_list, group, container_next);
3287
3288 return 0;
3289 }
3290
3291 static void vfio_disconnect_container(VFIOGroup *group)
3292 {
3293 VFIOContainer *container = group->container;
3294
3295 if (ioctl(group->fd, VFIO_GROUP_UNSET_CONTAINER, &container->fd)) {
3296 error_report("vfio: error disconnecting group %d from container",
3297 group->groupid);
3298 }
3299
3300 QLIST_REMOVE(group, container_next);
3301 group->container = NULL;
3302
3303 if (QLIST_EMPTY(&container->group_list)) {
3304 if (container->iommu_data.release) {
3305 container->iommu_data.release(container);
3306 }
3307 QLIST_REMOVE(container, next);
3308 DPRINTF("vfio_disconnect_container: close container->fd\n");
3309 close(container->fd);
3310 g_free(container);
3311 }
3312 }
3313
3314 static VFIOGroup *vfio_get_group(int groupid)
3315 {
3316 VFIOGroup *group;
3317 char path[32];
3318 struct vfio_group_status status = { .argsz = sizeof(status) };
3319
3320 QLIST_FOREACH(group, &group_list, next) {
3321 if (group->groupid == groupid) {
3322 return group;
3323 }
3324 }
3325
3326 group = g_malloc0(sizeof(*group));
3327
3328 snprintf(path, sizeof(path), "/dev/vfio/%d", groupid);
3329 group->fd = qemu_open(path, O_RDWR);
3330 if (group->fd < 0) {
3331 error_report("vfio: error opening %s: %m", path);
3332 g_free(group);
3333 return NULL;
3334 }
3335
3336 if (ioctl(group->fd, VFIO_GROUP_GET_STATUS, &status)) {
3337 error_report("vfio: error getting group status: %m");
3338 close(group->fd);
3339 g_free(group);
3340 return NULL;
3341 }
3342
3343 if (!(status.flags & VFIO_GROUP_FLAGS_VIABLE)) {
3344 error_report("vfio: error, group %d is not viable, please ensure "
3345 "all devices within the iommu_group are bound to their "
3346 "vfio bus driver.", groupid);
3347 close(group->fd);
3348 g_free(group);
3349 return NULL;
3350 }
3351
3352 group->groupid = groupid;
3353 QLIST_INIT(&group->device_list);
3354
3355 if (vfio_connect_container(group)) {
3356 error_report("vfio: failed to setup container for group %d", groupid);
3357 close(group->fd);
3358 g_free(group);
3359 return NULL;
3360 }
3361
3362 if (QLIST_EMPTY(&group_list)) {
3363 qemu_register_reset(vfio_pci_reset_handler, NULL);
3364 }
3365
3366 QLIST_INSERT_HEAD(&group_list, group, next);
3367
3368 vfio_kvm_device_add_group(group);
3369
3370 return group;
3371 }
3372
3373 static void vfio_put_group(VFIOGroup *group)
3374 {
3375 if (!QLIST_EMPTY(&group->device_list)) {
3376 return;
3377 }
3378
3379 vfio_kvm_device_del_group(group);
3380 vfio_disconnect_container(group);
3381 QLIST_REMOVE(group, next);
3382 DPRINTF("vfio_put_group: close group->fd\n");
3383 close(group->fd);
3384 g_free(group);
3385
3386 if (QLIST_EMPTY(&group_list)) {
3387 qemu_unregister_reset(vfio_pci_reset_handler, NULL);
3388 }
3389 }
3390
3391 static int vfio_get_device(VFIOGroup *group, const char *name, VFIODevice *vdev)
3392 {
3393 struct vfio_device_info dev_info = { .argsz = sizeof(dev_info) };
3394 struct vfio_region_info reg_info = { .argsz = sizeof(reg_info) };
3395 struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info) };
3396 int ret, i;
3397
3398 ret = ioctl(group->fd, VFIO_GROUP_GET_DEVICE_FD, name);
3399 if (ret < 0) {
3400 error_report("vfio: error getting device %s from group %d: %m",
3401 name, group->groupid);
3402 error_printf("Verify all devices in group %d are bound to vfio-pci "
3403 "or pci-stub and not already in use\n", group->groupid);
3404 return ret;
3405 }
3406
3407 vdev->fd = ret;
3408 vdev->group = group;
3409 QLIST_INSERT_HEAD(&group->device_list, vdev, next);
3410
3411 /* Sanity check device */
3412 ret = ioctl(vdev->fd, VFIO_DEVICE_GET_INFO, &dev_info);
3413 if (ret) {
3414 error_report("vfio: error getting device info: %m");
3415 goto error;
3416 }
3417
3418 DPRINTF("Device %s flags: %u, regions: %u, irgs: %u\n", name,
3419 dev_info.flags, dev_info.num_regions, dev_info.num_irqs);
3420
3421 if (!(dev_info.flags & VFIO_DEVICE_FLAGS_PCI)) {
3422 error_report("vfio: Um, this isn't a PCI device");
3423 goto error;
3424 }
3425
3426 vdev->reset_works = !!(dev_info.flags & VFIO_DEVICE_FLAGS_RESET);
3427
3428 if (dev_info.num_regions < VFIO_PCI_CONFIG_REGION_INDEX + 1) {
3429 error_report("vfio: unexpected number of io regions %u",
3430 dev_info.num_regions);
3431 goto error;
3432 }
3433
3434 if (dev_info.num_irqs < VFIO_PCI_MSIX_IRQ_INDEX + 1) {
3435 error_report("vfio: unexpected number of irqs %u", dev_info.num_irqs);
3436 goto error;
3437 }
3438
3439 for (i = VFIO_PCI_BAR0_REGION_INDEX; i < VFIO_PCI_ROM_REGION_INDEX; i++) {
3440 reg_info.index = i;
3441
3442 ret = ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info);
3443 if (ret) {
3444 error_report("vfio: Error getting region %d info: %m", i);
3445 goto error;
3446 }
3447
3448 DPRINTF("Device %s region %d:\n", name, i);
3449 DPRINTF(" size: 0x%lx, offset: 0x%lx, flags: 0x%lx\n",
3450 (unsigned long)reg_info.size, (unsigned long)reg_info.offset,
3451 (unsigned long)reg_info.flags);
3452
3453 vdev->bars[i].flags = reg_info.flags;
3454 vdev->bars[i].size = reg_info.size;
3455 vdev->bars[i].fd_offset = reg_info.offset;
3456 vdev->bars[i].fd = vdev->fd;
3457 vdev->bars[i].nr = i;
3458 QLIST_INIT(&vdev->bars[i].quirks);
3459 }
3460
3461 reg_info.index = VFIO_PCI_CONFIG_REGION_INDEX;
3462
3463 ret = ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info);
3464 if (ret) {
3465 error_report("vfio: Error getting config info: %m");
3466 goto error;
3467 }
3468
3469 DPRINTF("Device %s config:\n", name);
3470 DPRINTF(" size: 0x%lx, offset: 0x%lx, flags: 0x%lx\n",
3471 (unsigned long)reg_info.size, (unsigned long)reg_info.offset,
3472 (unsigned long)reg_info.flags);
3473
3474 vdev->config_size = reg_info.size;
3475 if (vdev->config_size == PCI_CONFIG_SPACE_SIZE) {
3476 vdev->pdev.cap_present &= ~QEMU_PCI_CAP_EXPRESS;
3477 }
3478 vdev->config_offset = reg_info.offset;
3479
3480 if ((vdev->features & VFIO_FEATURE_ENABLE_VGA) &&
3481 dev_info.num_regions > VFIO_PCI_VGA_REGION_INDEX) {
3482 struct vfio_region_info vga_info = {
3483 .argsz = sizeof(vga_info),
3484 .index = VFIO_PCI_VGA_REGION_INDEX,
3485 };
3486
3487 ret = ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, &vga_info);
3488 if (ret) {
3489 error_report(
3490 "vfio: Device does not support requested feature x-vga");
3491 goto error;
3492 }
3493
3494 if (!(vga_info.flags & VFIO_REGION_INFO_FLAG_READ) ||
3495 !(vga_info.flags & VFIO_REGION_INFO_FLAG_WRITE) ||
3496 vga_info.size < 0xbffff + 1) {
3497 error_report("vfio: Unexpected VGA info, flags 0x%lx, size 0x%lx",
3498 (unsigned long)vga_info.flags,
3499 (unsigned long)vga_info.size);
3500 goto error;
3501 }
3502
3503 vdev->vga.fd_offset = vga_info.offset;
3504 vdev->vga.fd = vdev->fd;
3505
3506 vdev->vga.region[QEMU_PCI_VGA_MEM].offset = QEMU_PCI_VGA_MEM_BASE;
3507 vdev->vga.region[QEMU_PCI_VGA_MEM].nr = QEMU_PCI_VGA_MEM;
3508 QLIST_INIT(&vdev->vga.region[QEMU_PCI_VGA_MEM].quirks);
3509
3510 vdev->vga.region[QEMU_PCI_VGA_IO_LO].offset = QEMU_PCI_VGA_IO_LO_BASE;
3511 vdev->vga.region[QEMU_PCI_VGA_IO_LO].nr = QEMU_PCI_VGA_IO_LO;
3512 QLIST_INIT(&vdev->vga.region[QEMU_PCI_VGA_IO_LO].quirks);
3513
3514 vdev->vga.region[QEMU_PCI_VGA_IO_HI].offset = QEMU_PCI_VGA_IO_HI_BASE;
3515 vdev->vga.region[QEMU_PCI_VGA_IO_HI].nr = QEMU_PCI_VGA_IO_HI;
3516 QLIST_INIT(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].quirks);
3517
3518 vdev->has_vga = true;
3519 }
3520 irq_info.index = VFIO_PCI_ERR_IRQ_INDEX;
3521
3522 ret = ioctl(vdev->fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info);
3523 if (ret) {
3524 /* This can fail for an old kernel or legacy PCI dev */
3525 DPRINTF("VFIO_DEVICE_GET_IRQ_INFO failure: %m\n");
3526 ret = 0;
3527 } else if (irq_info.count == 1) {
3528 vdev->pci_aer = true;
3529 } else {
3530 error_report("vfio: %04x:%02x:%02x.%x "
3531 "Could not enable error recovery for the device",
3532 vdev->host.domain, vdev->host.bus, vdev->host.slot,
3533 vdev->host.function);
3534 }
3535
3536 error:
3537 if (ret) {
3538 QLIST_REMOVE(vdev, next);
3539 vdev->group = NULL;
3540 close(vdev->fd);
3541 }
3542 return ret;
3543 }
3544
3545 static void vfio_put_device(VFIODevice *vdev)
3546 {
3547 QLIST_REMOVE(vdev, next);
3548 vdev->group = NULL;
3549 DPRINTF("vfio_put_device: close vdev->fd\n");
3550 close(vdev->fd);
3551 if (vdev->msix) {
3552 g_free(vdev->msix);
3553 vdev->msix = NULL;
3554 }
3555 }
3556
3557 static void vfio_err_notifier_handler(void *opaque)
3558 {
3559 VFIODevice *vdev = opaque;
3560
3561 if (!event_notifier_test_and_clear(&vdev->err_notifier)) {
3562 return;
3563 }
3564
3565 /*
3566 * TBD. Retrieve the error details and decide what action
3567 * needs to be taken. One of the actions could be to pass
3568 * the error to the guest and have the guest driver recover
3569 * from the error. This requires that PCIe capabilities be
3570 * exposed to the guest. For now, we just terminate the
3571 * guest to contain the error.
3572 */
3573
3574 error_report("%s(%04x:%02x:%02x.%x) Unrecoverable error detected. "
3575 "Please collect any data possible and then kill the guest",
3576 __func__, vdev->host.domain, vdev->host.bus,
3577 vdev->host.slot, vdev->host.function);
3578
3579 vm_stop(RUN_STATE_IO_ERROR);
3580 }
3581
3582 /*
3583 * Registers error notifier for devices supporting error recovery.
3584 * If we encounter a failure in this function, we report an error
3585 * and continue after disabling error recovery support for the
3586 * device.
3587 */
3588 static void vfio_register_err_notifier(VFIODevice *vdev)
3589 {
3590 int ret;
3591 int argsz;
3592 struct vfio_irq_set *irq_set;
3593 int32_t *pfd;
3594
3595 if (!vdev->pci_aer) {
3596 return;
3597 }
3598
3599 if (event_notifier_init(&vdev->err_notifier, 0)) {
3600 error_report("vfio: Unable to init event notifier for error detection");
3601 vdev->pci_aer = false;
3602 return;
3603 }
3604
3605 argsz = sizeof(*irq_set) + sizeof(*pfd);
3606
3607 irq_set = g_malloc0(argsz);
3608 irq_set->argsz = argsz;
3609 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
3610 VFIO_IRQ_SET_ACTION_TRIGGER;
3611 irq_set->index = VFIO_PCI_ERR_IRQ_INDEX;
3612 irq_set->start = 0;
3613 irq_set->count = 1;
3614 pfd = (int32_t *)&irq_set->data;
3615
3616 *pfd = event_notifier_get_fd(&vdev->err_notifier);
3617 qemu_set_fd_handler(*pfd, vfio_err_notifier_handler, NULL, vdev);
3618
3619 ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
3620 if (ret) {
3621 error_report("vfio: Failed to set up error notification");
3622 qemu_set_fd_handler(*pfd, NULL, NULL, vdev);
3623 event_notifier_cleanup(&vdev->err_notifier);
3624 vdev->pci_aer = false;
3625 }
3626 g_free(irq_set);
3627 }
3628
3629 static void vfio_unregister_err_notifier(VFIODevice *vdev)
3630 {
3631 int argsz;
3632 struct vfio_irq_set *irq_set;
3633 int32_t *pfd;
3634 int ret;
3635
3636 if (!vdev->pci_aer) {
3637 return;
3638 }
3639
3640 argsz = sizeof(*irq_set) + sizeof(*pfd);
3641
3642 irq_set = g_malloc0(argsz);
3643 irq_set->argsz = argsz;
3644 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
3645 VFIO_IRQ_SET_ACTION_TRIGGER;
3646 irq_set->index = VFIO_PCI_ERR_IRQ_INDEX;
3647 irq_set->start = 0;
3648 irq_set->count = 1;
3649 pfd = (int32_t *)&irq_set->data;
3650 *pfd = -1;
3651
3652 ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
3653 if (ret) {
3654 error_report("vfio: Failed to de-assign error fd: %m");
3655 }
3656 g_free(irq_set);
3657 qemu_set_fd_handler(event_notifier_get_fd(&vdev->err_notifier),
3658 NULL, NULL, vdev);
3659 event_notifier_cleanup(&vdev->err_notifier);
3660 }
3661
3662 static int vfio_initfn(PCIDevice *pdev)
3663 {
3664 VFIODevice *pvdev, *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
3665 VFIOGroup *group;
3666 char path[PATH_MAX], iommu_group_path[PATH_MAX], *group_name;
3667 ssize_t len;
3668 struct stat st;
3669 int groupid;
3670 int ret;
3671
3672 /* Check that the host device exists */
3673 snprintf(path, sizeof(path),
3674 "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/",
3675 vdev->host.domain, vdev->host.bus, vdev->host.slot,
3676 vdev->host.function);
3677 if (stat(path, &st) < 0) {
3678 error_report("vfio: error: no such host device: %s", path);
3679 return -errno;
3680 }
3681
3682 strncat(path, "iommu_group", sizeof(path) - strlen(path) - 1);
3683
3684 len = readlink(path, iommu_group_path, PATH_MAX);
3685 if (len <= 0) {
3686 error_report("vfio: error no iommu_group for device");
3687 return -errno;
3688 }
3689
3690 iommu_group_path[len] = 0;
3691 group_name = basename(iommu_group_path);
3692
3693 if (sscanf(group_name, "%d", &groupid) != 1) {
3694 error_report("vfio: error reading %s: %m", path);
3695 return -errno;
3696 }
3697
3698 DPRINTF("%s(%04x:%02x:%02x.%x) group %d\n", __func__, vdev->host.domain,
3699 vdev->host.bus, vdev->host.slot, vdev->host.function, groupid);
3700
3701 group = vfio_get_group(groupid);
3702 if (!group) {
3703 error_report("vfio: failed to get group %d", groupid);
3704 return -ENOENT;
3705 }
3706
3707 snprintf(path, sizeof(path), "%04x:%02x:%02x.%01x",
3708 vdev->host.domain, vdev->host.bus, vdev->host.slot,
3709 vdev->host.function);
3710
3711 QLIST_FOREACH(pvdev, &group->device_list, next) {
3712 if (pvdev->host.domain == vdev->host.domain &&
3713 pvdev->host.bus == vdev->host.bus &&
3714 pvdev->host.slot == vdev->host.slot &&
3715 pvdev->host.function == vdev->host.function) {
3716
3717 error_report("vfio: error: device %s is already attached", path);
3718 vfio_put_group(group);
3719 return -EBUSY;
3720 }
3721 }
3722
3723 ret = vfio_get_device(group, path, vdev);
3724 if (ret) {
3725 error_report("vfio: failed to get device %s", path);
3726 vfio_put_group(group);
3727 return ret;
3728 }
3729
3730 /* Get a copy of config space */
3731 ret = pread(vdev->fd, vdev->pdev.config,
3732 MIN(pci_config_size(&vdev->pdev), vdev->config_size),
3733 vdev->config_offset);
3734 if (ret < (int)MIN(pci_config_size(&vdev->pdev), vdev->config_size)) {
3735 ret = ret < 0 ? -errno : -EFAULT;
3736 error_report("vfio: Failed to read device config space");
3737 goto out_put;
3738 }
3739
3740 /* vfio emulates a lot for us, but some bits need extra love */
3741 vdev->emulated_config_bits = g_malloc0(vdev->config_size);
3742
3743 /* QEMU can choose to expose the ROM or not */
3744 memset(vdev->emulated_config_bits + PCI_ROM_ADDRESS, 0xff, 4);
3745
3746 /* QEMU can change multi-function devices to single function, or reverse */
3747 vdev->emulated_config_bits[PCI_HEADER_TYPE] =
3748 PCI_HEADER_TYPE_MULTI_FUNCTION;
3749
3750 /* Restore or clear multifunction, this is always controlled by QEMU */
3751 if (vdev->pdev.cap_present & QEMU_PCI_CAP_MULTIFUNCTION) {
3752 vdev->pdev.config[PCI_HEADER_TYPE] |= PCI_HEADER_TYPE_MULTI_FUNCTION;
3753 } else {
3754 vdev->pdev.config[PCI_HEADER_TYPE] &= ~PCI_HEADER_TYPE_MULTI_FUNCTION;
3755 }
3756
3757 /*
3758 * Clear host resource mapping info. If we choose not to register a
3759 * BAR, such as might be the case with the option ROM, we can get
3760 * confusing, unwritable, residual addresses from the host here.
3761 */
3762 memset(&vdev->pdev.config[PCI_BASE_ADDRESS_0], 0, 24);
3763 memset(&vdev->pdev.config[PCI_ROM_ADDRESS], 0, 4);
3764
3765 vfio_pci_size_rom(vdev);
3766
3767 ret = vfio_early_setup_msix(vdev);
3768 if (ret) {
3769 goto out_put;
3770 }
3771
3772 vfio_map_bars(vdev);
3773
3774 ret = vfio_add_capabilities(vdev);
3775 if (ret) {
3776 goto out_teardown;
3777 }
3778
3779 /* QEMU emulates all of MSI & MSIX */
3780 if (pdev->cap_present & QEMU_PCI_CAP_MSIX) {
3781 memset(vdev->emulated_config_bits + pdev->msix_cap, 0xff,
3782 MSIX_CAP_LENGTH);
3783 }
3784
3785 if (pdev->cap_present & QEMU_PCI_CAP_MSI) {
3786 memset(vdev->emulated_config_bits + pdev->msi_cap, 0xff,
3787 vdev->msi_cap_size);
3788 }
3789
3790 if (vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1)) {
3791 vdev->intx.mmap_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL,
3792 vfio_intx_mmap_enable, vdev);
3793 pci_device_set_intx_routing_notifier(&vdev->pdev, vfio_update_irq);
3794 ret = vfio_enable_intx(vdev);
3795 if (ret) {
3796 goto out_teardown;
3797 }
3798 }
3799
3800 add_boot_device_path(vdev->bootindex, &pdev->qdev, NULL);
3801 vfio_register_err_notifier(vdev);
3802
3803 return 0;
3804
3805 out_teardown:
3806 pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
3807 vfio_teardown_msi(vdev);
3808 vfio_unmap_bars(vdev);
3809 out_put:
3810 g_free(vdev->emulated_config_bits);
3811 vfio_put_device(vdev);
3812 vfio_put_group(group);
3813 return ret;
3814 }
3815
3816 static void vfio_exitfn(PCIDevice *pdev)
3817 {
3818 VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
3819 VFIOGroup *group = vdev->group;
3820
3821 vfio_unregister_err_notifier(vdev);
3822 pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
3823 vfio_disable_interrupts(vdev);
3824 if (vdev->intx.mmap_timer) {
3825 timer_free(vdev->intx.mmap_timer);
3826 }
3827 vfio_teardown_msi(vdev);
3828 vfio_unmap_bars(vdev);
3829 g_free(vdev->emulated_config_bits);
3830 g_free(vdev->rom);
3831 vfio_put_device(vdev);
3832 vfio_put_group(group);
3833 }
3834
3835 static void vfio_pci_reset(DeviceState *dev)
3836 {
3837 PCIDevice *pdev = DO_UPCAST(PCIDevice, qdev, dev);
3838 VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
3839
3840 DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
3841 vdev->host.bus, vdev->host.slot, vdev->host.function);
3842
3843 vfio_pci_pre_reset(vdev);
3844
3845 if (vdev->reset_works && (vdev->has_flr || !vdev->has_pm_reset) &&
3846 !ioctl(vdev->fd, VFIO_DEVICE_RESET)) {
3847 DPRINTF("%04x:%02x:%02x.%x FLR/VFIO_DEVICE_RESET\n", vdev->host.domain,
3848 vdev->host.bus, vdev->host.slot, vdev->host.function);
3849 goto post_reset;
3850 }
3851
3852 /* See if we can do our own bus reset */
3853 if (!vfio_pci_hot_reset_one(vdev)) {
3854 goto post_reset;
3855 }
3856
3857 /* If nothing else works and the device supports PM reset, use it */
3858 if (vdev->reset_works && vdev->has_pm_reset &&
3859 !ioctl(vdev->fd, VFIO_DEVICE_RESET)) {
3860 DPRINTF("%04x:%02x:%02x.%x PCI PM Reset\n", vdev->host.domain,
3861 vdev->host.bus, vdev->host.slot, vdev->host.function);
3862 goto post_reset;
3863 }
3864
3865 post_reset:
3866 vfio_pci_post_reset(vdev);
3867 }
3868
3869 static Property vfio_pci_dev_properties[] = {
3870 DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIODevice, host),
3871 DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIODevice,
3872 intx.mmap_timeout, 1100),
3873 DEFINE_PROP_BIT("x-vga", VFIODevice, features,
3874 VFIO_FEATURE_ENABLE_VGA_BIT, false),
3875 DEFINE_PROP_INT32("bootindex", VFIODevice, bootindex, -1),
3876 /*
3877 * TODO - support passed fds... is this necessary?
3878 * DEFINE_PROP_STRING("vfiofd", VFIODevice, vfiofd_name),
3879 * DEFINE_PROP_STRING("vfiogroupfd, VFIODevice, vfiogroupfd_name),
3880 */
3881 DEFINE_PROP_END_OF_LIST(),
3882 };
3883
3884 static const VMStateDescription vfio_pci_vmstate = {
3885 .name = "vfio-pci",
3886 .unmigratable = 1,
3887 };
3888
3889 static void vfio_pci_dev_class_init(ObjectClass *klass, void *data)
3890 {
3891 DeviceClass *dc = DEVICE_CLASS(klass);
3892 PCIDeviceClass *pdc = PCI_DEVICE_CLASS(klass);
3893
3894 dc->reset = vfio_pci_reset;
3895 dc->props = vfio_pci_dev_properties;
3896 dc->vmsd = &vfio_pci_vmstate;
3897 dc->desc = "VFIO-based PCI device assignment";
3898 set_bit(DEVICE_CATEGORY_MISC, dc->categories);
3899 pdc->init = vfio_initfn;
3900 pdc->exit = vfio_exitfn;
3901 pdc->config_read = vfio_pci_read_config;
3902 pdc->config_write = vfio_pci_write_config;
3903 pdc->is_express = 1; /* We might be */
3904 }
3905
3906 static const TypeInfo vfio_pci_dev_info = {
3907 .name = "vfio-pci",
3908 .parent = TYPE_PCI_DEVICE,
3909 .instance_size = sizeof(VFIODevice),
3910 .class_init = vfio_pci_dev_class_init,
3911 };
3912
3913 static void register_vfio_pci_dev_type(void)
3914 {
3915 type_register_static(&vfio_pci_dev_info);
3916 }
3917
3918 type_init(register_vfio_pci_dev_type)
QEmu