2 * Copyright (c) 2007, Intel Corporation.
4 * This work is licensed under the terms of the GNU GPL, version 2. See
5 * the COPYING file in the top-level directory.
7 * Jiang Yunhong <yunhong.jiang@intel.com>
9 * This file implements direct PCI assignment to a HVM guest
14 #include "hw/xen/xen_backend.h"
16 #include "hw/i386/apic-msidef.h"
19 #define XEN_PT_AUTO_ASSIGN -1
21 /* shift count for gflags */
22 #define XEN_PT_GFLAGS_SHIFT_DEST_ID 0
23 #define XEN_PT_GFLAGS_SHIFT_RH 8
24 #define XEN_PT_GFLAGS_SHIFT_DM 9
25 #define XEN_PT_GFLAGSSHIFT_DELIV_MODE 12
26 #define XEN_PT_GFLAGSSHIFT_TRG_MODE 15
28 #define latch(fld) latch[PCI_MSIX_ENTRY_##fld / sizeof(uint32_t)]
34 static inline uint8_t msi_vector(uint32_t data
)
36 return (data
& MSI_DATA_VECTOR_MASK
) >> MSI_DATA_VECTOR_SHIFT
;
39 static inline uint8_t msi_dest_id(uint32_t addr
)
41 return (addr
& MSI_ADDR_DEST_ID_MASK
) >> MSI_ADDR_DEST_ID_SHIFT
;
44 static inline uint32_t msi_ext_dest_id(uint32_t addr_hi
)
46 return addr_hi
& 0xffffff00;
49 static uint32_t msi_gflags(uint32_t data
, uint64_t addr
)
52 int rh
, dm
, dest_id
, deliv_mode
, trig_mode
;
54 rh
= (addr
>> MSI_ADDR_REDIRECTION_SHIFT
) & 0x1;
55 dm
= (addr
>> MSI_ADDR_DEST_MODE_SHIFT
) & 0x1;
56 dest_id
= msi_dest_id(addr
);
57 deliv_mode
= (data
>> MSI_DATA_DELIVERY_MODE_SHIFT
) & 0x7;
58 trig_mode
= (data
>> MSI_DATA_TRIGGER_SHIFT
) & 0x1;
60 result
= dest_id
| (rh
<< XEN_PT_GFLAGS_SHIFT_RH
)
61 | (dm
<< XEN_PT_GFLAGS_SHIFT_DM
)
62 | (deliv_mode
<< XEN_PT_GFLAGSSHIFT_DELIV_MODE
)
63 | (trig_mode
<< XEN_PT_GFLAGSSHIFT_TRG_MODE
);
68 static inline uint64_t msi_addr64(XenPTMSI
*msi
)
70 return (uint64_t)msi
->addr_hi
<< 32 | msi
->addr_lo
;
73 static int msi_msix_enable(XenPCIPassthroughState
*s
,
85 rc
= xen_host_pci_get_word(&s
->real_device
, address
, &val
);
87 XEN_PT_ERR(&s
->dev
, "Failed to read MSI/MSI-X register (0x%x), rc:%d\n",
96 rc
= xen_host_pci_set_word(&s
->real_device
, address
, val
);
98 XEN_PT_ERR(&s
->dev
, "Failed to write MSI/MSI-X register (0x%x), rc:%d\n",
104 static int msi_msix_setup(XenPCIPassthroughState
*s
,
112 uint8_t gvec
= msi_vector(data
);
115 assert((!is_msix
&& msix_entry
== 0) || is_msix
);
118 /* if gvec is 0, the guest is asking for a particular pirq that
119 * is passed as dest_id */
120 *ppirq
= msi_ext_dest_id(addr
>> 32) | msi_dest_id(addr
);
122 /* this probably identifies an misconfiguration of the guest,
123 * try the emulated path */
124 *ppirq
= XEN_PT_UNASSIGNED_PIRQ
;
126 XEN_PT_LOG(&s
->dev
, "requested pirq %d for MSI%s"
127 " (vec: %#x, entry: %#x)\n",
128 *ppirq
, is_msix
? "-X" : "", gvec
, msix_entry
);
133 uint64_t table_base
= 0;
136 table_base
= s
->msix
->table_base
;
139 rc
= xc_physdev_map_pirq_msi(xen_xc
, xen_domid
, XEN_PT_AUTO_ASSIGN
,
140 ppirq
, PCI_DEVFN(s
->real_device
.dev
,
141 s
->real_device
.func
),
143 msix_entry
, table_base
);
146 "Mapping of MSI%s (err: %i, vec: %#x, entry %#x)\n",
147 is_msix
? "-X" : "", errno
, gvec
, msix_entry
);
154 static int msi_msix_update(XenPCIPassthroughState
*s
,
162 PCIDevice
*d
= &s
->dev
;
163 uint8_t gvec
= msi_vector(data
);
164 uint32_t gflags
= msi_gflags(data
, addr
);
166 uint64_t table_addr
= 0;
168 XEN_PT_LOG(d
, "Updating MSI%s with pirq %d gvec %#x gflags %#x"
170 is_msix
? "-X" : "", pirq
, gvec
, gflags
, msix_entry
);
173 table_addr
= s
->msix
->mmio_base_addr
;
176 rc
= xc_domain_update_msi_irq(xen_xc
, xen_domid
, gvec
,
177 pirq
, gflags
, table_addr
);
180 XEN_PT_ERR(d
, "Updating of MSI%s failed. (err: %d)\n",
181 is_msix
? "-X" : "", errno
);
183 if (xc_physdev_unmap_pirq(xen_xc
, xen_domid
, *old_pirq
)) {
184 XEN_PT_ERR(d
, "Unmapping of MSI%s pirq %d failed. (err: %d)\n",
185 is_msix
? "-X" : "", *old_pirq
, errno
);
187 *old_pirq
= XEN_PT_UNASSIGNED_PIRQ
;
192 static int msi_msix_disable(XenPCIPassthroughState
*s
,
199 PCIDevice
*d
= &s
->dev
;
200 uint8_t gvec
= msi_vector(data
);
201 uint32_t gflags
= msi_gflags(data
, addr
);
204 if (pirq
== XEN_PT_UNASSIGNED_PIRQ
) {
209 XEN_PT_LOG(d
, "Unbind MSI%s with pirq %d, gvec %#x\n",
210 is_msix
? "-X" : "", pirq
, gvec
);
211 rc
= xc_domain_unbind_msi_irq(xen_xc
, xen_domid
, gvec
, pirq
, gflags
);
213 XEN_PT_ERR(d
, "Unbinding of MSI%s failed. (err: %d, pirq: %d, gvec: %#x)\n",
214 is_msix
? "-X" : "", errno
, pirq
, gvec
);
219 XEN_PT_LOG(d
, "Unmap MSI%s pirq %d\n", is_msix
? "-X" : "", pirq
);
220 rc
= xc_physdev_unmap_pirq(xen_xc
, xen_domid
, pirq
);
222 XEN_PT_ERR(d
, "Unmapping of MSI%s pirq %d failed. (err: %i)\n",
223 is_msix
? "-X" : "", pirq
, errno
);
231 * MSI virtualization functions
234 static int xen_pt_msi_set_enable(XenPCIPassthroughState
*s
, bool enable
)
236 XEN_PT_LOG(&s
->dev
, "%s MSI.\n", enable
? "enabling" : "disabling");
242 return msi_msix_enable(s
, s
->msi
->ctrl_offset
, PCI_MSI_FLAGS_ENABLE
,
246 /* setup physical msi, but don't enable it */
247 int xen_pt_msi_setup(XenPCIPassthroughState
*s
)
249 int pirq
= XEN_PT_UNASSIGNED_PIRQ
;
251 XenPTMSI
*msi
= s
->msi
;
253 if (msi
->initialized
) {
255 "Setup physical MSI when it has been properly initialized.\n");
259 rc
= msi_msix_setup(s
, msi_addr64(msi
), msi
->data
, &pirq
, false, 0, true);
265 XEN_PT_ERR(&s
->dev
, "Invalid pirq number: %d.\n", pirq
);
270 XEN_PT_LOG(&s
->dev
, "MSI mapped with pirq %d.\n", pirq
);
275 int xen_pt_msi_update(XenPCIPassthroughState
*s
)
277 XenPTMSI
*msi
= s
->msi
;
278 return msi_msix_update(s
, msi_addr64(msi
), msi
->data
, msi
->pirq
,
279 false, 0, &msi
->pirq
);
282 void xen_pt_msi_disable(XenPCIPassthroughState
*s
)
284 XenPTMSI
*msi
= s
->msi
;
290 (void)xen_pt_msi_set_enable(s
, false);
292 msi_msix_disable(s
, msi_addr64(msi
), msi
->data
, msi
->pirq
, false,
296 msi
->flags
&= ~PCI_MSI_FLAGS_ENABLE
;
297 msi
->initialized
= false;
299 msi
->pirq
= XEN_PT_UNASSIGNED_PIRQ
;
303 * MSI-X virtualization functions
306 static int msix_set_enable(XenPCIPassthroughState
*s
, bool enabled
)
308 XEN_PT_LOG(&s
->dev
, "%s MSI-X.\n", enabled
? "enabling" : "disabling");
314 return msi_msix_enable(s
, s
->msix
->ctrl_offset
, PCI_MSIX_FLAGS_ENABLE
,
318 static int xen_pt_msix_update_one(XenPCIPassthroughState
*s
, int entry_nr
,
321 XenPTMSIXEntry
*entry
= NULL
;
325 if (entry_nr
< 0 || entry_nr
>= s
->msix
->total_entries
) {
329 entry
= &s
->msix
->msix_entry
[entry_nr
];
331 if (!entry
->updated
) {
338 * Update the entry addr and data to the latest values only when the
339 * entry is masked or they are all masked, as required by the spec.
340 * Addr and data changes while the MSI-X entry is unmasked get deferred
341 * until the next masked -> unmasked transition.
343 if (pirq
== XEN_PT_UNASSIGNED_PIRQ
|| s
->msix
->maskall
||
344 (vec_ctrl
& PCI_MSIX_ENTRY_CTRL_MASKBIT
)) {
345 entry
->addr
= entry
->latch(LOWER_ADDR
) |
346 ((uint64_t)entry
->latch(UPPER_ADDR
) << 32);
347 entry
->data
= entry
->latch(DATA
);
350 rc
= msi_msix_setup(s
, entry
->addr
, entry
->data
, &pirq
, true, entry_nr
,
351 entry
->pirq
== XEN_PT_UNASSIGNED_PIRQ
);
355 if (entry
->pirq
== XEN_PT_UNASSIGNED_PIRQ
) {
359 rc
= msi_msix_update(s
, entry
->addr
, entry
->data
, pirq
, true,
360 entry_nr
, &entry
->pirq
);
363 entry
->updated
= false;
369 int xen_pt_msix_update(XenPCIPassthroughState
*s
)
371 XenPTMSIX
*msix
= s
->msix
;
374 for (i
= 0; i
< msix
->total_entries
; i
++) {
375 xen_pt_msix_update_one(s
, i
, msix
->msix_entry
[i
].latch(VECTOR_CTRL
));
381 void xen_pt_msix_disable(XenPCIPassthroughState
*s
)
385 msix_set_enable(s
, false);
387 for (i
= 0; i
< s
->msix
->total_entries
; i
++) {
388 XenPTMSIXEntry
*entry
= &s
->msix
->msix_entry
[i
];
390 msi_msix_disable(s
, entry
->addr
, entry
->data
, entry
->pirq
, true, true);
392 /* clear MSI-X info */
393 entry
->pirq
= XEN_PT_UNASSIGNED_PIRQ
;
394 entry
->updated
= false;
398 int xen_pt_msix_update_remap(XenPCIPassthroughState
*s
, int bar_index
)
400 XenPTMSIXEntry
*entry
;
403 if (!(s
->msix
&& s
->msix
->bar_index
== bar_index
)) {
407 for (i
= 0; i
< s
->msix
->total_entries
; i
++) {
408 entry
= &s
->msix
->msix_entry
[i
];
409 if (entry
->pirq
!= XEN_PT_UNASSIGNED_PIRQ
) {
410 ret
= xc_domain_unbind_pt_irq(xen_xc
, xen_domid
, entry
->pirq
,
411 PT_IRQ_TYPE_MSI
, 0, 0, 0, 0);
413 XEN_PT_ERR(&s
->dev
, "unbind MSI-X entry %d failed (err: %d)\n",
416 entry
->updated
= true;
419 return xen_pt_msix_update(s
);
422 static uint32_t get_entry_value(XenPTMSIXEntry
*e
, int offset
)
424 assert(!(offset
% sizeof(*e
->latch
)));
425 return e
->latch
[offset
/ sizeof(*e
->latch
)];
428 static void set_entry_value(XenPTMSIXEntry
*e
, int offset
, uint32_t val
)
430 assert(!(offset
% sizeof(*e
->latch
)));
431 e
->latch
[offset
/ sizeof(*e
->latch
)] = val
;
434 static void pci_msix_write(void *opaque
, hwaddr addr
,
435 uint64_t val
, unsigned size
)
437 XenPCIPassthroughState
*s
= opaque
;
438 XenPTMSIX
*msix
= s
->msix
;
439 XenPTMSIXEntry
*entry
;
440 unsigned int entry_nr
, offset
;
442 entry_nr
= addr
/ PCI_MSIX_ENTRY_SIZE
;
443 if (entry_nr
>= msix
->total_entries
) {
446 entry
= &msix
->msix_entry
[entry_nr
];
447 offset
= addr
% PCI_MSIX_ENTRY_SIZE
;
449 if (offset
!= PCI_MSIX_ENTRY_VECTOR_CTRL
) {
450 if (get_entry_value(entry
, offset
) == val
451 && entry
->pirq
!= XEN_PT_UNASSIGNED_PIRQ
) {
455 entry
->updated
= true;
456 } else if (msix
->enabled
&& entry
->updated
&&
457 !(val
& PCI_MSIX_ENTRY_CTRL_MASKBIT
)) {
458 const volatile uint32_t *vec_ctrl
;
461 * If Xen intercepts the mask bit access, entry->vec_ctrl may not be
462 * up-to-date. Read from hardware directly.
464 vec_ctrl
= s
->msix
->phys_iomem_base
+ entry_nr
* PCI_MSIX_ENTRY_SIZE
465 + PCI_MSIX_ENTRY_VECTOR_CTRL
;
466 xen_pt_msix_update_one(s
, entry_nr
, *vec_ctrl
);
469 set_entry_value(entry
, offset
, val
);
472 static uint64_t pci_msix_read(void *opaque
, hwaddr addr
,
475 XenPCIPassthroughState
*s
= opaque
;
476 XenPTMSIX
*msix
= s
->msix
;
477 int entry_nr
, offset
;
479 entry_nr
= addr
/ PCI_MSIX_ENTRY_SIZE
;
481 XEN_PT_ERR(&s
->dev
, "asked MSI-X entry '%i' invalid!\n", entry_nr
);
485 offset
= addr
% PCI_MSIX_ENTRY_SIZE
;
487 if (addr
< msix
->total_entries
* PCI_MSIX_ENTRY_SIZE
) {
488 return get_entry_value(&msix
->msix_entry
[entry_nr
], offset
);
490 /* Pending Bit Array (PBA) */
491 return *(uint32_t *)(msix
->phys_iomem_base
+ addr
);
495 static bool pci_msix_accepts(void *opaque
, hwaddr addr
,
496 unsigned size
, bool is_write
)
498 return !(addr
& (size
- 1));
501 static const MemoryRegionOps pci_msix_ops
= {
502 .read
= pci_msix_read
,
503 .write
= pci_msix_write
,
504 .endianness
= DEVICE_NATIVE_ENDIAN
,
506 .min_access_size
= 4,
507 .max_access_size
= 4,
509 .accepts
= pci_msix_accepts
512 .min_access_size
= 4,
513 .max_access_size
= 4,
518 int xen_pt_msix_init(XenPCIPassthroughState
*s
, uint32_t base
)
521 uint16_t control
= 0;
522 uint32_t table_off
= 0;
523 int i
, total_entries
, bar_index
;
524 XenHostPCIDevice
*hd
= &s
->real_device
;
525 PCIDevice
*d
= &s
->dev
;
527 XenPTMSIX
*msix
= NULL
;
530 rc
= xen_host_pci_get_byte(hd
, base
+ PCI_CAP_LIST_ID
, &id
);
535 if (id
!= PCI_CAP_ID_MSIX
) {
536 XEN_PT_ERR(d
, "Invalid id %#x base %#x\n", id
, base
);
540 xen_host_pci_get_word(hd
, base
+ PCI_MSIX_FLAGS
, &control
);
541 total_entries
= control
& PCI_MSIX_FLAGS_QSIZE
;
544 s
->msix
= g_malloc0(sizeof (XenPTMSIX
)
545 + total_entries
* sizeof (XenPTMSIXEntry
));
548 msix
->total_entries
= total_entries
;
549 for (i
= 0; i
< total_entries
; i
++) {
550 msix
->msix_entry
[i
].pirq
= XEN_PT_UNASSIGNED_PIRQ
;
553 memory_region_init_io(&msix
->mmio
, OBJECT(s
), &pci_msix_ops
,
554 s
, "xen-pci-pt-msix",
555 (total_entries
* PCI_MSIX_ENTRY_SIZE
559 xen_host_pci_get_long(hd
, base
+ PCI_MSIX_TABLE
, &table_off
);
560 bar_index
= msix
->bar_index
= table_off
& PCI_MSIX_FLAGS_BIRMASK
;
561 table_off
= table_off
& ~PCI_MSIX_FLAGS_BIRMASK
;
562 msix
->table_base
= s
->real_device
.io_regions
[bar_index
].base_addr
;
563 XEN_PT_LOG(d
, "get MSI-X table BAR base 0x%"PRIx64
"\n", msix
->table_base
);
565 fd
= open("/dev/mem", O_RDWR
);
568 XEN_PT_ERR(d
, "Can't open /dev/mem: %s\n", strerror(errno
));
571 XEN_PT_LOG(d
, "table_off = %#x, total_entries = %d\n",
572 table_off
, total_entries
);
573 msix
->table_offset_adjust
= table_off
& 0x0fff;
574 msix
->phys_iomem_base
=
576 total_entries
* PCI_MSIX_ENTRY_SIZE
+ msix
->table_offset_adjust
,
578 MAP_SHARED
| MAP_LOCKED
,
580 msix
->table_base
+ table_off
- msix
->table_offset_adjust
);
582 if (msix
->phys_iomem_base
== MAP_FAILED
) {
584 XEN_PT_ERR(d
, "Can't map physical MSI-X table: %s\n", strerror(errno
));
587 msix
->phys_iomem_base
= (char *)msix
->phys_iomem_base
588 + msix
->table_offset_adjust
;
590 XEN_PT_LOG(d
, "mapping physical MSI-X table to %p\n",
591 msix
->phys_iomem_base
);
593 memory_region_add_subregion_overlap(&s
->bar
[bar_index
], table_off
,
595 2); /* Priority: pci default + 1 */
605 void xen_pt_msix_unmap(XenPCIPassthroughState
*s
)
607 XenPTMSIX
*msix
= s
->msix
;
613 /* unmap the MSI-X memory mapped register area */
614 if (msix
->phys_iomem_base
) {
615 XEN_PT_LOG(&s
->dev
, "unmapping physical MSI-X table from %p\n",
616 msix
->phys_iomem_base
);
617 munmap(msix
->phys_iomem_base
, msix
->total_entries
* PCI_MSIX_ENTRY_SIZE
618 + msix
->table_offset_adjust
);
621 memory_region_del_subregion(&s
->bar
[msix
->bar_index
], &msix
->mmio
);
624 void xen_pt_msix_delete(XenPCIPassthroughState
*s
)
626 XenPTMSIX
*msix
= s
->msix
;
632 object_unparent(OBJECT(&msix
->mmio
));