4 * This module includes support for MSI-X in pci devices.
6 * Author: Michael S. Tsirkin <mst@redhat.com>
8 * Copyright (c) 2009, Red Hat Inc, Michael S. Tsirkin (mst@redhat.com)
10 * This work is licensed under the terms of the GNU GPL, version 2. See
11 * the COPYING file in the top-level directory.
13 * Contributions after 2012-01-13 are licensed under the terms of the
14 * GNU GPL, version 2 or (at your option) any later version.
24 #define MSIX_CAP_LENGTH 12
26 /* MSI enable bit and maskall bit are in byte 1 in FLAGS register */
27 #define MSIX_CONTROL_OFFSET (PCI_MSIX_FLAGS + 1)
28 #define MSIX_ENABLE_MASK (PCI_MSIX_FLAGS_ENABLE >> 8)
29 #define MSIX_MASKALL_MASK (PCI_MSIX_FLAGS_MASKALL >> 8)
31 /* How much space does an MSIX table need. */
32 /* The spec requires giving the table structure
33 * a 4K aligned region all by itself. */
34 #define MSIX_PAGE_SIZE 0x1000
35 /* Reserve second half of the page for pending bits */
36 #define MSIX_PAGE_PENDING (MSIX_PAGE_SIZE / 2)
37 #define MSIX_MAX_ENTRIES 32
40 /* KVM specific MSIX helpers */
41 static void kvm_msix_free(PCIDevice
*dev
)
43 int vector
, changed
= 0;
45 for (vector
= 0; vector
< dev
->msix_entries_nr
; ++vector
) {
46 if (dev
->msix_entry_used
[vector
]) {
47 kvm_msi_message_del(&dev
->msix_irq_entries
[vector
]);
52 kvm_commit_irq_routes();
56 static void kvm_msix_message_from_vector(PCIDevice
*dev
, unsigned vector
,
59 uint8_t *table_entry
= dev
->msix_table_page
+ vector
* PCI_MSIX_ENTRY_SIZE
;
61 kmm
->addr_lo
= pci_get_long(table_entry
+ PCI_MSIX_ENTRY_LOWER_ADDR
);
62 kmm
->addr_hi
= pci_get_long(table_entry
+ PCI_MSIX_ENTRY_UPPER_ADDR
);
63 kmm
->data
= pci_get_long(table_entry
+ PCI_MSIX_ENTRY_DATA
);
66 static void kvm_msix_update(PCIDevice
*dev
, int vector
,
67 int was_masked
, int is_masked
)
69 KVMMsiMessage new_entry
, *entry
;
70 int mask_cleared
= was_masked
&& !is_masked
;
73 /* It is only legal to change an entry when it is masked. Therefore, it is
74 * enough to update the routing in kernel when mask is being cleared. */
78 if (!dev
->msix_entry_used
[vector
]) {
82 entry
= dev
->msix_irq_entries
+ vector
;
83 kvm_msix_message_from_vector(dev
, vector
, &new_entry
);
84 r
= kvm_msi_message_update(entry
, &new_entry
);
86 fprintf(stderr
, "%s: kvm_update_msix failed: %s\n", __func__
,
92 r
= kvm_commit_irq_routes();
94 fprintf(stderr
, "%s: kvm_commit_irq_routes failed: %s\n", __func__
,
101 static int kvm_msix_vector_add(PCIDevice
*dev
, unsigned vector
)
103 KVMMsiMessage
*kmm
= dev
->msix_irq_entries
+ vector
;
106 kvm_msix_message_from_vector(dev
, vector
, kmm
);
107 r
= kvm_msi_message_add(kmm
);
109 fprintf(stderr
, "%s: kvm_add_msix failed: %s\n", __func__
, strerror(-r
));
113 r
= kvm_commit_irq_routes();
115 fprintf(stderr
, "%s: kvm_commit_irq_routes failed: %s\n", __func__
, strerror(-r
));
121 static void kvm_msix_vector_del(PCIDevice
*dev
, unsigned vector
)
123 kvm_msi_message_del(&dev
->msix_irq_entries
[vector
]);
124 kvm_commit_irq_routes();
127 /* Add MSI-X capability to the config space for the device. */
128 /* Given a bar and its size, add MSI-X table on top of it
129 * and fill MSI-X capability in the config space.
130 * Original bar size must be a power of 2 or 0.
131 * New bar size is returned. */
132 static int msix_add_config(struct PCIDevice
*pdev
, unsigned short nentries
,
133 unsigned bar_nr
, unsigned bar_size
)
138 pdev
->msix_bar_size
= bar_size
;
140 config_offset
= pci_find_capability(pdev
, PCI_CAP_ID_MSIX
);
142 if (!config_offset
) {
145 if (nentries
< 1 || nentries
> PCI_MSIX_FLAGS_QSIZE
+ 1)
147 if (bar_size
> 0x80000000)
150 /* Add space for MSI-X structures */
152 new_size
= MSIX_PAGE_SIZE
;
153 } else if (bar_size
< MSIX_PAGE_SIZE
) {
154 bar_size
= MSIX_PAGE_SIZE
;
155 new_size
= MSIX_PAGE_SIZE
* 2;
157 new_size
= bar_size
* 2;
160 pdev
->msix_bar_size
= new_size
;
161 config_offset
= pci_add_capability(pdev
, PCI_CAP_ID_MSIX
,
163 if (config_offset
< 0)
164 return config_offset
;
165 config
= pdev
->config
+ config_offset
;
167 pci_set_word(config
+ PCI_MSIX_FLAGS
, nentries
- 1);
168 /* Table on top of BAR */
169 pci_set_long(config
+ PCI_MSIX_TABLE
, bar_size
| bar_nr
);
170 /* Pending bits on top of that */
171 pci_set_long(config
+ PCI_MSIX_PBA
, (bar_size
+ MSIX_PAGE_PENDING
) |
174 pdev
->msix_cap
= config_offset
;
175 /* Make flags bit writable. */
176 pdev
->wmask
[config_offset
+ MSIX_CONTROL_OFFSET
] |= MSIX_ENABLE_MASK
|
178 pdev
->msix_function_masked
= true;
182 static uint64_t msix_mmio_read(void *opaque
, target_phys_addr_t addr
,
185 PCIDevice
*dev
= opaque
;
186 unsigned int offset
= addr
& (MSIX_PAGE_SIZE
- 1) & ~0x3;
187 void *page
= dev
->msix_table_page
;
189 return pci_get_long(page
+ offset
);
192 static uint8_t msix_pending_mask(int vector
)
194 return 1 << (vector
% 8);
197 static uint8_t *msix_pending_byte(PCIDevice
*dev
, int vector
)
199 return dev
->msix_table_page
+ MSIX_PAGE_PENDING
+ vector
/ 8;
202 static int msix_is_pending(PCIDevice
*dev
, int vector
)
204 return *msix_pending_byte(dev
, vector
) & msix_pending_mask(vector
);
207 static void msix_set_pending(PCIDevice
*dev
, int vector
)
209 *msix_pending_byte(dev
, vector
) |= msix_pending_mask(vector
);
212 static void msix_clr_pending(PCIDevice
*dev
, int vector
)
214 *msix_pending_byte(dev
, vector
) &= ~msix_pending_mask(vector
);
217 static bool msix_vector_masked(PCIDevice
*dev
, int vector
, bool fmask
)
219 unsigned offset
= vector
* PCI_MSIX_ENTRY_SIZE
+ PCI_MSIX_ENTRY_VECTOR_CTRL
;
220 return fmask
|| dev
->msix_table_page
[offset
] & PCI_MSIX_ENTRY_CTRL_MASKBIT
;
223 static bool msix_is_masked(PCIDevice
*dev
, int vector
)
225 return msix_vector_masked(dev
, vector
, dev
->msix_function_masked
);
228 static void msix_handle_mask_update(PCIDevice
*dev
, int vector
, bool was_masked
)
230 bool is_masked
= msix_is_masked(dev
, vector
);
231 if (is_masked
== was_masked
) {
235 if (dev
->msix_mask_notifier
) {
237 ret
= dev
->msix_mask_notifier(dev
, vector
, is_masked
);
241 if (!is_masked
&& msix_is_pending(dev
, vector
)) {
242 msix_clr_pending(dev
, vector
);
243 msix_notify(dev
, vector
);
247 static void msix_update_function_masked(PCIDevice
*dev
)
249 dev
->msix_function_masked
= !msix_enabled(dev
) ||
250 (dev
->config
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
] & MSIX_MASKALL_MASK
);
253 /* Handle MSI-X capability config write. */
254 void msix_write_config(PCIDevice
*dev
, uint32_t addr
,
255 uint32_t val
, int len
)
257 unsigned enable_pos
= dev
->msix_cap
+ MSIX_CONTROL_OFFSET
;
261 if (!range_covers_byte(addr
, len
, enable_pos
)) {
265 was_masked
= dev
->msix_function_masked
;
266 msix_update_function_masked(dev
);
268 if (!msix_enabled(dev
)) {
272 pci_device_deassert_intx(dev
);
274 if (dev
->msix_function_masked
== was_masked
) {
278 for (vector
= 0; vector
< dev
->msix_entries_nr
; ++vector
) {
279 msix_handle_mask_update(dev
, vector
,
280 msix_vector_masked(dev
, vector
, was_masked
));
284 static void msix_mmio_write(void *opaque
, target_phys_addr_t addr
,
285 uint64_t val
, unsigned size
)
287 PCIDevice
*dev
= opaque
;
288 unsigned int offset
= addr
& (MSIX_PAGE_SIZE
- 1) & ~0x3;
289 int vector
= offset
/ PCI_MSIX_ENTRY_SIZE
;
292 /* MSI-X page includes a read-only PBA and a writeable Vector Control. */
293 if (vector
>= dev
->msix_entries_nr
) {
297 was_masked
= msix_is_masked(dev
, vector
);
298 pci_set_long(dev
->msix_table_page
+ offset
, val
);
299 if (kvm_enabled() && kvm_irqchip_in_kernel()) {
300 kvm_msix_update(dev
, vector
, was_masked
, msix_is_masked(dev
, vector
));
302 msix_handle_mask_update(dev
, vector
, was_masked
);
305 static const MemoryRegionOps msix_mmio_ops
= {
306 .read
= msix_mmio_read
,
307 .write
= msix_mmio_write
,
308 .endianness
= DEVICE_NATIVE_ENDIAN
,
310 .min_access_size
= 4,
311 .max_access_size
= 4,
315 static void msix_mmio_setup(PCIDevice
*d
, MemoryRegion
*bar
)
317 uint8_t *config
= d
->config
+ d
->msix_cap
;
318 uint32_t table
= pci_get_long(config
+ PCI_MSIX_TABLE
);
319 uint32_t offset
= table
& ~(MSIX_PAGE_SIZE
- 1);
320 /* TODO: for assigned devices, we'll want to make it possible to map
321 * pending bits separately in case they are in a separate bar. */
323 memory_region_add_subregion(bar
, offset
, &d
->msix_mmio
);
326 static void msix_mask_all(struct PCIDevice
*dev
, unsigned nentries
)
329 for (vector
= 0; vector
< nentries
; ++vector
) {
331 vector
* PCI_MSIX_ENTRY_SIZE
+ PCI_MSIX_ENTRY_VECTOR_CTRL
;
332 int was_masked
= msix_is_masked(dev
, vector
);
333 dev
->msix_table_page
[offset
] |= PCI_MSIX_ENTRY_CTRL_MASKBIT
;
334 if (was_masked
!= msix_is_masked(dev
, vector
) &&
335 dev
->msix_mask_notifier
) {
336 r
= dev
->msix_mask_notifier(dev
, vector
,
337 msix_is_masked(dev
, vector
));
343 /* Initialize the MSI-X structures. Note: if MSI-X is supported, BAR size is
344 * modified, it should be retrieved with msix_bar_size. */
345 int msix_init(struct PCIDevice
*dev
, unsigned short nentries
,
347 unsigned bar_nr
, unsigned bar_size
)
351 /* Nothing to do if MSI is not supported by interrupt controller */
352 if (!msi_supported
||
353 (kvm_enabled() && kvm_irqchip_in_kernel() && !kvm_has_gsi_routing())) {
356 if (nentries
> MSIX_MAX_ENTRIES
)
359 dev
->msix_mask_notifier
= NULL
;
360 dev
->msix_entry_used
= g_malloc0(MSIX_MAX_ENTRIES
*
361 sizeof *dev
->msix_entry_used
);
363 dev
->msix_table_page
= g_malloc0(MSIX_PAGE_SIZE
);
364 msix_mask_all(dev
, nentries
);
366 memory_region_init_io(&dev
->msix_mmio
, &msix_mmio_ops
, dev
,
367 "msix", MSIX_PAGE_SIZE
);
369 dev
->msix_entries_nr
= nentries
;
370 ret
= msix_add_config(dev
, nentries
, bar_nr
, bar_size
);
374 if (kvm_enabled() && kvm_irqchip_in_kernel()) {
375 dev
->msix_irq_entries
= g_malloc(nentries
*
376 sizeof *dev
->msix_irq_entries
);
379 dev
->cap_present
|= QEMU_PCI_CAP_MSIX
;
380 msix_mmio_setup(dev
, bar
);
384 dev
->msix_entries_nr
= 0;
385 memory_region_destroy(&dev
->msix_mmio
);
386 g_free(dev
->msix_table_page
);
387 dev
->msix_table_page
= NULL
;
388 g_free(dev
->msix_entry_used
);
389 dev
->msix_entry_used
= NULL
;
393 static void msix_free_irq_entries(PCIDevice
*dev
)
397 if (kvm_enabled() && kvm_irqchip_in_kernel()) {
401 for (vector
= 0; vector
< dev
->msix_entries_nr
; ++vector
) {
402 dev
->msix_entry_used
[vector
] = 0;
403 msix_clr_pending(dev
, vector
);
407 /* Clean up resources for the device. */
408 int msix_uninit(PCIDevice
*dev
, MemoryRegion
*bar
)
410 if (!(dev
->cap_present
& QEMU_PCI_CAP_MSIX
))
412 pci_del_capability(dev
, PCI_CAP_ID_MSIX
, MSIX_CAP_LENGTH
);
414 msix_free_irq_entries(dev
);
415 dev
->msix_entries_nr
= 0;
416 memory_region_del_subregion(bar
, &dev
->msix_mmio
);
417 memory_region_destroy(&dev
->msix_mmio
);
418 g_free(dev
->msix_table_page
);
419 dev
->msix_table_page
= NULL
;
420 g_free(dev
->msix_entry_used
);
421 dev
->msix_entry_used
= NULL
;
422 g_free(dev
->msix_irq_entries
);
423 dev
->msix_irq_entries
= NULL
;
424 dev
->cap_present
&= ~QEMU_PCI_CAP_MSIX
;
428 void msix_save(PCIDevice
*dev
, QEMUFile
*f
)
430 unsigned n
= dev
->msix_entries_nr
;
432 if (!msi_supported
) {
436 if (!(dev
->cap_present
& QEMU_PCI_CAP_MSIX
)) {
439 qemu_put_buffer(f
, dev
->msix_table_page
, n
* PCI_MSIX_ENTRY_SIZE
);
440 qemu_put_buffer(f
, dev
->msix_table_page
+ MSIX_PAGE_PENDING
, (n
+ 7) / 8);
443 /* Should be called after restoring the config space. */
444 void msix_load(PCIDevice
*dev
, QEMUFile
*f
)
446 unsigned n
= dev
->msix_entries_nr
;
451 if (!(dev
->cap_present
& QEMU_PCI_CAP_MSIX
)) {
455 msix_free_irq_entries(dev
);
456 qemu_get_buffer(f
, dev
->msix_table_page
, n
* PCI_MSIX_ENTRY_SIZE
);
457 qemu_get_buffer(f
, dev
->msix_table_page
+ MSIX_PAGE_PENDING
, (n
+ 7) / 8);
458 msix_update_function_masked(dev
);
461 /* Does device support MSI-X? */
462 int msix_present(PCIDevice
*dev
)
464 return dev
->cap_present
& QEMU_PCI_CAP_MSIX
;
467 /* Is MSI-X enabled? */
468 int msix_enabled(PCIDevice
*dev
)
470 return (dev
->cap_present
& QEMU_PCI_CAP_MSIX
) &&
471 (dev
->config
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
] &
475 /* Size of bar where MSI-X table resides, or 0 if MSI-X not supported. */
476 uint32_t msix_bar_size(PCIDevice
*dev
)
478 return (dev
->cap_present
& QEMU_PCI_CAP_MSIX
) ?
479 dev
->msix_bar_size
: 0;
482 /* Send an MSI-X message */
483 void msix_notify(PCIDevice
*dev
, unsigned vector
)
485 uint8_t *table_entry
= dev
->msix_table_page
+ vector
* PCI_MSIX_ENTRY_SIZE
;
489 if (vector
>= dev
->msix_entries_nr
|| !dev
->msix_entry_used
[vector
])
491 if (msix_is_masked(dev
, vector
)) {
492 msix_set_pending(dev
, vector
);
496 if (kvm_enabled() && kvm_irqchip_in_kernel()) {
497 kvm_set_irq(dev
->msix_irq_entries
[vector
].gsi
, 1, NULL
);
501 address
= pci_get_quad(table_entry
+ PCI_MSIX_ENTRY_LOWER_ADDR
);
502 data
= pci_get_long(table_entry
+ PCI_MSIX_ENTRY_DATA
);
503 stl_le_phys(address
, data
);
506 void msix_reset(PCIDevice
*dev
)
508 if (!(dev
->cap_present
& QEMU_PCI_CAP_MSIX
))
510 msix_free_irq_entries(dev
);
511 dev
->config
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
] &=
512 ~dev
->wmask
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
];
513 memset(dev
->msix_table_page
, 0, MSIX_PAGE_SIZE
);
514 msix_mask_all(dev
, dev
->msix_entries_nr
);
517 /* PCI spec suggests that devices make it possible for software to configure
518 * less vectors than supported by the device, but does not specify a standard
519 * mechanism for devices to do so.
521 * We support this by asking devices to declare vectors software is going to
522 * actually use, and checking this on the notification path. Devices that
523 * don't want to follow the spec suggestion can declare all vectors as used. */
525 /* Mark vector as used. */
526 int msix_vector_use(PCIDevice
*dev
, unsigned vector
)
529 if (vector
>= dev
->msix_entries_nr
)
531 if (kvm_enabled() && kvm_irqchip_in_kernel() &&
532 !dev
->msix_entry_used
[vector
]) {
533 ret
= kvm_msix_vector_add(dev
, vector
);
538 ++dev
->msix_entry_used
[vector
];
542 /* Mark vector as unused. */
543 void msix_vector_unuse(PCIDevice
*dev
, unsigned vector
)
545 if (vector
>= dev
->msix_entries_nr
|| !dev
->msix_entry_used
[vector
]) {
548 if (--dev
->msix_entry_used
[vector
]) {
551 if (kvm_enabled() && kvm_irqchip_in_kernel()) {
552 kvm_msix_vector_del(dev
, vector
);
554 msix_clr_pending(dev
, vector
);
557 void msix_unuse_all_vectors(PCIDevice
*dev
)
559 if (!(dev
->cap_present
& QEMU_PCI_CAP_MSIX
))
561 msix_free_irq_entries(dev
);
564 /* Invoke the notifier if vector entry is used and unmasked. */
565 static int msix_notify_if_unmasked(PCIDevice
*dev
, unsigned vector
, int masked
)
567 assert(dev
->msix_mask_notifier
);
568 if (!dev
->msix_entry_used
[vector
] || msix_is_masked(dev
, vector
)) {
571 return dev
->msix_mask_notifier(dev
, vector
, masked
);
574 static int msix_set_mask_notifier_for_vector(PCIDevice
*dev
, unsigned vector
)
576 /* Notifier has been set. Invoke it on unmasked vectors. */
577 return msix_notify_if_unmasked(dev
, vector
, 0);
580 static int msix_unset_mask_notifier_for_vector(PCIDevice
*dev
, unsigned vector
)
582 /* Notifier will be unset. Invoke it to mask unmasked entries. */
583 return msix_notify_if_unmasked(dev
, vector
, 1);
586 int msix_set_mask_notifier(PCIDevice
*dev
, msix_mask_notifier_func f
)
589 assert(!dev
->msix_mask_notifier
);
590 dev
->msix_mask_notifier
= f
;
591 for (n
= 0; n
< dev
->msix_entries_nr
; ++n
) {
592 r
= msix_set_mask_notifier_for_vector(dev
, n
);
601 msix_unset_mask_notifier_for_vector(dev
, n
);
603 dev
->msix_mask_notifier
= NULL
;
607 int msix_unset_mask_notifier(PCIDevice
*dev
)
610 assert(dev
->msix_mask_notifier
);
611 for (n
= 0; n
< dev
->msix_entries_nr
; ++n
) {
612 r
= msix_unset_mask_notifier_for_vector(dev
, n
);
617 dev
->msix_mask_notifier
= NULL
;
622 msix_set_mask_notifier_for_vector(dev
, n
);