4 * This module includes support for MSI-X in pci devices.
6 * Author: Michael S. Tsirkin <mst@redhat.com>
8 * Copyright (c) 2009, Red Hat Inc, Michael S. Tsirkin (mst@redhat.com)
10 * This work is licensed under the terms of the GNU GPL, version 2. See
11 * the COPYING file in the top-level directory.
19 /* MSI-X capability structure */
20 #define MSIX_TABLE_OFFSET 4
21 #define MSIX_PBA_OFFSET 8
22 #define MSIX_CAP_LENGTH 12
24 /* MSI enable bit and maskall bit are in byte 1 in FLAGS register */
25 #define MSIX_CONTROL_OFFSET (PCI_MSIX_FLAGS + 1)
26 #define MSIX_ENABLE_MASK (PCI_MSIX_FLAGS_ENABLE >> 8)
27 #define MSIX_MASKALL_MASK (PCI_MSIX_FLAGS_MASKALL >> 8)
29 /* MSI-X table format */
30 #define MSIX_MSG_ADDR 0
31 #define MSIX_MSG_UPPER_ADDR 4
32 #define MSIX_MSG_DATA 8
33 #define MSIX_VECTOR_CTRL 12
34 #define MSIX_ENTRY_SIZE 16
35 #define MSIX_VECTOR_MASK 0x1
37 /* How much space does an MSIX table need. */
38 /* The spec requires giving the table structure
39 * a 4K aligned region all by itself. */
40 #define MSIX_PAGE_SIZE 0x1000
41 /* Reserve second half of the page for pending bits */
42 #define MSIX_PAGE_PENDING (MSIX_PAGE_SIZE / 2)
43 #define MSIX_MAX_ENTRIES 32
46 /* Flag for interrupt controller to declare MSI-X support */
50 /* KVM specific MSIX helpers */
51 static void kvm_msix_free(PCIDevice
*dev
)
53 int vector
, changed
= 0;
54 for (vector
= 0; vector
< dev
->msix_entries_nr
; ++vector
) {
55 if (dev
->msix_entry_used
[vector
]) {
56 kvm_del_routing_entry(kvm_context
, &dev
->msix_irq_entries
[vector
]);
61 kvm_commit_irq_routes(kvm_context
);
65 static void kvm_msix_routing_entry(PCIDevice
*dev
, unsigned vector
,
66 struct kvm_irq_routing_entry
*entry
)
68 uint8_t *table_entry
= dev
->msix_table_page
+ vector
* MSIX_ENTRY_SIZE
;
69 entry
->type
= KVM_IRQ_ROUTING_MSI
;
71 entry
->u
.msi
.address_lo
= pci_get_long(table_entry
+ MSIX_MSG_ADDR
);
72 entry
->u
.msi
.address_hi
= pci_get_long(table_entry
+ MSIX_MSG_UPPER_ADDR
);
73 entry
->u
.msi
.data
= pci_get_long(table_entry
+ MSIX_MSG_DATA
);
76 static void kvm_msix_update(PCIDevice
*dev
, int vector
,
77 int was_masked
, int is_masked
)
79 struct kvm_irq_routing_entry e
= {}, *entry
;
80 int mask_cleared
= was_masked
&& !is_masked
;
81 /* It is only legal to change an entry when it is masked. Therefore, it is
82 * enough to update the routing in kernel when mask is being cleared. */
86 if (!dev
->msix_entry_used
[vector
]) {
89 entry
= dev
->msix_irq_entries
+ vector
;
91 kvm_msix_routing_entry(dev
, vector
, &e
);
92 if (memcmp(&entry
->u
.msi
, &e
.u
.msi
, sizeof entry
->u
.msi
)) {
94 r
= kvm_update_routing_entry(kvm_context
, entry
, &e
);
96 fprintf(stderr
, "%s: kvm_update_routing_entry failed: %s\n", __func__
,
100 memcpy(&entry
->u
.msi
, &e
.u
.msi
, sizeof entry
->u
.msi
);
101 r
= kvm_commit_irq_routes(kvm_context
);
103 fprintf(stderr
, "%s: kvm_commit_irq_routes failed: %s\n", __func__
,
110 static int kvm_msix_add(PCIDevice
*dev
, unsigned vector
)
112 struct kvm_irq_routing_entry
*entry
= dev
->msix_irq_entries
+ vector
;
115 if (!kvm_has_gsi_routing(kvm_context
)) {
116 fprintf(stderr
, "Warning: no MSI-X support found. "
117 "At least kernel 2.6.30 is required for MSI-X support.\n"
122 r
= kvm_get_irq_route_gsi(kvm_context
);
124 fprintf(stderr
, "%s: kvm_get_irq_route_gsi failed: %s\n", __func__
, strerror(-r
));
128 kvm_msix_routing_entry(dev
, vector
, entry
);
129 r
= kvm_add_routing_entry(kvm_context
, entry
);
131 fprintf(stderr
, "%s: kvm_add_routing_entry failed: %s\n", __func__
, strerror(-r
));
135 r
= kvm_commit_irq_routes(kvm_context
);
137 fprintf(stderr
, "%s: kvm_commit_irq_routes failed: %s\n", __func__
, strerror(-r
));
143 static void kvm_msix_del(PCIDevice
*dev
, unsigned vector
)
145 if (dev
->msix_entry_used
[vector
]) {
148 kvm_del_routing_entry(kvm_context
, &dev
->msix_irq_entries
[vector
]);
149 kvm_commit_irq_routes(kvm_context
);
153 static void kvm_msix_free(PCIDevice
*dev
) {}
154 static void kvm_msix_update(PCIDevice
*dev
, int vector
,
155 int was_masked
, int is_masked
) {}
156 static int kvm_msix_add(PCIDevice
*dev
, unsigned vector
) { return -1; }
157 static void kvm_msix_del(PCIDevice
*dev
, unsigned vector
) {}
160 /* Add MSI-X capability to the config space for the device. */
161 /* Given a bar and its size, add MSI-X table on top of it
162 * and fill MSI-X capability in the config space.
163 * Original bar size must be a power of 2 or 0.
164 * New bar size is returned. */
165 static int msix_add_config(struct PCIDevice
*pdev
, unsigned short nentries
,
166 unsigned bar_nr
, unsigned bar_size
)
172 if (nentries
< 1 || nentries
> PCI_MSIX_FLAGS_QSIZE
+ 1)
174 if (bar_size
> 0x80000000)
177 /* Add space for MSI-X structures */
179 new_size
= MSIX_PAGE_SIZE
;
180 } else if (bar_size
< MSIX_PAGE_SIZE
) {
181 bar_size
= MSIX_PAGE_SIZE
;
182 new_size
= MSIX_PAGE_SIZE
* 2;
184 new_size
= bar_size
* 2;
187 pdev
->msix_bar_size
= new_size
;
188 config_offset
= pci_add_capability(pdev
, PCI_CAP_ID_MSIX
, MSIX_CAP_LENGTH
);
189 if (config_offset
< 0)
190 return config_offset
;
191 config
= pdev
->config
+ config_offset
;
193 pci_set_word(config
+ PCI_MSIX_FLAGS
, nentries
- 1);
194 /* Table on top of BAR */
195 pci_set_long(config
+ MSIX_TABLE_OFFSET
, bar_size
| bar_nr
);
196 /* Pending bits on top of that */
197 pci_set_long(config
+ MSIX_PBA_OFFSET
, (bar_size
+ MSIX_PAGE_PENDING
) |
199 pdev
->msix_cap
= config_offset
;
200 /* Make flags bit writeable. */
201 pdev
->wmask
[config_offset
+ MSIX_CONTROL_OFFSET
] |= MSIX_ENABLE_MASK
|
206 static uint32_t msix_mmio_readl(void *opaque
, target_phys_addr_t addr
)
208 PCIDevice
*dev
= opaque
;
209 unsigned int offset
= addr
& (MSIX_PAGE_SIZE
- 1) & ~0x3;
210 void *page
= dev
->msix_table_page
;
212 return pci_get_long(page
+ offset
);
215 static uint32_t msix_mmio_read_unallowed(void *opaque
, target_phys_addr_t addr
)
217 fprintf(stderr
, "MSI-X: only dword read is allowed!\n");
221 static uint8_t msix_pending_mask(int vector
)
223 return 1 << (vector
% 8);
226 static uint8_t *msix_pending_byte(PCIDevice
*dev
, int vector
)
228 return dev
->msix_table_page
+ MSIX_PAGE_PENDING
+ vector
/ 8;
231 static int msix_is_pending(PCIDevice
*dev
, int vector
)
233 return *msix_pending_byte(dev
, vector
) & msix_pending_mask(vector
);
236 static void msix_set_pending(PCIDevice
*dev
, int vector
)
238 *msix_pending_byte(dev
, vector
) |= msix_pending_mask(vector
);
241 static void msix_clr_pending(PCIDevice
*dev
, int vector
)
243 *msix_pending_byte(dev
, vector
) &= ~msix_pending_mask(vector
);
246 static int msix_function_masked(PCIDevice
*dev
)
248 return dev
->config
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
] & MSIX_MASKALL_MASK
;
251 static int msix_is_masked(PCIDevice
*dev
, int vector
)
253 unsigned offset
= vector
* MSIX_ENTRY_SIZE
+ MSIX_VECTOR_CTRL
;
254 return msix_function_masked(dev
) ||
255 dev
->msix_table_page
[offset
] & MSIX_VECTOR_MASK
;
258 static void msix_handle_mask_update(PCIDevice
*dev
, int vector
)
260 if (!msix_is_masked(dev
, vector
) && msix_is_pending(dev
, vector
)) {
261 msix_clr_pending(dev
, vector
);
262 msix_notify(dev
, vector
);
266 /* Handle MSI-X capability config write. */
267 void msix_write_config(PCIDevice
*dev
, uint32_t addr
,
268 uint32_t val
, int len
)
270 unsigned enable_pos
= dev
->msix_cap
+ MSIX_CONTROL_OFFSET
;
273 if (!range_covers_byte(addr
, len
, enable_pos
)) {
277 if (!msix_enabled(dev
)) {
281 qemu_set_irq(dev
->irq
[0], 0);
283 if (msix_function_masked(dev
)) {
287 for (vector
= 0; vector
< dev
->msix_entries_nr
; ++vector
) {
288 msix_handle_mask_update(dev
, vector
);
292 static void msix_mmio_writel(void *opaque
, target_phys_addr_t addr
,
295 PCIDevice
*dev
= opaque
;
296 unsigned int offset
= addr
& (MSIX_PAGE_SIZE
- 1) & ~0x3;
297 int vector
= offset
/ MSIX_ENTRY_SIZE
;
298 int was_masked
= msix_is_masked(dev
, vector
);
299 pci_set_long(dev
->msix_table_page
+ offset
, val
);
300 if (kvm_enabled() && kvm_irqchip_in_kernel()) {
301 kvm_msix_update(dev
, vector
, was_masked
, msix_is_masked(dev
, vector
));
303 if (was_masked
!= msix_is_masked(dev
, vector
) &&
304 dev
->msix_mask_notifier
&& dev
->msix_mask_notifier_opaque
[vector
]) {
305 int r
= dev
->msix_mask_notifier(dev
, vector
,
306 dev
->msix_mask_notifier_opaque
[vector
],
307 msix_is_masked(dev
, vector
));
310 msix_handle_mask_update(dev
, vector
);
313 static void msix_mmio_write_unallowed(void *opaque
, target_phys_addr_t addr
,
316 fprintf(stderr
, "MSI-X: only dword write is allowed!\n");
319 static CPUWriteMemoryFunc
* const msix_mmio_write
[] = {
320 msix_mmio_write_unallowed
, msix_mmio_write_unallowed
, msix_mmio_writel
323 static CPUReadMemoryFunc
* const msix_mmio_read
[] = {
324 msix_mmio_read_unallowed
, msix_mmio_read_unallowed
, msix_mmio_readl
327 /* Should be called from device's map method. */
328 void msix_mmio_map(PCIDevice
*d
, int region_num
,
329 pcibus_t addr
, pcibus_t size
, int type
)
331 uint8_t *config
= d
->config
+ d
->msix_cap
;
332 uint32_t table
= pci_get_long(config
+ MSIX_TABLE_OFFSET
);
333 uint32_t offset
= table
& ~(MSIX_PAGE_SIZE
- 1);
334 /* TODO: for assigned devices, we'll want to make it possible to map
335 * pending bits separately in case they are in a separate bar. */
336 int table_bir
= table
& PCI_MSIX_FLAGS_BIRMASK
;
338 if (table_bir
!= region_num
)
342 cpu_register_physical_memory(addr
+ offset
, size
- offset
,
346 static void msix_mask_all(struct PCIDevice
*dev
, unsigned nentries
)
349 for (vector
= 0; vector
< nentries
; ++vector
) {
350 unsigned offset
= vector
* MSIX_ENTRY_SIZE
+ MSIX_VECTOR_CTRL
;
351 int was_masked
= msix_is_masked(dev
, vector
);
352 dev
->msix_table_page
[offset
] |= MSIX_VECTOR_MASK
;
353 if (was_masked
!= msix_is_masked(dev
, vector
) &&
354 dev
->msix_mask_notifier
&& dev
->msix_mask_notifier_opaque
[vector
]) {
355 r
= dev
->msix_mask_notifier(dev
, vector
,
356 dev
->msix_mask_notifier_opaque
[vector
],
357 msix_is_masked(dev
, vector
));
363 /* Initialize the MSI-X structures. Note: if MSI-X is supported, BAR size is
364 * modified, it should be retrieved with msix_bar_size. */
365 int msix_init(struct PCIDevice
*dev
, unsigned short nentries
,
366 unsigned bar_nr
, unsigned bar_size
)
369 /* Nothing to do if MSI is not supported by interrupt controller */
373 if (nentries
> MSIX_MAX_ENTRIES
)
376 #ifdef KVM_CAP_IRQCHIP
377 if (kvm_enabled() && kvm_irqchip_in_kernel()) {
378 dev
->msix_irq_entries
= qemu_malloc(nentries
*
379 sizeof *dev
->msix_irq_entries
);
382 dev
->msix_mask_notifier_opaque
=
383 qemu_mallocz(nentries
* sizeof *dev
->msix_mask_notifier_opaque
);
384 dev
->msix_mask_notifier
= NULL
;
385 dev
->msix_entry_used
= qemu_mallocz(MSIX_MAX_ENTRIES
*
386 sizeof *dev
->msix_entry_used
);
388 dev
->msix_table_page
= qemu_mallocz(MSIX_PAGE_SIZE
);
389 msix_mask_all(dev
, nentries
);
391 dev
->msix_mmio_index
= cpu_register_io_memory(msix_mmio_read
,
392 msix_mmio_write
, dev
);
393 if (dev
->msix_mmio_index
== -1) {
398 dev
->msix_entries_nr
= nentries
;
399 ret
= msix_add_config(dev
, nentries
, bar_nr
, bar_size
);
403 dev
->cap_present
|= QEMU_PCI_CAP_MSIX
;
407 dev
->msix_entries_nr
= 0;
408 cpu_unregister_io_memory(dev
->msix_mmio_index
);
410 qemu_free(dev
->msix_table_page
);
411 dev
->msix_table_page
= NULL
;
412 qemu_free(dev
->msix_entry_used
);
413 dev
->msix_entry_used
= NULL
;
417 static void msix_free_irq_entries(PCIDevice
*dev
)
421 if (kvm_enabled() && kvm_irqchip_in_kernel()) {
425 for (vector
= 0; vector
< dev
->msix_entries_nr
; ++vector
) {
426 dev
->msix_entry_used
[vector
] = 0;
427 msix_clr_pending(dev
, vector
);
431 /* Clean up resources for the device. */
432 int msix_uninit(PCIDevice
*dev
)
434 if (!(dev
->cap_present
& QEMU_PCI_CAP_MSIX
))
436 pci_del_capability(dev
, PCI_CAP_ID_MSIX
, MSIX_CAP_LENGTH
);
438 msix_free_irq_entries(dev
);
439 dev
->msix_entries_nr
= 0;
440 cpu_unregister_io_memory(dev
->msix_mmio_index
);
441 qemu_free(dev
->msix_table_page
);
442 dev
->msix_table_page
= NULL
;
443 qemu_free(dev
->msix_entry_used
);
444 dev
->msix_entry_used
= NULL
;
445 qemu_free(dev
->msix_irq_entries
);
446 dev
->msix_irq_entries
= NULL
;
447 qemu_free(dev
->msix_mask_notifier_opaque
);
448 dev
->msix_mask_notifier_opaque
= NULL
;
449 dev
->cap_present
&= ~QEMU_PCI_CAP_MSIX
;
453 void msix_save(PCIDevice
*dev
, QEMUFile
*f
)
455 unsigned n
= dev
->msix_entries_nr
;
457 if (!msix_supported
) {
461 if (!(dev
->cap_present
& QEMU_PCI_CAP_MSIX
)) {
464 qemu_put_buffer(f
, dev
->msix_table_page
, n
* MSIX_ENTRY_SIZE
);
465 qemu_put_buffer(f
, dev
->msix_table_page
+ MSIX_PAGE_PENDING
, (n
+ 7) / 8);
468 /* Should be called after restoring the config space. */
469 void msix_load(PCIDevice
*dev
, QEMUFile
*f
)
471 unsigned n
= dev
->msix_entries_nr
;
476 if (!(dev
->cap_present
& QEMU_PCI_CAP_MSIX
)) {
480 msix_free_irq_entries(dev
);
481 qemu_get_buffer(f
, dev
->msix_table_page
, n
* MSIX_ENTRY_SIZE
);
482 qemu_get_buffer(f
, dev
->msix_table_page
+ MSIX_PAGE_PENDING
, (n
+ 7) / 8);
485 /* Does device support MSI-X? */
486 int msix_present(PCIDevice
*dev
)
488 return dev
->cap_present
& QEMU_PCI_CAP_MSIX
;
491 /* Is MSI-X enabled? */
492 int msix_enabled(PCIDevice
*dev
)
494 return (dev
->cap_present
& QEMU_PCI_CAP_MSIX
) &&
495 (dev
->config
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
] &
499 /* Size of bar where MSI-X table resides, or 0 if MSI-X not supported. */
500 uint32_t msix_bar_size(PCIDevice
*dev
)
502 return (dev
->cap_present
& QEMU_PCI_CAP_MSIX
) ?
503 dev
->msix_bar_size
: 0;
506 /* Send an MSI-X message */
507 void msix_notify(PCIDevice
*dev
, unsigned vector
)
509 uint8_t *table_entry
= dev
->msix_table_page
+ vector
* MSIX_ENTRY_SIZE
;
513 if (vector
>= dev
->msix_entries_nr
|| !dev
->msix_entry_used
[vector
])
515 if (msix_is_masked(dev
, vector
)) {
516 msix_set_pending(dev
, vector
);
520 #ifdef KVM_CAP_IRQCHIP
521 if (kvm_enabled() && kvm_irqchip_in_kernel()) {
522 kvm_set_irq(dev
->msix_irq_entries
[vector
].gsi
, 1, NULL
);
527 address
= pci_get_long(table_entry
+ MSIX_MSG_UPPER_ADDR
);
528 address
= (address
<< 32) | pci_get_long(table_entry
+ MSIX_MSG_ADDR
);
529 data
= pci_get_long(table_entry
+ MSIX_MSG_DATA
);
530 stl_phys(address
, data
);
533 void msix_reset(PCIDevice
*dev
)
535 if (!(dev
->cap_present
& QEMU_PCI_CAP_MSIX
))
537 msix_free_irq_entries(dev
);
538 dev
->config
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
] &=
539 ~dev
->wmask
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
];
540 memset(dev
->msix_table_page
, 0, MSIX_PAGE_SIZE
);
541 msix_mask_all(dev
, dev
->msix_entries_nr
);
544 /* PCI spec suggests that devices make it possible for software to configure
545 * less vectors than supported by the device, but does not specify a standard
546 * mechanism for devices to do so.
548 * We support this by asking devices to declare vectors software is going to
549 * actually use, and checking this on the notification path. Devices that
550 * don't want to follow the spec suggestion can declare all vectors as used. */
552 /* Mark vector as used. */
553 int msix_vector_use(PCIDevice
*dev
, unsigned vector
)
556 if (vector
>= dev
->msix_entries_nr
)
558 if (dev
->msix_entry_used
[vector
]) {
561 if (kvm_enabled() && kvm_irqchip_in_kernel()) {
562 ret
= kvm_msix_add(dev
, vector
);
567 ++dev
->msix_entry_used
[vector
];
571 /* Mark vector as unused. */
572 void msix_vector_unuse(PCIDevice
*dev
, unsigned vector
)
574 if (vector
>= dev
->msix_entries_nr
|| !dev
->msix_entry_used
[vector
]) {
577 if (--dev
->msix_entry_used
[vector
]) {
580 if (kvm_enabled() && kvm_irqchip_in_kernel()) {
581 kvm_msix_del(dev
, vector
);
583 msix_clr_pending(dev
, vector
);
586 void msix_unuse_all_vectors(PCIDevice
*dev
)
588 if (!(dev
->cap_present
& QEMU_PCI_CAP_MSIX
))
590 msix_free_irq_entries(dev
);
593 int msix_set_mask_notifier(PCIDevice
*dev
, unsigned vector
, void *opaque
)
596 if (vector
>= dev
->msix_entries_nr
|| !dev
->msix_entry_used
[vector
])
599 assert(dev
->msix_mask_notifier
);
601 assert(!dev
->msix_mask_notifier_opaque
[vector
]);
603 /* Unmask the new notifier unless vector is masked. */
604 if (!msix_is_masked(dev
, vector
)) {
605 r
= dev
->msix_mask_notifier(dev
, vector
, opaque
, false);
610 dev
->msix_mask_notifier_opaque
[vector
] = opaque
;
614 int msix_unset_mask_notifier(PCIDevice
*dev
, unsigned vector
)
618 if (vector
>= dev
->msix_entries_nr
|| !dev
->msix_entry_used
[vector
])
621 opaque
= dev
->msix_mask_notifier_opaque
[vector
];
623 assert(dev
->msix_mask_notifier
);
626 /* Mask the old notifier unless it is already masked. */
627 if (!msix_is_masked(dev
, vector
)) {
628 r
= dev
->msix_mask_notifier(dev
, vector
, opaque
, true);
633 dev
->msix_mask_notifier_opaque
[vector
] = NULL
;