4 * This module includes support for MSI-X in pci devices.
6 * Author: Michael S. Tsirkin <mst@redhat.com>
8 * Copyright (c) 2009, Red Hat Inc, Michael S. Tsirkin (mst@redhat.com)
10 * This work is licensed under the terms of the GNU GPL, version 2. See
11 * the COPYING file in the top-level directory.
17 #define QEMU_KVM_NO_CPU
20 /* Declaration from linux/pci_regs.h */
21 #define PCI_CAP_ID_MSIX 0x11 /* MSI-X */
22 #define PCI_MSIX_FLAGS 2 /* Table at lower 11 bits */
23 #define PCI_MSIX_FLAGS_QSIZE 0x7FF
24 #define PCI_MSIX_FLAGS_ENABLE (1 << 15)
25 #define PCI_MSIX_FLAGS_BIRMASK (7 << 0)
27 /* MSI-X capability structure */
28 #define MSIX_TABLE_OFFSET 4
29 #define MSIX_PBA_OFFSET 8
30 #define MSIX_CAP_LENGTH 12
32 /* MSI enable bit is in byte 1 in FLAGS register */
33 #define MSIX_ENABLE_OFFSET (PCI_MSIX_FLAGS + 1)
34 #define MSIX_ENABLE_MASK (PCI_MSIX_FLAGS_ENABLE >> 8)
36 /* MSI-X table format */
37 #define MSIX_MSG_ADDR 0
38 #define MSIX_MSG_UPPER_ADDR 4
39 #define MSIX_MSG_DATA 8
40 #define MSIX_VECTOR_CTRL 12
41 #define MSIX_ENTRY_SIZE 16
42 #define MSIX_VECTOR_MASK 0x1
44 /* How much space does an MSIX table need. */
45 /* The spec requires giving the table structure
46 * a 4K aligned region all by itself. */
47 #define MSIX_PAGE_SIZE 0x1000
48 /* Reserve second half of the page for pending bits */
49 #define MSIX_PAGE_PENDING (MSIX_PAGE_SIZE / 2)
50 #define MSIX_MAX_ENTRIES 32
54 #define DEBUG(fmt, ...) \
56 fprintf(stderr, "%s: " fmt, __func__ , __VA_ARGS__); \
59 #define DEBUG(fmt, ...) do { } while(0)
62 /* Flag for interrupt controller to declare MSI-X support */
66 /* KVM specific MSIX helpers */
67 static void kvm_msix_free(PCIDevice
*dev
)
69 int vector
, changed
= 0;
70 for (vector
= 0; vector
< dev
->msix_entries_nr
; ++vector
) {
71 if (dev
->msix_entry_used
[vector
]) {
72 kvm_del_routing_entry(kvm_context
, &dev
->msix_irq_entries
[vector
]);
77 kvm_commit_irq_routes(kvm_context
);
81 static void kvm_msix_routing_entry(PCIDevice
*dev
, unsigned vector
,
82 struct kvm_irq_routing_entry
*entry
)
84 uint8_t *table_entry
= dev
->msix_table_page
+ vector
* MSIX_ENTRY_SIZE
;
85 entry
->type
= KVM_IRQ_ROUTING_MSI
;
87 entry
->u
.msi
.address_lo
= pci_get_long(table_entry
+ MSIX_MSG_ADDR
);
88 entry
->u
.msi
.address_hi
= pci_get_long(table_entry
+ MSIX_MSG_UPPER_ADDR
);
89 entry
->u
.msi
.data
= pci_get_long(table_entry
+ MSIX_MSG_DATA
);
92 static void kvm_msix_update(PCIDevice
*dev
, int vector
,
93 int was_masked
, int is_masked
)
95 struct kvm_irq_routing_entry e
= {}, *entry
;
96 int mask_cleared
= was_masked
&& !is_masked
;
97 /* It is only legal to change an entry when it is masked. Therefore, it is
98 * enough to update the routing in kernel when mask is being cleared. */
102 if (!dev
->msix_entry_used
[vector
]) {
105 entry
= dev
->msix_irq_entries
+ vector
;
107 kvm_msix_routing_entry(dev
, vector
, &e
);
108 if (memcmp(&entry
->u
.msi
, &e
.u
.msi
, sizeof entry
->u
.msi
)) {
110 r
= kvm_update_routing_entry(kvm_context
, entry
, &e
);
112 fprintf(stderr
, "%s: kvm_update_routing_entry failed: %s\n", __func__
,
116 memcpy(&entry
->u
.msi
, &e
.u
.msi
, sizeof entry
->u
.msi
);
117 r
= kvm_commit_irq_routes(kvm_context
);
119 fprintf(stderr
, "%s: kvm_commit_irq_routes failed: %s\n", __func__
,
126 static int kvm_msix_add(PCIDevice
*dev
, unsigned vector
)
128 struct kvm_irq_routing_entry
*entry
= dev
->msix_irq_entries
+ vector
;
131 if (!kvm_has_gsi_routing(kvm_context
)) {
132 fprintf(stderr
, "Warning: no MSI-X support found. "
133 "At least kernel 2.6.30 is required for MSI-X support.\n"
138 r
= kvm_get_irq_route_gsi(kvm_context
);
140 fprintf(stderr
, "%s: kvm_get_irq_route_gsi failed: %s\n", __func__
, strerror(-r
));
144 kvm_msix_routing_entry(dev
, vector
, entry
);
145 r
= kvm_add_routing_entry(kvm_context
, entry
);
147 fprintf(stderr
, "%s: kvm_add_routing_entry failed: %s\n", __func__
, strerror(-r
));
151 r
= kvm_commit_irq_routes(kvm_context
);
153 fprintf(stderr
, "%s: kvm_commit_irq_routes failed: %s\n", __func__
, strerror(-r
));
159 static void kvm_msix_del(PCIDevice
*dev
, unsigned vector
)
161 if (dev
->msix_entry_used
[vector
]) {
164 kvm_del_routing_entry(kvm_context
, &dev
->msix_irq_entries
[vector
]);
165 kvm_commit_irq_routes(kvm_context
);
169 static void kvm_msix_free(PCIDevice
*dev
) {}
170 static void kvm_msix_update(PCIDevice
*dev
, int vector
,
171 int was_masked
, int is_masked
) {}
172 static int kvm_msix_add(PCIDevice
*dev
, unsigned vector
) { return -1; }
173 static void kvm_msix_del(PCIDevice
*dev
, unsigned vector
) {}
176 /* Add MSI-X capability to the config space for the device. */
177 /* Given a bar and its size, add MSI-X table on top of it
178 * and fill MSI-X capability in the config space.
179 * Original bar size must be a power of 2 or 0.
180 * New bar size is returned. */
181 static int msix_add_config(struct PCIDevice
*pdev
, unsigned short nentries
,
182 unsigned bar_nr
, unsigned bar_size
)
188 if (nentries
< 1 || nentries
> PCI_MSIX_FLAGS_QSIZE
+ 1)
190 if (bar_size
> 0x80000000)
193 /* Add space for MSI-X structures */
195 new_size
= MSIX_PAGE_SIZE
;
196 } else if (bar_size
< MSIX_PAGE_SIZE
) {
197 bar_size
= MSIX_PAGE_SIZE
;
198 new_size
= MSIX_PAGE_SIZE
* 2;
200 new_size
= bar_size
* 2;
203 pdev
->msix_bar_size
= new_size
;
204 config_offset
= pci_add_capability(pdev
, PCI_CAP_ID_MSIX
, MSIX_CAP_LENGTH
);
205 if (config_offset
< 0)
206 return config_offset
;
207 config
= pdev
->config
+ config_offset
;
209 pci_set_word(config
+ PCI_MSIX_FLAGS
, nentries
- 1);
210 /* Table on top of BAR */
211 pci_set_long(config
+ MSIX_TABLE_OFFSET
, bar_size
| bar_nr
);
212 /* Pending bits on top of that */
213 pci_set_long(config
+ MSIX_PBA_OFFSET
, (bar_size
+ MSIX_PAGE_PENDING
) |
215 pdev
->msix_cap
= config_offset
;
216 /* Make flags bit writeable. */
217 pdev
->wmask
[config_offset
+ MSIX_ENABLE_OFFSET
] |= MSIX_ENABLE_MASK
;
221 static void msix_free_irq_entries(PCIDevice
*dev
)
224 if (kvm_enabled() && kvm_irqchip_in_kernel()) {
228 for (vector
= 0; vector
< dev
->msix_entries_nr
; ++vector
)
229 dev
->msix_entry_used
[vector
] = 0;
232 /* Handle MSI-X capability config write. */
233 void msix_write_config(PCIDevice
*dev
, uint32_t addr
,
234 uint32_t val
, int len
)
236 unsigned enable_pos
= dev
->msix_cap
+ MSIX_ENABLE_OFFSET
;
237 if (addr
+ len
<= enable_pos
|| addr
> enable_pos
)
240 if (msix_enabled(dev
))
241 qemu_set_irq(dev
->irq
[0], 0);
244 static uint32_t msix_mmio_readl(void *opaque
, target_phys_addr_t addr
)
246 PCIDevice
*dev
= opaque
;
247 unsigned int offset
= addr
& (MSIX_PAGE_SIZE
- 1);
248 void *page
= dev
->msix_table_page
;
251 memcpy(&val
, (void *)((char *)page
+ offset
), 4);
256 static uint32_t msix_mmio_read_unallowed(void *opaque
, target_phys_addr_t addr
)
258 fprintf(stderr
, "MSI-X: only dword read is allowed!\n");
262 static uint8_t msix_pending_mask(int vector
)
264 return 1 << (vector
% 8);
267 static uint8_t *msix_pending_byte(PCIDevice
*dev
, int vector
)
269 return dev
->msix_table_page
+ MSIX_PAGE_PENDING
+ vector
/ 8;
272 static int msix_is_pending(PCIDevice
*dev
, int vector
)
274 return *msix_pending_byte(dev
, vector
) & msix_pending_mask(vector
);
277 static void msix_set_pending(PCIDevice
*dev
, int vector
)
279 *msix_pending_byte(dev
, vector
) |= msix_pending_mask(vector
);
282 static void msix_clr_pending(PCIDevice
*dev
, int vector
)
284 *msix_pending_byte(dev
, vector
) &= ~msix_pending_mask(vector
);
287 static int msix_is_masked(PCIDevice
*dev
, int vector
)
289 unsigned offset
= vector
* MSIX_ENTRY_SIZE
+ MSIX_VECTOR_CTRL
;
290 return dev
->msix_table_page
[offset
] & MSIX_VECTOR_MASK
;
293 static void msix_mmio_writel(void *opaque
, target_phys_addr_t addr
,
296 PCIDevice
*dev
= opaque
;
297 unsigned int offset
= addr
& (MSIX_PAGE_SIZE
- 1);
298 int vector
= offset
/ MSIX_ENTRY_SIZE
;
299 int was_masked
= msix_is_masked(dev
, vector
);
300 memcpy(dev
->msix_table_page
+ offset
, &val
, 4);
301 if (kvm_enabled() && kvm_irqchip_in_kernel()) {
302 kvm_msix_update(dev
, vector
, was_masked
, msix_is_masked(dev
, vector
));
304 if (!msix_is_masked(dev
, vector
) && msix_is_pending(dev
, vector
)) {
305 msix_clr_pending(dev
, vector
);
306 msix_notify(dev
, vector
);
310 static void msix_mmio_write_unallowed(void *opaque
, target_phys_addr_t addr
,
313 fprintf(stderr
, "MSI-X: only dword write is allowed!\n");
316 static CPUWriteMemoryFunc
* const msix_mmio_write
[] = {
317 msix_mmio_write_unallowed
, msix_mmio_write_unallowed
, msix_mmio_writel
320 static CPUReadMemoryFunc
* const msix_mmio_read
[] = {
321 msix_mmio_read_unallowed
, msix_mmio_read_unallowed
, msix_mmio_readl
324 /* Should be called from device's map method. */
325 void msix_mmio_map(PCIDevice
*d
, int region_num
,
326 uint32_t addr
, uint32_t size
, int type
)
328 uint8_t *config
= d
->config
+ d
->msix_cap
;
329 uint32_t table
= pci_get_long(config
+ MSIX_TABLE_OFFSET
);
330 uint32_t offset
= table
& ~(MSIX_PAGE_SIZE
- 1);
331 /* TODO: for assigned devices, we'll want to make it possible to map
332 * pending bits separately in case they are in a separate bar. */
333 int table_bir
= table
& PCI_MSIX_FLAGS_BIRMASK
;
335 if (table_bir
!= region_num
)
339 cpu_register_physical_memory(addr
+ offset
, size
- offset
,
343 /* Initialize the MSI-X structures. Note: if MSI-X is supported, BAR size is
344 * modified, it should be retrieved with msix_bar_size. */
345 int msix_init(struct PCIDevice
*dev
, unsigned short nentries
,
346 unsigned bar_nr
, unsigned bar_size
)
349 /* Nothing to do if MSI is not supported by interrupt controller */
353 if (nentries
> MSIX_MAX_ENTRIES
)
356 #ifdef KVM_CAP_IRQCHIP
357 if (kvm_enabled() && kvm_irqchip_in_kernel()) {
358 dev
->msix_irq_entries
= qemu_malloc(nentries
*
359 sizeof *dev
->msix_irq_entries
);
362 dev
->msix_entry_used
= qemu_mallocz(MSIX_MAX_ENTRIES
*
363 sizeof *dev
->msix_entry_used
);
365 dev
->msix_table_page
= qemu_mallocz(MSIX_PAGE_SIZE
);
367 dev
->msix_mmio_index
= cpu_register_io_memory(msix_mmio_read
,
368 msix_mmio_write
, dev
);
369 if (dev
->msix_mmio_index
== -1) {
374 dev
->msix_entries_nr
= nentries
;
375 ret
= msix_add_config(dev
, nentries
, bar_nr
, bar_size
);
379 dev
->cap_present
|= QEMU_PCI_CAP_MSIX
;
383 dev
->msix_entries_nr
= 0;
384 cpu_unregister_io_memory(dev
->msix_mmio_index
);
386 qemu_free(dev
->msix_table_page
);
387 dev
->msix_table_page
= NULL
;
388 qemu_free(dev
->msix_entry_used
);
389 dev
->msix_entry_used
= NULL
;
393 /* Clean up resources for the device. */
394 int msix_uninit(PCIDevice
*dev
)
396 if (!(dev
->cap_present
& QEMU_PCI_CAP_MSIX
))
398 pci_del_capability(dev
, PCI_CAP_ID_MSIX
, MSIX_CAP_LENGTH
);
400 msix_free_irq_entries(dev
);
401 dev
->msix_entries_nr
= 0;
402 cpu_unregister_io_memory(dev
->msix_mmio_index
);
403 qemu_free(dev
->msix_table_page
);
404 dev
->msix_table_page
= NULL
;
405 qemu_free(dev
->msix_entry_used
);
406 dev
->msix_entry_used
= NULL
;
407 qemu_free(dev
->msix_irq_entries
);
408 dev
->msix_irq_entries
= NULL
;
409 dev
->cap_present
&= ~QEMU_PCI_CAP_MSIX
;
413 void msix_save(PCIDevice
*dev
, QEMUFile
*f
)
415 unsigned n
= dev
->msix_entries_nr
;
417 if (!msix_supported
) {
421 if (!(dev
->cap_present
& QEMU_PCI_CAP_MSIX
)) {
424 qemu_put_buffer(f
, dev
->msix_table_page
, n
* MSIX_ENTRY_SIZE
);
425 qemu_put_buffer(f
, dev
->msix_table_page
+ MSIX_PAGE_PENDING
, (n
+ 7) / 8);
428 /* Should be called after restoring the config space. */
429 void msix_load(PCIDevice
*dev
, QEMUFile
*f
)
431 unsigned n
= dev
->msix_entries_nr
;
436 if (!(dev
->cap_present
& QEMU_PCI_CAP_MSIX
)) {
440 msix_free_irq_entries(dev
);
441 qemu_get_buffer(f
, dev
->msix_table_page
, n
* MSIX_ENTRY_SIZE
);
442 qemu_get_buffer(f
, dev
->msix_table_page
+ MSIX_PAGE_PENDING
, (n
+ 7) / 8);
445 /* Does device support MSI-X? */
446 int msix_present(PCIDevice
*dev
)
448 return dev
->cap_present
& QEMU_PCI_CAP_MSIX
;
451 /* Is MSI-X enabled? */
452 int msix_enabled(PCIDevice
*dev
)
454 return (dev
->cap_present
& QEMU_PCI_CAP_MSIX
) &&
455 (dev
->config
[dev
->msix_cap
+ MSIX_ENABLE_OFFSET
] &
459 /* Size of bar where MSI-X table resides, or 0 if MSI-X not supported. */
460 uint32_t msix_bar_size(PCIDevice
*dev
)
462 return (dev
->cap_present
& QEMU_PCI_CAP_MSIX
) ?
463 dev
->msix_bar_size
: 0;
466 /* Send an MSI-X message */
467 void msix_notify(PCIDevice
*dev
, unsigned vector
)
469 uint8_t *table_entry
= dev
->msix_table_page
+ vector
* MSIX_ENTRY_SIZE
;
473 if (vector
>= dev
->msix_entries_nr
|| !dev
->msix_entry_used
[vector
])
475 if (msix_is_masked(dev
, vector
)) {
476 msix_set_pending(dev
, vector
);
480 #ifdef KVM_CAP_IRQCHIP
481 if (kvm_enabled() && kvm_irqchip_in_kernel()) {
482 kvm_set_irq(dev
->msix_irq_entries
[vector
].gsi
, 1, NULL
);
487 address
= pci_get_long(table_entry
+ MSIX_MSG_UPPER_ADDR
);
488 address
= (address
<< 32) | pci_get_long(table_entry
+ MSIX_MSG_ADDR
);
489 data
= pci_get_long(table_entry
+ MSIX_MSG_DATA
);
490 stl_phys(address
, data
);
493 void msix_reset(PCIDevice
*dev
)
495 if (!(dev
->cap_present
& QEMU_PCI_CAP_MSIX
))
497 msix_free_irq_entries(dev
);
498 dev
->config
[dev
->msix_cap
+ MSIX_ENABLE_OFFSET
] &= MSIX_ENABLE_MASK
;
499 memset(dev
->msix_table_page
, 0, MSIX_PAGE_SIZE
);
502 /* PCI spec suggests that devices make it possible for software to configure
503 * less vectors than supported by the device, but does not specify a standard
504 * mechanism for devices to do so.
506 * We support this by asking devices to declare vectors software is going to
507 * actually use, and checking this on the notification path. Devices that
508 * don't want to follow the spec suggestion can declare all vectors as used. */
510 /* Mark vector as used. */
511 int msix_vector_use(PCIDevice
*dev
, unsigned vector
)
514 if (vector
>= dev
->msix_entries_nr
)
516 if (dev
->msix_entry_used
[vector
]) {
519 if (kvm_enabled() && kvm_irqchip_in_kernel()) {
520 ret
= kvm_msix_add(dev
, vector
);
525 ++dev
->msix_entry_used
[vector
];
529 /* Mark vector as unused. */
530 void msix_vector_unuse(PCIDevice
*dev
, unsigned vector
)
532 if (vector
< dev
->msix_entries_nr
&& dev
->msix_entry_used
[vector
]) {
533 --dev
->msix_entry_used
[vector
];
534 if (kvm_enabled() && kvm_irqchip_in_kernel()) {
535 kvm_msix_del(dev
, vector
);