4 * This module includes support for MSI-X in pci devices.
6 * Author: Michael S. Tsirkin <mst@redhat.com>
8 * Copyright (c) 2009, Red Hat Inc, Michael S. Tsirkin (mst@redhat.com)
10 * This work is licensed under the terms of the GNU GPL, version 2. See
11 * the COPYING file in the top-level directory.
17 #define QEMU_KVM_NO_CPU
20 /* Declaration from linux/pci_regs.h */
21 #define PCI_CAP_ID_MSIX 0x11 /* MSI-X */
22 #define PCI_MSIX_FLAGS 2 /* Table at lower 11 bits */
23 #define PCI_MSIX_FLAGS_QSIZE 0x7FF
24 #define PCI_MSIX_FLAGS_ENABLE (1 << 15)
25 #define PCI_MSIX_FLAGS_BIRMASK (7 << 0)
27 /* MSI-X capability structure */
28 #define MSIX_TABLE_OFFSET 4
29 #define MSIX_PBA_OFFSET 8
30 #define MSIX_CAP_LENGTH 12
32 /* MSI enable bit is in byte 1 in FLAGS register */
33 #define MSIX_ENABLE_OFFSET (PCI_MSIX_FLAGS + 1)
34 #define MSIX_ENABLE_MASK (PCI_MSIX_FLAGS_ENABLE >> 8)
36 /* MSI-X table format */
37 #define MSIX_MSG_ADDR 0
38 #define MSIX_MSG_UPPER_ADDR 4
39 #define MSIX_MSG_DATA 8
40 #define MSIX_VECTOR_CTRL 12
41 #define MSIX_ENTRY_SIZE 16
42 #define MSIX_VECTOR_MASK 0x1
43 #define MSIX_MAX_ENTRIES 32
47 #define DEBUG(fmt, ...) \
49 fprintf(stderr, "%s: " fmt, __func__ , __VA_ARGS__); \
52 #define DEBUG(fmt, ...) do { } while(0)
55 /* Flag for interrupt controller to declare MSI-X support */
59 /* KVM specific MSIX helpers */
60 static void kvm_msix_free(PCIDevice
*dev
)
62 int vector
, changed
= 0;
63 for (vector
= 0; vector
< dev
->msix_entries_nr
; ++vector
) {
64 if (dev
->msix_entry_used
[vector
]) {
65 kvm_del_routing_entry(kvm_context
, &dev
->msix_irq_entries
[vector
]);
70 kvm_commit_irq_routes(kvm_context
);
74 static void kvm_msix_routing_entry(PCIDevice
*dev
, unsigned vector
,
75 struct kvm_irq_routing_entry
*entry
)
77 uint8_t *table_entry
= dev
->msix_table_page
+ vector
* MSIX_ENTRY_SIZE
;
78 entry
->type
= KVM_IRQ_ROUTING_MSI
;
80 entry
->u
.msi
.address_lo
= pci_get_long(table_entry
+ MSIX_MSG_ADDR
);
81 entry
->u
.msi
.address_hi
= pci_get_long(table_entry
+ MSIX_MSG_UPPER_ADDR
);
82 entry
->u
.msi
.data
= pci_get_long(table_entry
+ MSIX_MSG_DATA
);
85 static void kvm_msix_update(PCIDevice
*dev
, int vector
,
86 int was_masked
, int is_masked
)
88 struct kvm_irq_routing_entry e
= {}, *entry
;
89 int mask_cleared
= was_masked
&& !is_masked
;
90 /* It is only legal to change an entry when it is masked. Therefore, it is
91 * enough to update the routing in kernel when mask is being cleared. */
95 if (!dev
->msix_entry_used
[vector
]) {
98 entry
= dev
->msix_irq_entries
+ vector
;
100 kvm_msix_routing_entry(dev
, vector
, &e
);
101 if (memcmp(&entry
->u
.msi
, &e
.u
.msi
, sizeof entry
->u
.msi
)) {
103 r
= kvm_update_routing_entry(kvm_context
, entry
, &e
);
105 fprintf(stderr
, "%s: kvm_update_routing_entry failed: %s\n", __func__
,
109 memcpy(&entry
->u
.msi
, &e
.u
.msi
, sizeof entry
->u
.msi
);
110 r
= kvm_commit_irq_routes(kvm_context
);
112 fprintf(stderr
, "%s: kvm_commit_irq_routes failed: %s\n", __func__
,
119 static int kvm_msix_add(PCIDevice
*dev
, unsigned vector
)
121 struct kvm_irq_routing_entry
*entry
= dev
->msix_irq_entries
+ vector
;
124 if (!kvm_has_gsi_routing(kvm_context
)) {
125 fprintf(stderr
, "Warning: no MSI-X support found. "
126 "At least kernel 2.6.30 is required for MSI-X support.\n"
131 r
= kvm_get_irq_route_gsi(kvm_context
);
133 fprintf(stderr
, "%s: kvm_get_irq_route_gsi failed: %s\n", __func__
, strerror(-r
));
137 kvm_msix_routing_entry(dev
, vector
, entry
);
138 r
= kvm_add_routing_entry(kvm_context
, entry
);
140 fprintf(stderr
, "%s: kvm_add_routing_entry failed: %s\n", __func__
, strerror(-r
));
144 r
= kvm_commit_irq_routes(kvm_context
);
146 fprintf(stderr
, "%s: kvm_commit_irq_routes failed: %s\n", __func__
, strerror(-r
));
152 static void kvm_msix_del(PCIDevice
*dev
, unsigned vector
)
154 if (dev
->msix_entry_used
[vector
]) {
157 kvm_del_routing_entry(kvm_context
, &dev
->msix_irq_entries
[vector
]);
158 kvm_commit_irq_routes(kvm_context
);
162 static void kvm_msix_free(PCIDevice
*dev
) {}
163 static void kvm_msix_update(PCIDevice
*dev
, int vector
,
164 int was_masked
, int is_masked
) {}
165 static int kvm_msix_add(PCIDevice
*dev
, unsigned vector
) { return -1; }
166 static void kvm_msix_del(PCIDevice
*dev
, unsigned vector
) {}
169 /* Reserve second half of the page for pending bits */
170 static int msix_page_pending(PCIDevice
*d
)
172 return (d
->msix_page_size
/ 2);
175 /* Add MSI-X capability to the config space for the device. */
176 /* Given a bar and its size, add MSI-X table on top of it
177 * and fill MSI-X capability in the config space.
178 * Original bar size must be a power of 2 or 0.
179 * New bar size is returned. */
180 static int msix_add_config(struct PCIDevice
*pdev
, unsigned short nentries
,
181 unsigned bar_nr
, unsigned bar_size
)
187 if (nentries
< 1 || nentries
> PCI_MSIX_FLAGS_QSIZE
+ 1)
189 if (bar_size
> 0x80000000)
192 /* Add space for MSI-X structures */
194 new_size
= pdev
->msix_page_size
;
195 } else if (bar_size
< pdev
->msix_page_size
) {
196 bar_size
= pdev
->msix_page_size
;
197 new_size
= pdev
->msix_page_size
* 2;
199 new_size
= bar_size
* 2;
201 pdev
->msix_bar_size
= new_size
;
202 config_offset
= pci_add_capability(pdev
, PCI_CAP_ID_MSIX
, MSIX_CAP_LENGTH
);
203 if (config_offset
< 0)
204 return config_offset
;
205 config
= pdev
->config
+ config_offset
;
207 pci_set_word(config
+ PCI_MSIX_FLAGS
, nentries
- 1);
208 /* Table on top of BAR */
209 pci_set_long(config
+ MSIX_TABLE_OFFSET
, bar_size
| bar_nr
);
210 /* Pending bits on top of that */
211 pci_set_long(config
+ MSIX_PBA_OFFSET
, (bar_size
+ msix_page_pending(pdev
))
213 pdev
->msix_cap
= config_offset
;
214 /* Make flags bit writeable. */
215 pdev
->wmask
[config_offset
+ MSIX_ENABLE_OFFSET
] |= MSIX_ENABLE_MASK
;
219 static void msix_free_irq_entries(PCIDevice
*dev
)
222 if (kvm_enabled() && qemu_kvm_irqchip_in_kernel()) {
226 for (vector
= 0; vector
< dev
->msix_entries_nr
; ++vector
)
227 dev
->msix_entry_used
[vector
] = 0;
230 /* Handle MSI-X capability config write. */
231 void msix_write_config(PCIDevice
*dev
, uint32_t addr
,
232 uint32_t val
, int len
)
234 unsigned enable_pos
= dev
->msix_cap
+ MSIX_ENABLE_OFFSET
;
235 if (addr
+ len
<= enable_pos
|| addr
> enable_pos
)
238 if (msix_enabled(dev
))
239 qemu_set_irq(dev
->irq
[0], 0);
242 static uint32_t msix_mmio_readl(void *opaque
, target_phys_addr_t addr
)
244 PCIDevice
*dev
= opaque
;
245 unsigned int offset
= addr
& (dev
->msix_page_size
- 1);
246 void *page
= dev
->msix_table_page
;
249 memcpy(&val
, (void *)((char *)page
+ offset
), 4);
254 static uint32_t msix_mmio_read_unallowed(void *opaque
, target_phys_addr_t addr
)
256 fprintf(stderr
, "MSI-X: only dword read is allowed!\n");
260 static uint8_t msix_pending_mask(int vector
)
262 return 1 << (vector
% 8);
265 static uint8_t *msix_pending_byte(PCIDevice
*dev
, int vector
)
267 return dev
->msix_table_page
+ msix_page_pending(dev
) + vector
/ 8;
270 static int msix_is_pending(PCIDevice
*dev
, int vector
)
272 return *msix_pending_byte(dev
, vector
) & msix_pending_mask(vector
);
275 static void msix_set_pending(PCIDevice
*dev
, int vector
)
277 *msix_pending_byte(dev
, vector
) |= msix_pending_mask(vector
);
280 static void msix_clr_pending(PCIDevice
*dev
, int vector
)
282 *msix_pending_byte(dev
, vector
) &= ~msix_pending_mask(vector
);
285 static int msix_is_masked(PCIDevice
*dev
, int vector
)
287 unsigned offset
= vector
* MSIX_ENTRY_SIZE
+ MSIX_VECTOR_CTRL
;
288 return dev
->msix_table_page
[offset
] & MSIX_VECTOR_MASK
;
291 static void msix_mmio_writel(void *opaque
, target_phys_addr_t addr
,
294 PCIDevice
*dev
= opaque
;
295 unsigned int offset
= addr
& (dev
->msix_page_size
- 1);
296 int vector
= offset
/ MSIX_ENTRY_SIZE
;
297 int was_masked
= msix_is_masked(dev
, vector
);
298 memcpy(dev
->msix_table_page
+ offset
, &val
, 4);
299 if (kvm_enabled() && qemu_kvm_irqchip_in_kernel()) {
300 kvm_msix_update(dev
, vector
, was_masked
, msix_is_masked(dev
, vector
));
302 if (!msix_is_masked(dev
, vector
) && msix_is_pending(dev
, vector
)) {
303 msix_clr_pending(dev
, vector
);
304 msix_notify(dev
, vector
);
308 static void msix_mmio_write_unallowed(void *opaque
, target_phys_addr_t addr
,
311 fprintf(stderr
, "MSI-X: only dword write is allowed!\n");
314 static CPUWriteMemoryFunc
* const msix_mmio_write
[] = {
315 msix_mmio_write_unallowed
, msix_mmio_write_unallowed
, msix_mmio_writel
318 static CPUReadMemoryFunc
* const msix_mmio_read
[] = {
319 msix_mmio_read_unallowed
, msix_mmio_read_unallowed
, msix_mmio_readl
322 /* Should be called from device's map method. */
323 void msix_mmio_map(PCIDevice
*d
, int region_num
,
324 uint32_t addr
, uint32_t size
, int type
)
326 uint8_t *config
= d
->config
+ d
->msix_cap
;
327 uint32_t table
= pci_get_long(config
+ MSIX_TABLE_OFFSET
);
328 uint32_t offset
= table
& ~(d
->msix_page_size
- 1);
329 /* TODO: for assigned devices, we'll want to make it possible to map
330 * pending bits separately in case they are in a separate bar. */
331 int table_bir
= table
& PCI_MSIX_FLAGS_BIRMASK
;
333 if (table_bir
!= region_num
)
337 cpu_register_physical_memory(addr
+ offset
, size
- offset
,
341 /* Initialize the MSI-X structures. Note: if MSI-X is supported, BAR size is
342 * modified, it should be retrieved with msix_bar_size. */
343 int msix_init(struct PCIDevice
*dev
, unsigned short nentries
,
344 unsigned bar_nr
, unsigned bar_size
, target_phys_addr_t page_size
)
347 /* Nothing to do if MSI is not supported by interrupt controller */
351 if (nentries
> MSIX_MAX_ENTRIES
)
354 #ifdef KVM_CAP_IRQCHIP
355 if (kvm_enabled() && qemu_kvm_irqchip_in_kernel()) {
356 dev
->msix_irq_entries
= qemu_malloc(nentries
*
357 sizeof *dev
->msix_irq_entries
);
360 dev
->msix_entry_used
= qemu_mallocz(MSIX_MAX_ENTRIES
*
361 sizeof *dev
->msix_entry_used
);
363 dev
->msix_page_size
= page_size
;
364 dev
->msix_table_page
= qemu_mallocz(dev
->msix_page_size
);
366 dev
->msix_mmio_index
= cpu_register_io_memory(msix_mmio_read
,
367 msix_mmio_write
, dev
);
368 if (dev
->msix_mmio_index
== -1) {
373 dev
->msix_entries_nr
= nentries
;
374 ret
= msix_add_config(dev
, nentries
, bar_nr
, bar_size
);
378 dev
->cap_present
|= QEMU_PCI_CAP_MSIX
;
382 dev
->msix_entries_nr
= 0;
383 cpu_unregister_io_memory(dev
->msix_mmio_index
);
385 qemu_free(dev
->msix_table_page
);
386 dev
->msix_table_page
= NULL
;
387 qemu_free(dev
->msix_entry_used
);
388 dev
->msix_entry_used
= NULL
;
392 /* Clean up resources for the device. */
393 int msix_uninit(PCIDevice
*dev
)
395 if (!(dev
->cap_present
& QEMU_PCI_CAP_MSIX
))
397 pci_del_capability(dev
, PCI_CAP_ID_MSIX
, MSIX_CAP_LENGTH
);
399 msix_free_irq_entries(dev
);
400 dev
->msix_entries_nr
= 0;
401 cpu_unregister_io_memory(dev
->msix_mmio_index
);
402 qemu_free(dev
->msix_table_page
);
403 dev
->msix_table_page
= NULL
;
404 qemu_free(dev
->msix_entry_used
);
405 dev
->msix_entry_used
= NULL
;
406 qemu_free(dev
->msix_irq_entries
);
407 dev
->msix_irq_entries
= NULL
;
408 dev
->cap_present
&= ~QEMU_PCI_CAP_MSIX
;
412 void msix_save(PCIDevice
*dev
, QEMUFile
*f
)
414 unsigned n
= dev
->msix_entries_nr
;
416 if (!msix_supported
) {
420 if (!(dev
->cap_present
& QEMU_PCI_CAP_MSIX
)) {
423 qemu_put_buffer(f
, dev
->msix_table_page
, n
* MSIX_ENTRY_SIZE
);
424 qemu_put_buffer(f
, dev
->msix_table_page
+ msix_page_pending(dev
),
428 /* Should be called after restoring the config space. */
429 void msix_load(PCIDevice
*dev
, QEMUFile
*f
)
431 unsigned n
= dev
->msix_entries_nr
;
436 if (!(dev
->cap_present
& QEMU_PCI_CAP_MSIX
)) {
440 msix_free_irq_entries(dev
);
441 qemu_get_buffer(f
, dev
->msix_table_page
, n
* MSIX_ENTRY_SIZE
);
442 qemu_get_buffer(f
, dev
->msix_table_page
+ msix_page_pending(dev
),
446 /* Does device support MSI-X? */
447 int msix_present(PCIDevice
*dev
)
449 return dev
->cap_present
& QEMU_PCI_CAP_MSIX
;
452 /* Is MSI-X enabled? */
453 int msix_enabled(PCIDevice
*dev
)
455 return (dev
->cap_present
& QEMU_PCI_CAP_MSIX
) &&
456 (dev
->config
[dev
->msix_cap
+ MSIX_ENABLE_OFFSET
] &
460 /* Size of bar where MSI-X table resides, or 0 if MSI-X not supported. */
461 uint32_t msix_bar_size(PCIDevice
*dev
)
463 return (dev
->cap_present
& QEMU_PCI_CAP_MSIX
) ?
464 dev
->msix_bar_size
: 0;
467 /* Send an MSI-X message */
468 void msix_notify(PCIDevice
*dev
, unsigned vector
)
470 uint8_t *table_entry
= dev
->msix_table_page
+ vector
* MSIX_ENTRY_SIZE
;
474 if (vector
>= dev
->msix_entries_nr
|| !dev
->msix_entry_used
[vector
])
476 if (msix_is_masked(dev
, vector
)) {
477 msix_set_pending(dev
, vector
);
481 #ifdef KVM_CAP_IRQCHIP
482 if (kvm_enabled() && qemu_kvm_irqchip_in_kernel()) {
483 kvm_set_irq(dev
->msix_irq_entries
[vector
].gsi
, 1, NULL
);
488 address
= pci_get_long(table_entry
+ MSIX_MSG_UPPER_ADDR
);
489 address
= (address
<< 32) | pci_get_long(table_entry
+ MSIX_MSG_ADDR
);
490 data
= pci_get_long(table_entry
+ MSIX_MSG_DATA
);
491 stl_phys(address
, data
);
494 void msix_reset(PCIDevice
*dev
)
496 if (!(dev
->cap_present
& QEMU_PCI_CAP_MSIX
))
498 msix_free_irq_entries(dev
);
499 dev
->config
[dev
->msix_cap
+ MSIX_ENABLE_OFFSET
] &= MSIX_ENABLE_MASK
;
500 memset(dev
->msix_table_page
, 0, dev
->msix_page_size
);
503 /* PCI spec suggests that devices make it possible for software to configure
504 * less vectors than supported by the device, but does not specify a standard
505 * mechanism for devices to do so.
507 * We support this by asking devices to declare vectors software is going to
508 * actually use, and checking this on the notification path. Devices that
509 * don't want to follow the spec suggestion can declare all vectors as used. */
511 /* Mark vector as used. */
512 int msix_vector_use(PCIDevice
*dev
, unsigned vector
)
515 if (vector
>= dev
->msix_entries_nr
)
517 if (dev
->msix_entry_used
[vector
]) {
520 if (kvm_enabled() && qemu_kvm_irqchip_in_kernel()) {
521 ret
= kvm_msix_add(dev
, vector
);
526 ++dev
->msix_entry_used
[vector
];
530 /* Mark vector as unused. */
531 void msix_vector_unuse(PCIDevice
*dev
, unsigned vector
)
533 if (vector
< dev
->msix_entries_nr
&& dev
->msix_entry_used
[vector
]) {
534 --dev
->msix_entry_used
[vector
];
535 if (kvm_enabled() && qemu_kvm_irqchip_in_kernel()) {
536 kvm_msix_del(dev
, vector
);