4 * This module includes support for MSI-X in pci devices.
6 * Author: Michael S. Tsirkin <mst@redhat.com>
8 * Copyright (c) 2009, Red Hat Inc, Michael S. Tsirkin (mst@redhat.com)
10 * This work is licensed under the terms of the GNU GPL, version 2. See
11 * the COPYING file in the top-level directory.
13 * Contributions after 2012-01-13 are licensed under the terms of the
14 * GNU GPL, version 2 or (at your option) any later version.
23 #define MSIX_CAP_LENGTH 12
25 /* MSI enable bit and maskall bit are in byte 1 in FLAGS register */
26 #define MSIX_CONTROL_OFFSET (PCI_MSIX_FLAGS + 1)
27 #define MSIX_ENABLE_MASK (PCI_MSIX_FLAGS_ENABLE >> 8)
28 #define MSIX_MASKALL_MASK (PCI_MSIX_FLAGS_MASKALL >> 8)
30 /* How much space does an MSIX table need. */
31 /* The spec requires giving the table structure
32 * a 4K aligned region all by itself. */
33 #define MSIX_PAGE_SIZE 0x1000
34 /* Reserve second half of the page for pending bits */
35 #define MSIX_PAGE_PENDING (MSIX_PAGE_SIZE / 2)
36 #define MSIX_MAX_ENTRIES 32
38 static MSIMessage
msix_get_message(PCIDevice
*dev
, unsigned vector
)
40 uint8_t *table_entry
= dev
->msix_table
+ vector
* PCI_MSIX_ENTRY_SIZE
;
43 msg
.address
= pci_get_quad(table_entry
+ PCI_MSIX_ENTRY_LOWER_ADDR
);
44 msg
.data
= pci_get_long(table_entry
+ PCI_MSIX_ENTRY_DATA
);
48 /* Add MSI-X capability to the config space for the device. */
49 /* Given a bar and its size, add MSI-X table on top of it
50 * and fill MSI-X capability in the config space.
51 * Original bar size must be a power of 2 or 0.
52 * New bar size is returned. */
53 static int msix_add_config(struct PCIDevice
*pdev
, unsigned short nentries
,
54 unsigned bar_nr
, unsigned bar_size
)
59 if (nentries
< 1 || nentries
> PCI_MSIX_FLAGS_QSIZE
+ 1)
61 if (bar_size
> 0x80000000)
64 /* Require aligned offset for MSI-X structures */
65 if (bar_size
& ~(MSIX_PAGE_SIZE
- 1)) {
69 config_offset
= pci_add_capability(pdev
, PCI_CAP_ID_MSIX
,
71 if (config_offset
< 0)
73 config
= pdev
->config
+ config_offset
;
75 pci_set_word(config
+ PCI_MSIX_FLAGS
, nentries
- 1);
76 /* Table on top of BAR */
77 pci_set_long(config
+ PCI_MSIX_TABLE
, bar_size
| bar_nr
);
78 /* Pending bits on top of that */
79 pci_set_long(config
+ PCI_MSIX_PBA
, (bar_size
+ MSIX_PAGE_PENDING
) |
81 pdev
->msix_cap
= config_offset
;
82 /* Make flags bit writable. */
83 pdev
->wmask
[config_offset
+ MSIX_CONTROL_OFFSET
] |= MSIX_ENABLE_MASK
|
85 pdev
->msix_function_masked
= true;
89 static uint8_t msix_pending_mask(int vector
)
91 return 1 << (vector
% 8);
94 static uint8_t *msix_pending_byte(PCIDevice
*dev
, int vector
)
96 return dev
->msix_pba
+ vector
/ 8;
99 static int msix_is_pending(PCIDevice
*dev
, int vector
)
101 return *msix_pending_byte(dev
, vector
) & msix_pending_mask(vector
);
104 static void msix_set_pending(PCIDevice
*dev
, int vector
)
106 *msix_pending_byte(dev
, vector
) |= msix_pending_mask(vector
);
109 static void msix_clr_pending(PCIDevice
*dev
, int vector
)
111 *msix_pending_byte(dev
, vector
) &= ~msix_pending_mask(vector
);
114 static bool msix_vector_masked(PCIDevice
*dev
, int vector
, bool fmask
)
116 unsigned offset
= vector
* PCI_MSIX_ENTRY_SIZE
+ PCI_MSIX_ENTRY_VECTOR_CTRL
;
117 return fmask
|| dev
->msix_table
[offset
] & PCI_MSIX_ENTRY_CTRL_MASKBIT
;
120 static bool msix_is_masked(PCIDevice
*dev
, int vector
)
122 return msix_vector_masked(dev
, vector
, dev
->msix_function_masked
);
125 static void msix_fire_vector_notifier(PCIDevice
*dev
,
126 unsigned int vector
, bool is_masked
)
131 if (!dev
->msix_vector_use_notifier
) {
135 dev
->msix_vector_release_notifier(dev
, vector
);
137 msg
= msix_get_message(dev
, vector
);
138 ret
= dev
->msix_vector_use_notifier(dev
, vector
, msg
);
143 static void msix_handle_mask_update(PCIDevice
*dev
, int vector
, bool was_masked
)
145 bool is_masked
= msix_is_masked(dev
, vector
);
147 if (is_masked
== was_masked
) {
151 msix_fire_vector_notifier(dev
, vector
, is_masked
);
153 if (!is_masked
&& msix_is_pending(dev
, vector
)) {
154 msix_clr_pending(dev
, vector
);
155 msix_notify(dev
, vector
);
159 static void msix_update_function_masked(PCIDevice
*dev
)
161 dev
->msix_function_masked
= !msix_enabled(dev
) ||
162 (dev
->config
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
] & MSIX_MASKALL_MASK
);
165 /* Handle MSI-X capability config write. */
166 void msix_write_config(PCIDevice
*dev
, uint32_t addr
,
167 uint32_t val
, int len
)
169 unsigned enable_pos
= dev
->msix_cap
+ MSIX_CONTROL_OFFSET
;
173 if (!msix_present(dev
) || !range_covers_byte(addr
, len
, enable_pos
)) {
177 was_masked
= dev
->msix_function_masked
;
178 msix_update_function_masked(dev
);
180 if (!msix_enabled(dev
)) {
184 pci_device_deassert_intx(dev
);
186 if (dev
->msix_function_masked
== was_masked
) {
190 for (vector
= 0; vector
< dev
->msix_entries_nr
; ++vector
) {
191 msix_handle_mask_update(dev
, vector
,
192 msix_vector_masked(dev
, vector
, was_masked
));
196 static uint64_t msix_table_mmio_read(void *opaque
, target_phys_addr_t addr
,
199 PCIDevice
*dev
= opaque
;
201 return pci_get_long(dev
->msix_table
+ addr
);
204 static void msix_table_mmio_write(void *opaque
, target_phys_addr_t addr
,
205 uint64_t val
, unsigned size
)
207 PCIDevice
*dev
= opaque
;
208 int vector
= addr
/ PCI_MSIX_ENTRY_SIZE
;
211 was_masked
= msix_is_masked(dev
, vector
);
212 pci_set_long(dev
->msix_table
+ addr
, val
);
213 msix_handle_mask_update(dev
, vector
, was_masked
);
216 static const MemoryRegionOps msix_table_mmio_ops
= {
217 .read
= msix_table_mmio_read
,
218 .write
= msix_table_mmio_write
,
219 /* TODO: MSIX should be LITTLE_ENDIAN. */
220 .endianness
= DEVICE_NATIVE_ENDIAN
,
222 .min_access_size
= 4,
223 .max_access_size
= 4,
227 static uint64_t msix_pba_mmio_read(void *opaque
, target_phys_addr_t addr
,
230 PCIDevice
*dev
= opaque
;
232 return pci_get_long(dev
->msix_pba
+ addr
);
235 static const MemoryRegionOps msix_pba_mmio_ops
= {
236 .read
= msix_pba_mmio_read
,
237 /* TODO: MSIX should be LITTLE_ENDIAN. */
238 .endianness
= DEVICE_NATIVE_ENDIAN
,
240 .min_access_size
= 4,
241 .max_access_size
= 4,
245 static void msix_mmio_setup(PCIDevice
*d
, MemoryRegion
*bar
)
247 uint8_t *config
= d
->config
+ d
->msix_cap
;
248 uint32_t table
= pci_get_long(config
+ PCI_MSIX_TABLE
);
249 uint32_t table_offset
= table
& ~PCI_MSIX_FLAGS_BIRMASK
;
250 uint32_t pba
= pci_get_long(config
+ PCI_MSIX_PBA
);
251 uint32_t pba_offset
= pba
& ~PCI_MSIX_FLAGS_BIRMASK
;
252 /* TODO: for assigned devices, we'll want to make it possible to map
253 * pending bits separately in case they are in a separate bar. */
255 memory_region_add_subregion(bar
, table_offset
, &d
->msix_table_mmio
);
256 memory_region_add_subregion(bar
, pba_offset
, &d
->msix_pba_mmio
);
259 static void msix_mask_all(struct PCIDevice
*dev
, unsigned nentries
)
263 for (vector
= 0; vector
< nentries
; ++vector
) {
265 vector
* PCI_MSIX_ENTRY_SIZE
+ PCI_MSIX_ENTRY_VECTOR_CTRL
;
266 bool was_masked
= msix_is_masked(dev
, vector
);
268 dev
->msix_table
[offset
] |= PCI_MSIX_ENTRY_CTRL_MASKBIT
;
269 msix_handle_mask_update(dev
, vector
, was_masked
);
273 /* Initialize the MSI-X structures. Note: if MSI-X is supported, BAR size is
274 * modified, it should be retrieved with msix_bar_size. */
275 int msix_init(struct PCIDevice
*dev
, unsigned short nentries
,
277 unsigned bar_nr
, unsigned bar_size
)
280 unsigned table_size
, pba_size
;
282 /* Nothing to do if MSI is not supported by interrupt controller */
283 if (!msi_supported
) {
286 if (nentries
> MSIX_MAX_ENTRIES
)
289 table_size
= nentries
* PCI_MSIX_ENTRY_SIZE
;
290 pba_size
= QEMU_ALIGN_UP(nentries
, 64) / 8;
292 dev
->msix_entry_used
= g_malloc0(MSIX_MAX_ENTRIES
*
293 sizeof *dev
->msix_entry_used
);
295 dev
->msix_table
= g_malloc0(table_size
);
296 dev
->msix_pba
= g_malloc0(pba_size
);
297 msix_mask_all(dev
, nentries
);
299 memory_region_init_io(&dev
->msix_table_mmio
, &msix_table_mmio_ops
, dev
,
300 "msix-table", table_size
);
301 memory_region_init_io(&dev
->msix_pba_mmio
, &msix_pba_mmio_ops
, dev
,
302 "msix-pba", pba_size
);
304 dev
->msix_entries_nr
= nentries
;
305 ret
= msix_add_config(dev
, nentries
, bar_nr
, bar_size
);
309 dev
->cap_present
|= QEMU_PCI_CAP_MSIX
;
310 msix_mmio_setup(dev
, bar
);
314 dev
->msix_entries_nr
= 0;
315 memory_region_destroy(&dev
->msix_pba_mmio
);
316 g_free(dev
->msix_pba
);
317 dev
->msix_pba
= NULL
;
318 memory_region_destroy(&dev
->msix_table_mmio
);
319 g_free(dev
->msix_table
);
320 dev
->msix_table
= NULL
;
321 g_free(dev
->msix_entry_used
);
322 dev
->msix_entry_used
= NULL
;
326 int msix_init_exclusive_bar(PCIDevice
*dev
, unsigned short nentries
,
333 * Migration compatibility dictates that this remains a 4k
334 * BAR with the vector table in the lower half and PBA in
335 * the upper half. Do not use these elsewhere!
337 #define MSIX_EXCLUSIVE_BAR_SIZE 4096
338 #define MSIX_EXCLUSIVE_BAR_PBA_OFFSET (MSIX_EXCLUSIVE_BAR_SIZE / 2)
340 if (nentries
* PCI_MSIX_ENTRY_SIZE
> MSIX_EXCLUSIVE_BAR_PBA_OFFSET
) {
344 if (asprintf(&name
, "%s-msix", dev
->name
) == -1) {
348 memory_region_init(&dev
->msix_exclusive_bar
, name
, MSIX_EXCLUSIVE_BAR_SIZE
);
352 ret
= msix_init(dev
, nentries
, &dev
->msix_exclusive_bar
, bar_nr
,
353 MSIX_EXCLUSIVE_BAR_SIZE
);
355 memory_region_destroy(&dev
->msix_exclusive_bar
);
359 pci_register_bar(dev
, bar_nr
, PCI_BASE_ADDRESS_SPACE_MEMORY
,
360 &dev
->msix_exclusive_bar
);
365 static void msix_free_irq_entries(PCIDevice
*dev
)
369 for (vector
= 0; vector
< dev
->msix_entries_nr
; ++vector
) {
370 dev
->msix_entry_used
[vector
] = 0;
371 msix_clr_pending(dev
, vector
);
375 /* Clean up resources for the device. */
376 int msix_uninit(PCIDevice
*dev
, MemoryRegion
*bar
)
378 if (!msix_present(dev
)) {
381 pci_del_capability(dev
, PCI_CAP_ID_MSIX
, MSIX_CAP_LENGTH
);
383 msix_free_irq_entries(dev
);
384 dev
->msix_entries_nr
= 0;
385 memory_region_del_subregion(bar
, &dev
->msix_pba_mmio
);
386 memory_region_destroy(&dev
->msix_pba_mmio
);
387 g_free(dev
->msix_pba
);
388 dev
->msix_pba
= NULL
;
389 memory_region_del_subregion(bar
, &dev
->msix_table_mmio
);
390 memory_region_destroy(&dev
->msix_table_mmio
);
391 g_free(dev
->msix_table
);
392 dev
->msix_table
= NULL
;
393 g_free(dev
->msix_entry_used
);
394 dev
->msix_entry_used
= NULL
;
395 dev
->cap_present
&= ~QEMU_PCI_CAP_MSIX
;
399 void msix_uninit_exclusive_bar(PCIDevice
*dev
)
401 if (msix_present(dev
)) {
402 msix_uninit(dev
, &dev
->msix_exclusive_bar
);
403 memory_region_destroy(&dev
->msix_exclusive_bar
);
407 void msix_save(PCIDevice
*dev
, QEMUFile
*f
)
409 unsigned n
= dev
->msix_entries_nr
;
411 if (!msix_present(dev
)) {
415 qemu_put_buffer(f
, dev
->msix_table
, n
* PCI_MSIX_ENTRY_SIZE
);
416 qemu_put_buffer(f
, dev
->msix_pba
, (n
+ 7) / 8);
419 /* Should be called after restoring the config space. */
420 void msix_load(PCIDevice
*dev
, QEMUFile
*f
)
422 unsigned n
= dev
->msix_entries_nr
;
425 if (!msix_present(dev
)) {
429 msix_free_irq_entries(dev
);
430 qemu_get_buffer(f
, dev
->msix_table
, n
* PCI_MSIX_ENTRY_SIZE
);
431 qemu_get_buffer(f
, dev
->msix_pba
, (n
+ 7) / 8);
432 msix_update_function_masked(dev
);
434 for (vector
= 0; vector
< n
; vector
++) {
435 msix_handle_mask_update(dev
, vector
, true);
439 /* Does device support MSI-X? */
440 int msix_present(PCIDevice
*dev
)
442 return dev
->cap_present
& QEMU_PCI_CAP_MSIX
;
445 /* Is MSI-X enabled? */
446 int msix_enabled(PCIDevice
*dev
)
448 return (dev
->cap_present
& QEMU_PCI_CAP_MSIX
) &&
449 (dev
->config
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
] &
453 /* Send an MSI-X message */
454 void msix_notify(PCIDevice
*dev
, unsigned vector
)
458 if (vector
>= dev
->msix_entries_nr
|| !dev
->msix_entry_used
[vector
])
460 if (msix_is_masked(dev
, vector
)) {
461 msix_set_pending(dev
, vector
);
465 msg
= msix_get_message(dev
, vector
);
467 stl_le_phys(msg
.address
, msg
.data
);
470 void msix_reset(PCIDevice
*dev
)
472 if (!msix_present(dev
)) {
475 msix_free_irq_entries(dev
);
476 dev
->config
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
] &=
477 ~dev
->wmask
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
];
478 memset(dev
->msix_table
, 0, dev
->msix_entries_nr
* PCI_MSIX_ENTRY_SIZE
);
479 memset(dev
->msix_pba
, 0, QEMU_ALIGN_UP(dev
->msix_entries_nr
, 64) / 8);
480 msix_mask_all(dev
, dev
->msix_entries_nr
);
483 /* PCI spec suggests that devices make it possible for software to configure
484 * less vectors than supported by the device, but does not specify a standard
485 * mechanism for devices to do so.
487 * We support this by asking devices to declare vectors software is going to
488 * actually use, and checking this on the notification path. Devices that
489 * don't want to follow the spec suggestion can declare all vectors as used. */
491 /* Mark vector as used. */
492 int msix_vector_use(PCIDevice
*dev
, unsigned vector
)
494 if (vector
>= dev
->msix_entries_nr
)
496 dev
->msix_entry_used
[vector
]++;
500 /* Mark vector as unused. */
501 void msix_vector_unuse(PCIDevice
*dev
, unsigned vector
)
503 if (vector
>= dev
->msix_entries_nr
|| !dev
->msix_entry_used
[vector
]) {
506 if (--dev
->msix_entry_used
[vector
]) {
509 msix_clr_pending(dev
, vector
);
512 void msix_unuse_all_vectors(PCIDevice
*dev
)
514 if (!msix_present(dev
)) {
517 msix_free_irq_entries(dev
);
520 unsigned int msix_nr_vectors_allocated(const PCIDevice
*dev
)
522 return dev
->msix_entries_nr
;
525 static int msix_set_notifier_for_vector(PCIDevice
*dev
, unsigned int vector
)
529 if (msix_is_masked(dev
, vector
)) {
532 msg
= msix_get_message(dev
, vector
);
533 return dev
->msix_vector_use_notifier(dev
, vector
, msg
);
536 static void msix_unset_notifier_for_vector(PCIDevice
*dev
, unsigned int vector
)
538 if (msix_is_masked(dev
, vector
)) {
541 dev
->msix_vector_release_notifier(dev
, vector
);
544 int msix_set_vector_notifiers(PCIDevice
*dev
,
545 MSIVectorUseNotifier use_notifier
,
546 MSIVectorReleaseNotifier release_notifier
)
550 assert(use_notifier
&& release_notifier
);
552 dev
->msix_vector_use_notifier
= use_notifier
;
553 dev
->msix_vector_release_notifier
= release_notifier
;
555 if ((dev
->config
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
] &
556 (MSIX_ENABLE_MASK
| MSIX_MASKALL_MASK
)) == MSIX_ENABLE_MASK
) {
557 for (vector
= 0; vector
< dev
->msix_entries_nr
; vector
++) {
558 ret
= msix_set_notifier_for_vector(dev
, vector
);
567 while (--vector
>= 0) {
568 msix_unset_notifier_for_vector(dev
, vector
);
570 dev
->msix_vector_use_notifier
= NULL
;
571 dev
->msix_vector_release_notifier
= NULL
;
575 void msix_unset_vector_notifiers(PCIDevice
*dev
)
579 assert(dev
->msix_vector_use_notifier
&&
580 dev
->msix_vector_release_notifier
);
582 if ((dev
->config
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
] &
583 (MSIX_ENABLE_MASK
| MSIX_MASKALL_MASK
)) == MSIX_ENABLE_MASK
) {
584 for (vector
= 0; vector
< dev
->msix_entries_nr
; vector
++) {
585 msix_unset_notifier_for_vector(dev
, vector
);
588 dev
->msix_vector_use_notifier
= NULL
;
589 dev
->msix_vector_release_notifier
= NULL
;