4 * This module includes support for MSI-X in pci devices.
6 * Author: Michael S. Tsirkin <mst@redhat.com>
8 * Copyright (c) 2009, Red Hat Inc, Michael S. Tsirkin (mst@redhat.com)
10 * This work is licensed under the terms of the GNU GPL, version 2. See
11 * the COPYING file in the top-level directory.
13 * Contributions after 2012-01-13 are licensed under the terms of the
14 * GNU GPL, version 2 or (at your option) any later version.
17 #include "qemu/osdep.h"
18 #include "hw/pci/msi.h"
19 #include "hw/pci/msix.h"
20 #include "hw/pci/pci.h"
21 #include "hw/xen/xen.h"
22 #include "migration/qemu-file-types.h"
23 #include "migration/vmstate.h"
24 #include "qemu/range.h"
25 #include "qapi/error.h"
28 /* MSI enable bit and maskall bit are in byte 1 in FLAGS register */
29 #define MSIX_CONTROL_OFFSET (PCI_MSIX_FLAGS + 1)
30 #define MSIX_ENABLE_MASK (PCI_MSIX_FLAGS_ENABLE >> 8)
31 #define MSIX_MASKALL_MASK (PCI_MSIX_FLAGS_MASKALL >> 8)
33 MSIMessage
msix_get_message(PCIDevice
*dev
, unsigned vector
)
35 uint8_t *table_entry
= dev
->msix_table
+ vector
* PCI_MSIX_ENTRY_SIZE
;
38 msg
.address
= pci_get_quad(table_entry
+ PCI_MSIX_ENTRY_LOWER_ADDR
);
39 msg
.data
= pci_get_long(table_entry
+ PCI_MSIX_ENTRY_DATA
);
44 * Special API for POWER to configure the vectors through
45 * a side channel. Should never be used by devices.
47 void msix_set_message(PCIDevice
*dev
, int vector
, struct MSIMessage msg
)
49 uint8_t *table_entry
= dev
->msix_table
+ vector
* PCI_MSIX_ENTRY_SIZE
;
51 pci_set_quad(table_entry
+ PCI_MSIX_ENTRY_LOWER_ADDR
, msg
.address
);
52 pci_set_long(table_entry
+ PCI_MSIX_ENTRY_DATA
, msg
.data
);
53 table_entry
[PCI_MSIX_ENTRY_VECTOR_CTRL
] &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT
;
56 static uint8_t msix_pending_mask(int vector
)
58 return 1 << (vector
% 8);
61 static uint8_t *msix_pending_byte(PCIDevice
*dev
, int vector
)
63 return dev
->msix_pba
+ vector
/ 8;
66 static int msix_is_pending(PCIDevice
*dev
, int vector
)
68 return *msix_pending_byte(dev
, vector
) & msix_pending_mask(vector
);
71 void msix_set_pending(PCIDevice
*dev
, unsigned int vector
)
73 *msix_pending_byte(dev
, vector
) |= msix_pending_mask(vector
);
76 void msix_clr_pending(PCIDevice
*dev
, int vector
)
78 *msix_pending_byte(dev
, vector
) &= ~msix_pending_mask(vector
);
81 static bool msix_vector_masked(PCIDevice
*dev
, unsigned int vector
, bool fmask
)
83 unsigned offset
= vector
* PCI_MSIX_ENTRY_SIZE
;
84 uint8_t *data
= &dev
->msix_table
[offset
+ PCI_MSIX_ENTRY_DATA
];
85 /* MSIs on Xen can be remapped into pirqs. In those cases, masking
86 * and unmasking go through the PV evtchn path. */
87 if (xen_enabled() && xen_is_pirq_msi(pci_get_long(data
))) {
90 return fmask
|| dev
->msix_table
[offset
+ PCI_MSIX_ENTRY_VECTOR_CTRL
] &
91 PCI_MSIX_ENTRY_CTRL_MASKBIT
;
94 bool msix_is_masked(PCIDevice
*dev
, unsigned int vector
)
96 return msix_vector_masked(dev
, vector
, dev
->msix_function_masked
);
99 static void msix_fire_vector_notifier(PCIDevice
*dev
,
100 unsigned int vector
, bool is_masked
)
105 if (!dev
->msix_vector_use_notifier
) {
109 dev
->msix_vector_release_notifier(dev
, vector
);
111 msg
= msix_get_message(dev
, vector
);
112 ret
= dev
->msix_vector_use_notifier(dev
, vector
, msg
);
117 static void msix_handle_mask_update(PCIDevice
*dev
, int vector
, bool was_masked
)
119 bool is_masked
= msix_is_masked(dev
, vector
);
121 if (is_masked
== was_masked
) {
125 msix_fire_vector_notifier(dev
, vector
, is_masked
);
127 if (!is_masked
&& msix_is_pending(dev
, vector
)) {
128 msix_clr_pending(dev
, vector
);
129 msix_notify(dev
, vector
);
133 static bool msix_masked(PCIDevice
*dev
)
135 return dev
->config
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
] & MSIX_MASKALL_MASK
;
138 static void msix_update_function_masked(PCIDevice
*dev
)
140 dev
->msix_function_masked
= !msix_enabled(dev
) || msix_masked(dev
);
143 /* Handle MSI-X capability config write. */
144 void msix_write_config(PCIDevice
*dev
, uint32_t addr
,
145 uint32_t val
, int len
)
147 unsigned enable_pos
= dev
->msix_cap
+ MSIX_CONTROL_OFFSET
;
151 if (!msix_present(dev
) || !range_covers_byte(addr
, len
, enable_pos
)) {
155 trace_msix_write_config(dev
->name
, msix_enabled(dev
), msix_masked(dev
));
157 was_masked
= dev
->msix_function_masked
;
158 msix_update_function_masked(dev
);
160 if (!msix_enabled(dev
)) {
164 pci_device_deassert_intx(dev
);
166 if (dev
->msix_function_masked
== was_masked
) {
170 for (vector
= 0; vector
< dev
->msix_entries_nr
; ++vector
) {
171 msix_handle_mask_update(dev
, vector
,
172 msix_vector_masked(dev
, vector
, was_masked
));
176 static uint64_t msix_table_mmio_read(void *opaque
, hwaddr addr
,
179 PCIDevice
*dev
= opaque
;
181 return pci_get_long(dev
->msix_table
+ addr
);
184 static void msix_table_mmio_write(void *opaque
, hwaddr addr
,
185 uint64_t val
, unsigned size
)
187 PCIDevice
*dev
= opaque
;
188 int vector
= addr
/ PCI_MSIX_ENTRY_SIZE
;
191 was_masked
= msix_is_masked(dev
, vector
);
192 pci_set_long(dev
->msix_table
+ addr
, val
);
193 msix_handle_mask_update(dev
, vector
, was_masked
);
196 static const MemoryRegionOps msix_table_mmio_ops
= {
197 .read
= msix_table_mmio_read
,
198 .write
= msix_table_mmio_write
,
199 .endianness
= DEVICE_LITTLE_ENDIAN
,
201 .min_access_size
= 4,
202 .max_access_size
= 8,
205 .max_access_size
= 4,
209 static uint64_t msix_pba_mmio_read(void *opaque
, hwaddr addr
,
212 PCIDevice
*dev
= opaque
;
213 if (dev
->msix_vector_poll_notifier
) {
214 unsigned vector_start
= addr
* 8;
215 unsigned vector_end
= MIN(addr
+ size
* 8, dev
->msix_entries_nr
);
216 dev
->msix_vector_poll_notifier(dev
, vector_start
, vector_end
);
219 return pci_get_long(dev
->msix_pba
+ addr
);
222 static void msix_pba_mmio_write(void *opaque
, hwaddr addr
,
223 uint64_t val
, unsigned size
)
227 static const MemoryRegionOps msix_pba_mmio_ops
= {
228 .read
= msix_pba_mmio_read
,
229 .write
= msix_pba_mmio_write
,
230 .endianness
= DEVICE_LITTLE_ENDIAN
,
232 .min_access_size
= 4,
233 .max_access_size
= 8,
236 .max_access_size
= 4,
240 static void msix_mask_all(struct PCIDevice
*dev
, unsigned nentries
)
244 for (vector
= 0; vector
< nentries
; ++vector
) {
246 vector
* PCI_MSIX_ENTRY_SIZE
+ PCI_MSIX_ENTRY_VECTOR_CTRL
;
247 bool was_masked
= msix_is_masked(dev
, vector
);
249 dev
->msix_table
[offset
] |= PCI_MSIX_ENTRY_CTRL_MASKBIT
;
250 msix_handle_mask_update(dev
, vector
, was_masked
);
255 * Make PCI device @dev MSI-X capable
256 * @nentries is the max number of MSI-X vectors that the device support.
257 * @table_bar is the MemoryRegion that MSI-X table structure resides.
258 * @table_bar_nr is number of base address register corresponding to @table_bar.
259 * @table_offset indicates the offset that the MSI-X table structure starts with
261 * @pba_bar is the MemoryRegion that the Pending Bit Array structure resides.
262 * @pba_bar_nr is number of base address register corresponding to @pba_bar.
263 * @pba_offset indicates the offset that the Pending Bit Array structure
264 * starts with in @pba_bar.
265 * Non-zero @cap_pos puts capability MSI-X at that offset in PCI config space.
266 * @errp is for returning errors.
268 * Return 0 on success; set @errp and return -errno on error:
269 * -ENOTSUP means lacking msi support for a msi-capable platform.
270 * -EINVAL means capability overlap, happens when @cap_pos is non-zero,
271 * also means a programming error, except device assignment, which can check
272 * if a real HW is broken.
274 int msix_init(struct PCIDevice
*dev
, unsigned short nentries
,
275 MemoryRegion
*table_bar
, uint8_t table_bar_nr
,
276 unsigned table_offset
, MemoryRegion
*pba_bar
,
277 uint8_t pba_bar_nr
, unsigned pba_offset
, uint8_t cap_pos
,
281 unsigned table_size
, pba_size
;
284 /* Nothing to do if MSI is not supported by interrupt controller */
285 if (!msi_nonbroken
) {
286 error_setg(errp
, "MSI-X is not supported by interrupt controller");
290 if (nentries
< 1 || nentries
> PCI_MSIX_FLAGS_QSIZE
+ 1) {
291 error_setg(errp
, "The number of MSI-X vectors is invalid");
295 table_size
= nentries
* PCI_MSIX_ENTRY_SIZE
;
296 pba_size
= QEMU_ALIGN_UP(nentries
, 64) / 8;
298 /* Sanity test: table & pba don't overlap, fit within BARs, min aligned */
299 if ((table_bar_nr
== pba_bar_nr
&&
300 ranges_overlap(table_offset
, table_size
, pba_offset
, pba_size
)) ||
301 table_offset
+ table_size
> memory_region_size(table_bar
) ||
302 pba_offset
+ pba_size
> memory_region_size(pba_bar
) ||
303 (table_offset
| pba_offset
) & PCI_MSIX_FLAGS_BIRMASK
) {
304 error_setg(errp
, "table & pba overlap, or they don't fit in BARs,"
309 cap
= pci_add_capability(dev
, PCI_CAP_ID_MSIX
,
310 cap_pos
, MSIX_CAP_LENGTH
, errp
);
316 dev
->cap_present
|= QEMU_PCI_CAP_MSIX
;
317 config
= dev
->config
+ cap
;
319 pci_set_word(config
+ PCI_MSIX_FLAGS
, nentries
- 1);
320 dev
->msix_entries_nr
= nentries
;
321 dev
->msix_function_masked
= true;
323 pci_set_long(config
+ PCI_MSIX_TABLE
, table_offset
| table_bar_nr
);
324 pci_set_long(config
+ PCI_MSIX_PBA
, pba_offset
| pba_bar_nr
);
326 /* Make flags bit writable. */
327 dev
->wmask
[cap
+ MSIX_CONTROL_OFFSET
] |= MSIX_ENABLE_MASK
|
330 dev
->msix_table
= g_malloc0(table_size
);
331 dev
->msix_pba
= g_malloc0(pba_size
);
332 dev
->msix_entry_used
= g_malloc0(nentries
* sizeof *dev
->msix_entry_used
);
334 msix_mask_all(dev
, nentries
);
336 memory_region_init_io(&dev
->msix_table_mmio
, OBJECT(dev
), &msix_table_mmio_ops
, dev
,
337 "msix-table", table_size
);
338 memory_region_add_subregion(table_bar
, table_offset
, &dev
->msix_table_mmio
);
339 memory_region_init_io(&dev
->msix_pba_mmio
, OBJECT(dev
), &msix_pba_mmio_ops
, dev
,
340 "msix-pba", pba_size
);
341 memory_region_add_subregion(pba_bar
, pba_offset
, &dev
->msix_pba_mmio
);
346 int msix_init_exclusive_bar(PCIDevice
*dev
, unsigned short nentries
,
347 uint8_t bar_nr
, Error
**errp
)
351 uint32_t bar_size
= 4096;
352 uint32_t bar_pba_offset
= bar_size
/ 2;
353 uint32_t bar_pba_size
= QEMU_ALIGN_UP(nentries
, 64) / 8;
356 * Migration compatibility dictates that this remains a 4k
357 * BAR with the vector table in the lower half and PBA in
358 * the upper half for nentries which is lower or equal to 128.
359 * No need to care about using more than 65 entries for legacy
360 * machine types who has at most 64 queues.
362 if (nentries
* PCI_MSIX_ENTRY_SIZE
> bar_pba_offset
) {
363 bar_pba_offset
= nentries
* PCI_MSIX_ENTRY_SIZE
;
366 if (bar_pba_offset
+ bar_pba_size
> 4096) {
367 bar_size
= bar_pba_offset
+ bar_pba_size
;
370 bar_size
= pow2ceil(bar_size
);
372 name
= g_strdup_printf("%s-msix", dev
->name
);
373 memory_region_init(&dev
->msix_exclusive_bar
, OBJECT(dev
), name
, bar_size
);
376 ret
= msix_init(dev
, nentries
, &dev
->msix_exclusive_bar
, bar_nr
,
377 0, &dev
->msix_exclusive_bar
,
378 bar_nr
, bar_pba_offset
,
384 pci_register_bar(dev
, bar_nr
, PCI_BASE_ADDRESS_SPACE_MEMORY
,
385 &dev
->msix_exclusive_bar
);
390 static void msix_free_irq_entries(PCIDevice
*dev
)
394 for (vector
= 0; vector
< dev
->msix_entries_nr
; ++vector
) {
395 dev
->msix_entry_used
[vector
] = 0;
396 msix_clr_pending(dev
, vector
);
400 static void msix_clear_all_vectors(PCIDevice
*dev
)
404 for (vector
= 0; vector
< dev
->msix_entries_nr
; ++vector
) {
405 msix_clr_pending(dev
, vector
);
409 /* Clean up resources for the device. */
410 void msix_uninit(PCIDevice
*dev
, MemoryRegion
*table_bar
, MemoryRegion
*pba_bar
)
412 if (!msix_present(dev
)) {
415 pci_del_capability(dev
, PCI_CAP_ID_MSIX
, MSIX_CAP_LENGTH
);
417 msix_free_irq_entries(dev
);
418 dev
->msix_entries_nr
= 0;
419 memory_region_del_subregion(pba_bar
, &dev
->msix_pba_mmio
);
420 g_free(dev
->msix_pba
);
421 dev
->msix_pba
= NULL
;
422 memory_region_del_subregion(table_bar
, &dev
->msix_table_mmio
);
423 g_free(dev
->msix_table
);
424 dev
->msix_table
= NULL
;
425 g_free(dev
->msix_entry_used
);
426 dev
->msix_entry_used
= NULL
;
427 dev
->cap_present
&= ~QEMU_PCI_CAP_MSIX
;
430 void msix_uninit_exclusive_bar(PCIDevice
*dev
)
432 if (msix_present(dev
)) {
433 msix_uninit(dev
, &dev
->msix_exclusive_bar
, &dev
->msix_exclusive_bar
);
437 void msix_save(PCIDevice
*dev
, QEMUFile
*f
)
439 unsigned n
= dev
->msix_entries_nr
;
441 if (!msix_present(dev
)) {
445 qemu_put_buffer(f
, dev
->msix_table
, n
* PCI_MSIX_ENTRY_SIZE
);
446 qemu_put_buffer(f
, dev
->msix_pba
, DIV_ROUND_UP(n
, 8));
449 /* Should be called after restoring the config space. */
450 void msix_load(PCIDevice
*dev
, QEMUFile
*f
)
452 unsigned n
= dev
->msix_entries_nr
;
455 if (!msix_present(dev
)) {
459 msix_clear_all_vectors(dev
);
460 qemu_get_buffer(f
, dev
->msix_table
, n
* PCI_MSIX_ENTRY_SIZE
);
461 qemu_get_buffer(f
, dev
->msix_pba
, DIV_ROUND_UP(n
, 8));
462 msix_update_function_masked(dev
);
464 for (vector
= 0; vector
< n
; vector
++) {
465 msix_handle_mask_update(dev
, vector
, true);
469 /* Does device support MSI-X? */
470 int msix_present(PCIDevice
*dev
)
472 return dev
->cap_present
& QEMU_PCI_CAP_MSIX
;
475 /* Is MSI-X enabled? */
476 int msix_enabled(PCIDevice
*dev
)
478 return (dev
->cap_present
& QEMU_PCI_CAP_MSIX
) &&
479 (dev
->config
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
] &
483 /* Send an MSI-X message */
484 void msix_notify(PCIDevice
*dev
, unsigned vector
)
488 if (vector
>= dev
->msix_entries_nr
|| !dev
->msix_entry_used
[vector
]) {
492 if (msix_is_masked(dev
, vector
)) {
493 msix_set_pending(dev
, vector
);
497 msg
= msix_get_message(dev
, vector
);
499 msi_send_message(dev
, msg
);
502 void msix_reset(PCIDevice
*dev
)
504 if (!msix_present(dev
)) {
507 msix_clear_all_vectors(dev
);
508 dev
->config
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
] &=
509 ~dev
->wmask
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
];
510 memset(dev
->msix_table
, 0, dev
->msix_entries_nr
* PCI_MSIX_ENTRY_SIZE
);
511 memset(dev
->msix_pba
, 0, QEMU_ALIGN_UP(dev
->msix_entries_nr
, 64) / 8);
512 msix_mask_all(dev
, dev
->msix_entries_nr
);
515 /* PCI spec suggests that devices make it possible for software to configure
516 * less vectors than supported by the device, but does not specify a standard
517 * mechanism for devices to do so.
519 * We support this by asking devices to declare vectors software is going to
520 * actually use, and checking this on the notification path. Devices that
521 * don't want to follow the spec suggestion can declare all vectors as used. */
523 /* Mark vector as used. */
524 int msix_vector_use(PCIDevice
*dev
, unsigned vector
)
526 if (vector
>= dev
->msix_entries_nr
) {
530 dev
->msix_entry_used
[vector
]++;
534 /* Mark vector as unused. */
535 void msix_vector_unuse(PCIDevice
*dev
, unsigned vector
)
537 if (vector
>= dev
->msix_entries_nr
|| !dev
->msix_entry_used
[vector
]) {
540 if (--dev
->msix_entry_used
[vector
]) {
543 msix_clr_pending(dev
, vector
);
546 void msix_unuse_all_vectors(PCIDevice
*dev
)
548 if (!msix_present(dev
)) {
551 msix_free_irq_entries(dev
);
554 unsigned int msix_nr_vectors_allocated(const PCIDevice
*dev
)
556 return dev
->msix_entries_nr
;
559 static int msix_set_notifier_for_vector(PCIDevice
*dev
, unsigned int vector
)
563 if (msix_is_masked(dev
, vector
)) {
566 msg
= msix_get_message(dev
, vector
);
567 return dev
->msix_vector_use_notifier(dev
, vector
, msg
);
570 static void msix_unset_notifier_for_vector(PCIDevice
*dev
, unsigned int vector
)
572 if (msix_is_masked(dev
, vector
)) {
575 dev
->msix_vector_release_notifier(dev
, vector
);
578 int msix_set_vector_notifiers(PCIDevice
*dev
,
579 MSIVectorUseNotifier use_notifier
,
580 MSIVectorReleaseNotifier release_notifier
,
581 MSIVectorPollNotifier poll_notifier
)
585 assert(use_notifier
&& release_notifier
);
587 dev
->msix_vector_use_notifier
= use_notifier
;
588 dev
->msix_vector_release_notifier
= release_notifier
;
589 dev
->msix_vector_poll_notifier
= poll_notifier
;
591 if ((dev
->config
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
] &
592 (MSIX_ENABLE_MASK
| MSIX_MASKALL_MASK
)) == MSIX_ENABLE_MASK
) {
593 for (vector
= 0; vector
< dev
->msix_entries_nr
; vector
++) {
594 ret
= msix_set_notifier_for_vector(dev
, vector
);
600 if (dev
->msix_vector_poll_notifier
) {
601 dev
->msix_vector_poll_notifier(dev
, 0, dev
->msix_entries_nr
);
606 while (--vector
>= 0) {
607 msix_unset_notifier_for_vector(dev
, vector
);
609 dev
->msix_vector_use_notifier
= NULL
;
610 dev
->msix_vector_release_notifier
= NULL
;
614 void msix_unset_vector_notifiers(PCIDevice
*dev
)
618 assert(dev
->msix_vector_use_notifier
&&
619 dev
->msix_vector_release_notifier
);
621 if ((dev
->config
[dev
->msix_cap
+ MSIX_CONTROL_OFFSET
] &
622 (MSIX_ENABLE_MASK
| MSIX_MASKALL_MASK
)) == MSIX_ENABLE_MASK
) {
623 for (vector
= 0; vector
< dev
->msix_entries_nr
; vector
++) {
624 msix_unset_notifier_for_vector(dev
, vector
);
627 dev
->msix_vector_use_notifier
= NULL
;
628 dev
->msix_vector_release_notifier
= NULL
;
629 dev
->msix_vector_poll_notifier
= NULL
;
632 static int put_msix_state(QEMUFile
*f
, void *pv
, size_t size
,
633 const VMStateField
*field
, QJSON
*vmdesc
)
640 static int get_msix_state(QEMUFile
*f
, void *pv
, size_t size
,
641 const VMStateField
*field
)
647 static VMStateInfo vmstate_info_msix
= {
648 .name
= "msix state",
649 .get
= get_msix_state
,
650 .put
= put_msix_state
,
653 const VMStateDescription vmstate_msix
= {
655 .fields
= (VMStateField
[]) {
659 .field_exists
= NULL
,
660 .size
= 0, /* ouch */
661 .info
= &vmstate_info_msix
,
665 VMSTATE_END_OF_LIST()