2 * Inter-VM Shared Memory PCI device.
5 * Cam Macdonell <cam@cs.ualberta.ca>
7 * Based On: cirrus_vga.c
8 * Copyright (c) 2004 Fabrice Bellard
9 * Copyright (c) 2004 Makoto Suzuki (suzu)
12 * Copyright (c) 2006 Igor Kovalenko
14 * This code is licensed under the GNU GPL v2.
16 * Contributions after 2012-01-13 are licensed under the terms of the
17 * GNU GPL, version 2 or (at your option) any later version.
19 #include "qemu/osdep.h"
20 #include "qemu/units.h"
21 #include "qapi/error.h"
22 #include "qemu/cutils.h"
24 #include "hw/pci/pci.h"
25 #include "hw/pci/msi.h"
26 #include "hw/pci/msix.h"
27 #include "sysemu/kvm.h"
28 #include "migration/blocker.h"
29 #include "qemu/error-report.h"
30 #include "qemu/event_notifier.h"
31 #include "qom/object_interfaces.h"
32 #include "chardev/char-fe.h"
33 #include "sysemu/hostmem.h"
34 #include "sysemu/qtest.h"
35 #include "qapi/visitor.h"
37 #include "hw/misc/ivshmem.h"
39 #define PCI_VENDOR_ID_IVSHMEM PCI_VENDOR_ID_REDHAT_QUMRANET
40 #define PCI_DEVICE_ID_IVSHMEM 0x1110
42 #define IVSHMEM_MAX_PEERS UINT16_MAX
43 #define IVSHMEM_IOEVENTFD 0
46 #define IVSHMEM_REG_BAR_SIZE 0x100
48 #define IVSHMEM_DEBUG 0
49 #define IVSHMEM_DPRINTF(fmt, ...) \
51 if (IVSHMEM_DEBUG) { \
52 printf("IVSHMEM: " fmt, ## __VA_ARGS__); \
56 #define TYPE_IVSHMEM_COMMON "ivshmem-common"
57 #define IVSHMEM_COMMON(obj) \
58 OBJECT_CHECK(IVShmemState, (obj), TYPE_IVSHMEM_COMMON)
60 #define TYPE_IVSHMEM_PLAIN "ivshmem-plain"
61 #define IVSHMEM_PLAIN(obj) \
62 OBJECT_CHECK(IVShmemState, (obj), TYPE_IVSHMEM_PLAIN)
64 #define TYPE_IVSHMEM_DOORBELL "ivshmem-doorbell"
65 #define IVSHMEM_DOORBELL(obj) \
66 OBJECT_CHECK(IVShmemState, (obj), TYPE_IVSHMEM_DOORBELL)
68 #define TYPE_IVSHMEM "ivshmem"
69 #define IVSHMEM(obj) \
70 OBJECT_CHECK(IVShmemState, (obj), TYPE_IVSHMEM)
74 EventNotifier
*eventfds
;
77 typedef struct MSIVector
{
83 typedef struct IVShmemState
{
90 /* exactly one of these two may be set */
91 HostMemoryBackend
*hostmem
; /* with interrupts */
92 CharBackend server_chr
; /* without interrupts */
100 MemoryRegion ivshmem_mmio
; /* BAR 0 (registers) */
101 MemoryRegion
*ivshmem_bar2
; /* BAR 2 (shared memory) */
102 MemoryRegion server_bar2
; /* used with server_chr */
104 /* interrupt support */
106 int nb_peers
; /* space in @peers[] */
108 MSIVector
*msi_vectors
;
109 uint64_t msg_buf
; /* buffer for receiving server messages */
110 int msg_buffered_bytes
; /* #bytes in @msg_buf */
112 /* migration stuff */
114 Error
*migration_blocker
;
121 uint32_t not_legacy_32bit
;
124 /* registers for the Inter-VM shared memory device */
125 enum ivshmem_registers
{
132 static inline uint32_t ivshmem_has_feature(IVShmemState
*ivs
,
133 unsigned int feature
) {
134 return (ivs
->features
& (1 << feature
));
137 static inline bool ivshmem_is_master(IVShmemState
*s
)
139 assert(s
->master
!= ON_OFF_AUTO_AUTO
);
140 return s
->master
== ON_OFF_AUTO_ON
;
143 static void ivshmem_update_irq(IVShmemState
*s
)
145 PCIDevice
*d
= PCI_DEVICE(s
);
146 uint32_t isr
= s
->intrstatus
& s
->intrmask
;
149 * Do nothing unless the device actually uses INTx. Here's how
150 * the device variants signal interrupts, what they put in PCI
152 * Device variant Interrupt Interrupt Pin MSI-X cap.
153 * ivshmem-plain none 0 no
154 * ivshmem-doorbell MSI-X 1 yes(1)
155 * ivshmem,msi=off INTx 1 no
156 * ivshmem,msi=on MSI-X 1(2) yes(1)
157 * (1) if guest enabled MSI-X
158 * (2) the device lies
159 * Leads to the condition for doing nothing:
161 if (ivshmem_has_feature(s
, IVSHMEM_MSI
)
162 || !d
->config
[PCI_INTERRUPT_PIN
]) {
166 /* don't print ISR resets */
168 IVSHMEM_DPRINTF("Set IRQ to %d (%04x %04x)\n",
169 isr
? 1 : 0, s
->intrstatus
, s
->intrmask
);
172 pci_set_irq(d
, isr
!= 0);
175 static void ivshmem_IntrMask_write(IVShmemState
*s
, uint32_t val
)
177 IVSHMEM_DPRINTF("IntrMask write(w) val = 0x%04x\n", val
);
180 ivshmem_update_irq(s
);
183 static uint32_t ivshmem_IntrMask_read(IVShmemState
*s
)
185 uint32_t ret
= s
->intrmask
;
187 IVSHMEM_DPRINTF("intrmask read(w) val = 0x%04x\n", ret
);
191 static void ivshmem_IntrStatus_write(IVShmemState
*s
, uint32_t val
)
193 IVSHMEM_DPRINTF("IntrStatus write(w) val = 0x%04x\n", val
);
196 ivshmem_update_irq(s
);
199 static uint32_t ivshmem_IntrStatus_read(IVShmemState
*s
)
201 uint32_t ret
= s
->intrstatus
;
203 /* reading ISR clears all interrupts */
205 ivshmem_update_irq(s
);
209 static void ivshmem_io_write(void *opaque
, hwaddr addr
,
210 uint64_t val
, unsigned size
)
212 IVShmemState
*s
= opaque
;
214 uint16_t dest
= val
>> 16;
215 uint16_t vector
= val
& 0xff;
219 IVSHMEM_DPRINTF("writing to addr " TARGET_FMT_plx
"\n", addr
);
223 ivshmem_IntrMask_write(s
, val
);
227 ivshmem_IntrStatus_write(s
, val
);
231 /* check that dest VM ID is reasonable */
232 if (dest
>= s
->nb_peers
) {
233 IVSHMEM_DPRINTF("Invalid destination VM ID (%d)\n", dest
);
237 /* check doorbell range */
238 if (vector
< s
->peers
[dest
].nb_eventfds
) {
239 IVSHMEM_DPRINTF("Notifying VM %d on vector %d\n", dest
, vector
);
240 event_notifier_set(&s
->peers
[dest
].eventfds
[vector
]);
242 IVSHMEM_DPRINTF("Invalid destination vector %d on VM %d\n",
247 IVSHMEM_DPRINTF("Unhandled write " TARGET_FMT_plx
"\n", addr
);
251 static uint64_t ivshmem_io_read(void *opaque
, hwaddr addr
,
255 IVShmemState
*s
= opaque
;
261 ret
= ivshmem_IntrMask_read(s
);
265 ret
= ivshmem_IntrStatus_read(s
);
273 IVSHMEM_DPRINTF("why are we reading " TARGET_FMT_plx
"\n", addr
);
280 static const MemoryRegionOps ivshmem_mmio_ops
= {
281 .read
= ivshmem_io_read
,
282 .write
= ivshmem_io_write
,
283 .endianness
= DEVICE_NATIVE_ENDIAN
,
285 .min_access_size
= 4,
286 .max_access_size
= 4,
290 static void ivshmem_vector_notify(void *opaque
)
292 MSIVector
*entry
= opaque
;
293 PCIDevice
*pdev
= entry
->pdev
;
294 IVShmemState
*s
= IVSHMEM_COMMON(pdev
);
295 int vector
= entry
- s
->msi_vectors
;
296 EventNotifier
*n
= &s
->peers
[s
->vm_id
].eventfds
[vector
];
298 if (!event_notifier_test_and_clear(n
)) {
302 IVSHMEM_DPRINTF("interrupt on vector %p %d\n", pdev
, vector
);
303 if (ivshmem_has_feature(s
, IVSHMEM_MSI
)) {
304 if (msix_enabled(pdev
)) {
305 msix_notify(pdev
, vector
);
308 ivshmem_IntrStatus_write(s
, 1);
312 static int ivshmem_vector_unmask(PCIDevice
*dev
, unsigned vector
,
315 IVShmemState
*s
= IVSHMEM_COMMON(dev
);
316 EventNotifier
*n
= &s
->peers
[s
->vm_id
].eventfds
[vector
];
317 MSIVector
*v
= &s
->msi_vectors
[vector
];
320 IVSHMEM_DPRINTF("vector unmask %p %d\n", dev
, vector
);
322 error_report("ivshmem: vector %d route does not exist", vector
);
325 assert(!v
->unmasked
);
327 ret
= kvm_irqchip_update_msi_route(kvm_state
, v
->virq
, msg
, dev
);
331 kvm_irqchip_commit_routes(kvm_state
);
333 ret
= kvm_irqchip_add_irqfd_notifier_gsi(kvm_state
, n
, NULL
, v
->virq
);
342 static void ivshmem_vector_mask(PCIDevice
*dev
, unsigned vector
)
344 IVShmemState
*s
= IVSHMEM_COMMON(dev
);
345 EventNotifier
*n
= &s
->peers
[s
->vm_id
].eventfds
[vector
];
346 MSIVector
*v
= &s
->msi_vectors
[vector
];
349 IVSHMEM_DPRINTF("vector mask %p %d\n", dev
, vector
);
351 error_report("ivshmem: vector %d route does not exist", vector
);
356 ret
= kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state
, n
, v
->virq
);
358 error_report("remove_irqfd_notifier_gsi failed");
364 static void ivshmem_vector_poll(PCIDevice
*dev
,
365 unsigned int vector_start
,
366 unsigned int vector_end
)
368 IVShmemState
*s
= IVSHMEM_COMMON(dev
);
371 IVSHMEM_DPRINTF("vector poll %p %d-%d\n", dev
, vector_start
, vector_end
);
373 vector_end
= MIN(vector_end
, s
->vectors
);
375 for (vector
= vector_start
; vector
< vector_end
; vector
++) {
376 EventNotifier
*notifier
= &s
->peers
[s
->vm_id
].eventfds
[vector
];
378 if (!msix_is_masked(dev
, vector
)) {
382 if (event_notifier_test_and_clear(notifier
)) {
383 msix_set_pending(dev
, vector
);
388 static void watch_vector_notifier(IVShmemState
*s
, EventNotifier
*n
,
391 int eventfd
= event_notifier_get_fd(n
);
393 assert(!s
->msi_vectors
[vector
].pdev
);
394 s
->msi_vectors
[vector
].pdev
= PCI_DEVICE(s
);
396 qemu_set_fd_handler(eventfd
, ivshmem_vector_notify
,
397 NULL
, &s
->msi_vectors
[vector
]);
400 static void ivshmem_add_eventfd(IVShmemState
*s
, int posn
, int i
)
402 memory_region_add_eventfd(&s
->ivshmem_mmio
,
407 &s
->peers
[posn
].eventfds
[i
]);
410 static void ivshmem_del_eventfd(IVShmemState
*s
, int posn
, int i
)
412 memory_region_del_eventfd(&s
->ivshmem_mmio
,
417 &s
->peers
[posn
].eventfds
[i
]);
420 static void close_peer_eventfds(IVShmemState
*s
, int posn
)
424 assert(posn
>= 0 && posn
< s
->nb_peers
);
425 n
= s
->peers
[posn
].nb_eventfds
;
427 if (ivshmem_has_feature(s
, IVSHMEM_IOEVENTFD
)) {
428 memory_region_transaction_begin();
429 for (i
= 0; i
< n
; i
++) {
430 ivshmem_del_eventfd(s
, posn
, i
);
432 memory_region_transaction_commit();
435 for (i
= 0; i
< n
; i
++) {
436 event_notifier_cleanup(&s
->peers
[posn
].eventfds
[i
]);
439 g_free(s
->peers
[posn
].eventfds
);
440 s
->peers
[posn
].nb_eventfds
= 0;
443 static void resize_peers(IVShmemState
*s
, int nb_peers
)
445 int old_nb_peers
= s
->nb_peers
;
448 assert(nb_peers
> old_nb_peers
);
449 IVSHMEM_DPRINTF("bumping storage to %d peers\n", nb_peers
);
451 s
->peers
= g_realloc(s
->peers
, nb_peers
* sizeof(Peer
));
452 s
->nb_peers
= nb_peers
;
454 for (i
= old_nb_peers
; i
< nb_peers
; i
++) {
455 s
->peers
[i
].eventfds
= g_new0(EventNotifier
, s
->vectors
);
456 s
->peers
[i
].nb_eventfds
= 0;
460 static void ivshmem_add_kvm_msi_virq(IVShmemState
*s
, int vector
,
463 PCIDevice
*pdev
= PCI_DEVICE(s
);
466 IVSHMEM_DPRINTF("ivshmem_add_kvm_msi_virq vector:%d\n", vector
);
467 assert(!s
->msi_vectors
[vector
].pdev
);
469 ret
= kvm_irqchip_add_msi_route(kvm_state
, vector
, pdev
);
471 error_setg(errp
, "kvm_irqchip_add_msi_route failed");
475 s
->msi_vectors
[vector
].virq
= ret
;
476 s
->msi_vectors
[vector
].pdev
= pdev
;
479 static void setup_interrupt(IVShmemState
*s
, int vector
, Error
**errp
)
481 EventNotifier
*n
= &s
->peers
[s
->vm_id
].eventfds
[vector
];
482 bool with_irqfd
= kvm_msi_via_irqfd_enabled() &&
483 ivshmem_has_feature(s
, IVSHMEM_MSI
);
484 PCIDevice
*pdev
= PCI_DEVICE(s
);
487 IVSHMEM_DPRINTF("setting up interrupt for vector: %d\n", vector
);
490 IVSHMEM_DPRINTF("with eventfd\n");
491 watch_vector_notifier(s
, n
, vector
);
492 } else if (msix_enabled(pdev
)) {
493 IVSHMEM_DPRINTF("with irqfd\n");
494 ivshmem_add_kvm_msi_virq(s
, vector
, &err
);
496 error_propagate(errp
, err
);
500 if (!msix_is_masked(pdev
, vector
)) {
501 kvm_irqchip_add_irqfd_notifier_gsi(kvm_state
, n
, NULL
,
502 s
->msi_vectors
[vector
].virq
);
503 /* TODO handle error */
506 /* it will be delayed until msix is enabled, in write_config */
507 IVSHMEM_DPRINTF("with irqfd, delayed until msix enabled\n");
511 static void process_msg_shmem(IVShmemState
*s
, int fd
, Error
**errp
)
513 Error
*local_err
= NULL
;
517 if (s
->ivshmem_bar2
) {
518 error_setg(errp
, "server sent unexpected shared memory message");
523 if (fstat(fd
, &buf
) < 0) {
524 error_setg_errno(errp
, errno
,
525 "can't determine size of shared memory sent by server");
533 if (s
->legacy_size
!= SIZE_MAX
) {
534 if (size
< s
->legacy_size
) {
535 error_setg(errp
, "server sent only %zd bytes of shared memory",
536 (size_t)buf
.st_size
);
540 size
= s
->legacy_size
;
543 /* mmap the region and map into the BAR2 */
544 memory_region_init_ram_from_fd(&s
->server_bar2
, OBJECT(s
),
545 "ivshmem.bar2", size
, true, fd
, &local_err
);
547 error_propagate(errp
, local_err
);
551 s
->ivshmem_bar2
= &s
->server_bar2
;
554 static void process_msg_disconnect(IVShmemState
*s
, uint16_t posn
,
557 IVSHMEM_DPRINTF("posn %d has gone away\n", posn
);
558 if (posn
>= s
->nb_peers
|| posn
== s
->vm_id
) {
559 error_setg(errp
, "invalid peer %d", posn
);
562 close_peer_eventfds(s
, posn
);
565 static void process_msg_connect(IVShmemState
*s
, uint16_t posn
, int fd
,
568 Peer
*peer
= &s
->peers
[posn
];
572 * The N-th connect message for this peer comes with the file
573 * descriptor for vector N-1. Count messages to find the vector.
575 if (peer
->nb_eventfds
>= s
->vectors
) {
576 error_setg(errp
, "Too many eventfd received, device has %d vectors",
581 vector
= peer
->nb_eventfds
++;
583 IVSHMEM_DPRINTF("eventfds[%d][%d] = %d\n", posn
, vector
, fd
);
584 event_notifier_init_fd(&peer
->eventfds
[vector
], fd
);
585 fcntl_setfl(fd
, O_NONBLOCK
); /* msix/irqfd poll non block */
587 if (posn
== s
->vm_id
) {
588 setup_interrupt(s
, vector
, errp
);
589 /* TODO do we need to handle the error? */
592 if (ivshmem_has_feature(s
, IVSHMEM_IOEVENTFD
)) {
593 ivshmem_add_eventfd(s
, posn
, vector
);
597 static void process_msg(IVShmemState
*s
, int64_t msg
, int fd
, Error
**errp
)
599 IVSHMEM_DPRINTF("posn is %" PRId64
", fd is %d\n", msg
, fd
);
601 if (msg
< -1 || msg
> IVSHMEM_MAX_PEERS
) {
602 error_setg(errp
, "server sent invalid message %" PRId64
, msg
);
608 process_msg_shmem(s
, fd
, errp
);
612 if (msg
>= s
->nb_peers
) {
613 resize_peers(s
, msg
+ 1);
617 process_msg_connect(s
, msg
, fd
, errp
);
619 process_msg_disconnect(s
, msg
, errp
);
623 static int ivshmem_can_receive(void *opaque
)
625 IVShmemState
*s
= opaque
;
627 assert(s
->msg_buffered_bytes
< sizeof(s
->msg_buf
));
628 return sizeof(s
->msg_buf
) - s
->msg_buffered_bytes
;
631 static void ivshmem_read(void *opaque
, const uint8_t *buf
, int size
)
633 IVShmemState
*s
= opaque
;
638 assert(size
>= 0 && s
->msg_buffered_bytes
+ size
<= sizeof(s
->msg_buf
));
639 memcpy((unsigned char *)&s
->msg_buf
+ s
->msg_buffered_bytes
, buf
, size
);
640 s
->msg_buffered_bytes
+= size
;
641 if (s
->msg_buffered_bytes
< sizeof(s
->msg_buf
)) {
644 msg
= le64_to_cpu(s
->msg_buf
);
645 s
->msg_buffered_bytes
= 0;
647 fd
= qemu_chr_fe_get_msgfd(&s
->server_chr
);
649 process_msg(s
, msg
, fd
, &err
);
651 error_report_err(err
);
655 static int64_t ivshmem_recv_msg(IVShmemState
*s
, int *pfd
, Error
**errp
)
662 ret
= qemu_chr_fe_read_all(&s
->server_chr
, (uint8_t *)&msg
+ n
,
668 error_setg_errno(errp
, -ret
, "read from server failed");
672 } while (n
< sizeof(msg
));
674 *pfd
= qemu_chr_fe_get_msgfd(&s
->server_chr
);
675 return le64_to_cpu(msg
);
678 static void ivshmem_recv_setup(IVShmemState
*s
, Error
**errp
)
684 msg
= ivshmem_recv_msg(s
, &fd
, &err
);
686 error_propagate(errp
, err
);
689 if (msg
!= IVSHMEM_PROTOCOL_VERSION
) {
690 error_setg(errp
, "server sent version %" PRId64
", expecting %d",
691 msg
, IVSHMEM_PROTOCOL_VERSION
);
695 error_setg(errp
, "server sent invalid version message");
700 * ivshmem-server sends the remaining initial messages in a fixed
701 * order, but the device has always accepted them in any order.
702 * Stay as compatible as practical, just in case people use
703 * servers that behave differently.
707 * ivshmem_device_spec.txt has always required the ID message
708 * right here, and ivshmem-server has always complied. However,
709 * older versions of the device accepted it out of order, but
710 * broke when an interrupt setup message arrived before it.
712 msg
= ivshmem_recv_msg(s
, &fd
, &err
);
714 error_propagate(errp
, err
);
717 if (fd
!= -1 || msg
< 0 || msg
> IVSHMEM_MAX_PEERS
) {
718 error_setg(errp
, "server sent invalid ID message");
724 * Receive more messages until we got shared memory.
727 msg
= ivshmem_recv_msg(s
, &fd
, &err
);
729 error_propagate(errp
, err
);
732 process_msg(s
, msg
, fd
, &err
);
734 error_propagate(errp
, err
);
740 * This function must either map the shared memory or fail. The
741 * loop above ensures that: it terminates normally only after it
742 * successfully processed the server's shared memory message.
743 * Assert that actually mapped the shared memory:
745 assert(s
->ivshmem_bar2
);
748 /* Select the MSI-X vectors used by device.
749 * ivshmem maps events to vectors statically, so
750 * we just enable all vectors on init and after reset. */
751 static void ivshmem_msix_vector_use(IVShmemState
*s
)
753 PCIDevice
*d
= PCI_DEVICE(s
);
756 for (i
= 0; i
< s
->vectors
; i
++) {
757 msix_vector_use(d
, i
);
761 static void ivshmem_disable_irqfd(IVShmemState
*s
);
763 static void ivshmem_reset(DeviceState
*d
)
765 IVShmemState
*s
= IVSHMEM_COMMON(d
);
767 ivshmem_disable_irqfd(s
);
771 if (ivshmem_has_feature(s
, IVSHMEM_MSI
)) {
772 ivshmem_msix_vector_use(s
);
776 static int ivshmem_setup_interrupts(IVShmemState
*s
, Error
**errp
)
778 /* allocate QEMU callback data for receiving interrupts */
779 s
->msi_vectors
= g_malloc0(s
->vectors
* sizeof(MSIVector
));
781 if (ivshmem_has_feature(s
, IVSHMEM_MSI
)) {
782 if (msix_init_exclusive_bar(PCI_DEVICE(s
), s
->vectors
, 1, errp
)) {
786 IVSHMEM_DPRINTF("msix initialized (%d vectors)\n", s
->vectors
);
787 ivshmem_msix_vector_use(s
);
793 static void ivshmem_remove_kvm_msi_virq(IVShmemState
*s
, int vector
)
795 IVSHMEM_DPRINTF("ivshmem_remove_kvm_msi_virq vector:%d\n", vector
);
797 if (s
->msi_vectors
[vector
].pdev
== NULL
) {
801 /* it was cleaned when masked in the frontend. */
802 kvm_irqchip_release_virq(kvm_state
, s
->msi_vectors
[vector
].virq
);
804 s
->msi_vectors
[vector
].pdev
= NULL
;
807 static void ivshmem_enable_irqfd(IVShmemState
*s
)
809 PCIDevice
*pdev
= PCI_DEVICE(s
);
812 for (i
= 0; i
< s
->peers
[s
->vm_id
].nb_eventfds
; i
++) {
815 ivshmem_add_kvm_msi_virq(s
, i
, &err
);
817 error_report_err(err
);
822 if (msix_set_vector_notifiers(pdev
,
823 ivshmem_vector_unmask
,
825 ivshmem_vector_poll
)) {
826 error_report("ivshmem: msix_set_vector_notifiers failed");
833 ivshmem_remove_kvm_msi_virq(s
, i
);
837 static void ivshmem_disable_irqfd(IVShmemState
*s
)
839 PCIDevice
*pdev
= PCI_DEVICE(s
);
842 if (!pdev
->msix_vector_use_notifier
) {
846 msix_unset_vector_notifiers(pdev
);
848 for (i
= 0; i
< s
->peers
[s
->vm_id
].nb_eventfds
; i
++) {
850 * MSI-X is already disabled here so msix_unset_vector_notifiers()
851 * didn't call our release notifier. Do it now to keep our masks and
854 if (s
->msi_vectors
[i
].unmasked
) {
855 ivshmem_vector_mask(pdev
, i
);
857 ivshmem_remove_kvm_msi_virq(s
, i
);
862 static void ivshmem_write_config(PCIDevice
*pdev
, uint32_t address
,
863 uint32_t val
, int len
)
865 IVShmemState
*s
= IVSHMEM_COMMON(pdev
);
866 int is_enabled
, was_enabled
= msix_enabled(pdev
);
868 pci_default_write_config(pdev
, address
, val
, len
);
869 is_enabled
= msix_enabled(pdev
);
871 if (kvm_msi_via_irqfd_enabled()) {
872 if (!was_enabled
&& is_enabled
) {
873 ivshmem_enable_irqfd(s
);
874 } else if (was_enabled
&& !is_enabled
) {
875 ivshmem_disable_irqfd(s
);
880 static void ivshmem_common_realize(PCIDevice
*dev
, Error
**errp
)
882 IVShmemState
*s
= IVSHMEM_COMMON(dev
);
885 uint8_t attr
= PCI_BASE_ADDRESS_SPACE_MEMORY
|
886 PCI_BASE_ADDRESS_MEM_PREFETCH
;
887 Error
*local_err
= NULL
;
889 /* IRQFD requires MSI */
890 if (ivshmem_has_feature(s
, IVSHMEM_IOEVENTFD
) &&
891 !ivshmem_has_feature(s
, IVSHMEM_MSI
)) {
892 error_setg(errp
, "ioeventfd/irqfd requires MSI");
896 pci_conf
= dev
->config
;
897 pci_conf
[PCI_COMMAND
] = PCI_COMMAND_IO
| PCI_COMMAND_MEMORY
;
899 memory_region_init_io(&s
->ivshmem_mmio
, OBJECT(s
), &ivshmem_mmio_ops
, s
,
900 "ivshmem-mmio", IVSHMEM_REG_BAR_SIZE
);
902 /* region for registers*/
903 pci_register_bar(dev
, 0, PCI_BASE_ADDRESS_SPACE_MEMORY
,
906 if (s
->not_legacy_32bit
) {
907 attr
|= PCI_BASE_ADDRESS_MEM_TYPE_64
;
910 if (s
->hostmem
!= NULL
) {
911 IVSHMEM_DPRINTF("using hostmem\n");
913 s
->ivshmem_bar2
= host_memory_backend_get_memory(s
->hostmem
);
914 host_memory_backend_set_mapped(s
->hostmem
, true);
916 Chardev
*chr
= qemu_chr_fe_get_driver(&s
->server_chr
);
919 IVSHMEM_DPRINTF("using shared memory server (socket = %s)\n",
922 /* we allocate enough space for 16 peers and grow as needed */
926 * Receive setup messages from server synchronously.
927 * Older versions did it asynchronously, but that creates a
928 * number of entertaining race conditions.
930 ivshmem_recv_setup(s
, &err
);
932 error_propagate(errp
, err
);
936 if (s
->master
== ON_OFF_AUTO_ON
&& s
->vm_id
!= 0) {
938 "master must connect to the server before any peers");
942 qemu_chr_fe_set_handlers(&s
->server_chr
, ivshmem_can_receive
,
943 ivshmem_read
, NULL
, NULL
, s
, NULL
, true);
945 if (ivshmem_setup_interrupts(s
, errp
) < 0) {
946 error_prepend(errp
, "Failed to initialize interrupts: ");
951 if (s
->master
== ON_OFF_AUTO_AUTO
) {
952 s
->master
= s
->vm_id
== 0 ? ON_OFF_AUTO_ON
: ON_OFF_AUTO_OFF
;
955 if (!ivshmem_is_master(s
)) {
956 error_setg(&s
->migration_blocker
,
957 "Migration is disabled when using feature 'peer mode' in device 'ivshmem'");
958 migrate_add_blocker(s
->migration_blocker
, &local_err
);
960 error_propagate(errp
, local_err
);
961 error_free(s
->migration_blocker
);
966 vmstate_register_ram(s
->ivshmem_bar2
, DEVICE(s
));
967 pci_register_bar(PCI_DEVICE(s
), 2, attr
, s
->ivshmem_bar2
);
970 static void ivshmem_exit(PCIDevice
*dev
)
972 IVShmemState
*s
= IVSHMEM_COMMON(dev
);
975 if (s
->migration_blocker
) {
976 migrate_del_blocker(s
->migration_blocker
);
977 error_free(s
->migration_blocker
);
980 if (memory_region_is_mapped(s
->ivshmem_bar2
)) {
982 void *addr
= memory_region_get_ram_ptr(s
->ivshmem_bar2
);
985 if (munmap(addr
, memory_region_size(s
->ivshmem_bar2
) == -1)) {
986 error_report("Failed to munmap shared memory %s",
990 fd
= memory_region_get_fd(s
->ivshmem_bar2
);
994 vmstate_unregister_ram(s
->ivshmem_bar2
, DEVICE(dev
));
998 host_memory_backend_set_mapped(s
->hostmem
, false);
1002 for (i
= 0; i
< s
->nb_peers
; i
++) {
1003 close_peer_eventfds(s
, i
);
1008 if (ivshmem_has_feature(s
, IVSHMEM_MSI
)) {
1009 msix_uninit_exclusive_bar(dev
);
1012 g_free(s
->msi_vectors
);
1015 static int ivshmem_pre_load(void *opaque
)
1017 IVShmemState
*s
= opaque
;
1019 if (!ivshmem_is_master(s
)) {
1020 error_report("'peer' devices are not migratable");
1027 static int ivshmem_post_load(void *opaque
, int version_id
)
1029 IVShmemState
*s
= opaque
;
1031 if (ivshmem_has_feature(s
, IVSHMEM_MSI
)) {
1032 ivshmem_msix_vector_use(s
);
1037 static void ivshmem_common_class_init(ObjectClass
*klass
, void *data
)
1039 DeviceClass
*dc
= DEVICE_CLASS(klass
);
1040 PCIDeviceClass
*k
= PCI_DEVICE_CLASS(klass
);
1042 k
->realize
= ivshmem_common_realize
;
1043 k
->exit
= ivshmem_exit
;
1044 k
->config_write
= ivshmem_write_config
;
1045 k
->vendor_id
= PCI_VENDOR_ID_IVSHMEM
;
1046 k
->device_id
= PCI_DEVICE_ID_IVSHMEM
;
1047 k
->class_id
= PCI_CLASS_MEMORY_RAM
;
1049 dc
->reset
= ivshmem_reset
;
1050 set_bit(DEVICE_CATEGORY_MISC
, dc
->categories
);
1051 dc
->desc
= "Inter-VM shared memory";
1054 static const TypeInfo ivshmem_common_info
= {
1055 .name
= TYPE_IVSHMEM_COMMON
,
1056 .parent
= TYPE_PCI_DEVICE
,
1057 .instance_size
= sizeof(IVShmemState
),
1059 .class_init
= ivshmem_common_class_init
,
1060 .interfaces
= (InterfaceInfo
[]) {
1061 { INTERFACE_CONVENTIONAL_PCI_DEVICE
},
1066 static const VMStateDescription ivshmem_plain_vmsd
= {
1067 .name
= TYPE_IVSHMEM_PLAIN
,
1069 .minimum_version_id
= 0,
1070 .pre_load
= ivshmem_pre_load
,
1071 .post_load
= ivshmem_post_load
,
1072 .fields
= (VMStateField
[]) {
1073 VMSTATE_PCI_DEVICE(parent_obj
, IVShmemState
),
1074 VMSTATE_UINT32(intrstatus
, IVShmemState
),
1075 VMSTATE_UINT32(intrmask
, IVShmemState
),
1076 VMSTATE_END_OF_LIST()
1080 static Property ivshmem_plain_properties
[] = {
1081 DEFINE_PROP_ON_OFF_AUTO("master", IVShmemState
, master
, ON_OFF_AUTO_OFF
),
1082 DEFINE_PROP_LINK("memdev", IVShmemState
, hostmem
, TYPE_MEMORY_BACKEND
,
1083 HostMemoryBackend
*),
1084 DEFINE_PROP_END_OF_LIST(),
1087 static void ivshmem_plain_init(Object
*obj
)
1089 IVShmemState
*s
= IVSHMEM_PLAIN(obj
);
1091 s
->not_legacy_32bit
= 1;
1094 static void ivshmem_plain_realize(PCIDevice
*dev
, Error
**errp
)
1096 IVShmemState
*s
= IVSHMEM_COMMON(dev
);
1099 error_setg(errp
, "You must specify a 'memdev'");
1101 } else if (host_memory_backend_is_mapped(s
->hostmem
)) {
1102 char *path
= object_get_canonical_path_component(OBJECT(s
->hostmem
));
1103 error_setg(errp
, "can't use already busy memdev: %s", path
);
1108 ivshmem_common_realize(dev
, errp
);
1111 static void ivshmem_plain_class_init(ObjectClass
*klass
, void *data
)
1113 DeviceClass
*dc
= DEVICE_CLASS(klass
);
1114 PCIDeviceClass
*k
= PCI_DEVICE_CLASS(klass
);
1116 k
->realize
= ivshmem_plain_realize
;
1117 dc
->props
= ivshmem_plain_properties
;
1118 dc
->vmsd
= &ivshmem_plain_vmsd
;
1121 static const TypeInfo ivshmem_plain_info
= {
1122 .name
= TYPE_IVSHMEM_PLAIN
,
1123 .parent
= TYPE_IVSHMEM_COMMON
,
1124 .instance_size
= sizeof(IVShmemState
),
1125 .instance_init
= ivshmem_plain_init
,
1126 .class_init
= ivshmem_plain_class_init
,
1129 static const VMStateDescription ivshmem_doorbell_vmsd
= {
1130 .name
= TYPE_IVSHMEM_DOORBELL
,
1132 .minimum_version_id
= 0,
1133 .pre_load
= ivshmem_pre_load
,
1134 .post_load
= ivshmem_post_load
,
1135 .fields
= (VMStateField
[]) {
1136 VMSTATE_PCI_DEVICE(parent_obj
, IVShmemState
),
1137 VMSTATE_MSIX(parent_obj
, IVShmemState
),
1138 VMSTATE_UINT32(intrstatus
, IVShmemState
),
1139 VMSTATE_UINT32(intrmask
, IVShmemState
),
1140 VMSTATE_END_OF_LIST()
1144 static Property ivshmem_doorbell_properties
[] = {
1145 DEFINE_PROP_CHR("chardev", IVShmemState
, server_chr
),
1146 DEFINE_PROP_UINT32("vectors", IVShmemState
, vectors
, 1),
1147 DEFINE_PROP_BIT("ioeventfd", IVShmemState
, features
, IVSHMEM_IOEVENTFD
,
1149 DEFINE_PROP_ON_OFF_AUTO("master", IVShmemState
, master
, ON_OFF_AUTO_OFF
),
1150 DEFINE_PROP_END_OF_LIST(),
1153 static void ivshmem_doorbell_init(Object
*obj
)
1155 IVShmemState
*s
= IVSHMEM_DOORBELL(obj
);
1157 s
->features
|= (1 << IVSHMEM_MSI
);
1158 s
->legacy_size
= SIZE_MAX
; /* whatever the server sends */
1159 s
->not_legacy_32bit
= 1;
1162 static void ivshmem_doorbell_realize(PCIDevice
*dev
, Error
**errp
)
1164 IVShmemState
*s
= IVSHMEM_COMMON(dev
);
1166 if (!qemu_chr_fe_backend_connected(&s
->server_chr
)) {
1167 error_setg(errp
, "You must specify a 'chardev'");
1171 ivshmem_common_realize(dev
, errp
);
1174 static void ivshmem_doorbell_class_init(ObjectClass
*klass
, void *data
)
1176 DeviceClass
*dc
= DEVICE_CLASS(klass
);
1177 PCIDeviceClass
*k
= PCI_DEVICE_CLASS(klass
);
1179 k
->realize
= ivshmem_doorbell_realize
;
1180 dc
->props
= ivshmem_doorbell_properties
;
1181 dc
->vmsd
= &ivshmem_doorbell_vmsd
;
1184 static const TypeInfo ivshmem_doorbell_info
= {
1185 .name
= TYPE_IVSHMEM_DOORBELL
,
1186 .parent
= TYPE_IVSHMEM_COMMON
,
1187 .instance_size
= sizeof(IVShmemState
),
1188 .instance_init
= ivshmem_doorbell_init
,
1189 .class_init
= ivshmem_doorbell_class_init
,
1192 static int ivshmem_load_old(QEMUFile
*f
, void *opaque
, int version_id
)
1194 IVShmemState
*s
= opaque
;
1195 PCIDevice
*pdev
= PCI_DEVICE(s
);
1198 IVSHMEM_DPRINTF("ivshmem_load_old\n");
1200 if (version_id
!= 0) {
1204 ret
= ivshmem_pre_load(s
);
1209 ret
= pci_device_load(pdev
, f
);
1214 if (ivshmem_has_feature(s
, IVSHMEM_MSI
)) {
1216 ivshmem_msix_vector_use(s
);
1218 s
->intrstatus
= qemu_get_be32(f
);
1219 s
->intrmask
= qemu_get_be32(f
);
1225 static bool test_msix(void *opaque
, int version_id
)
1227 IVShmemState
*s
= opaque
;
1229 return ivshmem_has_feature(s
, IVSHMEM_MSI
);
1232 static bool test_no_msix(void *opaque
, int version_id
)
1234 return !test_msix(opaque
, version_id
);
1237 static const VMStateDescription ivshmem_vmsd
= {
1240 .minimum_version_id
= 1,
1241 .pre_load
= ivshmem_pre_load
,
1242 .post_load
= ivshmem_post_load
,
1243 .fields
= (VMStateField
[]) {
1244 VMSTATE_PCI_DEVICE(parent_obj
, IVShmemState
),
1246 VMSTATE_MSIX_TEST(parent_obj
, IVShmemState
, test_msix
),
1247 VMSTATE_UINT32_TEST(intrstatus
, IVShmemState
, test_no_msix
),
1248 VMSTATE_UINT32_TEST(intrmask
, IVShmemState
, test_no_msix
),
1250 VMSTATE_END_OF_LIST()
1252 .load_state_old
= ivshmem_load_old
,
1253 .minimum_version_id_old
= 0
1256 static Property ivshmem_properties
[] = {
1257 DEFINE_PROP_CHR("chardev", IVShmemState
, server_chr
),
1258 DEFINE_PROP_STRING("size", IVShmemState
, sizearg
),
1259 DEFINE_PROP_UINT32("vectors", IVShmemState
, vectors
, 1),
1260 DEFINE_PROP_BIT("ioeventfd", IVShmemState
, features
, IVSHMEM_IOEVENTFD
,
1262 DEFINE_PROP_BIT("msi", IVShmemState
, features
, IVSHMEM_MSI
, true),
1263 DEFINE_PROP_STRING("shm", IVShmemState
, shmobj
),
1264 DEFINE_PROP_STRING("role", IVShmemState
, role
),
1265 DEFINE_PROP_UINT32("use64", IVShmemState
, not_legacy_32bit
, 1),
1266 DEFINE_PROP_END_OF_LIST(),
1269 static void desugar_shm(IVShmemState
*s
)
1274 obj
= object_new("memory-backend-file");
1275 path
= g_strdup_printf("/dev/shm/%s", s
->shmobj
);
1276 object_property_set_str(obj
, path
, "mem-path", &error_abort
);
1278 object_property_set_int(obj
, s
->legacy_size
, "size", &error_abort
);
1279 object_property_set_bool(obj
, true, "share", &error_abort
);
1280 object_property_add_child(OBJECT(s
), "internal-shm-backend", obj
,
1283 user_creatable_complete(USER_CREATABLE(obj
), &error_abort
);
1284 s
->hostmem
= MEMORY_BACKEND(obj
);
1287 static void ivshmem_realize(PCIDevice
*dev
, Error
**errp
)
1289 IVShmemState
*s
= IVSHMEM_COMMON(dev
);
1291 if (!qtest_enabled()) {
1292 warn_report("ivshmem is deprecated, please use ivshmem-plain"
1293 " or ivshmem-doorbell instead");
1296 if (qemu_chr_fe_backend_connected(&s
->server_chr
) + !!s
->shmobj
!= 1) {
1297 error_setg(errp
, "You must specify either 'shm' or 'chardev'");
1301 if (s
->sizearg
== NULL
) {
1302 s
->legacy_size
= 4 * MiB
; /* 4 MB default */
1307 ret
= qemu_strtosz_MiB(s
->sizearg
, NULL
, &size
);
1308 if (ret
< 0 || (size_t)size
!= size
|| !is_power_of_2(size
)) {
1309 error_setg(errp
, "Invalid size %s", s
->sizearg
);
1312 s
->legacy_size
= size
;
1315 /* check that role is reasonable */
1317 if (strncmp(s
->role
, "peer", 5) == 0) {
1318 s
->master
= ON_OFF_AUTO_OFF
;
1319 } else if (strncmp(s
->role
, "master", 7) == 0) {
1320 s
->master
= ON_OFF_AUTO_ON
;
1322 error_setg(errp
, "'role' must be 'peer' or 'master'");
1326 s
->master
= ON_OFF_AUTO_AUTO
;
1334 * Note: we don't use INTx with IVSHMEM_MSI at all, so this is a
1335 * bald-faced lie then. But it's a backwards compatible lie.
1337 pci_config_set_interrupt_pin(dev
->config
, 1);
1339 ivshmem_common_realize(dev
, errp
);
1342 static void ivshmem_class_init(ObjectClass
*klass
, void *data
)
1344 DeviceClass
*dc
= DEVICE_CLASS(klass
);
1345 PCIDeviceClass
*k
= PCI_DEVICE_CLASS(klass
);
1347 k
->realize
= ivshmem_realize
;
1349 dc
->desc
= "Inter-VM shared memory (legacy)";
1350 dc
->props
= ivshmem_properties
;
1351 dc
->vmsd
= &ivshmem_vmsd
;
1354 static const TypeInfo ivshmem_info
= {
1355 .name
= TYPE_IVSHMEM
,
1356 .parent
= TYPE_IVSHMEM_COMMON
,
1357 .instance_size
= sizeof(IVShmemState
),
1358 .class_init
= ivshmem_class_init
,
1361 static void ivshmem_register_types(void)
1363 type_register_static(&ivshmem_common_info
);
1364 type_register_static(&ivshmem_plain_info
);
1365 type_register_static(&ivshmem_doorbell_info
);
1366 type_register_static(&ivshmem_info
);
1369 type_init(ivshmem_register_types
)