2 * Inter-VM Shared Memory PCI device.
5 * Cam Macdonell <cam@cs.ualberta.ca>
7 * Based On: cirrus_vga.c
8 * Copyright (c) 2004 Fabrice Bellard
9 * Copyright (c) 2004 Makoto Suzuki (suzu)
12 * Copyright (c) 2006 Igor Kovalenko
14 * This code is licensed under the GNU GPL v2.
16 * Contributions after 2012-01-13 are licensed under the terms of the
17 * GNU GPL, version 2 or (at your option) any later version.
19 #include "qemu/osdep.h"
21 #include "hw/i386/pc.h"
22 #include "hw/pci/pci.h"
23 #include "hw/pci/msi.h"
24 #include "hw/pci/msix.h"
25 #include "sysemu/kvm.h"
26 #include "migration/migration.h"
27 #include "qemu/error-report.h"
28 #include "qemu/event_notifier.h"
29 #include "qemu/fifo8.h"
30 #include "sysemu/char.h"
31 #include "sysemu/hostmem.h"
32 #include "qapi/visitor.h"
33 #include "exec/ram_addr.h"
35 #include "hw/misc/ivshmem.h"
39 #define PCI_VENDOR_ID_IVSHMEM PCI_VENDOR_ID_REDHAT_QUMRANET
40 #define PCI_DEVICE_ID_IVSHMEM 0x1110
42 #define IVSHMEM_MAX_PEERS UINT16_MAX
43 #define IVSHMEM_IOEVENTFD 0
46 #define IVSHMEM_PEER 0
47 #define IVSHMEM_MASTER 1
49 #define IVSHMEM_REG_BAR_SIZE 0x100
51 #define IVSHMEM_DEBUG 0
52 #define IVSHMEM_DPRINTF(fmt, ...) \
54 if (IVSHMEM_DEBUG) { \
55 printf("IVSHMEM: " fmt, ## __VA_ARGS__); \
59 #define TYPE_IVSHMEM "ivshmem"
60 #define IVSHMEM(obj) \
61 OBJECT_CHECK(IVShmemState, (obj), TYPE_IVSHMEM)
65 EventNotifier
*eventfds
;
68 typedef struct MSIVector
{
73 typedef struct IVShmemState
{
78 HostMemoryBackend
*hostmem
;
82 CharDriverState
*server_chr
;
84 MemoryRegion ivshmem_mmio
;
86 /* We might need to register the BAR before we actually have the memory.
87 * So prepare a container MemoryRegion for the BAR immediately and
88 * add a subregion when we have the memory.
92 uint64_t ivshmem_size
; /* size of shared memory region */
93 uint32_t ivshmem_64bit
;
96 int nb_peers
; /* space in @peers[] */
101 MSIVector
*msi_vectors
;
103 Error
*migration_blocker
;
108 int role_val
; /* scalar to avoid multiple string comparisons */
111 /* registers for the Inter-VM shared memory device */
112 enum ivshmem_registers
{
119 static inline uint32_t ivshmem_has_feature(IVShmemState
*ivs
,
120 unsigned int feature
) {
121 return (ivs
->features
& (1 << feature
));
124 static void ivshmem_update_irq(IVShmemState
*s
)
126 PCIDevice
*d
= PCI_DEVICE(s
);
127 uint32_t isr
= s
->intrstatus
& s
->intrmask
;
129 /* No INTx with msi=on, whether the guest enabled MSI-X or not */
130 if (ivshmem_has_feature(s
, IVSHMEM_MSI
)) {
134 /* don't print ISR resets */
136 IVSHMEM_DPRINTF("Set IRQ to %d (%04x %04x)\n",
137 isr
? 1 : 0, s
->intrstatus
, s
->intrmask
);
140 pci_set_irq(d
, isr
!= 0);
143 static void ivshmem_IntrMask_write(IVShmemState
*s
, uint32_t val
)
145 IVSHMEM_DPRINTF("IntrMask write(w) val = 0x%04x\n", val
);
148 ivshmem_update_irq(s
);
151 static uint32_t ivshmem_IntrMask_read(IVShmemState
*s
)
153 uint32_t ret
= s
->intrmask
;
155 IVSHMEM_DPRINTF("intrmask read(w) val = 0x%04x\n", ret
);
159 static void ivshmem_IntrStatus_write(IVShmemState
*s
, uint32_t val
)
161 IVSHMEM_DPRINTF("IntrStatus write(w) val = 0x%04x\n", val
);
164 ivshmem_update_irq(s
);
167 static uint32_t ivshmem_IntrStatus_read(IVShmemState
*s
)
169 uint32_t ret
= s
->intrstatus
;
171 /* reading ISR clears all interrupts */
173 ivshmem_update_irq(s
);
177 static void ivshmem_io_write(void *opaque
, hwaddr addr
,
178 uint64_t val
, unsigned size
)
180 IVShmemState
*s
= opaque
;
182 uint16_t dest
= val
>> 16;
183 uint16_t vector
= val
& 0xff;
187 IVSHMEM_DPRINTF("writing to addr " TARGET_FMT_plx
"\n", addr
);
191 ivshmem_IntrMask_write(s
, val
);
195 ivshmem_IntrStatus_write(s
, val
);
199 /* check that dest VM ID is reasonable */
200 if (dest
>= s
->nb_peers
) {
201 IVSHMEM_DPRINTF("Invalid destination VM ID (%d)\n", dest
);
205 /* check doorbell range */
206 if (vector
< s
->peers
[dest
].nb_eventfds
) {
207 IVSHMEM_DPRINTF("Notifying VM %d on vector %d\n", dest
, vector
);
208 event_notifier_set(&s
->peers
[dest
].eventfds
[vector
]);
210 IVSHMEM_DPRINTF("Invalid destination vector %d on VM %d\n",
215 IVSHMEM_DPRINTF("Unhandled write " TARGET_FMT_plx
"\n", addr
);
219 static uint64_t ivshmem_io_read(void *opaque
, hwaddr addr
,
223 IVShmemState
*s
= opaque
;
229 ret
= ivshmem_IntrMask_read(s
);
233 ret
= ivshmem_IntrStatus_read(s
);
241 IVSHMEM_DPRINTF("why are we reading " TARGET_FMT_plx
"\n", addr
);
248 static const MemoryRegionOps ivshmem_mmio_ops
= {
249 .read
= ivshmem_io_read
,
250 .write
= ivshmem_io_write
,
251 .endianness
= DEVICE_NATIVE_ENDIAN
,
253 .min_access_size
= 4,
254 .max_access_size
= 4,
258 static int ivshmem_can_receive(void * opaque
)
260 return sizeof(int64_t);
263 static void ivshmem_vector_notify(void *opaque
)
265 MSIVector
*entry
= opaque
;
266 PCIDevice
*pdev
= entry
->pdev
;
267 IVShmemState
*s
= IVSHMEM(pdev
);
268 int vector
= entry
- s
->msi_vectors
;
269 EventNotifier
*n
= &s
->peers
[s
->vm_id
].eventfds
[vector
];
271 if (!event_notifier_test_and_clear(n
)) {
275 IVSHMEM_DPRINTF("interrupt on vector %p %d\n", pdev
, vector
);
276 if (ivshmem_has_feature(s
, IVSHMEM_MSI
)) {
277 if (msix_enabled(pdev
)) {
278 msix_notify(pdev
, vector
);
281 ivshmem_IntrStatus_write(s
, 1);
285 static int ivshmem_vector_unmask(PCIDevice
*dev
, unsigned vector
,
288 IVShmemState
*s
= IVSHMEM(dev
);
289 EventNotifier
*n
= &s
->peers
[s
->vm_id
].eventfds
[vector
];
290 MSIVector
*v
= &s
->msi_vectors
[vector
];
293 IVSHMEM_DPRINTF("vector unmask %p %d\n", dev
, vector
);
295 ret
= kvm_irqchip_update_msi_route(kvm_state
, v
->virq
, msg
, dev
);
300 return kvm_irqchip_add_irqfd_notifier_gsi(kvm_state
, n
, NULL
, v
->virq
);
303 static void ivshmem_vector_mask(PCIDevice
*dev
, unsigned vector
)
305 IVShmemState
*s
= IVSHMEM(dev
);
306 EventNotifier
*n
= &s
->peers
[s
->vm_id
].eventfds
[vector
];
309 IVSHMEM_DPRINTF("vector mask %p %d\n", dev
, vector
);
311 ret
= kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state
, n
,
312 s
->msi_vectors
[vector
].virq
);
314 error_report("remove_irqfd_notifier_gsi failed");
318 static void ivshmem_vector_poll(PCIDevice
*dev
,
319 unsigned int vector_start
,
320 unsigned int vector_end
)
322 IVShmemState
*s
= IVSHMEM(dev
);
325 IVSHMEM_DPRINTF("vector poll %p %d-%d\n", dev
, vector_start
, vector_end
);
327 vector_end
= MIN(vector_end
, s
->vectors
);
329 for (vector
= vector_start
; vector
< vector_end
; vector
++) {
330 EventNotifier
*notifier
= &s
->peers
[s
->vm_id
].eventfds
[vector
];
332 if (!msix_is_masked(dev
, vector
)) {
336 if (event_notifier_test_and_clear(notifier
)) {
337 msix_set_pending(dev
, vector
);
342 static void watch_vector_notifier(IVShmemState
*s
, EventNotifier
*n
,
345 int eventfd
= event_notifier_get_fd(n
);
347 assert(!s
->msi_vectors
[vector
].pdev
);
348 s
->msi_vectors
[vector
].pdev
= PCI_DEVICE(s
);
350 qemu_set_fd_handler(eventfd
, ivshmem_vector_notify
,
351 NULL
, &s
->msi_vectors
[vector
]);
354 static int check_shm_size(IVShmemState
*s
, int fd
, Error
**errp
)
356 /* check that the guest isn't going to try and map more memory than the
357 * the object has allocated return -1 to indicate error */
361 if (fstat(fd
, &buf
) < 0) {
362 error_setg(errp
, "exiting: fstat on fd %d failed: %s",
363 fd
, strerror(errno
));
367 if (s
->ivshmem_size
> buf
.st_size
) {
368 error_setg(errp
, "Requested memory size greater"
369 " than shared object size (%" PRIu64
" > %" PRIu64
")",
370 s
->ivshmem_size
, (uint64_t)buf
.st_size
);
377 /* create the shared memory BAR when we are not using the server, so we can
378 * create the BAR and map the memory immediately */
379 static int create_shared_memory_BAR(IVShmemState
*s
, int fd
, uint8_t attr
,
384 ptr
= mmap(0, s
->ivshmem_size
, PROT_READ
|PROT_WRITE
, MAP_SHARED
, fd
, 0);
385 if (ptr
== MAP_FAILED
) {
386 error_setg_errno(errp
, errno
, "Failed to mmap shared memory");
390 memory_region_init_ram_ptr(&s
->ivshmem
, OBJECT(s
), "ivshmem.bar2",
391 s
->ivshmem_size
, ptr
);
392 qemu_set_ram_fd(memory_region_get_ram_addr(&s
->ivshmem
), fd
);
393 vmstate_register_ram(&s
->ivshmem
, DEVICE(s
));
394 memory_region_add_subregion(&s
->bar
, 0, &s
->ivshmem
);
396 /* region for shared memory */
397 pci_register_bar(PCI_DEVICE(s
), 2, attr
, &s
->bar
);
402 static void ivshmem_add_eventfd(IVShmemState
*s
, int posn
, int i
)
404 memory_region_add_eventfd(&s
->ivshmem_mmio
,
409 &s
->peers
[posn
].eventfds
[i
]);
412 static void ivshmem_del_eventfd(IVShmemState
*s
, int posn
, int i
)
414 memory_region_del_eventfd(&s
->ivshmem_mmio
,
419 &s
->peers
[posn
].eventfds
[i
]);
422 static void close_peer_eventfds(IVShmemState
*s
, int posn
)
426 assert(posn
>= 0 && posn
< s
->nb_peers
);
427 n
= s
->peers
[posn
].nb_eventfds
;
429 if (ivshmem_has_feature(s
, IVSHMEM_IOEVENTFD
)) {
430 memory_region_transaction_begin();
431 for (i
= 0; i
< n
; i
++) {
432 ivshmem_del_eventfd(s
, posn
, i
);
434 memory_region_transaction_commit();
437 for (i
= 0; i
< n
; i
++) {
438 event_notifier_cleanup(&s
->peers
[posn
].eventfds
[i
]);
441 g_free(s
->peers
[posn
].eventfds
);
442 s
->peers
[posn
].nb_eventfds
= 0;
445 static void resize_peers(IVShmemState
*s
, int nb_peers
)
447 int old_nb_peers
= s
->nb_peers
;
450 assert(nb_peers
> old_nb_peers
);
451 IVSHMEM_DPRINTF("bumping storage to %d peers\n", nb_peers
);
453 s
->peers
= g_realloc(s
->peers
, nb_peers
* sizeof(Peer
));
454 s
->nb_peers
= nb_peers
;
456 for (i
= old_nb_peers
; i
< nb_peers
; i
++) {
457 s
->peers
[i
].eventfds
= g_new0(EventNotifier
, s
->vectors
);
458 s
->peers
[i
].nb_eventfds
= 0;
462 static bool fifo_update_and_get(IVShmemState
*s
, const uint8_t *buf
, int size
,
463 void *data
, size_t len
)
468 assert(len
<= sizeof(int64_t)); /* limitation of the fifo */
469 if (fifo8_is_empty(&s
->incoming_fifo
) && size
== len
) {
470 memcpy(data
, buf
, size
);
474 IVSHMEM_DPRINTF("short read of %d bytes\n", size
);
476 num
= MIN(size
, sizeof(int64_t) - fifo8_num_used(&s
->incoming_fifo
));
477 fifo8_push_all(&s
->incoming_fifo
, buf
, num
);
479 if (fifo8_num_used(&s
->incoming_fifo
) < len
) {
486 p
= fifo8_pop_buf(&s
->incoming_fifo
, len
, &num
);
489 memcpy(data
, p
, len
);
492 fifo8_push_all(&s
->incoming_fifo
, buf
, size
);
498 static bool fifo_update_and_get_i64(IVShmemState
*s
,
499 const uint8_t *buf
, int size
, int64_t *i64
)
501 if (fifo_update_and_get(s
, buf
, size
, i64
, sizeof(*i64
))) {
502 *i64
= GINT64_FROM_LE(*i64
);
509 static void ivshmem_add_kvm_msi_virq(IVShmemState
*s
, int vector
,
512 PCIDevice
*pdev
= PCI_DEVICE(s
);
513 MSIMessage msg
= msix_get_message(pdev
, vector
);
516 IVSHMEM_DPRINTF("ivshmem_add_kvm_msi_virq vector:%d\n", vector
);
517 assert(!s
->msi_vectors
[vector
].pdev
);
519 ret
= kvm_irqchip_add_msi_route(kvm_state
, msg
, pdev
);
521 error_setg(errp
, "kvm_irqchip_add_msi_route failed");
525 s
->msi_vectors
[vector
].virq
= ret
;
526 s
->msi_vectors
[vector
].pdev
= pdev
;
529 static void setup_interrupt(IVShmemState
*s
, int vector
, Error
**errp
)
531 EventNotifier
*n
= &s
->peers
[s
->vm_id
].eventfds
[vector
];
532 bool with_irqfd
= kvm_msi_via_irqfd_enabled() &&
533 ivshmem_has_feature(s
, IVSHMEM_MSI
);
534 PCIDevice
*pdev
= PCI_DEVICE(s
);
537 IVSHMEM_DPRINTF("setting up interrupt for vector: %d\n", vector
);
540 IVSHMEM_DPRINTF("with eventfd\n");
541 watch_vector_notifier(s
, n
, vector
);
542 } else if (msix_enabled(pdev
)) {
543 IVSHMEM_DPRINTF("with irqfd\n");
544 ivshmem_add_kvm_msi_virq(s
, vector
, &err
);
546 error_propagate(errp
, err
);
550 if (!msix_is_masked(pdev
, vector
)) {
551 kvm_irqchip_add_irqfd_notifier_gsi(kvm_state
, n
, NULL
,
552 s
->msi_vectors
[vector
].virq
);
553 /* TODO handle error */
556 /* it will be delayed until msix is enabled, in write_config */
557 IVSHMEM_DPRINTF("with irqfd, delayed until msix enabled\n");
561 static void process_msg_shmem(IVShmemState
*s
, int fd
, Error
**errp
)
566 if (memory_region_is_mapped(&s
->ivshmem
)) {
567 error_setg(errp
, "server sent unexpected shared memory message");
572 if (check_shm_size(s
, fd
, &err
) == -1) {
573 error_propagate(errp
, err
);
578 /* mmap the region and map into the BAR2 */
579 ptr
= mmap(0, s
->ivshmem_size
, PROT_READ
| PROT_WRITE
, MAP_SHARED
, fd
, 0);
580 if (ptr
== MAP_FAILED
) {
581 error_setg_errno(errp
, errno
, "Failed to mmap shared memory");
585 memory_region_init_ram_ptr(&s
->ivshmem
, OBJECT(s
),
586 "ivshmem.bar2", s
->ivshmem_size
, ptr
);
587 qemu_set_ram_fd(memory_region_get_ram_addr(&s
->ivshmem
), fd
);
588 vmstate_register_ram(&s
->ivshmem
, DEVICE(s
));
589 memory_region_add_subregion(&s
->bar
, 0, &s
->ivshmem
);
592 static void process_msg_disconnect(IVShmemState
*s
, uint16_t posn
,
595 IVSHMEM_DPRINTF("posn %d has gone away\n", posn
);
596 if (posn
>= s
->nb_peers
|| posn
== s
->vm_id
) {
597 error_setg(errp
, "invalid peer %d", posn
);
600 close_peer_eventfds(s
, posn
);
603 static void process_msg_connect(IVShmemState
*s
, uint16_t posn
, int fd
,
606 Peer
*peer
= &s
->peers
[posn
];
610 * The N-th connect message for this peer comes with the file
611 * descriptor for vector N-1. Count messages to find the vector.
613 if (peer
->nb_eventfds
>= s
->vectors
) {
614 error_setg(errp
, "Too many eventfd received, device has %d vectors",
619 vector
= peer
->nb_eventfds
++;
621 IVSHMEM_DPRINTF("eventfds[%d][%d] = %d\n", posn
, vector
, fd
);
622 event_notifier_init_fd(&peer
->eventfds
[vector
], fd
);
623 fcntl_setfl(fd
, O_NONBLOCK
); /* msix/irqfd poll non block */
625 if (posn
== s
->vm_id
) {
626 setup_interrupt(s
, vector
, errp
);
627 /* TODO do we need to handle the error? */
630 if (ivshmem_has_feature(s
, IVSHMEM_IOEVENTFD
)) {
631 ivshmem_add_eventfd(s
, posn
, vector
);
635 static void process_msg(IVShmemState
*s
, int64_t msg
, int fd
, Error
**errp
)
637 IVSHMEM_DPRINTF("posn is %" PRId64
", fd is %d\n", msg
, fd
);
639 if (msg
< -1 || msg
> IVSHMEM_MAX_PEERS
) {
640 error_setg(errp
, "server sent invalid message %" PRId64
, msg
);
646 process_msg_shmem(s
, fd
, errp
);
650 if (msg
>= s
->nb_peers
) {
651 resize_peers(s
, msg
+ 1);
655 process_msg_connect(s
, msg
, fd
, errp
);
657 process_msg_disconnect(s
, msg
, errp
);
661 static void ivshmem_read(void *opaque
, const uint8_t *buf
, int size
)
663 IVShmemState
*s
= opaque
;
668 if (!fifo_update_and_get_i64(s
, buf
, size
, &msg
)) {
672 fd
= qemu_chr_fe_get_msgfd(s
->server_chr
);
673 IVSHMEM_DPRINTF("posn is %" PRId64
", fd is %d\n", msg
, fd
);
675 process_msg(s
, msg
, fd
, &err
);
677 error_report_err(err
);
681 static int64_t ivshmem_recv_msg(IVShmemState
*s
, int *pfd
, Error
**errp
)
688 ret
= qemu_chr_fe_read_all(s
->server_chr
, (uint8_t *)&msg
+ n
,
690 if (ret
< 0 && ret
!= -EINTR
) {
691 error_setg_errno(errp
, -ret
, "read from server failed");
695 } while (n
< sizeof(msg
));
697 *pfd
= qemu_chr_fe_get_msgfd(s
->server_chr
);
701 static void ivshmem_recv_setup(IVShmemState
*s
, Error
**errp
)
707 msg
= ivshmem_recv_msg(s
, &fd
, &err
);
709 error_propagate(errp
, err
);
712 if (msg
!= IVSHMEM_PROTOCOL_VERSION
) {
713 error_setg(errp
, "server sent version %" PRId64
", expecting %d",
714 msg
, IVSHMEM_PROTOCOL_VERSION
);
718 error_setg(errp
, "server sent invalid version message");
723 * ivshmem-server sends the remaining initial messages in a fixed
724 * order, but the device has always accepted them in any order.
725 * Stay as compatible as practical, just in case people use
726 * servers that behave differently.
730 * ivshmem_device_spec.txt has always required the ID message
731 * right here, and ivshmem-server has always complied. However,
732 * older versions of the device accepted it out of order, but
733 * broke when an interrupt setup message arrived before it.
735 msg
= ivshmem_recv_msg(s
, &fd
, &err
);
737 error_propagate(errp
, err
);
740 if (fd
!= -1 || msg
< 0 || msg
> IVSHMEM_MAX_PEERS
) {
741 error_setg(errp
, "server sent invalid ID message");
747 * Receive more messages until we got shared memory.
750 msg
= ivshmem_recv_msg(s
, &fd
, &err
);
752 error_propagate(errp
, err
);
755 process_msg(s
, msg
, fd
, &err
);
757 error_propagate(errp
, err
);
763 * This function must either map the shared memory or fail. The
764 * loop above ensures that: it terminates normally only after it
765 * successfully processed the server's shared memory message.
766 * Assert that actually mapped the shared memory:
768 assert(memory_region_is_mapped(&s
->ivshmem
));
771 /* Select the MSI-X vectors used by device.
772 * ivshmem maps events to vectors statically, so
773 * we just enable all vectors on init and after reset. */
774 static void ivshmem_msix_vector_use(IVShmemState
*s
)
776 PCIDevice
*d
= PCI_DEVICE(s
);
779 for (i
= 0; i
< s
->vectors
; i
++) {
780 msix_vector_use(d
, i
);
784 static void ivshmem_reset(DeviceState
*d
)
786 IVShmemState
*s
= IVSHMEM(d
);
790 if (ivshmem_has_feature(s
, IVSHMEM_MSI
)) {
791 ivshmem_msix_vector_use(s
);
795 static int ivshmem_setup_interrupts(IVShmemState
*s
)
797 /* allocate QEMU callback data for receiving interrupts */
798 s
->msi_vectors
= g_malloc0(s
->vectors
* sizeof(MSIVector
));
800 if (ivshmem_has_feature(s
, IVSHMEM_MSI
)) {
801 if (msix_init_exclusive_bar(PCI_DEVICE(s
), s
->vectors
, 1)) {
805 IVSHMEM_DPRINTF("msix initialized (%d vectors)\n", s
->vectors
);
806 ivshmem_msix_vector_use(s
);
812 static void ivshmem_enable_irqfd(IVShmemState
*s
)
814 PCIDevice
*pdev
= PCI_DEVICE(s
);
817 for (i
= 0; i
< s
->peers
[s
->vm_id
].nb_eventfds
; i
++) {
820 ivshmem_add_kvm_msi_virq(s
, i
, &err
);
822 error_report_err(err
);
823 /* TODO do we need to handle the error? */
827 if (msix_set_vector_notifiers(pdev
,
828 ivshmem_vector_unmask
,
830 ivshmem_vector_poll
)) {
831 error_report("ivshmem: msix_set_vector_notifiers failed");
835 static void ivshmem_remove_kvm_msi_virq(IVShmemState
*s
, int vector
)
837 IVSHMEM_DPRINTF("ivshmem_remove_kvm_msi_virq vector:%d\n", vector
);
839 if (s
->msi_vectors
[vector
].pdev
== NULL
) {
843 /* it was cleaned when masked in the frontend. */
844 kvm_irqchip_release_virq(kvm_state
, s
->msi_vectors
[vector
].virq
);
846 s
->msi_vectors
[vector
].pdev
= NULL
;
849 static void ivshmem_disable_irqfd(IVShmemState
*s
)
851 PCIDevice
*pdev
= PCI_DEVICE(s
);
854 for (i
= 0; i
< s
->peers
[s
->vm_id
].nb_eventfds
; i
++) {
855 ivshmem_remove_kvm_msi_virq(s
, i
);
858 msix_unset_vector_notifiers(pdev
);
861 static void ivshmem_write_config(PCIDevice
*pdev
, uint32_t address
,
862 uint32_t val
, int len
)
864 IVShmemState
*s
= IVSHMEM(pdev
);
865 int is_enabled
, was_enabled
= msix_enabled(pdev
);
867 pci_default_write_config(pdev
, address
, val
, len
);
868 is_enabled
= msix_enabled(pdev
);
870 if (kvm_msi_via_irqfd_enabled()) {
871 if (!was_enabled
&& is_enabled
) {
872 ivshmem_enable_irqfd(s
);
873 } else if (was_enabled
&& !is_enabled
) {
874 ivshmem_disable_irqfd(s
);
879 static void pci_ivshmem_realize(PCIDevice
*dev
, Error
**errp
)
881 IVShmemState
*s
= IVSHMEM(dev
);
884 uint8_t attr
= PCI_BASE_ADDRESS_SPACE_MEMORY
|
885 PCI_BASE_ADDRESS_MEM_PREFETCH
;
887 if (!!s
->server_chr
+ !!s
->shmobj
+ !!s
->hostmem
!= 1) {
889 "You must specify either 'shm', 'chardev' or 'x-memdev'");
897 g_warning("size argument ignored with hostmem");
900 mr
= host_memory_backend_get_memory(s
->hostmem
, &error_abort
);
901 s
->ivshmem_size
= memory_region_size(mr
);
902 } else if (s
->sizearg
== NULL
) {
903 s
->ivshmem_size
= 4 << 20; /* 4 MB default */
906 int64_t size
= qemu_strtosz(s
->sizearg
, &end
);
907 if (size
< 0 || *end
!= '\0' || !is_power_of_2(size
)) {
908 error_setg(errp
, "Invalid size %s", s
->sizearg
);
911 s
->ivshmem_size
= size
;
914 /* IRQFD requires MSI */
915 if (ivshmem_has_feature(s
, IVSHMEM_IOEVENTFD
) &&
916 !ivshmem_has_feature(s
, IVSHMEM_MSI
)) {
917 error_setg(errp
, "ioeventfd/irqfd requires MSI");
921 /* check that role is reasonable */
923 if (strncmp(s
->role
, "peer", 5) == 0) {
924 s
->role_val
= IVSHMEM_PEER
;
925 } else if (strncmp(s
->role
, "master", 7) == 0) {
926 s
->role_val
= IVSHMEM_MASTER
;
928 error_setg(errp
, "'role' must be 'peer' or 'master'");
932 s
->role_val
= IVSHMEM_MASTER
; /* default */
935 pci_conf
= dev
->config
;
936 pci_conf
[PCI_COMMAND
] = PCI_COMMAND_IO
| PCI_COMMAND_MEMORY
;
939 * Note: we don't use INTx with IVSHMEM_MSI at all, so this is a
940 * bald-faced lie then. But it's a backwards compatible lie.
942 pci_config_set_interrupt_pin(pci_conf
, 1);
944 memory_region_init_io(&s
->ivshmem_mmio
, OBJECT(s
), &ivshmem_mmio_ops
, s
,
945 "ivshmem-mmio", IVSHMEM_REG_BAR_SIZE
);
947 /* region for registers*/
948 pci_register_bar(dev
, 0, PCI_BASE_ADDRESS_SPACE_MEMORY
,
951 memory_region_init(&s
->bar
, OBJECT(s
), "ivshmem-bar2-container", s
->ivshmem_size
);
952 if (s
->ivshmem_64bit
) {
953 attr
|= PCI_BASE_ADDRESS_MEM_TYPE_64
;
956 if (s
->hostmem
!= NULL
) {
959 IVSHMEM_DPRINTF("using hostmem\n");
961 mr
= host_memory_backend_get_memory(MEMORY_BACKEND(s
->hostmem
),
963 vmstate_register_ram(mr
, DEVICE(s
));
964 memory_region_add_subregion(&s
->bar
, 0, mr
);
965 pci_register_bar(PCI_DEVICE(s
), 2, attr
, &s
->bar
);
966 } else if (s
->server_chr
!= NULL
) {
967 IVSHMEM_DPRINTF("using shared memory server (socket = %s)\n",
968 s
->server_chr
->filename
);
970 /* we allocate enough space for 16 peers and grow as needed */
973 pci_register_bar(dev
, 2, attr
, &s
->bar
);
976 * Receive setup messages from server synchronously.
977 * Older versions did it asynchronously, but that creates a
978 * number of entertaining race conditions.
980 ivshmem_recv_setup(s
, &err
);
982 error_propagate(errp
, err
);
986 qemu_chr_add_handlers(s
->server_chr
, ivshmem_can_receive
,
987 ivshmem_read
, NULL
, s
);
989 if (ivshmem_setup_interrupts(s
) < 0) {
990 error_setg(errp
, "failed to initialize interrupts");
994 /* just map the file immediately, we're not using a server */
997 IVSHMEM_DPRINTF("using shm_open (shm object = %s)\n", s
->shmobj
);
999 /* try opening with O_EXCL and if it succeeds zero the memory
1000 * by truncating to 0 */
1001 if ((fd
= shm_open(s
->shmobj
, O_CREAT
|O_RDWR
|O_EXCL
,
1002 S_IRWXU
|S_IRWXG
|S_IRWXO
)) > 0) {
1003 /* truncate file to length PCI device's memory */
1004 if (ftruncate(fd
, s
->ivshmem_size
) != 0) {
1005 error_report("could not truncate shared file");
1008 } else if ((fd
= shm_open(s
->shmobj
, O_CREAT
|O_RDWR
,
1009 S_IRWXU
|S_IRWXG
|S_IRWXO
)) < 0) {
1010 error_setg(errp
, "could not open shared file");
1014 if (check_shm_size(s
, fd
, errp
) == -1) {
1018 create_shared_memory_BAR(s
, fd
, attr
, &err
);
1020 error_propagate(errp
, err
);
1025 fifo8_create(&s
->incoming_fifo
, sizeof(int64_t));
1027 if (s
->role_val
== IVSHMEM_PEER
) {
1028 error_setg(&s
->migration_blocker
,
1029 "Migration is disabled when using feature 'peer mode' in device 'ivshmem'");
1030 migrate_add_blocker(s
->migration_blocker
);
1034 static void pci_ivshmem_exit(PCIDevice
*dev
)
1036 IVShmemState
*s
= IVSHMEM(dev
);
1039 fifo8_destroy(&s
->incoming_fifo
);
1041 if (s
->migration_blocker
) {
1042 migrate_del_blocker(s
->migration_blocker
);
1043 error_free(s
->migration_blocker
);
1046 if (memory_region_is_mapped(&s
->ivshmem
)) {
1048 void *addr
= memory_region_get_ram_ptr(&s
->ivshmem
);
1051 if (munmap(addr
, s
->ivshmem_size
) == -1) {
1052 error_report("Failed to munmap shared memory %s",
1056 fd
= qemu_get_ram_fd(memory_region_get_ram_addr(&s
->ivshmem
));
1062 vmstate_unregister_ram(&s
->ivshmem
, DEVICE(dev
));
1063 memory_region_del_subregion(&s
->bar
, &s
->ivshmem
);
1067 for (i
= 0; i
< s
->nb_peers
; i
++) {
1068 close_peer_eventfds(s
, i
);
1073 if (ivshmem_has_feature(s
, IVSHMEM_MSI
)) {
1074 msix_uninit_exclusive_bar(dev
);
1077 g_free(s
->msi_vectors
);
1080 static bool test_msix(void *opaque
, int version_id
)
1082 IVShmemState
*s
= opaque
;
1084 return ivshmem_has_feature(s
, IVSHMEM_MSI
);
1087 static bool test_no_msix(void *opaque
, int version_id
)
1089 return !test_msix(opaque
, version_id
);
1092 static int ivshmem_pre_load(void *opaque
)
1094 IVShmemState
*s
= opaque
;
1096 if (s
->role_val
== IVSHMEM_PEER
) {
1097 error_report("'peer' devices are not migratable");
1104 static int ivshmem_post_load(void *opaque
, int version_id
)
1106 IVShmemState
*s
= opaque
;
1108 if (ivshmem_has_feature(s
, IVSHMEM_MSI
)) {
1109 ivshmem_msix_vector_use(s
);
1114 static int ivshmem_load_old(QEMUFile
*f
, void *opaque
, int version_id
)
1116 IVShmemState
*s
= opaque
;
1117 PCIDevice
*pdev
= PCI_DEVICE(s
);
1120 IVSHMEM_DPRINTF("ivshmem_load_old\n");
1122 if (version_id
!= 0) {
1126 if (s
->role_val
== IVSHMEM_PEER
) {
1127 error_report("'peer' devices are not migratable");
1131 ret
= pci_device_load(pdev
, f
);
1136 if (ivshmem_has_feature(s
, IVSHMEM_MSI
)) {
1138 ivshmem_msix_vector_use(s
);
1140 s
->intrstatus
= qemu_get_be32(f
);
1141 s
->intrmask
= qemu_get_be32(f
);
1147 static const VMStateDescription ivshmem_vmsd
= {
1150 .minimum_version_id
= 1,
1151 .pre_load
= ivshmem_pre_load
,
1152 .post_load
= ivshmem_post_load
,
1153 .fields
= (VMStateField
[]) {
1154 VMSTATE_PCI_DEVICE(parent_obj
, IVShmemState
),
1156 VMSTATE_MSIX_TEST(parent_obj
, IVShmemState
, test_msix
),
1157 VMSTATE_UINT32_TEST(intrstatus
, IVShmemState
, test_no_msix
),
1158 VMSTATE_UINT32_TEST(intrmask
, IVShmemState
, test_no_msix
),
1160 VMSTATE_END_OF_LIST()
1162 .load_state_old
= ivshmem_load_old
,
1163 .minimum_version_id_old
= 0
1166 static Property ivshmem_properties
[] = {
1167 DEFINE_PROP_CHR("chardev", IVShmemState
, server_chr
),
1168 DEFINE_PROP_STRING("size", IVShmemState
, sizearg
),
1169 DEFINE_PROP_UINT32("vectors", IVShmemState
, vectors
, 1),
1170 DEFINE_PROP_BIT("ioeventfd", IVShmemState
, features
, IVSHMEM_IOEVENTFD
, false),
1171 DEFINE_PROP_BIT("msi", IVShmemState
, features
, IVSHMEM_MSI
, true),
1172 DEFINE_PROP_STRING("shm", IVShmemState
, shmobj
),
1173 DEFINE_PROP_STRING("role", IVShmemState
, role
),
1174 DEFINE_PROP_UINT32("use64", IVShmemState
, ivshmem_64bit
, 1),
1175 DEFINE_PROP_END_OF_LIST(),
1178 static void ivshmem_class_init(ObjectClass
*klass
, void *data
)
1180 DeviceClass
*dc
= DEVICE_CLASS(klass
);
1181 PCIDeviceClass
*k
= PCI_DEVICE_CLASS(klass
);
1183 k
->realize
= pci_ivshmem_realize
;
1184 k
->exit
= pci_ivshmem_exit
;
1185 k
->config_write
= ivshmem_write_config
;
1186 k
->vendor_id
= PCI_VENDOR_ID_IVSHMEM
;
1187 k
->device_id
= PCI_DEVICE_ID_IVSHMEM
;
1188 k
->class_id
= PCI_CLASS_MEMORY_RAM
;
1189 dc
->reset
= ivshmem_reset
;
1190 dc
->props
= ivshmem_properties
;
1191 dc
->vmsd
= &ivshmem_vmsd
;
1192 set_bit(DEVICE_CATEGORY_MISC
, dc
->categories
);
1193 dc
->desc
= "Inter-VM shared memory";
1196 static void ivshmem_check_memdev_is_busy(Object
*obj
, const char *name
,
1197 Object
*val
, Error
**errp
)
1201 mr
= host_memory_backend_get_memory(MEMORY_BACKEND(val
), &error_abort
);
1202 if (memory_region_is_mapped(mr
)) {
1203 char *path
= object_get_canonical_path_component(val
);
1204 error_setg(errp
, "can't use already busy memdev: %s", path
);
1207 qdev_prop_allow_set_link_before_realize(obj
, name
, val
, errp
);
1211 static void ivshmem_init(Object
*obj
)
1213 IVShmemState
*s
= IVSHMEM(obj
);
1215 object_property_add_link(obj
, "x-memdev", TYPE_MEMORY_BACKEND
,
1216 (Object
**)&s
->hostmem
,
1217 ivshmem_check_memdev_is_busy
,
1218 OBJ_PROP_LINK_UNREF_ON_RELEASE
,
1222 static const TypeInfo ivshmem_info
= {
1223 .name
= TYPE_IVSHMEM
,
1224 .parent
= TYPE_PCI_DEVICE
,
1225 .instance_size
= sizeof(IVShmemState
),
1226 .instance_init
= ivshmem_init
,
1227 .class_init
= ivshmem_class_init
,
1230 static void ivshmem_register_types(void)
1232 type_register_static(&ivshmem_info
);
1235 type_init(ivshmem_register_types
)