tcg-ppc64: Fix add2_i64
[qemu/kevin.git] / kvm-all.c
blob405480ef594ca0314bb67b8ff8c057f571c1a46e
1 /*
2 * QEMU KVM support
4 * Copyright IBM, Corp. 2008
5 * Red Hat, Inc. 2008
7 * Authors:
8 * Anthony Liguori <aliguori@us.ibm.com>
9 * Glauber Costa <gcosta@redhat.com>
11 * This work is licensed under the terms of the GNU GPL, version 2 or later.
12 * See the COPYING file in the top-level directory.
16 #include <sys/types.h>
17 #include <sys/ioctl.h>
18 #include <sys/mman.h>
19 #include <stdarg.h>
21 #include <linux/kvm.h>
23 #include "qemu-common.h"
24 #include "qemu/atomic.h"
25 #include "qemu/option.h"
26 #include "qemu/config-file.h"
27 #include "sysemu/sysemu.h"
28 #include "hw/hw.h"
29 #include "hw/pci/msi.h"
30 #include "exec/gdbstub.h"
31 #include "sysemu/kvm.h"
32 #include "qemu/bswap.h"
33 #include "exec/memory.h"
34 #include "exec/address-spaces.h"
35 #include "qemu/event_notifier.h"
36 #include "trace.h"
38 /* This check must be after config-host.h is included */
39 #ifdef CONFIG_EVENTFD
40 #include <sys/eventfd.h>
41 #endif
43 #ifdef CONFIG_VALGRIND_H
44 #include <valgrind/memcheck.h>
45 #endif
47 /* KVM uses PAGE_SIZE in its definition of COALESCED_MMIO_MAX */
48 #define PAGE_SIZE TARGET_PAGE_SIZE
50 //#define DEBUG_KVM
52 #ifdef DEBUG_KVM
53 #define DPRINTF(fmt, ...) \
54 do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
55 #else
56 #define DPRINTF(fmt, ...) \
57 do { } while (0)
58 #endif
60 #define KVM_MSI_HASHTAB_SIZE 256
62 typedef struct KVMSlot
64 hwaddr start_addr;
65 ram_addr_t memory_size;
66 void *ram;
67 int slot;
68 int flags;
69 } KVMSlot;
71 typedef struct kvm_dirty_log KVMDirtyLog;
73 struct KVMState
75 KVMSlot slots[32];
76 int fd;
77 int vmfd;
78 int coalesced_mmio;
79 struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
80 bool coalesced_flush_in_progress;
81 int broken_set_mem_region;
82 int migration_log;
83 int vcpu_events;
84 int robust_singlestep;
85 int debugregs;
86 #ifdef KVM_CAP_SET_GUEST_DEBUG
87 struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
88 #endif
89 int pit_state2;
90 int xsave, xcrs;
91 int many_ioeventfds;
92 int intx_set_mask;
93 /* The man page (and posix) say ioctl numbers are signed int, but
94 * they're not. Linux, glibc and *BSD all treat ioctl numbers as
95 * unsigned, and treating them as signed here can break things */
96 unsigned irq_set_ioctl;
97 #ifdef KVM_CAP_IRQ_ROUTING
98 struct kvm_irq_routing *irq_routes;
99 int nr_allocated_irq_routes;
100 uint32_t *used_gsi_bitmap;
101 unsigned int gsi_count;
102 QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
103 bool direct_msi;
104 #endif
107 KVMState *kvm_state;
108 bool kvm_kernel_irqchip;
109 bool kvm_async_interrupts_allowed;
110 bool kvm_irqfds_allowed;
111 bool kvm_msi_via_irqfd_allowed;
112 bool kvm_gsi_routing_allowed;
113 bool kvm_allowed;
114 bool kvm_readonly_mem_allowed;
116 static const KVMCapabilityInfo kvm_required_capabilites[] = {
117 KVM_CAP_INFO(USER_MEMORY),
118 KVM_CAP_INFO(DESTROY_MEMORY_REGION_WORKS),
119 KVM_CAP_LAST_INFO
122 static KVMSlot *kvm_alloc_slot(KVMState *s)
124 int i;
126 for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
127 if (s->slots[i].memory_size == 0) {
128 return &s->slots[i];
132 fprintf(stderr, "%s: no free slot available\n", __func__);
133 abort();
136 static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
137 hwaddr start_addr,
138 hwaddr end_addr)
140 int i;
142 for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
143 KVMSlot *mem = &s->slots[i];
145 if (start_addr == mem->start_addr &&
146 end_addr == mem->start_addr + mem->memory_size) {
147 return mem;
151 return NULL;
155 * Find overlapping slot with lowest start address
157 static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
158 hwaddr start_addr,
159 hwaddr end_addr)
161 KVMSlot *found = NULL;
162 int i;
164 for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
165 KVMSlot *mem = &s->slots[i];
167 if (mem->memory_size == 0 ||
168 (found && found->start_addr < mem->start_addr)) {
169 continue;
172 if (end_addr > mem->start_addr &&
173 start_addr < mem->start_addr + mem->memory_size) {
174 found = mem;
178 return found;
181 int kvm_physical_memory_addr_from_host(KVMState *s, void *ram,
182 hwaddr *phys_addr)
184 int i;
186 for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
187 KVMSlot *mem = &s->slots[i];
189 if (ram >= mem->ram && ram < mem->ram + mem->memory_size) {
190 *phys_addr = mem->start_addr + (ram - mem->ram);
191 return 1;
195 return 0;
198 static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
200 struct kvm_userspace_memory_region mem;
202 mem.slot = slot->slot;
203 mem.guest_phys_addr = slot->start_addr;
204 mem.userspace_addr = (unsigned long)slot->ram;
205 mem.flags = slot->flags;
206 if (s->migration_log) {
207 mem.flags |= KVM_MEM_LOG_DIRTY_PAGES;
210 if (slot->memory_size && mem.flags & KVM_MEM_READONLY) {
211 /* Set the slot size to 0 before setting the slot to the desired
212 * value. This is needed based on KVM commit 75d61fbc. */
213 mem.memory_size = 0;
214 kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
216 mem.memory_size = slot->memory_size;
217 return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
220 static void kvm_reset_vcpu(void *opaque)
222 CPUState *cpu = opaque;
224 kvm_arch_reset_vcpu(cpu);
227 int kvm_init_vcpu(CPUState *cpu)
229 KVMState *s = kvm_state;
230 long mmap_size;
231 int ret;
233 DPRINTF("kvm_init_vcpu\n");
235 ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)kvm_arch_vcpu_id(cpu));
236 if (ret < 0) {
237 DPRINTF("kvm_create_vcpu failed\n");
238 goto err;
241 cpu->kvm_fd = ret;
242 cpu->kvm_state = s;
243 cpu->kvm_vcpu_dirty = true;
245 mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
246 if (mmap_size < 0) {
247 ret = mmap_size;
248 DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
249 goto err;
252 cpu->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
253 cpu->kvm_fd, 0);
254 if (cpu->kvm_run == MAP_FAILED) {
255 ret = -errno;
256 DPRINTF("mmap'ing vcpu state failed\n");
257 goto err;
260 if (s->coalesced_mmio && !s->coalesced_mmio_ring) {
261 s->coalesced_mmio_ring =
262 (void *)cpu->kvm_run + s->coalesced_mmio * PAGE_SIZE;
265 ret = kvm_arch_init_vcpu(cpu);
266 if (ret == 0) {
267 qemu_register_reset(kvm_reset_vcpu, cpu);
268 kvm_arch_reset_vcpu(cpu);
270 err:
271 return ret;
275 * dirty pages logging control
278 static int kvm_mem_flags(KVMState *s, bool log_dirty, bool readonly)
280 int flags = 0;
281 flags = log_dirty ? KVM_MEM_LOG_DIRTY_PAGES : 0;
282 if (readonly && kvm_readonly_mem_allowed) {
283 flags |= KVM_MEM_READONLY;
285 return flags;
288 static int kvm_slot_dirty_pages_log_change(KVMSlot *mem, bool log_dirty)
290 KVMState *s = kvm_state;
291 int flags, mask = KVM_MEM_LOG_DIRTY_PAGES;
292 int old_flags;
294 old_flags = mem->flags;
296 flags = (mem->flags & ~mask) | kvm_mem_flags(s, log_dirty, false);
297 mem->flags = flags;
299 /* If nothing changed effectively, no need to issue ioctl */
300 if (s->migration_log) {
301 flags |= KVM_MEM_LOG_DIRTY_PAGES;
304 if (flags == old_flags) {
305 return 0;
308 return kvm_set_user_memory_region(s, mem);
311 static int kvm_dirty_pages_log_change(hwaddr phys_addr,
312 ram_addr_t size, bool log_dirty)
314 KVMState *s = kvm_state;
315 KVMSlot *mem = kvm_lookup_matching_slot(s, phys_addr, phys_addr + size);
317 if (mem == NULL) {
318 fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
319 TARGET_FMT_plx "\n", __func__, phys_addr,
320 (hwaddr)(phys_addr + size - 1));
321 return -EINVAL;
323 return kvm_slot_dirty_pages_log_change(mem, log_dirty);
326 static void kvm_log_start(MemoryListener *listener,
327 MemoryRegionSection *section)
329 int r;
331 r = kvm_dirty_pages_log_change(section->offset_within_address_space,
332 section->size, true);
333 if (r < 0) {
334 abort();
338 static void kvm_log_stop(MemoryListener *listener,
339 MemoryRegionSection *section)
341 int r;
343 r = kvm_dirty_pages_log_change(section->offset_within_address_space,
344 section->size, false);
345 if (r < 0) {
346 abort();
350 static int kvm_set_migration_log(int enable)
352 KVMState *s = kvm_state;
353 KVMSlot *mem;
354 int i, err;
356 s->migration_log = enable;
358 for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
359 mem = &s->slots[i];
361 if (!mem->memory_size) {
362 continue;
364 if (!!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) == enable) {
365 continue;
367 err = kvm_set_user_memory_region(s, mem);
368 if (err) {
369 return err;
372 return 0;
375 /* get kvm's dirty pages bitmap and update qemu's */
376 static int kvm_get_dirty_pages_log_range(MemoryRegionSection *section,
377 unsigned long *bitmap)
379 unsigned int i, j;
380 unsigned long page_number, c;
381 hwaddr addr, addr1;
382 unsigned int len = ((section->size / getpagesize()) + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
383 unsigned long hpratio = getpagesize() / TARGET_PAGE_SIZE;
386 * bitmap-traveling is faster than memory-traveling (for addr...)
387 * especially when most of the memory is not dirty.
389 for (i = 0; i < len; i++) {
390 if (bitmap[i] != 0) {
391 c = leul_to_cpu(bitmap[i]);
392 do {
393 j = ffsl(c) - 1;
394 c &= ~(1ul << j);
395 page_number = (i * HOST_LONG_BITS + j) * hpratio;
396 addr1 = page_number * TARGET_PAGE_SIZE;
397 addr = section->offset_within_region + addr1;
398 memory_region_set_dirty(section->mr, addr,
399 TARGET_PAGE_SIZE * hpratio);
400 } while (c != 0);
403 return 0;
406 #define ALIGN(x, y) (((x)+(y)-1) & ~((y)-1))
409 * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
410 * This function updates qemu's dirty bitmap using
411 * memory_region_set_dirty(). This means all bits are set
412 * to dirty.
414 * @start_add: start of logged region.
415 * @end_addr: end of logged region.
417 static int kvm_physical_sync_dirty_bitmap(MemoryRegionSection *section)
419 KVMState *s = kvm_state;
420 unsigned long size, allocated_size = 0;
421 KVMDirtyLog d;
422 KVMSlot *mem;
423 int ret = 0;
424 hwaddr start_addr = section->offset_within_address_space;
425 hwaddr end_addr = start_addr + section->size;
427 d.dirty_bitmap = NULL;
428 while (start_addr < end_addr) {
429 mem = kvm_lookup_overlapping_slot(s, start_addr, end_addr);
430 if (mem == NULL) {
431 break;
434 /* XXX bad kernel interface alert
435 * For dirty bitmap, kernel allocates array of size aligned to
436 * bits-per-long. But for case when the kernel is 64bits and
437 * the userspace is 32bits, userspace can't align to the same
438 * bits-per-long, since sizeof(long) is different between kernel
439 * and user space. This way, userspace will provide buffer which
440 * may be 4 bytes less than the kernel will use, resulting in
441 * userspace memory corruption (which is not detectable by valgrind
442 * too, in most cases).
443 * So for now, let's align to 64 instead of HOST_LONG_BITS here, in
444 * a hope that sizeof(long) wont become >8 any time soon.
446 size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS),
447 /*HOST_LONG_BITS*/ 64) / 8;
448 if (!d.dirty_bitmap) {
449 d.dirty_bitmap = g_malloc(size);
450 } else if (size > allocated_size) {
451 d.dirty_bitmap = g_realloc(d.dirty_bitmap, size);
453 allocated_size = size;
454 memset(d.dirty_bitmap, 0, allocated_size);
456 d.slot = mem->slot;
458 if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
459 DPRINTF("ioctl failed %d\n", errno);
460 ret = -1;
461 break;
464 kvm_get_dirty_pages_log_range(section, d.dirty_bitmap);
465 start_addr = mem->start_addr + mem->memory_size;
467 g_free(d.dirty_bitmap);
469 return ret;
472 static void kvm_coalesce_mmio_region(MemoryListener *listener,
473 MemoryRegionSection *secion,
474 hwaddr start, hwaddr size)
476 KVMState *s = kvm_state;
478 if (s->coalesced_mmio) {
479 struct kvm_coalesced_mmio_zone zone;
481 zone.addr = start;
482 zone.size = size;
483 zone.pad = 0;
485 (void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
489 static void kvm_uncoalesce_mmio_region(MemoryListener *listener,
490 MemoryRegionSection *secion,
491 hwaddr start, hwaddr size)
493 KVMState *s = kvm_state;
495 if (s->coalesced_mmio) {
496 struct kvm_coalesced_mmio_zone zone;
498 zone.addr = start;
499 zone.size = size;
500 zone.pad = 0;
502 (void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
506 int kvm_check_extension(KVMState *s, unsigned int extension)
508 int ret;
510 ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
511 if (ret < 0) {
512 ret = 0;
515 return ret;
518 static int kvm_set_ioeventfd_mmio(int fd, uint32_t addr, uint32_t val,
519 bool assign, uint32_t size, bool datamatch)
521 int ret;
522 struct kvm_ioeventfd iofd;
524 iofd.datamatch = datamatch ? val : 0;
525 iofd.addr = addr;
526 iofd.len = size;
527 iofd.flags = 0;
528 iofd.fd = fd;
530 if (!kvm_enabled()) {
531 return -ENOSYS;
534 if (datamatch) {
535 iofd.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH;
537 if (!assign) {
538 iofd.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
541 ret = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &iofd);
543 if (ret < 0) {
544 return -errno;
547 return 0;
550 static int kvm_set_ioeventfd_pio(int fd, uint16_t addr, uint16_t val,
551 bool assign, uint32_t size, bool datamatch)
553 struct kvm_ioeventfd kick = {
554 .datamatch = datamatch ? val : 0,
555 .addr = addr,
556 .flags = KVM_IOEVENTFD_FLAG_PIO,
557 .len = size,
558 .fd = fd,
560 int r;
561 if (!kvm_enabled()) {
562 return -ENOSYS;
564 if (datamatch) {
565 kick.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH;
567 if (!assign) {
568 kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
570 r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
571 if (r < 0) {
572 return r;
574 return 0;
578 static int kvm_check_many_ioeventfds(void)
580 /* Userspace can use ioeventfd for io notification. This requires a host
581 * that supports eventfd(2) and an I/O thread; since eventfd does not
582 * support SIGIO it cannot interrupt the vcpu.
584 * Older kernels have a 6 device limit on the KVM io bus. Find out so we
585 * can avoid creating too many ioeventfds.
587 #if defined(CONFIG_EVENTFD)
588 int ioeventfds[7];
589 int i, ret = 0;
590 for (i = 0; i < ARRAY_SIZE(ioeventfds); i++) {
591 ioeventfds[i] = eventfd(0, EFD_CLOEXEC);
592 if (ioeventfds[i] < 0) {
593 break;
595 ret = kvm_set_ioeventfd_pio(ioeventfds[i], 0, i, true, 2, true);
596 if (ret < 0) {
597 close(ioeventfds[i]);
598 break;
602 /* Decide whether many devices are supported or not */
603 ret = i == ARRAY_SIZE(ioeventfds);
605 while (i-- > 0) {
606 kvm_set_ioeventfd_pio(ioeventfds[i], 0, i, false, 2, true);
607 close(ioeventfds[i]);
609 return ret;
610 #else
611 return 0;
612 #endif
615 static const KVMCapabilityInfo *
616 kvm_check_extension_list(KVMState *s, const KVMCapabilityInfo *list)
618 while (list->name) {
619 if (!kvm_check_extension(s, list->value)) {
620 return list;
622 list++;
624 return NULL;
627 static void kvm_set_phys_mem(MemoryRegionSection *section, bool add)
629 KVMState *s = kvm_state;
630 KVMSlot *mem, old;
631 int err;
632 MemoryRegion *mr = section->mr;
633 bool log_dirty = memory_region_is_logging(mr);
634 bool writeable = !mr->readonly && !mr->rom_device;
635 bool readonly_flag = mr->readonly || memory_region_is_romd(mr);
636 hwaddr start_addr = section->offset_within_address_space;
637 ram_addr_t size = section->size;
638 void *ram = NULL;
639 unsigned delta;
641 /* kvm works in page size chunks, but the function may be called
642 with sub-page size and unaligned start address. */
643 delta = TARGET_PAGE_ALIGN(size) - size;
644 if (delta > size) {
645 return;
647 start_addr += delta;
648 size -= delta;
649 size &= TARGET_PAGE_MASK;
650 if (!size || (start_addr & ~TARGET_PAGE_MASK)) {
651 return;
654 if (!memory_region_is_ram(mr)) {
655 if (writeable || !kvm_readonly_mem_allowed) {
656 return;
657 } else if (!mr->romd_mode) {
658 /* If the memory device is not in romd_mode, then we actually want
659 * to remove the kvm memory slot so all accesses will trap. */
660 add = false;
664 ram = memory_region_get_ram_ptr(mr) + section->offset_within_region + delta;
666 while (1) {
667 mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size);
668 if (!mem) {
669 break;
672 if (add && start_addr >= mem->start_addr &&
673 (start_addr + size <= mem->start_addr + mem->memory_size) &&
674 (ram - start_addr == mem->ram - mem->start_addr)) {
675 /* The new slot fits into the existing one and comes with
676 * identical parameters - update flags and done. */
677 kvm_slot_dirty_pages_log_change(mem, log_dirty);
678 return;
681 old = *mem;
683 if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) {
684 kvm_physical_sync_dirty_bitmap(section);
687 /* unregister the overlapping slot */
688 mem->memory_size = 0;
689 err = kvm_set_user_memory_region(s, mem);
690 if (err) {
691 fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
692 __func__, strerror(-err));
693 abort();
696 /* Workaround for older KVM versions: we can't join slots, even not by
697 * unregistering the previous ones and then registering the larger
698 * slot. We have to maintain the existing fragmentation. Sigh.
700 * This workaround assumes that the new slot starts at the same
701 * address as the first existing one. If not or if some overlapping
702 * slot comes around later, we will fail (not seen in practice so far)
703 * - and actually require a recent KVM version. */
704 if (s->broken_set_mem_region &&
705 old.start_addr == start_addr && old.memory_size < size && add) {
706 mem = kvm_alloc_slot(s);
707 mem->memory_size = old.memory_size;
708 mem->start_addr = old.start_addr;
709 mem->ram = old.ram;
710 mem->flags = kvm_mem_flags(s, log_dirty, readonly_flag);
712 err = kvm_set_user_memory_region(s, mem);
713 if (err) {
714 fprintf(stderr, "%s: error updating slot: %s\n", __func__,
715 strerror(-err));
716 abort();
719 start_addr += old.memory_size;
720 ram += old.memory_size;
721 size -= old.memory_size;
722 continue;
725 /* register prefix slot */
726 if (old.start_addr < start_addr) {
727 mem = kvm_alloc_slot(s);
728 mem->memory_size = start_addr - old.start_addr;
729 mem->start_addr = old.start_addr;
730 mem->ram = old.ram;
731 mem->flags = kvm_mem_flags(s, log_dirty, readonly_flag);
733 err = kvm_set_user_memory_region(s, mem);
734 if (err) {
735 fprintf(stderr, "%s: error registering prefix slot: %s\n",
736 __func__, strerror(-err));
737 #ifdef TARGET_PPC
738 fprintf(stderr, "%s: This is probably because your kernel's " \
739 "PAGE_SIZE is too big. Please try to use 4k " \
740 "PAGE_SIZE!\n", __func__);
741 #endif
742 abort();
746 /* register suffix slot */
747 if (old.start_addr + old.memory_size > start_addr + size) {
748 ram_addr_t size_delta;
750 mem = kvm_alloc_slot(s);
751 mem->start_addr = start_addr + size;
752 size_delta = mem->start_addr - old.start_addr;
753 mem->memory_size = old.memory_size - size_delta;
754 mem->ram = old.ram + size_delta;
755 mem->flags = kvm_mem_flags(s, log_dirty, readonly_flag);
757 err = kvm_set_user_memory_region(s, mem);
758 if (err) {
759 fprintf(stderr, "%s: error registering suffix slot: %s\n",
760 __func__, strerror(-err));
761 abort();
766 /* in case the KVM bug workaround already "consumed" the new slot */
767 if (!size) {
768 return;
770 if (!add) {
771 return;
773 mem = kvm_alloc_slot(s);
774 mem->memory_size = size;
775 mem->start_addr = start_addr;
776 mem->ram = ram;
777 mem->flags = kvm_mem_flags(s, log_dirty, readonly_flag);
779 err = kvm_set_user_memory_region(s, mem);
780 if (err) {
781 fprintf(stderr, "%s: error registering slot: %s\n", __func__,
782 strerror(-err));
783 abort();
787 static void kvm_region_add(MemoryListener *listener,
788 MemoryRegionSection *section)
790 kvm_set_phys_mem(section, true);
793 static void kvm_region_del(MemoryListener *listener,
794 MemoryRegionSection *section)
796 kvm_set_phys_mem(section, false);
799 static void kvm_log_sync(MemoryListener *listener,
800 MemoryRegionSection *section)
802 int r;
804 r = kvm_physical_sync_dirty_bitmap(section);
805 if (r < 0) {
806 abort();
810 static void kvm_log_global_start(struct MemoryListener *listener)
812 int r;
814 r = kvm_set_migration_log(1);
815 assert(r >= 0);
818 static void kvm_log_global_stop(struct MemoryListener *listener)
820 int r;
822 r = kvm_set_migration_log(0);
823 assert(r >= 0);
826 static void kvm_mem_ioeventfd_add(MemoryListener *listener,
827 MemoryRegionSection *section,
828 bool match_data, uint64_t data,
829 EventNotifier *e)
831 int fd = event_notifier_get_fd(e);
832 int r;
834 r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
835 data, true, section->size, match_data);
836 if (r < 0) {
837 abort();
841 static void kvm_mem_ioeventfd_del(MemoryListener *listener,
842 MemoryRegionSection *section,
843 bool match_data, uint64_t data,
844 EventNotifier *e)
846 int fd = event_notifier_get_fd(e);
847 int r;
849 r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
850 data, false, section->size, match_data);
851 if (r < 0) {
852 abort();
856 static void kvm_io_ioeventfd_add(MemoryListener *listener,
857 MemoryRegionSection *section,
858 bool match_data, uint64_t data,
859 EventNotifier *e)
861 int fd = event_notifier_get_fd(e);
862 int r;
864 r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space,
865 data, true, section->size, match_data);
866 if (r < 0) {
867 abort();
871 static void kvm_io_ioeventfd_del(MemoryListener *listener,
872 MemoryRegionSection *section,
873 bool match_data, uint64_t data,
874 EventNotifier *e)
877 int fd = event_notifier_get_fd(e);
878 int r;
880 r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space,
881 data, false, section->size, match_data);
882 if (r < 0) {
883 abort();
887 static MemoryListener kvm_memory_listener = {
888 .region_add = kvm_region_add,
889 .region_del = kvm_region_del,
890 .log_start = kvm_log_start,
891 .log_stop = kvm_log_stop,
892 .log_sync = kvm_log_sync,
893 .log_global_start = kvm_log_global_start,
894 .log_global_stop = kvm_log_global_stop,
895 .eventfd_add = kvm_mem_ioeventfd_add,
896 .eventfd_del = kvm_mem_ioeventfd_del,
897 .coalesced_mmio_add = kvm_coalesce_mmio_region,
898 .coalesced_mmio_del = kvm_uncoalesce_mmio_region,
899 .priority = 10,
902 static MemoryListener kvm_io_listener = {
903 .eventfd_add = kvm_io_ioeventfd_add,
904 .eventfd_del = kvm_io_ioeventfd_del,
905 .priority = 10,
908 static void kvm_handle_interrupt(CPUState *cpu, int mask)
910 cpu->interrupt_request |= mask;
912 if (!qemu_cpu_is_self(cpu)) {
913 qemu_cpu_kick(cpu);
917 int kvm_set_irq(KVMState *s, int irq, int level)
919 struct kvm_irq_level event;
920 int ret;
922 assert(kvm_async_interrupts_enabled());
924 event.level = level;
925 event.irq = irq;
926 ret = kvm_vm_ioctl(s, s->irq_set_ioctl, &event);
927 if (ret < 0) {
928 perror("kvm_set_irq");
929 abort();
932 return (s->irq_set_ioctl == KVM_IRQ_LINE) ? 1 : event.status;
935 #ifdef KVM_CAP_IRQ_ROUTING
936 typedef struct KVMMSIRoute {
937 struct kvm_irq_routing_entry kroute;
938 QTAILQ_ENTRY(KVMMSIRoute) entry;
939 } KVMMSIRoute;
941 static void set_gsi(KVMState *s, unsigned int gsi)
943 s->used_gsi_bitmap[gsi / 32] |= 1U << (gsi % 32);
946 static void clear_gsi(KVMState *s, unsigned int gsi)
948 s->used_gsi_bitmap[gsi / 32] &= ~(1U << (gsi % 32));
951 static void kvm_init_irq_routing(KVMState *s)
953 int gsi_count, i;
955 gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING);
956 if (gsi_count > 0) {
957 unsigned int gsi_bits, i;
959 /* Round up so we can search ints using ffs */
960 gsi_bits = ALIGN(gsi_count, 32);
961 s->used_gsi_bitmap = g_malloc0(gsi_bits / 8);
962 s->gsi_count = gsi_count;
964 /* Mark any over-allocated bits as already in use */
965 for (i = gsi_count; i < gsi_bits; i++) {
966 set_gsi(s, i);
970 s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
971 s->nr_allocated_irq_routes = 0;
973 if (!s->direct_msi) {
974 for (i = 0; i < KVM_MSI_HASHTAB_SIZE; i++) {
975 QTAILQ_INIT(&s->msi_hashtab[i]);
979 kvm_arch_init_irq_routing(s);
982 static void kvm_irqchip_commit_routes(KVMState *s)
984 int ret;
986 s->irq_routes->flags = 0;
987 ret = kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes);
988 assert(ret == 0);
991 static void kvm_add_routing_entry(KVMState *s,
992 struct kvm_irq_routing_entry *entry)
994 struct kvm_irq_routing_entry *new;
995 int n, size;
997 if (s->irq_routes->nr == s->nr_allocated_irq_routes) {
998 n = s->nr_allocated_irq_routes * 2;
999 if (n < 64) {
1000 n = 64;
1002 size = sizeof(struct kvm_irq_routing);
1003 size += n * sizeof(*new);
1004 s->irq_routes = g_realloc(s->irq_routes, size);
1005 s->nr_allocated_irq_routes = n;
1007 n = s->irq_routes->nr++;
1008 new = &s->irq_routes->entries[n];
1009 memset(new, 0, sizeof(*new));
1010 new->gsi = entry->gsi;
1011 new->type = entry->type;
1012 new->flags = entry->flags;
1013 new->u = entry->u;
1015 set_gsi(s, entry->gsi);
1017 kvm_irqchip_commit_routes(s);
1020 static int kvm_update_routing_entry(KVMState *s,
1021 struct kvm_irq_routing_entry *new_entry)
1023 struct kvm_irq_routing_entry *entry;
1024 int n;
1026 for (n = 0; n < s->irq_routes->nr; n++) {
1027 entry = &s->irq_routes->entries[n];
1028 if (entry->gsi != new_entry->gsi) {
1029 continue;
1032 entry->type = new_entry->type;
1033 entry->flags = new_entry->flags;
1034 entry->u = new_entry->u;
1036 kvm_irqchip_commit_routes(s);
1038 return 0;
1041 return -ESRCH;
1044 void kvm_irqchip_add_irq_route(KVMState *s, int irq, int irqchip, int pin)
1046 struct kvm_irq_routing_entry e;
1048 assert(pin < s->gsi_count);
1050 e.gsi = irq;
1051 e.type = KVM_IRQ_ROUTING_IRQCHIP;
1052 e.flags = 0;
1053 e.u.irqchip.irqchip = irqchip;
1054 e.u.irqchip.pin = pin;
1055 kvm_add_routing_entry(s, &e);
1058 void kvm_irqchip_release_virq(KVMState *s, int virq)
1060 struct kvm_irq_routing_entry *e;
1061 int i;
1063 for (i = 0; i < s->irq_routes->nr; i++) {
1064 e = &s->irq_routes->entries[i];
1065 if (e->gsi == virq) {
1066 s->irq_routes->nr--;
1067 *e = s->irq_routes->entries[s->irq_routes->nr];
1070 clear_gsi(s, virq);
1073 static unsigned int kvm_hash_msi(uint32_t data)
1075 /* This is optimized for IA32 MSI layout. However, no other arch shall
1076 * repeat the mistake of not providing a direct MSI injection API. */
1077 return data & 0xff;
1080 static void kvm_flush_dynamic_msi_routes(KVMState *s)
1082 KVMMSIRoute *route, *next;
1083 unsigned int hash;
1085 for (hash = 0; hash < KVM_MSI_HASHTAB_SIZE; hash++) {
1086 QTAILQ_FOREACH_SAFE(route, &s->msi_hashtab[hash], entry, next) {
1087 kvm_irqchip_release_virq(s, route->kroute.gsi);
1088 QTAILQ_REMOVE(&s->msi_hashtab[hash], route, entry);
1089 g_free(route);
1094 static int kvm_irqchip_get_virq(KVMState *s)
1096 uint32_t *word = s->used_gsi_bitmap;
1097 int max_words = ALIGN(s->gsi_count, 32) / 32;
1098 int i, bit;
1099 bool retry = true;
1101 again:
1102 /* Return the lowest unused GSI in the bitmap */
1103 for (i = 0; i < max_words; i++) {
1104 bit = ffs(~word[i]);
1105 if (!bit) {
1106 continue;
1109 return bit - 1 + i * 32;
1111 if (!s->direct_msi && retry) {
1112 retry = false;
1113 kvm_flush_dynamic_msi_routes(s);
1114 goto again;
1116 return -ENOSPC;
1120 static KVMMSIRoute *kvm_lookup_msi_route(KVMState *s, MSIMessage msg)
1122 unsigned int hash = kvm_hash_msi(msg.data);
1123 KVMMSIRoute *route;
1125 QTAILQ_FOREACH(route, &s->msi_hashtab[hash], entry) {
1126 if (route->kroute.u.msi.address_lo == (uint32_t)msg.address &&
1127 route->kroute.u.msi.address_hi == (msg.address >> 32) &&
1128 route->kroute.u.msi.data == msg.data) {
1129 return route;
1132 return NULL;
1135 int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
1137 struct kvm_msi msi;
1138 KVMMSIRoute *route;
1140 if (s->direct_msi) {
1141 msi.address_lo = (uint32_t)msg.address;
1142 msi.address_hi = msg.address >> 32;
1143 msi.data = msg.data;
1144 msi.flags = 0;
1145 memset(msi.pad, 0, sizeof(msi.pad));
1147 return kvm_vm_ioctl(s, KVM_SIGNAL_MSI, &msi);
1150 route = kvm_lookup_msi_route(s, msg);
1151 if (!route) {
1152 int virq;
1154 virq = kvm_irqchip_get_virq(s);
1155 if (virq < 0) {
1156 return virq;
1159 route = g_malloc(sizeof(KVMMSIRoute));
1160 route->kroute.gsi = virq;
1161 route->kroute.type = KVM_IRQ_ROUTING_MSI;
1162 route->kroute.flags = 0;
1163 route->kroute.u.msi.address_lo = (uint32_t)msg.address;
1164 route->kroute.u.msi.address_hi = msg.address >> 32;
1165 route->kroute.u.msi.data = msg.data;
1167 kvm_add_routing_entry(s, &route->kroute);
1169 QTAILQ_INSERT_TAIL(&s->msi_hashtab[kvm_hash_msi(msg.data)], route,
1170 entry);
1173 assert(route->kroute.type == KVM_IRQ_ROUTING_MSI);
1175 return kvm_set_irq(s, route->kroute.gsi, 1);
1178 int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
1180 struct kvm_irq_routing_entry kroute;
1181 int virq;
1183 if (!kvm_gsi_routing_enabled()) {
1184 return -ENOSYS;
1187 virq = kvm_irqchip_get_virq(s);
1188 if (virq < 0) {
1189 return virq;
1192 kroute.gsi = virq;
1193 kroute.type = KVM_IRQ_ROUTING_MSI;
1194 kroute.flags = 0;
1195 kroute.u.msi.address_lo = (uint32_t)msg.address;
1196 kroute.u.msi.address_hi = msg.address >> 32;
1197 kroute.u.msi.data = msg.data;
1199 kvm_add_routing_entry(s, &kroute);
1201 return virq;
1204 int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
1206 struct kvm_irq_routing_entry kroute;
1208 if (!kvm_irqchip_in_kernel()) {
1209 return -ENOSYS;
1212 kroute.gsi = virq;
1213 kroute.type = KVM_IRQ_ROUTING_MSI;
1214 kroute.flags = 0;
1215 kroute.u.msi.address_lo = (uint32_t)msg.address;
1216 kroute.u.msi.address_hi = msg.address >> 32;
1217 kroute.u.msi.data = msg.data;
1219 return kvm_update_routing_entry(s, &kroute);
1222 static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
1224 struct kvm_irqfd irqfd = {
1225 .fd = fd,
1226 .gsi = virq,
1227 .flags = assign ? 0 : KVM_IRQFD_FLAG_DEASSIGN,
1230 if (!kvm_irqfds_enabled()) {
1231 return -ENOSYS;
1234 return kvm_vm_ioctl(s, KVM_IRQFD, &irqfd);
1237 #else /* !KVM_CAP_IRQ_ROUTING */
1239 static void kvm_init_irq_routing(KVMState *s)
1243 void kvm_irqchip_release_virq(KVMState *s, int virq)
1247 int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
1249 abort();
1252 int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
1254 return -ENOSYS;
1257 static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
1259 abort();
1262 int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
1264 return -ENOSYS;
1266 #endif /* !KVM_CAP_IRQ_ROUTING */
1268 int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n, int virq)
1270 return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n), virq, true);
1273 int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n, int virq)
1275 return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n), virq, false);
1278 static int kvm_irqchip_create(KVMState *s)
1280 QemuOptsList *list = qemu_find_opts("machine");
1281 int ret;
1283 if (QTAILQ_EMPTY(&list->head) ||
1284 !qemu_opt_get_bool(QTAILQ_FIRST(&list->head),
1285 "kernel_irqchip", true) ||
1286 !kvm_check_extension(s, KVM_CAP_IRQCHIP)) {
1287 return 0;
1290 ret = kvm_vm_ioctl(s, KVM_CREATE_IRQCHIP);
1291 if (ret < 0) {
1292 fprintf(stderr, "Create kernel irqchip failed\n");
1293 return ret;
1296 kvm_kernel_irqchip = true;
1297 /* If we have an in-kernel IRQ chip then we must have asynchronous
1298 * interrupt delivery (though the reverse is not necessarily true)
1300 kvm_async_interrupts_allowed = true;
1302 kvm_init_irq_routing(s);
1304 return 0;
1307 static int kvm_max_vcpus(KVMState *s)
1309 int ret;
1311 /* Find number of supported CPUs using the recommended
1312 * procedure from the kernel API documentation to cope with
1313 * older kernels that may be missing capabilities.
1315 ret = kvm_check_extension(s, KVM_CAP_MAX_VCPUS);
1316 if (ret) {
1317 return ret;
1319 ret = kvm_check_extension(s, KVM_CAP_NR_VCPUS);
1320 if (ret) {
1321 return ret;
1324 return 4;
1327 int kvm_init(void)
1329 static const char upgrade_note[] =
1330 "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
1331 "(see http://sourceforge.net/projects/kvm).\n";
1332 KVMState *s;
1333 const KVMCapabilityInfo *missing_cap;
1334 int ret;
1335 int i;
1336 int max_vcpus;
1338 s = g_malloc0(sizeof(KVMState));
1341 * On systems where the kernel can support different base page
1342 * sizes, host page size may be different from TARGET_PAGE_SIZE,
1343 * even with KVM. TARGET_PAGE_SIZE is assumed to be the minimum
1344 * page size for the system though.
1346 assert(TARGET_PAGE_SIZE <= getpagesize());
1348 #ifdef KVM_CAP_SET_GUEST_DEBUG
1349 QTAILQ_INIT(&s->kvm_sw_breakpoints);
1350 #endif
1351 for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
1352 s->slots[i].slot = i;
1354 s->vmfd = -1;
1355 s->fd = qemu_open("/dev/kvm", O_RDWR);
1356 if (s->fd == -1) {
1357 fprintf(stderr, "Could not access KVM kernel module: %m\n");
1358 ret = -errno;
1359 goto err;
1362 ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
1363 if (ret < KVM_API_VERSION) {
1364 if (ret > 0) {
1365 ret = -EINVAL;
1367 fprintf(stderr, "kvm version too old\n");
1368 goto err;
1371 if (ret > KVM_API_VERSION) {
1372 ret = -EINVAL;
1373 fprintf(stderr, "kvm version not supported\n");
1374 goto err;
1377 max_vcpus = kvm_max_vcpus(s);
1378 if (smp_cpus > max_vcpus) {
1379 ret = -EINVAL;
1380 fprintf(stderr, "Number of SMP cpus requested (%d) exceeds max cpus "
1381 "supported by KVM (%d)\n", smp_cpus, max_vcpus);
1382 goto err;
1385 s->vmfd = kvm_ioctl(s, KVM_CREATE_VM, 0);
1386 if (s->vmfd < 0) {
1387 #ifdef TARGET_S390X
1388 fprintf(stderr, "Please add the 'switch_amode' kernel parameter to "
1389 "your host kernel command line\n");
1390 #endif
1391 ret = s->vmfd;
1392 goto err;
1395 missing_cap = kvm_check_extension_list(s, kvm_required_capabilites);
1396 if (!missing_cap) {
1397 missing_cap =
1398 kvm_check_extension_list(s, kvm_arch_required_capabilities);
1400 if (missing_cap) {
1401 ret = -EINVAL;
1402 fprintf(stderr, "kvm does not support %s\n%s",
1403 missing_cap->name, upgrade_note);
1404 goto err;
1407 s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
1409 s->broken_set_mem_region = 1;
1410 ret = kvm_check_extension(s, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
1411 if (ret > 0) {
1412 s->broken_set_mem_region = 0;
1415 #ifdef KVM_CAP_VCPU_EVENTS
1416 s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
1417 #endif
1419 s->robust_singlestep =
1420 kvm_check_extension(s, KVM_CAP_X86_ROBUST_SINGLESTEP);
1422 #ifdef KVM_CAP_DEBUGREGS
1423 s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS);
1424 #endif
1426 #ifdef KVM_CAP_XSAVE
1427 s->xsave = kvm_check_extension(s, KVM_CAP_XSAVE);
1428 #endif
1430 #ifdef KVM_CAP_XCRS
1431 s->xcrs = kvm_check_extension(s, KVM_CAP_XCRS);
1432 #endif
1434 #ifdef KVM_CAP_PIT_STATE2
1435 s->pit_state2 = kvm_check_extension(s, KVM_CAP_PIT_STATE2);
1436 #endif
1438 #ifdef KVM_CAP_IRQ_ROUTING
1439 s->direct_msi = (kvm_check_extension(s, KVM_CAP_SIGNAL_MSI) > 0);
1440 #endif
1442 s->intx_set_mask = kvm_check_extension(s, KVM_CAP_PCI_2_3);
1444 s->irq_set_ioctl = KVM_IRQ_LINE;
1445 if (kvm_check_extension(s, KVM_CAP_IRQ_INJECT_STATUS)) {
1446 s->irq_set_ioctl = KVM_IRQ_LINE_STATUS;
1449 #ifdef KVM_CAP_READONLY_MEM
1450 kvm_readonly_mem_allowed =
1451 (kvm_check_extension(s, KVM_CAP_READONLY_MEM) > 0);
1452 #endif
1454 ret = kvm_arch_init(s);
1455 if (ret < 0) {
1456 goto err;
1459 ret = kvm_irqchip_create(s);
1460 if (ret < 0) {
1461 goto err;
1464 kvm_state = s;
1465 memory_listener_register(&kvm_memory_listener, &address_space_memory);
1466 memory_listener_register(&kvm_io_listener, &address_space_io);
1468 s->many_ioeventfds = kvm_check_many_ioeventfds();
1470 cpu_interrupt_handler = kvm_handle_interrupt;
1472 return 0;
1474 err:
1475 if (s->vmfd >= 0) {
1476 close(s->vmfd);
1478 if (s->fd != -1) {
1479 close(s->fd);
1481 g_free(s);
1483 return ret;
1486 static void kvm_handle_io(uint16_t port, void *data, int direction, int size,
1487 uint32_t count)
1489 int i;
1490 uint8_t *ptr = data;
1492 for (i = 0; i < count; i++) {
1493 if (direction == KVM_EXIT_IO_IN) {
1494 switch (size) {
1495 case 1:
1496 stb_p(ptr, cpu_inb(port));
1497 break;
1498 case 2:
1499 stw_p(ptr, cpu_inw(port));
1500 break;
1501 case 4:
1502 stl_p(ptr, cpu_inl(port));
1503 break;
1505 } else {
1506 switch (size) {
1507 case 1:
1508 cpu_outb(port, ldub_p(ptr));
1509 break;
1510 case 2:
1511 cpu_outw(port, lduw_p(ptr));
1512 break;
1513 case 4:
1514 cpu_outl(port, ldl_p(ptr));
1515 break;
1519 ptr += size;
1523 static int kvm_handle_internal_error(CPUArchState *env, struct kvm_run *run)
1525 CPUState *cpu = ENV_GET_CPU(env);
1527 fprintf(stderr, "KVM internal error.");
1528 if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) {
1529 int i;
1531 fprintf(stderr, " Suberror: %d\n", run->internal.suberror);
1532 for (i = 0; i < run->internal.ndata; ++i) {
1533 fprintf(stderr, "extra data[%d]: %"PRIx64"\n",
1534 i, (uint64_t)run->internal.data[i]);
1536 } else {
1537 fprintf(stderr, "\n");
1539 if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) {
1540 fprintf(stderr, "emulation failure\n");
1541 if (!kvm_arch_stop_on_emulation_error(cpu)) {
1542 cpu_dump_state(env, stderr, fprintf, CPU_DUMP_CODE);
1543 return EXCP_INTERRUPT;
1546 /* FIXME: Should trigger a qmp message to let management know
1547 * something went wrong.
1549 return -1;
1552 void kvm_flush_coalesced_mmio_buffer(void)
1554 KVMState *s = kvm_state;
1556 if (s->coalesced_flush_in_progress) {
1557 return;
1560 s->coalesced_flush_in_progress = true;
1562 if (s->coalesced_mmio_ring) {
1563 struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
1564 while (ring->first != ring->last) {
1565 struct kvm_coalesced_mmio *ent;
1567 ent = &ring->coalesced_mmio[ring->first];
1569 cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
1570 smp_wmb();
1571 ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
1575 s->coalesced_flush_in_progress = false;
1578 static void do_kvm_cpu_synchronize_state(void *arg)
1580 CPUState *cpu = arg;
1582 if (!cpu->kvm_vcpu_dirty) {
1583 kvm_arch_get_registers(cpu);
1584 cpu->kvm_vcpu_dirty = true;
1588 void kvm_cpu_synchronize_state(CPUArchState *env)
1590 CPUState *cpu = ENV_GET_CPU(env);
1592 if (!cpu->kvm_vcpu_dirty) {
1593 run_on_cpu(cpu, do_kvm_cpu_synchronize_state, cpu);
1597 void kvm_cpu_synchronize_post_reset(CPUState *cpu)
1599 kvm_arch_put_registers(cpu, KVM_PUT_RESET_STATE);
1600 cpu->kvm_vcpu_dirty = false;
1603 void kvm_cpu_synchronize_post_init(CPUState *cpu)
1605 kvm_arch_put_registers(cpu, KVM_PUT_FULL_STATE);
1606 cpu->kvm_vcpu_dirty = false;
1609 int kvm_cpu_exec(CPUArchState *env)
1611 CPUState *cpu = ENV_GET_CPU(env);
1612 struct kvm_run *run = cpu->kvm_run;
1613 int ret, run_ret;
1615 DPRINTF("kvm_cpu_exec()\n");
1617 if (kvm_arch_process_async_events(cpu)) {
1618 cpu->exit_request = 0;
1619 return EXCP_HLT;
1622 do {
1623 if (cpu->kvm_vcpu_dirty) {
1624 kvm_arch_put_registers(cpu, KVM_PUT_RUNTIME_STATE);
1625 cpu->kvm_vcpu_dirty = false;
1628 kvm_arch_pre_run(cpu, run);
1629 if (cpu->exit_request) {
1630 DPRINTF("interrupt exit requested\n");
1632 * KVM requires us to reenter the kernel after IO exits to complete
1633 * instruction emulation. This self-signal will ensure that we
1634 * leave ASAP again.
1636 qemu_cpu_kick_self();
1638 qemu_mutex_unlock_iothread();
1640 run_ret = kvm_vcpu_ioctl(cpu, KVM_RUN, 0);
1642 qemu_mutex_lock_iothread();
1643 kvm_arch_post_run(cpu, run);
1645 if (run_ret < 0) {
1646 if (run_ret == -EINTR || run_ret == -EAGAIN) {
1647 DPRINTF("io window exit\n");
1648 ret = EXCP_INTERRUPT;
1649 break;
1651 fprintf(stderr, "error: kvm run failed %s\n",
1652 strerror(-run_ret));
1653 abort();
1656 trace_kvm_run_exit(cpu->cpu_index, run->exit_reason);
1657 switch (run->exit_reason) {
1658 case KVM_EXIT_IO:
1659 DPRINTF("handle_io\n");
1660 kvm_handle_io(run->io.port,
1661 (uint8_t *)run + run->io.data_offset,
1662 run->io.direction,
1663 run->io.size,
1664 run->io.count);
1665 ret = 0;
1666 break;
1667 case KVM_EXIT_MMIO:
1668 DPRINTF("handle_mmio\n");
1669 cpu_physical_memory_rw(run->mmio.phys_addr,
1670 run->mmio.data,
1671 run->mmio.len,
1672 run->mmio.is_write);
1673 ret = 0;
1674 break;
1675 case KVM_EXIT_IRQ_WINDOW_OPEN:
1676 DPRINTF("irq_window_open\n");
1677 ret = EXCP_INTERRUPT;
1678 break;
1679 case KVM_EXIT_SHUTDOWN:
1680 DPRINTF("shutdown\n");
1681 qemu_system_reset_request();
1682 ret = EXCP_INTERRUPT;
1683 break;
1684 case KVM_EXIT_UNKNOWN:
1685 fprintf(stderr, "KVM: unknown exit, hardware reason %" PRIx64 "\n",
1686 (uint64_t)run->hw.hardware_exit_reason);
1687 ret = -1;
1688 break;
1689 case KVM_EXIT_INTERNAL_ERROR:
1690 ret = kvm_handle_internal_error(env, run);
1691 break;
1692 default:
1693 DPRINTF("kvm_arch_handle_exit\n");
1694 ret = kvm_arch_handle_exit(cpu, run);
1695 break;
1697 } while (ret == 0);
1699 if (ret < 0) {
1700 cpu_dump_state(env, stderr, fprintf, CPU_DUMP_CODE);
1701 vm_stop(RUN_STATE_INTERNAL_ERROR);
1704 cpu->exit_request = 0;
1705 return ret;
1708 int kvm_ioctl(KVMState *s, int type, ...)
1710 int ret;
1711 void *arg;
1712 va_list ap;
1714 va_start(ap, type);
1715 arg = va_arg(ap, void *);
1716 va_end(ap);
1718 trace_kvm_ioctl(type, arg);
1719 ret = ioctl(s->fd, type, arg);
1720 if (ret == -1) {
1721 ret = -errno;
1723 return ret;
1726 int kvm_vm_ioctl(KVMState *s, int type, ...)
1728 int ret;
1729 void *arg;
1730 va_list ap;
1732 va_start(ap, type);
1733 arg = va_arg(ap, void *);
1734 va_end(ap);
1736 trace_kvm_vm_ioctl(type, arg);
1737 ret = ioctl(s->vmfd, type, arg);
1738 if (ret == -1) {
1739 ret = -errno;
1741 return ret;
1744 int kvm_vcpu_ioctl(CPUState *cpu, int type, ...)
1746 int ret;
1747 void *arg;
1748 va_list ap;
1750 va_start(ap, type);
1751 arg = va_arg(ap, void *);
1752 va_end(ap);
1754 trace_kvm_vcpu_ioctl(cpu->cpu_index, type, arg);
1755 ret = ioctl(cpu->kvm_fd, type, arg);
1756 if (ret == -1) {
1757 ret = -errno;
1759 return ret;
1762 int kvm_has_sync_mmu(void)
1764 return kvm_check_extension(kvm_state, KVM_CAP_SYNC_MMU);
1767 int kvm_has_vcpu_events(void)
1769 return kvm_state->vcpu_events;
1772 int kvm_has_robust_singlestep(void)
1774 return kvm_state->robust_singlestep;
1777 int kvm_has_debugregs(void)
1779 return kvm_state->debugregs;
1782 int kvm_has_xsave(void)
1784 return kvm_state->xsave;
1787 int kvm_has_xcrs(void)
1789 return kvm_state->xcrs;
1792 int kvm_has_pit_state2(void)
1794 return kvm_state->pit_state2;
1797 int kvm_has_many_ioeventfds(void)
1799 if (!kvm_enabled()) {
1800 return 0;
1802 return kvm_state->many_ioeventfds;
1805 int kvm_has_gsi_routing(void)
1807 #ifdef KVM_CAP_IRQ_ROUTING
1808 return kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING);
1809 #else
1810 return false;
1811 #endif
1814 int kvm_has_intx_set_mask(void)
1816 return kvm_state->intx_set_mask;
1819 void *kvm_ram_alloc(ram_addr_t size)
1821 #ifdef TARGET_S390X
1822 void *mem;
1824 mem = kvm_arch_ram_alloc(size);
1825 if (mem) {
1826 return mem;
1828 #endif
1829 return qemu_anon_ram_alloc(size);
1832 void kvm_setup_guest_memory(void *start, size_t size)
1834 #ifdef CONFIG_VALGRIND_H
1835 VALGRIND_MAKE_MEM_DEFINED(start, size);
1836 #endif
1837 if (!kvm_has_sync_mmu()) {
1838 int ret = qemu_madvise(start, size, QEMU_MADV_DONTFORK);
1840 if (ret) {
1841 perror("qemu_madvise");
1842 fprintf(stderr,
1843 "Need MADV_DONTFORK in absence of synchronous KVM MMU\n");
1844 exit(1);
1849 #ifdef KVM_CAP_SET_GUEST_DEBUG
1850 struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *cpu,
1851 target_ulong pc)
1853 struct kvm_sw_breakpoint *bp;
1855 QTAILQ_FOREACH(bp, &cpu->kvm_state->kvm_sw_breakpoints, entry) {
1856 if (bp->pc == pc) {
1857 return bp;
1860 return NULL;
1863 int kvm_sw_breakpoints_active(CPUState *cpu)
1865 return !QTAILQ_EMPTY(&cpu->kvm_state->kvm_sw_breakpoints);
1868 struct kvm_set_guest_debug_data {
1869 struct kvm_guest_debug dbg;
1870 CPUState *cpu;
1871 int err;
1874 static void kvm_invoke_set_guest_debug(void *data)
1876 struct kvm_set_guest_debug_data *dbg_data = data;
1878 dbg_data->err = kvm_vcpu_ioctl(dbg_data->cpu, KVM_SET_GUEST_DEBUG,
1879 &dbg_data->dbg);
1882 int kvm_update_guest_debug(CPUArchState *env, unsigned long reinject_trap)
1884 CPUState *cpu = ENV_GET_CPU(env);
1885 struct kvm_set_guest_debug_data data;
1887 data.dbg.control = reinject_trap;
1889 if (env->singlestep_enabled) {
1890 data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
1892 kvm_arch_update_guest_debug(cpu, &data.dbg);
1893 data.cpu = cpu;
1895 run_on_cpu(cpu, kvm_invoke_set_guest_debug, &data);
1896 return data.err;
1899 int kvm_insert_breakpoint(CPUArchState *current_env, target_ulong addr,
1900 target_ulong len, int type)
1902 CPUState *current_cpu = ENV_GET_CPU(current_env);
1903 struct kvm_sw_breakpoint *bp;
1904 CPUArchState *env;
1905 int err;
1907 if (type == GDB_BREAKPOINT_SW) {
1908 bp = kvm_find_sw_breakpoint(current_cpu, addr);
1909 if (bp) {
1910 bp->use_count++;
1911 return 0;
1914 bp = g_malloc(sizeof(struct kvm_sw_breakpoint));
1915 if (!bp) {
1916 return -ENOMEM;
1919 bp->pc = addr;
1920 bp->use_count = 1;
1921 err = kvm_arch_insert_sw_breakpoint(current_cpu, bp);
1922 if (err) {
1923 g_free(bp);
1924 return err;
1927 QTAILQ_INSERT_HEAD(&current_cpu->kvm_state->kvm_sw_breakpoints,
1928 bp, entry);
1929 } else {
1930 err = kvm_arch_insert_hw_breakpoint(addr, len, type);
1931 if (err) {
1932 return err;
1936 for (env = first_cpu; env != NULL; env = env->next_cpu) {
1937 err = kvm_update_guest_debug(env, 0);
1938 if (err) {
1939 return err;
1942 return 0;
1945 int kvm_remove_breakpoint(CPUArchState *current_env, target_ulong addr,
1946 target_ulong len, int type)
1948 CPUState *current_cpu = ENV_GET_CPU(current_env);
1949 struct kvm_sw_breakpoint *bp;
1950 CPUArchState *env;
1951 int err;
1953 if (type == GDB_BREAKPOINT_SW) {
1954 bp = kvm_find_sw_breakpoint(current_cpu, addr);
1955 if (!bp) {
1956 return -ENOENT;
1959 if (bp->use_count > 1) {
1960 bp->use_count--;
1961 return 0;
1964 err = kvm_arch_remove_sw_breakpoint(current_cpu, bp);
1965 if (err) {
1966 return err;
1969 QTAILQ_REMOVE(&current_cpu->kvm_state->kvm_sw_breakpoints, bp, entry);
1970 g_free(bp);
1971 } else {
1972 err = kvm_arch_remove_hw_breakpoint(addr, len, type);
1973 if (err) {
1974 return err;
1978 for (env = first_cpu; env != NULL; env = env->next_cpu) {
1979 err = kvm_update_guest_debug(env, 0);
1980 if (err) {
1981 return err;
1984 return 0;
1987 void kvm_remove_all_breakpoints(CPUArchState *current_env)
1989 CPUState *current_cpu = ENV_GET_CPU(current_env);
1990 struct kvm_sw_breakpoint *bp, *next;
1991 KVMState *s = current_cpu->kvm_state;
1992 CPUArchState *env;
1993 CPUState *cpu;
1995 QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
1996 if (kvm_arch_remove_sw_breakpoint(current_cpu, bp) != 0) {
1997 /* Try harder to find a CPU that currently sees the breakpoint. */
1998 for (env = first_cpu; env != NULL; env = env->next_cpu) {
1999 cpu = ENV_GET_CPU(env);
2000 if (kvm_arch_remove_sw_breakpoint(cpu, bp) == 0) {
2001 break;
2005 QTAILQ_REMOVE(&s->kvm_sw_breakpoints, bp, entry);
2006 g_free(bp);
2008 kvm_arch_remove_all_hw_breakpoints();
2010 for (env = first_cpu; env != NULL; env = env->next_cpu) {
2011 kvm_update_guest_debug(env, 0);
2015 #else /* !KVM_CAP_SET_GUEST_DEBUG */
2017 int kvm_update_guest_debug(CPUArchState *env, unsigned long reinject_trap)
2019 return -EINVAL;
2022 int kvm_insert_breakpoint(CPUArchState *current_env, target_ulong addr,
2023 target_ulong len, int type)
2025 return -EINVAL;
2028 int kvm_remove_breakpoint(CPUArchState *current_env, target_ulong addr,
2029 target_ulong len, int type)
2031 return -EINVAL;
2034 void kvm_remove_all_breakpoints(CPUArchState *current_env)
2037 #endif /* !KVM_CAP_SET_GUEST_DEBUG */
2039 int kvm_set_signal_mask(CPUArchState *env, const sigset_t *sigset)
2041 CPUState *cpu = ENV_GET_CPU(env);
2042 struct kvm_signal_mask *sigmask;
2043 int r;
2045 if (!sigset) {
2046 return kvm_vcpu_ioctl(cpu, KVM_SET_SIGNAL_MASK, NULL);
2049 sigmask = g_malloc(sizeof(*sigmask) + sizeof(*sigset));
2051 sigmask->len = 8;
2052 memcpy(sigmask->sigset, sigset, sizeof(*sigset));
2053 r = kvm_vcpu_ioctl(cpu, KVM_SET_SIGNAL_MASK, sigmask);
2054 g_free(sigmask);
2056 return r;
2058 int kvm_on_sigbus_vcpu(CPUState *cpu, int code, void *addr)
2060 return kvm_arch_on_sigbus_vcpu(cpu, code, addr);
2063 int kvm_on_sigbus(int code, void *addr)
2065 return kvm_arch_on_sigbus(code, addr);