QCOW2: bug fix - read base image beyond its size
[qemu/cris-port.git] / kvm-all.c
blob2ec9e0980da816939bf7c8a897325e48ccae3777
1 /*
2 * QEMU KVM support
4 * Copyright IBM, Corp. 2008
5 * Red Hat, Inc. 2008
7 * Authors:
8 * Anthony Liguori <aliguori@us.ibm.com>
9 * Glauber Costa <gcosta@redhat.com>
11 * This work is licensed under the terms of the GNU GPL, version 2 or later.
12 * See the COPYING file in the top-level directory.
16 #include <sys/types.h>
17 #include <sys/ioctl.h>
18 #include <sys/mman.h>
19 #include <stdarg.h>
21 #include <linux/kvm.h>
23 #include "qemu-common.h"
24 #include "qemu-barrier.h"
25 #include "sysemu.h"
26 #include "hw/hw.h"
27 #include "gdbstub.h"
28 #include "kvm.h"
29 #include "bswap.h"
31 /* This check must be after config-host.h is included */
32 #ifdef CONFIG_EVENTFD
33 #include <sys/eventfd.h>
34 #endif
36 /* KVM uses PAGE_SIZE in it's definition of COALESCED_MMIO_MAX */
37 #define PAGE_SIZE TARGET_PAGE_SIZE
39 //#define DEBUG_KVM
41 #ifdef DEBUG_KVM
42 #define DPRINTF(fmt, ...) \
43 do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
44 #else
45 #define DPRINTF(fmt, ...) \
46 do { } while (0)
47 #endif
49 typedef struct KVMSlot
51 target_phys_addr_t start_addr;
52 ram_addr_t memory_size;
53 ram_addr_t phys_offset;
54 int slot;
55 int flags;
56 } KVMSlot;
58 typedef struct kvm_dirty_log KVMDirtyLog;
60 struct KVMState
62 KVMSlot slots[32];
63 int fd;
64 int vmfd;
65 int coalesced_mmio;
66 struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
67 int broken_set_mem_region;
68 int migration_log;
69 int vcpu_events;
70 int robust_singlestep;
71 int debugregs;
72 #ifdef KVM_CAP_SET_GUEST_DEBUG
73 struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
74 #endif
75 int irqchip_in_kernel;
76 int pit_in_kernel;
77 int xsave, xcrs;
78 int many_ioeventfds;
81 static KVMState *kvm_state;
83 static const KVMCapabilityInfo kvm_required_capabilites[] = {
84 KVM_CAP_INFO(USER_MEMORY),
85 KVM_CAP_INFO(DESTROY_MEMORY_REGION_WORKS),
86 KVM_CAP_LAST_INFO
89 static KVMSlot *kvm_alloc_slot(KVMState *s)
91 int i;
93 for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
94 /* KVM private memory slots */
95 if (i >= 8 && i < 12) {
96 continue;
98 if (s->slots[i].memory_size == 0) {
99 return &s->slots[i];
103 fprintf(stderr, "%s: no free slot available\n", __func__);
104 abort();
107 static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
108 target_phys_addr_t start_addr,
109 target_phys_addr_t end_addr)
111 int i;
113 for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
114 KVMSlot *mem = &s->slots[i];
116 if (start_addr == mem->start_addr &&
117 end_addr == mem->start_addr + mem->memory_size) {
118 return mem;
122 return NULL;
126 * Find overlapping slot with lowest start address
128 static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
129 target_phys_addr_t start_addr,
130 target_phys_addr_t end_addr)
132 KVMSlot *found = NULL;
133 int i;
135 for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
136 KVMSlot *mem = &s->slots[i];
138 if (mem->memory_size == 0 ||
139 (found && found->start_addr < mem->start_addr)) {
140 continue;
143 if (end_addr > mem->start_addr &&
144 start_addr < mem->start_addr + mem->memory_size) {
145 found = mem;
149 return found;
152 int kvm_physical_memory_addr_from_ram(KVMState *s, ram_addr_t ram_addr,
153 target_phys_addr_t *phys_addr)
155 int i;
157 for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
158 KVMSlot *mem = &s->slots[i];
160 if (ram_addr >= mem->phys_offset &&
161 ram_addr < mem->phys_offset + mem->memory_size) {
162 *phys_addr = mem->start_addr + (ram_addr - mem->phys_offset);
163 return 1;
167 return 0;
170 static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
172 struct kvm_userspace_memory_region mem;
174 mem.slot = slot->slot;
175 mem.guest_phys_addr = slot->start_addr;
176 mem.memory_size = slot->memory_size;
177 mem.userspace_addr = (unsigned long)qemu_safe_ram_ptr(slot->phys_offset);
178 mem.flags = slot->flags;
179 if (s->migration_log) {
180 mem.flags |= KVM_MEM_LOG_DIRTY_PAGES;
182 return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
185 static void kvm_reset_vcpu(void *opaque)
187 CPUState *env = opaque;
189 kvm_arch_reset_vcpu(env);
192 int kvm_irqchip_in_kernel(void)
194 return kvm_state->irqchip_in_kernel;
197 int kvm_pit_in_kernel(void)
199 return kvm_state->pit_in_kernel;
203 int kvm_init_vcpu(CPUState *env)
205 KVMState *s = kvm_state;
206 long mmap_size;
207 int ret;
209 DPRINTF("kvm_init_vcpu\n");
211 ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, env->cpu_index);
212 if (ret < 0) {
213 DPRINTF("kvm_create_vcpu failed\n");
214 goto err;
217 env->kvm_fd = ret;
218 env->kvm_state = s;
220 mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
221 if (mmap_size < 0) {
222 DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
223 goto err;
226 env->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
227 env->kvm_fd, 0);
228 if (env->kvm_run == MAP_FAILED) {
229 ret = -errno;
230 DPRINTF("mmap'ing vcpu state failed\n");
231 goto err;
234 if (s->coalesced_mmio && !s->coalesced_mmio_ring) {
235 s->coalesced_mmio_ring =
236 (void *)env->kvm_run + s->coalesced_mmio * PAGE_SIZE;
239 ret = kvm_arch_init_vcpu(env);
240 if (ret == 0) {
241 qemu_register_reset(kvm_reset_vcpu, env);
242 kvm_arch_reset_vcpu(env);
244 err:
245 return ret;
249 * dirty pages logging control
251 static int kvm_dirty_pages_log_change(target_phys_addr_t phys_addr,
252 ram_addr_t size, int flags, int mask)
254 KVMState *s = kvm_state;
255 KVMSlot *mem = kvm_lookup_matching_slot(s, phys_addr, phys_addr + size);
256 int old_flags;
258 if (mem == NULL) {
259 fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
260 TARGET_FMT_plx "\n", __func__, phys_addr,
261 (target_phys_addr_t)(phys_addr + size - 1));
262 return -EINVAL;
265 old_flags = mem->flags;
267 flags = (mem->flags & ~mask) | flags;
268 mem->flags = flags;
270 /* If nothing changed effectively, no need to issue ioctl */
271 if (s->migration_log) {
272 flags |= KVM_MEM_LOG_DIRTY_PAGES;
274 if (flags == old_flags) {
275 return 0;
278 return kvm_set_user_memory_region(s, mem);
281 int kvm_log_start(target_phys_addr_t phys_addr, ram_addr_t size)
283 return kvm_dirty_pages_log_change(phys_addr, size, KVM_MEM_LOG_DIRTY_PAGES,
284 KVM_MEM_LOG_DIRTY_PAGES);
287 int kvm_log_stop(target_phys_addr_t phys_addr, ram_addr_t size)
289 return kvm_dirty_pages_log_change(phys_addr, size, 0,
290 KVM_MEM_LOG_DIRTY_PAGES);
293 static int kvm_set_migration_log(int enable)
295 KVMState *s = kvm_state;
296 KVMSlot *mem;
297 int i, err;
299 s->migration_log = enable;
301 for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
302 mem = &s->slots[i];
304 if (!mem->memory_size) {
305 continue;
307 if (!!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) == enable) {
308 continue;
310 err = kvm_set_user_memory_region(s, mem);
311 if (err) {
312 return err;
315 return 0;
318 /* get kvm's dirty pages bitmap and update qemu's */
319 static int kvm_get_dirty_pages_log_range(unsigned long start_addr,
320 unsigned long *bitmap,
321 unsigned long offset,
322 unsigned long mem_size)
324 unsigned int i, j;
325 unsigned long page_number, addr, addr1, c;
326 ram_addr_t ram_addr;
327 unsigned int len = ((mem_size / TARGET_PAGE_SIZE) + HOST_LONG_BITS - 1) /
328 HOST_LONG_BITS;
331 * bitmap-traveling is faster than memory-traveling (for addr...)
332 * especially when most of the memory is not dirty.
334 for (i = 0; i < len; i++) {
335 if (bitmap[i] != 0) {
336 c = leul_to_cpu(bitmap[i]);
337 do {
338 j = ffsl(c) - 1;
339 c &= ~(1ul << j);
340 page_number = i * HOST_LONG_BITS + j;
341 addr1 = page_number * TARGET_PAGE_SIZE;
342 addr = offset + addr1;
343 ram_addr = cpu_get_physical_page_desc(addr);
344 cpu_physical_memory_set_dirty(ram_addr);
345 } while (c != 0);
348 return 0;
351 #define ALIGN(x, y) (((x)+(y)-1) & ~((y)-1))
354 * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
355 * This function updates qemu's dirty bitmap using cpu_physical_memory_set_dirty().
356 * This means all bits are set to dirty.
358 * @start_add: start of logged region.
359 * @end_addr: end of logged region.
361 static int kvm_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
362 target_phys_addr_t end_addr)
364 KVMState *s = kvm_state;
365 unsigned long size, allocated_size = 0;
366 KVMDirtyLog d;
367 KVMSlot *mem;
368 int ret = 0;
370 d.dirty_bitmap = NULL;
371 while (start_addr < end_addr) {
372 mem = kvm_lookup_overlapping_slot(s, start_addr, end_addr);
373 if (mem == NULL) {
374 break;
377 size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS), HOST_LONG_BITS) / 8;
378 if (!d.dirty_bitmap) {
379 d.dirty_bitmap = qemu_malloc(size);
380 } else if (size > allocated_size) {
381 d.dirty_bitmap = qemu_realloc(d.dirty_bitmap, size);
383 allocated_size = size;
384 memset(d.dirty_bitmap, 0, allocated_size);
386 d.slot = mem->slot;
388 if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
389 DPRINTF("ioctl failed %d\n", errno);
390 ret = -1;
391 break;
394 kvm_get_dirty_pages_log_range(mem->start_addr, d.dirty_bitmap,
395 mem->start_addr, mem->memory_size);
396 start_addr = mem->start_addr + mem->memory_size;
398 qemu_free(d.dirty_bitmap);
400 return ret;
403 int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
405 int ret = -ENOSYS;
406 KVMState *s = kvm_state;
408 if (s->coalesced_mmio) {
409 struct kvm_coalesced_mmio_zone zone;
411 zone.addr = start;
412 zone.size = size;
414 ret = kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
417 return ret;
420 int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
422 int ret = -ENOSYS;
423 KVMState *s = kvm_state;
425 if (s->coalesced_mmio) {
426 struct kvm_coalesced_mmio_zone zone;
428 zone.addr = start;
429 zone.size = size;
431 ret = kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
434 return ret;
437 int kvm_check_extension(KVMState *s, unsigned int extension)
439 int ret;
441 ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
442 if (ret < 0) {
443 ret = 0;
446 return ret;
449 static int kvm_check_many_ioeventfds(void)
451 /* Userspace can use ioeventfd for io notification. This requires a host
452 * that supports eventfd(2) and an I/O thread; since eventfd does not
453 * support SIGIO it cannot interrupt the vcpu.
455 * Older kernels have a 6 device limit on the KVM io bus. Find out so we
456 * can avoid creating too many ioeventfds.
458 #if defined(CONFIG_EVENTFD) && defined(CONFIG_IOTHREAD)
459 int ioeventfds[7];
460 int i, ret = 0;
461 for (i = 0; i < ARRAY_SIZE(ioeventfds); i++) {
462 ioeventfds[i] = eventfd(0, EFD_CLOEXEC);
463 if (ioeventfds[i] < 0) {
464 break;
466 ret = kvm_set_ioeventfd_pio_word(ioeventfds[i], 0, i, true);
467 if (ret < 0) {
468 close(ioeventfds[i]);
469 break;
473 /* Decide whether many devices are supported or not */
474 ret = i == ARRAY_SIZE(ioeventfds);
476 while (i-- > 0) {
477 kvm_set_ioeventfd_pio_word(ioeventfds[i], 0, i, false);
478 close(ioeventfds[i]);
480 return ret;
481 #else
482 return 0;
483 #endif
486 static const KVMCapabilityInfo *
487 kvm_check_extension_list(KVMState *s, const KVMCapabilityInfo *list)
489 while (list->name) {
490 if (!kvm_check_extension(s, list->value)) {
491 return list;
493 list++;
495 return NULL;
498 static void kvm_set_phys_mem(target_phys_addr_t start_addr, ram_addr_t size,
499 ram_addr_t phys_offset)
501 KVMState *s = kvm_state;
502 ram_addr_t flags = phys_offset & ~TARGET_PAGE_MASK;
503 KVMSlot *mem, old;
504 int err;
506 /* kvm works in page size chunks, but the function may be called
507 with sub-page size and unaligned start address. */
508 size = TARGET_PAGE_ALIGN(size);
509 start_addr = TARGET_PAGE_ALIGN(start_addr);
511 /* KVM does not support read-only slots */
512 phys_offset &= ~IO_MEM_ROM;
514 while (1) {
515 mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size);
516 if (!mem) {
517 break;
520 if (flags < IO_MEM_UNASSIGNED && start_addr >= mem->start_addr &&
521 (start_addr + size <= mem->start_addr + mem->memory_size) &&
522 (phys_offset - start_addr == mem->phys_offset - mem->start_addr)) {
523 /* The new slot fits into the existing one and comes with
524 * identical parameters - nothing to be done. */
525 return;
528 old = *mem;
530 /* unregister the overlapping slot */
531 mem->memory_size = 0;
532 err = kvm_set_user_memory_region(s, mem);
533 if (err) {
534 fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
535 __func__, strerror(-err));
536 abort();
539 /* Workaround for older KVM versions: we can't join slots, even not by
540 * unregistering the previous ones and then registering the larger
541 * slot. We have to maintain the existing fragmentation. Sigh.
543 * This workaround assumes that the new slot starts at the same
544 * address as the first existing one. If not or if some overlapping
545 * slot comes around later, we will fail (not seen in practice so far)
546 * - and actually require a recent KVM version. */
547 if (s->broken_set_mem_region &&
548 old.start_addr == start_addr && old.memory_size < size &&
549 flags < IO_MEM_UNASSIGNED) {
550 mem = kvm_alloc_slot(s);
551 mem->memory_size = old.memory_size;
552 mem->start_addr = old.start_addr;
553 mem->phys_offset = old.phys_offset;
554 mem->flags = 0;
556 err = kvm_set_user_memory_region(s, mem);
557 if (err) {
558 fprintf(stderr, "%s: error updating slot: %s\n", __func__,
559 strerror(-err));
560 abort();
563 start_addr += old.memory_size;
564 phys_offset += old.memory_size;
565 size -= old.memory_size;
566 continue;
569 /* register prefix slot */
570 if (old.start_addr < start_addr) {
571 mem = kvm_alloc_slot(s);
572 mem->memory_size = start_addr - old.start_addr;
573 mem->start_addr = old.start_addr;
574 mem->phys_offset = old.phys_offset;
575 mem->flags = 0;
577 err = kvm_set_user_memory_region(s, mem);
578 if (err) {
579 fprintf(stderr, "%s: error registering prefix slot: %s\n",
580 __func__, strerror(-err));
581 abort();
585 /* register suffix slot */
586 if (old.start_addr + old.memory_size > start_addr + size) {
587 ram_addr_t size_delta;
589 mem = kvm_alloc_slot(s);
590 mem->start_addr = start_addr + size;
591 size_delta = mem->start_addr - old.start_addr;
592 mem->memory_size = old.memory_size - size_delta;
593 mem->phys_offset = old.phys_offset + size_delta;
594 mem->flags = 0;
596 err = kvm_set_user_memory_region(s, mem);
597 if (err) {
598 fprintf(stderr, "%s: error registering suffix slot: %s\n",
599 __func__, strerror(-err));
600 abort();
605 /* in case the KVM bug workaround already "consumed" the new slot */
606 if (!size) {
607 return;
609 /* KVM does not need to know about this memory */
610 if (flags >= IO_MEM_UNASSIGNED) {
611 return;
613 mem = kvm_alloc_slot(s);
614 mem->memory_size = size;
615 mem->start_addr = start_addr;
616 mem->phys_offset = phys_offset;
617 mem->flags = 0;
619 err = kvm_set_user_memory_region(s, mem);
620 if (err) {
621 fprintf(stderr, "%s: error registering slot: %s\n", __func__,
622 strerror(-err));
623 abort();
627 static void kvm_client_set_memory(struct CPUPhysMemoryClient *client,
628 target_phys_addr_t start_addr,
629 ram_addr_t size, ram_addr_t phys_offset)
631 kvm_set_phys_mem(start_addr, size, phys_offset);
634 static int kvm_client_sync_dirty_bitmap(struct CPUPhysMemoryClient *client,
635 target_phys_addr_t start_addr,
636 target_phys_addr_t end_addr)
638 return kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
641 static int kvm_client_migration_log(struct CPUPhysMemoryClient *client,
642 int enable)
644 return kvm_set_migration_log(enable);
647 static CPUPhysMemoryClient kvm_cpu_phys_memory_client = {
648 .set_memory = kvm_client_set_memory,
649 .sync_dirty_bitmap = kvm_client_sync_dirty_bitmap,
650 .migration_log = kvm_client_migration_log,
653 int kvm_init(void)
655 static const char upgrade_note[] =
656 "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
657 "(see http://sourceforge.net/projects/kvm).\n";
658 KVMState *s;
659 const KVMCapabilityInfo *missing_cap;
660 int ret;
661 int i;
663 s = qemu_mallocz(sizeof(KVMState));
665 #ifdef KVM_CAP_SET_GUEST_DEBUG
666 QTAILQ_INIT(&s->kvm_sw_breakpoints);
667 #endif
668 for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
669 s->slots[i].slot = i;
671 s->vmfd = -1;
672 s->fd = qemu_open("/dev/kvm", O_RDWR);
673 if (s->fd == -1) {
674 fprintf(stderr, "Could not access KVM kernel module: %m\n");
675 ret = -errno;
676 goto err;
679 ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
680 if (ret < KVM_API_VERSION) {
681 if (ret > 0) {
682 ret = -EINVAL;
684 fprintf(stderr, "kvm version too old\n");
685 goto err;
688 if (ret > KVM_API_VERSION) {
689 ret = -EINVAL;
690 fprintf(stderr, "kvm version not supported\n");
691 goto err;
694 s->vmfd = kvm_ioctl(s, KVM_CREATE_VM, 0);
695 if (s->vmfd < 0) {
696 #ifdef TARGET_S390X
697 fprintf(stderr, "Please add the 'switch_amode' kernel parameter to "
698 "your host kernel command line\n");
699 #endif
700 goto err;
703 missing_cap = kvm_check_extension_list(s, kvm_required_capabilites);
704 if (!missing_cap) {
705 missing_cap =
706 kvm_check_extension_list(s, kvm_arch_required_capabilities);
708 if (missing_cap) {
709 ret = -EINVAL;
710 fprintf(stderr, "kvm does not support %s\n%s",
711 missing_cap->name, upgrade_note);
712 goto err;
715 s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
717 s->broken_set_mem_region = 1;
718 #ifdef KVM_CAP_JOIN_MEMORY_REGIONS_WORKS
719 ret = kvm_check_extension(s, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
720 if (ret > 0) {
721 s->broken_set_mem_region = 0;
723 #endif
725 s->vcpu_events = 0;
726 #ifdef KVM_CAP_VCPU_EVENTS
727 s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
728 #endif
730 s->robust_singlestep = 0;
731 #ifdef KVM_CAP_X86_ROBUST_SINGLESTEP
732 s->robust_singlestep =
733 kvm_check_extension(s, KVM_CAP_X86_ROBUST_SINGLESTEP);
734 #endif
736 s->debugregs = 0;
737 #ifdef KVM_CAP_DEBUGREGS
738 s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS);
739 #endif
741 s->xsave = 0;
742 #ifdef KVM_CAP_XSAVE
743 s->xsave = kvm_check_extension(s, KVM_CAP_XSAVE);
744 #endif
746 s->xcrs = 0;
747 #ifdef KVM_CAP_XCRS
748 s->xcrs = kvm_check_extension(s, KVM_CAP_XCRS);
749 #endif
751 ret = kvm_arch_init(s);
752 if (ret < 0) {
753 goto err;
756 kvm_state = s;
757 cpu_register_phys_memory_client(&kvm_cpu_phys_memory_client);
759 s->many_ioeventfds = kvm_check_many_ioeventfds();
761 return 0;
763 err:
764 if (s) {
765 if (s->vmfd != -1) {
766 close(s->vmfd);
768 if (s->fd != -1) {
769 close(s->fd);
772 qemu_free(s);
774 return ret;
777 static int kvm_handle_io(uint16_t port, void *data, int direction, int size,
778 uint32_t count)
780 int i;
781 uint8_t *ptr = data;
783 for (i = 0; i < count; i++) {
784 if (direction == KVM_EXIT_IO_IN) {
785 switch (size) {
786 case 1:
787 stb_p(ptr, cpu_inb(port));
788 break;
789 case 2:
790 stw_p(ptr, cpu_inw(port));
791 break;
792 case 4:
793 stl_p(ptr, cpu_inl(port));
794 break;
796 } else {
797 switch (size) {
798 case 1:
799 cpu_outb(port, ldub_p(ptr));
800 break;
801 case 2:
802 cpu_outw(port, lduw_p(ptr));
803 break;
804 case 4:
805 cpu_outl(port, ldl_p(ptr));
806 break;
810 ptr += size;
813 return 1;
816 #ifdef KVM_CAP_INTERNAL_ERROR_DATA
817 static int kvm_handle_internal_error(CPUState *env, struct kvm_run *run)
819 fprintf(stderr, "KVM internal error.");
820 if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) {
821 int i;
823 fprintf(stderr, " Suberror: %d\n", run->internal.suberror);
824 for (i = 0; i < run->internal.ndata; ++i) {
825 fprintf(stderr, "extra data[%d]: %"PRIx64"\n",
826 i, (uint64_t)run->internal.data[i]);
828 } else {
829 fprintf(stderr, "\n");
831 if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) {
832 fprintf(stderr, "emulation failure\n");
833 if (!kvm_arch_stop_on_emulation_error(env)) {
834 cpu_dump_state(env, stderr, fprintf, CPU_DUMP_CODE);
835 return 0;
838 /* FIXME: Should trigger a qmp message to let management know
839 * something went wrong.
841 return -1;
843 #endif
845 void kvm_flush_coalesced_mmio_buffer(void)
847 KVMState *s = kvm_state;
848 if (s->coalesced_mmio_ring) {
849 struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
850 while (ring->first != ring->last) {
851 struct kvm_coalesced_mmio *ent;
853 ent = &ring->coalesced_mmio[ring->first];
855 cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
856 smp_wmb();
857 ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
862 static void do_kvm_cpu_synchronize_state(void *_env)
864 CPUState *env = _env;
866 if (!env->kvm_vcpu_dirty) {
867 kvm_arch_get_registers(env);
868 env->kvm_vcpu_dirty = 1;
872 void kvm_cpu_synchronize_state(CPUState *env)
874 if (!env->kvm_vcpu_dirty) {
875 run_on_cpu(env, do_kvm_cpu_synchronize_state, env);
879 void kvm_cpu_synchronize_post_reset(CPUState *env)
881 kvm_arch_put_registers(env, KVM_PUT_RESET_STATE);
882 env->kvm_vcpu_dirty = 0;
885 void kvm_cpu_synchronize_post_init(CPUState *env)
887 kvm_arch_put_registers(env, KVM_PUT_FULL_STATE);
888 env->kvm_vcpu_dirty = 0;
891 int kvm_cpu_exec(CPUState *env)
893 struct kvm_run *run = env->kvm_run;
894 int ret;
896 DPRINTF("kvm_cpu_exec()\n");
898 do {
899 #ifndef CONFIG_IOTHREAD
900 if (env->exit_request) {
901 DPRINTF("interrupt exit requested\n");
902 ret = 0;
903 break;
905 #endif
907 if (kvm_arch_process_irqchip_events(env)) {
908 ret = 0;
909 break;
912 if (env->kvm_vcpu_dirty) {
913 kvm_arch_put_registers(env, KVM_PUT_RUNTIME_STATE);
914 env->kvm_vcpu_dirty = 0;
917 kvm_arch_pre_run(env, run);
918 cpu_single_env = NULL;
919 qemu_mutex_unlock_iothread();
920 ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);
921 qemu_mutex_lock_iothread();
922 cpu_single_env = env;
923 kvm_arch_post_run(env, run);
925 kvm_flush_coalesced_mmio_buffer();
927 if (ret == -EINTR || ret == -EAGAIN) {
928 cpu_exit(env);
929 DPRINTF("io window exit\n");
930 ret = 0;
931 break;
934 if (ret < 0) {
935 DPRINTF("kvm run failed %s\n", strerror(-ret));
936 abort();
939 ret = 0; /* exit loop */
940 switch (run->exit_reason) {
941 case KVM_EXIT_IO:
942 DPRINTF("handle_io\n");
943 ret = kvm_handle_io(run->io.port,
944 (uint8_t *)run + run->io.data_offset,
945 run->io.direction,
946 run->io.size,
947 run->io.count);
948 break;
949 case KVM_EXIT_MMIO:
950 DPRINTF("handle_mmio\n");
951 cpu_physical_memory_rw(run->mmio.phys_addr,
952 run->mmio.data,
953 run->mmio.len,
954 run->mmio.is_write);
955 ret = 1;
956 break;
957 case KVM_EXIT_IRQ_WINDOW_OPEN:
958 DPRINTF("irq_window_open\n");
959 break;
960 case KVM_EXIT_SHUTDOWN:
961 DPRINTF("shutdown\n");
962 qemu_system_reset_request();
963 ret = 1;
964 break;
965 case KVM_EXIT_UNKNOWN:
966 fprintf(stderr, "KVM: unknown exit, hardware reason %" PRIx64 "\n",
967 (uint64_t)run->hw.hardware_exit_reason);
968 ret = -1;
969 break;
970 #ifdef KVM_CAP_INTERNAL_ERROR_DATA
971 case KVM_EXIT_INTERNAL_ERROR:
972 ret = kvm_handle_internal_error(env, run);
973 break;
974 #endif
975 case KVM_EXIT_DEBUG:
976 DPRINTF("kvm_exit_debug\n");
977 #ifdef KVM_CAP_SET_GUEST_DEBUG
978 if (kvm_arch_debug(&run->debug.arch)) {
979 env->exception_index = EXCP_DEBUG;
980 return 0;
982 /* re-enter, this exception was guest-internal */
983 ret = 1;
984 #endif /* KVM_CAP_SET_GUEST_DEBUG */
985 break;
986 default:
987 DPRINTF("kvm_arch_handle_exit\n");
988 ret = kvm_arch_handle_exit(env, run);
989 break;
991 } while (ret > 0);
993 if (ret < 0) {
994 cpu_dump_state(env, stderr, fprintf, CPU_DUMP_CODE);
995 vm_stop(0);
996 env->exit_request = 1;
998 if (env->exit_request) {
999 env->exit_request = 0;
1000 env->exception_index = EXCP_INTERRUPT;
1003 return ret;
1006 int kvm_ioctl(KVMState *s, int type, ...)
1008 int ret;
1009 void *arg;
1010 va_list ap;
1012 va_start(ap, type);
1013 arg = va_arg(ap, void *);
1014 va_end(ap);
1016 ret = ioctl(s->fd, type, arg);
1017 if (ret == -1) {
1018 ret = -errno;
1020 return ret;
1023 int kvm_vm_ioctl(KVMState *s, int type, ...)
1025 int ret;
1026 void *arg;
1027 va_list ap;
1029 va_start(ap, type);
1030 arg = va_arg(ap, void *);
1031 va_end(ap);
1033 ret = ioctl(s->vmfd, type, arg);
1034 if (ret == -1) {
1035 ret = -errno;
1037 return ret;
1040 int kvm_vcpu_ioctl(CPUState *env, int type, ...)
1042 int ret;
1043 void *arg;
1044 va_list ap;
1046 va_start(ap, type);
1047 arg = va_arg(ap, void *);
1048 va_end(ap);
1050 ret = ioctl(env->kvm_fd, type, arg);
1051 if (ret == -1) {
1052 ret = -errno;
1054 return ret;
1057 int kvm_has_sync_mmu(void)
1059 return kvm_check_extension(kvm_state, KVM_CAP_SYNC_MMU);
1062 int kvm_has_vcpu_events(void)
1064 return kvm_state->vcpu_events;
1067 int kvm_has_robust_singlestep(void)
1069 return kvm_state->robust_singlestep;
1072 int kvm_has_debugregs(void)
1074 return kvm_state->debugregs;
1077 int kvm_has_xsave(void)
1079 return kvm_state->xsave;
1082 int kvm_has_xcrs(void)
1084 return kvm_state->xcrs;
1087 int kvm_has_many_ioeventfds(void)
1089 if (!kvm_enabled()) {
1090 return 0;
1092 return kvm_state->many_ioeventfds;
1095 void kvm_setup_guest_memory(void *start, size_t size)
1097 if (!kvm_has_sync_mmu()) {
1098 int ret = qemu_madvise(start, size, QEMU_MADV_DONTFORK);
1100 if (ret) {
1101 perror("qemu_madvise");
1102 fprintf(stderr,
1103 "Need MADV_DONTFORK in absence of synchronous KVM MMU\n");
1104 exit(1);
1109 #ifdef KVM_CAP_SET_GUEST_DEBUG
1110 struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *env,
1111 target_ulong pc)
1113 struct kvm_sw_breakpoint *bp;
1115 QTAILQ_FOREACH(bp, &env->kvm_state->kvm_sw_breakpoints, entry) {
1116 if (bp->pc == pc) {
1117 return bp;
1120 return NULL;
1123 int kvm_sw_breakpoints_active(CPUState *env)
1125 return !QTAILQ_EMPTY(&env->kvm_state->kvm_sw_breakpoints);
1128 struct kvm_set_guest_debug_data {
1129 struct kvm_guest_debug dbg;
1130 CPUState *env;
1131 int err;
1134 static void kvm_invoke_set_guest_debug(void *data)
1136 struct kvm_set_guest_debug_data *dbg_data = data;
1137 CPUState *env = dbg_data->env;
1139 dbg_data->err = kvm_vcpu_ioctl(env, KVM_SET_GUEST_DEBUG, &dbg_data->dbg);
1142 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1144 struct kvm_set_guest_debug_data data;
1146 data.dbg.control = reinject_trap;
1148 if (env->singlestep_enabled) {
1149 data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
1151 kvm_arch_update_guest_debug(env, &data.dbg);
1152 data.env = env;
1154 run_on_cpu(env, kvm_invoke_set_guest_debug, &data);
1155 return data.err;
1158 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1159 target_ulong len, int type)
1161 struct kvm_sw_breakpoint *bp;
1162 CPUState *env;
1163 int err;
1165 if (type == GDB_BREAKPOINT_SW) {
1166 bp = kvm_find_sw_breakpoint(current_env, addr);
1167 if (bp) {
1168 bp->use_count++;
1169 return 0;
1172 bp = qemu_malloc(sizeof(struct kvm_sw_breakpoint));
1173 if (!bp) {
1174 return -ENOMEM;
1177 bp->pc = addr;
1178 bp->use_count = 1;
1179 err = kvm_arch_insert_sw_breakpoint(current_env, bp);
1180 if (err) {
1181 free(bp);
1182 return err;
1185 QTAILQ_INSERT_HEAD(&current_env->kvm_state->kvm_sw_breakpoints,
1186 bp, entry);
1187 } else {
1188 err = kvm_arch_insert_hw_breakpoint(addr, len, type);
1189 if (err) {
1190 return err;
1194 for (env = first_cpu; env != NULL; env = env->next_cpu) {
1195 err = kvm_update_guest_debug(env, 0);
1196 if (err) {
1197 return err;
1200 return 0;
1203 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1204 target_ulong len, int type)
1206 struct kvm_sw_breakpoint *bp;
1207 CPUState *env;
1208 int err;
1210 if (type == GDB_BREAKPOINT_SW) {
1211 bp = kvm_find_sw_breakpoint(current_env, addr);
1212 if (!bp) {
1213 return -ENOENT;
1216 if (bp->use_count > 1) {
1217 bp->use_count--;
1218 return 0;
1221 err = kvm_arch_remove_sw_breakpoint(current_env, bp);
1222 if (err) {
1223 return err;
1226 QTAILQ_REMOVE(&current_env->kvm_state->kvm_sw_breakpoints, bp, entry);
1227 qemu_free(bp);
1228 } else {
1229 err = kvm_arch_remove_hw_breakpoint(addr, len, type);
1230 if (err) {
1231 return err;
1235 for (env = first_cpu; env != NULL; env = env->next_cpu) {
1236 err = kvm_update_guest_debug(env, 0);
1237 if (err) {
1238 return err;
1241 return 0;
1244 void kvm_remove_all_breakpoints(CPUState *current_env)
1246 struct kvm_sw_breakpoint *bp, *next;
1247 KVMState *s = current_env->kvm_state;
1248 CPUState *env;
1250 QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
1251 if (kvm_arch_remove_sw_breakpoint(current_env, bp) != 0) {
1252 /* Try harder to find a CPU that currently sees the breakpoint. */
1253 for (env = first_cpu; env != NULL; env = env->next_cpu) {
1254 if (kvm_arch_remove_sw_breakpoint(env, bp) == 0) {
1255 break;
1260 kvm_arch_remove_all_hw_breakpoints();
1262 for (env = first_cpu; env != NULL; env = env->next_cpu) {
1263 kvm_update_guest_debug(env, 0);
1267 #else /* !KVM_CAP_SET_GUEST_DEBUG */
1269 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1271 return -EINVAL;
1274 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1275 target_ulong len, int type)
1277 return -EINVAL;
1280 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1281 target_ulong len, int type)
1283 return -EINVAL;
1286 void kvm_remove_all_breakpoints(CPUState *current_env)
1289 #endif /* !KVM_CAP_SET_GUEST_DEBUG */
1291 int kvm_set_signal_mask(CPUState *env, const sigset_t *sigset)
1293 struct kvm_signal_mask *sigmask;
1294 int r;
1296 if (!sigset) {
1297 return kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, NULL);
1300 sigmask = qemu_malloc(sizeof(*sigmask) + sizeof(*sigset));
1302 sigmask->len = 8;
1303 memcpy(sigmask->sigset, sigset, sizeof(*sigset));
1304 r = kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, sigmask);
1305 free(sigmask);
1307 return r;
1310 int kvm_set_ioeventfd_mmio_long(int fd, uint32_t addr, uint32_t val, bool assign)
1312 #ifdef KVM_IOEVENTFD
1313 int ret;
1314 struct kvm_ioeventfd iofd;
1316 iofd.datamatch = val;
1317 iofd.addr = addr;
1318 iofd.len = 4;
1319 iofd.flags = KVM_IOEVENTFD_FLAG_DATAMATCH;
1320 iofd.fd = fd;
1322 if (!kvm_enabled()) {
1323 return -ENOSYS;
1326 if (!assign) {
1327 iofd.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
1330 ret = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &iofd);
1332 if (ret < 0) {
1333 return -errno;
1336 return 0;
1337 #else
1338 return -ENOSYS;
1339 #endif
1342 int kvm_set_ioeventfd_pio_word(int fd, uint16_t addr, uint16_t val, bool assign)
1344 #ifdef KVM_IOEVENTFD
1345 struct kvm_ioeventfd kick = {
1346 .datamatch = val,
1347 .addr = addr,
1348 .len = 2,
1349 .flags = KVM_IOEVENTFD_FLAG_DATAMATCH | KVM_IOEVENTFD_FLAG_PIO,
1350 .fd = fd,
1352 int r;
1353 if (!kvm_enabled()) {
1354 return -ENOSYS;
1356 if (!assign) {
1357 kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
1359 r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
1360 if (r < 0) {
1361 return r;
1363 return 0;
1364 #else
1365 return -ENOSYS;
1366 #endif