4 * Copyright IBM, Corp. 2008
8 * Anthony Liguori <aliguori@us.ibm.com>
9 * Glauber Costa <gcosta@redhat.com>
11 * This work is licensed under the terms of the GNU GPL, version 2 or later.
12 * See the COPYING file in the top-level directory.
16 #include <sys/types.h>
17 #include <sys/ioctl.h>
21 #include <linux/kvm.h>
23 #include "qemu-common.h"
28 /* KVM uses PAGE_SIZE in it's definition of COALESCED_MMIO_MAX */
29 #define PAGE_SIZE TARGET_PAGE_SIZE
34 #define dprintf(fmt, ...) \
35 do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
37 #define dprintf(fmt, ...) \
41 typedef struct KVMSlot
43 target_phys_addr_t start_addr
;
44 ram_addr_t memory_size
;
45 ram_addr_t phys_offset
;
50 typedef struct kvm_dirty_log KVMDirtyLog
;
60 #ifdef KVM_CAP_SET_GUEST_DEBUG
61 struct kvm_sw_breakpoint_head kvm_sw_breakpoints
;
65 static KVMState
*kvm_state
;
67 static KVMSlot
*kvm_alloc_slot(KVMState
*s
)
71 for (i
= 0; i
< ARRAY_SIZE(s
->slots
); i
++) {
72 /* KVM private memory slots */
75 if (s
->slots
[i
].memory_size
== 0)
79 fprintf(stderr
, "%s: no free slot available\n", __func__
);
83 static KVMSlot
*kvm_lookup_matching_slot(KVMState
*s
,
84 target_phys_addr_t start_addr
,
85 target_phys_addr_t end_addr
)
89 for (i
= 0; i
< ARRAY_SIZE(s
->slots
); i
++) {
90 KVMSlot
*mem
= &s
->slots
[i
];
92 if (start_addr
== mem
->start_addr
&&
93 end_addr
== mem
->start_addr
+ mem
->memory_size
) {
102 * Find overlapping slot with lowest start address
104 static KVMSlot
*kvm_lookup_overlapping_slot(KVMState
*s
,
105 target_phys_addr_t start_addr
,
106 target_phys_addr_t end_addr
)
108 KVMSlot
*found
= NULL
;
111 for (i
= 0; i
< ARRAY_SIZE(s
->slots
); i
++) {
112 KVMSlot
*mem
= &s
->slots
[i
];
114 if (mem
->memory_size
== 0 ||
115 (found
&& found
->start_addr
< mem
->start_addr
)) {
119 if (end_addr
> mem
->start_addr
&&
120 start_addr
< mem
->start_addr
+ mem
->memory_size
) {
128 static int kvm_set_user_memory_region(KVMState
*s
, KVMSlot
*slot
)
130 struct kvm_userspace_memory_region mem
;
132 mem
.slot
= slot
->slot
;
133 mem
.guest_phys_addr
= slot
->start_addr
;
134 mem
.memory_size
= slot
->memory_size
;
135 mem
.userspace_addr
= (unsigned long)qemu_get_ram_ptr(slot
->phys_offset
);
136 mem
.flags
= slot
->flags
;
138 return kvm_vm_ioctl(s
, KVM_SET_USER_MEMORY_REGION
, &mem
);
142 int kvm_init_vcpu(CPUState
*env
)
144 KVMState
*s
= kvm_state
;
148 dprintf("kvm_init_vcpu\n");
150 ret
= kvm_vm_ioctl(s
, KVM_CREATE_VCPU
, env
->cpu_index
);
152 dprintf("kvm_create_vcpu failed\n");
159 mmap_size
= kvm_ioctl(s
, KVM_GET_VCPU_MMAP_SIZE
, 0);
161 dprintf("KVM_GET_VCPU_MMAP_SIZE failed\n");
165 env
->kvm_run
= mmap(NULL
, mmap_size
, PROT_READ
| PROT_WRITE
, MAP_SHARED
,
167 if (env
->kvm_run
== MAP_FAILED
) {
169 dprintf("mmap'ing vcpu state failed\n");
173 ret
= kvm_arch_init_vcpu(env
);
179 int kvm_sync_vcpus(void)
183 for (env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
186 ret
= kvm_arch_put_registers(env
);
195 * dirty pages logging control
197 static int kvm_dirty_pages_log_change(target_phys_addr_t phys_addr
,
198 ram_addr_t size
, unsigned flags
,
201 KVMState
*s
= kvm_state
;
202 KVMSlot
*mem
= kvm_lookup_matching_slot(s
, phys_addr
, phys_addr
+ size
);
204 fprintf(stderr
, "BUG: %s: invalid parameters " TARGET_FMT_plx
"-"
205 TARGET_FMT_plx
"\n", __func__
, phys_addr
,
206 phys_addr
+ size
- 1);
210 flags
= (mem
->flags
& ~mask
) | flags
;
211 /* Nothing changed, no need to issue ioctl */
212 if (flags
== mem
->flags
)
217 return kvm_set_user_memory_region(s
, mem
);
220 int kvm_log_start(target_phys_addr_t phys_addr
, ram_addr_t size
)
222 return kvm_dirty_pages_log_change(phys_addr
, size
,
223 KVM_MEM_LOG_DIRTY_PAGES
,
224 KVM_MEM_LOG_DIRTY_PAGES
);
227 int kvm_log_stop(target_phys_addr_t phys_addr
, ram_addr_t size
)
229 return kvm_dirty_pages_log_change(phys_addr
, size
,
231 KVM_MEM_LOG_DIRTY_PAGES
);
235 * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
236 * This function updates qemu's dirty bitmap using cpu_physical_memory_set_dirty().
237 * This means all bits are set to dirty.
239 * @start_add: start of logged region.
240 * @end_addr: end of logged region.
242 void kvm_physical_sync_dirty_bitmap(target_phys_addr_t start_addr
,
243 target_phys_addr_t end_addr
)
245 KVMState
*s
= kvm_state
;
247 KVMSlot
*mem
= kvm_lookup_matching_slot(s
, start_addr
, end_addr
);
248 unsigned long alloc_size
;
250 target_phys_addr_t phys_addr
= start_addr
;
252 dprintf("sync addr: " TARGET_FMT_lx
" into %lx\n", start_addr
,
255 fprintf(stderr
, "BUG: %s: invalid parameters " TARGET_FMT_plx
"-"
256 TARGET_FMT_plx
"\n", __func__
, phys_addr
, end_addr
- 1);
260 alloc_size
= mem
->memory_size
>> TARGET_PAGE_BITS
/ sizeof(d
.dirty_bitmap
);
261 d
.dirty_bitmap
= qemu_mallocz(alloc_size
);
264 dprintf("slot %d, phys_addr %llx, uaddr: %llx\n",
265 d
.slot
, mem
->start_addr
, mem
->phys_offset
);
267 if (kvm_vm_ioctl(s
, KVM_GET_DIRTY_LOG
, &d
) == -1) {
268 dprintf("ioctl failed %d\n", errno
);
272 phys_addr
= start_addr
;
273 for (addr
= mem
->phys_offset
; phys_addr
< end_addr
; phys_addr
+= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
274 unsigned long *bitmap
= (unsigned long *)d
.dirty_bitmap
;
275 unsigned nr
= (phys_addr
- start_addr
) >> TARGET_PAGE_BITS
;
276 unsigned word
= nr
/ (sizeof(*bitmap
) * 8);
277 unsigned bit
= nr
% (sizeof(*bitmap
) * 8);
278 if ((bitmap
[word
] >> bit
) & 1)
279 cpu_physical_memory_set_dirty(addr
);
282 qemu_free(d
.dirty_bitmap
);
285 int kvm_coalesce_mmio_region(target_phys_addr_t start
, ram_addr_t size
)
288 #ifdef KVM_CAP_COALESCED_MMIO
289 KVMState
*s
= kvm_state
;
291 if (s
->coalesced_mmio
) {
292 struct kvm_coalesced_mmio_zone zone
;
297 ret
= kvm_vm_ioctl(s
, KVM_REGISTER_COALESCED_MMIO
, &zone
);
304 int kvm_uncoalesce_mmio_region(target_phys_addr_t start
, ram_addr_t size
)
307 #ifdef KVM_CAP_COALESCED_MMIO
308 KVMState
*s
= kvm_state
;
310 if (s
->coalesced_mmio
) {
311 struct kvm_coalesced_mmio_zone zone
;
316 ret
= kvm_vm_ioctl(s
, KVM_UNREGISTER_COALESCED_MMIO
, &zone
);
323 int kvm_check_extension(KVMState
*s
, unsigned int extension
)
327 ret
= kvm_ioctl(s
, KVM_CHECK_EXTENSION
, extension
);
335 int kvm_init(int smp_cpus
)
342 fprintf(stderr
, "No SMP KVM support, use '-smp 1'\n");
346 s
= qemu_mallocz(sizeof(KVMState
));
348 #ifdef KVM_CAP_SET_GUEST_DEBUG
349 TAILQ_INIT(&s
->kvm_sw_breakpoints
);
351 for (i
= 0; i
< ARRAY_SIZE(s
->slots
); i
++)
352 s
->slots
[i
].slot
= i
;
355 s
->fd
= open("/dev/kvm", O_RDWR
);
357 fprintf(stderr
, "Could not access KVM kernel module: %m\n");
362 ret
= kvm_ioctl(s
, KVM_GET_API_VERSION
, 0);
363 if (ret
< KVM_API_VERSION
) {
366 fprintf(stderr
, "kvm version too old\n");
370 if (ret
> KVM_API_VERSION
) {
372 fprintf(stderr
, "kvm version not supported\n");
376 s
->vmfd
= kvm_ioctl(s
, KVM_CREATE_VM
, 0);
380 /* initially, KVM allocated its own memory and we had to jump through
381 * hooks to make phys_ram_base point to this. Modern versions of KVM
382 * just use a user allocated buffer so we can use regular pages
383 * unmodified. Make sure we have a sufficiently modern version of KVM.
385 if (!kvm_check_extension(s
, KVM_CAP_USER_MEMORY
)) {
387 fprintf(stderr
, "kvm does not support KVM_CAP_USER_MEMORY\n");
391 /* There was a nasty bug in < kvm-80 that prevents memory slots from being
392 * destroyed properly. Since we rely on this capability, refuse to work
393 * with any kernel without this capability. */
394 if (!kvm_check_extension(s
, KVM_CAP_DESTROY_MEMORY_REGION_WORKS
)) {
398 "KVM kernel module broken (DESTROY_MEMORY_REGION)\n"
399 "Please upgrade to at least kvm-81.\n");
403 #ifdef KVM_CAP_COALESCED_MMIO
404 s
->coalesced_mmio
= kvm_check_extension(s
, KVM_CAP_COALESCED_MMIO
);
406 s
->coalesced_mmio
= 0;
409 ret
= kvm_arch_init(s
, smp_cpus
);
429 static int kvm_handle_io(CPUState
*env
, uint16_t port
, void *data
,
430 int direction
, int size
, uint32_t count
)
435 for (i
= 0; i
< count
; i
++) {
436 if (direction
== KVM_EXIT_IO_IN
) {
439 stb_p(ptr
, cpu_inb(env
, port
));
442 stw_p(ptr
, cpu_inw(env
, port
));
445 stl_p(ptr
, cpu_inl(env
, port
));
451 cpu_outb(env
, port
, ldub_p(ptr
));
454 cpu_outw(env
, port
, lduw_p(ptr
));
457 cpu_outl(env
, port
, ldl_p(ptr
));
468 static void kvm_run_coalesced_mmio(CPUState
*env
, struct kvm_run
*run
)
470 #ifdef KVM_CAP_COALESCED_MMIO
471 KVMState
*s
= kvm_state
;
472 if (s
->coalesced_mmio
) {
473 struct kvm_coalesced_mmio_ring
*ring
;
475 ring
= (void *)run
+ (s
->coalesced_mmio
* TARGET_PAGE_SIZE
);
476 while (ring
->first
!= ring
->last
) {
477 struct kvm_coalesced_mmio
*ent
;
479 ent
= &ring
->coalesced_mmio
[ring
->first
];
481 cpu_physical_memory_write(ent
->phys_addr
, ent
->data
, ent
->len
);
482 /* FIXME smp_wmb() */
483 ring
->first
= (ring
->first
+ 1) % KVM_COALESCED_MMIO_MAX
;
489 int kvm_cpu_exec(CPUState
*env
)
491 struct kvm_run
*run
= env
->kvm_run
;
494 dprintf("kvm_cpu_exec()\n");
497 kvm_arch_pre_run(env
, run
);
499 if (env
->exit_request
) {
500 dprintf("interrupt exit requested\n");
505 ret
= kvm_vcpu_ioctl(env
, KVM_RUN
, 0);
506 kvm_arch_post_run(env
, run
);
508 if (ret
== -EINTR
|| ret
== -EAGAIN
) {
509 dprintf("io window exit\n");
515 dprintf("kvm run failed %s\n", strerror(-ret
));
519 kvm_run_coalesced_mmio(env
, run
);
521 ret
= 0; /* exit loop */
522 switch (run
->exit_reason
) {
524 dprintf("handle_io\n");
525 ret
= kvm_handle_io(env
, run
->io
.port
,
526 (uint8_t *)run
+ run
->io
.data_offset
,
532 dprintf("handle_mmio\n");
533 cpu_physical_memory_rw(run
->mmio
.phys_addr
,
539 case KVM_EXIT_IRQ_WINDOW_OPEN
:
540 dprintf("irq_window_open\n");
542 case KVM_EXIT_SHUTDOWN
:
543 dprintf("shutdown\n");
544 qemu_system_reset_request();
547 case KVM_EXIT_UNKNOWN
:
548 dprintf("kvm_exit_unknown\n");
550 case KVM_EXIT_FAIL_ENTRY
:
551 dprintf("kvm_exit_fail_entry\n");
553 case KVM_EXIT_EXCEPTION
:
554 dprintf("kvm_exit_exception\n");
557 dprintf("kvm_exit_debug\n");
558 #ifdef KVM_CAP_SET_GUEST_DEBUG
559 if (kvm_arch_debug(&run
->debug
.arch
)) {
560 gdb_set_stop_cpu(env
);
562 env
->exception_index
= EXCP_DEBUG
;
565 /* re-enter, this exception was guest-internal */
567 #endif /* KVM_CAP_SET_GUEST_DEBUG */
570 dprintf("kvm_arch_handle_exit\n");
571 ret
= kvm_arch_handle_exit(env
, run
);
576 if (env
->exit_request
) {
577 env
->exit_request
= 0;
578 env
->exception_index
= EXCP_INTERRUPT
;
584 void kvm_set_phys_mem(target_phys_addr_t start_addr
,
586 ram_addr_t phys_offset
)
588 KVMState
*s
= kvm_state
;
589 ram_addr_t flags
= phys_offset
& ~TARGET_PAGE_MASK
;
593 if (start_addr
& ~TARGET_PAGE_MASK
) {
594 if (flags
>= IO_MEM_UNASSIGNED
) {
595 if (!kvm_lookup_overlapping_slot(s
, start_addr
,
596 start_addr
+ size
)) {
599 fprintf(stderr
, "Unaligned split of a KVM memory slot\n");
601 fprintf(stderr
, "Only page-aligned memory slots supported\n");
606 /* KVM does not support read-only slots */
607 phys_offset
&= ~IO_MEM_ROM
;
610 mem
= kvm_lookup_overlapping_slot(s
, start_addr
, start_addr
+ size
);
615 if (flags
< IO_MEM_UNASSIGNED
&& start_addr
>= mem
->start_addr
&&
616 (start_addr
+ size
<= mem
->start_addr
+ mem
->memory_size
) &&
617 (phys_offset
- start_addr
== mem
->phys_offset
- mem
->start_addr
)) {
618 /* The new slot fits into the existing one and comes with
619 * identical parameters - nothing to be done. */
625 /* unregister the overlapping slot */
626 mem
->memory_size
= 0;
627 err
= kvm_set_user_memory_region(s
, mem
);
629 fprintf(stderr
, "%s: error unregistering overlapping slot: %s\n",
630 __func__
, strerror(-err
));
634 /* Workaround for older KVM versions: we can't join slots, even not by
635 * unregistering the previous ones and then registering the larger
636 * slot. We have to maintain the existing fragmentation. Sigh.
638 * This workaround assumes that the new slot starts at the same
639 * address as the first existing one. If not or if some overlapping
640 * slot comes around later, we will fail (not seen in practice so far)
641 * - and actually require a recent KVM version. */
642 if (old
.start_addr
== start_addr
&& old
.memory_size
< size
&&
643 flags
< IO_MEM_UNASSIGNED
) {
644 mem
= kvm_alloc_slot(s
);
645 mem
->memory_size
= old
.memory_size
;
646 mem
->start_addr
= old
.start_addr
;
647 mem
->phys_offset
= old
.phys_offset
;
650 err
= kvm_set_user_memory_region(s
, mem
);
652 fprintf(stderr
, "%s: error updating slot: %s\n", __func__
,
657 start_addr
+= old
.memory_size
;
658 phys_offset
+= old
.memory_size
;
659 size
-= old
.memory_size
;
663 /* register prefix slot */
664 if (old
.start_addr
< start_addr
) {
665 mem
= kvm_alloc_slot(s
);
666 mem
->memory_size
= start_addr
- old
.start_addr
;
667 mem
->start_addr
= old
.start_addr
;
668 mem
->phys_offset
= old
.phys_offset
;
671 err
= kvm_set_user_memory_region(s
, mem
);
673 fprintf(stderr
, "%s: error registering prefix slot: %s\n",
674 __func__
, strerror(-err
));
679 /* register suffix slot */
680 if (old
.start_addr
+ old
.memory_size
> start_addr
+ size
) {
681 ram_addr_t size_delta
;
683 mem
= kvm_alloc_slot(s
);
684 mem
->start_addr
= start_addr
+ size
;
685 size_delta
= mem
->start_addr
- old
.start_addr
;
686 mem
->memory_size
= old
.memory_size
- size_delta
;
687 mem
->phys_offset
= old
.phys_offset
+ size_delta
;
690 err
= kvm_set_user_memory_region(s
, mem
);
692 fprintf(stderr
, "%s: error registering suffix slot: %s\n",
693 __func__
, strerror(-err
));
699 /* in case the KVM bug workaround already "consumed" the new slot */
703 /* KVM does not need to know about this memory */
704 if (flags
>= IO_MEM_UNASSIGNED
)
707 mem
= kvm_alloc_slot(s
);
708 mem
->memory_size
= size
;
709 mem
->start_addr
= start_addr
;
710 mem
->phys_offset
= phys_offset
;
713 err
= kvm_set_user_memory_region(s
, mem
);
715 fprintf(stderr
, "%s: error registering slot: %s\n", __func__
,
721 int kvm_ioctl(KVMState
*s
, int type
, ...)
728 arg
= va_arg(ap
, void *);
731 ret
= ioctl(s
->fd
, type
, arg
);
738 int kvm_vm_ioctl(KVMState
*s
, int type
, ...)
745 arg
= va_arg(ap
, void *);
748 ret
= ioctl(s
->vmfd
, type
, arg
);
755 int kvm_vcpu_ioctl(CPUState
*env
, int type
, ...)
762 arg
= va_arg(ap
, void *);
765 ret
= ioctl(env
->kvm_fd
, type
, arg
);
772 int kvm_has_sync_mmu(void)
774 #ifdef KVM_CAP_SYNC_MMU
775 KVMState
*s
= kvm_state
;
777 return kvm_check_extension(s
, KVM_CAP_SYNC_MMU
);
783 void kvm_setup_guest_memory(void *start
, size_t size
)
785 if (!kvm_has_sync_mmu()) {
787 int ret
= madvise(start
, size
, MADV_DONTFORK
);
795 "Need MADV_DONTFORK in absence of synchronous KVM MMU\n");
801 #ifdef KVM_CAP_SET_GUEST_DEBUG
802 struct kvm_sw_breakpoint
*kvm_find_sw_breakpoint(CPUState
*env
,
805 struct kvm_sw_breakpoint
*bp
;
807 TAILQ_FOREACH(bp
, &env
->kvm_state
->kvm_sw_breakpoints
, entry
) {
814 int kvm_sw_breakpoints_active(CPUState
*env
)
816 return !TAILQ_EMPTY(&env
->kvm_state
->kvm_sw_breakpoints
);
819 int kvm_update_guest_debug(CPUState
*env
, unsigned long reinject_trap
)
821 struct kvm_guest_debug dbg
;
824 if (env
->singlestep_enabled
)
825 dbg
.control
= KVM_GUESTDBG_ENABLE
| KVM_GUESTDBG_SINGLESTEP
;
827 kvm_arch_update_guest_debug(env
, &dbg
);
828 dbg
.control
|= reinject_trap
;
830 return kvm_vcpu_ioctl(env
, KVM_SET_GUEST_DEBUG
, &dbg
);
833 int kvm_insert_breakpoint(CPUState
*current_env
, target_ulong addr
,
834 target_ulong len
, int type
)
836 struct kvm_sw_breakpoint
*bp
;
840 if (type
== GDB_BREAKPOINT_SW
) {
841 bp
= kvm_find_sw_breakpoint(current_env
, addr
);
847 bp
= qemu_malloc(sizeof(struct kvm_sw_breakpoint
));
853 err
= kvm_arch_insert_sw_breakpoint(current_env
, bp
);
859 TAILQ_INSERT_HEAD(¤t_env
->kvm_state
->kvm_sw_breakpoints
,
862 err
= kvm_arch_insert_hw_breakpoint(addr
, len
, type
);
867 for (env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
868 err
= kvm_update_guest_debug(env
, 0);
875 int kvm_remove_breakpoint(CPUState
*current_env
, target_ulong addr
,
876 target_ulong len
, int type
)
878 struct kvm_sw_breakpoint
*bp
;
882 if (type
== GDB_BREAKPOINT_SW
) {
883 bp
= kvm_find_sw_breakpoint(current_env
, addr
);
887 if (bp
->use_count
> 1) {
892 err
= kvm_arch_remove_sw_breakpoint(current_env
, bp
);
896 TAILQ_REMOVE(¤t_env
->kvm_state
->kvm_sw_breakpoints
, bp
, entry
);
899 err
= kvm_arch_remove_hw_breakpoint(addr
, len
, type
);
904 for (env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
905 err
= kvm_update_guest_debug(env
, 0);
912 void kvm_remove_all_breakpoints(CPUState
*current_env
)
914 struct kvm_sw_breakpoint
*bp
, *next
;
915 KVMState
*s
= current_env
->kvm_state
;
918 TAILQ_FOREACH_SAFE(bp
, &s
->kvm_sw_breakpoints
, entry
, next
) {
919 if (kvm_arch_remove_sw_breakpoint(current_env
, bp
) != 0) {
920 /* Try harder to find a CPU that currently sees the breakpoint. */
921 for (env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
922 if (kvm_arch_remove_sw_breakpoint(env
, bp
) == 0)
927 kvm_arch_remove_all_hw_breakpoints();
929 for (env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
)
930 kvm_update_guest_debug(env
, 0);
933 #else /* !KVM_CAP_SET_GUEST_DEBUG */
935 int kvm_update_guest_debug(CPUState
*env
, unsigned long reinject_trap
)
940 int kvm_insert_breakpoint(CPUState
*current_env
, target_ulong addr
,
941 target_ulong len
, int type
)
946 int kvm_remove_breakpoint(CPUState
*current_env
, target_ulong addr
,
947 target_ulong len
, int type
)
952 void kvm_remove_all_breakpoints(CPUState
*current_env
)
955 #endif /* !KVM_CAP_SET_GUEST_DEBUG */