4 * Copyright IBM, Corp. 2008
8 * Anthony Liguori <aliguori@us.ibm.com>
9 * Glauber Costa <gcosta@redhat.com>
11 * This work is licensed under the terms of the GNU GPL, version 2 or later.
12 * See the COPYING file in the top-level directory.
16 #include <sys/types.h>
17 #include <sys/ioctl.h>
21 #include <linux/kvm.h>
23 #include "qemu-common.h"
24 #include "qemu-barrier.h"
31 /* This check must be after config-host.h is included */
33 #include <sys/eventfd.h>
36 /* KVM uses PAGE_SIZE in it's definition of COALESCED_MMIO_MAX */
37 #define PAGE_SIZE TARGET_PAGE_SIZE
42 #define DPRINTF(fmt, ...) \
43 do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
45 #define DPRINTF(fmt, ...) \
49 #ifdef OBSOLETE_KVM_IMPL
51 typedef struct KVMSlot
53 target_phys_addr_t start_addr
;
54 ram_addr_t memory_size
;
55 ram_addr_t phys_offset
;
60 typedef struct kvm_dirty_log KVMDirtyLog
;
68 struct kvm_coalesced_mmio_ring
*coalesced_mmio_ring
;
69 int broken_set_mem_region
;
72 int robust_singlestep
;
74 #ifdef KVM_CAP_SET_GUEST_DEBUG
75 struct kvm_sw_breakpoint_head kvm_sw_breakpoints
;
77 int irqchip_in_kernel
;
86 static const KVMCapabilityInfo kvm_required_capabilites
[] = {
87 KVM_CAP_INFO(USER_MEMORY
),
88 KVM_CAP_INFO(DESTROY_MEMORY_REGION_WORKS
),
94 static KVMSlot
*kvm_alloc_slot(KVMState
*s
)
98 for (i
= 0; i
< ARRAY_SIZE(s
->slots
); i
++) {
99 if (s
->slots
[i
].memory_size
== 0) {
104 fprintf(stderr
, "%s: no free slot available\n", __func__
);
108 static KVMSlot
*kvm_lookup_matching_slot(KVMState
*s
,
109 target_phys_addr_t start_addr
,
110 target_phys_addr_t end_addr
)
114 for (i
= 0; i
< ARRAY_SIZE(s
->slots
); i
++) {
115 KVMSlot
*mem
= &s
->slots
[i
];
117 if (start_addr
== mem
->start_addr
&&
118 end_addr
== mem
->start_addr
+ mem
->memory_size
) {
127 * Find overlapping slot with lowest start address
129 static KVMSlot
*kvm_lookup_overlapping_slot(KVMState
*s
,
130 target_phys_addr_t start_addr
,
131 target_phys_addr_t end_addr
)
133 KVMSlot
*found
= NULL
;
136 for (i
= 0; i
< ARRAY_SIZE(s
->slots
); i
++) {
137 KVMSlot
*mem
= &s
->slots
[i
];
139 if (mem
->memory_size
== 0 ||
140 (found
&& found
->start_addr
< mem
->start_addr
)) {
144 if (end_addr
> mem
->start_addr
&&
145 start_addr
< mem
->start_addr
+ mem
->memory_size
) {
153 int kvm_physical_memory_addr_from_ram(KVMState
*s
, ram_addr_t ram_addr
,
154 target_phys_addr_t
*phys_addr
)
158 for (i
= 0; i
< ARRAY_SIZE(s
->slots
); i
++) {
159 KVMSlot
*mem
= &s
->slots
[i
];
161 if (ram_addr
>= mem
->phys_offset
&&
162 ram_addr
< mem
->phys_offset
+ mem
->memory_size
) {
163 *phys_addr
= mem
->start_addr
+ (ram_addr
- mem
->phys_offset
);
171 static int kvm_set_user_memory_region(KVMState
*s
, KVMSlot
*slot
)
173 struct kvm_userspace_memory_region mem
;
175 mem
.slot
= slot
->slot
;
176 mem
.guest_phys_addr
= slot
->start_addr
;
177 mem
.memory_size
= slot
->memory_size
;
178 mem
.userspace_addr
= (unsigned long)qemu_safe_ram_ptr(slot
->phys_offset
);
179 mem
.flags
= slot
->flags
;
180 if (s
->migration_log
) {
181 mem
.flags
|= KVM_MEM_LOG_DIRTY_PAGES
;
183 return kvm_vm_ioctl(s
, KVM_SET_USER_MEMORY_REGION
, &mem
);
186 #ifdef OBSOLETE_KVM_IMPL
187 static void kvm_reset_vcpu(void *opaque
)
189 CPUState
*env
= opaque
;
191 kvm_arch_reset_vcpu(env
);
195 int kvm_irqchip_in_kernel(void)
197 return kvm_state
->irqchip_in_kernel
;
200 int kvm_pit_in_kernel(void)
202 return kvm_state
->pit_in_kernel
;
205 #ifdef OBSOLETE_KVM_IMPL
206 int kvm_init_vcpu(CPUState
*env
)
208 KVMState
*s
= kvm_state
;
212 DPRINTF("kvm_init_vcpu\n");
214 ret
= kvm_vm_ioctl(s
, KVM_CREATE_VCPU
, env
->cpu_index
);
216 DPRINTF("kvm_create_vcpu failed\n");
222 env
->kvm_vcpu_dirty
= 1;
224 mmap_size
= kvm_ioctl(s
, KVM_GET_VCPU_MMAP_SIZE
, 0);
227 DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
231 env
->kvm_run
= mmap(NULL
, mmap_size
, PROT_READ
| PROT_WRITE
, MAP_SHARED
,
233 if (env
->kvm_run
== MAP_FAILED
) {
235 DPRINTF("mmap'ing vcpu state failed\n");
239 if (s
->coalesced_mmio
&& !s
->coalesced_mmio_ring
) {
240 s
->coalesced_mmio_ring
=
241 (void *)env
->kvm_run
+ s
->coalesced_mmio
* PAGE_SIZE
;
244 ret
= kvm_arch_init_vcpu(env
);
246 qemu_register_reset(kvm_reset_vcpu
, env
);
247 kvm_arch_reset_vcpu(env
);
255 * dirty pages logging control
257 static int kvm_dirty_pages_log_change(target_phys_addr_t phys_addr
,
258 ram_addr_t size
, int flags
, int mask
)
260 KVMState
*s
= kvm_state
;
261 KVMSlot
*mem
= kvm_lookup_matching_slot(s
, phys_addr
, phys_addr
+ size
);
265 fprintf(stderr
, "BUG: %s: invalid parameters " TARGET_FMT_plx
"-"
266 TARGET_FMT_plx
"\n", __func__
, phys_addr
,
267 (target_phys_addr_t
)(phys_addr
+ size
- 1));
271 old_flags
= mem
->flags
;
273 flags
= (mem
->flags
& ~mask
) | flags
;
276 /* If nothing changed effectively, no need to issue ioctl */
277 if (s
->migration_log
) {
278 flags
|= KVM_MEM_LOG_DIRTY_PAGES
;
280 if (flags
== old_flags
) {
284 return kvm_set_user_memory_region(s
, mem
);
287 static int kvm_log_start(CPUPhysMemoryClient
*client
,
288 target_phys_addr_t phys_addr
, ram_addr_t size
)
290 return kvm_dirty_pages_log_change(phys_addr
, size
, KVM_MEM_LOG_DIRTY_PAGES
,
291 KVM_MEM_LOG_DIRTY_PAGES
);
294 static int kvm_log_stop(CPUPhysMemoryClient
*client
,
295 target_phys_addr_t phys_addr
, ram_addr_t size
)
297 return kvm_dirty_pages_log_change(phys_addr
, size
, 0,
298 KVM_MEM_LOG_DIRTY_PAGES
);
301 static int kvm_set_migration_log(int enable
)
303 KVMState
*s
= kvm_state
;
307 s
->migration_log
= enable
;
309 for (i
= 0; i
< ARRAY_SIZE(s
->slots
); i
++) {
312 if (!mem
->memory_size
) {
315 if (!!(mem
->flags
& KVM_MEM_LOG_DIRTY_PAGES
) == enable
) {
318 err
= kvm_set_user_memory_region(s
, mem
);
326 /* get kvm's dirty pages bitmap and update qemu's */
327 static int kvm_get_dirty_pages_log_range(unsigned long start_addr
,
328 unsigned long *bitmap
,
329 unsigned long offset
,
330 unsigned long mem_size
)
333 unsigned long page_number
, addr
, addr1
, c
;
335 unsigned int len
= ((mem_size
/ TARGET_PAGE_SIZE
) + HOST_LONG_BITS
- 1) /
339 * bitmap-traveling is faster than memory-traveling (for addr...)
340 * especially when most of the memory is not dirty.
342 for (i
= 0; i
< len
; i
++) {
343 if (bitmap
[i
] != 0) {
344 c
= leul_to_cpu(bitmap
[i
]);
348 page_number
= i
* HOST_LONG_BITS
+ j
;
349 addr1
= page_number
* TARGET_PAGE_SIZE
;
350 addr
= offset
+ addr1
;
351 ram_addr
= cpu_get_physical_page_desc(addr
);
352 cpu_physical_memory_set_dirty(ram_addr
);
359 #define ALIGN(x, y) (((x)+(y)-1) & ~((y)-1))
362 * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
363 * This function updates qemu's dirty bitmap using cpu_physical_memory_set_dirty().
364 * This means all bits are set to dirty.
366 * @start_add: start of logged region.
367 * @end_addr: end of logged region.
369 static int kvm_physical_sync_dirty_bitmap(target_phys_addr_t start_addr
,
370 target_phys_addr_t end_addr
)
372 KVMState
*s
= kvm_state
;
373 unsigned long size
, allocated_size
= 0;
378 d
.dirty_bitmap
= NULL
;
379 while (start_addr
< end_addr
) {
380 mem
= kvm_lookup_overlapping_slot(s
, start_addr
, end_addr
);
385 size
= ALIGN(((mem
->memory_size
) >> TARGET_PAGE_BITS
), HOST_LONG_BITS
) / 8;
386 if (!d
.dirty_bitmap
) {
387 d
.dirty_bitmap
= qemu_malloc(size
);
388 } else if (size
> allocated_size
) {
389 d
.dirty_bitmap
= qemu_realloc(d
.dirty_bitmap
, size
);
391 allocated_size
= size
;
392 memset(d
.dirty_bitmap
, 0, allocated_size
);
396 if (kvm_vm_ioctl(s
, KVM_GET_DIRTY_LOG
, &d
) == -1) {
397 DPRINTF("ioctl failed %d\n", errno
);
402 kvm_get_dirty_pages_log_range(mem
->start_addr
, d
.dirty_bitmap
,
403 mem
->start_addr
, mem
->memory_size
);
404 start_addr
= mem
->start_addr
+ mem
->memory_size
;
406 qemu_free(d
.dirty_bitmap
);
411 int kvm_coalesce_mmio_region(target_phys_addr_t start
, ram_addr_t size
)
414 KVMState
*s
= kvm_state
;
416 if (s
->coalesced_mmio
) {
417 struct kvm_coalesced_mmio_zone zone
;
422 ret
= kvm_vm_ioctl(s
, KVM_REGISTER_COALESCED_MMIO
, &zone
);
428 int kvm_uncoalesce_mmio_region(target_phys_addr_t start
, ram_addr_t size
)
431 KVMState
*s
= kvm_state
;
433 if (s
->coalesced_mmio
) {
434 struct kvm_coalesced_mmio_zone zone
;
439 ret
= kvm_vm_ioctl(s
, KVM_UNREGISTER_COALESCED_MMIO
, &zone
);
445 int kvm_check_extension(KVMState
*s
, unsigned int extension
)
449 ret
= kvm_ioctl(s
, KVM_CHECK_EXTENSION
, extension
);
457 static int kvm_check_many_ioeventfds(void)
459 /* Userspace can use ioeventfd for io notification. This requires a host
460 * that supports eventfd(2) and an I/O thread; since eventfd does not
461 * support SIGIO it cannot interrupt the vcpu.
463 * Older kernels have a 6 device limit on the KVM io bus. Find out so we
464 * can avoid creating too many ioeventfds.
466 #if defined(CONFIG_EVENTFD) /* && defined(CONFIG_IOTHREAD) */
469 for (i
= 0; i
< ARRAY_SIZE(ioeventfds
); i
++) {
470 ioeventfds
[i
] = eventfd(0, EFD_CLOEXEC
);
471 if (ioeventfds
[i
] < 0) {
474 ret
= kvm_set_ioeventfd_pio_word(ioeventfds
[i
], 0, i
, true);
476 close(ioeventfds
[i
]);
481 /* Decide whether many devices are supported or not */
482 ret
= i
== ARRAY_SIZE(ioeventfds
);
485 kvm_set_ioeventfd_pio_word(ioeventfds
[i
], 0, i
, false);
486 close(ioeventfds
[i
]);
494 #ifdef OBSOLETE_KVM_IMPL
495 static const KVMCapabilityInfo
*
496 kvm_check_extension_list(KVMState
*s
, const KVMCapabilityInfo
*list
)
499 if (!kvm_check_extension(s
, list
->value
)) {
508 static void kvm_set_phys_mem(target_phys_addr_t start_addr
, ram_addr_t size
,
509 ram_addr_t phys_offset
)
511 KVMState
*s
= kvm_state
;
512 ram_addr_t flags
= phys_offset
& ~TARGET_PAGE_MASK
;
516 /* kvm works in page size chunks, but the function may be called
517 with sub-page size and unaligned start address. */
518 size
= TARGET_PAGE_ALIGN(size
);
519 start_addr
= TARGET_PAGE_ALIGN(start_addr
);
521 /* KVM does not support read-only slots */
522 phys_offset
&= ~IO_MEM_ROM
;
525 mem
= kvm_lookup_overlapping_slot(s
, start_addr
, start_addr
+ size
);
530 if (flags
< IO_MEM_UNASSIGNED
&& start_addr
>= mem
->start_addr
&&
531 (start_addr
+ size
<= mem
->start_addr
+ mem
->memory_size
) &&
532 (phys_offset
- start_addr
== mem
->phys_offset
- mem
->start_addr
)) {
533 /* The new slot fits into the existing one and comes with
534 * identical parameters - nothing to be done. */
540 /* unregister the overlapping slot */
541 mem
->memory_size
= 0;
542 err
= kvm_set_user_memory_region(s
, mem
);
544 fprintf(stderr
, "%s: error unregistering overlapping slot: %s\n",
545 __func__
, strerror(-err
));
549 /* Workaround for older KVM versions: we can't join slots, even not by
550 * unregistering the previous ones and then registering the larger
551 * slot. We have to maintain the existing fragmentation. Sigh.
553 * This workaround assumes that the new slot starts at the same
554 * address as the first existing one. If not or if some overlapping
555 * slot comes around later, we will fail (not seen in practice so far)
556 * - and actually require a recent KVM version. */
557 if (s
->broken_set_mem_region
&&
558 old
.start_addr
== start_addr
&& old
.memory_size
< size
&&
559 flags
< IO_MEM_UNASSIGNED
) {
560 mem
= kvm_alloc_slot(s
);
561 mem
->memory_size
= old
.memory_size
;
562 mem
->start_addr
= old
.start_addr
;
563 mem
->phys_offset
= old
.phys_offset
;
566 err
= kvm_set_user_memory_region(s
, mem
);
568 fprintf(stderr
, "%s: error updating slot: %s\n", __func__
,
573 start_addr
+= old
.memory_size
;
574 phys_offset
+= old
.memory_size
;
575 size
-= old
.memory_size
;
579 /* register prefix slot */
580 if (old
.start_addr
< start_addr
) {
581 mem
= kvm_alloc_slot(s
);
582 mem
->memory_size
= start_addr
- old
.start_addr
;
583 mem
->start_addr
= old
.start_addr
;
584 mem
->phys_offset
= old
.phys_offset
;
587 err
= kvm_set_user_memory_region(s
, mem
);
589 fprintf(stderr
, "%s: error registering prefix slot: %s\n",
590 __func__
, strerror(-err
));
595 /* register suffix slot */
596 if (old
.start_addr
+ old
.memory_size
> start_addr
+ size
) {
597 ram_addr_t size_delta
;
599 mem
= kvm_alloc_slot(s
);
600 mem
->start_addr
= start_addr
+ size
;
601 size_delta
= mem
->start_addr
- old
.start_addr
;
602 mem
->memory_size
= old
.memory_size
- size_delta
;
603 mem
->phys_offset
= old
.phys_offset
+ size_delta
;
606 err
= kvm_set_user_memory_region(s
, mem
);
608 fprintf(stderr
, "%s: error registering suffix slot: %s\n",
609 __func__
, strerror(-err
));
615 /* in case the KVM bug workaround already "consumed" the new slot */
619 /* KVM does not need to know about this memory */
620 if (flags
>= IO_MEM_UNASSIGNED
) {
623 mem
= kvm_alloc_slot(s
);
624 mem
->memory_size
= size
;
625 mem
->start_addr
= start_addr
;
626 mem
->phys_offset
= phys_offset
;
629 err
= kvm_set_user_memory_region(s
, mem
);
631 fprintf(stderr
, "%s: error registering slot: %s\n", __func__
,
637 static void kvm_client_set_memory(struct CPUPhysMemoryClient
*client
,
638 target_phys_addr_t start_addr
,
639 ram_addr_t size
, ram_addr_t phys_offset
)
641 kvm_set_phys_mem(start_addr
, size
, phys_offset
);
644 static int kvm_client_sync_dirty_bitmap(struct CPUPhysMemoryClient
*client
,
645 target_phys_addr_t start_addr
,
646 target_phys_addr_t end_addr
)
648 return kvm_physical_sync_dirty_bitmap(start_addr
, end_addr
);
651 static int kvm_client_migration_log(struct CPUPhysMemoryClient
*client
,
654 return kvm_set_migration_log(enable
);
657 static CPUPhysMemoryClient kvm_cpu_phys_memory_client
= {
658 .set_memory
= kvm_client_set_memory
,
659 .sync_dirty_bitmap
= kvm_client_sync_dirty_bitmap
,
660 .migration_log
= kvm_client_migration_log
,
661 .log_start
= kvm_log_start
,
662 .log_stop
= kvm_log_stop
,
665 void kvm_cpu_register_phys_memory_client(void)
667 cpu_register_phys_memory_client(&kvm_cpu_phys_memory_client
);
670 #ifdef OBSOLETE_KVM_IMPL
674 static const char upgrade_note
[] =
675 "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
676 "(see http://sourceforge.net/projects/kvm).\n";
678 const KVMCapabilityInfo
*missing_cap
;
682 s
= qemu_mallocz(sizeof(KVMState
));
684 #ifdef KVM_CAP_SET_GUEST_DEBUG
685 QTAILQ_INIT(&s
->kvm_sw_breakpoints
);
687 for (i
= 0; i
< ARRAY_SIZE(s
->slots
); i
++) {
688 s
->slots
[i
].slot
= i
;
691 s
->fd
= qemu_open("/dev/kvm", O_RDWR
);
693 fprintf(stderr
, "Could not access KVM kernel module: %m\n");
698 ret
= kvm_ioctl(s
, KVM_GET_API_VERSION
, 0);
699 if (ret
< KVM_API_VERSION
) {
703 fprintf(stderr
, "kvm version too old\n");
707 if (ret
> KVM_API_VERSION
) {
709 fprintf(stderr
, "kvm version not supported\n");
713 s
->vmfd
= kvm_ioctl(s
, KVM_CREATE_VM
, 0);
716 fprintf(stderr
, "Please add the 'switch_amode' kernel parameter to "
717 "your host kernel command line\n");
722 missing_cap
= kvm_check_extension_list(s
, kvm_required_capabilites
);
725 kvm_check_extension_list(s
, kvm_arch_required_capabilities
);
729 fprintf(stderr
, "kvm does not support %s\n%s",
730 missing_cap
->name
, upgrade_note
);
734 s
->coalesced_mmio
= kvm_check_extension(s
, KVM_CAP_COALESCED_MMIO
);
736 s
->broken_set_mem_region
= 1;
737 #ifdef KVM_CAP_JOIN_MEMORY_REGIONS_WORKS
738 ret
= kvm_check_extension(s
, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS
);
740 s
->broken_set_mem_region
= 0;
745 #ifdef KVM_CAP_VCPU_EVENTS
746 s
->vcpu_events
= kvm_check_extension(s
, KVM_CAP_VCPU_EVENTS
);
749 s
->robust_singlestep
= 0;
750 #ifdef KVM_CAP_X86_ROBUST_SINGLESTEP
751 s
->robust_singlestep
=
752 kvm_check_extension(s
, KVM_CAP_X86_ROBUST_SINGLESTEP
);
756 #ifdef KVM_CAP_DEBUGREGS
757 s
->debugregs
= kvm_check_extension(s
, KVM_CAP_DEBUGREGS
);
762 s
->xsave
= kvm_check_extension(s
, KVM_CAP_XSAVE
);
767 s
->xcrs
= kvm_check_extension(s
, KVM_CAP_XCRS
);
770 ret
= kvm_arch_init(s
);
776 cpu_register_phys_memory_client(&kvm_cpu_phys_memory_client
);
778 s
->many_ioeventfds
= kvm_check_many_ioeventfds();
797 static void kvm_handle_io(uint16_t port
, void *data
, int direction
, int size
,
803 for (i
= 0; i
< count
; i
++) {
804 if (direction
== KVM_EXIT_IO_IN
) {
807 stb_p(ptr
, cpu_inb(port
));
810 stw_p(ptr
, cpu_inw(port
));
813 stl_p(ptr
, cpu_inl(port
));
819 cpu_outb(port
, ldub_p(ptr
));
822 cpu_outw(port
, lduw_p(ptr
));
825 cpu_outl(port
, ldl_p(ptr
));
834 #ifdef KVM_CAP_INTERNAL_ERROR_DATA
835 static int kvm_handle_internal_error(CPUState
*env
, struct kvm_run
*run
)
837 fprintf(stderr
, "KVM internal error.");
838 if (kvm_check_extension(kvm_state
, KVM_CAP_INTERNAL_ERROR_DATA
)) {
841 fprintf(stderr
, " Suberror: %d\n", run
->internal
.suberror
);
842 for (i
= 0; i
< run
->internal
.ndata
; ++i
) {
843 fprintf(stderr
, "extra data[%d]: %"PRIx64
"\n",
844 i
, (uint64_t)run
->internal
.data
[i
]);
847 fprintf(stderr
, "\n");
849 if (run
->internal
.suberror
== KVM_INTERNAL_ERROR_EMULATION
) {
850 fprintf(stderr
, "emulation failure\n");
851 if (!kvm_arch_stop_on_emulation_error(env
)) {
852 cpu_dump_state(env
, stderr
, fprintf
, CPU_DUMP_CODE
);
853 return EXCP_INTERRUPT
;
856 /* FIXME: Should trigger a qmp message to let management know
857 * something went wrong.
863 void kvm_flush_coalesced_mmio_buffer(void)
865 KVMState
*s
= kvm_state
;
866 if (s
->coalesced_mmio_ring
) {
867 struct kvm_coalesced_mmio_ring
*ring
= s
->coalesced_mmio_ring
;
868 while (ring
->first
!= ring
->last
) {
869 struct kvm_coalesced_mmio
*ent
;
871 ent
= &ring
->coalesced_mmio
[ring
->first
];
873 cpu_physical_memory_write(ent
->phys_addr
, ent
->data
, ent
->len
);
875 ring
->first
= (ring
->first
+ 1) % KVM_COALESCED_MMIO_MAX
;
880 #ifdef OBSOLETE_KVM_IMPL
882 static void do_kvm_cpu_synchronize_state(void *_env
)
884 CPUState
*env
= _env
;
886 if (!env
->kvm_vcpu_dirty
) {
887 kvm_arch_get_registers(env
);
888 env
->kvm_vcpu_dirty
= 1;
892 void kvm_cpu_synchronize_state(CPUState
*env
)
894 if (!env
->kvm_vcpu_dirty
) {
895 run_on_cpu(env
, do_kvm_cpu_synchronize_state
, env
);
899 void kvm_cpu_synchronize_post_reset(CPUState
*env
)
901 kvm_arch_put_registers(env
, KVM_PUT_RESET_STATE
);
902 env
->kvm_vcpu_dirty
= 0;
905 void kvm_cpu_synchronize_post_init(CPUState
*env
)
907 kvm_arch_put_registers(env
, KVM_PUT_FULL_STATE
);
908 env
->kvm_vcpu_dirty
= 0;
911 int kvm_cpu_exec(CPUState
*env
)
913 struct kvm_run
*run
= env
->kvm_run
;
916 DPRINTF("kvm_cpu_exec()\n");
918 if (kvm_arch_process_async_events(env
)) {
919 env
->exit_request
= 0;
923 cpu_single_env
= env
;
926 if (env
->kvm_vcpu_dirty
) {
927 kvm_arch_put_registers(env
, KVM_PUT_RUNTIME_STATE
);
928 env
->kvm_vcpu_dirty
= 0;
931 kvm_arch_pre_run(env
, run
);
932 if (env
->exit_request
) {
933 DPRINTF("interrupt exit requested\n");
935 * KVM requires us to reenter the kernel after IO exits to complete
936 * instruction emulation. This self-signal will ensure that we
939 qemu_cpu_kick_self();
941 cpu_single_env
= NULL
;
942 qemu_mutex_unlock_iothread();
944 run_ret
= kvm_vcpu_ioctl(env
, KVM_RUN
, 0);
946 qemu_mutex_lock_iothread();
947 cpu_single_env
= env
;
948 kvm_arch_post_run(env
, run
);
950 kvm_flush_coalesced_mmio_buffer();
953 if (run_ret
== -EINTR
|| run_ret
== -EAGAIN
) {
954 DPRINTF("io window exit\n");
955 ret
= EXCP_INTERRUPT
;
958 DPRINTF("kvm run failed %s\n", strerror(-run_ret
));
962 switch (run
->exit_reason
) {
964 DPRINTF("handle_io\n");
965 kvm_handle_io(run
->io
.port
,
966 (uint8_t *)run
+ run
->io
.data_offset
,
973 DPRINTF("handle_mmio\n");
974 cpu_physical_memory_rw(run
->mmio
.phys_addr
,
980 case KVM_EXIT_IRQ_WINDOW_OPEN
:
981 DPRINTF("irq_window_open\n");
982 ret
= EXCP_INTERRUPT
;
984 case KVM_EXIT_SHUTDOWN
:
985 DPRINTF("shutdown\n");
986 qemu_system_reset_request();
987 ret
= EXCP_INTERRUPT
;
989 case KVM_EXIT_UNKNOWN
:
990 fprintf(stderr
, "KVM: unknown exit, hardware reason %" PRIx64
"\n",
991 (uint64_t)run
->hw
.hardware_exit_reason
);
994 #ifdef KVM_CAP_INTERNAL_ERROR_DATA
995 case KVM_EXIT_INTERNAL_ERROR
:
996 ret
= kvm_handle_internal_error(env
, run
);
1000 DPRINTF("kvm_arch_handle_exit\n");
1001 ret
= kvm_arch_handle_exit(env
, run
);
1007 cpu_dump_state(env
, stderr
, fprintf
, CPU_DUMP_CODE
);
1008 vm_stop(VMSTOP_PANIC
);
1011 env
->exit_request
= 0;
1012 cpu_single_env
= NULL
;
1017 int kvm_ioctl(KVMState
*s
, int type
, ...)
1024 arg
= va_arg(ap
, void *);
1027 ret
= ioctl(s
->fd
, type
, arg
);
1034 int kvm_vm_ioctl(KVMState
*s
, int type
, ...)
1041 arg
= va_arg(ap
, void *);
1044 ret
= ioctl(s
->vmfd
, type
, arg
);
1051 int kvm_vcpu_ioctl(CPUState
*env
, int type
, ...)
1058 arg
= va_arg(ap
, void *);
1061 ret
= ioctl(env
->kvm_fd
, type
, arg
);
1068 int kvm_has_sync_mmu(void)
1070 return kvm_check_extension(kvm_state
, KVM_CAP_SYNC_MMU
);
1073 int kvm_has_vcpu_events(void)
1075 return kvm_state
->vcpu_events
;
1078 int kvm_has_robust_singlestep(void)
1080 return kvm_state
->robust_singlestep
;
1083 int kvm_has_debugregs(void)
1085 return kvm_state
->debugregs
;
1088 int kvm_has_xsave(void)
1090 return kvm_state
->xsave
;
1093 int kvm_has_xcrs(void)
1095 return kvm_state
->xcrs
;
1098 int kvm_has_many_ioeventfds(void)
1100 if (!kvm_enabled()) {
1103 return kvm_state
->many_ioeventfds
;
1106 void kvm_setup_guest_memory(void *start
, size_t size
)
1108 if (!kvm_has_sync_mmu()) {
1109 int ret
= qemu_madvise(start
, size
, QEMU_MADV_DONTFORK
);
1112 perror("qemu_madvise");
1114 "Need MADV_DONTFORK in absence of synchronous KVM MMU\n");
1120 #ifdef KVM_CAP_SET_GUEST_DEBUG
1121 #ifndef OBSOLETE_KVM_IMPL
1122 #define run_on_cpu on_vcpu
1123 #endif /* !OBSOLETE_KVM_IMPL */
1125 struct kvm_sw_breakpoint
*kvm_find_sw_breakpoint(CPUState
*env
,
1128 struct kvm_sw_breakpoint
*bp
;
1130 QTAILQ_FOREACH(bp
, &env
->kvm_state
->kvm_sw_breakpoints
, entry
) {
1138 int kvm_sw_breakpoints_active(CPUState
*env
)
1140 return !QTAILQ_EMPTY(&env
->kvm_state
->kvm_sw_breakpoints
);
1143 struct kvm_set_guest_debug_data
{
1144 struct kvm_guest_debug dbg
;
1149 static void kvm_invoke_set_guest_debug(void *data
)
1151 struct kvm_set_guest_debug_data
*dbg_data
= data
;
1152 CPUState
*env
= dbg_data
->env
;
1154 dbg_data
->err
= kvm_vcpu_ioctl(env
, KVM_SET_GUEST_DEBUG
, &dbg_data
->dbg
);
1157 int kvm_update_guest_debug(CPUState
*env
, unsigned long reinject_trap
)
1159 struct kvm_set_guest_debug_data data
;
1161 data
.dbg
.control
= reinject_trap
;
1163 if (env
->singlestep_enabled
) {
1164 data
.dbg
.control
|= KVM_GUESTDBG_ENABLE
| KVM_GUESTDBG_SINGLESTEP
;
1166 kvm_arch_update_guest_debug(env
, &data
.dbg
);
1169 run_on_cpu(env
, kvm_invoke_set_guest_debug
, &data
);
1173 int kvm_insert_breakpoint(CPUState
*current_env
, target_ulong addr
,
1174 target_ulong len
, int type
)
1176 struct kvm_sw_breakpoint
*bp
;
1180 if (type
== GDB_BREAKPOINT_SW
) {
1181 bp
= kvm_find_sw_breakpoint(current_env
, addr
);
1187 bp
= qemu_malloc(sizeof(struct kvm_sw_breakpoint
));
1194 err
= kvm_arch_insert_sw_breakpoint(current_env
, bp
);
1200 QTAILQ_INSERT_HEAD(¤t_env
->kvm_state
->kvm_sw_breakpoints
,
1203 err
= kvm_arch_insert_hw_breakpoint(addr
, len
, type
);
1209 for (env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
1210 err
= kvm_update_guest_debug(env
, 0);
1218 int kvm_remove_breakpoint(CPUState
*current_env
, target_ulong addr
,
1219 target_ulong len
, int type
)
1221 struct kvm_sw_breakpoint
*bp
;
1225 if (type
== GDB_BREAKPOINT_SW
) {
1226 bp
= kvm_find_sw_breakpoint(current_env
, addr
);
1231 if (bp
->use_count
> 1) {
1236 err
= kvm_arch_remove_sw_breakpoint(current_env
, bp
);
1241 QTAILQ_REMOVE(¤t_env
->kvm_state
->kvm_sw_breakpoints
, bp
, entry
);
1244 err
= kvm_arch_remove_hw_breakpoint(addr
, len
, type
);
1250 for (env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
1251 err
= kvm_update_guest_debug(env
, 0);
1259 void kvm_remove_all_breakpoints(CPUState
*current_env
)
1261 struct kvm_sw_breakpoint
*bp
, *next
;
1262 KVMState
*s
= current_env
->kvm_state
;
1265 QTAILQ_FOREACH_SAFE(bp
, &s
->kvm_sw_breakpoints
, entry
, next
) {
1266 if (kvm_arch_remove_sw_breakpoint(current_env
, bp
) != 0) {
1267 /* Try harder to find a CPU that currently sees the breakpoint. */
1268 for (env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
1269 if (kvm_arch_remove_sw_breakpoint(env
, bp
) == 0) {
1275 kvm_arch_remove_all_hw_breakpoints();
1277 for (env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
1278 kvm_update_guest_debug(env
, 0);
1282 #else /* !KVM_CAP_SET_GUEST_DEBUG */
1284 int kvm_update_guest_debug(CPUState
*env
, unsigned long reinject_trap
)
1289 int kvm_insert_breakpoint(CPUState
*current_env
, target_ulong addr
,
1290 target_ulong len
, int type
)
1295 int kvm_remove_breakpoint(CPUState
*current_env
, target_ulong addr
,
1296 target_ulong len
, int type
)
1301 void kvm_remove_all_breakpoints(CPUState
*current_env
)
1304 #endif /* !KVM_CAP_SET_GUEST_DEBUG */
1306 int kvm_set_signal_mask(CPUState
*env
, const sigset_t
*sigset
)
1308 struct kvm_signal_mask
*sigmask
;
1312 return kvm_vcpu_ioctl(env
, KVM_SET_SIGNAL_MASK
, NULL
);
1315 sigmask
= qemu_malloc(sizeof(*sigmask
) + sizeof(*sigset
));
1318 memcpy(sigmask
->sigset
, sigset
, sizeof(*sigset
));
1319 r
= kvm_vcpu_ioctl(env
, KVM_SET_SIGNAL_MASK
, sigmask
);
1325 int kvm_set_ioeventfd_mmio_long(int fd
, uint32_t addr
, uint32_t val
, bool assign
)
1327 #ifdef KVM_IOEVENTFD
1329 struct kvm_ioeventfd iofd
;
1331 iofd
.datamatch
= val
;
1334 iofd
.flags
= KVM_IOEVENTFD_FLAG_DATAMATCH
;
1337 if (!kvm_enabled()) {
1342 iofd
.flags
|= KVM_IOEVENTFD_FLAG_DEASSIGN
;
1345 ret
= kvm_vm_ioctl(kvm_state
, KVM_IOEVENTFD
, &iofd
);
1357 int kvm_set_ioeventfd_pio_word(int fd
, uint16_t addr
, uint16_t val
, bool assign
)
1359 #ifdef KVM_IOEVENTFD
1360 struct kvm_ioeventfd kick
= {
1364 .flags
= KVM_IOEVENTFD_FLAG_DATAMATCH
| KVM_IOEVENTFD_FLAG_PIO
,
1368 if (!kvm_enabled()) {
1372 kick
.flags
|= KVM_IOEVENTFD_FLAG_DEASSIGN
;
1374 r
= kvm_vm_ioctl(kvm_state
, KVM_IOEVENTFD
, &kick
);
1384 #if defined(KVM_IRQFD)
1385 int kvm_set_irqfd(int gsi
, int fd
, bool assigned
)
1387 struct kvm_irqfd irqfd
= {
1390 .flags
= assigned
? 0 : KVM_IRQFD_FLAG_DEASSIGN
,
1393 if (!kvm_enabled() || !kvm_irqchip_in_kernel())
1396 r
= kvm_vm_ioctl(kvm_state
, KVM_IRQFD
, &irqfd
);
1403 int kvm_on_sigbus_vcpu(CPUState
*env
, int code
, void *addr
)
1405 return kvm_arch_on_sigbus_vcpu(env
, code
, addr
);
1408 int kvm_on_sigbus(int code
, void *addr
)
1410 return kvm_arch_on_sigbus(code
, addr
);
1414 #include "qemu-kvm.c"