?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge commit '253d0942fac33c5c15c9a7f8657f55f125dc5816' into upstream-merge
[qemu-kvm/fedora.git] / qemu-kvm.c
blobc5cd03899b3510771484c725b38f7d3fe5a27dc9
1 /*
2 * qemu/kvm integration
3 *
4 * Copyright (C) 2006-2008 Qumranet Technologies
5 *
6 * Licensed under the terms of the GNU GPL version 2 or higher.
7 */
8 #include "config.h"
9 #include "config-host.h"
10
11 #include <assert.h>
12 #include <string.h>
13 #include "hw/hw.h"
14 #include "sysemu.h"
15 #include "qemu-common.h"
16 #include "console.h"
17 #include "block.h"
18 #include "compatfd.h"
19 #include "gdbstub.h"
20
21 #include "qemu-kvm.h"
22 #include "libkvm-all.h"
23 #include "libkvm.h"
24
25 #include <pthread.h>
26 #include <sys/utsname.h>
27 #include <sys/syscall.h>
28 #include <sys/mman.h>
29 #include <sys/ioctl.h>
30
31 #define false 0
32 #define true 1
33
34 #define EXPECTED_KVM_API_VERSION 12
35
36 #if EXPECTED_KVM_API_VERSION != KVM_API_VERSION
37 #error libkvm: userspace and kernel version mismatch
38 #endif
39
40 int kvm_allowed = 1;
41 int kvm_irqchip = 1;
42 int kvm_pit = 1;
43 int kvm_pit_reinject = 1;
44 int kvm_nested = 0;
45 kvm_context_t kvm_context;
46
47 pthread_mutex_t qemu_mutex = PTHREAD_MUTEX_INITIALIZER;
48 pthread_cond_t qemu_vcpu_cond = PTHREAD_COND_INITIALIZER;
49 pthread_cond_t qemu_system_cond = PTHREAD_COND_INITIALIZER;
50 pthread_cond_t qemu_pause_cond = PTHREAD_COND_INITIALIZER;
51 pthread_cond_t qemu_work_cond = PTHREAD_COND_INITIALIZER;
52 __thread CPUState *current_env;
53
54 static int qemu_system_ready;
55
56 #define SIG_IPI (SIGRTMIN+4)
57
58 pthread_t io_thread;
59 static int io_thread_fd = -1;
60 static int io_thread_sigfd = -1;
61
62 static CPUState *kvm_debug_cpu_requested;
63
64 static uint64_t phys_ram_size;
65
66 /* The list of ioperm_data */
67 static LIST_HEAD(, ioperm_data) ioperm_head;
68
69 //#define DEBUG_MEMREG
70 #ifdef DEBUG_MEMREG
71 #define DPRINTF(fmt, args...) \
72 do { fprintf(stderr, "%s:%d " fmt , __func__, __LINE__, ##args); } while (0)
73 #else
74 #define DPRINTF(fmt, args...) do {} while (0)
75 #endif
76
77 #define ALIGN(x, y) (((x)+(y)-1) & ~((y)-1))
78
79 int kvm_abi = EXPECTED_KVM_API_VERSION;
80 int kvm_page_size;
81
82 static inline void set_gsi(kvm_context_t kvm, unsigned int gsi)
83 {
84 uint32_t *bitmap = kvm->used_gsi_bitmap;
85
86 if (gsi < kvm->max_gsi)
87 bitmap[gsi / 32] |= 1U << (gsi % 32);
88 else
89 DPRINTF("Invalid GSI %d\n");
90 }
91
92 static inline void clear_gsi(kvm_context_t kvm, unsigned int gsi)
93 {
94 uint32_t *bitmap = kvm->used_gsi_bitmap;
95
96 if (gsi < kvm->max_gsi)
97 bitmap[gsi / 32] &= ~(1U << (gsi % 32));
98 else
99 DPRINTF("Invalid GSI %d\n");
100 }
101
102 struct slot_info {
103 unsigned long phys_addr;
104 unsigned long len;
105 unsigned long userspace_addr;
106 unsigned flags;
107 int logging_count;
108 };
109
110 struct slot_info slots[KVM_MAX_NUM_MEM_REGIONS];
111
112 static void init_slots(void)
113 {
114 int i;
115
116 for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS; ++i)
117 slots[i].len = 0;
118 }
119
120 static int get_free_slot(kvm_context_t kvm)
121 {
122 int i;
123 int tss_ext;
124
125 #if defined(KVM_CAP_SET_TSS_ADDR) && !defined(__s390__)
126 tss_ext = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_SET_TSS_ADDR);
127 #else
128 tss_ext = 0;
129 #endif
130
131 /*
132 * on older kernels where the set tss ioctl is not supprted we must save
133 * slot 0 to hold the extended memory, as the vmx will use the last 3
134 * pages of this slot.
135 */
136 if (tss_ext > 0)
137 i = 0;
138 else
139 i = 1;
140
141 for (; i < KVM_MAX_NUM_MEM_REGIONS; ++i)
142 if (!slots[i].len)
143 return i;
144 return -1;
145 }
146
147 static void register_slot(int slot, unsigned long phys_addr, unsigned long len,
148 unsigned long userspace_addr, unsigned flags)
149 {
150 slots[slot].phys_addr = phys_addr;
151 slots[slot].len = len;
152 slots[slot].userspace_addr = userspace_addr;
153 slots[slot].flags = flags;
154 }
155
156 static void free_slot(int slot)
157 {
158 slots[slot].len = 0;
159 slots[slot].logging_count = 0;
160 }
161
162 static int get_slot(unsigned long phys_addr)
163 {
164 int i;
165
166 for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS ; ++i) {
167 if (slots[i].len && slots[i].phys_addr <= phys_addr &&
168 (slots[i].phys_addr + slots[i].len-1) >= phys_addr)
169 return i;
170 }
171 return -1;
172 }
173
174 /* Returns -1 if this slot is not totally contained on any other,
175 * and the number of the slot otherwise */
176 static int get_container_slot(uint64_t phys_addr, unsigned long size)
177 {
178 int i;
179
180 for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS ; ++i)
181 if (slots[i].len && slots[i].phys_addr <= phys_addr &&
182 (slots[i].phys_addr + slots[i].len) >= phys_addr + size)
183 return i;
184 return -1;
185 }
186
187 int kvm_is_containing_region(kvm_context_t kvm, unsigned long phys_addr, unsigned long size)
188 {
189 int slot = get_container_slot(phys_addr, size);
190 if (slot == -1)
191 return 0;
192 return 1;
193 }
194
195 /*
196 * dirty pages logging control
197 */
198 static int kvm_dirty_pages_log_change(kvm_context_t kvm,
199 unsigned long phys_addr,
200 unsigned flags,
201 unsigned mask)
202 {
203 int r = -1;
204 int slot = get_slot(phys_addr);
205
206 if (slot == -1) {
207 fprintf(stderr, "BUG: %s: invalid parameters\n", __FUNCTION__);
208 return 1;
209 }
210
211 flags = (slots[slot].flags & ~mask) | flags;
212 if (flags == slots[slot].flags)
213 return 0;
214 slots[slot].flags = flags;
215
216 {
217 struct kvm_userspace_memory_region mem = {
218 .slot = slot,
219 .memory_size = slots[slot].len,
220 .guest_phys_addr = slots[slot].phys_addr,
221 .userspace_addr = slots[slot].userspace_addr,
222 .flags = slots[slot].flags,
223 };
224
225
226 DPRINTF("slot %d start %llx len %llx flags %x\n",
227 mem.slot,
228 mem.guest_phys_addr,
229 mem.memory_size,
230 mem.flags);
231 r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &mem);
232 if (r == -1)
233 fprintf(stderr, "%s: %m\n", __FUNCTION__);
234 }
235 return r;
236 }
237
238 static int kvm_dirty_pages_log_change_all(kvm_context_t kvm,
239 int (*change)(kvm_context_t kvm,
240 uint64_t start,
241 uint64_t len))
242 {
243 int i, r;
244
245 for (i=r=0; i<KVM_MAX_NUM_MEM_REGIONS && r==0; i++) {
246 if (slots[i].len)
247 r = change(kvm, slots[i].phys_addr, slots[i].len);
248 }
249 return r;
250 }
251
252 int kvm_dirty_pages_log_enable_slot(kvm_context_t kvm,
253 uint64_t phys_addr,
254 uint64_t len)
255 {
256 int slot = get_slot(phys_addr);
257
258 DPRINTF("start %"PRIx64" len %"PRIx64"\n", phys_addr, len);
259 if (slot == -1) {
260 fprintf(stderr, "BUG: %s: invalid parameters\n", __func__);
261 return -EINVAL;
262 }
263
264 if (slots[slot].logging_count++)
265 return 0;
266
267 return kvm_dirty_pages_log_change(kvm, slots[slot].phys_addr,
268 KVM_MEM_LOG_DIRTY_PAGES,
269 KVM_MEM_LOG_DIRTY_PAGES);
270 }
271
272 int kvm_dirty_pages_log_disable_slot(kvm_context_t kvm,
273 uint64_t phys_addr,
274 uint64_t len)
275 {
276 int slot = get_slot(phys_addr);
277
278 if (slot == -1) {
279 fprintf(stderr, "BUG: %s: invalid parameters\n", __func__);
280 return -EINVAL;
281 }
282
283 if (--slots[slot].logging_count)
284 return 0;
285
286 return kvm_dirty_pages_log_change(kvm, slots[slot].phys_addr,
287 0,
288 KVM_MEM_LOG_DIRTY_PAGES);
289 }
290
291 /**
292 * Enable dirty page logging for all memory regions
293 */
294 int kvm_dirty_pages_log_enable_all(kvm_context_t kvm)
295 {
296 if (kvm->dirty_pages_log_all)
297 return 0;
298 kvm->dirty_pages_log_all = 1;
299 return kvm_dirty_pages_log_change_all(kvm,
300 kvm_dirty_pages_log_enable_slot);
301 }
302
303 /**
304 * Enable dirty page logging only for memory regions that were created with
305 * dirty logging enabled (disable for all other memory regions).
306 */
307 int kvm_dirty_pages_log_reset(kvm_context_t kvm)
308 {
309 if (!kvm->dirty_pages_log_all)
310 return 0;
311 kvm->dirty_pages_log_all = 0;
312 return kvm_dirty_pages_log_change_all(kvm,
313 kvm_dirty_pages_log_disable_slot);
314 }
315
316
317 kvm_context_t kvm_init(struct kvm_callbacks *callbacks,
318 void *opaque)
319 {
320 int fd;
321 kvm_context_t kvm;
322 int r, gsi_count;
323
324 fd = open("/dev/kvm", O_RDWR);
325 if (fd == -1) {
326 perror("open /dev/kvm");
327 return NULL;
328 }
329 r = ioctl(fd, KVM_GET_API_VERSION, 0);
330 if (r == -1) {
331 fprintf(stderr, "kvm kernel version too old: "
332 "KVM_GET_API_VERSION ioctl not supported\n");
333 goto out_close;
334 }
335 if (r < EXPECTED_KVM_API_VERSION) {
336 fprintf(stderr, "kvm kernel version too old: "
337 "We expect API version %d or newer, but got "
338 "version %d\n",
339 EXPECTED_KVM_API_VERSION, r);
340 goto out_close;
341 }
342 if (r > EXPECTED_KVM_API_VERSION) {
343 fprintf(stderr, "kvm userspace version too old\n");
344 goto out_close;
345 }
346 kvm_abi = r;
347 kvm_page_size = getpagesize();
348 kvm = malloc(sizeof(*kvm));
349 if (kvm == NULL)
350 goto out_close;
351 memset(kvm, 0, sizeof(*kvm));
352 kvm->fd = fd;
353 kvm->vm_fd = -1;
354 kvm->callbacks = callbacks;
355 kvm->opaque = opaque;
356 kvm->dirty_pages_log_all = 0;
357 kvm->no_irqchip_creation = 0;
358 kvm->no_pit_creation = 0;
359
360 gsi_count = kvm_get_gsi_count(kvm);
361 if (gsi_count > 0) {
362 int gsi_bits, i;
363
364 /* Round up so we can search ints using ffs */
365 gsi_bits = ALIGN(gsi_count, 32);
366 kvm->used_gsi_bitmap = malloc(gsi_bits / 8);
367 if (!kvm->used_gsi_bitmap)
368 goto out_close;
369 memset(kvm->used_gsi_bitmap, 0, gsi_bits / 8);
370 kvm->max_gsi = gsi_bits;
371
372 /* Mark any over-allocated bits as already in use */
373 for (i = gsi_count; i < gsi_bits; i++)
374 set_gsi(kvm, i);
375 }
376
377 return kvm;
378 out_close:
379 close(fd);
380 return NULL;
381 }
382
383 void kvm_finalize(kvm_context_t kvm)
384 {
385 /* FIXME
386 if (kvm->vcpu_fd[0] != -1)
387 close(kvm->vcpu_fd[0]);
388 if (kvm->vm_fd != -1)
389 close(kvm->vm_fd);
390 */
391 close(kvm->fd);
392 free(kvm);
393 }
394
395 void kvm_disable_irqchip_creation(kvm_context_t kvm)
396 {
397 kvm->no_irqchip_creation = 1;
398 }
399
400 void kvm_disable_pit_creation(kvm_context_t kvm)
401 {
402 kvm->no_pit_creation = 1;
403 }
404
405 kvm_vcpu_context_t kvm_create_vcpu(kvm_context_t kvm, int id)
406 {
407 long mmap_size;
408 int r;
409 kvm_vcpu_context_t vcpu_ctx = malloc(sizeof(struct kvm_vcpu_context));
410
411 if (!vcpu_ctx) {
412 errno = ENOMEM;
413 return NULL;
414 }
415
416 vcpu_ctx->kvm = kvm;
417 vcpu_ctx->id = id;
418
419 r = ioctl(kvm->vm_fd, KVM_CREATE_VCPU, id);
420 if (r == -1) {
421 fprintf(stderr, "kvm_create_vcpu: %m\n");
422 goto err;
423 }
424 vcpu_ctx->fd = r;
425 mmap_size = ioctl(kvm->fd, KVM_GET_VCPU_MMAP_SIZE, 0);
426 if (mmap_size == -1) {
427 fprintf(stderr, "get vcpu mmap size: %m\n");
428 goto err_fd;
429 }
430 vcpu_ctx->run = mmap(NULL, mmap_size, PROT_READ|PROT_WRITE, MAP_SHARED,
431 vcpu_ctx->fd, 0);
432 if (vcpu_ctx->run == MAP_FAILED) {
433 fprintf(stderr, "mmap vcpu area: %m\n");
434 goto err_fd;
435 }
436 return vcpu_ctx;
437 err_fd:
438 close(vcpu_ctx->fd);
439 err:
440 free(vcpu_ctx);
441 return NULL;
442 }
443
444 int kvm_create_vm(kvm_context_t kvm)
445 {
446 int fd = kvm->fd;
447
448 #ifdef KVM_CAP_IRQ_ROUTING
449 kvm->irq_routes = malloc(sizeof(*kvm->irq_routes));
450 if (!kvm->irq_routes)
451 return -ENOMEM;
452 memset(kvm->irq_routes, 0, sizeof(*kvm->irq_routes));
453 kvm->nr_allocated_irq_routes = 0;
454 #endif
455
456 fd = ioctl(fd, KVM_CREATE_VM, 0);
457 if (fd == -1) {
458 fprintf(stderr, "kvm_create_vm: %m\n");
459 return -1;
460 }
461 kvm->vm_fd = fd;
462 return 0;
463 }
464
465 static int kvm_create_default_phys_mem(kvm_context_t kvm,
466 unsigned long phys_mem_bytes,
467 void **vm_mem)
468 {
469 #ifdef KVM_CAP_USER_MEMORY
470 int r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_USER_MEMORY);
471 if (r > 0)
472 return 0;
473 fprintf(stderr, "Hypervisor too old: KVM_CAP_USER_MEMORY extension not supported\n");
474 #else
475 #error Hypervisor too old: KVM_CAP_USER_MEMORY extension not supported
476 #endif
477 return -1;
478 }
479
480 int kvm_check_extension(kvm_context_t kvm, int ext)
481 {
482 int ret;
483
484 ret = ioctl(kvm->fd, KVM_CHECK_EXTENSION, ext);
485 if (ret > 0)
486 return ret;
487 return 0;
488 }
489
490 void kvm_create_irqchip(kvm_context_t kvm)
491 {
492 int r;
493
494 kvm->irqchip_in_kernel = 0;
495 #ifdef KVM_CAP_IRQCHIP
496 if (!kvm->no_irqchip_creation) {
497 r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_IRQCHIP);
498 if (r > 0) { /* kernel irqchip supported */
499 r = ioctl(kvm->vm_fd, KVM_CREATE_IRQCHIP);
500 if (r >= 0) {
501 kvm->irqchip_inject_ioctl = KVM_IRQ_LINE;
502 #if defined(KVM_CAP_IRQ_INJECT_STATUS) && defined(KVM_IRQ_LINE_STATUS)
503 r = ioctl(kvm->fd, KVM_CHECK_EXTENSION,
504 KVM_CAP_IRQ_INJECT_STATUS);
505 if (r > 0)
506 kvm->irqchip_inject_ioctl = KVM_IRQ_LINE_STATUS;
507 #endif
508 kvm->irqchip_in_kernel = 1;
509 }
510 else
511 fprintf(stderr, "Create kernel PIC irqchip failed\n");
512 }
513 }
514 #endif
515 }
516
517 int kvm_create(kvm_context_t kvm, unsigned long phys_mem_bytes, void **vm_mem)
518 {
519 int r;
520
521 r = kvm_create_vm(kvm);
522 if (r < 0)
523 return r;
524 r = kvm_arch_create(kvm, phys_mem_bytes, vm_mem);
525 if (r < 0)
526 return r;
527 init_slots();
528 r = kvm_create_default_phys_mem(kvm, phys_mem_bytes, vm_mem);
529 if (r < 0)
530 return r;
531 kvm_create_irqchip(kvm);
532
533 return 0;
534 }
535
536
537 void *kvm_create_phys_mem(kvm_context_t kvm, unsigned long phys_start,
538 unsigned long len, int log, int writable)
539 {
540 int r;
541 int prot = PROT_READ;
542 void *ptr;
543 struct kvm_userspace_memory_region memory = {
544 .memory_size = len,
545 .guest_phys_addr = phys_start,
546 .flags = log ? KVM_MEM_LOG_DIRTY_PAGES : 0,
547 };
548
549 if (writable)
550 prot |= PROT_WRITE;
551
552 #if !defined(__s390__)
553 ptr = mmap(NULL, len, prot, MAP_ANONYMOUS | MAP_SHARED, -1, 0);
554 #else
555 ptr = mmap(LIBKVM_S390_ORIGIN, len, prot | PROT_EXEC,
556 MAP_FIXED | MAP_SHARED | MAP_ANONYMOUS, -1, 0);
557 #endif
558 if (ptr == MAP_FAILED) {
559 fprintf(stderr, "%s: %s", __func__, strerror(errno));
560 return 0;
561 }
562
563 memset(ptr, 0, len);
564
565 memory.userspace_addr = (unsigned long)ptr;
566 memory.slot = get_free_slot(kvm);
567 DPRINTF("slot %d start %llx len %llx flags %x\n",
568 memory.slot,
569 memory.guest_phys_addr,
570 memory.memory_size,
571 memory.flags);
572 r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &memory);
573 if (r == -1) {
574 fprintf(stderr, "%s: %s", __func__, strerror(errno));
575 return 0;
576 }
577 register_slot(memory.slot, memory.guest_phys_addr, memory.memory_size,
578 memory.userspace_addr, memory.flags);
579
580 return ptr;
581 }
582
583 int kvm_register_phys_mem(kvm_context_t kvm,
584 unsigned long phys_start, void *userspace_addr,
585 unsigned long len, int log)
586 {
587
588 struct kvm_userspace_memory_region memory = {
589 .memory_size = len,
590 .guest_phys_addr = phys_start,
591 .userspace_addr = (unsigned long)(intptr_t)userspace_addr,
592 .flags = log ? KVM_MEM_LOG_DIRTY_PAGES : 0,
593 };
594 int r;
595
596 memory.slot = get_free_slot(kvm);
597 DPRINTF("memory: gpa: %llx, size: %llx, uaddr: %llx, slot: %x, flags: %lx\n",
598 memory.guest_phys_addr, memory.memory_size,
599 memory.userspace_addr, memory.slot, memory.flags);
600 r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &memory);
601 if (r == -1) {
602 fprintf(stderr, "create_userspace_phys_mem: %s\n", strerror(errno));
603 return -1;
604 }
605 register_slot(memory.slot, memory.guest_phys_addr, memory.memory_size,
606 memory.userspace_addr, memory.flags);
607 return 0;
608 }
609
610
611 /* destroy/free a whole slot.
612 * phys_start, len and slot are the params passed to kvm_create_phys_mem()
613 */
614 void kvm_destroy_phys_mem(kvm_context_t kvm, unsigned long phys_start,
615 unsigned long len)
616 {
617 int slot;
618 int r;
619 struct kvm_userspace_memory_region memory = {
620 .memory_size = 0,
621 .guest_phys_addr = phys_start,
622 .userspace_addr = 0,
623 .flags = 0,
624 };
625
626 slot = get_slot(phys_start);
627
628 if ((slot >= KVM_MAX_NUM_MEM_REGIONS) || (slot == -1)) {
629 fprintf(stderr, "BUG: %s: invalid parameters (slot=%d)\n",
630 __FUNCTION__, slot);
631 return;
632 }
633 if (phys_start != slots[slot].phys_addr) {
634 fprintf(stderr,
635 "WARNING: %s: phys_start is 0x%lx expecting 0x%lx\n",
636 __FUNCTION__, phys_start, slots[slot].phys_addr);
637 phys_start = slots[slot].phys_addr;
638 }
639
640 memory.slot = slot;
641 DPRINTF("slot %d start %llx len %llx flags %x\n",
642 memory.slot,
643 memory.guest_phys_addr,
644 memory.memory_size,
645 memory.flags);
646 r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &memory);
647 if (r == -1) {
648 fprintf(stderr, "destroy_userspace_phys_mem: %s",
649 strerror(errno));
650 return;
651 }
652
653 free_slot(memory.slot);
654 }
655
656 void kvm_unregister_memory_area(kvm_context_t kvm, uint64_t phys_addr, unsigned long size)
657 {
658
659 int slot = get_container_slot(phys_addr, size);
660
661 if (slot != -1) {
662 DPRINTF("Unregistering memory region %llx (%lx)\n", phys_addr, size);
663 kvm_destroy_phys_mem(kvm, phys_addr, size);
664 return;
665 }
666 }
667
668 static int kvm_get_map(kvm_context_t kvm, int ioctl_num, int slot, void *buf)
669 {
670 int r;
671 struct kvm_dirty_log log = {
672 .slot = slot,
673 };
674
675 log.dirty_bitmap = buf;
676
677 r = ioctl(kvm->vm_fd, ioctl_num, &log);
678 if (r == -1)
679 return -errno;
680 return 0;
681 }
682
683 int kvm_get_dirty_pages(kvm_context_t kvm, unsigned long phys_addr, void *buf)
684 {
685 int slot;
686
687 slot = get_slot(phys_addr);
688 return kvm_get_map(kvm, KVM_GET_DIRTY_LOG, slot, buf);
689 }
690
691 int kvm_get_dirty_pages_range(kvm_context_t kvm, unsigned long phys_addr,
692 unsigned long len, void *opaque,
693 int (*cb)(unsigned long start, unsigned long len,
694 void*bitmap, void *opaque))
695 {
696 int i;
697 int r;
698 unsigned long end_addr = phys_addr + len;
699 void *buf;
700
701 for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS; ++i) {
702 if ((slots[i].len && (uint64_t)slots[i].phys_addr >= phys_addr)
703 && ((uint64_t)slots[i].phys_addr + slots[i].len <= end_addr)) {
704 buf = qemu_malloc((slots[i].len / 4096 + 7) / 8 + 2);
705 r = kvm_get_map(kvm, KVM_GET_DIRTY_LOG, i, buf);
706 if (r) {
707 qemu_free(buf);
708 return r;
709 }
710 r = cb(slots[i].phys_addr, slots[i].len, buf, opaque);
711 qemu_free(buf);
712 if (r)
713 return r;
714 }
715 }
716 return 0;
717 }
718
719 #ifdef KVM_CAP_IRQCHIP
720
721 int kvm_set_irq_level(kvm_context_t kvm, int irq, int level, int *status)
722 {
723 struct kvm_irq_level event;
724 int r;
725
726 if (!kvm->irqchip_in_kernel)
727 return 0;
728 event.level = level;
729 event.irq = irq;
730 r = ioctl(kvm->vm_fd, kvm->irqchip_inject_ioctl, &event);
731 if (r == -1)
732 perror("kvm_set_irq_level");
733
734 if (status) {
735 #ifdef KVM_CAP_IRQ_INJECT_STATUS
736 *status = (kvm->irqchip_inject_ioctl == KVM_IRQ_LINE) ?
737 1 : event.status;
738 #else
739 *status = 1;
740 #endif
741 }
742
743 return 1;
744 }
745
746 int kvm_get_irqchip(kvm_context_t kvm, struct kvm_irqchip *chip)
747 {
748 int r;
749
750 if (!kvm->irqchip_in_kernel)
751 return 0;
752 r = ioctl(kvm->vm_fd, KVM_GET_IRQCHIP, chip);
753 if (r == -1) {
754 r = -errno;
755 perror("kvm_get_irqchip\n");
756 }
757 return r;
758 }
759
760 int kvm_set_irqchip(kvm_context_t kvm, struct kvm_irqchip *chip)
761 {
762 int r;
763
764 if (!kvm->irqchip_in_kernel)
765 return 0;
766 r = ioctl(kvm->vm_fd, KVM_SET_IRQCHIP, chip);
767 if (r == -1) {
768 r = -errno;
769 perror("kvm_set_irqchip\n");
770 }
771 return r;
772 }
773
774 #endif
775
776 static int handle_io(kvm_vcpu_context_t vcpu)
777 {
778 struct kvm_run *run = vcpu->run;
779 kvm_context_t kvm = vcpu->kvm;
780 uint16_t addr = run->io.port;
781 int r;
782 int i;
783 void *p = (void *)run + run->io.data_offset;
784
785 for (i = 0; i < run->io.count; ++i) {
786 switch (run->io.direction) {
787 case KVM_EXIT_IO_IN:
788 switch (run->io.size) {
789 case 1:
790 r = kvm->callbacks->inb(kvm->opaque, addr, p);
791 break;
792 case 2:
793 r = kvm->callbacks->inw(kvm->opaque, addr, p);
794 break;
795 case 4:
796 r = kvm->callbacks->inl(kvm->opaque, addr, p);
797 break;
798 default:
799 fprintf(stderr, "bad I/O size %d\n", run->io.size);
800 return -EMSGSIZE;
801 }
802 break;
803 case KVM_EXIT_IO_OUT:
804 switch (run->io.size) {
805 case 1:
806 r = kvm->callbacks->outb(kvm->opaque, addr,
807 *(uint8_t *)p);
808 break;
809 case 2:
810 r = kvm->callbacks->outw(kvm->opaque, addr,
811 *(uint16_t *)p);
812 break;
813 case 4:
814 r = kvm->callbacks->outl(kvm->opaque, addr,
815 *(uint32_t *)p);
816 break;
817 default:
818 fprintf(stderr, "bad I/O size %d\n", run->io.size);
819 return -EMSGSIZE;
820 }
821 break;
822 default:
823 fprintf(stderr, "bad I/O direction %d\n", run->io.direction);
824 return -EPROTO;
825 }
826
827 p += run->io.size;
828 }
829
830 return 0;
831 }
832
833 int handle_debug(kvm_vcpu_context_t vcpu, void *env)
834 {
835 #ifdef KVM_CAP_SET_GUEST_DEBUG
836 struct kvm_run *run = vcpu->run;
837 kvm_context_t kvm = vcpu->kvm;
838
839 return kvm->callbacks->debug(kvm->opaque, env, &run->debug.arch);
840 #else
841 return 0;
842 #endif
843 }
844
845 int kvm_get_regs(kvm_vcpu_context_t vcpu, struct kvm_regs *regs)
846 {
847 return ioctl(vcpu->fd, KVM_GET_REGS, regs);
848 }
849
850 int kvm_set_regs(kvm_vcpu_context_t vcpu, struct kvm_regs *regs)
851 {
852 return ioctl(vcpu->fd, KVM_SET_REGS, regs);
853 }
854
855 int kvm_get_fpu(kvm_vcpu_context_t vcpu, struct kvm_fpu *fpu)
856 {
857 return ioctl(vcpu->fd, KVM_GET_FPU, fpu);
858 }
859
860 int kvm_set_fpu(kvm_vcpu_context_t vcpu, struct kvm_fpu *fpu)
861 {
862 return ioctl(vcpu->fd, KVM_SET_FPU, fpu);
863 }
864
865 int kvm_get_sregs(kvm_vcpu_context_t vcpu, struct kvm_sregs *sregs)
866 {
867 return ioctl(vcpu->fd, KVM_GET_SREGS, sregs);
868 }
869
870 int kvm_set_sregs(kvm_vcpu_context_t vcpu, struct kvm_sregs *sregs)
871 {
872 return ioctl(vcpu->fd, KVM_SET_SREGS, sregs);
873 }
874
875 #ifdef KVM_CAP_MP_STATE
876 int kvm_get_mpstate(kvm_vcpu_context_t vcpu, struct kvm_mp_state *mp_state)
877 {
878 int r;
879
880 r = ioctl(vcpu->kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_MP_STATE);
881 if (r > 0)
882 return ioctl(vcpu->fd, KVM_GET_MP_STATE, mp_state);
883 return -ENOSYS;
884 }
885
886 int kvm_set_mpstate(kvm_vcpu_context_t vcpu, struct kvm_mp_state *mp_state)
887 {
888 int r;
889
890 r = ioctl(vcpu->kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_MP_STATE);
891 if (r > 0)
892 return ioctl(vcpu->fd, KVM_SET_MP_STATE, mp_state);
893 return -ENOSYS;
894 }
895 #endif
896
897 static int handle_mmio(kvm_vcpu_context_t vcpu)
898 {
899 unsigned long addr = vcpu->run->mmio.phys_addr;
900 kvm_context_t kvm = vcpu->kvm;
901 struct kvm_run *kvm_run = vcpu->run;
902 void *data = kvm_run->mmio.data;
903
904 /* hack: Red Hat 7.1 generates these weird accesses. */
905 if ((addr > 0xa0000-4 && addr <= 0xa0000) && kvm_run->mmio.len == 3)
906 return 0;
907
908 if (kvm_run->mmio.is_write)
909 return kvm->callbacks->mmio_write(kvm->opaque, addr, data,
910 kvm_run->mmio.len);
911 else
912 return kvm->callbacks->mmio_read(kvm->opaque, addr, data,
913 kvm_run->mmio.len);
914 }
915
916 int handle_io_window(kvm_context_t kvm)
917 {
918 return kvm->callbacks->io_window(kvm->opaque);
919 }
920
921 int handle_halt(kvm_vcpu_context_t vcpu)
922 {
923 return vcpu->kvm->callbacks->halt(vcpu->kvm->opaque, vcpu);
924 }
925
926 int handle_shutdown(kvm_context_t kvm, void *env)
927 {
928 return kvm->callbacks->shutdown(kvm->opaque, env);
929 }
930
931 int try_push_interrupts(kvm_context_t kvm)
932 {
933 return kvm->callbacks->try_push_interrupts(kvm->opaque);
934 }
935
936 static inline void push_nmi(kvm_context_t kvm)
937 {
938 #ifdef KVM_CAP_USER_NMI
939 kvm->callbacks->push_nmi(kvm->opaque);
940 #endif /* KVM_CAP_USER_NMI */
941 }
942
943 void post_kvm_run(kvm_context_t kvm, void *env)
944 {
945 kvm->callbacks->post_kvm_run(kvm->opaque, env);
946 }
947
948 int pre_kvm_run(kvm_context_t kvm, void *env)
949 {
950 return kvm->callbacks->pre_kvm_run(kvm->opaque, env);
951 }
952
953 int kvm_get_interrupt_flag(kvm_vcpu_context_t vcpu)
954 {
955 return vcpu->run->if_flag;
956 }
957
958 int kvm_is_ready_for_interrupt_injection(kvm_vcpu_context_t vcpu)
959 {
960 return vcpu->run->ready_for_interrupt_injection;
961 }
962
963 int kvm_run(kvm_vcpu_context_t vcpu, void *env)
964 {
965 int r;
966 int fd = vcpu->fd;
967 struct kvm_run *run = vcpu->run;
968 kvm_context_t kvm = vcpu->kvm;
969
970 again:
971 push_nmi(kvm);
972 #if !defined(__s390__)
973 if (!kvm->irqchip_in_kernel)
974 run->request_interrupt_window = try_push_interrupts(kvm);
975 #endif
976 r = pre_kvm_run(kvm, env);
977 if (r)
978 return r;
979 r = ioctl(fd, KVM_RUN, 0);
980
981 if (r == -1 && errno != EINTR && errno != EAGAIN) {
982 r = -errno;
983 post_kvm_run(kvm, env);
984 fprintf(stderr, "kvm_run: %s\n", strerror(-r));
985 return r;
986 }
987
988 post_kvm_run(kvm, env);
989
990 #if defined(KVM_CAP_COALESCED_MMIO)
991 if (kvm->coalesced_mmio) {
992 struct kvm_coalesced_mmio_ring *ring = (void *)run +
993 kvm->coalesced_mmio * PAGE_SIZE;
994 while (ring->first != ring->last) {
995 kvm->callbacks->mmio_write(kvm->opaque,
996 ring->coalesced_mmio[ring->first].phys_addr,
997 &ring->coalesced_mmio[ring->first].data[0],
998 ring->coalesced_mmio[ring->first].len);
999 smp_wmb();
1000 ring->first = (ring->first + 1) %
1001 KVM_COALESCED_MMIO_MAX;
1002 }
1003 }
1004 #endif
1005
1006 #if !defined(__s390__)
1007 if (r == -1) {
1008 r = handle_io_window(kvm);
1009 goto more;
1010 }
1011 #endif
1012 if (1) {
1013 switch (run->exit_reason) {
1014 case KVM_EXIT_UNKNOWN:
1015 r = kvm->callbacks->unhandled(kvm, vcpu,
1016 run->hw.hardware_exit_reason);
1017 break;
1018 case KVM_EXIT_FAIL_ENTRY:
1019 r = kvm->callbacks->unhandled(kvm, vcpu,
1020 run->fail_entry.hardware_entry_failure_reason);
1021 break;
1022 case KVM_EXIT_EXCEPTION:
1023 fprintf(stderr, "exception %d (%x)\n",
1024 run->ex.exception,
1025 run->ex.error_code);
1026 kvm_show_regs(vcpu);
1027 kvm_show_code(vcpu);
1028 abort();
1029 break;
1030 case KVM_EXIT_IO:
1031 r = handle_io(vcpu);
1032 break;
1033 case KVM_EXIT_DEBUG:
1034 r = handle_debug(vcpu, env);
1035 break;
1036 case KVM_EXIT_MMIO:
1037 r = handle_mmio(vcpu);
1038 break;
1039 case KVM_EXIT_HLT:
1040 r = handle_halt(vcpu);
1041 break;
1042 case KVM_EXIT_IRQ_WINDOW_OPEN:
1043 break;
1044 case KVM_EXIT_SHUTDOWN:
1045 r = handle_shutdown(kvm, env);
1046 break;
1047 #if defined(__s390__)
1048 case KVM_EXIT_S390_SIEIC:
1049 r = kvm->callbacks->s390_handle_intercept(kvm, vcpu,
1050 run);
1051 break;
1052 case KVM_EXIT_S390_RESET:
1053 r = kvm->callbacks->s390_handle_reset(kvm, vcpu, run);
1054 break;
1055 #endif
1056 default:
1057 if (kvm_arch_run(vcpu)) {
1058 fprintf(stderr, "unhandled vm exit: 0x%x\n",
1059 run->exit_reason);
1060 kvm_show_regs(vcpu);
1061 abort();
1062 }
1063 break;
1064 }
1065 }
1066 more:
1067 if (!r)
1068 goto again;
1069 return r;
1070 }
1071
1072 int kvm_inject_irq(kvm_vcpu_context_t vcpu, unsigned irq)
1073 {
1074 struct kvm_interrupt intr;
1075
1076 intr.irq = irq;
1077 return ioctl(vcpu->fd, KVM_INTERRUPT, &intr);
1078 }
1079
1080 #ifdef KVM_CAP_SET_GUEST_DEBUG
1081 int kvm_set_guest_debug(kvm_vcpu_context_t vcpu, struct kvm_guest_debug *dbg)
1082 {
1083 return ioctl(vcpu->fd, KVM_SET_GUEST_DEBUG, dbg);
1084 }
1085 #endif
1086
1087 int kvm_set_signal_mask(kvm_vcpu_context_t vcpu, const sigset_t *sigset)
1088 {
1089 struct kvm_signal_mask *sigmask;
1090 int r;
1091
1092 if (!sigset) {
1093 r = ioctl(vcpu->fd, KVM_SET_SIGNAL_MASK, NULL);
1094 if (r == -1)
1095 r = -errno;
1096 return r;
1097 }
1098 sigmask = malloc(sizeof(*sigmask) + sizeof(*sigset));
1099 if (!sigmask)
1100 return -ENOMEM;
1101
1102 sigmask->len = 8;
1103 memcpy(sigmask->sigset, sigset, sizeof(*sigset));
1104 r = ioctl(vcpu->fd, KVM_SET_SIGNAL_MASK, sigmask);
1105 if (r == -1)
1106 r = -errno;
1107 free(sigmask);
1108 return r;
1109 }
1110
1111 int kvm_irqchip_in_kernel(kvm_context_t kvm)
1112 {
1113 return kvm->irqchip_in_kernel;
1114 }
1115
1116 int kvm_pit_in_kernel(kvm_context_t kvm)
1117 {
1118 return kvm->pit_in_kernel;
1119 }
1120
1121 int kvm_has_sync_mmu(void)
1122 {
1123 int r = 0;
1124 #ifdef KVM_CAP_SYNC_MMU
1125 r = ioctl(kvm_context->fd, KVM_CHECK_EXTENSION, KVM_CAP_SYNC_MMU);
1126 #endif
1127 return r;
1128 }
1129
1130 int kvm_inject_nmi(kvm_vcpu_context_t vcpu)
1131 {
1132 #ifdef KVM_CAP_USER_NMI
1133 return ioctl(vcpu->fd, KVM_NMI);
1134 #else
1135 return -ENOSYS;
1136 #endif
1137 }
1138
1139 int kvm_init_coalesced_mmio(kvm_context_t kvm)
1140 {
1141 int r = 0;
1142 kvm->coalesced_mmio = 0;
1143 #ifdef KVM_CAP_COALESCED_MMIO
1144 r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_COALESCED_MMIO);
1145 if (r > 0) {
1146 kvm->coalesced_mmio = r;
1147 return 0;
1148 }
1149 #endif
1150 return r;
1151 }
1152
1153 int kvm_register_coalesced_mmio(kvm_context_t kvm, uint64_t addr, uint32_t size)
1154 {
1155 #ifdef KVM_CAP_COALESCED_MMIO
1156 struct kvm_coalesced_mmio_zone zone;
1157 int r;
1158
1159 if (kvm->coalesced_mmio) {
1160
1161 zone.addr = addr;
1162 zone.size = size;
1163
1164 r = ioctl(kvm->vm_fd, KVM_REGISTER_COALESCED_MMIO, &zone);
1165 if (r == -1) {
1166 perror("kvm_register_coalesced_mmio_zone");
1167 return -errno;
1168 }
1169 return 0;
1170 }
1171 #endif
1172 return -ENOSYS;
1173 }
1174
1175 int kvm_unregister_coalesced_mmio(kvm_context_t kvm, uint64_t addr, uint32_t size)
1176 {
1177 #ifdef KVM_CAP_COALESCED_MMIO
1178 struct kvm_coalesced_mmio_zone zone;
1179 int r;
1180
1181 if (kvm->coalesced_mmio) {
1182
1183 zone.addr = addr;
1184 zone.size = size;
1185
1186 r = ioctl(kvm->vm_fd, KVM_UNREGISTER_COALESCED_MMIO, &zone);
1187 if (r == -1) {
1188 perror("kvm_unregister_coalesced_mmio_zone");
1189 return -errno;
1190 }
1191 DPRINTF("Unregistered coalesced mmio region for %llx (%lx)\n", addr, size);
1192 return 0;
1193 }
1194 #endif
1195 return -ENOSYS;
1196 }
1197
1198 #ifdef KVM_CAP_DEVICE_ASSIGNMENT
1199 int kvm_assign_pci_device(kvm_context_t kvm,
1200 struct kvm_assigned_pci_dev *assigned_dev)
1201 {
1202 int ret;
1203
1204 ret = ioctl(kvm->vm_fd, KVM_ASSIGN_PCI_DEVICE, assigned_dev);
1205 if (ret < 0)
1206 return -errno;
1207
1208 return ret;
1209 }
1210
1211 static int kvm_old_assign_irq(kvm_context_t kvm,
1212 struct kvm_assigned_irq *assigned_irq)
1213 {
1214 int ret;
1215
1216 ret = ioctl(kvm->vm_fd, KVM_ASSIGN_IRQ, assigned_irq);
1217 if (ret < 0)
1218 return -errno;
1219
1220 return ret;
1221 }
1222
1223 #ifdef KVM_CAP_ASSIGN_DEV_IRQ
1224 int kvm_assign_irq(kvm_context_t kvm,
1225 struct kvm_assigned_irq *assigned_irq)
1226 {
1227 int ret;
1228
1229 ret = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_ASSIGN_DEV_IRQ);
1230 if (ret > 0) {
1231 ret = ioctl(kvm->vm_fd, KVM_ASSIGN_DEV_IRQ, assigned_irq);
1232 if (ret < 0)
1233 return -errno;
1234 return ret;
1235 }
1236
1237 return kvm_old_assign_irq(kvm, assigned_irq);
1238 }
1239
1240 int kvm_deassign_irq(kvm_context_t kvm,
1241 struct kvm_assigned_irq *assigned_irq)
1242 {
1243 int ret;
1244
1245 ret = ioctl(kvm->vm_fd, KVM_DEASSIGN_DEV_IRQ, assigned_irq);
1246 if (ret < 0)
1247 return -errno;
1248
1249 return ret;
1250 }
1251 #else
1252 int kvm_assign_irq(kvm_context_t kvm,
1253 struct kvm_assigned_irq *assigned_irq)
1254 {
1255 return kvm_old_assign_irq(kvm, assigned_irq);
1256 }
1257 #endif
1258 #endif
1259
1260 #ifdef KVM_CAP_DEVICE_DEASSIGNMENT
1261 int kvm_deassign_pci_device(kvm_context_t kvm,
1262 struct kvm_assigned_pci_dev *assigned_dev)
1263 {
1264 int ret;
1265
1266 ret = ioctl(kvm->vm_fd, KVM_DEASSIGN_PCI_DEVICE, assigned_dev);
1267 if (ret < 0)
1268 return -errno;
1269
1270 return ret;
1271 }
1272 #endif
1273
1274 int kvm_destroy_memory_region_works(kvm_context_t kvm)
1275 {
1276 int ret = 0;
1277
1278 #ifdef KVM_CAP_DESTROY_MEMORY_REGION_WORKS
1279 ret = ioctl(kvm->fd, KVM_CHECK_EXTENSION,
1280 KVM_CAP_DESTROY_MEMORY_REGION_WORKS);
1281 if (ret <= 0)
1282 ret = 0;
1283 #endif
1284 return ret;
1285 }
1286
1287 int kvm_reinject_control(kvm_context_t kvm, int pit_reinject)
1288 {
1289 #ifdef KVM_CAP_REINJECT_CONTROL
1290 int r;
1291 struct kvm_reinject_control control;
1292
1293 control.pit_reinject = pit_reinject;
1294
1295 r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_REINJECT_CONTROL);
1296 if (r > 0) {
1297 r = ioctl(kvm->vm_fd, KVM_REINJECT_CONTROL, &control);
1298 if (r == -1)
1299 return -errno;
1300 return r;
1301 }
1302 #endif
1303 return -ENOSYS;
1304 }
1305
1306 int kvm_has_gsi_routing(kvm_context_t kvm)
1307 {
1308 int r = 0;
1309
1310 #ifdef KVM_CAP_IRQ_ROUTING
1311 r = kvm_check_extension(kvm, KVM_CAP_IRQ_ROUTING);
1312 #endif
1313 return r;
1314 }
1315
1316 int kvm_get_gsi_count(kvm_context_t kvm)
1317 {
1318 #ifdef KVM_CAP_IRQ_ROUTING
1319 return kvm_check_extension(kvm, KVM_CAP_IRQ_ROUTING);
1320 #else
1321 return -EINVAL;
1322 #endif
1323 }
1324
1325 int kvm_clear_gsi_routes(kvm_context_t kvm)
1326 {
1327 #ifdef KVM_CAP_IRQ_ROUTING
1328 kvm->irq_routes->nr = 0;
1329 return 0;
1330 #else
1331 return -EINVAL;
1332 #endif
1333 }
1334
1335 int kvm_add_routing_entry(kvm_context_t kvm,
1336 struct kvm_irq_routing_entry* entry)
1337 {
1338 #ifdef KVM_CAP_IRQ_ROUTING
1339 struct kvm_irq_routing *z;
1340 struct kvm_irq_routing_entry *new;
1341 int n, size;
1342
1343 if (kvm->irq_routes->nr == kvm->nr_allocated_irq_routes) {
1344 n = kvm->nr_allocated_irq_routes * 2;
1345 if (n < 64)
1346 n = 64;
1347 size = sizeof(struct kvm_irq_routing);
1348 size += n * sizeof(*new);
1349 z = realloc(kvm->irq_routes, size);
1350 if (!z)
1351 return -ENOMEM;
1352 kvm->nr_allocated_irq_routes = n;
1353 kvm->irq_routes = z;
1354 }
1355 n = kvm->irq_routes->nr++;
1356 new = &kvm->irq_routes->entries[n];
1357 memset(new, 0, sizeof(*new));
1358 new->gsi = entry->gsi;
1359 new->type = entry->type;
1360 new->flags = entry->flags;
1361 new->u = entry->u;
1362
1363 set_gsi(kvm, entry->gsi);
1364
1365 return 0;
1366 #else
1367 return -ENOSYS;
1368 #endif
1369 }
1370
1371 int kvm_add_irq_route(kvm_context_t kvm, int gsi, int irqchip, int pin)
1372 {
1373 #ifdef KVM_CAP_IRQ_ROUTING
1374 struct kvm_irq_routing_entry e;
1375
1376 e.gsi = gsi;
1377 e.type = KVM_IRQ_ROUTING_IRQCHIP;
1378 e.flags = 0;
1379 e.u.irqchip.irqchip = irqchip;
1380 e.u.irqchip.pin = pin;
1381 return kvm_add_routing_entry(kvm, &e);
1382 #else
1383 return -ENOSYS;
1384 #endif
1385 }
1386
1387 int kvm_del_routing_entry(kvm_context_t kvm,
1388 struct kvm_irq_routing_entry* entry)
1389 {
1390 #ifdef KVM_CAP_IRQ_ROUTING
1391 struct kvm_irq_routing_entry *e, *p;
1392 int i, gsi, found = 0;
1393
1394 gsi = entry->gsi;
1395
1396 for (i = 0; i < kvm->irq_routes->nr; ++i) {
1397 e = &kvm->irq_routes->entries[i];
1398 if (e->type == entry->type
1399 && e->gsi == gsi) {
1400 switch (e->type)
1401 {
1402 case KVM_IRQ_ROUTING_IRQCHIP: {
1403 if (e->u.irqchip.irqchip ==
1404 entry->u.irqchip.irqchip
1405 && e->u.irqchip.pin ==
1406 entry->u.irqchip.pin) {
1407 p = &kvm->irq_routes->
1408 entries[--kvm->irq_routes->nr];
1409 *e = *p;
1410 found = 1;
1411 }
1412 break;
1413 }
1414 case KVM_IRQ_ROUTING_MSI: {
1415 if (e->u.msi.address_lo ==
1416 entry->u.msi.address_lo
1417 && e->u.msi.address_hi ==
1418 entry->u.msi.address_hi
1419 && e->u.msi.data == entry->u.msi.data) {
1420 p = &kvm->irq_routes->
1421 entries[--kvm->irq_routes->nr];
1422 *e = *p;
1423 found = 1;
1424 }
1425 break;
1426 }
1427 default:
1428 break;
1429 }
1430 if (found) {
1431 /* If there are no other users of this GSI
1432 * mark it available in the bitmap */
1433 for (i = 0; i < kvm->irq_routes->nr; i++) {
1434 e = &kvm->irq_routes->entries[i];
1435 if (e->gsi == gsi)
1436 break;
1437 }
1438 if (i == kvm->irq_routes->nr)
1439 clear_gsi(kvm, gsi);
1440
1441 return 0;
1442 }
1443 }
1444 }
1445 return -ESRCH;
1446 #else
1447 return -ENOSYS;
1448 #endif
1449 }
1450
1451 int kvm_del_irq_route(kvm_context_t kvm, int gsi, int irqchip, int pin)
1452 {
1453 #ifdef KVM_CAP_IRQ_ROUTING
1454 struct kvm_irq_routing_entry e;
1455
1456 e.gsi = gsi;
1457 e.type = KVM_IRQ_ROUTING_IRQCHIP;
1458 e.flags = 0;
1459 e.u.irqchip.irqchip = irqchip;
1460 e.u.irqchip.pin = pin;
1461 return kvm_del_routing_entry(kvm, &e);
1462 #else
1463 return -ENOSYS;
1464 #endif
1465 }
1466
1467 int kvm_commit_irq_routes(kvm_context_t kvm)
1468 {
1469 #ifdef KVM_CAP_IRQ_ROUTING
1470 int r;
1471
1472 kvm->irq_routes->flags = 0;
1473 r = ioctl(kvm->vm_fd, KVM_SET_GSI_ROUTING, kvm->irq_routes);
1474 if (r == -1)
1475 r = -errno;
1476 return r;
1477 #else
1478 return -ENOSYS;
1479 #endif
1480 }
1481
1482 int kvm_get_irq_route_gsi(kvm_context_t kvm)
1483 {
1484 int i, bit;
1485 uint32_t *buf = kvm->used_gsi_bitmap;
1486
1487 /* Return the lowest unused GSI in the bitmap */
1488 for (i = 0; i < kvm->max_gsi / 32; i++) {
1489 bit = ffs(~buf[i]);
1490 if (!bit)
1491 continue;
1492
1493 return bit - 1 + i * 32;
1494 }
1495
1496 return -ENOSPC;
1497 }
1498
1499 #ifdef KVM_CAP_DEVICE_MSIX
1500 int kvm_assign_set_msix_nr(kvm_context_t kvm,
1501 struct kvm_assigned_msix_nr *msix_nr)
1502 {
1503 int ret;
1504
1505 ret = ioctl(kvm->vm_fd, KVM_ASSIGN_SET_MSIX_NR, msix_nr);
1506 if (ret < 0)
1507 return -errno;
1508
1509 return ret;
1510 }
1511
1512 int kvm_assign_set_msix_entry(kvm_context_t kvm,
1513 struct kvm_assigned_msix_entry *entry)
1514 {
1515 int ret;
1516
1517 ret = ioctl(kvm->vm_fd, KVM_ASSIGN_SET_MSIX_ENTRY, entry);
1518 if (ret < 0)
1519 return -errno;
1520
1521 return ret;
1522 }
1523 #endif
1524
1525 #if defined(KVM_CAP_IRQFD) && defined(CONFIG_eventfd)
1526
1527 #include <sys/eventfd.h>
1528
1529 static int _kvm_irqfd(kvm_context_t kvm, int fd, int gsi, int flags)
1530 {
1531 int r;
1532 struct kvm_irqfd data = {
1533 .fd = fd,
1534 .gsi = gsi,
1535 .flags = flags,
1536 };
1537
1538 r = ioctl(kvm->vm_fd, KVM_IRQFD, &data);
1539 if (r == -1)
1540 r = -errno;
1541 return r;
1542 }
1543
1544 int kvm_irqfd(kvm_context_t kvm, int gsi, int flags)
1545 {
1546 int r;
1547 int fd;
1548
1549 if (!kvm_check_extension(kvm, KVM_CAP_IRQFD))
1550 return -ENOENT;
1551
1552 fd = eventfd(0, 0);
1553 if (fd < 0)
1554 return -errno;
1555
1556 r = _kvm_irqfd(kvm, fd, gsi, 0);
1557 if (r < 0) {
1558 close(fd);
1559 return -errno;
1560 }
1561
1562 return fd;
1563 }
1564
1565 #else /* KVM_CAP_IRQFD */
1566
1567 int kvm_irqfd(kvm_context_t kvm, int gsi, int flags)
1568 {
1569 return -ENOSYS;
1570 }
1571
1572 #endif /* KVM_CAP_IRQFD */
1573 static inline unsigned long kvm_get_thread_id(void)
1574 {
1575 return syscall(SYS_gettid);
1576 }
1577
1578 static void qemu_cond_wait(pthread_cond_t *cond)
1579 {
1580 CPUState *env = cpu_single_env;
1581 static const struct timespec ts = {
1582 .tv_sec = 0,
1583 .tv_nsec = 100000,
1584 };
1585
1586 pthread_cond_timedwait(cond, &qemu_mutex, &ts);
1587 cpu_single_env = env;
1588 }
1589
1590 static void sig_ipi_handler(int n)
1591 {
1592 }
1593
1594 static void on_vcpu(CPUState *env, void (*func)(void *data), void *data)
1595 {
1596 struct qemu_work_item wi;
1597
1598 if (env == current_env) {
1599 func(data);
1600 return;
1601 }
1602
1603 wi.func = func;
1604 wi.data = data;
1605 if (!env->kvm_cpu_state.queued_work_first)
1606 env->kvm_cpu_state.queued_work_first = &wi;
1607 else
1608 env->kvm_cpu_state.queued_work_last->next = &wi;
1609 env->kvm_cpu_state.queued_work_last = &wi;
1610 wi.next = NULL;
1611 wi.done = false;
1612
1613 pthread_kill(env->kvm_cpu_state.thread, SIG_IPI);
1614 while (!wi.done)
1615 qemu_cond_wait(&qemu_work_cond);
1616 }
1617
1618 static void inject_interrupt(void *data)
1619 {
1620 cpu_interrupt(current_env, (long)data);
1621 }
1622
1623 void kvm_inject_interrupt(CPUState *env, int mask)
1624 {
1625 on_vcpu(env, inject_interrupt, (void *)(long)mask);
1626 }
1627
1628 void kvm_update_interrupt_request(CPUState *env)
1629 {
1630 int signal = 0;
1631
1632 if (env) {
1633 if (!current_env || !current_env->kvm_cpu_state.created)
1634 signal = 1;
1635 /*
1636 * Testing for created here is really redundant
1637 */
1638 if (current_env && current_env->kvm_cpu_state.created &&
1639 env != current_env && !env->kvm_cpu_state.signalled)
1640 signal = 1;
1641
1642 if (signal) {
1643 env->kvm_cpu_state.signalled = 1;
1644 if (env->kvm_cpu_state.thread)
1645 pthread_kill(env->kvm_cpu_state.thread, SIG_IPI);
1646 }
1647 }
1648 }
1649
1650 #include <signal.h>
1651
1652 static int kvm_try_push_interrupts(void *opaque)
1653 {
1654 return kvm_arch_try_push_interrupts(opaque);
1655 }
1656
1657 static void kvm_post_run(void *opaque, void *data)
1658 {
1659 CPUState *env = (CPUState *)data;
1660
1661 pthread_mutex_lock(&qemu_mutex);
1662 kvm_arch_post_kvm_run(opaque, env);
1663 }
1664
1665 static int kvm_pre_run(void *opaque, void *data)
1666 {
1667 CPUState *env = (CPUState *)data;
1668
1669 kvm_arch_pre_kvm_run(opaque, env);
1670
1671 pthread_mutex_unlock(&qemu_mutex);
1672 return 0;
1673 }
1674
1675 static void kvm_do_load_registers(void *_env)
1676 {
1677 CPUState *env = _env;
1678
1679 kvm_arch_load_regs(env);
1680 }
1681
1682 void kvm_load_registers(CPUState *env)
1683 {
1684 if (kvm_enabled() && qemu_system_ready)
1685 on_vcpu(env, kvm_do_load_registers, env);
1686 }
1687
1688 static void kvm_do_save_registers(void *_env)
1689 {
1690 CPUState *env = _env;
1691
1692 kvm_arch_save_regs(env);
1693 }
1694
1695 void kvm_save_registers(CPUState *env)
1696 {
1697 if (kvm_enabled())
1698 on_vcpu(env, kvm_do_save_registers, env);
1699 }
1700
1701 static void kvm_do_load_mpstate(void *_env)
1702 {
1703 CPUState *env = _env;
1704
1705 kvm_arch_load_mpstate(env);
1706 }
1707
1708 void kvm_load_mpstate(CPUState *env)
1709 {
1710 if (kvm_enabled() && qemu_system_ready)
1711 on_vcpu(env, kvm_do_load_mpstate, env);
1712 }
1713
1714 static void kvm_do_save_mpstate(void *_env)
1715 {
1716 CPUState *env = _env;
1717
1718 kvm_arch_save_mpstate(env);
1719 env->halted = (env->mp_state == KVM_MP_STATE_HALTED);
1720 }
1721
1722 void kvm_save_mpstate(CPUState *env)
1723 {
1724 if (kvm_enabled())
1725 on_vcpu(env, kvm_do_save_mpstate, env);
1726 }
1727
1728 int kvm_cpu_exec(CPUState *env)
1729 {
1730 int r;
1731
1732 r = kvm_run(env->kvm_cpu_state.vcpu_ctx, env);
1733 if (r < 0) {
1734 printf("kvm_run returned %d\n", r);
1735 vm_stop(0);
1736 }
1737
1738 return 0;
1739 }
1740
1741 static int is_cpu_stopped(CPUState *env)
1742 {
1743 return !vm_running || env->kvm_cpu_state.stopped;
1744 }
1745
1746 static void flush_queued_work(CPUState *env)
1747 {
1748 struct qemu_work_item *wi;
1749
1750 if (!env->kvm_cpu_state.queued_work_first)
1751 return;
1752
1753 while ((wi = env->kvm_cpu_state.queued_work_first)) {
1754 env->kvm_cpu_state.queued_work_first = wi->next;
1755 wi->func(wi->data);
1756 wi->done = true;
1757 }
1758 env->kvm_cpu_state.queued_work_last = NULL;
1759 pthread_cond_broadcast(&qemu_work_cond);
1760 }
1761
1762 static void kvm_main_loop_wait(CPUState *env, int timeout)
1763 {
1764 struct timespec ts;
1765 int r, e;
1766 siginfo_t siginfo;
1767 sigset_t waitset;
1768
1769 pthread_mutex_unlock(&qemu_mutex);
1770
1771 ts.tv_sec = timeout / 1000;
1772 ts.tv_nsec = (timeout % 1000) * 1000000;
1773 sigemptyset(&waitset);
1774 sigaddset(&waitset, SIG_IPI);
1775
1776 r = sigtimedwait(&waitset, &siginfo, &ts);
1777 e = errno;
1778
1779 pthread_mutex_lock(&qemu_mutex);
1780
1781 if (r == -1 && !(e == EAGAIN || e == EINTR)) {
1782 printf("sigtimedwait: %s\n", strerror(e));
1783 exit(1);
1784 }
1785
1786 cpu_single_env = env;
1787 flush_queued_work(env);
1788
1789 if (env->kvm_cpu_state.stop) {
1790 env->kvm_cpu_state.stop = 0;
1791 env->kvm_cpu_state.stopped = 1;
1792 pthread_cond_signal(&qemu_pause_cond);
1793 }
1794
1795 env->kvm_cpu_state.signalled = 0;
1796 }
1797
1798 static int all_threads_paused(void)
1799 {
1800 CPUState *penv = first_cpu;
1801
1802 while (penv) {
1803 if (penv->kvm_cpu_state.stop)
1804 return 0;
1805 penv = (CPUState *)penv->next_cpu;
1806 }
1807
1808 return 1;
1809 }
1810
1811 static void pause_all_threads(void)
1812 {
1813 CPUState *penv = first_cpu;
1814
1815 while (penv) {
1816 if (penv != cpu_single_env) {
1817 penv->kvm_cpu_state.stop = 1;
1818 pthread_kill(penv->kvm_cpu_state.thread, SIG_IPI);
1819 } else {
1820 penv->kvm_cpu_state.stop = 0;
1821 penv->kvm_cpu_state.stopped = 1;
1822 cpu_exit(penv);
1823 }
1824 penv = (CPUState *)penv->next_cpu;
1825 }
1826
1827 while (!all_threads_paused())
1828 qemu_cond_wait(&qemu_pause_cond);
1829 }
1830
1831 static void resume_all_threads(void)
1832 {
1833 CPUState *penv = first_cpu;
1834
1835 assert(!cpu_single_env);
1836
1837 while (penv) {
1838 penv->kvm_cpu_state.stop = 0;
1839 penv->kvm_cpu_state.stopped = 0;
1840 pthread_kill(penv->kvm_cpu_state.thread, SIG_IPI);
1841 penv = (CPUState *)penv->next_cpu;
1842 }
1843 }
1844
1845 static void kvm_vm_state_change_handler(void *context, int running, int reason)
1846 {
1847 if (running)
1848 resume_all_threads();
1849 else
1850 pause_all_threads();
1851 }
1852
1853 static void setup_kernel_sigmask(CPUState *env)
1854 {
1855 sigset_t set;
1856
1857 sigemptyset(&set);
1858 sigaddset(&set, SIGUSR2);
1859 sigaddset(&set, SIGIO);
1860 sigaddset(&set, SIGALRM);
1861 sigprocmask(SIG_BLOCK, &set, NULL);
1862
1863 sigprocmask(SIG_BLOCK, NULL, &set);
1864 sigdelset(&set, SIG_IPI);
1865
1866 kvm_set_signal_mask(env->kvm_cpu_state.vcpu_ctx, &set);
1867 }
1868
1869 static void qemu_kvm_system_reset(void)
1870 {
1871 CPUState *penv = first_cpu;
1872
1873 pause_all_threads();
1874
1875 qemu_system_reset();
1876
1877 while (penv) {
1878 kvm_arch_cpu_reset(penv);
1879 penv = (CPUState *)penv->next_cpu;
1880 }
1881
1882 resume_all_threads();
1883 }
1884
1885 static void process_irqchip_events(CPUState *env)
1886 {
1887 kvm_arch_process_irqchip_events(env);
1888 if (kvm_arch_has_work(env))
1889 env->halted = 0;
1890 }
1891
1892 static int kvm_main_loop_cpu(CPUState *env)
1893 {
1894 setup_kernel_sigmask(env);
1895
1896 pthread_mutex_lock(&qemu_mutex);
1897
1898 kvm_qemu_init_env(env);
1899 #ifdef TARGET_I386
1900 kvm_tpr_vcpu_start(env);
1901 #endif
1902
1903 cpu_single_env = env;
1904 kvm_arch_load_regs(env);
1905
1906 while (1) {
1907 int run_cpu = !is_cpu_stopped(env);
1908 if (run_cpu && !kvm_irqchip_in_kernel(kvm_context)) {
1909 process_irqchip_events(env);
1910 run_cpu = !env->halted;
1911 }
1912 if (run_cpu) {
1913 kvm_main_loop_wait(env, 0);
1914 kvm_cpu_exec(env);
1915 } else {
1916 kvm_main_loop_wait(env, 1000);
1917 }
1918 }
1919 pthread_mutex_unlock(&qemu_mutex);
1920 return 0;
1921 }
1922
1923 static void *ap_main_loop(void *_env)
1924 {
1925 CPUState *env = _env;
1926 sigset_t signals;
1927 struct ioperm_data *data = NULL;
1928
1929 current_env = env;
1930 env->thread_id = kvm_get_thread_id();
1931 sigfillset(&signals);
1932 sigprocmask(SIG_BLOCK, &signals, NULL);
1933 env->kvm_cpu_state.vcpu_ctx = kvm_create_vcpu(kvm_context, env->cpu_index);
1934
1935 #ifdef USE_KVM_DEVICE_ASSIGNMENT
1936 /* do ioperm for io ports of assigned devices */
1937 LIST_FOREACH(data, &ioperm_head, entries)
1938 on_vcpu(env, kvm_arch_do_ioperm, data);
1939 #endif
1940
1941 /* signal VCPU creation */
1942 pthread_mutex_lock(&qemu_mutex);
1943 current_env->kvm_cpu_state.created = 1;
1944 pthread_cond_signal(&qemu_vcpu_cond);
1945
1946 /* and wait for machine initialization */
1947 while (!qemu_system_ready)
1948 qemu_cond_wait(&qemu_system_cond);
1949 pthread_mutex_unlock(&qemu_mutex);
1950
1951 kvm_main_loop_cpu(env);
1952 return NULL;
1953 }
1954
1955 void kvm_init_vcpu(CPUState *env)
1956 {
1957 pthread_create(&env->kvm_cpu_state.thread, NULL, ap_main_loop, env);
1958
1959 while (env->kvm_cpu_state.created == 0)
1960 qemu_cond_wait(&qemu_vcpu_cond);
1961 }
1962
1963 int kvm_vcpu_inited(CPUState *env)
1964 {
1965 return env->kvm_cpu_state.created;
1966 }
1967
1968 int kvm_init_ap(void)
1969 {
1970 #ifdef TARGET_I386
1971 kvm_tpr_opt_setup();
1972 #endif
1973 qemu_add_vm_change_state_handler(kvm_vm_state_change_handler, NULL);
1974
1975 signal(SIG_IPI, sig_ipi_handler);
1976 return 0;
1977 }
1978
1979 void qemu_kvm_notify_work(void)
1980 {
1981 uint64_t value = 1;
1982 char buffer[8];
1983 size_t offset = 0;
1984
1985 if (io_thread_fd == -1)
1986 return;
1987
1988 memcpy(buffer, &value, sizeof(value));
1989
1990 while (offset < 8) {
1991 ssize_t len;
1992
1993 len = write(io_thread_fd, buffer + offset, 8 - offset);
1994 if (len == -1 && errno == EINTR)
1995 continue;
1996
1997 if (len <= 0)
1998 break;
1999
2000 offset += len;
2001 }
2002
2003 if (offset != 8)
2004 fprintf(stderr, "failed to notify io thread\n");
2005 }
2006
2007 /* If we have signalfd, we mask out the signals we want to handle and then
2008 * use signalfd to listen for them. We rely on whatever the current signal
2009 * handler is to dispatch the signals when we receive them.
2010 */
2011
2012 static void sigfd_handler(void *opaque)
2013 {
2014 int fd = (unsigned long)opaque;
2015 struct qemu_signalfd_siginfo info;
2016 struct sigaction action;
2017 ssize_t len;
2018
2019 while (1) {
2020 do {
2021 len = read(fd, &info, sizeof(info));
2022 } while (len == -1 && errno == EINTR);
2023
2024 if (len == -1 && errno == EAGAIN)
2025 break;
2026
2027 if (len != sizeof(info)) {
2028 printf("read from sigfd returned %zd: %m\n", len);
2029 return;
2030 }
2031
2032 sigaction(info.ssi_signo, NULL, &action);
2033 if (action.sa_handler)
2034 action.sa_handler(info.ssi_signo);
2035
2036 }
2037 }
2038
2039 /* Used to break IO thread out of select */
2040 static void io_thread_wakeup(void *opaque)
2041 {
2042 int fd = (unsigned long)opaque;
2043 char buffer[8];
2044 size_t offset = 0;
2045
2046 while (offset < 8) {
2047 ssize_t len;
2048
2049 len = read(fd, buffer + offset, 8 - offset);
2050 if (len == -1 && errno == EINTR)
2051 continue;
2052
2053 if (len <= 0)
2054 break;
2055
2056 offset += len;
2057 }
2058 }
2059
2060 int kvm_main_loop(void)
2061 {
2062 int fds[2];
2063 sigset_t mask;
2064 int sigfd;
2065
2066 io_thread = pthread_self();
2067 qemu_system_ready = 1;
2068
2069 if (qemu_eventfd(fds) == -1) {
2070 fprintf(stderr, "failed to create eventfd\n");
2071 return -errno;
2072 }
2073
2074 qemu_set_fd_handler2(fds[0], NULL, io_thread_wakeup, NULL,
2075 (void *)(unsigned long)fds[0]);
2076
2077 io_thread_fd = fds[1];
2078
2079 sigemptyset(&mask);
2080 sigaddset(&mask, SIGIO);
2081 sigaddset(&mask, SIGALRM);
2082 sigprocmask(SIG_BLOCK, &mask, NULL);
2083
2084 sigfd = qemu_signalfd(&mask);
2085 if (sigfd == -1) {
2086 fprintf(stderr, "failed to create signalfd\n");
2087 return -errno;
2088 }
2089
2090 fcntl(sigfd, F_SETFL, O_NONBLOCK);
2091
2092 qemu_set_fd_handler2(sigfd, NULL, sigfd_handler, NULL,
2093 (void *)(unsigned long)sigfd);
2094
2095 pthread_cond_broadcast(&qemu_system_cond);
2096
2097 io_thread_sigfd = sigfd;
2098 cpu_single_env = NULL;
2099
2100 while (1) {
2101 main_loop_wait(1000);
2102 if (qemu_shutdown_requested()) {
2103 if (qemu_no_shutdown()) {
2104 vm_stop(0);
2105 } else
2106 break;
2107 } else if (qemu_powerdown_requested())
2108 qemu_system_powerdown();
2109 else if (qemu_reset_requested())
2110 qemu_kvm_system_reset();
2111 else if (kvm_debug_cpu_requested) {
2112 gdb_set_stop_cpu(kvm_debug_cpu_requested);
2113 vm_stop(EXCP_DEBUG);
2114 kvm_debug_cpu_requested = NULL;
2115 }
2116 }
2117
2118 pause_all_threads();
2119 pthread_mutex_unlock(&qemu_mutex);
2120
2121 return 0;
2122 }
2123
2124 #ifdef KVM_CAP_SET_GUEST_DEBUG
2125 static int kvm_debug(void *opaque, void *data,
2126 struct kvm_debug_exit_arch *arch_info)
2127 {
2128 int handle = kvm_arch_debug(arch_info);
2129 CPUState *env = data;
2130
2131 if (handle) {
2132 kvm_debug_cpu_requested = env;
2133 env->kvm_cpu_state.stopped = 1;
2134 }
2135 return handle;
2136 }
2137 #endif
2138
2139 static int kvm_inb(void *opaque, uint16_t addr, uint8_t *data)
2140 {
2141 *data = cpu_inb(0, addr);
2142 return 0;
2143 }
2144
2145 static int kvm_inw(void *opaque, uint16_t addr, uint16_t *data)
2146 {
2147 *data = cpu_inw(0, addr);
2148 return 0;
2149 }
2150
2151 static int kvm_inl(void *opaque, uint16_t addr, uint32_t *data)
2152 {
2153 *data = cpu_inl(0, addr);
2154 return 0;
2155 }
2156
2157 #define PM_IO_BASE 0xb000
2158
2159 static int kvm_outb(void *opaque, uint16_t addr, uint8_t data)
2160 {
2161 if (addr == 0xb2) {
2162 switch (data) {
2163 case 0: {
2164 cpu_outb(0, 0xb3, 0);
2165 break;
2166 }
2167 case 0xf0: {
2168 unsigned x;
2169
2170 /* enable acpi */
2171 x = cpu_inw(0, PM_IO_BASE + 4);
2172 x &= ~1;
2173 cpu_outw(0, PM_IO_BASE + 4, x);
2174 break;
2175 }
2176 case 0xf1: {
2177 unsigned x;
2178
2179 /* enable acpi */
2180 x = cpu_inw(0, PM_IO_BASE + 4);
2181 x |= 1;
2182 cpu_outw(0, PM_IO_BASE + 4, x);
2183 break;
2184 }
2185 default:
2186 break;
2187 }
2188 return 0;
2189 }
2190 cpu_outb(0, addr, data);
2191 return 0;
2192 }
2193
2194 static int kvm_outw(void *opaque, uint16_t addr, uint16_t data)
2195 {
2196 cpu_outw(0, addr, data);
2197 return 0;
2198 }
2199
2200 static int kvm_outl(void *opaque, uint16_t addr, uint32_t data)
2201 {
2202 cpu_outl(0, addr, data);
2203 return 0;
2204 }
2205
2206 static int kvm_mmio_read(void *opaque, uint64_t addr, uint8_t *data, int len)
2207 {
2208 cpu_physical_memory_rw(addr, data, len, 0);
2209 return 0;
2210 }
2211
2212 static int kvm_mmio_write(void *opaque, uint64_t addr, uint8_t *data, int len)
2213 {
2214 cpu_physical_memory_rw(addr, data, len, 1);
2215 return 0;
2216 }
2217
2218 static int kvm_io_window(void *opaque)
2219 {
2220 return 1;
2221 }
2222
2223
2224 static int kvm_halt(void *opaque, kvm_vcpu_context_t vcpu)
2225 {
2226 return kvm_arch_halt(opaque, vcpu);
2227 }
2228
2229 static int kvm_shutdown(void *opaque, void *data)
2230 {
2231 CPUState *env = (CPUState *)data;
2232
2233 /* stop the current vcpu from going back to guest mode */
2234 env->kvm_cpu_state.stopped = 1;
2235
2236 qemu_system_reset_request();
2237 return 1;
2238 }
2239
2240 static int handle_unhandled(kvm_context_t kvm, kvm_vcpu_context_t vcpu,
2241 uint64_t reason)
2242 {
2243 fprintf(stderr, "kvm: unhandled exit %"PRIx64"\n", reason);
2244 return -EINVAL;
2245 }
2246
2247 static struct kvm_callbacks qemu_kvm_ops = {
2248 #ifdef KVM_CAP_SET_GUEST_DEBUG
2249 .debug = kvm_debug,
2250 #endif
2251 .inb = kvm_inb,
2252 .inw = kvm_inw,
2253 .inl = kvm_inl,
2254 .outb = kvm_outb,
2255 .outw = kvm_outw,
2256 .outl = kvm_outl,
2257 .mmio_read = kvm_mmio_read,
2258 .mmio_write = kvm_mmio_write,
2259 .halt = kvm_halt,
2260 .shutdown = kvm_shutdown,
2261 .io_window = kvm_io_window,
2262 .try_push_interrupts = kvm_try_push_interrupts,
2263 #ifdef KVM_CAP_USER_NMI
2264 .push_nmi = kvm_arch_push_nmi,
2265 #endif
2266 .post_kvm_run = kvm_post_run,
2267 .pre_kvm_run = kvm_pre_run,
2268 #ifdef TARGET_I386
2269 .tpr_access = handle_tpr_access,
2270 #endif
2271 .unhandled = handle_unhandled,
2272 };
2273
2274 int kvm_qemu_init()
2275 {
2276 /* Try to initialize kvm */
2277 kvm_context = kvm_init(&qemu_kvm_ops, cpu_single_env);
2278 if (!kvm_context) {
2279 return -1;
2280 }
2281 pthread_mutex_lock(&qemu_mutex);
2282
2283 return 0;
2284 }
2285
2286 #ifdef TARGET_I386
2287 static int destroy_region_works = 0;
2288 #endif
2289
2290
2291 #if !defined(TARGET_I386)
2292 int kvm_arch_init_irq_routing(void)
2293 {
2294 return 0;
2295 }
2296 #endif
2297
2298 int kvm_qemu_create_context(void)
2299 {
2300 int r;
2301
2302 if (!kvm_irqchip) {
2303 kvm_disable_irqchip_creation(kvm_context);
2304 }
2305 if (!kvm_pit) {
2306 kvm_disable_pit_creation(kvm_context);
2307 }
2308 if (kvm_create(kvm_context, 0, NULL) < 0) {
2309 kvm_qemu_destroy();
2310 return -1;
2311 }
2312 r = kvm_arch_qemu_create_context();
2313 if(r <0)
2314 kvm_qemu_destroy();
2315 if (kvm_pit && !kvm_pit_reinject) {
2316 if (kvm_reinject_control(kvm_context, 0)) {
2317 fprintf(stderr, "failure to disable in-kernel PIT reinjection\n");
2318 return -1;
2319 }
2320 }
2321 #ifdef TARGET_I386
2322 destroy_region_works = kvm_destroy_memory_region_works(kvm_context);
2323 #endif
2324
2325 r = kvm_arch_init_irq_routing();
2326 if (r < 0) {
2327 return r;
2328 }
2329
2330 return 0;
2331 }
2332
2333 void kvm_qemu_destroy(void)
2334 {
2335 kvm_finalize(kvm_context);
2336 }
2337
2338 #ifdef TARGET_I386
2339 static int must_use_aliases_source(target_phys_addr_t addr)
2340 {
2341 if (destroy_region_works)
2342 return false;
2343 if (addr == 0xa0000 || addr == 0xa8000)
2344 return true;
2345 return false;
2346 }
2347
2348 static int must_use_aliases_target(target_phys_addr_t addr)
2349 {
2350 if (destroy_region_works)
2351 return false;
2352 if (addr >= 0xe0000000 && addr < 0x100000000ull)
2353 return true;
2354 return false;
2355 }
2356
2357 static struct mapping {
2358 target_phys_addr_t phys;
2359 ram_addr_t ram;
2360 ram_addr_t len;
2361 } mappings[50];
2362 static int nr_mappings;
2363
2364 static struct mapping *find_ram_mapping(ram_addr_t ram_addr)
2365 {
2366 struct mapping *p;
2367
2368 for (p = mappings; p < mappings + nr_mappings; ++p) {
2369 if (p->ram <= ram_addr && ram_addr < p->ram + p->len) {
2370 return p;
2371 }
2372 }
2373 return NULL;
2374 }
2375
2376 static struct mapping *find_mapping(target_phys_addr_t start_addr)
2377 {
2378 struct mapping *p;
2379
2380 for (p = mappings; p < mappings + nr_mappings; ++p) {
2381 if (p->phys <= start_addr && start_addr < p->phys + p->len) {
2382 return p;
2383 }
2384 }
2385 return NULL;
2386 }
2387
2388 static void drop_mapping(target_phys_addr_t start_addr)
2389 {
2390 struct mapping *p = find_mapping(start_addr);
2391
2392 if (p)
2393 *p = mappings[--nr_mappings];
2394 }
2395 #endif
2396
2397 void kvm_cpu_register_physical_memory(target_phys_addr_t start_addr,
2398 unsigned long size,
2399 unsigned long phys_offset)
2400 {
2401 int r = 0;
2402 unsigned long area_flags;
2403 #ifdef TARGET_I386
2404 struct mapping *p;
2405 #endif
2406
2407 if (start_addr + size > phys_ram_size) {
2408 phys_ram_size = start_addr + size;
2409 }
2410
2411 phys_offset &= ~IO_MEM_ROM;
2412 area_flags = phys_offset & ~TARGET_PAGE_MASK;
2413
2414 if (area_flags != IO_MEM_RAM) {
2415 #ifdef TARGET_I386
2416 if (must_use_aliases_source(start_addr)) {
2417 kvm_destroy_memory_alias(kvm_context, start_addr);
2418 return;
2419 }
2420 if (must_use_aliases_target(start_addr))
2421 return;
2422 #endif
2423 while (size > 0) {
2424 p = find_mapping(start_addr);
2425 if (p) {
2426 kvm_unregister_memory_area(kvm_context, p->phys, p->len);
2427 drop_mapping(p->phys);
2428 }
2429 start_addr += TARGET_PAGE_SIZE;
2430 if (size > TARGET_PAGE_SIZE) {
2431 size -= TARGET_PAGE_SIZE;
2432 } else {
2433 size = 0;
2434 }
2435 }
2436 return;
2437 }
2438
2439 r = kvm_is_containing_region(kvm_context, start_addr, size);
2440 if (r)
2441 return;
2442
2443 if (area_flags >= TLB_MMIO)
2444 return;
2445
2446 #ifdef TARGET_I386
2447 if (must_use_aliases_source(start_addr)) {
2448 p = find_ram_mapping(phys_offset);
2449 if (p) {
2450 kvm_create_memory_alias(kvm_context, start_addr, size,
2451 p->phys + (phys_offset - p->ram));
2452 }
2453 return;
2454 }
2455 #endif
2456
2457 r = kvm_register_phys_mem(kvm_context, start_addr,
2458 qemu_get_ram_ptr(phys_offset),
2459 size, 0);
2460 if (r < 0) {
2461 printf("kvm_cpu_register_physical_memory: failed\n");
2462 exit(1);
2463 }
2464
2465 #ifdef TARGET_I386
2466 drop_mapping(start_addr);
2467 p = &mappings[nr_mappings++];
2468 p->phys = start_addr;
2469 p->ram = phys_offset;
2470 p->len = size;
2471 #endif
2472
2473 return;
2474 }
2475
2476 void kvm_cpu_unregister_physical_memory(target_phys_addr_t start_addr,
2477 target_phys_addr_t size,
2478 unsigned long phys_offset)
2479 {
2480 kvm_unregister_memory_area(kvm_context, start_addr, size);
2481 }
2482
2483 int kvm_setup_guest_memory(void *area, unsigned long size)
2484 {
2485 int ret = 0;
2486
2487 #ifdef MADV_DONTFORK
2488 if (kvm_enabled() && !kvm_has_sync_mmu())
2489 ret = madvise(area, size, MADV_DONTFORK);
2490 #endif
2491
2492 if (ret)
2493 perror ("madvise");
2494
2495 return ret;
2496 }
2497
2498 int kvm_qemu_check_extension(int ext)
2499 {
2500 return kvm_check_extension(kvm_context, ext);
2501 }
2502
2503 int kvm_qemu_init_env(CPUState *cenv)
2504 {
2505 return kvm_arch_qemu_init_env(cenv);
2506 }
2507
2508 #ifdef KVM_CAP_SET_GUEST_DEBUG
2509 struct kvm_sw_breakpoint_head kvm_sw_breakpoints =
2510 TAILQ_HEAD_INITIALIZER(kvm_sw_breakpoints);
2511
2512 struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(target_ulong pc)
2513 {
2514 struct kvm_sw_breakpoint *bp;
2515
2516 TAILQ_FOREACH(bp, &kvm_sw_breakpoints, entry) {
2517 if (bp->pc == pc)
2518 return bp;
2519 }
2520 return NULL;
2521 }
2522
2523 struct kvm_set_guest_debug_data {
2524 struct kvm_guest_debug dbg;
2525 int err;
2526 };
2527
2528 static void kvm_invoke_set_guest_debug(void *data)
2529 {
2530 struct kvm_set_guest_debug_data *dbg_data = data;
2531
2532 dbg_data->err = kvm_set_guest_debug(cpu_single_env->kvm_cpu_state.vcpu_ctx,
2533 &dbg_data->dbg);
2534 }
2535
2536 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
2537 {
2538 struct kvm_set_guest_debug_data data;
2539
2540 data.dbg.control = 0;
2541 if (env->singlestep_enabled)
2542 data.dbg.control = KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
2543
2544 kvm_arch_update_guest_debug(env, &data.dbg);
2545 data.dbg.control |= reinject_trap;
2546
2547 on_vcpu(env, kvm_invoke_set_guest_debug, &data);
2548 return data.err;
2549 }
2550
2551 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
2552 target_ulong len, int type)
2553 {
2554 struct kvm_sw_breakpoint *bp;
2555 CPUState *env;
2556 int err;
2557
2558 if (type == GDB_BREAKPOINT_SW) {
2559 bp = kvm_find_sw_breakpoint(addr);
2560 if (bp) {
2561 bp->use_count++;
2562 return 0;
2563 }
2564
2565 bp = qemu_malloc(sizeof(struct kvm_sw_breakpoint));
2566 if (!bp)
2567 return -ENOMEM;
2568
2569 bp->pc = addr;
2570 bp->use_count = 1;
2571 err = kvm_arch_insert_sw_breakpoint(current_env, bp);
2572 if (err) {
2573 free(bp);
2574 return err;
2575 }
2576
2577 TAILQ_INSERT_HEAD(&kvm_sw_breakpoints, bp, entry);
2578 } else {
2579 err = kvm_arch_insert_hw_breakpoint(addr, len, type);
2580 if (err)
2581 return err;
2582 }
2583
2584 for (env = first_cpu; env != NULL; env = env->next_cpu) {
2585 err = kvm_update_guest_debug(env, 0);
2586 if (err)
2587 return err;
2588 }
2589 return 0;
2590 }
2591
2592 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
2593 target_ulong len, int type)
2594 {
2595 struct kvm_sw_breakpoint *bp;
2596 CPUState *env;
2597 int err;
2598
2599 if (type == GDB_BREAKPOINT_SW) {
2600 bp = kvm_find_sw_breakpoint(addr);
2601 if (!bp)
2602 return -ENOENT;
2603
2604 if (bp->use_count > 1) {
2605 bp->use_count--;
2606 return 0;
2607 }
2608
2609 err = kvm_arch_remove_sw_breakpoint(current_env, bp);
2610 if (err)
2611 return err;
2612
2613 TAILQ_REMOVE(&kvm_sw_breakpoints, bp, entry);
2614 qemu_free(bp);
2615 } else {
2616 err = kvm_arch_remove_hw_breakpoint(addr, len, type);
2617 if (err)
2618 return err;
2619 }
2620
2621 for (env = first_cpu; env != NULL; env = env->next_cpu) {
2622 err = kvm_update_guest_debug(env, 0);
2623 if (err)
2624 return err;
2625 }
2626 return 0;
2627 }
2628
2629 void kvm_remove_all_breakpoints(CPUState *current_env)
2630 {
2631 struct kvm_sw_breakpoint *bp, *next;
2632 CPUState *env;
2633
2634 TAILQ_FOREACH_SAFE(bp, &kvm_sw_breakpoints, entry, next) {
2635 if (kvm_arch_remove_sw_breakpoint(current_env, bp) != 0) {
2636 /* Try harder to find a CPU that currently sees the breakpoint. */
2637 for (env = first_cpu; env != NULL; env = env->next_cpu) {
2638 if (kvm_arch_remove_sw_breakpoint(env, bp) == 0)
2639 break;
2640 }
2641 }
2642 }
2643 kvm_arch_remove_all_hw_breakpoints();
2644
2645 for (env = first_cpu; env != NULL; env = env->next_cpu)
2646 kvm_update_guest_debug(env, 0);
2647 }
2648
2649 #else /* !KVM_CAP_SET_GUEST_DEBUG */
2650
2651 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
2652 {
2653 return -EINVAL;
2654 }
2655
2656 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
2657 target_ulong len, int type)
2658 {
2659 return -EINVAL;
2660 }
2661
2662 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
2663 target_ulong len, int type)
2664 {
2665 return -EINVAL;
2666 }
2667
2668 void kvm_remove_all_breakpoints(CPUState *current_env)
2669 {
2670 }
2671 #endif /* !KVM_CAP_SET_GUEST_DEBUG */
2672
2673 /*
2674 * dirty pages logging
2675 */
2676 /* FIXME: use unsigned long pointer instead of unsigned char */
2677 unsigned char *kvm_dirty_bitmap = NULL;
2678 int kvm_physical_memory_set_dirty_tracking(int enable)
2679 {
2680 int r = 0;
2681
2682 if (!kvm_enabled())
2683 return 0;
2684
2685 if (enable) {
2686 if (!kvm_dirty_bitmap) {
2687 unsigned bitmap_size = BITMAP_SIZE(phys_ram_size);
2688 kvm_dirty_bitmap = qemu_malloc(bitmap_size);
2689 if (kvm_dirty_bitmap == NULL) {
2690 perror("Failed to allocate dirty pages bitmap");
2691 r=-1;
2692 }
2693 else {
2694 r = kvm_dirty_pages_log_enable_all(kvm_context);
2695 }
2696 }
2697 }
2698 else {
2699 if (kvm_dirty_bitmap) {
2700 r = kvm_dirty_pages_log_reset(kvm_context);
2701 qemu_free(kvm_dirty_bitmap);
2702 kvm_dirty_bitmap = NULL;
2703 }
2704 }
2705 return r;
2706 }
2707
2708 /* get kvm's dirty pages bitmap and update qemu's */
2709 static int kvm_get_dirty_pages_log_range(unsigned long start_addr,
2710 unsigned char *bitmap,
2711 unsigned long offset,
2712 unsigned long mem_size)
2713 {
2714 unsigned int i, j, n=0;
2715 unsigned char c;
2716 unsigned long page_number, addr, addr1;
2717 ram_addr_t ram_addr;
2718 unsigned int len = ((mem_size/TARGET_PAGE_SIZE) + 7) / 8;
2719
2720 /*
2721 * bitmap-traveling is faster than memory-traveling (for addr...)
2722 * especially when most of the memory is not dirty.
2723 */
2724 for (i=0; i<len; i++) {
2725 c = bitmap[i];
2726 while (c>0) {
2727 j = ffsl(c) - 1;
2728 c &= ~(1u<<j);
2729 page_number = i * 8 + j;
2730 addr1 = page_number * TARGET_PAGE_SIZE;
2731 addr = offset + addr1;
2732 ram_addr = cpu_get_physical_page_desc(addr);
2733 cpu_physical_memory_set_dirty(ram_addr);
2734 n++;
2735 }
2736 }
2737 return 0;
2738 }
2739 static int kvm_get_dirty_bitmap_cb(unsigned long start, unsigned long len,
2740 void *bitmap, void *opaque)
2741 {
2742 return kvm_get_dirty_pages_log_range(start, bitmap, start, len);
2743 }
2744
2745 /*
2746 * get kvm's dirty pages bitmap and update qemu's
2747 * we only care about physical ram, which resides in slots 0 and 3
2748 */
2749 int kvm_update_dirty_pages_log(void)
2750 {
2751 int r = 0;
2752
2753
2754 r = kvm_get_dirty_pages_range(kvm_context, 0, -1UL,
2755 NULL,
2756 kvm_get_dirty_bitmap_cb);
2757 return r;
2758 }
2759
2760 void kvm_qemu_log_memory(target_phys_addr_t start, target_phys_addr_t size,
2761 int log)
2762 {
2763 if (log)
2764 kvm_dirty_pages_log_enable_slot(kvm_context, start, size);
2765 else {
2766 #ifdef TARGET_I386
2767 if (must_use_aliases_target(start))
2768 return;
2769 #endif
2770 kvm_dirty_pages_log_disable_slot(kvm_context, start, size);
2771 }
2772 }
2773
2774 int kvm_get_phys_ram_page_bitmap(unsigned char *bitmap)
2775 {
2776 unsigned int bsize = BITMAP_SIZE(phys_ram_size);
2777 unsigned int brsize = BITMAP_SIZE(ram_size);
2778 unsigned int extra_pages = (phys_ram_size - ram_size) / TARGET_PAGE_SIZE;
2779 unsigned int extra_bytes = (extra_pages +7)/8;
2780 unsigned int hole_start = BITMAP_SIZE(0xa0000);
2781 unsigned int hole_end = BITMAP_SIZE(0xc0000);
2782
2783 memset(bitmap, 0xFF, brsize + extra_bytes);
2784 memset(bitmap + hole_start, 0, hole_end - hole_start);
2785 memset(bitmap + brsize + extra_bytes, 0, bsize - brsize - extra_bytes);
2786
2787 return 0;
2788 }
2789
2790 #ifdef KVM_CAP_IRQCHIP
2791
2792 int kvm_set_irq(int irq, int level, int *status)
2793 {
2794 return kvm_set_irq_level(kvm_context, irq, level, status);
2795 }
2796
2797 #endif
2798
2799 int qemu_kvm_get_dirty_pages(unsigned long phys_addr, void *buf)
2800 {
2801 return kvm_get_dirty_pages(kvm_context, phys_addr, buf);
2802 }
2803
2804 void *kvm_cpu_create_phys_mem(target_phys_addr_t start_addr,
2805 unsigned long size, int log, int writable)
2806 {
2807 return kvm_create_phys_mem(kvm_context, start_addr, size, log, writable);
2808 }
2809
2810 void kvm_cpu_destroy_phys_mem(target_phys_addr_t start_addr,
2811 unsigned long size)
2812 {
2813 kvm_destroy_phys_mem(kvm_context, start_addr, size);
2814 }
2815
2816 void kvm_mutex_unlock(void)
2817 {
2818 assert(!cpu_single_env);
2819 pthread_mutex_unlock(&qemu_mutex);
2820 }
2821
2822 void kvm_mutex_lock(void)
2823 {
2824 pthread_mutex_lock(&qemu_mutex);
2825 cpu_single_env = NULL;
2826 }
2827
2828 int qemu_kvm_register_coalesced_mmio(target_phys_addr_t addr, unsigned int size)
2829 {
2830 return kvm_register_coalesced_mmio(kvm_context, addr, size);
2831 }
2832
2833 int qemu_kvm_unregister_coalesced_mmio(target_phys_addr_t addr,
2834 unsigned int size)
2835 {
2836 return kvm_unregister_coalesced_mmio(kvm_context, addr, size);
2837 }
2838
2839 int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
2840 {
2841 return kvm_register_coalesced_mmio(kvm_context, start, size);
2842 }
2843
2844 int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
2845 {
2846 return kvm_unregister_coalesced_mmio(kvm_context, start, size);
2847 }
2848
2849 #ifdef USE_KVM_DEVICE_ASSIGNMENT
2850 void kvm_add_ioperm_data(struct ioperm_data *data)
2851 {
2852 LIST_INSERT_HEAD(&ioperm_head, data, entries);
2853 }
2854
2855 void kvm_remove_ioperm_data(unsigned long start_port, unsigned long num)
2856 {
2857 struct ioperm_data *data;
2858
2859 data = LIST_FIRST(&ioperm_head);
2860 while (data) {
2861 struct ioperm_data *next = LIST_NEXT(data, entries);
2862
2863 if (data->start_port == start_port && data->num == num) {
2864 LIST_REMOVE(data, entries);
2865 qemu_free(data);
2866 }
2867
2868 data = next;
2869 }
2870 }
2871
2872 void kvm_ioperm(CPUState *env, void *data)
2873 {
2874 if (kvm_enabled() && qemu_system_ready)
2875 on_vcpu(env, kvm_arch_do_ioperm, data);
2876 }
2877
2878 #endif
2879
2880 int kvm_physical_sync_dirty_bitmap(target_phys_addr_t start_addr, target_phys_addr_t end_addr)
2881 {
2882 #ifndef TARGET_IA64
2883
2884 #ifdef TARGET_I386
2885 if (must_use_aliases_source(start_addr))
2886 return 0;
2887 #endif
2888
2889 kvm_get_dirty_pages_range(kvm_context, start_addr, end_addr - start_addr,
2890 NULL, kvm_get_dirty_bitmap_cb);
2891 #endif
2892 return 0;
2893 }
2894
2895 int kvm_log_start(target_phys_addr_t phys_addr, target_phys_addr_t len)
2896 {
2897 #ifdef TARGET_I386
2898 if (must_use_aliases_source(phys_addr))
2899 return 0;
2900 #endif
2901
2902 #ifndef TARGET_IA64
2903 kvm_qemu_log_memory(phys_addr, len, 1);
2904 #endif
2905 return 0;
2906 }
2907
2908 int kvm_log_stop(target_phys_addr_t phys_addr, target_phys_addr_t len)
2909 {
2910 #ifdef TARGET_I386
2911 if (must_use_aliases_source(phys_addr))
2912 return 0;
2913 #endif
2914
2915 #ifndef TARGET_IA64
2916 kvm_qemu_log_memory(phys_addr, len, 0);
2917 #endif
2918 return 0;
2919 }
2920
2921 void qemu_kvm_cpu_stop(CPUState *env)
2922 {
2923 if (kvm_enabled())
2924 env->kvm_cpu_state.stopped = 1;
2925 }
Patches shipped by Fedora