qemu-kvm.c

   1 /*
   2  * qemu/kvm integration
   3  *
   4  * Copyright (C) 2006-2008 Qumranet Technologies
   5  *
   6  * Licensed under the terms of the GNU GPL version 2 or higher.
   7  */
   8 #include "config.h"
   9 #include "config-host.h"
  10
  11 int kvm_allowed = 1;
  12 int kvm_irqchip = 1;
  13 int kvm_pit = 1;
  14
  15 #include <assert.h>
  16 #include <string.h>
  17 #include "hw/hw.h"
  18 #include "sysemu.h"
  19 #include "qemu-common.h"
  20 #include "console.h"
  21 #include "block.h"
  22 #include "compatfd.h"
  23
  24 #include "qemu-kvm.h"
  25 #include <libkvm.h>
  26 #include <pthread.h>
  27 #include <sys/utsname.h>
  28 #include <sys/syscall.h>
  29 #include <sys/mman.h>
  30
  31 #define bool _Bool
  32 #define false 0
  33 #define true 1
  34
  35 extern void perror(const char *s);
  36
  37 kvm_context_t kvm_context;
  38
  39 extern int smp_cpus;
  40
  41 pthread_mutex_t qemu_mutex = PTHREAD_MUTEX_INITIALIZER;
  42 pthread_cond_t qemu_vcpu_cond = PTHREAD_COND_INITIALIZER;
  43 pthread_cond_t qemu_system_cond = PTHREAD_COND_INITIALIZER;
  44 pthread_cond_t qemu_pause_cond = PTHREAD_COND_INITIALIZER;
  45 pthread_cond_t qemu_work_cond = PTHREAD_COND_INITIALIZER;
  46 __thread struct vcpu_info *vcpu;
  47
  48 static int qemu_system_ready;
  49
  50 #define SIG_IPI (SIGRTMIN+4)
  51
  52 struct qemu_kvm_work_item {
  53     struct qemu_kvm_work_item *next;
  54     void (*func)(void *data);
  55     void *data;
  56     bool done;
  57 };
  58
  59 struct vcpu_info {
  60     CPUState *env;
  61     int sipi_needed;
  62     int init;
  63     pthread_t thread;
  64     int signalled;
  65     int stop;
  66     int stopped;
  67     int created;
  68     struct qemu_kvm_work_item *queued_work_first, *queued_work_last;
  69 } vcpu_info[256];
  70
  71 pthread_t io_thread;
  72 static int io_thread_fd = -1;
  73 static int io_thread_sigfd = -1;
  74
  75 static int kvm_debug_stop_requested;
  76
  77 /* The list of ioperm_data */
  78 static LIST_HEAD(, ioperm_data) ioperm_head;
  79
  80 static inline unsigned long kvm_get_thread_id(void)
  81 {
  82     return syscall(SYS_gettid);
  83 }
  84
  85 static void qemu_cond_wait(pthread_cond_t *cond)
  86 {
  87     CPUState *env = cpu_single_env;
  88     static const struct timespec ts = {
  89         .tv_sec = 0,
  90         .tv_nsec = 100000,
  91     };
  92
  93     pthread_cond_timedwait(cond, &qemu_mutex, &ts);
  94     cpu_single_env = env;
  95 }
  96
  97 CPUState *qemu_kvm_cpu_env(int index)
  98 {
  99     return vcpu_info[index].env;
 100 }
 101
 102 static void sig_ipi_handler(int n)
 103 {
 104 }
 105
 106 static void on_vcpu(CPUState *env, void (*func)(void *data), void *data)
 107 {
 108     struct vcpu_info *vi = &vcpu_info[env->cpu_index];
 109     struct qemu_kvm_work_item wi;
 110
 111     if (vi == vcpu) {
 112         func(data);
 113         return;
 114     }
 115
 116     wi.func = func;
 117     wi.data = data;
 118     if (!vi->queued_work_first)
 119         vi->queued_work_first = &wi;
 120     else
 121         vi->queued_work_last->next = &wi;
 122     vi->queued_work_last = &wi;
 123     wi.next = NULL;
 124     wi.done = false;
 125
 126     pthread_kill(vi->thread, SIG_IPI);
 127     while (!wi.done)
 128         qemu_cond_wait(&qemu_work_cond);
 129 }
 130
 131 static void inject_interrupt(void *data)
 132 {
 133     cpu_interrupt(vcpu->env, (int)data);
 134 }
 135
 136 void kvm_inject_interrupt(CPUState *env, int mask)
 137 {
 138     on_vcpu(env, inject_interrupt, (void *)mask);
 139 }
 140
 141 void kvm_update_interrupt_request(CPUState *env)
 142 {
 143     int signal = 0;
 144
 145     if (env) {
 146         if (!vcpu)
 147             signal = 1;
 148         if (vcpu && env != vcpu->env && !vcpu_info[env->cpu_index].signalled)
 149             signal = 1;
 150
 151         if (signal) {
 152             vcpu_info[env->cpu_index].signalled = 1;
 153                 if (vcpu_info[env->cpu_index].thread)
 154                     pthread_kill(vcpu_info[env->cpu_index].thread, SIG_IPI);
 155         }
 156     }
 157 }
 158
 159 void kvm_update_after_sipi(CPUState *env)
 160 {
 161     vcpu_info[env->cpu_index].sipi_needed = 1;
 162     kvm_update_interrupt_request(env);
 163 }
 164
 165 void kvm_apic_init(CPUState *env)
 166 {
 167     if (env->cpu_index != 0)
 168         vcpu_info[env->cpu_index].init = 1;
 169     kvm_update_interrupt_request(env);
 170 }
 171
 172 #include <signal.h>
 173
 174 static int try_push_interrupts(void *opaque)
 175 {
 176     return kvm_arch_try_push_interrupts(opaque);
 177 }
 178
 179 static int try_push_nmi(void *opaque)
 180 {
 181     return kvm_arch_try_push_nmi(opaque);
 182 }
 183
 184 static void post_kvm_run(void *opaque, int vcpu)
 185 {
 186
 187     pthread_mutex_lock(&qemu_mutex);
 188     kvm_arch_post_kvm_run(opaque, vcpu);
 189 }
 190
 191 static int pre_kvm_run(void *opaque, int vcpu)
 192 {
 193     CPUState *env = qemu_kvm_cpu_env(vcpu);
 194
 195     kvm_arch_pre_kvm_run(opaque, vcpu);
 196
 197     if (env->interrupt_request & CPU_INTERRUPT_EXIT)
 198         return 1;
 199     pthread_mutex_unlock(&qemu_mutex);
 200     return 0;
 201 }
 202
 203 static void kvm_do_load_registers(void *_env)
 204 {
 205     CPUState *env = _env;
 206
 207     kvm_arch_load_regs(env);
 208 }
 209
 210 void kvm_load_registers(CPUState *env)
 211 {
 212     if (kvm_enabled() && qemu_system_ready)
 213         on_vcpu(env, kvm_do_load_registers, env);
 214 }
 215
 216 static void kvm_do_save_registers(void *_env)
 217 {
 218     CPUState *env = _env;
 219
 220     kvm_arch_save_regs(env);
 221 }
 222
 223 void kvm_save_registers(CPUState *env)
 224 {
 225     if (kvm_enabled())
 226         on_vcpu(env, kvm_do_save_registers, env);
 227 }
 228
 229 int kvm_cpu_exec(CPUState *env)
 230 {
 231     int r;
 232
 233     r = kvm_run(kvm_context, env->cpu_index);
 234     if (r < 0) {
 235         printf("kvm_run returned %d\n", r);
 236         exit(1);
 237     }
 238
 239     return 0;
 240 }
 241
 242 extern int vm_running;
 243
 244 static int has_work(CPUState *env)
 245 {
 246     if (!vm_running || (env && vcpu_info[env->cpu_index].stopped))
 247         return 0;
 248     if (!env->halted)
 249         return 1;
 250     return kvm_arch_has_work(env);
 251 }
 252
 253 static void flush_queued_work(CPUState *env)
 254 {
 255     struct vcpu_info *vi = &vcpu_info[env->cpu_index];
 256     struct qemu_kvm_work_item *wi;
 257
 258     if (!vi->queued_work_first)
 259         return;
 260
 261     while ((wi = vi->queued_work_first)) {
 262         vi->queued_work_first = wi->next;
 263         wi->func(wi->data);
 264         wi->done = true;
 265     }
 266     vi->queued_work_last = NULL;
 267     pthread_cond_broadcast(&qemu_work_cond);
 268 }
 269
 270 static void kvm_main_loop_wait(CPUState *env, int timeout)
 271 {
 272     struct timespec ts;
 273     int r, e;
 274     siginfo_t siginfo;
 275     sigset_t waitset;
 276
 277     pthread_mutex_unlock(&qemu_mutex);
 278
 279     ts.tv_sec = timeout / 1000;
 280     ts.tv_nsec = (timeout % 1000) * 1000000;
 281     sigemptyset(&waitset);
 282     sigaddset(&waitset, SIG_IPI);
 283
 284     r = sigtimedwait(&waitset, &siginfo, &ts);
 285     e = errno;
 286
 287     pthread_mutex_lock(&qemu_mutex);
 288
 289     if (r == -1 && !(e == EAGAIN || e == EINTR)) {
 290         printf("sigtimedwait: %s\n", strerror(e));
 291         exit(1);
 292     }
 293
 294     cpu_single_env = env;
 295     flush_queued_work(env);
 296
 297     if (vcpu_info[env->cpu_index].stop) {
 298         vcpu_info[env->cpu_index].stop = 0;
 299         vcpu_info[env->cpu_index].stopped = 1;
 300         pthread_cond_signal(&qemu_pause_cond);
 301     }
 302
 303     vcpu_info[env->cpu_index].signalled = 0;
 304 }
 305
 306 static int all_threads_paused(void)
 307 {
 308     int i;
 309
 310     for (i = 0; i < smp_cpus; ++i)
 311         if (vcpu_info[i].stop)
 312             return 0;
 313     return 1;
 314 }
 315
 316 static void pause_all_threads(void)
 317 {
 318     int i;
 319
 320     assert(!cpu_single_env);
 321
 322     for (i = 0; i < smp_cpus; ++i) {
 323         vcpu_info[i].stop = 1;
 324         pthread_kill(vcpu_info[i].thread, SIG_IPI);
 325     }
 326     while (!all_threads_paused())
 327         qemu_cond_wait(&qemu_pause_cond);
 328 }
 329
 330 static void resume_all_threads(void)
 331 {
 332     int i;
 333
 334     assert(!cpu_single_env);
 335
 336     for (i = 0; i < smp_cpus; ++i) {
 337         vcpu_info[i].stop = 0;
 338         vcpu_info[i].stopped = 0;
 339         pthread_kill(vcpu_info[i].thread, SIG_IPI);
 340     }
 341 }
 342
 343 static void kvm_vm_state_change_handler(void *context, int running)
 344 {
 345     if (running)
 346         resume_all_threads();
 347     else
 348         pause_all_threads();
 349 }
 350
 351 static void update_regs_for_sipi(CPUState *env)
 352 {
 353     kvm_arch_update_regs_for_sipi(env);
 354     vcpu_info[env->cpu_index].sipi_needed = 0;
 355 }
 356
 357 static void update_regs_for_init(CPUState *env)
 358 {
 359 #ifdef TARGET_I386
 360     SegmentCache cs = env->segs[R_CS];
 361 #endif
 362
 363     cpu_reset(env);
 364
 365 #ifdef TARGET_I386
 366     /* restore SIPI vector */
 367     if(vcpu_info[env->cpu_index].sipi_needed)
 368         env->segs[R_CS] = cs;
 369
 370     vcpu_info[env->cpu_index].init = 0;
 371 #endif
 372     kvm_arch_load_regs(env);
 373 }
 374
 375 static void setup_kernel_sigmask(CPUState *env)
 376 {
 377     sigset_t set;
 378
 379     sigemptyset(&set);
 380     sigaddset(&set, SIGUSR2);
 381     sigaddset(&set, SIGIO);
 382     sigaddset(&set, SIGALRM);
 383     sigprocmask(SIG_BLOCK, &set, NULL);
 384
 385     sigprocmask(SIG_BLOCK, NULL, &set);
 386     sigdelset(&set, SIG_IPI);
 387
 388     kvm_set_signal_mask(kvm_context, env->cpu_index, &set);
 389 }
 390
 391 void qemu_kvm_system_reset(void)
 392 {
 393     int i;
 394
 395     pause_all_threads();
 396
 397     qemu_system_reset();
 398
 399     for (i = 0; i < smp_cpus; ++i)
 400         kvm_arch_cpu_reset(vcpu_info[i].env);
 401
 402     resume_all_threads();
 403 }
 404
 405 static int kvm_main_loop_cpu(CPUState *env)
 406 {
 407     struct vcpu_info *info = &vcpu_info[env->cpu_index];
 408
 409     setup_kernel_sigmask(env);
 410
 411     pthread_mutex_lock(&qemu_mutex);
 412     if (kvm_irqchip_in_kernel(kvm_context))
 413         env->halted = 0;
 414
 415     kvm_qemu_init_env(env);
 416 #ifdef TARGET_I386
 417     kvm_tpr_vcpu_start(env);
 418 #endif
 419
 420     cpu_single_env = env;
 421     kvm_load_registers(env);
 422
 423     while (1) {
 424         while (!has_work(env))
 425             kvm_main_loop_wait(env, 1000);
 426         if (env->interrupt_request & (CPU_INTERRUPT_HARD | CPU_INTERRUPT_NMI))
 427             env->halted = 0;
 428     if (!kvm_irqchip_in_kernel(kvm_context)) {
 429             if (info->init)
 430                 update_regs_for_init(env);
 431             if (info->sipi_needed)
 432                 update_regs_for_sipi(env);
 433     }
 434         if (!env->halted && !info->init)
 435             kvm_cpu_exec(env);
 436         env->interrupt_request &= ~CPU_INTERRUPT_EXIT;
 437         kvm_main_loop_wait(env, 0);
 438     }
 439     pthread_mutex_unlock(&qemu_mutex);
 440     return 0;
 441 }
 442
 443 static void *ap_main_loop(void *_env)
 444 {
 445     CPUState *env = _env;
 446     sigset_t signals;
 447     struct ioperm_data *data = NULL;
 448
 449     vcpu = &vcpu_info[env->cpu_index];
 450     vcpu->env = env;
 451     vcpu->env->thread_id = kvm_get_thread_id();
 452     sigfillset(&signals);
 453     sigprocmask(SIG_BLOCK, &signals, NULL);
 454     kvm_create_vcpu(kvm_context, env->cpu_index);
 455     kvm_qemu_init_env(env);
 456
 457 #ifdef USE_KVM_DEVICE_ASSIGNMENT
 458     /* do ioperm for io ports of assigned devices */
 459     LIST_FOREACH(data, &ioperm_head, entries)
 460         on_vcpu(env, kvm_arch_do_ioperm, data);
 461 #endif
 462
 463     /* signal VCPU creation */
 464     pthread_mutex_lock(&qemu_mutex);
 465     vcpu->created = 1;
 466     pthread_cond_signal(&qemu_vcpu_cond);
 467
 468     /* and wait for machine initialization */
 469     while (!qemu_system_ready)
 470         qemu_cond_wait(&qemu_system_cond);
 471     pthread_mutex_unlock(&qemu_mutex);
 472
 473     kvm_main_loop_cpu(env);
 474     return NULL;
 475 }
 476
 477 void kvm_init_vcpu(CPUState *env)
 478 {
 479     int cpu = env->cpu_index;
 480     pthread_create(&vcpu_info[cpu].thread, NULL, ap_main_loop, env);
 481
 482     while (vcpu_info[cpu].created == 0)
 483         qemu_cond_wait(&qemu_vcpu_cond);
 484 }
 485
 486 int kvm_init_ap(void)
 487 {
 488 #ifdef TARGET_I386
 489     kvm_tpr_opt_setup();
 490 #endif
 491     qemu_add_vm_change_state_handler(kvm_vm_state_change_handler, NULL);
 492
 493     signal(SIG_IPI, sig_ipi_handler);
 494     return 0;
 495 }
 496
 497 void qemu_kvm_notify_work(void)
 498 {
 499     uint64_t value = 1;
 500     char buffer[8];
 501     size_t offset = 0;
 502
 503     if (io_thread_fd == -1)
 504         return;
 505
 506     memcpy(buffer, &value, sizeof(value));
 507
 508     while (offset < 8) {
 509         ssize_t len;
 510
 511         len = write(io_thread_fd, buffer + offset, 8 - offset);
 512         if (len == -1 && errno == EINTR)
 513             continue;
 514
 515         if (len <= 0)
 516             break;
 517
 518         offset += len;
 519     }
 520
 521     if (offset != 8)
 522         fprintf(stderr, "failed to notify io thread\n");
 523 }
 524
 525 /* If we have signalfd, we mask out the signals we want to handle and then
 526  * use signalfd to listen for them.  We rely on whatever the current signal
 527  * handler is to dispatch the signals when we receive them.
 528  */
 529
 530 static void sigfd_handler(void *opaque)
 531 {
 532     int fd = (unsigned long)opaque;
 533     struct qemu_signalfd_siginfo info;
 534     struct sigaction action;
 535     ssize_t len;
 536
 537     while (1) {
 538         do {
 539             len = read(fd, &info, sizeof(info));
 540         } while (len == -1 && errno == EINTR);
 541
 542         if (len == -1 && errno == EAGAIN)
 543             break;
 544
 545         if (len != sizeof(info)) {
 546             printf("read from sigfd returned %ld: %m\n", len);
 547             return;
 548         }
 549
 550         sigaction(info.ssi_signo, NULL, &action);
 551         if (action.sa_handler)
 552             action.sa_handler(info.ssi_signo);
 553
 554     }
 555 }
 556
 557 /* Used to break IO thread out of select */
 558 static void io_thread_wakeup(void *opaque)
 559 {
 560     int fd = (unsigned long)opaque;
 561     char buffer[8];
 562     size_t offset = 0;
 563
 564     while (offset < 8) {
 565         ssize_t len;
 566
 567         len = read(fd, buffer + offset, 8 - offset);
 568         if (len == -1 && errno == EINTR)
 569             continue;
 570
 571         if (len <= 0)
 572             break;
 573
 574         offset += len;
 575     }
 576 }
 577
 578 int kvm_main_loop(void)
 579 {
 580     int fds[2];
 581     sigset_t mask;
 582     int sigfd;
 583
 584     io_thread = pthread_self();
 585     qemu_system_ready = 1;
 586
 587     if (qemu_eventfd(fds) == -1) {
 588         fprintf(stderr, "failed to create eventfd\n");
 589         return -errno;
 590     }
 591
 592     qemu_set_fd_handler2(fds[0], NULL, io_thread_wakeup, NULL,
 593                          (void *)(unsigned long)fds[0]);
 594
 595     io_thread_fd = fds[1];
 596
 597     sigemptyset(&mask);
 598     sigaddset(&mask, SIGIO);
 599     sigaddset(&mask, SIGALRM);
 600     sigprocmask(SIG_BLOCK, &mask, NULL);
 601
 602     sigfd = qemu_signalfd(&mask);
 603     if (sigfd == -1) {
 604         fprintf(stderr, "failed to create signalfd\n");
 605         return -errno;
 606     }
 607
 608     fcntl(sigfd, F_SETFL, O_NONBLOCK);
 609
 610     qemu_set_fd_handler2(sigfd, NULL, sigfd_handler, NULL,
 611                          (void *)(unsigned long)sigfd);
 612
 613     pthread_cond_broadcast(&qemu_system_cond);
 614
 615     io_thread_sigfd = sigfd;
 616     cpu_single_env = NULL;
 617
 618     while (1) {
 619         main_loop_wait(1000);
 620         if (qemu_shutdown_requested())
 621             break;
 622         else if (qemu_powerdown_requested())
 623             qemu_system_powerdown();
 624         else if (qemu_reset_requested())
 625             qemu_kvm_system_reset();
 626         else if (kvm_debug_stop_requested) {
 627             vm_stop(EXCP_DEBUG);
 628             kvm_debug_stop_requested = 0;
 629         }
 630     }
 631
 632     pause_all_threads();
 633     pthread_mutex_unlock(&qemu_mutex);
 634
 635     return 0;
 636 }
 637
 638 static int kvm_debug(void *opaque, int vcpu)
 639 {
 640     kvm_debug_stop_requested = 1;
 641     vcpu_info[vcpu].stopped = 1;
 642     return 1;
 643 }
 644
 645 static int kvm_inb(void *opaque, uint16_t addr, uint8_t *data)
 646 {
 647     *data = cpu_inb(0, addr);
 648     return 0;
 649 }
 650
 651 static int kvm_inw(void *opaque, uint16_t addr, uint16_t *data)
 652 {
 653     *data = cpu_inw(0, addr);
 654     return 0;
 655 }
 656
 657 static int kvm_inl(void *opaque, uint16_t addr, uint32_t *data)
 658 {
 659     *data = cpu_inl(0, addr);
 660     return 0;
 661 }
 662
 663 #define PM_IO_BASE 0xb000
 664
 665 static int kvm_outb(void *opaque, uint16_t addr, uint8_t data)
 666 {
 667     if (addr == 0xb2) {
 668         switch (data) {
 669         case 0: {
 670             cpu_outb(0, 0xb3, 0);
 671             break;
 672         }
 673         case 0xf0: {
 674             unsigned x;
 675
 676             /* enable acpi */
 677             x = cpu_inw(0, PM_IO_BASE + 4);
 678             x &= ~1;
 679             cpu_outw(0, PM_IO_BASE + 4, x);
 680             break;
 681         }
 682         case 0xf1: {
 683             unsigned x;
 684
 685             /* enable acpi */
 686             x = cpu_inw(0, PM_IO_BASE + 4);
 687             x |= 1;
 688             cpu_outw(0, PM_IO_BASE + 4, x);
 689             break;
 690         }
 691         default:
 692             break;
 693         }
 694         return 0;
 695     }
 696     cpu_outb(0, addr, data);
 697     return 0;
 698 }
 699
 700 static int kvm_outw(void *opaque, uint16_t addr, uint16_t data)
 701 {
 702     cpu_outw(0, addr, data);
 703     return 0;
 704 }
 705
 706 static int kvm_outl(void *opaque, uint16_t addr, uint32_t data)
 707 {
 708     cpu_outl(0, addr, data);
 709     return 0;
 710 }
 711
 712 static int kvm_mmio_read(void *opaque, uint64_t addr, uint8_t *data, int len)
 713 {
 714         cpu_physical_memory_rw(addr, data, len, 0);
 715         return 0;
 716 }
 717
 718 static int kvm_mmio_write(void *opaque, uint64_t addr, uint8_t *data, int len)
 719 {
 720         cpu_physical_memory_rw(addr, data, len, 1);
 721         return 0;
 722 }
 723
 724 static int kvm_io_window(void *opaque)
 725 {
 726     return 1;
 727 }
 728
 729
 730 static int kvm_halt(void *opaque, int vcpu)
 731 {
 732     return kvm_arch_halt(opaque, vcpu);
 733 }
 734
 735 static int kvm_shutdown(void *opaque, int vcpu)
 736 {
 737     /* stop the current vcpu from going back to guest mode */
 738     vcpu_info[cpu_single_env->cpu_index].stopped = 1;
 739
 740     qemu_system_reset_request();
 741     return 1;
 742 }
 743
 744 static struct kvm_callbacks qemu_kvm_ops = {
 745     .debug = kvm_debug,
 746     .inb   = kvm_inb,
 747     .inw   = kvm_inw,
 748     .inl   = kvm_inl,
 749     .outb  = kvm_outb,
 750     .outw  = kvm_outw,
 751     .outl  = kvm_outl,
 752     .mmio_read = kvm_mmio_read,
 753     .mmio_write = kvm_mmio_write,
 754     .halt  = kvm_halt,
 755     .shutdown = kvm_shutdown,
 756     .io_window = kvm_io_window,
 757     .try_push_interrupts = try_push_interrupts,
 758     .try_push_nmi = try_push_nmi,
 759     .post_kvm_run = post_kvm_run,
 760     .pre_kvm_run = pre_kvm_run,
 761 #ifdef TARGET_I386
 762     .tpr_access = handle_tpr_access,
 763 #endif
 764 #ifdef TARGET_PPC
 765     .powerpc_dcr_read = handle_powerpc_dcr_read,
 766     .powerpc_dcr_write = handle_powerpc_dcr_write,
 767 #endif
 768 };
 769
 770 int kvm_qemu_init()
 771 {
 772     /* Try to initialize kvm */
 773     kvm_context = kvm_init(&qemu_kvm_ops, cpu_single_env);
 774     if (!kvm_context) {
 775         return -1;
 776     }
 777     pthread_mutex_lock(&qemu_mutex);
 778
 779     return 0;
 780 }
 781
 782 int kvm_qemu_create_context(void)
 783 {
 784     int r;
 785     if (!kvm_irqchip) {
 786         kvm_disable_irqchip_creation(kvm_context);
 787     }
 788     if (!kvm_pit) {
 789         kvm_disable_pit_creation(kvm_context);
 790     }
 791     if (kvm_create(kvm_context, phys_ram_size, (void**)&phys_ram_base) < 0) {
 792         kvm_qemu_destroy();
 793         return -1;
 794     }
 795     r = kvm_arch_qemu_create_context();
 796     if(r <0)
 797         kvm_qemu_destroy();
 798     return 0;
 799 }
 800
 801 void kvm_qemu_destroy(void)
 802 {
 803     kvm_finalize(kvm_context);
 804 }
 805
 806 void kvm_cpu_register_physical_memory(target_phys_addr_t start_addr,
 807                                       unsigned long size,
 808                                       unsigned long phys_offset)
 809 {
 810     int r = 0;
 811     unsigned long area_flags = phys_offset & ~TARGET_PAGE_MASK;
 812
 813     phys_offset &= ~IO_MEM_ROM;
 814
 815     if (area_flags == IO_MEM_UNASSIGNED) {
 816         kvm_unregister_memory_area(kvm_context, start_addr, size);
 817         return;
 818     }
 819
 820     r = kvm_is_containing_region(kvm_context, start_addr, size);
 821     if (r)
 822         return;
 823
 824     if (area_flags >= TLB_MMIO)
 825         return;
 826
 827     r = kvm_register_phys_mem(kvm_context, start_addr,
 828                               phys_ram_base + phys_offset,
 829                               size, 0);
 830     if (r < 0) {
 831         printf("kvm_cpu_register_physical_memory: failed\n");
 832         exit(1);
 833     }
 834     return;
 835 }
 836
 837 void kvm_cpu_unregister_physical_memory(target_phys_addr_t start_addr,
 838                                         target_phys_addr_t size,
 839                                         unsigned long phys_offset)
 840 {
 841     kvm_unregister_memory_area(kvm_context, start_addr, size);
 842 }
 843
 844 int kvm_setup_guest_memory(void *area, unsigned long size)
 845 {
 846     int ret = 0;
 847
 848 #ifdef MADV_DONTFORK
 849     if (kvm_enabled() && !kvm_has_sync_mmu(kvm_context))
 850         ret = madvise(area, size, MADV_DONTFORK);
 851 #endif
 852
 853     if (ret)
 854         perror ("madvise");
 855
 856     return ret;
 857 }
 858
 859 int kvm_qemu_check_extension(int ext)
 860 {
 861     return kvm_check_extension(kvm_context, ext);
 862 }
 863
 864 int kvm_qemu_init_env(CPUState *cenv)
 865 {
 866     return kvm_arch_qemu_init_env(cenv);
 867 }
 868
 869 struct kvm_guest_debug_data {
 870     struct kvm_debug_guest dbg;
 871     int err;
 872 };
 873
 874 void kvm_invoke_guest_debug(void *data)
 875 {
 876     struct kvm_guest_debug_data *dbg_data = data;
 877
 878     dbg_data->err = kvm_guest_debug(kvm_context, cpu_single_env->cpu_index,
 879                                     &dbg_data->dbg);
 880 }
 881
 882 int kvm_update_debugger(CPUState *env)
 883 {
 884     struct kvm_guest_debug_data data;
 885     int i;
 886
 887     memset(data.dbg.breakpoints, 0, sizeof(data.dbg.breakpoints));
 888
 889     data.dbg.enabled = 0;
 890     if (env->nb_breakpoints || env->singlestep_enabled) {
 891         data.dbg.enabled = 1;
 892         for (i = 0; i < 4 && i < env->nb_breakpoints; ++i) {
 893             data.dbg.breakpoints[i].enabled = 1;
 894             data.dbg.breakpoints[i].address = env->breakpoints[i];
 895         }
 896         data.dbg.singlestep = env->singlestep_enabled;
 897     }
 898     on_vcpu(env, kvm_invoke_guest_debug, &data);
 899     return data.err;
 900 }
 901
 902
 903 /*
 904  * dirty pages logging
 905  */
 906 /* FIXME: use unsigned long pointer instead of unsigned char */
 907 unsigned char *kvm_dirty_bitmap = NULL;
 908 int kvm_physical_memory_set_dirty_tracking(int enable)
 909 {
 910     int r = 0;
 911
 912     if (!kvm_enabled())
 913         return 0;
 914
 915     if (enable) {
 916         if (!kvm_dirty_bitmap) {
 917             unsigned bitmap_size = BITMAP_SIZE(phys_ram_size);
 918             kvm_dirty_bitmap = qemu_malloc(bitmap_size);
 919             if (kvm_dirty_bitmap == NULL) {
 920                 perror("Failed to allocate dirty pages bitmap");
 921                 r=-1;
 922             }
 923             else {
 924                 r = kvm_dirty_pages_log_enable_all(kvm_context);
 925             }
 926         }
 927     }
 928     else {
 929         if (kvm_dirty_bitmap) {
 930             r = kvm_dirty_pages_log_reset(kvm_context);
 931             qemu_free(kvm_dirty_bitmap);
 932             kvm_dirty_bitmap = NULL;
 933         }
 934     }
 935     return r;
 936 }
 937
 938 /* get kvm's dirty pages bitmap and update qemu's */
 939 int kvm_get_dirty_pages_log_range(unsigned long start_addr,
 940                                   unsigned char *bitmap,
 941                                   unsigned int offset,
 942                                   unsigned long mem_size)
 943 {
 944     unsigned int i, j, n=0;
 945     unsigned char c;
 946     unsigned page_number, addr, addr1;
 947     unsigned int len = ((mem_size/TARGET_PAGE_SIZE) + 7) / 8;
 948
 949     /*
 950      * bitmap-traveling is faster than memory-traveling (for addr...)
 951      * especially when most of the memory is not dirty.
 952      */
 953     for (i=0; i<len; i++) {
 954         c = bitmap[i];
 955         while (c>0) {
 956             j = ffsl(c) - 1;
 957             c &= ~(1u<<j);
 958             page_number = i * 8 + j;
 959             addr1 = page_number * TARGET_PAGE_SIZE;
 960             addr  = offset + addr1;
 961             cpu_physical_memory_set_dirty(addr);
 962             n++;
 963         }
 964     }
 965     return 0;
 966 }
 967 int kvm_get_dirty_bitmap_cb(unsigned long start, unsigned long len,
 968                             void *bitmap, void *opaque)
 969 {
 970     return kvm_get_dirty_pages_log_range(start, bitmap, start, len);
 971 }
 972
 973 /*
 974  * get kvm's dirty pages bitmap and update qemu's
 975  * we only care about physical ram, which resides in slots 0 and 3
 976  */
 977 int kvm_update_dirty_pages_log(void)
 978 {
 979     int r = 0;
 980
 981
 982     r = kvm_get_dirty_pages_range(kvm_context, 0, phys_ram_size,
 983                                   kvm_dirty_bitmap, NULL,
 984                                   kvm_get_dirty_bitmap_cb);
 985     return r;
 986 }
 987
 988 void kvm_qemu_log_memory(target_phys_addr_t start, target_phys_addr_t size,
 989                          int log)
 990 {
 991     if (log)
 992         kvm_dirty_pages_log_enable_slot(kvm_context, start, size);
 993     else
 994         kvm_dirty_pages_log_disable_slot(kvm_context, start, size);
 995 }
 996
 997 int kvm_get_phys_ram_page_bitmap(unsigned char *bitmap)
 998 {
 999     unsigned int bsize  = BITMAP_SIZE(phys_ram_size);
1000     unsigned int brsize = BITMAP_SIZE(ram_size);
1001     unsigned int extra_pages = (phys_ram_size - ram_size) / TARGET_PAGE_SIZE;
1002     unsigned int extra_bytes = (extra_pages +7)/8;
1003     unsigned int hole_start = BITMAP_SIZE(0xa0000);
1004     unsigned int hole_end   = BITMAP_SIZE(0xc0000);
1005
1006     memset(bitmap, 0xFF, brsize + extra_bytes);
1007     memset(bitmap + hole_start, 0, hole_end - hole_start);
1008     memset(bitmap + brsize + extra_bytes, 0, bsize - brsize - extra_bytes);
1009
1010     return 0;
1011 }
1012
1013 #ifdef KVM_CAP_IRQCHIP
1014
1015 int kvm_set_irq(int irq, int level)
1016 {
1017     return kvm_set_irq_level(kvm_context, irq, level);
1018 }
1019
1020 #endif
1021
1022 int qemu_kvm_get_dirty_pages(unsigned long phys_addr, void *buf)
1023 {
1024     return kvm_get_dirty_pages(kvm_context, phys_addr, buf);
1025 }
1026
1027 void *kvm_cpu_create_phys_mem(target_phys_addr_t start_addr,
1028                               unsigned long size, int log, int writable)
1029 {
1030     return kvm_create_phys_mem(kvm_context, start_addr, size, log, writable);
1031 }
1032
1033 void kvm_cpu_destroy_phys_mem(target_phys_addr_t start_addr,
1034                               unsigned long size)
1035 {
1036     kvm_destroy_phys_mem(kvm_context, start_addr, size);
1037 }
1038
1039 void kvm_mutex_unlock(void)
1040 {
1041     assert(!cpu_single_env);
1042     pthread_mutex_unlock(&qemu_mutex);
1043 }
1044
1045 void kvm_mutex_lock(void)
1046 {
1047     pthread_mutex_lock(&qemu_mutex);
1048     cpu_single_env = NULL;
1049 }
1050
1051 int qemu_kvm_register_coalesced_mmio(target_phys_addr_t addr, unsigned int size)
1052 {
1053     return kvm_register_coalesced_mmio(kvm_context, addr, size);
1054 }
1055
1056 int qemu_kvm_unregister_coalesced_mmio(target_phys_addr_t addr,
1057                                        unsigned int size)
1058 {
1059     return kvm_unregister_coalesced_mmio(kvm_context, addr, size);
1060 }
1061
1062 #ifdef USE_KVM_DEVICE_ASSIGNMENT
1063 void kvm_add_ioperm_data(struct ioperm_data *data)
1064 {
1065     LIST_INSERT_HEAD(&ioperm_head, data, entries);
1066 }
1067
1068 void kvm_ioperm(CPUState *env, void *data)
1069 {
1070     if (kvm_enabled() && qemu_system_ready)
1071         on_vcpu(env, kvm_arch_do_ioperm, data);
1072 }
1073 #endif
1074