qemu-kvm.c

   1 /*
   2  * qemu/kvm integration
   3  *
   4  * Copyright (C) 2006-2008 Qumranet Technologies
   5  *
   6  * Licensed under the terms of the GNU GPL version 2 or higher.
   7  */
   8 #include "config.h"
   9 #include "config-host.h"
  10
  11 int kvm_allowed = 1;
  12 int kvm_irqchip = 1;
  13 int kvm_pit = 1;
  14
  15 #include <assert.h>
  16 #include <string.h>
  17 #include "hw/hw.h"
  18 #include "sysemu.h"
  19 #include "qemu-common.h"
  20 #include "console.h"
  21 #include "block.h"
  22
  23 #include "qemu-kvm.h"
  24 #include <libkvm.h>
  25 #include <pthread.h>
  26 #include <sys/utsname.h>
  27 #include <sys/syscall.h>
  28
  29 #define bool _Bool
  30 #define false 0
  31 #define true 1
  32
  33 extern void perror(const char *s);
  34
  35 kvm_context_t kvm_context;
  36
  37 extern int smp_cpus;
  38
  39 pthread_mutex_t qemu_mutex = PTHREAD_MUTEX_INITIALIZER;
  40 pthread_cond_t qemu_aio_cond = PTHREAD_COND_INITIALIZER;
  41 pthread_cond_t qemu_vcpu_cond = PTHREAD_COND_INITIALIZER;
  42 pthread_cond_t qemu_system_cond = PTHREAD_COND_INITIALIZER;
  43 pthread_cond_t qemu_pause_cond = PTHREAD_COND_INITIALIZER;
  44 pthread_cond_t qemu_work_cond = PTHREAD_COND_INITIALIZER;
  45 __thread struct vcpu_info *vcpu;
  46
  47 static int qemu_system_ready;
  48
  49 #define SIG_IPI (SIGRTMIN+4)
  50
  51 struct qemu_kvm_work_item {
  52     struct qemu_kvm_work_item *next;
  53     void (*func)(void *data);
  54     void *data;
  55     bool done;
  56 };
  57
  58 struct vcpu_info {
  59     CPUState *env;
  60     int sipi_needed;
  61     int init;
  62     pthread_t thread;
  63     int signalled;
  64     int stop;
  65     int stopped;
  66     int created;
  67     struct qemu_kvm_work_item *queued_work_first, *queued_work_last;
  68 } vcpu_info[256];
  69
  70 pthread_t io_thread;
  71 static int io_thread_fd = -1;
  72 static int io_thread_sigfd = -1;
  73
  74 static inline unsigned long kvm_get_thread_id(void)
  75 {
  76     return syscall(SYS_gettid);
  77 }
  78
  79 static void qemu_cond_wait(pthread_cond_t *cond)
  80 {
  81     CPUState *env = cpu_single_env;
  82
  83     pthread_cond_wait(cond, &qemu_mutex);
  84     cpu_single_env = env;
  85 }
  86
  87 CPUState *qemu_kvm_cpu_env(int index)
  88 {
  89     return vcpu_info[index].env;
  90 }
  91
  92 static void sig_ipi_handler(int n)
  93 {
  94 }
  95
  96 static void on_vcpu(CPUState *env, void (*func)(void *data), void *data)
  97 {
  98     struct vcpu_info *vi = &vcpu_info[env->cpu_index];
  99     struct qemu_kvm_work_item wi;
 100
 101     if (vi == vcpu) {
 102         func(data);
 103         return;
 104     }
 105
 106     wi.func = func;
 107     wi.data = data;
 108     if (!vi->queued_work_first)
 109         vi->queued_work_first = &wi;
 110     else
 111         vi->queued_work_last->next = &wi;
 112     vi->queued_work_last = &wi;
 113     wi.next = NULL;
 114     wi.done = false;
 115
 116     pthread_kill(vi->thread, SIG_IPI);
 117     while (!wi.done)
 118         qemu_cond_wait(&qemu_work_cond);
 119 }
 120
 121 void kvm_update_interrupt_request(CPUState *env)
 122 {
 123     int signal = 0;
 124
 125     if (env) {
 126         if (!vcpu)
 127             signal = 1;
 128         if (vcpu && env != vcpu->env && !vcpu_info[env->cpu_index].signalled)
 129             signal = 1;
 130
 131         if (signal) {
 132             vcpu_info[env->cpu_index].signalled = 1;
 133                 if (vcpu_info[env->cpu_index].thread)
 134                     pthread_kill(vcpu_info[env->cpu_index].thread, SIG_IPI);
 135         }
 136     }
 137 }
 138
 139 void kvm_update_after_sipi(CPUState *env)
 140 {
 141     vcpu_info[env->cpu_index].sipi_needed = 1;
 142     kvm_update_interrupt_request(env);
 143 }
 144
 145 void kvm_apic_init(CPUState *env)
 146 {
 147     if (env->cpu_index != 0)
 148         vcpu_info[env->cpu_index].init = 1;
 149     kvm_update_interrupt_request(env);
 150 }
 151
 152 #include <signal.h>
 153
 154 static int try_push_interrupts(void *opaque)
 155 {
 156     return kvm_arch_try_push_interrupts(opaque);
 157 }
 158
 159 static void post_kvm_run(void *opaque, int vcpu)
 160 {
 161
 162     pthread_mutex_lock(&qemu_mutex);
 163     kvm_arch_post_kvm_run(opaque, vcpu);
 164 }
 165
 166 static int pre_kvm_run(void *opaque, int vcpu)
 167 {
 168     CPUState *env = qemu_kvm_cpu_env(vcpu);
 169
 170     kvm_arch_pre_kvm_run(opaque, vcpu);
 171
 172     if (env->interrupt_request & CPU_INTERRUPT_EXIT)
 173         return 1;
 174     pthread_mutex_unlock(&qemu_mutex);
 175     return 0;
 176 }
 177
 178 static void kvm_do_load_registers(void *_env)
 179 {
 180     CPUState *env = _env;
 181
 182     kvm_arch_load_regs(env);
 183 }
 184
 185 void kvm_load_registers(CPUState *env)
 186 {
 187     if (kvm_enabled())
 188         on_vcpu(env->cpu_index, kvm_do_load_registers, env);
 189 }
 190
 191 static void kvm_do_save_registers(void *_env)
 192 {
 193     CPUState *env = _env;
 194
 195     kvm_arch_save_regs(env);
 196 }
 197
 198 void kvm_save_registers(CPUState *env)
 199 {
 200     if (kvm_enabled())
 201         on_vcpu(env, kvm_do_save_registers, env);
 202 }
 203
 204 int kvm_cpu_exec(CPUState *env)
 205 {
 206     int r;
 207
 208     r = kvm_run(kvm_context, env->cpu_index);
 209     if (r < 0) {
 210         printf("kvm_run returned %d\n", r);
 211         exit(1);
 212     }
 213
 214     return 0;
 215 }
 216
 217 extern int vm_running;
 218
 219 static int has_work(CPUState *env)
 220 {
 221     if (!vm_running || (env && vcpu_info[env->cpu_index].stopped))
 222         return 0;
 223     if (!(env->hflags & HF_HALTED_MASK))
 224         return 1;
 225     return kvm_arch_has_work(env);
 226 }
 227
 228 static void flush_queued_work(CPUState *env)
 229 {
 230     struct vcpu_info *vi = &vcpu_info[env->cpu_index];
 231     struct qemu_kvm_work_item *wi;
 232
 233     if (!vi->queued_work_first)
 234         return;
 235
 236     while ((wi = vi->queued_work_first)) {
 237         vi->queued_work_first = wi->next;
 238         wi->func(wi->data);
 239         wi->done = true;
 240     }
 241     vi->queued_work_last = NULL;
 242     pthread_cond_broadcast(&qemu_work_cond);
 243 }
 244
 245 static void kvm_main_loop_wait(CPUState *env, int timeout)
 246 {
 247     struct timespec ts;
 248     int r, e;
 249     siginfo_t siginfo;
 250     sigset_t waitset;
 251
 252     pthread_mutex_unlock(&qemu_mutex);
 253
 254     ts.tv_sec = timeout / 1000;
 255     ts.tv_nsec = (timeout % 1000) * 1000000;
 256     sigemptyset(&waitset);
 257     sigaddset(&waitset, SIG_IPI);
 258
 259     r = sigtimedwait(&waitset, &siginfo, &ts);
 260     e = errno;
 261
 262     pthread_mutex_lock(&qemu_mutex);
 263
 264     if (r == -1 && !(e == EAGAIN || e == EINTR)) {
 265         printf("sigtimedwait: %s\n", strerror(e));
 266         exit(1);
 267     }
 268
 269
 270     flush_queued_work(env);
 271
 272     if (vcpu_info[env->cpu_index].stop) {
 273         vcpu_info[env->cpu_index].stop = 0;
 274         vcpu_info[env->cpu_index].stopped = 1;
 275         pthread_cond_signal(&qemu_pause_cond);
 276     }
 277     cpu_single_env = env;
 278
 279     vcpu_info[env->cpu_index].signalled = 0;
 280 }
 281
 282 static int all_threads_paused(void)
 283 {
 284     int i;
 285
 286     for (i = 0; i < smp_cpus; ++i)
 287         if (vcpu_info[i].stop)
 288             return 0;
 289     return 1;
 290 }
 291
 292 static void pause_all_threads(void)
 293 {
 294     int i;
 295
 296     assert(!cpu_single_env);
 297
 298     for (i = 0; i < smp_cpus; ++i) {
 299         vcpu_info[i].stop = 1;
 300         pthread_kill(vcpu_info[i].thread, SIG_IPI);
 301     }
 302     while (!all_threads_paused())
 303         qemu_cond_wait(&qemu_pause_cond);
 304 }
 305
 306 static void resume_all_threads(void)
 307 {
 308     int i;
 309
 310     assert(!cpu_single_env);
 311
 312     for (i = 0; i < smp_cpus; ++i) {
 313         vcpu_info[i].stop = 0;
 314         vcpu_info[i].stopped = 0;
 315         pthread_kill(vcpu_info[i].thread, SIG_IPI);
 316     }
 317 }
 318
 319 static void kvm_vm_state_change_handler(void *context, int running)
 320 {
 321     if (running)
 322         resume_all_threads();
 323     else
 324         pause_all_threads();
 325 }
 326
 327 static void update_regs_for_sipi(CPUState *env)
 328 {
 329     kvm_arch_update_regs_for_sipi(env);
 330     vcpu_info[env->cpu_index].sipi_needed = 0;
 331     vcpu_info[env->cpu_index].init = 0;
 332 }
 333
 334 static void update_regs_for_init(CPUState *env)
 335 {
 336     cpu_reset(env);
 337     kvm_arch_load_regs(env);
 338 }
 339
 340 static void setup_kernel_sigmask(CPUState *env)
 341 {
 342     sigset_t set;
 343
 344     sigemptyset(&set);
 345     sigaddset(&set, SIGUSR2);
 346     sigaddset(&set, SIGIO);
 347     sigaddset(&set, SIGALRM);
 348     sigprocmask(SIG_BLOCK, &set, NULL);
 349
 350     sigprocmask(SIG_BLOCK, NULL, &set);
 351     sigdelset(&set, SIG_IPI);
 352
 353     kvm_set_signal_mask(kvm_context, env->cpu_index, &set);
 354 }
 355
 356 void qemu_kvm_system_reset(void)
 357 {
 358     int i;
 359
 360     pause_all_threads();
 361
 362     qemu_system_reset();
 363
 364     for (i = 0; i < smp_cpus; ++i)
 365         kvm_arch_cpu_reset(vcpu_info[i].env);
 366
 367     resume_all_threads();
 368 }
 369
 370 static int kvm_main_loop_cpu(CPUState *env)
 371 {
 372     struct vcpu_info *info = &vcpu_info[env->cpu_index];
 373
 374     setup_kernel_sigmask(env);
 375
 376     pthread_mutex_lock(&qemu_mutex);
 377     if (kvm_irqchip_in_kernel(kvm_context))
 378         env->hflags &= ~HF_HALTED_MASK;
 379
 380     kvm_qemu_init_env(env);
 381     env->ready_for_interrupt_injection = 1;
 382 #ifdef TARGET_I386
 383     kvm_tpr_vcpu_start(env);
 384 #endif
 385
 386     cpu_single_env = env;
 387     while (1) {
 388         while (!has_work(env))
 389             kvm_main_loop_wait(env, 1000);
 390         if (env->interrupt_request & CPU_INTERRUPT_HARD)
 391             env->hflags &= ~HF_HALTED_MASK;
 392         if (!kvm_irqchip_in_kernel(kvm_context) && info->sipi_needed)
 393             update_regs_for_sipi(env);
 394         if (!kvm_irqchip_in_kernel(kvm_context) && info->init)
 395             update_regs_for_init(env);
 396         if (!(env->hflags & HF_HALTED_MASK) && !info->init)
 397             kvm_cpu_exec(env);
 398         env->interrupt_request &= ~CPU_INTERRUPT_EXIT;
 399         kvm_main_loop_wait(env, 0);
 400     }
 401     pthread_mutex_unlock(&qemu_mutex);
 402     return 0;
 403 }
 404
 405 static void *ap_main_loop(void *_env)
 406 {
 407     CPUState *env = _env;
 408     sigset_t signals;
 409
 410     vcpu = &vcpu_info[env->cpu_index];
 411     vcpu->env = env;
 412     vcpu->env->thread_id = kvm_get_thread_id();
 413     sigfillset(&signals);
 414     sigprocmask(SIG_BLOCK, &signals, NULL);
 415     kvm_create_vcpu(kvm_context, env->cpu_index);
 416     kvm_qemu_init_env(env);
 417
 418     /* signal VCPU creation */
 419     pthread_mutex_lock(&qemu_mutex);
 420     vcpu->created = 1;
 421     pthread_cond_signal(&qemu_vcpu_cond);
 422
 423     /* and wait for machine initialization */
 424     while (!qemu_system_ready)
 425         qemu_cond_wait(&qemu_system_cond);
 426     pthread_mutex_unlock(&qemu_mutex);
 427
 428     kvm_main_loop_cpu(env);
 429     return NULL;
 430 }
 431
 432 void kvm_init_new_ap(int cpu, CPUState *env)
 433 {
 434     pthread_create(&vcpu_info[cpu].thread, NULL, ap_main_loop, env);
 435
 436     while (vcpu_info[cpu].created == 0)
 437         qemu_cond_wait(&qemu_vcpu_cond);
 438 }
 439
 440 int kvm_init_ap(void)
 441 {
 442 #ifdef TARGET_I386
 443     kvm_tpr_opt_setup();
 444 #endif
 445     qemu_add_vm_change_state_handler(kvm_vm_state_change_handler, NULL);
 446
 447     signal(SIG_IPI, sig_ipi_handler);
 448     return 0;
 449 }
 450
 451 void qemu_kvm_notify_work(void)
 452 {
 453     uint64_t value = 1;
 454     char buffer[8];
 455     size_t offset = 0;
 456
 457     if (io_thread_fd == -1)
 458         return;
 459
 460     memcpy(buffer, &value, sizeof(value));
 461
 462     while (offset < 8) {
 463         ssize_t len;
 464
 465         len = write(io_thread_fd, buffer + offset, 8 - offset);
 466         if (len == -1 && errno == EINTR)
 467             continue;
 468
 469         if (len <= 0)
 470             break;
 471
 472         offset += len;
 473     }
 474
 475     if (offset != 8)
 476         fprintf(stderr, "failed to notify io thread\n");
 477 }
 478
 479 /* If we have signalfd, we mask out the signals we want to handle and then
 480  * use signalfd to listen for them.  We rely on whatever the current signal
 481  * handler is to dispatch the signals when we receive them.
 482  */
 483
 484 static void sigfd_handler(void *opaque)
 485 {
 486     int fd = (unsigned long)opaque;
 487     struct signalfd_siginfo info;
 488     struct sigaction action;
 489     ssize_t len;
 490
 491     while (1) {
 492         do {
 493             len = read(fd, &info, sizeof(info));
 494         } while (len == -1 && errno == EINTR);
 495
 496         if (len == -1 && errno == EAGAIN)
 497             break;
 498
 499         if (len != sizeof(info)) {
 500             printf("read from sigfd returned %ld: %m\n", len);
 501             return;
 502         }
 503
 504         sigaction(info.ssi_signo, NULL, &action);
 505         if (action.sa_handler)
 506             action.sa_handler(info.ssi_signo);
 507
 508         if (info.ssi_signo == SIGUSR2) {
 509             pthread_cond_signal(&qemu_aio_cond);
 510         }
 511     }
 512 }
 513
 514 /* Used to break IO thread out of select */
 515 static void io_thread_wakeup(void *opaque)
 516 {
 517     int fd = (unsigned long)opaque;
 518     char buffer[8];
 519     size_t offset = 0;
 520
 521     while (offset < 8) {
 522         ssize_t len;
 523
 524         len = read(fd, buffer + offset, 8 - offset);
 525         if (len == -1 && errno == EINTR)
 526             continue;
 527
 528         if (len <= 0)
 529             break;
 530
 531         offset += len;
 532     }
 533 }
 534
 535 int kvm_main_loop(void)
 536 {
 537     int fds[2];
 538     sigset_t mask;
 539     int sigfd;
 540
 541     io_thread = pthread_self();
 542     qemu_system_ready = 1;
 543
 544     if (kvm_eventfd(fds) == -1) {
 545         fprintf(stderr, "failed to create eventfd\n");
 546         return -errno;
 547     }
 548
 549     qemu_set_fd_handler2(fds[0], NULL, io_thread_wakeup, NULL,
 550                          (void *)(unsigned long)fds[0]);
 551
 552     io_thread_fd = fds[1];
 553
 554     sigemptyset(&mask);
 555     sigaddset(&mask, SIGIO);
 556     sigaddset(&mask, SIGALRM);
 557     sigaddset(&mask, SIGUSR2);
 558     sigprocmask(SIG_BLOCK, &mask, NULL);
 559
 560     sigfd = kvm_signalfd(&mask);
 561     if (sigfd == -1) {
 562         fprintf(stderr, "failed to create signalfd\n");
 563         return -errno;
 564     }
 565
 566     fcntl(sigfd, F_SETFL, O_NONBLOCK);
 567
 568     qemu_set_fd_handler2(sigfd, NULL, sigfd_handler, NULL,
 569                          (void *)(unsigned long)sigfd);
 570
 571     pthread_cond_broadcast(&qemu_system_cond);
 572
 573     io_thread_sigfd = sigfd;
 574     cpu_single_env = NULL;
 575
 576     while (1) {
 577         main_loop_wait(1000);
 578         if (qemu_shutdown_requested())
 579             break;
 580         else if (qemu_powerdown_requested())
 581             qemu_system_powerdown();
 582         else if (qemu_reset_requested())
 583             qemu_kvm_system_reset();
 584     }
 585
 586     pause_all_threads();
 587     pthread_mutex_unlock(&qemu_mutex);
 588
 589     return 0;
 590 }
 591
 592 static int kvm_debug(void *opaque, int vcpu)
 593 {
 594     CPUState *env = cpu_single_env;
 595
 596     env->exception_index = EXCP_DEBUG;
 597     return 1;
 598 }
 599
 600 static int kvm_inb(void *opaque, uint16_t addr, uint8_t *data)
 601 {
 602     *data = cpu_inb(0, addr);
 603     return 0;
 604 }
 605
 606 static int kvm_inw(void *opaque, uint16_t addr, uint16_t *data)
 607 {
 608     *data = cpu_inw(0, addr);
 609     return 0;
 610 }
 611
 612 static int kvm_inl(void *opaque, uint16_t addr, uint32_t *data)
 613 {
 614     *data = cpu_inl(0, addr);
 615     return 0;
 616 }
 617
 618 #define PM_IO_BASE 0xb000
 619
 620 static int kvm_outb(void *opaque, uint16_t addr, uint8_t data)
 621 {
 622     if (addr == 0xb2) {
 623         switch (data) {
 624         case 0: {
 625             cpu_outb(0, 0xb3, 0);
 626             break;
 627         }
 628         case 0xf0: {
 629             unsigned x;
 630
 631             /* enable acpi */
 632             x = cpu_inw(0, PM_IO_BASE + 4);
 633             x &= ~1;
 634             cpu_outw(0, PM_IO_BASE + 4, x);
 635             break;
 636         }
 637         case 0xf1: {
 638             unsigned x;
 639
 640             /* enable acpi */
 641             x = cpu_inw(0, PM_IO_BASE + 4);
 642             x |= 1;
 643             cpu_outw(0, PM_IO_BASE + 4, x);
 644             break;
 645         }
 646         default:
 647             break;
 648         }
 649         return 0;
 650     }
 651     cpu_outb(0, addr, data);
 652     return 0;
 653 }
 654
 655 static int kvm_outw(void *opaque, uint16_t addr, uint16_t data)
 656 {
 657     cpu_outw(0, addr, data);
 658     return 0;
 659 }
 660
 661 static int kvm_outl(void *opaque, uint16_t addr, uint32_t data)
 662 {
 663     cpu_outl(0, addr, data);
 664     return 0;
 665 }
 666
 667 static int kvm_mmio_read(void *opaque, uint64_t addr, uint8_t *data, int len)
 668 {
 669         cpu_physical_memory_rw(addr, data, len, 0);
 670         return 0;
 671 }
 672
 673 static int kvm_mmio_write(void *opaque, uint64_t addr, uint8_t *data, int len)
 674 {
 675         cpu_physical_memory_rw(addr, data, len, 1);
 676         return 0;
 677 }
 678
 679 static int kvm_io_window(void *opaque)
 680 {
 681     return 1;
 682 }
 683
 684
 685 static int kvm_halt(void *opaque, int vcpu)
 686 {
 687     return kvm_arch_halt(opaque, vcpu);
 688 }
 689
 690 static int kvm_shutdown(void *opaque, int vcpu)
 691 {
 692     /* stop the current vcpu from going back to guest mode */
 693     vcpu_info[cpu_single_env->cpu_index].stopped = 1;
 694
 695     qemu_system_reset_request();
 696     return 1;
 697 }
 698
 699 static struct kvm_callbacks qemu_kvm_ops = {
 700     .debug = kvm_debug,
 701     .inb   = kvm_inb,
 702     .inw   = kvm_inw,
 703     .inl   = kvm_inl,
 704     .outb  = kvm_outb,
 705     .outw  = kvm_outw,
 706     .outl  = kvm_outl,
 707     .mmio_read = kvm_mmio_read,
 708     .mmio_write = kvm_mmio_write,
 709     .halt  = kvm_halt,
 710     .shutdown = kvm_shutdown,
 711     .io_window = kvm_io_window,
 712     .try_push_interrupts = try_push_interrupts,
 713     .post_kvm_run = post_kvm_run,
 714     .pre_kvm_run = pre_kvm_run,
 715 #ifdef TARGET_I386
 716     .tpr_access = handle_tpr_access,
 717 #endif
 718 #ifdef TARGET_PPC
 719     .powerpc_dcr_read = handle_powerpc_dcr_read,
 720     .powerpc_dcr_write = handle_powerpc_dcr_write,
 721 #endif
 722 };
 723
 724 int kvm_qemu_init()
 725 {
 726     /* Try to initialize kvm */
 727     kvm_context = kvm_init(&qemu_kvm_ops, cpu_single_env);
 728     if (!kvm_context) {
 729         return -1;
 730     }
 731     pthread_mutex_lock(&qemu_mutex);
 732
 733     return 0;
 734 }
 735
 736 int kvm_qemu_create_context(void)
 737 {
 738     int r;
 739     if (!kvm_irqchip) {
 740         kvm_disable_irqchip_creation(kvm_context);
 741     }
 742     if (!kvm_pit) {
 743         kvm_disable_pit_creation(kvm_context);
 744     }
 745     if (kvm_create(kvm_context, phys_ram_size, (void**)&phys_ram_base) < 0) {
 746         kvm_qemu_destroy();
 747         return -1;
 748     }
 749     r = kvm_arch_qemu_create_context();
 750     if(r <0)
 751         kvm_qemu_destroy();
 752     return 0;
 753 }
 754
 755 void kvm_qemu_destroy(void)
 756 {
 757     kvm_finalize(kvm_context);
 758 }
 759
 760 void kvm_cpu_register_physical_memory(target_phys_addr_t start_addr,
 761                                       unsigned long size,
 762                                       unsigned long phys_offset)
 763 {
 764 #ifdef KVM_CAP_USER_MEMORY
 765     int r = 0;
 766
 767     r = kvm_check_extension(kvm_context, KVM_CAP_USER_MEMORY);
 768     if (r) {
 769         if (!(phys_offset & ~TARGET_PAGE_MASK)) {
 770                 r = kvm_is_allocated_mem(kvm_context, start_addr, size);
 771             if (r)
 772                 return;
 773             r = kvm_is_intersecting_mem(kvm_context, start_addr);
 774             if (r)
 775                 kvm_create_mem_hole(kvm_context, start_addr, size);
 776             r = kvm_register_userspace_phys_mem(kvm_context, start_addr,
 777                                                 phys_ram_base + phys_offset,
 778                                                 size, 0);
 779         }
 780         if (phys_offset & IO_MEM_ROM) {
 781             phys_offset &= ~IO_MEM_ROM;
 782             r = kvm_is_intersecting_mem(kvm_context, start_addr);
 783             if (r)
 784                 kvm_create_mem_hole(kvm_context, start_addr, size);
 785             r = kvm_register_userspace_phys_mem(kvm_context, start_addr,
 786                                                 phys_ram_base + phys_offset,
 787                                                 size, 0);
 788         }
 789         if (r < 0) {
 790             printf("kvm_cpu_register_physical_memory: failed\n");
 791             exit(1);
 792         }
 793         return;
 794     }
 795 #endif
 796     if (phys_offset & IO_MEM_ROM) {
 797         phys_offset &= ~IO_MEM_ROM;
 798         memcpy(phys_ram_base + start_addr, phys_ram_base + phys_offset, size);
 799     }
 800 }
 801
 802 int kvm_qemu_check_extension(int ext)
 803 {
 804     return kvm_check_extension(kvm_context, ext);
 805 }
 806
 807 int kvm_qemu_init_env(CPUState *cenv)
 808 {
 809     return kvm_arch_qemu_init_env(cenv);
 810 }
 811
 812 int kvm_update_debugger(CPUState *env)
 813 {
 814     struct kvm_debug_guest dbg;
 815     int i;
 816
 817     memset(dbg.breakpoints, 0, sizeof(dbg.breakpoints));
 818
 819     dbg.enabled = 0;
 820     if (env->nb_breakpoints || env->singlestep_enabled) {
 821         dbg.enabled = 1;
 822         for (i = 0; i < 4 && i < env->nb_breakpoints; ++i) {
 823             dbg.breakpoints[i].enabled = 1;
 824             dbg.breakpoints[i].address = env->breakpoints[i];
 825         }
 826         dbg.singlestep = env->singlestep_enabled;
 827     }
 828     return kvm_guest_debug(kvm_context, env->cpu_index, &dbg);
 829 }
 830
 831
 832 /*
 833  * dirty pages logging
 834  */
 835 /* FIXME: use unsigned long pointer instead of unsigned char */
 836 unsigned char *kvm_dirty_bitmap = NULL;
 837 int kvm_physical_memory_set_dirty_tracking(int enable)
 838 {
 839     int r = 0;
 840
 841     if (!kvm_enabled())
 842         return 0;
 843
 844     if (enable) {
 845         if (!kvm_dirty_bitmap) {
 846             unsigned bitmap_size = BITMAP_SIZE(phys_ram_size);
 847             kvm_dirty_bitmap = qemu_malloc(bitmap_size);
 848             if (kvm_dirty_bitmap == NULL) {
 849                 perror("Failed to allocate dirty pages bitmap");
 850                 r=-1;
 851             }
 852             else {
 853                 r = kvm_dirty_pages_log_enable_all(kvm_context);
 854             }
 855         }
 856     }
 857     else {
 858         if (kvm_dirty_bitmap) {
 859             r = kvm_dirty_pages_log_reset(kvm_context);
 860             qemu_free(kvm_dirty_bitmap);
 861             kvm_dirty_bitmap = NULL;
 862         }
 863     }
 864     return r;
 865 }
 866
 867 /* get kvm's dirty pages bitmap and update qemu's */
 868 int kvm_get_dirty_pages_log_range(unsigned long start_addr,
 869                                   unsigned char *bitmap,
 870                                   unsigned int offset,
 871                                   unsigned long mem_size)
 872 {
 873     unsigned int i, j, n=0;
 874     unsigned char c;
 875     unsigned page_number, addr, addr1;
 876     unsigned int len = ((mem_size/TARGET_PAGE_SIZE) + 7) / 8;
 877
 878     /*
 879      * bitmap-traveling is faster than memory-traveling (for addr...)
 880      * especially when most of the memory is not dirty.
 881      */
 882     for (i=0; i<len; i++) {
 883         c = bitmap[i];
 884         while (c>0) {
 885             j = ffsl(c) - 1;
 886             c &= ~(1u<<j);
 887             page_number = i * 8 + j;
 888             addr1 = page_number * TARGET_PAGE_SIZE;
 889             addr  = offset + addr1;
 890             cpu_physical_memory_set_dirty(addr);
 891             n++;
 892         }
 893     }
 894     return 0;
 895 }
 896 int kvm_get_dirty_bitmap_cb(unsigned long start, unsigned long len,
 897                             void *bitmap, void *opaque)
 898 {
 899     return kvm_get_dirty_pages_log_range(start, bitmap, start, len);
 900 }
 901
 902 /*
 903  * get kvm's dirty pages bitmap and update qemu's
 904  * we only care about physical ram, which resides in slots 0 and 3
 905  */
 906 int kvm_update_dirty_pages_log(void)
 907 {
 908     int r = 0;
 909
 910
 911     r = kvm_get_dirty_pages_range(kvm_context, 0, phys_ram_size,
 912                                   kvm_dirty_bitmap, NULL,
 913                                   kvm_get_dirty_bitmap_cb);
 914     return r;
 915 }
 916
 917 int kvm_get_phys_ram_page_bitmap(unsigned char *bitmap)
 918 {
 919     unsigned int bsize  = BITMAP_SIZE(phys_ram_size);
 920     unsigned int brsize = BITMAP_SIZE(ram_size);
 921     unsigned int extra_pages = (phys_ram_size - ram_size) / TARGET_PAGE_SIZE;
 922     unsigned int extra_bytes = (extra_pages +7)/8;
 923     unsigned int hole_start = BITMAP_SIZE(0xa0000);
 924     unsigned int hole_end   = BITMAP_SIZE(0xc0000);
 925
 926     memset(bitmap, 0xFF, brsize + extra_bytes);
 927     memset(bitmap + hole_start, 0, hole_end - hole_start);
 928     memset(bitmap + brsize + extra_bytes, 0, bsize - brsize - extra_bytes);
 929
 930     return 0;
 931 }
 932
 933 #ifdef KVM_CAP_IRQCHIP
 934
 935 int kvm_set_irq(int irq, int level)
 936 {
 937     return kvm_set_irq_level(kvm_context, irq, level);
 938 }
 939
 940 #endif
 941
 942 void qemu_kvm_aio_wait_start(void)
 943 {
 944 }
 945
 946 void qemu_kvm_aio_wait(void)
 947 {
 948     if (!cpu_single_env) {
 949         if (io_thread_sigfd != -1) {
 950             fd_set rfds;
 951             int ret;
 952
 953             FD_ZERO(&rfds);
 954             FD_SET(io_thread_sigfd, &rfds);
 955
 956             /* this is a rare case where we do want to hold qemu_mutex
 957              * while sleeping.  We cannot allow anything else to run
 958              * right now. */
 959             ret = select(io_thread_sigfd + 1, &rfds, NULL, NULL, NULL);
 960             if (ret > 0 && FD_ISSET(io_thread_sigfd, &rfds))
 961                 sigfd_handler((void *)(unsigned long)io_thread_sigfd);
 962         }
 963         qemu_aio_poll();
 964     } else
 965         qemu_cond_wait(&qemu_aio_cond);
 966 }
 967
 968 void qemu_kvm_aio_wait_end(void)
 969 {
 970 }
 971
 972 int qemu_kvm_get_dirty_pages(unsigned long phys_addr, void *buf)
 973 {
 974     return kvm_get_dirty_pages(kvm_context, phys_addr, buf);
 975 }
 976
 977 void *kvm_cpu_create_phys_mem(target_phys_addr_t start_addr,
 978                               unsigned long size, int log, int writable)
 979 {
 980     return kvm_create_phys_mem(kvm_context, start_addr, size, log, writable);
 981 }
 982
 983 void kvm_cpu_destroy_phys_mem(target_phys_addr_t start_addr,
 984                               unsigned long size)
 985 {
 986     kvm_destroy_phys_mem(kvm_context, start_addr, size);
 987 }
 988
 989 void kvm_mutex_unlock(void)
 990 {
 991     assert(!cpu_single_env);
 992     pthread_mutex_unlock(&qemu_mutex);
 993 }
 994
 995 void kvm_mutex_lock(void)
 996 {
 997     pthread_mutex_lock(&qemu_mutex);
 998     cpu_single_env = NULL;
 999 }