kvm-all.c

   1 /*
   2  * QEMU KVM support
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *           Red Hat, Inc. 2008
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *  Glauber Costa     <gcosta@redhat.com>
  10  *
  11  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  12  * See the COPYING file in the top-level directory.
  13  *
  14  */
  15
  16 #include <sys/types.h>
  17 #include <sys/ioctl.h>
  18 #include <sys/mman.h>
  19 #include <stdarg.h>
  20
  21 #include <linux/kvm.h>
  22
  23 #include "qemu-common.h"
  24 #include "sysemu.h"
  25 #include "hw/hw.h"
  26 #include "gdbstub.h"
  27 #include "kvm.h"
  28
  29 /* KVM uses PAGE_SIZE in it's definition of COALESCED_MMIO_MAX */
  30 #define PAGE_SIZE TARGET_PAGE_SIZE
  31
  32 //#define DEBUG_KVM
  33
  34 #ifdef DEBUG_KVM
  35 #define dprintf(fmt, ...) \
  36     do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
  37 #else
  38 #define dprintf(fmt, ...) \
  39     do { } while (0)
  40 #endif
  41
  42 typedef struct KVMSlot
  43 {
  44     target_phys_addr_t start_addr;
  45     ram_addr_t memory_size;
  46     ram_addr_t phys_offset;
  47     int slot;
  48     int flags;
  49 } KVMSlot;
  50
  51 typedef struct kvm_dirty_log KVMDirtyLog;
  52
  53 int kvm_allowed = 0;
  54
  55 struct KVMState
  56 {
  57     KVMSlot slots[32];
  58     int fd;
  59     int vmfd;
  60     int regs_modified;
  61     int coalesced_mmio;
  62 #ifdef KVM_CAP_COALESCED_MMIO
  63     struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
  64 #endif
  65     int broken_set_mem_region;
  66     int migration_log;
  67     int vcpu_events;
  68 #ifdef KVM_CAP_SET_GUEST_DEBUG
  69     struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
  70 #endif
  71     int irqchip_in_kernel;
  72     int pit_in_kernel;
  73 };
  74
  75 static KVMState *kvm_state;
  76
  77 static KVMSlot *kvm_alloc_slot(KVMState *s)
  78 {
  79     int i;
  80
  81     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
  82         /* KVM private memory slots */
  83         if (i >= 8 && i < 12)
  84             continue;
  85         if (s->slots[i].memory_size == 0)
  86             return &s->slots[i];
  87     }
  88
  89     fprintf(stderr, "%s: no free slot available\n", __func__);
  90     abort();
  91 }
  92
  93 static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
  94                                          target_phys_addr_t start_addr,
  95                                          target_phys_addr_t end_addr)
  96 {
  97     int i;
  98
  99     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 100         KVMSlot *mem = &s->slots[i];
 101
 102         if (start_addr == mem->start_addr &&
 103             end_addr == mem->start_addr + mem->memory_size) {
 104             return mem;
 105         }
 106     }
 107
 108     return NULL;
 109 }
 110
 111 /*
 112  * Find overlapping slot with lowest start address
 113  */
 114 static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
 115                                             target_phys_addr_t start_addr,
 116                                             target_phys_addr_t end_addr)
 117 {
 118     KVMSlot *found = NULL;
 119     int i;
 120
 121     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 122         KVMSlot *mem = &s->slots[i];
 123
 124         if (mem->memory_size == 0 ||
 125             (found && found->start_addr < mem->start_addr)) {
 126             continue;
 127         }
 128
 129         if (end_addr > mem->start_addr &&
 130             start_addr < mem->start_addr + mem->memory_size) {
 131             found = mem;
 132         }
 133     }
 134
 135     return found;
 136 }
 137
 138 static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
 139 {
 140     struct kvm_userspace_memory_region mem;
 141
 142     mem.slot = slot->slot;
 143     mem.guest_phys_addr = slot->start_addr;
 144     mem.memory_size = slot->memory_size;
 145     mem.userspace_addr = (unsigned long)qemu_get_ram_ptr(slot->phys_offset);
 146     mem.flags = slot->flags;
 147     if (s->migration_log) {
 148         mem.flags |= KVM_MEM_LOG_DIRTY_PAGES;
 149     }
 150     return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
 151 }
 152
 153 static void kvm_reset_vcpu(void *opaque)
 154 {
 155     CPUState *env = opaque;
 156
 157     kvm_arch_reset_vcpu(env);
 158     if (kvm_arch_put_registers(env)) {
 159         fprintf(stderr, "Fatal: kvm vcpu reset failed\n");
 160         abort();
 161     }
 162 }
 163
 164 int kvm_irqchip_in_kernel(void)
 165 {
 166     return kvm_state->irqchip_in_kernel;
 167 }
 168
 169 int kvm_pit_in_kernel(void)
 170 {
 171     return kvm_state->pit_in_kernel;
 172 }
 173
 174
 175 int kvm_init_vcpu(CPUState *env)
 176 {
 177     KVMState *s = kvm_state;
 178     long mmap_size;
 179     int ret;
 180
 181     dprintf("kvm_init_vcpu\n");
 182
 183     ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, env->cpu_index);
 184     if (ret < 0) {
 185         dprintf("kvm_create_vcpu failed\n");
 186         goto err;
 187     }
 188
 189     env->kvm_fd = ret;
 190     env->kvm_state = s;
 191
 192     mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
 193     if (mmap_size < 0) {
 194         dprintf("KVM_GET_VCPU_MMAP_SIZE failed\n");
 195         goto err;
 196     }
 197
 198     env->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
 199                         env->kvm_fd, 0);
 200     if (env->kvm_run == MAP_FAILED) {
 201         ret = -errno;
 202         dprintf("mmap'ing vcpu state failed\n");
 203         goto err;
 204     }
 205
 206 #ifdef KVM_CAP_COALESCED_MMIO
 207     if (s->coalesced_mmio && !s->coalesced_mmio_ring)
 208         s->coalesced_mmio_ring = (void *) env->kvm_run +
 209                 s->coalesced_mmio * PAGE_SIZE;
 210 #endif
 211
 212     ret = kvm_arch_init_vcpu(env);
 213     if (ret == 0) {
 214         qemu_register_reset(kvm_reset_vcpu, env);
 215         kvm_arch_reset_vcpu(env);
 216         ret = kvm_arch_put_registers(env);
 217     }
 218 err:
 219     return ret;
 220 }
 221
 222 /*
 223  * dirty pages logging control
 224  */
 225 static int kvm_dirty_pages_log_change(target_phys_addr_t phys_addr,
 226                                       ram_addr_t size, int flags, int mask)
 227 {
 228     KVMState *s = kvm_state;
 229     KVMSlot *mem = kvm_lookup_matching_slot(s, phys_addr, phys_addr + size);
 230     int old_flags;
 231
 232     if (mem == NULL)  {
 233             fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
 234                     TARGET_FMT_plx "\n", __func__, phys_addr,
 235                     (target_phys_addr_t)(phys_addr + size - 1));
 236             return -EINVAL;
 237     }
 238
 239     old_flags = mem->flags;
 240
 241     flags = (mem->flags & ~mask) | flags;
 242     mem->flags = flags;
 243
 244     /* If nothing changed effectively, no need to issue ioctl */
 245     if (s->migration_log) {
 246         flags |= KVM_MEM_LOG_DIRTY_PAGES;
 247     }
 248     if (flags == old_flags) {
 249             return 0;
 250     }
 251
 252     return kvm_set_user_memory_region(s, mem);
 253 }
 254
 255 int kvm_log_start(target_phys_addr_t phys_addr, ram_addr_t size)
 256 {
 257         return kvm_dirty_pages_log_change(phys_addr, size,
 258                                           KVM_MEM_LOG_DIRTY_PAGES,
 259                                           KVM_MEM_LOG_DIRTY_PAGES);
 260 }
 261
 262 int kvm_log_stop(target_phys_addr_t phys_addr, ram_addr_t size)
 263 {
 264         return kvm_dirty_pages_log_change(phys_addr, size,
 265                                           0,
 266                                           KVM_MEM_LOG_DIRTY_PAGES);
 267 }
 268
 269 int kvm_set_migration_log(int enable)
 270 {
 271     KVMState *s = kvm_state;
 272     KVMSlot *mem;
 273     int i, err;
 274
 275     s->migration_log = enable;
 276
 277     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 278         mem = &s->slots[i];
 279
 280         if (!!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) == enable) {
 281             continue;
 282         }
 283         err = kvm_set_user_memory_region(s, mem);
 284         if (err) {
 285             return err;
 286         }
 287     }
 288     return 0;
 289 }
 290
 291 static int test_le_bit(unsigned long nr, unsigned char *addr)
 292 {
 293     return (addr[nr >> 3] >> (nr & 7)) & 1;
 294 }
 295
 296 /**
 297  * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
 298  * This function updates qemu's dirty bitmap using cpu_physical_memory_set_dirty().
 299  * This means all bits are set to dirty.
 300  *
 301  * @start_add: start of logged region.
 302  * @end_addr: end of logged region.
 303  */
 304 int kvm_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
 305                                    target_phys_addr_t end_addr)
 306 {
 307     KVMState *s = kvm_state;
 308     unsigned long size, allocated_size = 0;
 309     target_phys_addr_t phys_addr;
 310     ram_addr_t addr;
 311     KVMDirtyLog d;
 312     KVMSlot *mem;
 313     int ret = 0;
 314
 315     d.dirty_bitmap = NULL;
 316     while (start_addr < end_addr) {
 317         mem = kvm_lookup_overlapping_slot(s, start_addr, end_addr);
 318         if (mem == NULL) {
 319             break;
 320         }
 321
 322         size = ((mem->memory_size >> TARGET_PAGE_BITS) + 7) / 8;
 323         if (!d.dirty_bitmap) {
 324             d.dirty_bitmap = qemu_malloc(size);
 325         } else if (size > allocated_size) {
 326             d.dirty_bitmap = qemu_realloc(d.dirty_bitmap, size);
 327         }
 328         allocated_size = size;
 329         memset(d.dirty_bitmap, 0, allocated_size);
 330
 331         d.slot = mem->slot;
 332
 333         if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
 334             dprintf("ioctl failed %d\n", errno);
 335             ret = -1;
 336             break;
 337         }
 338
 339         for (phys_addr = mem->start_addr, addr = mem->phys_offset;
 340              phys_addr < mem->start_addr + mem->memory_size;
 341              phys_addr += TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
 342             unsigned char *bitmap = (unsigned char *)d.dirty_bitmap;
 343             unsigned nr = (phys_addr - mem->start_addr) >> TARGET_PAGE_BITS;
 344
 345             if (test_le_bit(nr, bitmap)) {
 346                 cpu_physical_memory_set_dirty(addr);
 347             }
 348         }
 349         start_addr = phys_addr;
 350     }
 351     qemu_free(d.dirty_bitmap);
 352
 353     return ret;
 354 }
 355
 356 int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
 357 {
 358     int ret = -ENOSYS;
 359 #ifdef KVM_CAP_COALESCED_MMIO
 360     KVMState *s = kvm_state;
 361
 362     if (s->coalesced_mmio) {
 363         struct kvm_coalesced_mmio_zone zone;
 364
 365         zone.addr = start;
 366         zone.size = size;
 367
 368         ret = kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
 369     }
 370 #endif
 371
 372     return ret;
 373 }
 374
 375 int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
 376 {
 377     int ret = -ENOSYS;
 378 #ifdef KVM_CAP_COALESCED_MMIO
 379     KVMState *s = kvm_state;
 380
 381     if (s->coalesced_mmio) {
 382         struct kvm_coalesced_mmio_zone zone;
 383
 384         zone.addr = start;
 385         zone.size = size;
 386
 387         ret = kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
 388     }
 389 #endif
 390
 391     return ret;
 392 }
 393
 394 int kvm_check_extension(KVMState *s, unsigned int extension)
 395 {
 396     int ret;
 397
 398     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
 399     if (ret < 0) {
 400         ret = 0;
 401     }
 402
 403     return ret;
 404 }
 405
 406 int kvm_init(int smp_cpus)
 407 {
 408     static const char upgrade_note[] =
 409         "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
 410         "(see http://sourceforge.net/projects/kvm).\n";
 411     KVMState *s;
 412     int ret;
 413     int i;
 414
 415     if (smp_cpus > 1) {
 416         fprintf(stderr, "No SMP KVM support, use '-smp 1'\n");
 417         return -EINVAL;
 418     }
 419
 420     s = qemu_mallocz(sizeof(KVMState));
 421
 422 #ifdef KVM_CAP_SET_GUEST_DEBUG
 423     QTAILQ_INIT(&s->kvm_sw_breakpoints);
 424 #endif
 425     for (i = 0; i < ARRAY_SIZE(s->slots); i++)
 426         s->slots[i].slot = i;
 427
 428     s->vmfd = -1;
 429     s->fd = qemu_open("/dev/kvm", O_RDWR);
 430     if (s->fd == -1) {
 431         fprintf(stderr, "Could not access KVM kernel module: %m\n");
 432         ret = -errno;
 433         goto err;
 434     }
 435
 436     ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
 437     if (ret < KVM_API_VERSION) {
 438         if (ret > 0)
 439             ret = -EINVAL;
 440         fprintf(stderr, "kvm version too old\n");
 441         goto err;
 442     }
 443
 444     if (ret > KVM_API_VERSION) {
 445         ret = -EINVAL;
 446         fprintf(stderr, "kvm version not supported\n");
 447         goto err;
 448     }
 449
 450     s->vmfd = kvm_ioctl(s, KVM_CREATE_VM, 0);
 451     if (s->vmfd < 0)
 452         goto err;
 453
 454     /* initially, KVM allocated its own memory and we had to jump through
 455      * hooks to make phys_ram_base point to this.  Modern versions of KVM
 456      * just use a user allocated buffer so we can use regular pages
 457      * unmodified.  Make sure we have a sufficiently modern version of KVM.
 458      */
 459     if (!kvm_check_extension(s, KVM_CAP_USER_MEMORY)) {
 460         ret = -EINVAL;
 461         fprintf(stderr, "kvm does not support KVM_CAP_USER_MEMORY\n%s",
 462                 upgrade_note);
 463         goto err;
 464     }
 465
 466     /* There was a nasty bug in < kvm-80 that prevents memory slots from being
 467      * destroyed properly.  Since we rely on this capability, refuse to work
 468      * with any kernel without this capability. */
 469     if (!kvm_check_extension(s, KVM_CAP_DESTROY_MEMORY_REGION_WORKS)) {
 470         ret = -EINVAL;
 471
 472         fprintf(stderr,
 473                 "KVM kernel module broken (DESTROY_MEMORY_REGION).\n%s",
 474                 upgrade_note);
 475         goto err;
 476     }
 477
 478     s->coalesced_mmio = 0;
 479 #ifdef KVM_CAP_COALESCED_MMIO
 480     s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
 481     s->coalesced_mmio_ring = NULL;
 482 #endif
 483
 484     s->broken_set_mem_region = 1;
 485 #ifdef KVM_CAP_JOIN_MEMORY_REGIONS_WORKS
 486     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
 487     if (ret > 0) {
 488         s->broken_set_mem_region = 0;
 489     }
 490 #endif
 491
 492     s->vcpu_events = 0;
 493 #ifdef KVM_CAP_VCPU_EVENTS
 494     s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
 495 #endif
 496
 497     ret = kvm_arch_init(s, smp_cpus);
 498     if (ret < 0)
 499         goto err;
 500
 501     kvm_state = s;
 502
 503     return 0;
 504
 505 err:
 506     if (s) {
 507         if (s->vmfd != -1)
 508             close(s->vmfd);
 509         if (s->fd != -1)
 510             close(s->fd);
 511     }
 512     qemu_free(s);
 513
 514     return ret;
 515 }
 516
 517 static int kvm_handle_io(uint16_t port, void *data, int direction, int size,
 518                          uint32_t count)
 519 {
 520     int i;
 521     uint8_t *ptr = data;
 522
 523     for (i = 0; i < count; i++) {
 524         if (direction == KVM_EXIT_IO_IN) {
 525             switch (size) {
 526             case 1:
 527                 stb_p(ptr, cpu_inb(port));
 528                 break;
 529             case 2:
 530                 stw_p(ptr, cpu_inw(port));
 531                 break;
 532             case 4:
 533                 stl_p(ptr, cpu_inl(port));
 534                 break;
 535             }
 536         } else {
 537             switch (size) {
 538             case 1:
 539                 cpu_outb(port, ldub_p(ptr));
 540                 break;
 541             case 2:
 542                 cpu_outw(port, lduw_p(ptr));
 543                 break;
 544             case 4:
 545                 cpu_outl(port, ldl_p(ptr));
 546                 break;
 547             }
 548         }
 549
 550         ptr += size;
 551     }
 552
 553     return 1;
 554 }
 555
 556 void kvm_flush_coalesced_mmio_buffer(void)
 557 {
 558 #ifdef KVM_CAP_COALESCED_MMIO
 559     KVMState *s = kvm_state;
 560     if (s->coalesced_mmio_ring) {
 561         struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
 562         while (ring->first != ring->last) {
 563             struct kvm_coalesced_mmio *ent;
 564
 565             ent = &ring->coalesced_mmio[ring->first];
 566
 567             cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
 568             /* FIXME smp_wmb() */
 569             ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
 570         }
 571     }
 572 #endif
 573 }
 574
 575 void kvm_cpu_synchronize_state(CPUState *env)
 576 {
 577     if (!env->kvm_state->regs_modified) {
 578         kvm_arch_get_registers(env);
 579         env->kvm_state->regs_modified = 1;
 580     }
 581 }
 582
 583 int kvm_cpu_exec(CPUState *env)
 584 {
 585     struct kvm_run *run = env->kvm_run;
 586     int ret;
 587
 588     dprintf("kvm_cpu_exec()\n");
 589
 590     do {
 591         if (env->exit_request) {
 592             dprintf("interrupt exit requested\n");
 593             ret = 0;
 594             break;
 595         }
 596
 597         if (env->kvm_state->regs_modified) {
 598             kvm_arch_put_registers(env);
 599             env->kvm_state->regs_modified = 0;
 600         }
 601
 602         kvm_arch_pre_run(env, run);
 603         qemu_mutex_unlock_iothread();
 604         ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);
 605         qemu_mutex_lock_iothread();
 606         kvm_arch_post_run(env, run);
 607
 608         if (ret == -EINTR || ret == -EAGAIN) {
 609             dprintf("io window exit\n");
 610             ret = 0;
 611             break;
 612         }
 613
 614         if (ret < 0) {
 615             dprintf("kvm run failed %s\n", strerror(-ret));
 616             abort();
 617         }
 618
 619         kvm_flush_coalesced_mmio_buffer();
 620
 621         ret = 0; /* exit loop */
 622         switch (run->exit_reason) {
 623         case KVM_EXIT_IO:
 624             dprintf("handle_io\n");
 625             ret = kvm_handle_io(run->io.port,
 626                                 (uint8_t *)run + run->io.data_offset,
 627                                 run->io.direction,
 628                                 run->io.size,
 629                                 run->io.count);
 630             break;
 631         case KVM_EXIT_MMIO:
 632             dprintf("handle_mmio\n");
 633             cpu_physical_memory_rw(run->mmio.phys_addr,
 634                                    run->mmio.data,
 635                                    run->mmio.len,
 636                                    run->mmio.is_write);
 637             ret = 1;
 638             break;
 639         case KVM_EXIT_IRQ_WINDOW_OPEN:
 640             dprintf("irq_window_open\n");
 641             break;
 642         case KVM_EXIT_SHUTDOWN:
 643             dprintf("shutdown\n");
 644             qemu_system_reset_request();
 645             ret = 1;
 646             break;
 647         case KVM_EXIT_UNKNOWN:
 648             dprintf("kvm_exit_unknown\n");
 649             break;
 650         case KVM_EXIT_FAIL_ENTRY:
 651             dprintf("kvm_exit_fail_entry\n");
 652             break;
 653         case KVM_EXIT_EXCEPTION:
 654             dprintf("kvm_exit_exception\n");
 655             break;
 656         case KVM_EXIT_DEBUG:
 657             dprintf("kvm_exit_debug\n");
 658 #ifdef KVM_CAP_SET_GUEST_DEBUG
 659             if (kvm_arch_debug(&run->debug.arch)) {
 660                 gdb_set_stop_cpu(env);
 661                 vm_stop(EXCP_DEBUG);
 662                 env->exception_index = EXCP_DEBUG;
 663                 return 0;
 664             }
 665             /* re-enter, this exception was guest-internal */
 666             ret = 1;
 667 #endif /* KVM_CAP_SET_GUEST_DEBUG */
 668             break;
 669         default:
 670             dprintf("kvm_arch_handle_exit\n");
 671             ret = kvm_arch_handle_exit(env, run);
 672             break;
 673         }
 674     } while (ret > 0);
 675
 676     if (env->exit_request) {
 677         env->exit_request = 0;
 678         env->exception_index = EXCP_INTERRUPT;
 679     }
 680
 681     return ret;
 682 }
 683
 684 void kvm_set_phys_mem(target_phys_addr_t start_addr,
 685                       ram_addr_t size,
 686                       ram_addr_t phys_offset)
 687 {
 688     KVMState *s = kvm_state;
 689     ram_addr_t flags = phys_offset & ~TARGET_PAGE_MASK;
 690     KVMSlot *mem, old;
 691     int err;
 692
 693     if (start_addr & ~TARGET_PAGE_MASK) {
 694         if (flags >= IO_MEM_UNASSIGNED) {
 695             if (!kvm_lookup_overlapping_slot(s, start_addr,
 696                                              start_addr + size)) {
 697                 return;
 698             }
 699             fprintf(stderr, "Unaligned split of a KVM memory slot\n");
 700         } else {
 701             fprintf(stderr, "Only page-aligned memory slots supported\n");
 702         }
 703         abort();
 704     }
 705
 706     /* KVM does not support read-only slots */
 707     phys_offset &= ~IO_MEM_ROM;
 708
 709     while (1) {
 710         mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size);
 711         if (!mem) {
 712             break;
 713         }
 714
 715         if (flags < IO_MEM_UNASSIGNED && start_addr >= mem->start_addr &&
 716             (start_addr + size <= mem->start_addr + mem->memory_size) &&
 717             (phys_offset - start_addr == mem->phys_offset - mem->start_addr)) {
 718             /* The new slot fits into the existing one and comes with
 719              * identical parameters - nothing to be done. */
 720             return;
 721         }
 722
 723         old = *mem;
 724
 725         /* unregister the overlapping slot */
 726         mem->memory_size = 0;
 727         err = kvm_set_user_memory_region(s, mem);
 728         if (err) {
 729             fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
 730                     __func__, strerror(-err));
 731             abort();
 732         }
 733
 734         /* Workaround for older KVM versions: we can't join slots, even not by
 735          * unregistering the previous ones and then registering the larger
 736          * slot. We have to maintain the existing fragmentation. Sigh.
 737          *
 738          * This workaround assumes that the new slot starts at the same
 739          * address as the first existing one. If not or if some overlapping
 740          * slot comes around later, we will fail (not seen in practice so far)
 741          * - and actually require a recent KVM version. */
 742         if (s->broken_set_mem_region &&
 743             old.start_addr == start_addr && old.memory_size < size &&
 744             flags < IO_MEM_UNASSIGNED) {
 745             mem = kvm_alloc_slot(s);
 746             mem->memory_size = old.memory_size;
 747             mem->start_addr = old.start_addr;
 748             mem->phys_offset = old.phys_offset;
 749             mem->flags = 0;
 750
 751             err = kvm_set_user_memory_region(s, mem);
 752             if (err) {
 753                 fprintf(stderr, "%s: error updating slot: %s\n", __func__,
 754                         strerror(-err));
 755                 abort();
 756             }
 757
 758             start_addr += old.memory_size;
 759             phys_offset += old.memory_size;
 760             size -= old.memory_size;
 761             continue;
 762         }
 763
 764         /* register prefix slot */
 765         if (old.start_addr < start_addr) {
 766             mem = kvm_alloc_slot(s);
 767             mem->memory_size = start_addr - old.start_addr;
 768             mem->start_addr = old.start_addr;
 769             mem->phys_offset = old.phys_offset;
 770             mem->flags = 0;
 771
 772             err = kvm_set_user_memory_region(s, mem);
 773             if (err) {
 774                 fprintf(stderr, "%s: error registering prefix slot: %s\n",
 775                         __func__, strerror(-err));
 776                 abort();
 777             }
 778         }
 779
 780         /* register suffix slot */
 781         if (old.start_addr + old.memory_size > start_addr + size) {
 782             ram_addr_t size_delta;
 783
 784             mem = kvm_alloc_slot(s);
 785             mem->start_addr = start_addr + size;
 786             size_delta = mem->start_addr - old.start_addr;
 787             mem->memory_size = old.memory_size - size_delta;
 788             mem->phys_offset = old.phys_offset + size_delta;
 789             mem->flags = 0;
 790
 791             err = kvm_set_user_memory_region(s, mem);
 792             if (err) {
 793                 fprintf(stderr, "%s: error registering suffix slot: %s\n",
 794                         __func__, strerror(-err));
 795                 abort();
 796             }
 797         }
 798     }
 799
 800     /* in case the KVM bug workaround already "consumed" the new slot */
 801     if (!size)
 802         return;
 803
 804     /* KVM does not need to know about this memory */
 805     if (flags >= IO_MEM_UNASSIGNED)
 806         return;
 807
 808     mem = kvm_alloc_slot(s);
 809     mem->memory_size = size;
 810     mem->start_addr = start_addr;
 811     mem->phys_offset = phys_offset;
 812     mem->flags = 0;
 813
 814     err = kvm_set_user_memory_region(s, mem);
 815     if (err) {
 816         fprintf(stderr, "%s: error registering slot: %s\n", __func__,
 817                 strerror(-err));
 818         abort();
 819     }
 820 }
 821
 822 int kvm_ioctl(KVMState *s, int type, ...)
 823 {
 824     int ret;
 825     void *arg;
 826     va_list ap;
 827
 828     va_start(ap, type);
 829     arg = va_arg(ap, void *);
 830     va_end(ap);
 831
 832     ret = ioctl(s->fd, type, arg);
 833     if (ret == -1)
 834         ret = -errno;
 835
 836     return ret;
 837 }
 838
 839 int kvm_vm_ioctl(KVMState *s, int type, ...)
 840 {
 841     int ret;
 842     void *arg;
 843     va_list ap;
 844
 845     va_start(ap, type);
 846     arg = va_arg(ap, void *);
 847     va_end(ap);
 848
 849     ret = ioctl(s->vmfd, type, arg);
 850     if (ret == -1)
 851         ret = -errno;
 852
 853     return ret;
 854 }
 855
 856 int kvm_vcpu_ioctl(CPUState *env, int type, ...)
 857 {
 858     int ret;
 859     void *arg;
 860     va_list ap;
 861
 862     va_start(ap, type);
 863     arg = va_arg(ap, void *);
 864     va_end(ap);
 865
 866     ret = ioctl(env->kvm_fd, type, arg);
 867     if (ret == -1)
 868         ret = -errno;
 869
 870     return ret;
 871 }
 872
 873 int kvm_has_sync_mmu(void)
 874 {
 875 #ifdef KVM_CAP_SYNC_MMU
 876     KVMState *s = kvm_state;
 877
 878     return kvm_check_extension(s, KVM_CAP_SYNC_MMU);
 879 #else
 880     return 0;
 881 #endif
 882 }
 883
 884 int kvm_has_vcpu_events(void)
 885 {
 886     return kvm_state->vcpu_events;
 887 }
 888
 889 void kvm_setup_guest_memory(void *start, size_t size)
 890 {
 891     if (!kvm_has_sync_mmu()) {
 892 #ifdef MADV_DONTFORK
 893         int ret = madvise(start, size, MADV_DONTFORK);
 894
 895         if (ret) {
 896             perror("madvice");
 897             exit(1);
 898         }
 899 #else
 900         fprintf(stderr,
 901                 "Need MADV_DONTFORK in absence of synchronous KVM MMU\n");
 902         exit(1);
 903 #endif
 904     }
 905 }
 906
 907 #ifdef KVM_CAP_SET_GUEST_DEBUG
 908 static void on_vcpu(CPUState *env, void (*func)(void *data), void *data)
 909 {
 910 #ifdef CONFIG_IOTHREAD
 911     if (env == cpu_single_env) {
 912         func(data);
 913         return;
 914     }
 915     abort();
 916 #else
 917     func(data);
 918 #endif
 919 }
 920
 921 struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *env,
 922                                                  target_ulong pc)
 923 {
 924     struct kvm_sw_breakpoint *bp;
 925
 926     QTAILQ_FOREACH(bp, &env->kvm_state->kvm_sw_breakpoints, entry) {
 927         if (bp->pc == pc)
 928             return bp;
 929     }
 930     return NULL;
 931 }
 932
 933 int kvm_sw_breakpoints_active(CPUState *env)
 934 {
 935     return !QTAILQ_EMPTY(&env->kvm_state->kvm_sw_breakpoints);
 936 }
 937
 938 struct kvm_set_guest_debug_data {
 939     struct kvm_guest_debug dbg;
 940     CPUState *env;
 941     int err;
 942 };
 943
 944 static void kvm_invoke_set_guest_debug(void *data)
 945 {
 946     struct kvm_set_guest_debug_data *dbg_data = data;
 947     CPUState *env = dbg_data->env;
 948
 949     if (env->kvm_state->regs_modified) {
 950         kvm_arch_put_registers(env);
 951         env->kvm_state->regs_modified = 0;
 952     }
 953     dbg_data->err = kvm_vcpu_ioctl(env, KVM_SET_GUEST_DEBUG, &dbg_data->dbg);
 954 }
 955
 956 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
 957 {
 958     struct kvm_set_guest_debug_data data;
 959
 960     data.dbg.control = 0;
 961     if (env->singlestep_enabled)
 962         data.dbg.control = KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
 963
 964     kvm_arch_update_guest_debug(env, &data.dbg);
 965     data.dbg.control |= reinject_trap;
 966     data.env = env;
 967
 968     on_vcpu(env, kvm_invoke_set_guest_debug, &data);
 969     return data.err;
 970 }
 971
 972 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
 973                           target_ulong len, int type)
 974 {
 975     struct kvm_sw_breakpoint *bp;
 976     CPUState *env;
 977     int err;
 978
 979     if (type == GDB_BREAKPOINT_SW) {
 980         bp = kvm_find_sw_breakpoint(current_env, addr);
 981         if (bp) {
 982             bp->use_count++;
 983             return 0;
 984         }
 985
 986         bp = qemu_malloc(sizeof(struct kvm_sw_breakpoint));
 987         if (!bp)
 988             return -ENOMEM;
 989
 990         bp->pc = addr;
 991         bp->use_count = 1;
 992         err = kvm_arch_insert_sw_breakpoint(current_env, bp);
 993         if (err) {
 994             free(bp);
 995             return err;
 996         }
 997
 998         QTAILQ_INSERT_HEAD(&current_env->kvm_state->kvm_sw_breakpoints,
 999                           bp, entry);
1000     } else {
1001         err = kvm_arch_insert_hw_breakpoint(addr, len, type);
1002         if (err)
1003             return err;
1004     }
1005
1006     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1007         err = kvm_update_guest_debug(env, 0);
1008         if (err)
1009             return err;
1010     }
1011     return 0;
1012 }
1013
1014 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1015                           target_ulong len, int type)
1016 {
1017     struct kvm_sw_breakpoint *bp;
1018     CPUState *env;
1019     int err;
1020
1021     if (type == GDB_BREAKPOINT_SW) {
1022         bp = kvm_find_sw_breakpoint(current_env, addr);
1023         if (!bp)
1024             return -ENOENT;
1025
1026         if (bp->use_count > 1) {
1027             bp->use_count--;
1028             return 0;
1029         }
1030
1031         err = kvm_arch_remove_sw_breakpoint(current_env, bp);
1032         if (err)
1033             return err;
1034
1035         QTAILQ_REMOVE(&current_env->kvm_state->kvm_sw_breakpoints, bp, entry);
1036         qemu_free(bp);
1037     } else {
1038         err = kvm_arch_remove_hw_breakpoint(addr, len, type);
1039         if (err)
1040             return err;
1041     }
1042
1043     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1044         err = kvm_update_guest_debug(env, 0);
1045         if (err)
1046             return err;
1047     }
1048     return 0;
1049 }
1050
1051 void kvm_remove_all_breakpoints(CPUState *current_env)
1052 {
1053     struct kvm_sw_breakpoint *bp, *next;
1054     KVMState *s = current_env->kvm_state;
1055     CPUState *env;
1056
1057     QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
1058         if (kvm_arch_remove_sw_breakpoint(current_env, bp) != 0) {
1059             /* Try harder to find a CPU that currently sees the breakpoint. */
1060             for (env = first_cpu; env != NULL; env = env->next_cpu) {
1061                 if (kvm_arch_remove_sw_breakpoint(env, bp) == 0)
1062                     break;
1063             }
1064         }
1065     }
1066     kvm_arch_remove_all_hw_breakpoints();
1067
1068     for (env = first_cpu; env != NULL; env = env->next_cpu)
1069         kvm_update_guest_debug(env, 0);
1070 }
1071
1072 #else /* !KVM_CAP_SET_GUEST_DEBUG */
1073
1074 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1075 {
1076     return -EINVAL;
1077 }
1078
1079 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1080                           target_ulong len, int type)
1081 {
1082     return -EINVAL;
1083 }
1084
1085 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1086                           target_ulong len, int type)
1087 {
1088     return -EINVAL;
1089 }
1090
1091 void kvm_remove_all_breakpoints(CPUState *current_env)
1092 {
1093 }
1094 #endif /* !KVM_CAP_SET_GUEST_DEBUG */