kvm-all.c

   1 /*
   2  * QEMU KVM support
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *           Red Hat, Inc. 2008
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *  Glauber Costa     <gcosta@redhat.com>
  10  *
  11  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  12  * See the COPYING file in the top-level directory.
  13  *
  14  */
  15
  16 #include <sys/types.h>
  17 #include <sys/ioctl.h>
  18 #include <sys/mman.h>
  19 #include <stdarg.h>
  20
  21 #include <linux/kvm.h>
  22
  23 #include "qemu-common.h"
  24 #include "sysemu.h"
  25 #include "hw/hw.h"
  26 #include "gdbstub.h"
  27 #include "kvm.h"
  28
  29 #ifdef KVM_UPSTREAM
  30 /* KVM uses PAGE_SIZE in it's definition of COALESCED_MMIO_MAX */
  31 #define PAGE_SIZE TARGET_PAGE_SIZE
  32
  33 //#define DEBUG_KVM
  34
  35 #ifdef DEBUG_KVM
  36 #define dprintf(fmt, ...) \
  37     do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
  38 #else
  39 #define dprintf(fmt, ...) \
  40     do { } while (0)
  41 #endif
  42
  43 typedef struct KVMSlot
  44 {
  45     target_phys_addr_t start_addr;
  46     ram_addr_t memory_size;
  47     ram_addr_t phys_offset;
  48     int slot;
  49     int flags;
  50 } KVMSlot;
  51
  52 typedef struct kvm_dirty_log KVMDirtyLog;
  53
  54 int kvm_allowed = 0;
  55
  56 struct KVMState
  57 {
  58     KVMSlot slots[32];
  59     int fd;
  60     int vmfd;
  61     int coalesced_mmio;
  62     int broken_set_mem_region;
  63     int migration_log;
  64     int vcpu_events;
  65 #ifdef KVM_CAP_SET_GUEST_DEBUG
  66     struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
  67 #endif
  68     int irqchip_in_kernel;
  69     int pit_in_kernel;
  70 };
  71
  72 static KVMState *kvm_state;
  73
  74 static KVMSlot *kvm_alloc_slot(KVMState *s)
  75 {
  76     int i;
  77
  78     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
  79         /* KVM private memory slots */
  80         if (i >= 8 && i < 12)
  81             continue;
  82         if (s->slots[i].memory_size == 0)
  83             return &s->slots[i];
  84     }
  85
  86     fprintf(stderr, "%s: no free slot available\n", __func__);
  87     abort();
  88 }
  89
  90 static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
  91                                          target_phys_addr_t start_addr,
  92                                          target_phys_addr_t end_addr)
  93 {
  94     int i;
  95
  96     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
  97         KVMSlot *mem = &s->slots[i];
  98
  99         if (start_addr == mem->start_addr &&
 100             end_addr == mem->start_addr + mem->memory_size) {
 101             return mem;
 102         }
 103     }
 104
 105     return NULL;
 106 }
 107
 108 /*
 109  * Find overlapping slot with lowest start address
 110  */
 111 static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
 112                                             target_phys_addr_t start_addr,
 113                                             target_phys_addr_t end_addr)
 114 {
 115     KVMSlot *found = NULL;
 116     int i;
 117
 118     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 119         KVMSlot *mem = &s->slots[i];
 120
 121         if (mem->memory_size == 0 ||
 122             (found && found->start_addr < mem->start_addr)) {
 123             continue;
 124         }
 125
 126         if (end_addr > mem->start_addr &&
 127             start_addr < mem->start_addr + mem->memory_size) {
 128             found = mem;
 129         }
 130     }
 131
 132     return found;
 133 }
 134
 135 static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
 136 {
 137     struct kvm_userspace_memory_region mem;
 138
 139     mem.slot = slot->slot;
 140     mem.guest_phys_addr = slot->start_addr;
 141     mem.memory_size = slot->memory_size;
 142     mem.userspace_addr = (unsigned long)qemu_get_ram_ptr(slot->phys_offset);
 143     mem.flags = slot->flags;
 144     if (s->migration_log) {
 145         mem.flags |= KVM_MEM_LOG_DIRTY_PAGES;
 146     }
 147     return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
 148 }
 149
 150 static void kvm_reset_vcpu(void *opaque)
 151 {
 152     CPUState *env = opaque;
 153
 154     kvm_arch_reset_vcpu(env);
 155     if (kvm_arch_put_registers(env)) {
 156         fprintf(stderr, "Fatal: kvm vcpu reset failed\n");
 157         abort();
 158     }
 159 }
 160 #endif
 161
 162 int kvm_irqchip_in_kernel(void)
 163 {
 164     return kvm_state->irqchip_in_kernel;
 165 }
 166
 167 #ifdef KVM_UPSTREAM
 168 int kvm_pit_in_kernel(void)
 169 {
 170     return kvm_state->pit_in_kernel;
 171 }
 172
 173
 174 int kvm_init_vcpu(CPUState *env)
 175 {
 176     KVMState *s = kvm_state;
 177     long mmap_size;
 178     int ret;
 179
 180     dprintf("kvm_init_vcpu\n");
 181
 182     ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, env->cpu_index);
 183     if (ret < 0) {
 184         dprintf("kvm_create_vcpu failed\n");
 185         goto err;
 186     }
 187
 188     env->kvm_fd = ret;
 189     env->kvm_state = s;
 190
 191     mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
 192     if (mmap_size < 0) {
 193         dprintf("KVM_GET_VCPU_MMAP_SIZE failed\n");
 194         goto err;
 195     }
 196
 197     env->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
 198                         env->kvm_fd, 0);
 199     if (env->kvm_run == MAP_FAILED) {
 200         ret = -errno;
 201         dprintf("mmap'ing vcpu state failed\n");
 202         goto err;
 203     }
 204
 205     ret = kvm_arch_init_vcpu(env);
 206     if (ret == 0) {
 207         qemu_register_reset(kvm_reset_vcpu, env);
 208         kvm_arch_reset_vcpu(env);
 209         ret = kvm_arch_put_registers(env);
 210     }
 211 err:
 212     return ret;
 213 }
 214
 215 /*
 216  * dirty pages logging control
 217  */
 218 static int kvm_dirty_pages_log_change(target_phys_addr_t phys_addr,
 219                                       ram_addr_t size, int flags, int mask)
 220 {
 221     KVMState *s = kvm_state;
 222     KVMSlot *mem = kvm_lookup_matching_slot(s, phys_addr, phys_addr + size);
 223     int old_flags;
 224
 225     if (mem == NULL)  {
 226             fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
 227                     TARGET_FMT_plx "\n", __func__, phys_addr,
 228                     (target_phys_addr_t)(phys_addr + size - 1));
 229             return -EINVAL;
 230     }
 231
 232     old_flags = mem->flags;
 233
 234     flags = (mem->flags & ~mask) | flags;
 235     mem->flags = flags;
 236
 237     /* If nothing changed effectively, no need to issue ioctl */
 238     if (s->migration_log) {
 239         flags |= KVM_MEM_LOG_DIRTY_PAGES;
 240     }
 241     if (flags == old_flags) {
 242             return 0;
 243     }
 244
 245     return kvm_set_user_memory_region(s, mem);
 246 }
 247
 248 int kvm_log_start(target_phys_addr_t phys_addr, ram_addr_t size)
 249 {
 250         return kvm_dirty_pages_log_change(phys_addr, size,
 251                                           KVM_MEM_LOG_DIRTY_PAGES,
 252                                           KVM_MEM_LOG_DIRTY_PAGES);
 253 }
 254
 255 int kvm_log_stop(target_phys_addr_t phys_addr, ram_addr_t size)
 256 {
 257         return kvm_dirty_pages_log_change(phys_addr, size,
 258                                           0,
 259                                           KVM_MEM_LOG_DIRTY_PAGES);
 260 }
 261
 262 int kvm_set_migration_log(int enable)
 263 {
 264     KVMState *s = kvm_state;
 265     KVMSlot *mem;
 266     int i, err;
 267
 268     s->migration_log = enable;
 269
 270     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 271         mem = &s->slots[i];
 272
 273         if (!!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) == enable) {
 274             continue;
 275         }
 276         err = kvm_set_user_memory_region(s, mem);
 277         if (err) {
 278             return err;
 279         }
 280     }
 281     return 0;
 282 }
 283
 284 static int test_le_bit(unsigned long nr, unsigned char *addr)
 285 {
 286     return (addr[nr >> 3] >> (nr & 7)) & 1;
 287 }
 288
 289 /**
 290  * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
 291  * This function updates qemu's dirty bitmap using cpu_physical_memory_set_dirty().
 292  * This means all bits are set to dirty.
 293  *
 294  * @start_add: start of logged region.
 295  * @end_addr: end of logged region.
 296  */
 297 int kvm_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
 298                                    target_phys_addr_t end_addr)
 299 {
 300     KVMState *s = kvm_state;
 301     unsigned long size, allocated_size = 0;
 302     target_phys_addr_t phys_addr;
 303     ram_addr_t addr;
 304     KVMDirtyLog d;
 305     KVMSlot *mem;
 306     int ret = 0;
 307
 308     d.dirty_bitmap = NULL;
 309     while (start_addr < end_addr) {
 310         mem = kvm_lookup_overlapping_slot(s, start_addr, end_addr);
 311         if (mem == NULL) {
 312             break;
 313         }
 314
 315         size = ((mem->memory_size >> TARGET_PAGE_BITS) + 7) / 8;
 316         if (!d.dirty_bitmap) {
 317             d.dirty_bitmap = qemu_malloc(size);
 318         } else if (size > allocated_size) {
 319             d.dirty_bitmap = qemu_realloc(d.dirty_bitmap, size);
 320         }
 321         allocated_size = size;
 322         memset(d.dirty_bitmap, 0, allocated_size);
 323
 324         d.slot = mem->slot;
 325
 326         if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
 327             dprintf("ioctl failed %d\n", errno);
 328             ret = -1;
 329             break;
 330         }
 331
 332         for (phys_addr = mem->start_addr, addr = mem->phys_offset;
 333              phys_addr < mem->start_addr + mem->memory_size;
 334              phys_addr += TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
 335             unsigned char *bitmap = (unsigned char *)d.dirty_bitmap;
 336             unsigned nr = (phys_addr - mem->start_addr) >> TARGET_PAGE_BITS;
 337
 338             if (test_le_bit(nr, bitmap)) {
 339                 cpu_physical_memory_set_dirty(addr);
 340             }
 341         }
 342         start_addr = phys_addr;
 343     }
 344     qemu_free(d.dirty_bitmap);
 345
 346     return ret;
 347 }
 348 #endif
 349
 350 int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
 351 {
 352     int ret = -ENOSYS;
 353 #ifdef KVM_CAP_COALESCED_MMIO
 354     KVMState *s = kvm_state;
 355
 356     if (s->coalesced_mmio) {
 357         struct kvm_coalesced_mmio_zone zone;
 358
 359         zone.addr = start;
 360         zone.size = size;
 361
 362         ret = kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
 363     }
 364 #endif
 365
 366     return ret;
 367 }
 368
 369 int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
 370 {
 371     int ret = -ENOSYS;
 372 #ifdef KVM_CAP_COALESCED_MMIO
 373     KVMState *s = kvm_state;
 374
 375     if (s->coalesced_mmio) {
 376         struct kvm_coalesced_mmio_zone zone;
 377
 378         zone.addr = start;
 379         zone.size = size;
 380
 381         ret = kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
 382     }
 383 #endif
 384
 385     return ret;
 386 }
 387
 388 int kvm_check_extension(KVMState *s, unsigned int extension)
 389 {
 390     int ret;
 391
 392     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
 393     if (ret < 0) {
 394         ret = 0;
 395     }
 396
 397     return ret;
 398 }
 399 #ifdef KVM_UPSTREAM
 400
 401 int kvm_init(int smp_cpus)
 402 {
 403     static const char upgrade_note[] =
 404         "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
 405         "(see http://sourceforge.net/projects/kvm).\n";
 406     KVMState *s;
 407     int ret;
 408     int i;
 409
 410     if (smp_cpus > 1) {
 411         fprintf(stderr, "No SMP KVM support, use '-smp 1'\n");
 412         return -EINVAL;
 413     }
 414
 415     s = qemu_mallocz(sizeof(KVMState));
 416
 417 #ifdef KVM_CAP_SET_GUEST_DEBUG
 418     QTAILQ_INIT(&s->kvm_sw_breakpoints);
 419 #endif
 420     for (i = 0; i < ARRAY_SIZE(s->slots); i++)
 421         s->slots[i].slot = i;
 422
 423     s->vmfd = -1;
 424     s->fd = qemu_open("/dev/kvm", O_RDWR);
 425     if (s->fd == -1) {
 426         fprintf(stderr, "Could not access KVM kernel module: %m\n");
 427         ret = -errno;
 428         goto err;
 429     }
 430
 431     ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
 432     if (ret < KVM_API_VERSION) {
 433         if (ret > 0)
 434             ret = -EINVAL;
 435         fprintf(stderr, "kvm version too old\n");
 436         goto err;
 437     }
 438
 439     if (ret > KVM_API_VERSION) {
 440         ret = -EINVAL;
 441         fprintf(stderr, "kvm version not supported\n");
 442         goto err;
 443     }
 444
 445     s->vmfd = kvm_ioctl(s, KVM_CREATE_VM, 0);
 446     if (s->vmfd < 0)
 447         goto err;
 448
 449     /* initially, KVM allocated its own memory and we had to jump through
 450      * hooks to make phys_ram_base point to this.  Modern versions of KVM
 451      * just use a user allocated buffer so we can use regular pages
 452      * unmodified.  Make sure we have a sufficiently modern version of KVM.
 453      */
 454     if (!kvm_check_extension(s, KVM_CAP_USER_MEMORY)) {
 455         ret = -EINVAL;
 456         fprintf(stderr, "kvm does not support KVM_CAP_USER_MEMORY\n%s",
 457                 upgrade_note);
 458         goto err;
 459     }
 460
 461     /* There was a nasty bug in < kvm-80 that prevents memory slots from being
 462      * destroyed properly.  Since we rely on this capability, refuse to work
 463      * with any kernel without this capability. */
 464     if (!kvm_check_extension(s, KVM_CAP_DESTROY_MEMORY_REGION_WORKS)) {
 465         ret = -EINVAL;
 466
 467         fprintf(stderr,
 468                 "KVM kernel module broken (DESTROY_MEMORY_REGION).\n%s",
 469                 upgrade_note);
 470         goto err;
 471     }
 472
 473 #ifdef KVM_CAP_COALESCED_MMIO
 474     s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
 475 #else
 476     s->coalesced_mmio = 0;
 477 #endif
 478
 479     s->broken_set_mem_region = 1;
 480 #ifdef KVM_CAP_JOIN_MEMORY_REGIONS_WORKS
 481     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
 482     if (ret > 0) {
 483         s->broken_set_mem_region = 0;
 484     }
 485 #endif
 486
 487     s->vcpu_events = 0;
 488 #ifdef KVM_CAP_VCPU_EVENTS
 489     s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
 490 #endif
 491
 492     ret = kvm_arch_init(s, smp_cpus);
 493     if (ret < 0)
 494         goto err;
 495
 496     kvm_state = s;
 497
 498     return 0;
 499
 500 err:
 501     if (s) {
 502         if (s->vmfd != -1)
 503             close(s->vmfd);
 504         if (s->fd != -1)
 505             close(s->fd);
 506     }
 507     qemu_free(s);
 508
 509     return ret;
 510 }
 511 #endif
 512
 513 static int kvm_handle_io(uint16_t port, void *data, int direction, int size,
 514                          uint32_t count)
 515 {
 516     int i;
 517     uint8_t *ptr = data;
 518
 519     for (i = 0; i < count; i++) {
 520         if (direction == KVM_EXIT_IO_IN) {
 521             switch (size) {
 522             case 1:
 523                 stb_p(ptr, cpu_inb(port));
 524                 break;
 525             case 2:
 526                 stw_p(ptr, cpu_inw(port));
 527                 break;
 528             case 4:
 529                 stl_p(ptr, cpu_inl(port));
 530                 break;
 531             }
 532         } else {
 533             switch (size) {
 534             case 1:
 535                 cpu_outb(port, ldub_p(ptr));
 536                 break;
 537             case 2:
 538                 cpu_outw(port, lduw_p(ptr));
 539                 break;
 540             case 4:
 541                 cpu_outl(port, ldl_p(ptr));
 542                 break;
 543             }
 544         }
 545
 546         ptr += size;
 547     }
 548
 549     return 1;
 550 }
 551
 552 #ifdef KVM_UPSTREAM
 553 static void kvm_run_coalesced_mmio(CPUState *env, struct kvm_run *run)
 554 {
 555 #ifdef KVM_CAP_COALESCED_MMIO
 556     KVMState *s = kvm_state;
 557     if (s->coalesced_mmio) {
 558         struct kvm_coalesced_mmio_ring *ring;
 559
 560         ring = (void *)run + (s->coalesced_mmio * TARGET_PAGE_SIZE);
 561         while (ring->first != ring->last) {
 562             struct kvm_coalesced_mmio *ent;
 563
 564             ent = &ring->coalesced_mmio[ring->first];
 565
 566             cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
 567             /* FIXME smp_wmb() */
 568             ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
 569         }
 570     }
 571 #endif
 572 }
 573
 574 void kvm_cpu_synchronize_state(CPUState *env)
 575 {
 576     if (!env->kvm_state->regs_modified) {
 577         kvm_arch_get_registers(env);
 578         env->kvm_state->regs_modified = 1;
 579     }
 580 }
 581
 582 int kvm_cpu_exec(CPUState *env)
 583 {
 584     struct kvm_run *run = env->kvm_run;
 585     int ret;
 586
 587     dprintf("kvm_cpu_exec()\n");
 588
 589     do {
 590         if (env->exit_request) {
 591             dprintf("interrupt exit requested\n");
 592             ret = 0;
 593             break;
 594         }
 595
 596         if (env->kvm_state->regs_modified) {
 597             kvm_arch_put_registers(env);
 598             env->kvm_state->regs_modified = 0;
 599         }
 600
 601         kvm_arch_pre_run(env, run);
 602         qemu_mutex_unlock_iothread();
 603         ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);
 604         qemu_mutex_lock_iothread();
 605         kvm_arch_post_run(env, run);
 606
 607         if (ret == -EINTR || ret == -EAGAIN) {
 608             dprintf("io window exit\n");
 609             ret = 0;
 610             break;
 611         }
 612
 613         if (ret < 0) {
 614             dprintf("kvm run failed %s\n", strerror(-ret));
 615             abort();
 616         }
 617
 618         kvm_run_coalesced_mmio(env, run);
 619
 620         ret = 0; /* exit loop */
 621         switch (run->exit_reason) {
 622         case KVM_EXIT_IO:
 623             dprintf("handle_io\n");
 624             ret = kvm_handle_io(run->io.port,
 625                                 (uint8_t *)run + run->io.data_offset,
 626                                 run->io.direction,
 627                                 run->io.size,
 628                                 run->io.count);
 629             break;
 630         case KVM_EXIT_MMIO:
 631             dprintf("handle_mmio\n");
 632             cpu_physical_memory_rw(run->mmio.phys_addr,
 633                                    run->mmio.data,
 634                                    run->mmio.len,
 635                                    run->mmio.is_write);
 636             ret = 1;
 637             break;
 638         case KVM_EXIT_IRQ_WINDOW_OPEN:
 639             dprintf("irq_window_open\n");
 640             break;
 641         case KVM_EXIT_SHUTDOWN:
 642             dprintf("shutdown\n");
 643             qemu_system_reset_request();
 644             ret = 1;
 645             break;
 646         case KVM_EXIT_UNKNOWN:
 647             dprintf("kvm_exit_unknown\n");
 648             break;
 649         case KVM_EXIT_FAIL_ENTRY:
 650             dprintf("kvm_exit_fail_entry\n");
 651             break;
 652         case KVM_EXIT_EXCEPTION:
 653             dprintf("kvm_exit_exception\n");
 654             break;
 655         case KVM_EXIT_DEBUG:
 656             dprintf("kvm_exit_debug\n");
 657 #ifdef KVM_CAP_SET_GUEST_DEBUG
 658             if (kvm_arch_debug(&run->debug.arch)) {
 659                 gdb_set_stop_cpu(env);
 660                 vm_stop(EXCP_DEBUG);
 661                 env->exception_index = EXCP_DEBUG;
 662                 return 0;
 663             }
 664             /* re-enter, this exception was guest-internal */
 665             ret = 1;
 666 #endif /* KVM_CAP_SET_GUEST_DEBUG */
 667             break;
 668         default:
 669             dprintf("kvm_arch_handle_exit\n");
 670             ret = kvm_arch_handle_exit(env, run);
 671             break;
 672         }
 673     } while (ret > 0);
 674
 675     if (env->exit_request) {
 676         env->exit_request = 0;
 677         env->exception_index = EXCP_INTERRUPT;
 678     }
 679
 680     return ret;
 681 }
 682
 683 void kvm_set_phys_mem(target_phys_addr_t start_addr,
 684                       ram_addr_t size,
 685                       ram_addr_t phys_offset)
 686 {
 687     KVMState *s = kvm_state;
 688     ram_addr_t flags = phys_offset & ~TARGET_PAGE_MASK;
 689     KVMSlot *mem, old;
 690     int err;
 691
 692     if (start_addr & ~TARGET_PAGE_MASK) {
 693         if (flags >= IO_MEM_UNASSIGNED) {
 694             if (!kvm_lookup_overlapping_slot(s, start_addr,
 695                                              start_addr + size)) {
 696                 return;
 697             }
 698             fprintf(stderr, "Unaligned split of a KVM memory slot\n");
 699         } else {
 700             fprintf(stderr, "Only page-aligned memory slots supported\n");
 701         }
 702         abort();
 703     }
 704
 705     /* KVM does not support read-only slots */
 706     phys_offset &= ~IO_MEM_ROM;
 707
 708     while (1) {
 709         mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size);
 710         if (!mem) {
 711             break;
 712         }
 713
 714         if (flags < IO_MEM_UNASSIGNED && start_addr >= mem->start_addr &&
 715             (start_addr + size <= mem->start_addr + mem->memory_size) &&
 716             (phys_offset - start_addr == mem->phys_offset - mem->start_addr)) {
 717             /* The new slot fits into the existing one and comes with
 718              * identical parameters - nothing to be done. */
 719             return;
 720         }
 721
 722         old = *mem;
 723
 724         /* unregister the overlapping slot */
 725         mem->memory_size = 0;
 726         err = kvm_set_user_memory_region(s, mem);
 727         if (err) {
 728             fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
 729                     __func__, strerror(-err));
 730             abort();
 731         }
 732
 733         /* Workaround for older KVM versions: we can't join slots, even not by
 734          * unregistering the previous ones and then registering the larger
 735          * slot. We have to maintain the existing fragmentation. Sigh.
 736          *
 737          * This workaround assumes that the new slot starts at the same
 738          * address as the first existing one. If not or if some overlapping
 739          * slot comes around later, we will fail (not seen in practice so far)
 740          * - and actually require a recent KVM version. */
 741         if (s->broken_set_mem_region &&
 742             old.start_addr == start_addr && old.memory_size < size &&
 743             flags < IO_MEM_UNASSIGNED) {
 744             mem = kvm_alloc_slot(s);
 745             mem->memory_size = old.memory_size;
 746             mem->start_addr = old.start_addr;
 747             mem->phys_offset = old.phys_offset;
 748             mem->flags = 0;
 749
 750             err = kvm_set_user_memory_region(s, mem);
 751             if (err) {
 752                 fprintf(stderr, "%s: error updating slot: %s\n", __func__,
 753                         strerror(-err));
 754                 abort();
 755             }
 756
 757             start_addr += old.memory_size;
 758             phys_offset += old.memory_size;
 759             size -= old.memory_size;
 760             continue;
 761         }
 762
 763         /* register prefix slot */
 764         if (old.start_addr < start_addr) {
 765             mem = kvm_alloc_slot(s);
 766             mem->memory_size = start_addr - old.start_addr;
 767             mem->start_addr = old.start_addr;
 768             mem->phys_offset = old.phys_offset;
 769             mem->flags = 0;
 770
 771             err = kvm_set_user_memory_region(s, mem);
 772             if (err) {
 773                 fprintf(stderr, "%s: error registering prefix slot: %s\n",
 774                         __func__, strerror(-err));
 775                 abort();
 776             }
 777         }
 778
 779         /* register suffix slot */
 780         if (old.start_addr + old.memory_size > start_addr + size) {
 781             ram_addr_t size_delta;
 782
 783             mem = kvm_alloc_slot(s);
 784             mem->start_addr = start_addr + size;
 785             size_delta = mem->start_addr - old.start_addr;
 786             mem->memory_size = old.memory_size - size_delta;
 787             mem->phys_offset = old.phys_offset + size_delta;
 788             mem->flags = 0;
 789
 790             err = kvm_set_user_memory_region(s, mem);
 791             if (err) {
 792                 fprintf(stderr, "%s: error registering suffix slot: %s\n",
 793                         __func__, strerror(-err));
 794                 abort();
 795             }
 796         }
 797     }
 798
 799     /* in case the KVM bug workaround already "consumed" the new slot */
 800     if (!size)
 801         return;
 802
 803     /* KVM does not need to know about this memory */
 804     if (flags >= IO_MEM_UNASSIGNED)
 805         return;
 806
 807     mem = kvm_alloc_slot(s);
 808     mem->memory_size = size;
 809     mem->start_addr = start_addr;
 810     mem->phys_offset = phys_offset;
 811     mem->flags = 0;
 812
 813     err = kvm_set_user_memory_region(s, mem);
 814     if (err) {
 815         fprintf(stderr, "%s: error registering slot: %s\n", __func__,
 816                 strerror(-err));
 817         abort();
 818     }
 819 }
 820
 821 #endif
 822 int kvm_ioctl(KVMState *s, int type, ...)
 823 {
 824     int ret;
 825     void *arg;
 826     va_list ap;
 827
 828     va_start(ap, type);
 829     arg = va_arg(ap, void *);
 830     va_end(ap);
 831
 832     ret = ioctl(s->fd, type, arg);
 833     if (ret == -1)
 834         ret = -errno;
 835
 836     return ret;
 837 }
 838
 839 int kvm_vm_ioctl(KVMState *s, int type, ...)
 840 {
 841     int ret;
 842     void *arg;
 843     va_list ap;
 844
 845     va_start(ap, type);
 846     arg = va_arg(ap, void *);
 847     va_end(ap);
 848
 849     ret = ioctl(s->vmfd, type, arg);
 850     if (ret == -1)
 851         ret = -errno;
 852
 853     return ret;
 854 }
 855
 856 int kvm_vcpu_ioctl(CPUState *env, int type, ...)
 857 {
 858     int ret;
 859     void *arg;
 860     va_list ap;
 861
 862     va_start(ap, type);
 863     arg = va_arg(ap, void *);
 864     va_end(ap);
 865
 866     ret = ioctl(env->kvm_fd, type, arg);
 867     if (ret == -1)
 868         ret = -errno;
 869
 870     return ret;
 871 }
 872
 873 int kvm_has_sync_mmu(void)
 874 {
 875 #ifdef KVM_CAP_SYNC_MMU
 876     KVMState *s = kvm_state;
 877
 878     return kvm_check_extension(s, KVM_CAP_SYNC_MMU);
 879 #else
 880     return 0;
 881 #endif
 882 }
 883
 884 int kvm_has_vcpu_events(void)
 885 {
 886     return kvm_state->vcpu_events;
 887 }
 888
 889 #ifdef KVM_UPSTREAM
 890 void kvm_setup_guest_memory(void *start, size_t size)
 891 {
 892     if (!kvm_has_sync_mmu()) {
 893 #ifdef MADV_DONTFORK
 894         int ret = madvise(start, size, MADV_DONTFORK);
 895
 896         if (ret) {
 897             perror("madvice");
 898             exit(1);
 899         }
 900 #else
 901         fprintf(stderr,
 902                 "Need MADV_DONTFORK in absence of synchronous KVM MMU\n");
 903         exit(1);
 904 #endif
 905     }
 906 }
 907
 908 #endif /* KVM_UPSTREAM */
 909
 910 #ifdef KVM_CAP_SET_GUEST_DEBUG
 911
 912 #ifdef KVM_UPSTREAM
 913 static void on_vcpu(CPUState *env, void (*func)(void *data), void *data)
 914 {
 915 #ifdef CONFIG_IOTHREAD
 916     if (env == cpu_single_env) {
 917         func(data);
 918         return;
 919     }
 920     abort();
 921 #else
 922     func(data);
 923 #endif
 924 }
 925 #endif /* KVM_UPSTREAM */
 926
 927 struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *env,
 928                                                  target_ulong pc)
 929 {
 930     struct kvm_sw_breakpoint *bp;
 931
 932     QTAILQ_FOREACH(bp, &env->kvm_state->kvm_sw_breakpoints, entry) {
 933         if (bp->pc == pc)
 934             return bp;
 935     }
 936     return NULL;
 937 }
 938
 939 int kvm_sw_breakpoints_active(CPUState *env)
 940 {
 941     return !QTAILQ_EMPTY(&env->kvm_state->kvm_sw_breakpoints);
 942 }
 943
 944 #ifdef KVM_UPSTREAM
 945
 946 struct kvm_set_guest_debug_data {
 947     struct kvm_guest_debug dbg;
 948     CPUState *env;
 949     int err;
 950 };
 951
 952 static void kvm_invoke_set_guest_debug(void *data)
 953 {
 954     struct kvm_set_guest_debug_data *dbg_data = data;
 955     CPUState *env = dbg_data->env;
 956
 957     if (env->kvm_state->regs_modified) {
 958         kvm_arch_put_registers(env);
 959         env->kvm_state->regs_modified = 0;
 960     }
 961     dbg_data->err = kvm_vcpu_ioctl(env, KVM_SET_GUEST_DEBUG, &dbg_data->dbg);
 962 }
 963
 964 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
 965 {
 966     struct kvm_set_guest_debug_data data;
 967
 968     data.dbg.control = 0;
 969     if (env->singlestep_enabled)
 970         data.dbg.control = KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
 971
 972     kvm_arch_update_guest_debug(env, &data.dbg);
 973     data.dbg.control |= reinject_trap;
 974     data.env = env;
 975
 976     on_vcpu(env, kvm_invoke_set_guest_debug, &data);
 977     return data.err;
 978 }
 979 #endif
 980
 981 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
 982                           target_ulong len, int type)
 983 {
 984     struct kvm_sw_breakpoint *bp;
 985     CPUState *env;
 986     int err;
 987
 988     if (type == GDB_BREAKPOINT_SW) {
 989         bp = kvm_find_sw_breakpoint(current_env, addr);
 990         if (bp) {
 991             bp->use_count++;
 992             return 0;
 993         }
 994
 995         bp = qemu_malloc(sizeof(struct kvm_sw_breakpoint));
 996         if (!bp)
 997             return -ENOMEM;
 998
 999         bp->pc = addr;
1000         bp->use_count = 1;
1001         err = kvm_arch_insert_sw_breakpoint(current_env, bp);
1002         if (err) {
1003             free(bp);
1004             return err;
1005         }
1006
1007         QTAILQ_INSERT_HEAD(&current_env->kvm_state->kvm_sw_breakpoints,
1008                           bp, entry);
1009     } else {
1010         err = kvm_arch_insert_hw_breakpoint(addr, len, type);
1011         if (err)
1012             return err;
1013     }
1014
1015     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1016         err = kvm_update_guest_debug(env, 0);
1017         if (err)
1018             return err;
1019     }
1020     return 0;
1021 }
1022
1023 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1024                           target_ulong len, int type)
1025 {
1026     struct kvm_sw_breakpoint *bp;
1027     CPUState *env;
1028     int err;
1029
1030     if (type == GDB_BREAKPOINT_SW) {
1031         bp = kvm_find_sw_breakpoint(current_env, addr);
1032         if (!bp)
1033             return -ENOENT;
1034
1035         if (bp->use_count > 1) {
1036             bp->use_count--;
1037             return 0;
1038         }
1039
1040         err = kvm_arch_remove_sw_breakpoint(current_env, bp);
1041         if (err)
1042             return err;
1043
1044         QTAILQ_REMOVE(&current_env->kvm_state->kvm_sw_breakpoints, bp, entry);
1045         qemu_free(bp);
1046     } else {
1047         err = kvm_arch_remove_hw_breakpoint(addr, len, type);
1048         if (err)
1049             return err;
1050     }
1051
1052     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1053         err = kvm_update_guest_debug(env, 0);
1054         if (err)
1055             return err;
1056     }
1057     return 0;
1058 }
1059
1060 void kvm_remove_all_breakpoints(CPUState *current_env)
1061 {
1062     struct kvm_sw_breakpoint *bp, *next;
1063     KVMState *s = current_env->kvm_state;
1064     CPUState *env;
1065
1066     QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
1067         if (kvm_arch_remove_sw_breakpoint(current_env, bp) != 0) {
1068             /* Try harder to find a CPU that currently sees the breakpoint. */
1069             for (env = first_cpu; env != NULL; env = env->next_cpu) {
1070                 if (kvm_arch_remove_sw_breakpoint(env, bp) == 0)
1071                     break;
1072             }
1073         }
1074     }
1075     kvm_arch_remove_all_hw_breakpoints();
1076
1077     for (env = first_cpu; env != NULL; env = env->next_cpu)
1078         kvm_update_guest_debug(env, 0);
1079 }
1080
1081 #else /* !KVM_CAP_SET_GUEST_DEBUG */
1082
1083 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1084 {
1085     return -EINVAL;
1086 }
1087
1088 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1089                           target_ulong len, int type)
1090 {
1091     return -EINVAL;
1092 }
1093
1094 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1095                           target_ulong len, int type)
1096 {
1097     return -EINVAL;
1098 }
1099
1100 void kvm_remove_all_breakpoints(CPUState *current_env)
1101 {
1102 }
1103 #endif /* !KVM_CAP_SET_GUEST_DEBUG */
1104
1105 #include "qemu-kvm.c"