kvm-all.c

   1 /*
   2  * QEMU KVM support
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *           Red Hat, Inc. 2008
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *  Glauber Costa     <gcosta@redhat.com>
  10  *
  11  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  12  * See the COPYING file in the top-level directory.
  13  *
  14  */
  15
  16 #include <sys/types.h>
  17 #include <sys/ioctl.h>
  18 #include <sys/mman.h>
  19 #include <stdarg.h>
  20
  21 #include <linux/kvm.h>
  22
  23 #include "qemu-common.h"
  24 #include "qemu-barrier.h"
  25 #include "sysemu.h"
  26 #include "hw/hw.h"
  27 #include "gdbstub.h"
  28 #include "kvm.h"
  29 #include "bswap.h"
  30
  31 /* This check must be after config-host.h is included */
  32 #ifdef CONFIG_EVENTFD
  33 #include <sys/eventfd.h>
  34 #endif
  35
  36 /* KVM uses PAGE_SIZE in it's definition of COALESCED_MMIO_MAX */
  37 #define PAGE_SIZE TARGET_PAGE_SIZE
  38
  39 //#define DEBUG_KVM
  40
  41 #ifdef DEBUG_KVM
  42 #define DPRINTF(fmt, ...) \
  43     do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
  44 #else
  45 #define DPRINTF(fmt, ...) \
  46     do { } while (0)
  47 #endif
  48
  49 typedef struct KVMSlot
  50 {
  51     target_phys_addr_t start_addr;
  52     ram_addr_t memory_size;
  53     ram_addr_t phys_offset;
  54     int slot;
  55     int flags;
  56 } KVMSlot;
  57
  58 typedef struct kvm_dirty_log KVMDirtyLog;
  59
  60 struct KVMState
  61 {
  62     KVMSlot slots[32];
  63     int fd;
  64     int vmfd;
  65     int coalesced_mmio;
  66 #ifdef KVM_CAP_COALESCED_MMIO
  67     struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
  68 #endif
  69     int broken_set_mem_region;
  70     int migration_log;
  71     int vcpu_events;
  72     int robust_singlestep;
  73     int debugregs;
  74 #ifdef KVM_CAP_SET_GUEST_DEBUG
  75     struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
  76 #endif
  77     int irqchip_in_kernel;
  78     int pit_in_kernel;
  79     int xsave, xcrs;
  80     int many_ioeventfds;
  81 };
  82
  83 static KVMState *kvm_state;
  84
  85 static KVMSlot *kvm_alloc_slot(KVMState *s)
  86 {
  87     int i;
  88
  89     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
  90         /* KVM private memory slots */
  91         if (i >= 8 && i < 12)
  92             continue;
  93         if (s->slots[i].memory_size == 0)
  94             return &s->slots[i];
  95     }
  96
  97     fprintf(stderr, "%s: no free slot available\n", __func__);
  98     abort();
  99 }
 100
 101 static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
 102                                          target_phys_addr_t start_addr,
 103                                          target_phys_addr_t end_addr)
 104 {
 105     int i;
 106
 107     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 108         KVMSlot *mem = &s->slots[i];
 109
 110         if (start_addr == mem->start_addr &&
 111             end_addr == mem->start_addr + mem->memory_size) {
 112             return mem;
 113         }
 114     }
 115
 116     return NULL;
 117 }
 118
 119 /*
 120  * Find overlapping slot with lowest start address
 121  */
 122 static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
 123                                             target_phys_addr_t start_addr,
 124                                             target_phys_addr_t end_addr)
 125 {
 126     KVMSlot *found = NULL;
 127     int i;
 128
 129     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 130         KVMSlot *mem = &s->slots[i];
 131
 132         if (mem->memory_size == 0 ||
 133             (found && found->start_addr < mem->start_addr)) {
 134             continue;
 135         }
 136
 137         if (end_addr > mem->start_addr &&
 138             start_addr < mem->start_addr + mem->memory_size) {
 139             found = mem;
 140         }
 141     }
 142
 143     return found;
 144 }
 145
 146 int kvm_physical_memory_addr_from_ram(KVMState *s, ram_addr_t ram_addr,
 147                                       target_phys_addr_t *phys_addr)
 148 {
 149     int i;
 150
 151     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 152         KVMSlot *mem = &s->slots[i];
 153
 154         if (ram_addr >= mem->phys_offset &&
 155             ram_addr < mem->phys_offset + mem->memory_size) {
 156             *phys_addr = mem->start_addr + (ram_addr - mem->phys_offset);
 157             return 1;
 158         }
 159     }
 160
 161     return 0;
 162 }
 163
 164 static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
 165 {
 166     struct kvm_userspace_memory_region mem;
 167
 168     mem.slot = slot->slot;
 169     mem.guest_phys_addr = slot->start_addr;
 170     mem.memory_size = slot->memory_size;
 171     mem.userspace_addr = (unsigned long)qemu_safe_ram_ptr(slot->phys_offset);
 172     mem.flags = slot->flags;
 173     if (s->migration_log) {
 174         mem.flags |= KVM_MEM_LOG_DIRTY_PAGES;
 175     }
 176     return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
 177 }
 178
 179 static void kvm_reset_vcpu(void *opaque)
 180 {
 181     CPUState *env = opaque;
 182
 183     kvm_arch_reset_vcpu(env);
 184 }
 185
 186 int kvm_irqchip_in_kernel(void)
 187 {
 188     return kvm_state->irqchip_in_kernel;
 189 }
 190
 191 int kvm_pit_in_kernel(void)
 192 {
 193     return kvm_state->pit_in_kernel;
 194 }
 195
 196
 197 int kvm_init_vcpu(CPUState *env)
 198 {
 199     KVMState *s = kvm_state;
 200     long mmap_size;
 201     int ret;
 202
 203     DPRINTF("kvm_init_vcpu\n");
 204
 205     ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, env->cpu_index);
 206     if (ret < 0) {
 207         DPRINTF("kvm_create_vcpu failed\n");
 208         goto err;
 209     }
 210
 211     env->kvm_fd = ret;
 212     env->kvm_state = s;
 213
 214     mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
 215     if (mmap_size < 0) {
 216         DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
 217         goto err;
 218     }
 219
 220     env->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
 221                         env->kvm_fd, 0);
 222     if (env->kvm_run == MAP_FAILED) {
 223         ret = -errno;
 224         DPRINTF("mmap'ing vcpu state failed\n");
 225         goto err;
 226     }
 227
 228 #ifdef KVM_CAP_COALESCED_MMIO
 229     if (s->coalesced_mmio && !s->coalesced_mmio_ring)
 230         s->coalesced_mmio_ring = (void *) env->kvm_run +
 231                 s->coalesced_mmio * PAGE_SIZE;
 232 #endif
 233
 234     ret = kvm_arch_init_vcpu(env);
 235     if (ret == 0) {
 236         qemu_register_reset(kvm_reset_vcpu, env);
 237         kvm_arch_reset_vcpu(env);
 238     }
 239 err:
 240     return ret;
 241 }
 242
 243 /*
 244  * dirty pages logging control
 245  */
 246 static int kvm_dirty_pages_log_change(target_phys_addr_t phys_addr,
 247                                       ram_addr_t size, int flags, int mask)
 248 {
 249     KVMState *s = kvm_state;
 250     KVMSlot *mem = kvm_lookup_matching_slot(s, phys_addr, phys_addr + size);
 251     int old_flags;
 252
 253     if (mem == NULL)  {
 254             fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
 255                     TARGET_FMT_plx "\n", __func__, phys_addr,
 256                     (target_phys_addr_t)(phys_addr + size - 1));
 257             return -EINVAL;
 258     }
 259
 260     old_flags = mem->flags;
 261
 262     flags = (mem->flags & ~mask) | flags;
 263     mem->flags = flags;
 264
 265     /* If nothing changed effectively, no need to issue ioctl */
 266     if (s->migration_log) {
 267         flags |= KVM_MEM_LOG_DIRTY_PAGES;
 268     }
 269     if (flags == old_flags) {
 270             return 0;
 271     }
 272
 273     return kvm_set_user_memory_region(s, mem);
 274 }
 275
 276 int kvm_log_start(target_phys_addr_t phys_addr, ram_addr_t size)
 277 {
 278         return kvm_dirty_pages_log_change(phys_addr, size,
 279                                           KVM_MEM_LOG_DIRTY_PAGES,
 280                                           KVM_MEM_LOG_DIRTY_PAGES);
 281 }
 282
 283 int kvm_log_stop(target_phys_addr_t phys_addr, ram_addr_t size)
 284 {
 285         return kvm_dirty_pages_log_change(phys_addr, size,
 286                                           0,
 287                                           KVM_MEM_LOG_DIRTY_PAGES);
 288 }
 289
 290 static int kvm_set_migration_log(int enable)
 291 {
 292     KVMState *s = kvm_state;
 293     KVMSlot *mem;
 294     int i, err;
 295
 296     s->migration_log = enable;
 297
 298     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 299         mem = &s->slots[i];
 300
 301         if (!mem->memory_size) {
 302             continue;
 303         }
 304         if (!!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) == enable) {
 305             continue;
 306         }
 307         err = kvm_set_user_memory_region(s, mem);
 308         if (err) {
 309             return err;
 310         }
 311     }
 312     return 0;
 313 }
 314
 315 /* get kvm's dirty pages bitmap and update qemu's */
 316 static int kvm_get_dirty_pages_log_range(unsigned long start_addr,
 317                                          unsigned long *bitmap,
 318                                          unsigned long offset,
 319                                          unsigned long mem_size)
 320 {
 321     unsigned int i, j;
 322     unsigned long page_number, addr, addr1, c;
 323     ram_addr_t ram_addr;
 324     unsigned int len = ((mem_size / TARGET_PAGE_SIZE) + HOST_LONG_BITS - 1) /
 325         HOST_LONG_BITS;
 326
 327     /*
 328      * bitmap-traveling is faster than memory-traveling (for addr...)
 329      * especially when most of the memory is not dirty.
 330      */
 331     for (i = 0; i < len; i++) {
 332         if (bitmap[i] != 0) {
 333             c = leul_to_cpu(bitmap[i]);
 334             do {
 335                 j = ffsl(c) - 1;
 336                 c &= ~(1ul << j);
 337                 page_number = i * HOST_LONG_BITS + j;
 338                 addr1 = page_number * TARGET_PAGE_SIZE;
 339                 addr = offset + addr1;
 340                 ram_addr = cpu_get_physical_page_desc(addr);
 341                 cpu_physical_memory_set_dirty(ram_addr);
 342             } while (c != 0);
 343         }
 344     }
 345     return 0;
 346 }
 347
 348 #define ALIGN(x, y)  (((x)+(y)-1) & ~((y)-1))
 349
 350 /**
 351  * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
 352  * This function updates qemu's dirty bitmap using cpu_physical_memory_set_dirty().
 353  * This means all bits are set to dirty.
 354  *
 355  * @start_add: start of logged region.
 356  * @end_addr: end of logged region.
 357  */
 358 static int kvm_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
 359                                           target_phys_addr_t end_addr)
 360 {
 361     KVMState *s = kvm_state;
 362     unsigned long size, allocated_size = 0;
 363     KVMDirtyLog d;
 364     KVMSlot *mem;
 365     int ret = 0;
 366
 367     d.dirty_bitmap = NULL;
 368     while (start_addr < end_addr) {
 369         mem = kvm_lookup_overlapping_slot(s, start_addr, end_addr);
 370         if (mem == NULL) {
 371             break;
 372         }
 373
 374         size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS), HOST_LONG_BITS) / 8;
 375         if (!d.dirty_bitmap) {
 376             d.dirty_bitmap = qemu_malloc(size);
 377         } else if (size > allocated_size) {
 378             d.dirty_bitmap = qemu_realloc(d.dirty_bitmap, size);
 379         }
 380         allocated_size = size;
 381         memset(d.dirty_bitmap, 0, allocated_size);
 382
 383         d.slot = mem->slot;
 384
 385         if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
 386             DPRINTF("ioctl failed %d\n", errno);
 387             ret = -1;
 388             break;
 389         }
 390
 391         kvm_get_dirty_pages_log_range(mem->start_addr, d.dirty_bitmap,
 392                                       mem->start_addr, mem->memory_size);
 393         start_addr = mem->start_addr + mem->memory_size;
 394     }
 395     qemu_free(d.dirty_bitmap);
 396
 397     return ret;
 398 }
 399
 400 int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
 401 {
 402     int ret = -ENOSYS;
 403 #ifdef KVM_CAP_COALESCED_MMIO
 404     KVMState *s = kvm_state;
 405
 406     if (s->coalesced_mmio) {
 407         struct kvm_coalesced_mmio_zone zone;
 408
 409         zone.addr = start;
 410         zone.size = size;
 411
 412         ret = kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
 413     }
 414 #endif
 415
 416     return ret;
 417 }
 418
 419 int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
 420 {
 421     int ret = -ENOSYS;
 422 #ifdef KVM_CAP_COALESCED_MMIO
 423     KVMState *s = kvm_state;
 424
 425     if (s->coalesced_mmio) {
 426         struct kvm_coalesced_mmio_zone zone;
 427
 428         zone.addr = start;
 429         zone.size = size;
 430
 431         ret = kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
 432     }
 433 #endif
 434
 435     return ret;
 436 }
 437
 438 int kvm_check_extension(KVMState *s, unsigned int extension)
 439 {
 440     int ret;
 441
 442     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
 443     if (ret < 0) {
 444         ret = 0;
 445     }
 446
 447     return ret;
 448 }
 449
 450 static int kvm_check_many_ioeventfds(void)
 451 {
 452     /* Userspace can use ioeventfd for io notification.  This requires a host
 453      * that supports eventfd(2) and an I/O thread; since eventfd does not
 454      * support SIGIO it cannot interrupt the vcpu.
 455      *
 456      * Older kernels have a 6 device limit on the KVM io bus.  Find out so we
 457      * can avoid creating too many ioeventfds.
 458      */
 459 #if defined(CONFIG_EVENTFD) && defined(CONFIG_IOTHREAD)
 460     int ioeventfds[7];
 461     int i, ret = 0;
 462     for (i = 0; i < ARRAY_SIZE(ioeventfds); i++) {
 463         ioeventfds[i] = eventfd(0, EFD_CLOEXEC);
 464         if (ioeventfds[i] < 0) {
 465             break;
 466         }
 467         ret = kvm_set_ioeventfd_pio_word(ioeventfds[i], 0, i, true);
 468         if (ret < 0) {
 469             close(ioeventfds[i]);
 470             break;
 471         }
 472     }
 473
 474     /* Decide whether many devices are supported or not */
 475     ret = i == ARRAY_SIZE(ioeventfds);
 476
 477     while (i-- > 0) {
 478         kvm_set_ioeventfd_pio_word(ioeventfds[i], 0, i, false);
 479         close(ioeventfds[i]);
 480     }
 481     return ret;
 482 #else
 483     return 0;
 484 #endif
 485 }
 486
 487 static void kvm_set_phys_mem(target_phys_addr_t start_addr,
 488                              ram_addr_t size,
 489                              ram_addr_t phys_offset)
 490 {
 491     KVMState *s = kvm_state;
 492     ram_addr_t flags = phys_offset & ~TARGET_PAGE_MASK;
 493     KVMSlot *mem, old;
 494     int err;
 495
 496     /* kvm works in page size chunks, but the function may be called
 497        with sub-page size and unaligned start address. */
 498     size = TARGET_PAGE_ALIGN(size);
 499     start_addr = TARGET_PAGE_ALIGN(start_addr);
 500
 501     /* KVM does not support read-only slots */
 502     phys_offset &= ~IO_MEM_ROM;
 503
 504     while (1) {
 505         mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size);
 506         if (!mem) {
 507             break;
 508         }
 509
 510         if (flags < IO_MEM_UNASSIGNED && start_addr >= mem->start_addr &&
 511             (start_addr + size <= mem->start_addr + mem->memory_size) &&
 512             (phys_offset - start_addr == mem->phys_offset - mem->start_addr)) {
 513             /* The new slot fits into the existing one and comes with
 514              * identical parameters - nothing to be done. */
 515             return;
 516         }
 517
 518         old = *mem;
 519
 520         /* unregister the overlapping slot */
 521         mem->memory_size = 0;
 522         err = kvm_set_user_memory_region(s, mem);
 523         if (err) {
 524             fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
 525                     __func__, strerror(-err));
 526             abort();
 527         }
 528
 529         /* Workaround for older KVM versions: we can't join slots, even not by
 530          * unregistering the previous ones and then registering the larger
 531          * slot. We have to maintain the existing fragmentation. Sigh.
 532          *
 533          * This workaround assumes that the new slot starts at the same
 534          * address as the first existing one. If not or if some overlapping
 535          * slot comes around later, we will fail (not seen in practice so far)
 536          * - and actually require a recent KVM version. */
 537         if (s->broken_set_mem_region &&
 538             old.start_addr == start_addr && old.memory_size < size &&
 539             flags < IO_MEM_UNASSIGNED) {
 540             mem = kvm_alloc_slot(s);
 541             mem->memory_size = old.memory_size;
 542             mem->start_addr = old.start_addr;
 543             mem->phys_offset = old.phys_offset;
 544             mem->flags = 0;
 545
 546             err = kvm_set_user_memory_region(s, mem);
 547             if (err) {
 548                 fprintf(stderr, "%s: error updating slot: %s\n", __func__,
 549                         strerror(-err));
 550                 abort();
 551             }
 552
 553             start_addr += old.memory_size;
 554             phys_offset += old.memory_size;
 555             size -= old.memory_size;
 556             continue;
 557         }
 558
 559         /* register prefix slot */
 560         if (old.start_addr < start_addr) {
 561             mem = kvm_alloc_slot(s);
 562             mem->memory_size = start_addr - old.start_addr;
 563             mem->start_addr = old.start_addr;
 564             mem->phys_offset = old.phys_offset;
 565             mem->flags = 0;
 566
 567             err = kvm_set_user_memory_region(s, mem);
 568             if (err) {
 569                 fprintf(stderr, "%s: error registering prefix slot: %s\n",
 570                         __func__, strerror(-err));
 571                 abort();
 572             }
 573         }
 574
 575         /* register suffix slot */
 576         if (old.start_addr + old.memory_size > start_addr + size) {
 577             ram_addr_t size_delta;
 578
 579             mem = kvm_alloc_slot(s);
 580             mem->start_addr = start_addr + size;
 581             size_delta = mem->start_addr - old.start_addr;
 582             mem->memory_size = old.memory_size - size_delta;
 583             mem->phys_offset = old.phys_offset + size_delta;
 584             mem->flags = 0;
 585
 586             err = kvm_set_user_memory_region(s, mem);
 587             if (err) {
 588                 fprintf(stderr, "%s: error registering suffix slot: %s\n",
 589                         __func__, strerror(-err));
 590                 abort();
 591             }
 592         }
 593     }
 594
 595     /* in case the KVM bug workaround already "consumed" the new slot */
 596     if (!size)
 597         return;
 598
 599     /* KVM does not need to know about this memory */
 600     if (flags >= IO_MEM_UNASSIGNED)
 601         return;
 602
 603     mem = kvm_alloc_slot(s);
 604     mem->memory_size = size;
 605     mem->start_addr = start_addr;
 606     mem->phys_offset = phys_offset;
 607     mem->flags = 0;
 608
 609     err = kvm_set_user_memory_region(s, mem);
 610     if (err) {
 611         fprintf(stderr, "%s: error registering slot: %s\n", __func__,
 612                 strerror(-err));
 613         abort();
 614     }
 615 }
 616
 617 static void kvm_client_set_memory(struct CPUPhysMemoryClient *client,
 618                                   target_phys_addr_t start_addr,
 619                                   ram_addr_t size,
 620                                   ram_addr_t phys_offset)
 621 {
 622         kvm_set_phys_mem(start_addr, size, phys_offset);
 623 }
 624
 625 static int kvm_client_sync_dirty_bitmap(struct CPUPhysMemoryClient *client,
 626                                         target_phys_addr_t start_addr,
 627                                         target_phys_addr_t end_addr)
 628 {
 629         return kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
 630 }
 631
 632 static int kvm_client_migration_log(struct CPUPhysMemoryClient *client,
 633                                     int enable)
 634 {
 635         return kvm_set_migration_log(enable);
 636 }
 637
 638 static CPUPhysMemoryClient kvm_cpu_phys_memory_client = {
 639         .set_memory = kvm_client_set_memory,
 640         .sync_dirty_bitmap = kvm_client_sync_dirty_bitmap,
 641         .migration_log = kvm_client_migration_log,
 642 };
 643
 644 int kvm_init(int smp_cpus)
 645 {
 646     static const char upgrade_note[] =
 647         "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
 648         "(see http://sourceforge.net/projects/kvm).\n";
 649     KVMState *s;
 650     int ret;
 651     int i;
 652
 653     s = qemu_mallocz(sizeof(KVMState));
 654
 655 #ifdef KVM_CAP_SET_GUEST_DEBUG
 656     QTAILQ_INIT(&s->kvm_sw_breakpoints);
 657 #endif
 658     for (i = 0; i < ARRAY_SIZE(s->slots); i++)
 659         s->slots[i].slot = i;
 660
 661     s->vmfd = -1;
 662     s->fd = qemu_open("/dev/kvm", O_RDWR);
 663     if (s->fd == -1) {
 664         fprintf(stderr, "Could not access KVM kernel module: %m\n");
 665         ret = -errno;
 666         goto err;
 667     }
 668
 669     ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
 670     if (ret < KVM_API_VERSION) {
 671         if (ret > 0)
 672             ret = -EINVAL;
 673         fprintf(stderr, "kvm version too old\n");
 674         goto err;
 675     }
 676
 677     if (ret > KVM_API_VERSION) {
 678         ret = -EINVAL;
 679         fprintf(stderr, "kvm version not supported\n");
 680         goto err;
 681     }
 682
 683     s->vmfd = kvm_ioctl(s, KVM_CREATE_VM, 0);
 684     if (s->vmfd < 0) {
 685 #ifdef TARGET_S390X
 686         fprintf(stderr, "Please add the 'switch_amode' kernel parameter to "
 687                         "your host kernel command line\n");
 688 #endif
 689         goto err;
 690     }
 691
 692     /* initially, KVM allocated its own memory and we had to jump through
 693      * hooks to make phys_ram_base point to this.  Modern versions of KVM
 694      * just use a user allocated buffer so we can use regular pages
 695      * unmodified.  Make sure we have a sufficiently modern version of KVM.
 696      */
 697     if (!kvm_check_extension(s, KVM_CAP_USER_MEMORY)) {
 698         ret = -EINVAL;
 699         fprintf(stderr, "kvm does not support KVM_CAP_USER_MEMORY\n%s",
 700                 upgrade_note);
 701         goto err;
 702     }
 703
 704     /* There was a nasty bug in < kvm-80 that prevents memory slots from being
 705      * destroyed properly.  Since we rely on this capability, refuse to work
 706      * with any kernel without this capability. */
 707     if (!kvm_check_extension(s, KVM_CAP_DESTROY_MEMORY_REGION_WORKS)) {
 708         ret = -EINVAL;
 709
 710         fprintf(stderr,
 711                 "KVM kernel module broken (DESTROY_MEMORY_REGION).\n%s",
 712                 upgrade_note);
 713         goto err;
 714     }
 715
 716     s->coalesced_mmio = 0;
 717 #ifdef KVM_CAP_COALESCED_MMIO
 718     s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
 719     s->coalesced_mmio_ring = NULL;
 720 #endif
 721
 722     s->broken_set_mem_region = 1;
 723 #ifdef KVM_CAP_JOIN_MEMORY_REGIONS_WORKS
 724     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
 725     if (ret > 0) {
 726         s->broken_set_mem_region = 0;
 727     }
 728 #endif
 729
 730     s->vcpu_events = 0;
 731 #ifdef KVM_CAP_VCPU_EVENTS
 732     s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
 733 #endif
 734
 735     s->robust_singlestep = 0;
 736 #ifdef KVM_CAP_X86_ROBUST_SINGLESTEP
 737     s->robust_singlestep =
 738         kvm_check_extension(s, KVM_CAP_X86_ROBUST_SINGLESTEP);
 739 #endif
 740
 741     s->debugregs = 0;
 742 #ifdef KVM_CAP_DEBUGREGS
 743     s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS);
 744 #endif
 745
 746     s->xsave = 0;
 747 #ifdef KVM_CAP_XSAVE
 748     s->xsave = kvm_check_extension(s, KVM_CAP_XSAVE);
 749 #endif
 750
 751     s->xcrs = 0;
 752 #ifdef KVM_CAP_XCRS
 753     s->xcrs = kvm_check_extension(s, KVM_CAP_XCRS);
 754 #endif
 755
 756     ret = kvm_arch_init(s, smp_cpus);
 757     if (ret < 0)
 758         goto err;
 759
 760     kvm_state = s;
 761     cpu_register_phys_memory_client(&kvm_cpu_phys_memory_client);
 762
 763     s->many_ioeventfds = kvm_check_many_ioeventfds();
 764
 765     return 0;
 766
 767 err:
 768     if (s) {
 769         if (s->vmfd != -1)
 770             close(s->vmfd);
 771         if (s->fd != -1)
 772             close(s->fd);
 773     }
 774     qemu_free(s);
 775
 776     return ret;
 777 }
 778
 779 static int kvm_handle_io(uint16_t port, void *data, int direction, int size,
 780                          uint32_t count)
 781 {
 782     int i;
 783     uint8_t *ptr = data;
 784
 785     for (i = 0; i < count; i++) {
 786         if (direction == KVM_EXIT_IO_IN) {
 787             switch (size) {
 788             case 1:
 789                 stb_p(ptr, cpu_inb(port));
 790                 break;
 791             case 2:
 792                 stw_p(ptr, cpu_inw(port));
 793                 break;
 794             case 4:
 795                 stl_p(ptr, cpu_inl(port));
 796                 break;
 797             }
 798         } else {
 799             switch (size) {
 800             case 1:
 801                 cpu_outb(port, ldub_p(ptr));
 802                 break;
 803             case 2:
 804                 cpu_outw(port, lduw_p(ptr));
 805                 break;
 806             case 4:
 807                 cpu_outl(port, ldl_p(ptr));
 808                 break;
 809             }
 810         }
 811
 812         ptr += size;
 813     }
 814
 815     return 1;
 816 }
 817
 818 #ifdef KVM_CAP_INTERNAL_ERROR_DATA
 819 static void kvm_handle_internal_error(CPUState *env, struct kvm_run *run)
 820 {
 821
 822     if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) {
 823         int i;
 824
 825         fprintf(stderr, "KVM internal error. Suberror: %d\n",
 826                 run->internal.suberror);
 827
 828         for (i = 0; i < run->internal.ndata; ++i) {
 829             fprintf(stderr, "extra data[%d]: %"PRIx64"\n",
 830                     i, (uint64_t)run->internal.data[i]);
 831         }
 832     }
 833     cpu_dump_state(env, stderr, fprintf, 0);
 834     if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) {
 835         fprintf(stderr, "emulation failure\n");
 836         if (!kvm_arch_stop_on_emulation_error(env))
 837                 return;
 838     }
 839     /* FIXME: Should trigger a qmp message to let management know
 840      * something went wrong.
 841      */
 842     vm_stop(0);
 843 }
 844 #endif
 845
 846 void kvm_flush_coalesced_mmio_buffer(void)
 847 {
 848 #ifdef KVM_CAP_COALESCED_MMIO
 849     KVMState *s = kvm_state;
 850     if (s->coalesced_mmio_ring) {
 851         struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
 852         while (ring->first != ring->last) {
 853             struct kvm_coalesced_mmio *ent;
 854
 855             ent = &ring->coalesced_mmio[ring->first];
 856
 857             cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
 858             smp_wmb();
 859             ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
 860         }
 861     }
 862 #endif
 863 }
 864
 865 static void do_kvm_cpu_synchronize_state(void *_env)
 866 {
 867     CPUState *env = _env;
 868
 869     if (!env->kvm_vcpu_dirty) {
 870         kvm_arch_get_registers(env);
 871         env->kvm_vcpu_dirty = 1;
 872     }
 873 }
 874
 875 void kvm_cpu_synchronize_state(CPUState *env)
 876 {
 877     if (!env->kvm_vcpu_dirty)
 878         run_on_cpu(env, do_kvm_cpu_synchronize_state, env);
 879 }
 880
 881 void kvm_cpu_synchronize_post_reset(CPUState *env)
 882 {
 883     kvm_arch_put_registers(env, KVM_PUT_RESET_STATE);
 884     env->kvm_vcpu_dirty = 0;
 885 }
 886
 887 void kvm_cpu_synchronize_post_init(CPUState *env)
 888 {
 889     kvm_arch_put_registers(env, KVM_PUT_FULL_STATE);
 890     env->kvm_vcpu_dirty = 0;
 891 }
 892
 893 int kvm_cpu_exec(CPUState *env)
 894 {
 895     struct kvm_run *run = env->kvm_run;
 896     int ret;
 897
 898     DPRINTF("kvm_cpu_exec()\n");
 899
 900     do {
 901 #ifndef CONFIG_IOTHREAD
 902         if (env->exit_request) {
 903             DPRINTF("interrupt exit requested\n");
 904             ret = 0;
 905             break;
 906         }
 907 #endif
 908
 909         if (kvm_arch_process_irqchip_events(env)) {
 910             ret = 0;
 911             break;
 912         }
 913
 914         if (env->kvm_vcpu_dirty) {
 915             kvm_arch_put_registers(env, KVM_PUT_RUNTIME_STATE);
 916             env->kvm_vcpu_dirty = 0;
 917         }
 918
 919         kvm_arch_pre_run(env, run);
 920         cpu_single_env = NULL;
 921         qemu_mutex_unlock_iothread();
 922         ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);
 923         qemu_mutex_lock_iothread();
 924         cpu_single_env = env;
 925         kvm_arch_post_run(env, run);
 926
 927         if (ret == -EINTR || ret == -EAGAIN) {
 928             cpu_exit(env);
 929             DPRINTF("io window exit\n");
 930             ret = 0;
 931             break;
 932         }
 933
 934         if (ret < 0) {
 935             DPRINTF("kvm run failed %s\n", strerror(-ret));
 936             abort();
 937         }
 938
 939         kvm_flush_coalesced_mmio_buffer();
 940
 941         ret = 0; /* exit loop */
 942         switch (run->exit_reason) {
 943         case KVM_EXIT_IO:
 944             DPRINTF("handle_io\n");
 945             ret = kvm_handle_io(run->io.port,
 946                                 (uint8_t *)run + run->io.data_offset,
 947                                 run->io.direction,
 948                                 run->io.size,
 949                                 run->io.count);
 950             break;
 951         case KVM_EXIT_MMIO:
 952             DPRINTF("handle_mmio\n");
 953             cpu_physical_memory_rw(run->mmio.phys_addr,
 954                                    run->mmio.data,
 955                                    run->mmio.len,
 956                                    run->mmio.is_write);
 957             ret = 1;
 958             break;
 959         case KVM_EXIT_IRQ_WINDOW_OPEN:
 960             DPRINTF("irq_window_open\n");
 961             break;
 962         case KVM_EXIT_SHUTDOWN:
 963             DPRINTF("shutdown\n");
 964             qemu_system_reset_request();
 965             ret = 1;
 966             break;
 967         case KVM_EXIT_UNKNOWN:
 968             DPRINTF("kvm_exit_unknown\n");
 969             break;
 970         case KVM_EXIT_FAIL_ENTRY:
 971             DPRINTF("kvm_exit_fail_entry\n");
 972             break;
 973         case KVM_EXIT_EXCEPTION:
 974             DPRINTF("kvm_exit_exception\n");
 975             break;
 976 #ifdef KVM_CAP_INTERNAL_ERROR_DATA
 977         case KVM_EXIT_INTERNAL_ERROR:
 978             kvm_handle_internal_error(env, run);
 979             break;
 980 #endif
 981         case KVM_EXIT_DEBUG:
 982             DPRINTF("kvm_exit_debug\n");
 983 #ifdef KVM_CAP_SET_GUEST_DEBUG
 984             if (kvm_arch_debug(&run->debug.arch)) {
 985                 env->exception_index = EXCP_DEBUG;
 986                 return 0;
 987             }
 988             /* re-enter, this exception was guest-internal */
 989             ret = 1;
 990 #endif /* KVM_CAP_SET_GUEST_DEBUG */
 991             break;
 992         default:
 993             DPRINTF("kvm_arch_handle_exit\n");
 994             ret = kvm_arch_handle_exit(env, run);
 995             break;
 996         }
 997     } while (ret > 0);
 998
 999     if (env->exit_request) {
1000         env->exit_request = 0;
1001         env->exception_index = EXCP_INTERRUPT;
1002     }
1003
1004     return ret;
1005 }
1006
1007 int kvm_ioctl(KVMState *s, int type, ...)
1008 {
1009     int ret;
1010     void *arg;
1011     va_list ap;
1012
1013     va_start(ap, type);
1014     arg = va_arg(ap, void *);
1015     va_end(ap);
1016
1017     ret = ioctl(s->fd, type, arg);
1018     if (ret == -1)
1019         ret = -errno;
1020
1021     return ret;
1022 }
1023
1024 int kvm_vm_ioctl(KVMState *s, int type, ...)
1025 {
1026     int ret;
1027     void *arg;
1028     va_list ap;
1029
1030     va_start(ap, type);
1031     arg = va_arg(ap, void *);
1032     va_end(ap);
1033
1034     ret = ioctl(s->vmfd, type, arg);
1035     if (ret == -1)
1036         ret = -errno;
1037
1038     return ret;
1039 }
1040
1041 int kvm_vcpu_ioctl(CPUState *env, int type, ...)
1042 {
1043     int ret;
1044     void *arg;
1045     va_list ap;
1046
1047     va_start(ap, type);
1048     arg = va_arg(ap, void *);
1049     va_end(ap);
1050
1051     ret = ioctl(env->kvm_fd, type, arg);
1052     if (ret == -1)
1053         ret = -errno;
1054
1055     return ret;
1056 }
1057
1058 int kvm_has_sync_mmu(void)
1059 {
1060 #ifdef KVM_CAP_SYNC_MMU
1061     KVMState *s = kvm_state;
1062
1063     return kvm_check_extension(s, KVM_CAP_SYNC_MMU);
1064 #else
1065     return 0;
1066 #endif
1067 }
1068
1069 int kvm_has_vcpu_events(void)
1070 {
1071     return kvm_state->vcpu_events;
1072 }
1073
1074 int kvm_has_robust_singlestep(void)
1075 {
1076     return kvm_state->robust_singlestep;
1077 }
1078
1079 int kvm_has_debugregs(void)
1080 {
1081     return kvm_state->debugregs;
1082 }
1083
1084 int kvm_has_xsave(void)
1085 {
1086     return kvm_state->xsave;
1087 }
1088
1089 int kvm_has_xcrs(void)
1090 {
1091     return kvm_state->xcrs;
1092 }
1093
1094 int kvm_has_many_ioeventfds(void)
1095 {
1096     if (!kvm_enabled()) {
1097         return 0;
1098     }
1099     return kvm_state->many_ioeventfds;
1100 }
1101
1102 void kvm_setup_guest_memory(void *start, size_t size)
1103 {
1104     if (!kvm_has_sync_mmu()) {
1105         int ret = qemu_madvise(start, size, QEMU_MADV_DONTFORK);
1106
1107         if (ret) {
1108             perror("qemu_madvise");
1109             fprintf(stderr,
1110                     "Need MADV_DONTFORK in absence of synchronous KVM MMU\n");
1111             exit(1);
1112         }
1113     }
1114 }
1115
1116 #ifdef KVM_CAP_SET_GUEST_DEBUG
1117 struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *env,
1118                                                  target_ulong pc)
1119 {
1120     struct kvm_sw_breakpoint *bp;
1121
1122     QTAILQ_FOREACH(bp, &env->kvm_state->kvm_sw_breakpoints, entry) {
1123         if (bp->pc == pc)
1124             return bp;
1125     }
1126     return NULL;
1127 }
1128
1129 int kvm_sw_breakpoints_active(CPUState *env)
1130 {
1131     return !QTAILQ_EMPTY(&env->kvm_state->kvm_sw_breakpoints);
1132 }
1133
1134 struct kvm_set_guest_debug_data {
1135     struct kvm_guest_debug dbg;
1136     CPUState *env;
1137     int err;
1138 };
1139
1140 static void kvm_invoke_set_guest_debug(void *data)
1141 {
1142     struct kvm_set_guest_debug_data *dbg_data = data;
1143     CPUState *env = dbg_data->env;
1144
1145     dbg_data->err = kvm_vcpu_ioctl(env, KVM_SET_GUEST_DEBUG, &dbg_data->dbg);
1146 }
1147
1148 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1149 {
1150     struct kvm_set_guest_debug_data data;
1151
1152     data.dbg.control = reinject_trap;
1153
1154     if (env->singlestep_enabled) {
1155         data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
1156     }
1157     kvm_arch_update_guest_debug(env, &data.dbg);
1158     data.env = env;
1159
1160     run_on_cpu(env, kvm_invoke_set_guest_debug, &data);
1161     return data.err;
1162 }
1163
1164 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1165                           target_ulong len, int type)
1166 {
1167     struct kvm_sw_breakpoint *bp;
1168     CPUState *env;
1169     int err;
1170
1171     if (type == GDB_BREAKPOINT_SW) {
1172         bp = kvm_find_sw_breakpoint(current_env, addr);
1173         if (bp) {
1174             bp->use_count++;
1175             return 0;
1176         }
1177
1178         bp = qemu_malloc(sizeof(struct kvm_sw_breakpoint));
1179         if (!bp)
1180             return -ENOMEM;
1181
1182         bp->pc = addr;
1183         bp->use_count = 1;
1184         err = kvm_arch_insert_sw_breakpoint(current_env, bp);
1185         if (err) {
1186             free(bp);
1187             return err;
1188         }
1189
1190         QTAILQ_INSERT_HEAD(&current_env->kvm_state->kvm_sw_breakpoints,
1191                           bp, entry);
1192     } else {
1193         err = kvm_arch_insert_hw_breakpoint(addr, len, type);
1194         if (err)
1195             return err;
1196     }
1197
1198     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1199         err = kvm_update_guest_debug(env, 0);
1200         if (err)
1201             return err;
1202     }
1203     return 0;
1204 }
1205
1206 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1207                           target_ulong len, int type)
1208 {
1209     struct kvm_sw_breakpoint *bp;
1210     CPUState *env;
1211     int err;
1212
1213     if (type == GDB_BREAKPOINT_SW) {
1214         bp = kvm_find_sw_breakpoint(current_env, addr);
1215         if (!bp)
1216             return -ENOENT;
1217
1218         if (bp->use_count > 1) {
1219             bp->use_count--;
1220             return 0;
1221         }
1222
1223         err = kvm_arch_remove_sw_breakpoint(current_env, bp);
1224         if (err)
1225             return err;
1226
1227         QTAILQ_REMOVE(&current_env->kvm_state->kvm_sw_breakpoints, bp, entry);
1228         qemu_free(bp);
1229     } else {
1230         err = kvm_arch_remove_hw_breakpoint(addr, len, type);
1231         if (err)
1232             return err;
1233     }
1234
1235     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1236         err = kvm_update_guest_debug(env, 0);
1237         if (err)
1238             return err;
1239     }
1240     return 0;
1241 }
1242
1243 void kvm_remove_all_breakpoints(CPUState *current_env)
1244 {
1245     struct kvm_sw_breakpoint *bp, *next;
1246     KVMState *s = current_env->kvm_state;
1247     CPUState *env;
1248
1249     QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
1250         if (kvm_arch_remove_sw_breakpoint(current_env, bp) != 0) {
1251             /* Try harder to find a CPU that currently sees the breakpoint. */
1252             for (env = first_cpu; env != NULL; env = env->next_cpu) {
1253                 if (kvm_arch_remove_sw_breakpoint(env, bp) == 0)
1254                     break;
1255             }
1256         }
1257     }
1258     kvm_arch_remove_all_hw_breakpoints();
1259
1260     for (env = first_cpu; env != NULL; env = env->next_cpu)
1261         kvm_update_guest_debug(env, 0);
1262 }
1263
1264 #else /* !KVM_CAP_SET_GUEST_DEBUG */
1265
1266 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1267 {
1268     return -EINVAL;
1269 }
1270
1271 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1272                           target_ulong len, int type)
1273 {
1274     return -EINVAL;
1275 }
1276
1277 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1278                           target_ulong len, int type)
1279 {
1280     return -EINVAL;
1281 }
1282
1283 void kvm_remove_all_breakpoints(CPUState *current_env)
1284 {
1285 }
1286 #endif /* !KVM_CAP_SET_GUEST_DEBUG */
1287
1288 int kvm_set_signal_mask(CPUState *env, const sigset_t *sigset)
1289 {
1290     struct kvm_signal_mask *sigmask;
1291     int r;
1292
1293     if (!sigset)
1294         return kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, NULL);
1295
1296     sigmask = qemu_malloc(sizeof(*sigmask) + sizeof(*sigset));
1297
1298     sigmask->len = 8;
1299     memcpy(sigmask->sigset, sigset, sizeof(*sigset));
1300     r = kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, sigmask);
1301     free(sigmask);
1302
1303     return r;
1304 }
1305
1306 int kvm_set_ioeventfd_mmio_long(int fd, uint32_t addr, uint32_t val, bool assign)
1307 {
1308 #ifdef KVM_IOEVENTFD
1309     int ret;
1310     struct kvm_ioeventfd iofd;
1311
1312     iofd.datamatch = val;
1313     iofd.addr = addr;
1314     iofd.len = 4;
1315     iofd.flags = KVM_IOEVENTFD_FLAG_DATAMATCH;
1316     iofd.fd = fd;
1317
1318     if (!kvm_enabled()) {
1319         return -ENOSYS;
1320     }
1321
1322     if (!assign) {
1323         iofd.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
1324     }
1325
1326     ret = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &iofd);
1327
1328     if (ret < 0) {
1329         return -errno;
1330     }
1331
1332     return 0;
1333 #else
1334     return -ENOSYS;
1335 #endif
1336 }
1337
1338 int kvm_set_ioeventfd_pio_word(int fd, uint16_t addr, uint16_t val, bool assign)
1339 {
1340 #ifdef KVM_IOEVENTFD
1341     struct kvm_ioeventfd kick = {
1342         .datamatch = val,
1343         .addr = addr,
1344         .len = 2,
1345         .flags = KVM_IOEVENTFD_FLAG_DATAMATCH | KVM_IOEVENTFD_FLAG_PIO,
1346         .fd = fd,
1347     };
1348     int r;
1349     if (!kvm_enabled())
1350         return -ENOSYS;
1351     if (!assign)
1352         kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
1353     r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
1354     if (r < 0)
1355         return r;
1356     return 0;
1357 #else
1358     return -ENOSYS;
1359 #endif
1360 }