kvm-all.c

   1 /*
   2  * QEMU KVM support
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *           Red Hat, Inc. 2008
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *  Glauber Costa     <gcosta@redhat.com>
  10  *
  11  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  12  * See the COPYING file in the top-level directory.
  13  *
  14  */
  15
  16 #include <sys/types.h>
  17 #include <sys/ioctl.h>
  18 #include <sys/mman.h>
  19 #include <stdarg.h>
  20
  21 #include <linux/kvm.h>
  22
  23 #include "qemu-common.h"
  24 #include "qemu-barrier.h"
  25 #include "sysemu.h"
  26 #include "hw/hw.h"
  27 #include "gdbstub.h"
  28 #include "kvm.h"
  29 #include "bswap.h"
  30
  31 /* This check must be after config-host.h is included */
  32 #ifdef CONFIG_EVENTFD
  33 #include <sys/eventfd.h>
  34 #endif
  35
  36 /* KVM uses PAGE_SIZE in it's definition of COALESCED_MMIO_MAX */
  37 #define PAGE_SIZE TARGET_PAGE_SIZE
  38
  39 //#define DEBUG_KVM
  40
  41 #ifdef DEBUG_KVM
  42 #define DPRINTF(fmt, ...) \
  43     do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
  44 #else
  45 #define DPRINTF(fmt, ...) \
  46     do { } while (0)
  47 #endif
  48
  49 typedef struct KVMSlot
  50 {
  51     target_phys_addr_t start_addr;
  52     ram_addr_t memory_size;
  53     ram_addr_t phys_offset;
  54     int slot;
  55     int flags;
  56 } KVMSlot;
  57
  58 typedef struct kvm_dirty_log KVMDirtyLog;
  59
  60 struct KVMState
  61 {
  62     KVMSlot slots[32];
  63     int fd;
  64     int vmfd;
  65     int coalesced_mmio;
  66 #ifdef KVM_CAP_COALESCED_MMIO
  67     struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
  68 #endif
  69     int broken_set_mem_region;
  70     int migration_log;
  71     int vcpu_events;
  72     int robust_singlestep;
  73     int debugregs;
  74 #ifdef KVM_CAP_SET_GUEST_DEBUG
  75     struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
  76 #endif
  77     int irqchip_in_kernel;
  78     int pit_in_kernel;
  79     int xsave, xcrs;
  80     int many_ioeventfds;
  81 };
  82
  83 static KVMState *kvm_state;
  84
  85 static KVMSlot *kvm_alloc_slot(KVMState *s)
  86 {
  87     int i;
  88
  89     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
  90         /* KVM private memory slots */
  91         if (i >= 8 && i < 12)
  92             continue;
  93         if (s->slots[i].memory_size == 0)
  94             return &s->slots[i];
  95     }
  96
  97     fprintf(stderr, "%s: no free slot available\n", __func__);
  98     abort();
  99 }
 100
 101 static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
 102                                          target_phys_addr_t start_addr,
 103                                          target_phys_addr_t end_addr)
 104 {
 105     int i;
 106
 107     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 108         KVMSlot *mem = &s->slots[i];
 109
 110         if (start_addr == mem->start_addr &&
 111             end_addr == mem->start_addr + mem->memory_size) {
 112             return mem;
 113         }
 114     }
 115
 116     return NULL;
 117 }
 118
 119 /*
 120  * Find overlapping slot with lowest start address
 121  */
 122 static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
 123                                             target_phys_addr_t start_addr,
 124                                             target_phys_addr_t end_addr)
 125 {
 126     KVMSlot *found = NULL;
 127     int i;
 128
 129     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 130         KVMSlot *mem = &s->slots[i];
 131
 132         if (mem->memory_size == 0 ||
 133             (found && found->start_addr < mem->start_addr)) {
 134             continue;
 135         }
 136
 137         if (end_addr > mem->start_addr &&
 138             start_addr < mem->start_addr + mem->memory_size) {
 139             found = mem;
 140         }
 141     }
 142
 143     return found;
 144 }
 145
 146 int kvm_physical_memory_addr_from_ram(KVMState *s, ram_addr_t ram_addr,
 147                                       target_phys_addr_t *phys_addr)
 148 {
 149     int i;
 150
 151     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 152         KVMSlot *mem = &s->slots[i];
 153
 154         if (ram_addr >= mem->phys_offset &&
 155             ram_addr < mem->phys_offset + mem->memory_size) {
 156             *phys_addr = mem->start_addr + (ram_addr - mem->phys_offset);
 157             return 1;
 158         }
 159     }
 160
 161     return 0;
 162 }
 163
 164 static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
 165 {
 166     struct kvm_userspace_memory_region mem;
 167
 168     mem.slot = slot->slot;
 169     mem.guest_phys_addr = slot->start_addr;
 170     mem.memory_size = slot->memory_size;
 171     mem.userspace_addr = (unsigned long)qemu_safe_ram_ptr(slot->phys_offset);
 172     mem.flags = slot->flags;
 173     if (s->migration_log) {
 174         mem.flags |= KVM_MEM_LOG_DIRTY_PAGES;
 175     }
 176     return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
 177 }
 178
 179 static void kvm_reset_vcpu(void *opaque)
 180 {
 181     CPUState *env = opaque;
 182
 183     kvm_arch_reset_vcpu(env);
 184 }
 185
 186 int kvm_irqchip_in_kernel(void)
 187 {
 188     return kvm_state->irqchip_in_kernel;
 189 }
 190
 191 int kvm_pit_in_kernel(void)
 192 {
 193     return kvm_state->pit_in_kernel;
 194 }
 195
 196
 197 int kvm_init_vcpu(CPUState *env)
 198 {
 199     KVMState *s = kvm_state;
 200     long mmap_size;
 201     int ret;
 202
 203     DPRINTF("kvm_init_vcpu\n");
 204
 205     ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, env->cpu_index);
 206     if (ret < 0) {
 207         DPRINTF("kvm_create_vcpu failed\n");
 208         goto err;
 209     }
 210
 211     env->kvm_fd = ret;
 212     env->kvm_state = s;
 213
 214     mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
 215     if (mmap_size < 0) {
 216         DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
 217         goto err;
 218     }
 219
 220     env->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
 221                         env->kvm_fd, 0);
 222     if (env->kvm_run == MAP_FAILED) {
 223         ret = -errno;
 224         DPRINTF("mmap'ing vcpu state failed\n");
 225         goto err;
 226     }
 227
 228 #ifdef KVM_CAP_COALESCED_MMIO
 229     if (s->coalesced_mmio && !s->coalesced_mmio_ring)
 230         s->coalesced_mmio_ring = (void *) env->kvm_run +
 231                 s->coalesced_mmio * PAGE_SIZE;
 232 #endif
 233
 234     ret = kvm_arch_init_vcpu(env);
 235     if (ret == 0) {
 236         qemu_register_reset(kvm_reset_vcpu, env);
 237         kvm_arch_reset_vcpu(env);
 238     }
 239 err:
 240     return ret;
 241 }
 242
 243 /*
 244  * dirty pages logging control
 245  */
 246 static int kvm_dirty_pages_log_change(target_phys_addr_t phys_addr,
 247                                       ram_addr_t size, int flags, int mask)
 248 {
 249     KVMState *s = kvm_state;
 250     KVMSlot *mem = kvm_lookup_matching_slot(s, phys_addr, phys_addr + size);
 251     int old_flags;
 252
 253     if (mem == NULL)  {
 254             fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
 255                     TARGET_FMT_plx "\n", __func__, phys_addr,
 256                     (target_phys_addr_t)(phys_addr + size - 1));
 257             return -EINVAL;
 258     }
 259
 260     old_flags = mem->flags;
 261
 262     flags = (mem->flags & ~mask) | flags;
 263     mem->flags = flags;
 264
 265     /* If nothing changed effectively, no need to issue ioctl */
 266     if (s->migration_log) {
 267         flags |= KVM_MEM_LOG_DIRTY_PAGES;
 268     }
 269     if (flags == old_flags) {
 270             return 0;
 271     }
 272
 273     return kvm_set_user_memory_region(s, mem);
 274 }
 275
 276 int kvm_log_start(target_phys_addr_t phys_addr, ram_addr_t size)
 277 {
 278         return kvm_dirty_pages_log_change(phys_addr, size,
 279                                           KVM_MEM_LOG_DIRTY_PAGES,
 280                                           KVM_MEM_LOG_DIRTY_PAGES);
 281 }
 282
 283 int kvm_log_stop(target_phys_addr_t phys_addr, ram_addr_t size)
 284 {
 285         return kvm_dirty_pages_log_change(phys_addr, size,
 286                                           0,
 287                                           KVM_MEM_LOG_DIRTY_PAGES);
 288 }
 289
 290 static int kvm_set_migration_log(int enable)
 291 {
 292     KVMState *s = kvm_state;
 293     KVMSlot *mem;
 294     int i, err;
 295
 296     s->migration_log = enable;
 297
 298     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 299         mem = &s->slots[i];
 300
 301         if (!mem->memory_size) {
 302             continue;
 303         }
 304         if (!!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) == enable) {
 305             continue;
 306         }
 307         err = kvm_set_user_memory_region(s, mem);
 308         if (err) {
 309             return err;
 310         }
 311     }
 312     return 0;
 313 }
 314
 315 /* get kvm's dirty pages bitmap and update qemu's */
 316 static int kvm_get_dirty_pages_log_range(unsigned long start_addr,
 317                                          unsigned long *bitmap,
 318                                          unsigned long offset,
 319                                          unsigned long mem_size)
 320 {
 321     unsigned int i, j;
 322     unsigned long page_number, addr, addr1, c;
 323     ram_addr_t ram_addr;
 324     unsigned int len = ((mem_size / TARGET_PAGE_SIZE) + HOST_LONG_BITS - 1) /
 325         HOST_LONG_BITS;
 326
 327     /*
 328      * bitmap-traveling is faster than memory-traveling (for addr...)
 329      * especially when most of the memory is not dirty.
 330      */
 331     for (i = 0; i < len; i++) {
 332         if (bitmap[i] != 0) {
 333             c = leul_to_cpu(bitmap[i]);
 334             do {
 335                 j = ffsl(c) - 1;
 336                 c &= ~(1ul << j);
 337                 page_number = i * HOST_LONG_BITS + j;
 338                 addr1 = page_number * TARGET_PAGE_SIZE;
 339                 addr = offset + addr1;
 340                 ram_addr = cpu_get_physical_page_desc(addr);
 341                 cpu_physical_memory_set_dirty(ram_addr);
 342             } while (c != 0);
 343         }
 344     }
 345     return 0;
 346 }
 347
 348 #define ALIGN(x, y)  (((x)+(y)-1) & ~((y)-1))
 349
 350 /**
 351  * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
 352  * This function updates qemu's dirty bitmap using cpu_physical_memory_set_dirty().
 353  * This means all bits are set to dirty.
 354  *
 355  * @start_add: start of logged region.
 356  * @end_addr: end of logged region.
 357  */
 358 static int kvm_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
 359                                           target_phys_addr_t end_addr)
 360 {
 361     KVMState *s = kvm_state;
 362     unsigned long size, allocated_size = 0;
 363     KVMDirtyLog d;
 364     KVMSlot *mem;
 365     int ret = 0;
 366
 367     d.dirty_bitmap = NULL;
 368     while (start_addr < end_addr) {
 369         mem = kvm_lookup_overlapping_slot(s, start_addr, end_addr);
 370         if (mem == NULL) {
 371             break;
 372         }
 373
 374         size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS), HOST_LONG_BITS) / 8;
 375         if (!d.dirty_bitmap) {
 376             d.dirty_bitmap = qemu_malloc(size);
 377         } else if (size > allocated_size) {
 378             d.dirty_bitmap = qemu_realloc(d.dirty_bitmap, size);
 379         }
 380         allocated_size = size;
 381         memset(d.dirty_bitmap, 0, allocated_size);
 382
 383         d.slot = mem->slot;
 384
 385         if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
 386             DPRINTF("ioctl failed %d\n", errno);
 387             ret = -1;
 388             break;
 389         }
 390
 391         kvm_get_dirty_pages_log_range(mem->start_addr, d.dirty_bitmap,
 392                                       mem->start_addr, mem->memory_size);
 393         start_addr = mem->start_addr + mem->memory_size;
 394     }
 395     qemu_free(d.dirty_bitmap);
 396
 397     return ret;
 398 }
 399
 400 int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
 401 {
 402     int ret = -ENOSYS;
 403 #ifdef KVM_CAP_COALESCED_MMIO
 404     KVMState *s = kvm_state;
 405
 406     if (s->coalesced_mmio) {
 407         struct kvm_coalesced_mmio_zone zone;
 408
 409         zone.addr = start;
 410         zone.size = size;
 411
 412         ret = kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
 413     }
 414 #endif
 415
 416     return ret;
 417 }
 418
 419 int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
 420 {
 421     int ret = -ENOSYS;
 422 #ifdef KVM_CAP_COALESCED_MMIO
 423     KVMState *s = kvm_state;
 424
 425     if (s->coalesced_mmio) {
 426         struct kvm_coalesced_mmio_zone zone;
 427
 428         zone.addr = start;
 429         zone.size = size;
 430
 431         ret = kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
 432     }
 433 #endif
 434
 435     return ret;
 436 }
 437
 438 int kvm_check_extension(KVMState *s, unsigned int extension)
 439 {
 440     int ret;
 441
 442     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
 443     if (ret < 0) {
 444         ret = 0;
 445     }
 446
 447     return ret;
 448 }
 449
 450 static int kvm_check_many_ioeventfds(void)
 451 {
 452     /* Older kernels have a 6 device limit on the KVM io bus.  Find out so we
 453      * can avoid creating too many ioeventfds.
 454      */
 455 #ifdef CONFIG_EVENTFD
 456     int ioeventfds[7];
 457     int i, ret = 0;
 458     for (i = 0; i < ARRAY_SIZE(ioeventfds); i++) {
 459         ioeventfds[i] = eventfd(0, EFD_CLOEXEC);
 460         if (ioeventfds[i] < 0) {
 461             break;
 462         }
 463         ret = kvm_set_ioeventfd_pio_word(ioeventfds[i], 0, i, true);
 464         if (ret < 0) {
 465             close(ioeventfds[i]);
 466             break;
 467         }
 468     }
 469
 470     /* Decide whether many devices are supported or not */
 471     ret = i == ARRAY_SIZE(ioeventfds);
 472
 473     while (i-- > 0) {
 474         kvm_set_ioeventfd_pio_word(ioeventfds[i], 0, i, false);
 475         close(ioeventfds[i]);
 476     }
 477     return ret;
 478 #else
 479     return 0;
 480 #endif
 481 }
 482
 483 static void kvm_set_phys_mem(target_phys_addr_t start_addr,
 484                              ram_addr_t size,
 485                              ram_addr_t phys_offset)
 486 {
 487     KVMState *s = kvm_state;
 488     ram_addr_t flags = phys_offset & ~TARGET_PAGE_MASK;
 489     KVMSlot *mem, old;
 490     int err;
 491
 492     /* kvm works in page size chunks, but the function may be called
 493        with sub-page size and unaligned start address. */
 494     size = TARGET_PAGE_ALIGN(size);
 495     start_addr = TARGET_PAGE_ALIGN(start_addr);
 496
 497     /* KVM does not support read-only slots */
 498     phys_offset &= ~IO_MEM_ROM;
 499
 500     while (1) {
 501         mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size);
 502         if (!mem) {
 503             break;
 504         }
 505
 506         if (flags < IO_MEM_UNASSIGNED && start_addr >= mem->start_addr &&
 507             (start_addr + size <= mem->start_addr + mem->memory_size) &&
 508             (phys_offset - start_addr == mem->phys_offset - mem->start_addr)) {
 509             /* The new slot fits into the existing one and comes with
 510              * identical parameters - nothing to be done. */
 511             return;
 512         }
 513
 514         old = *mem;
 515
 516         /* unregister the overlapping slot */
 517         mem->memory_size = 0;
 518         err = kvm_set_user_memory_region(s, mem);
 519         if (err) {
 520             fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
 521                     __func__, strerror(-err));
 522             abort();
 523         }
 524
 525         /* Workaround for older KVM versions: we can't join slots, even not by
 526          * unregistering the previous ones and then registering the larger
 527          * slot. We have to maintain the existing fragmentation. Sigh.
 528          *
 529          * This workaround assumes that the new slot starts at the same
 530          * address as the first existing one. If not or if some overlapping
 531          * slot comes around later, we will fail (not seen in practice so far)
 532          * - and actually require a recent KVM version. */
 533         if (s->broken_set_mem_region &&
 534             old.start_addr == start_addr && old.memory_size < size &&
 535             flags < IO_MEM_UNASSIGNED) {
 536             mem = kvm_alloc_slot(s);
 537             mem->memory_size = old.memory_size;
 538             mem->start_addr = old.start_addr;
 539             mem->phys_offset = old.phys_offset;
 540             mem->flags = 0;
 541
 542             err = kvm_set_user_memory_region(s, mem);
 543             if (err) {
 544                 fprintf(stderr, "%s: error updating slot: %s\n", __func__,
 545                         strerror(-err));
 546                 abort();
 547             }
 548
 549             start_addr += old.memory_size;
 550             phys_offset += old.memory_size;
 551             size -= old.memory_size;
 552             continue;
 553         }
 554
 555         /* register prefix slot */
 556         if (old.start_addr < start_addr) {
 557             mem = kvm_alloc_slot(s);
 558             mem->memory_size = start_addr - old.start_addr;
 559             mem->start_addr = old.start_addr;
 560             mem->phys_offset = old.phys_offset;
 561             mem->flags = 0;
 562
 563             err = kvm_set_user_memory_region(s, mem);
 564             if (err) {
 565                 fprintf(stderr, "%s: error registering prefix slot: %s\n",
 566                         __func__, strerror(-err));
 567                 abort();
 568             }
 569         }
 570
 571         /* register suffix slot */
 572         if (old.start_addr + old.memory_size > start_addr + size) {
 573             ram_addr_t size_delta;
 574
 575             mem = kvm_alloc_slot(s);
 576             mem->start_addr = start_addr + size;
 577             size_delta = mem->start_addr - old.start_addr;
 578             mem->memory_size = old.memory_size - size_delta;
 579             mem->phys_offset = old.phys_offset + size_delta;
 580             mem->flags = 0;
 581
 582             err = kvm_set_user_memory_region(s, mem);
 583             if (err) {
 584                 fprintf(stderr, "%s: error registering suffix slot: %s\n",
 585                         __func__, strerror(-err));
 586                 abort();
 587             }
 588         }
 589     }
 590
 591     /* in case the KVM bug workaround already "consumed" the new slot */
 592     if (!size)
 593         return;
 594
 595     /* KVM does not need to know about this memory */
 596     if (flags >= IO_MEM_UNASSIGNED)
 597         return;
 598
 599     mem = kvm_alloc_slot(s);
 600     mem->memory_size = size;
 601     mem->start_addr = start_addr;
 602     mem->phys_offset = phys_offset;
 603     mem->flags = 0;
 604
 605     err = kvm_set_user_memory_region(s, mem);
 606     if (err) {
 607         fprintf(stderr, "%s: error registering slot: %s\n", __func__,
 608                 strerror(-err));
 609         abort();
 610     }
 611 }
 612
 613 static void kvm_client_set_memory(struct CPUPhysMemoryClient *client,
 614                                   target_phys_addr_t start_addr,
 615                                   ram_addr_t size,
 616                                   ram_addr_t phys_offset)
 617 {
 618         kvm_set_phys_mem(start_addr, size, phys_offset);
 619 }
 620
 621 static int kvm_client_sync_dirty_bitmap(struct CPUPhysMemoryClient *client,
 622                                         target_phys_addr_t start_addr,
 623                                         target_phys_addr_t end_addr)
 624 {
 625         return kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
 626 }
 627
 628 static int kvm_client_migration_log(struct CPUPhysMemoryClient *client,
 629                                     int enable)
 630 {
 631         return kvm_set_migration_log(enable);
 632 }
 633
 634 static CPUPhysMemoryClient kvm_cpu_phys_memory_client = {
 635         .set_memory = kvm_client_set_memory,
 636         .sync_dirty_bitmap = kvm_client_sync_dirty_bitmap,
 637         .migration_log = kvm_client_migration_log,
 638 };
 639
 640 int kvm_init(int smp_cpus)
 641 {
 642     static const char upgrade_note[] =
 643         "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
 644         "(see http://sourceforge.net/projects/kvm).\n";
 645     KVMState *s;
 646     int ret;
 647     int i;
 648
 649     s = qemu_mallocz(sizeof(KVMState));
 650
 651 #ifdef KVM_CAP_SET_GUEST_DEBUG
 652     QTAILQ_INIT(&s->kvm_sw_breakpoints);
 653 #endif
 654     for (i = 0; i < ARRAY_SIZE(s->slots); i++)
 655         s->slots[i].slot = i;
 656
 657     s->vmfd = -1;
 658     s->fd = qemu_open("/dev/kvm", O_RDWR);
 659     if (s->fd == -1) {
 660         fprintf(stderr, "Could not access KVM kernel module: %m\n");
 661         ret = -errno;
 662         goto err;
 663     }
 664
 665     ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
 666     if (ret < KVM_API_VERSION) {
 667         if (ret > 0)
 668             ret = -EINVAL;
 669         fprintf(stderr, "kvm version too old\n");
 670         goto err;
 671     }
 672
 673     if (ret > KVM_API_VERSION) {
 674         ret = -EINVAL;
 675         fprintf(stderr, "kvm version not supported\n");
 676         goto err;
 677     }
 678
 679     s->vmfd = kvm_ioctl(s, KVM_CREATE_VM, 0);
 680     if (s->vmfd < 0) {
 681 #ifdef TARGET_S390X
 682         fprintf(stderr, "Please add the 'switch_amode' kernel parameter to "
 683                         "your host kernel command line\n");
 684 #endif
 685         goto err;
 686     }
 687
 688     /* initially, KVM allocated its own memory and we had to jump through
 689      * hooks to make phys_ram_base point to this.  Modern versions of KVM
 690      * just use a user allocated buffer so we can use regular pages
 691      * unmodified.  Make sure we have a sufficiently modern version of KVM.
 692      */
 693     if (!kvm_check_extension(s, KVM_CAP_USER_MEMORY)) {
 694         ret = -EINVAL;
 695         fprintf(stderr, "kvm does not support KVM_CAP_USER_MEMORY\n%s",
 696                 upgrade_note);
 697         goto err;
 698     }
 699
 700     /* There was a nasty bug in < kvm-80 that prevents memory slots from being
 701      * destroyed properly.  Since we rely on this capability, refuse to work
 702      * with any kernel without this capability. */
 703     if (!kvm_check_extension(s, KVM_CAP_DESTROY_MEMORY_REGION_WORKS)) {
 704         ret = -EINVAL;
 705
 706         fprintf(stderr,
 707                 "KVM kernel module broken (DESTROY_MEMORY_REGION).\n%s",
 708                 upgrade_note);
 709         goto err;
 710     }
 711
 712     s->coalesced_mmio = 0;
 713 #ifdef KVM_CAP_COALESCED_MMIO
 714     s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
 715     s->coalesced_mmio_ring = NULL;
 716 #endif
 717
 718     s->broken_set_mem_region = 1;
 719 #ifdef KVM_CAP_JOIN_MEMORY_REGIONS_WORKS
 720     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
 721     if (ret > 0) {
 722         s->broken_set_mem_region = 0;
 723     }
 724 #endif
 725
 726     s->vcpu_events = 0;
 727 #ifdef KVM_CAP_VCPU_EVENTS
 728     s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
 729 #endif
 730
 731     s->robust_singlestep = 0;
 732 #ifdef KVM_CAP_X86_ROBUST_SINGLESTEP
 733     s->robust_singlestep =
 734         kvm_check_extension(s, KVM_CAP_X86_ROBUST_SINGLESTEP);
 735 #endif
 736
 737     s->debugregs = 0;
 738 #ifdef KVM_CAP_DEBUGREGS
 739     s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS);
 740 #endif
 741
 742     s->xsave = 0;
 743 #ifdef KVM_CAP_XSAVE
 744     s->xsave = kvm_check_extension(s, KVM_CAP_XSAVE);
 745 #endif
 746
 747     s->xcrs = 0;
 748 #ifdef KVM_CAP_XCRS
 749     s->xcrs = kvm_check_extension(s, KVM_CAP_XCRS);
 750 #endif
 751
 752     ret = kvm_arch_init(s, smp_cpus);
 753     if (ret < 0)
 754         goto err;
 755
 756     kvm_state = s;
 757     cpu_register_phys_memory_client(&kvm_cpu_phys_memory_client);
 758
 759     s->many_ioeventfds = kvm_check_many_ioeventfds();
 760
 761     return 0;
 762
 763 err:
 764     if (s) {
 765         if (s->vmfd != -1)
 766             close(s->vmfd);
 767         if (s->fd != -1)
 768             close(s->fd);
 769     }
 770     qemu_free(s);
 771
 772     return ret;
 773 }
 774
 775 static int kvm_handle_io(uint16_t port, void *data, int direction, int size,
 776                          uint32_t count)
 777 {
 778     int i;
 779     uint8_t *ptr = data;
 780
 781     for (i = 0; i < count; i++) {
 782         if (direction == KVM_EXIT_IO_IN) {
 783             switch (size) {
 784             case 1:
 785                 stb_p(ptr, cpu_inb(port));
 786                 break;
 787             case 2:
 788                 stw_p(ptr, cpu_inw(port));
 789                 break;
 790             case 4:
 791                 stl_p(ptr, cpu_inl(port));
 792                 break;
 793             }
 794         } else {
 795             switch (size) {
 796             case 1:
 797                 cpu_outb(port, ldub_p(ptr));
 798                 break;
 799             case 2:
 800                 cpu_outw(port, lduw_p(ptr));
 801                 break;
 802             case 4:
 803                 cpu_outl(port, ldl_p(ptr));
 804                 break;
 805             }
 806         }
 807
 808         ptr += size;
 809     }
 810
 811     return 1;
 812 }
 813
 814 #ifdef KVM_CAP_INTERNAL_ERROR_DATA
 815 static void kvm_handle_internal_error(CPUState *env, struct kvm_run *run)
 816 {
 817
 818     if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) {
 819         int i;
 820
 821         fprintf(stderr, "KVM internal error. Suberror: %d\n",
 822                 run->internal.suberror);
 823
 824         for (i = 0; i < run->internal.ndata; ++i) {
 825             fprintf(stderr, "extra data[%d]: %"PRIx64"\n",
 826                     i, (uint64_t)run->internal.data[i]);
 827         }
 828     }
 829     cpu_dump_state(env, stderr, fprintf, 0);
 830     if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) {
 831         fprintf(stderr, "emulation failure\n");
 832         if (!kvm_arch_stop_on_emulation_error(env))
 833                 return;
 834     }
 835     /* FIXME: Should trigger a qmp message to let management know
 836      * something went wrong.
 837      */
 838     vm_stop(0);
 839 }
 840 #endif
 841
 842 void kvm_flush_coalesced_mmio_buffer(void)
 843 {
 844 #ifdef KVM_CAP_COALESCED_MMIO
 845     KVMState *s = kvm_state;
 846     if (s->coalesced_mmio_ring) {
 847         struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
 848         while (ring->first != ring->last) {
 849             struct kvm_coalesced_mmio *ent;
 850
 851             ent = &ring->coalesced_mmio[ring->first];
 852
 853             cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
 854             smp_wmb();
 855             ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
 856         }
 857     }
 858 #endif
 859 }
 860
 861 static void do_kvm_cpu_synchronize_state(void *_env)
 862 {
 863     CPUState *env = _env;
 864
 865     if (!env->kvm_vcpu_dirty) {
 866         kvm_arch_get_registers(env);
 867         env->kvm_vcpu_dirty = 1;
 868     }
 869 }
 870
 871 void kvm_cpu_synchronize_state(CPUState *env)
 872 {
 873     if (!env->kvm_vcpu_dirty)
 874         run_on_cpu(env, do_kvm_cpu_synchronize_state, env);
 875 }
 876
 877 void kvm_cpu_synchronize_post_reset(CPUState *env)
 878 {
 879     kvm_arch_put_registers(env, KVM_PUT_RESET_STATE);
 880     env->kvm_vcpu_dirty = 0;
 881 }
 882
 883 void kvm_cpu_synchronize_post_init(CPUState *env)
 884 {
 885     kvm_arch_put_registers(env, KVM_PUT_FULL_STATE);
 886     env->kvm_vcpu_dirty = 0;
 887 }
 888
 889 int kvm_cpu_exec(CPUState *env)
 890 {
 891     struct kvm_run *run = env->kvm_run;
 892     int ret;
 893
 894     DPRINTF("kvm_cpu_exec()\n");
 895
 896     do {
 897 #ifndef CONFIG_IOTHREAD
 898         if (env->exit_request) {
 899             DPRINTF("interrupt exit requested\n");
 900             ret = 0;
 901             break;
 902         }
 903 #endif
 904
 905         if (kvm_arch_process_irqchip_events(env)) {
 906             ret = 0;
 907             break;
 908         }
 909
 910         if (env->kvm_vcpu_dirty) {
 911             kvm_arch_put_registers(env, KVM_PUT_RUNTIME_STATE);
 912             env->kvm_vcpu_dirty = 0;
 913         }
 914
 915         kvm_arch_pre_run(env, run);
 916         cpu_single_env = NULL;
 917         qemu_mutex_unlock_iothread();
 918         ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);
 919         qemu_mutex_lock_iothread();
 920         cpu_single_env = env;
 921         kvm_arch_post_run(env, run);
 922
 923         if (ret == -EINTR || ret == -EAGAIN) {
 924             cpu_exit(env);
 925             DPRINTF("io window exit\n");
 926             ret = 0;
 927             break;
 928         }
 929
 930         if (ret < 0) {
 931             DPRINTF("kvm run failed %s\n", strerror(-ret));
 932             abort();
 933         }
 934
 935         kvm_flush_coalesced_mmio_buffer();
 936
 937         ret = 0; /* exit loop */
 938         switch (run->exit_reason) {
 939         case KVM_EXIT_IO:
 940             DPRINTF("handle_io\n");
 941             ret = kvm_handle_io(run->io.port,
 942                                 (uint8_t *)run + run->io.data_offset,
 943                                 run->io.direction,
 944                                 run->io.size,
 945                                 run->io.count);
 946             break;
 947         case KVM_EXIT_MMIO:
 948             DPRINTF("handle_mmio\n");
 949             cpu_physical_memory_rw(run->mmio.phys_addr,
 950                                    run->mmio.data,
 951                                    run->mmio.len,
 952                                    run->mmio.is_write);
 953             ret = 1;
 954             break;
 955         case KVM_EXIT_IRQ_WINDOW_OPEN:
 956             DPRINTF("irq_window_open\n");
 957             break;
 958         case KVM_EXIT_SHUTDOWN:
 959             DPRINTF("shutdown\n");
 960             qemu_system_reset_request();
 961             ret = 1;
 962             break;
 963         case KVM_EXIT_UNKNOWN:
 964             DPRINTF("kvm_exit_unknown\n");
 965             break;
 966         case KVM_EXIT_FAIL_ENTRY:
 967             DPRINTF("kvm_exit_fail_entry\n");
 968             break;
 969         case KVM_EXIT_EXCEPTION:
 970             DPRINTF("kvm_exit_exception\n");
 971             break;
 972 #ifdef KVM_CAP_INTERNAL_ERROR_DATA
 973         case KVM_EXIT_INTERNAL_ERROR:
 974             kvm_handle_internal_error(env, run);
 975             break;
 976 #endif
 977         case KVM_EXIT_DEBUG:
 978             DPRINTF("kvm_exit_debug\n");
 979 #ifdef KVM_CAP_SET_GUEST_DEBUG
 980             if (kvm_arch_debug(&run->debug.arch)) {
 981                 env->exception_index = EXCP_DEBUG;
 982                 return 0;
 983             }
 984             /* re-enter, this exception was guest-internal */
 985             ret = 1;
 986 #endif /* KVM_CAP_SET_GUEST_DEBUG */
 987             break;
 988         default:
 989             DPRINTF("kvm_arch_handle_exit\n");
 990             ret = kvm_arch_handle_exit(env, run);
 991             break;
 992         }
 993     } while (ret > 0);
 994
 995     if (env->exit_request) {
 996         env->exit_request = 0;
 997         env->exception_index = EXCP_INTERRUPT;
 998     }
 999
1000     return ret;
1001 }
1002
1003 int kvm_ioctl(KVMState *s, int type, ...)
1004 {
1005     int ret;
1006     void *arg;
1007     va_list ap;
1008
1009     va_start(ap, type);
1010     arg = va_arg(ap, void *);
1011     va_end(ap);
1012
1013     ret = ioctl(s->fd, type, arg);
1014     if (ret == -1)
1015         ret = -errno;
1016
1017     return ret;
1018 }
1019
1020 int kvm_vm_ioctl(KVMState *s, int type, ...)
1021 {
1022     int ret;
1023     void *arg;
1024     va_list ap;
1025
1026     va_start(ap, type);
1027     arg = va_arg(ap, void *);
1028     va_end(ap);
1029
1030     ret = ioctl(s->vmfd, type, arg);
1031     if (ret == -1)
1032         ret = -errno;
1033
1034     return ret;
1035 }
1036
1037 int kvm_vcpu_ioctl(CPUState *env, int type, ...)
1038 {
1039     int ret;
1040     void *arg;
1041     va_list ap;
1042
1043     va_start(ap, type);
1044     arg = va_arg(ap, void *);
1045     va_end(ap);
1046
1047     ret = ioctl(env->kvm_fd, type, arg);
1048     if (ret == -1)
1049         ret = -errno;
1050
1051     return ret;
1052 }
1053
1054 int kvm_has_sync_mmu(void)
1055 {
1056 #ifdef KVM_CAP_SYNC_MMU
1057     KVMState *s = kvm_state;
1058
1059     return kvm_check_extension(s, KVM_CAP_SYNC_MMU);
1060 #else
1061     return 0;
1062 #endif
1063 }
1064
1065 int kvm_has_vcpu_events(void)
1066 {
1067     return kvm_state->vcpu_events;
1068 }
1069
1070 int kvm_has_robust_singlestep(void)
1071 {
1072     return kvm_state->robust_singlestep;
1073 }
1074
1075 int kvm_has_debugregs(void)
1076 {
1077     return kvm_state->debugregs;
1078 }
1079
1080 int kvm_has_xsave(void)
1081 {
1082     return kvm_state->xsave;
1083 }
1084
1085 int kvm_has_xcrs(void)
1086 {
1087     return kvm_state->xcrs;
1088 }
1089
1090 int kvm_has_many_ioeventfds(void)
1091 {
1092     if (!kvm_enabled()) {
1093         return 0;
1094     }
1095     return kvm_state->many_ioeventfds;
1096 }
1097
1098 void kvm_setup_guest_memory(void *start, size_t size)
1099 {
1100     if (!kvm_has_sync_mmu()) {
1101         int ret = qemu_madvise(start, size, QEMU_MADV_DONTFORK);
1102
1103         if (ret) {
1104             perror("qemu_madvise");
1105             fprintf(stderr,
1106                     "Need MADV_DONTFORK in absence of synchronous KVM MMU\n");
1107             exit(1);
1108         }
1109     }
1110 }
1111
1112 #ifdef KVM_CAP_SET_GUEST_DEBUG
1113 struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *env,
1114                                                  target_ulong pc)
1115 {
1116     struct kvm_sw_breakpoint *bp;
1117
1118     QTAILQ_FOREACH(bp, &env->kvm_state->kvm_sw_breakpoints, entry) {
1119         if (bp->pc == pc)
1120             return bp;
1121     }
1122     return NULL;
1123 }
1124
1125 int kvm_sw_breakpoints_active(CPUState *env)
1126 {
1127     return !QTAILQ_EMPTY(&env->kvm_state->kvm_sw_breakpoints);
1128 }
1129
1130 struct kvm_set_guest_debug_data {
1131     struct kvm_guest_debug dbg;
1132     CPUState *env;
1133     int err;
1134 };
1135
1136 static void kvm_invoke_set_guest_debug(void *data)
1137 {
1138     struct kvm_set_guest_debug_data *dbg_data = data;
1139     CPUState *env = dbg_data->env;
1140
1141     dbg_data->err = kvm_vcpu_ioctl(env, KVM_SET_GUEST_DEBUG, &dbg_data->dbg);
1142 }
1143
1144 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1145 {
1146     struct kvm_set_guest_debug_data data;
1147
1148     data.dbg.control = reinject_trap;
1149
1150     if (env->singlestep_enabled) {
1151         data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
1152     }
1153     kvm_arch_update_guest_debug(env, &data.dbg);
1154     data.env = env;
1155
1156     run_on_cpu(env, kvm_invoke_set_guest_debug, &data);
1157     return data.err;
1158 }
1159
1160 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1161                           target_ulong len, int type)
1162 {
1163     struct kvm_sw_breakpoint *bp;
1164     CPUState *env;
1165     int err;
1166
1167     if (type == GDB_BREAKPOINT_SW) {
1168         bp = kvm_find_sw_breakpoint(current_env, addr);
1169         if (bp) {
1170             bp->use_count++;
1171             return 0;
1172         }
1173
1174         bp = qemu_malloc(sizeof(struct kvm_sw_breakpoint));
1175         if (!bp)
1176             return -ENOMEM;
1177
1178         bp->pc = addr;
1179         bp->use_count = 1;
1180         err = kvm_arch_insert_sw_breakpoint(current_env, bp);
1181         if (err) {
1182             free(bp);
1183             return err;
1184         }
1185
1186         QTAILQ_INSERT_HEAD(&current_env->kvm_state->kvm_sw_breakpoints,
1187                           bp, entry);
1188     } else {
1189         err = kvm_arch_insert_hw_breakpoint(addr, len, type);
1190         if (err)
1191             return err;
1192     }
1193
1194     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1195         err = kvm_update_guest_debug(env, 0);
1196         if (err)
1197             return err;
1198     }
1199     return 0;
1200 }
1201
1202 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1203                           target_ulong len, int type)
1204 {
1205     struct kvm_sw_breakpoint *bp;
1206     CPUState *env;
1207     int err;
1208
1209     if (type == GDB_BREAKPOINT_SW) {
1210         bp = kvm_find_sw_breakpoint(current_env, addr);
1211         if (!bp)
1212             return -ENOENT;
1213
1214         if (bp->use_count > 1) {
1215             bp->use_count--;
1216             return 0;
1217         }
1218
1219         err = kvm_arch_remove_sw_breakpoint(current_env, bp);
1220         if (err)
1221             return err;
1222
1223         QTAILQ_REMOVE(&current_env->kvm_state->kvm_sw_breakpoints, bp, entry);
1224         qemu_free(bp);
1225     } else {
1226         err = kvm_arch_remove_hw_breakpoint(addr, len, type);
1227         if (err)
1228             return err;
1229     }
1230
1231     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1232         err = kvm_update_guest_debug(env, 0);
1233         if (err)
1234             return err;
1235     }
1236     return 0;
1237 }
1238
1239 void kvm_remove_all_breakpoints(CPUState *current_env)
1240 {
1241     struct kvm_sw_breakpoint *bp, *next;
1242     KVMState *s = current_env->kvm_state;
1243     CPUState *env;
1244
1245     QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
1246         if (kvm_arch_remove_sw_breakpoint(current_env, bp) != 0) {
1247             /* Try harder to find a CPU that currently sees the breakpoint. */
1248             for (env = first_cpu; env != NULL; env = env->next_cpu) {
1249                 if (kvm_arch_remove_sw_breakpoint(env, bp) == 0)
1250                     break;
1251             }
1252         }
1253     }
1254     kvm_arch_remove_all_hw_breakpoints();
1255
1256     for (env = first_cpu; env != NULL; env = env->next_cpu)
1257         kvm_update_guest_debug(env, 0);
1258 }
1259
1260 #else /* !KVM_CAP_SET_GUEST_DEBUG */
1261
1262 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1263 {
1264     return -EINVAL;
1265 }
1266
1267 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1268                           target_ulong len, int type)
1269 {
1270     return -EINVAL;
1271 }
1272
1273 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1274                           target_ulong len, int type)
1275 {
1276     return -EINVAL;
1277 }
1278
1279 void kvm_remove_all_breakpoints(CPUState *current_env)
1280 {
1281 }
1282 #endif /* !KVM_CAP_SET_GUEST_DEBUG */
1283
1284 int kvm_set_signal_mask(CPUState *env, const sigset_t *sigset)
1285 {
1286     struct kvm_signal_mask *sigmask;
1287     int r;
1288
1289     if (!sigset)
1290         return kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, NULL);
1291
1292     sigmask = qemu_malloc(sizeof(*sigmask) + sizeof(*sigset));
1293
1294     sigmask->len = 8;
1295     memcpy(sigmask->sigset, sigset, sizeof(*sigset));
1296     r = kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, sigmask);
1297     free(sigmask);
1298
1299     return r;
1300 }
1301
1302 int kvm_set_ioeventfd_mmio_long(int fd, uint32_t addr, uint32_t val, bool assign)
1303 {
1304 #ifdef KVM_IOEVENTFD
1305     int ret;
1306     struct kvm_ioeventfd iofd;
1307
1308     iofd.datamatch = val;
1309     iofd.addr = addr;
1310     iofd.len = 4;
1311     iofd.flags = KVM_IOEVENTFD_FLAG_DATAMATCH;
1312     iofd.fd = fd;
1313
1314     if (!kvm_enabled()) {
1315         return -ENOSYS;
1316     }
1317
1318     if (!assign) {
1319         iofd.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
1320     }
1321
1322     ret = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &iofd);
1323
1324     if (ret < 0) {
1325         return -errno;
1326     }
1327
1328     return 0;
1329 #else
1330     return -ENOSYS;
1331 #endif
1332 }
1333
1334 int kvm_set_ioeventfd_pio_word(int fd, uint16_t addr, uint16_t val, bool assign)
1335 {
1336 #ifdef KVM_IOEVENTFD
1337     struct kvm_ioeventfd kick = {
1338         .datamatch = val,
1339         .addr = addr,
1340         .len = 2,
1341         .flags = KVM_IOEVENTFD_FLAG_DATAMATCH | KVM_IOEVENTFD_FLAG_PIO,
1342         .fd = fd,
1343     };
1344     int r;
1345     if (!kvm_enabled())
1346         return -ENOSYS;
1347     if (!assign)
1348         kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
1349     r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
1350     if (r < 0)
1351         return r;
1352     return 0;
1353 #else
1354     return -ENOSYS;
1355 #endif
1356 }