kvm-all.c

   1 /*
   2  * QEMU KVM support
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *           Red Hat, Inc. 2008
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *  Glauber Costa     <gcosta@redhat.com>
  10  *
  11  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  12  * See the COPYING file in the top-level directory.
  13  *
  14  */
  15
  16 #include <sys/types.h>
  17 #include <sys/ioctl.h>
  18 #include <sys/mman.h>
  19 #include <stdarg.h>
  20
  21 #include <linux/kvm.h>
  22
  23 #include "qemu-common.h"
  24 #include "qemu-barrier.h"
  25 #include "sysemu.h"
  26 #include "hw/hw.h"
  27 #include "gdbstub.h"
  28 #include "kvm.h"
  29 #include "bswap.h"
  30
  31 /* KVM uses PAGE_SIZE in it's definition of COALESCED_MMIO_MAX */
  32 #define PAGE_SIZE TARGET_PAGE_SIZE
  33
  34 //#define DEBUG_KVM
  35
  36 #ifdef DEBUG_KVM
  37 #define DPRINTF(fmt, ...) \
  38     do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
  39 #else
  40 #define DPRINTF(fmt, ...) \
  41     do { } while (0)
  42 #endif
  43
  44 typedef struct KVMSlot
  45 {
  46     target_phys_addr_t start_addr;
  47     ram_addr_t memory_size;
  48     ram_addr_t phys_offset;
  49     int slot;
  50     int flags;
  51 } KVMSlot;
  52
  53 typedef struct kvm_dirty_log KVMDirtyLog;
  54
  55 struct KVMState
  56 {
  57     KVMSlot slots[32];
  58     int fd;
  59     int vmfd;
  60     int coalesced_mmio;
  61 #ifdef KVM_CAP_COALESCED_MMIO
  62     struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
  63 #endif
  64     int broken_set_mem_region;
  65     int migration_log;
  66     int vcpu_events;
  67     int robust_singlestep;
  68     int debugregs;
  69 #ifdef KVM_CAP_SET_GUEST_DEBUG
  70     struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
  71 #endif
  72     int irqchip_in_kernel;
  73     int pit_in_kernel;
  74 };
  75
  76 static KVMState *kvm_state;
  77
  78 static KVMSlot *kvm_alloc_slot(KVMState *s)
  79 {
  80     int i;
  81
  82     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
  83         /* KVM private memory slots */
  84         if (i >= 8 && i < 12)
  85             continue;
  86         if (s->slots[i].memory_size == 0)
  87             return &s->slots[i];
  88     }
  89
  90     fprintf(stderr, "%s: no free slot available\n", __func__);
  91     abort();
  92 }
  93
  94 static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
  95                                          target_phys_addr_t start_addr,
  96                                          target_phys_addr_t end_addr)
  97 {
  98     int i;
  99
 100     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 101         KVMSlot *mem = &s->slots[i];
 102
 103         if (start_addr == mem->start_addr &&
 104             end_addr == mem->start_addr + mem->memory_size) {
 105             return mem;
 106         }
 107     }
 108
 109     return NULL;
 110 }
 111
 112 /*
 113  * Find overlapping slot with lowest start address
 114  */
 115 static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
 116                                             target_phys_addr_t start_addr,
 117                                             target_phys_addr_t end_addr)
 118 {
 119     KVMSlot *found = NULL;
 120     int i;
 121
 122     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 123         KVMSlot *mem = &s->slots[i];
 124
 125         if (mem->memory_size == 0 ||
 126             (found && found->start_addr < mem->start_addr)) {
 127             continue;
 128         }
 129
 130         if (end_addr > mem->start_addr &&
 131             start_addr < mem->start_addr + mem->memory_size) {
 132             found = mem;
 133         }
 134     }
 135
 136     return found;
 137 }
 138
 139 static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
 140 {
 141     struct kvm_userspace_memory_region mem;
 142
 143     mem.slot = slot->slot;
 144     mem.guest_phys_addr = slot->start_addr;
 145     mem.memory_size = slot->memory_size;
 146     mem.userspace_addr = (unsigned long)qemu_get_ram_ptr(slot->phys_offset);
 147     mem.flags = slot->flags;
 148     if (s->migration_log) {
 149         mem.flags |= KVM_MEM_LOG_DIRTY_PAGES;
 150     }
 151     return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
 152 }
 153
 154 static void kvm_reset_vcpu(void *opaque)
 155 {
 156     CPUState *env = opaque;
 157
 158     kvm_arch_reset_vcpu(env);
 159 }
 160
 161 int kvm_irqchip_in_kernel(void)
 162 {
 163     return kvm_state->irqchip_in_kernel;
 164 }
 165
 166 int kvm_pit_in_kernel(void)
 167 {
 168     return kvm_state->pit_in_kernel;
 169 }
 170
 171
 172 int kvm_init_vcpu(CPUState *env)
 173 {
 174     KVMState *s = kvm_state;
 175     long mmap_size;
 176     int ret;
 177
 178     DPRINTF("kvm_init_vcpu\n");
 179
 180     ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, env->cpu_index);
 181     if (ret < 0) {
 182         DPRINTF("kvm_create_vcpu failed\n");
 183         goto err;
 184     }
 185
 186     env->kvm_fd = ret;
 187     env->kvm_state = s;
 188
 189     mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
 190     if (mmap_size < 0) {
 191         DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
 192         goto err;
 193     }
 194
 195     env->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
 196                         env->kvm_fd, 0);
 197     if (env->kvm_run == MAP_FAILED) {
 198         ret = -errno;
 199         DPRINTF("mmap'ing vcpu state failed\n");
 200         goto err;
 201     }
 202
 203 #ifdef KVM_CAP_COALESCED_MMIO
 204     if (s->coalesced_mmio && !s->coalesced_mmio_ring)
 205         s->coalesced_mmio_ring = (void *) env->kvm_run +
 206                 s->coalesced_mmio * PAGE_SIZE;
 207 #endif
 208
 209     ret = kvm_arch_init_vcpu(env);
 210     if (ret == 0) {
 211         qemu_register_reset(kvm_reset_vcpu, env);
 212         kvm_arch_reset_vcpu(env);
 213     }
 214 err:
 215     return ret;
 216 }
 217
 218 /*
 219  * dirty pages logging control
 220  */
 221 static int kvm_dirty_pages_log_change(target_phys_addr_t phys_addr,
 222                                       ram_addr_t size, int flags, int mask)
 223 {
 224     KVMState *s = kvm_state;
 225     KVMSlot *mem = kvm_lookup_matching_slot(s, phys_addr, phys_addr + size);
 226     int old_flags;
 227
 228     if (mem == NULL)  {
 229             fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
 230                     TARGET_FMT_plx "\n", __func__, phys_addr,
 231                     (target_phys_addr_t)(phys_addr + size - 1));
 232             return -EINVAL;
 233     }
 234
 235     old_flags = mem->flags;
 236
 237     flags = (mem->flags & ~mask) | flags;
 238     mem->flags = flags;
 239
 240     /* If nothing changed effectively, no need to issue ioctl */
 241     if (s->migration_log) {
 242         flags |= KVM_MEM_LOG_DIRTY_PAGES;
 243     }
 244     if (flags == old_flags) {
 245             return 0;
 246     }
 247
 248     return kvm_set_user_memory_region(s, mem);
 249 }
 250
 251 int kvm_log_start(target_phys_addr_t phys_addr, ram_addr_t size)
 252 {
 253         return kvm_dirty_pages_log_change(phys_addr, size,
 254                                           KVM_MEM_LOG_DIRTY_PAGES,
 255                                           KVM_MEM_LOG_DIRTY_PAGES);
 256 }
 257
 258 int kvm_log_stop(target_phys_addr_t phys_addr, ram_addr_t size)
 259 {
 260         return kvm_dirty_pages_log_change(phys_addr, size,
 261                                           0,
 262                                           KVM_MEM_LOG_DIRTY_PAGES);
 263 }
 264
 265 static int kvm_set_migration_log(int enable)
 266 {
 267     KVMState *s = kvm_state;
 268     KVMSlot *mem;
 269     int i, err;
 270
 271     s->migration_log = enable;
 272
 273     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 274         mem = &s->slots[i];
 275
 276         if (!!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) == enable) {
 277             continue;
 278         }
 279         err = kvm_set_user_memory_region(s, mem);
 280         if (err) {
 281             return err;
 282         }
 283     }
 284     return 0;
 285 }
 286
 287 /* get kvm's dirty pages bitmap and update qemu's */
 288 static int kvm_get_dirty_pages_log_range(unsigned long start_addr,
 289                                          unsigned long *bitmap,
 290                                          unsigned long offset,
 291                                          unsigned long mem_size)
 292 {
 293     unsigned int i, j;
 294     unsigned long page_number, addr, addr1, c;
 295     ram_addr_t ram_addr;
 296     unsigned int len = ((mem_size / TARGET_PAGE_SIZE) + HOST_LONG_BITS - 1) /
 297         HOST_LONG_BITS;
 298
 299     /*
 300      * bitmap-traveling is faster than memory-traveling (for addr...)
 301      * especially when most of the memory is not dirty.
 302      */
 303     for (i = 0; i < len; i++) {
 304         if (bitmap[i] != 0) {
 305             c = leul_to_cpu(bitmap[i]);
 306             do {
 307                 j = ffsl(c) - 1;
 308                 c &= ~(1ul << j);
 309                 page_number = i * HOST_LONG_BITS + j;
 310                 addr1 = page_number * TARGET_PAGE_SIZE;
 311                 addr = offset + addr1;
 312                 ram_addr = cpu_get_physical_page_desc(addr);
 313                 cpu_physical_memory_set_dirty(ram_addr);
 314             } while (c != 0);
 315         }
 316     }
 317     return 0;
 318 }
 319
 320 #define ALIGN(x, y)  (((x)+(y)-1) & ~((y)-1))
 321
 322 /**
 323  * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
 324  * This function updates qemu's dirty bitmap using cpu_physical_memory_set_dirty().
 325  * This means all bits are set to dirty.
 326  *
 327  * @start_add: start of logged region.
 328  * @end_addr: end of logged region.
 329  */
 330 static int kvm_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
 331                                           target_phys_addr_t end_addr)
 332 {
 333     KVMState *s = kvm_state;
 334     unsigned long size, allocated_size = 0;
 335     KVMDirtyLog d;
 336     KVMSlot *mem;
 337     int ret = 0;
 338
 339     d.dirty_bitmap = NULL;
 340     while (start_addr < end_addr) {
 341         mem = kvm_lookup_overlapping_slot(s, start_addr, end_addr);
 342         if (mem == NULL) {
 343             break;
 344         }
 345
 346         size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS), HOST_LONG_BITS) / 8;
 347         if (!d.dirty_bitmap) {
 348             d.dirty_bitmap = qemu_malloc(size);
 349         } else if (size > allocated_size) {
 350             d.dirty_bitmap = qemu_realloc(d.dirty_bitmap, size);
 351         }
 352         allocated_size = size;
 353         memset(d.dirty_bitmap, 0, allocated_size);
 354
 355         d.slot = mem->slot;
 356
 357         if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
 358             DPRINTF("ioctl failed %d\n", errno);
 359             ret = -1;
 360             break;
 361         }
 362
 363         kvm_get_dirty_pages_log_range(mem->start_addr, d.dirty_bitmap,
 364                                       mem->start_addr, mem->memory_size);
 365         start_addr = mem->start_addr + mem->memory_size;
 366     }
 367     qemu_free(d.dirty_bitmap);
 368
 369     return ret;
 370 }
 371
 372 int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
 373 {
 374     int ret = -ENOSYS;
 375 #ifdef KVM_CAP_COALESCED_MMIO
 376     KVMState *s = kvm_state;
 377
 378     if (s->coalesced_mmio) {
 379         struct kvm_coalesced_mmio_zone zone;
 380
 381         zone.addr = start;
 382         zone.size = size;
 383
 384         ret = kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
 385     }
 386 #endif
 387
 388     return ret;
 389 }
 390
 391 int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
 392 {
 393     int ret = -ENOSYS;
 394 #ifdef KVM_CAP_COALESCED_MMIO
 395     KVMState *s = kvm_state;
 396
 397     if (s->coalesced_mmio) {
 398         struct kvm_coalesced_mmio_zone zone;
 399
 400         zone.addr = start;
 401         zone.size = size;
 402
 403         ret = kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
 404     }
 405 #endif
 406
 407     return ret;
 408 }
 409
 410 int kvm_check_extension(KVMState *s, unsigned int extension)
 411 {
 412     int ret;
 413
 414     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
 415     if (ret < 0) {
 416         ret = 0;
 417     }
 418
 419     return ret;
 420 }
 421
 422 static void kvm_set_phys_mem(target_phys_addr_t start_addr,
 423                              ram_addr_t size,
 424                              ram_addr_t phys_offset)
 425 {
 426     KVMState *s = kvm_state;
 427     ram_addr_t flags = phys_offset & ~TARGET_PAGE_MASK;
 428     KVMSlot *mem, old;
 429     int err;
 430
 431     if (start_addr & ~TARGET_PAGE_MASK) {
 432         if (flags >= IO_MEM_UNASSIGNED) {
 433             if (!kvm_lookup_overlapping_slot(s, start_addr,
 434                                              start_addr + size)) {
 435                 return;
 436             }
 437             fprintf(stderr, "Unaligned split of a KVM memory slot\n");
 438         } else {
 439             fprintf(stderr, "Only page-aligned memory slots supported\n");
 440         }
 441         abort();
 442     }
 443
 444     /* KVM does not support read-only slots */
 445     phys_offset &= ~IO_MEM_ROM;
 446
 447     while (1) {
 448         mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size);
 449         if (!mem) {
 450             break;
 451         }
 452
 453         if (flags < IO_MEM_UNASSIGNED && start_addr >= mem->start_addr &&
 454             (start_addr + size <= mem->start_addr + mem->memory_size) &&
 455             (phys_offset - start_addr == mem->phys_offset - mem->start_addr)) {
 456             /* The new slot fits into the existing one and comes with
 457              * identical parameters - nothing to be done. */
 458             return;
 459         }
 460
 461         old = *mem;
 462
 463         /* unregister the overlapping slot */
 464         mem->memory_size = 0;
 465         err = kvm_set_user_memory_region(s, mem);
 466         if (err) {
 467             fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
 468                     __func__, strerror(-err));
 469             abort();
 470         }
 471
 472         /* Workaround for older KVM versions: we can't join slots, even not by
 473          * unregistering the previous ones and then registering the larger
 474          * slot. We have to maintain the existing fragmentation. Sigh.
 475          *
 476          * This workaround assumes that the new slot starts at the same
 477          * address as the first existing one. If not or if some overlapping
 478          * slot comes around later, we will fail (not seen in practice so far)
 479          * - and actually require a recent KVM version. */
 480         if (s->broken_set_mem_region &&
 481             old.start_addr == start_addr && old.memory_size < size &&
 482             flags < IO_MEM_UNASSIGNED) {
 483             mem = kvm_alloc_slot(s);
 484             mem->memory_size = old.memory_size;
 485             mem->start_addr = old.start_addr;
 486             mem->phys_offset = old.phys_offset;
 487             mem->flags = 0;
 488
 489             err = kvm_set_user_memory_region(s, mem);
 490             if (err) {
 491                 fprintf(stderr, "%s: error updating slot: %s\n", __func__,
 492                         strerror(-err));
 493                 abort();
 494             }
 495
 496             start_addr += old.memory_size;
 497             phys_offset += old.memory_size;
 498             size -= old.memory_size;
 499             continue;
 500         }
 501
 502         /* register prefix slot */
 503         if (old.start_addr < start_addr) {
 504             mem = kvm_alloc_slot(s);
 505             mem->memory_size = start_addr - old.start_addr;
 506             mem->start_addr = old.start_addr;
 507             mem->phys_offset = old.phys_offset;
 508             mem->flags = 0;
 509
 510             err = kvm_set_user_memory_region(s, mem);
 511             if (err) {
 512                 fprintf(stderr, "%s: error registering prefix slot: %s\n",
 513                         __func__, strerror(-err));
 514                 abort();
 515             }
 516         }
 517
 518         /* register suffix slot */
 519         if (old.start_addr + old.memory_size > start_addr + size) {
 520             ram_addr_t size_delta;
 521
 522             mem = kvm_alloc_slot(s);
 523             mem->start_addr = start_addr + size;
 524             size_delta = mem->start_addr - old.start_addr;
 525             mem->memory_size = old.memory_size - size_delta;
 526             mem->phys_offset = old.phys_offset + size_delta;
 527             mem->flags = 0;
 528
 529             err = kvm_set_user_memory_region(s, mem);
 530             if (err) {
 531                 fprintf(stderr, "%s: error registering suffix slot: %s\n",
 532                         __func__, strerror(-err));
 533                 abort();
 534             }
 535         }
 536     }
 537
 538     /* in case the KVM bug workaround already "consumed" the new slot */
 539     if (!size)
 540         return;
 541
 542     /* KVM does not need to know about this memory */
 543     if (flags >= IO_MEM_UNASSIGNED)
 544         return;
 545
 546     mem = kvm_alloc_slot(s);
 547     mem->memory_size = size;
 548     mem->start_addr = start_addr;
 549     mem->phys_offset = phys_offset;
 550     mem->flags = 0;
 551
 552     err = kvm_set_user_memory_region(s, mem);
 553     if (err) {
 554         fprintf(stderr, "%s: error registering slot: %s\n", __func__,
 555                 strerror(-err));
 556         abort();
 557     }
 558 }
 559
 560 static void kvm_client_set_memory(struct CPUPhysMemoryClient *client,
 561                                   target_phys_addr_t start_addr,
 562                                   ram_addr_t size,
 563                                   ram_addr_t phys_offset)
 564 {
 565         kvm_set_phys_mem(start_addr, size, phys_offset);
 566 }
 567
 568 static int kvm_client_sync_dirty_bitmap(struct CPUPhysMemoryClient *client,
 569                                         target_phys_addr_t start_addr,
 570                                         target_phys_addr_t end_addr)
 571 {
 572         return kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
 573 }
 574
 575 static int kvm_client_migration_log(struct CPUPhysMemoryClient *client,
 576                                     int enable)
 577 {
 578         return kvm_set_migration_log(enable);
 579 }
 580
 581 static CPUPhysMemoryClient kvm_cpu_phys_memory_client = {
 582         .set_memory = kvm_client_set_memory,
 583         .sync_dirty_bitmap = kvm_client_sync_dirty_bitmap,
 584         .migration_log = kvm_client_migration_log,
 585 };
 586
 587 int kvm_init(int smp_cpus)
 588 {
 589     static const char upgrade_note[] =
 590         "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
 591         "(see http://sourceforge.net/projects/kvm).\n";
 592     KVMState *s;
 593     int ret;
 594     int i;
 595
 596     if (smp_cpus > 1) {
 597         fprintf(stderr, "No SMP KVM support, use '-smp 1'\n");
 598         return -EINVAL;
 599     }
 600
 601     s = qemu_mallocz(sizeof(KVMState));
 602
 603 #ifdef KVM_CAP_SET_GUEST_DEBUG
 604     QTAILQ_INIT(&s->kvm_sw_breakpoints);
 605 #endif
 606     for (i = 0; i < ARRAY_SIZE(s->slots); i++)
 607         s->slots[i].slot = i;
 608
 609     s->vmfd = -1;
 610     s->fd = qemu_open("/dev/kvm", O_RDWR);
 611     if (s->fd == -1) {
 612         fprintf(stderr, "Could not access KVM kernel module: %m\n");
 613         ret = -errno;
 614         goto err;
 615     }
 616
 617     ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
 618     if (ret < KVM_API_VERSION) {
 619         if (ret > 0)
 620             ret = -EINVAL;
 621         fprintf(stderr, "kvm version too old\n");
 622         goto err;
 623     }
 624
 625     if (ret > KVM_API_VERSION) {
 626         ret = -EINVAL;
 627         fprintf(stderr, "kvm version not supported\n");
 628         goto err;
 629     }
 630
 631     s->vmfd = kvm_ioctl(s, KVM_CREATE_VM, 0);
 632     if (s->vmfd < 0) {
 633 #ifdef TARGET_S390X
 634         fprintf(stderr, "Please add the 'switch_amode' kernel parameter to "
 635                         "your host kernel command line\n");
 636 #endif
 637         goto err;
 638     }
 639
 640     /* initially, KVM allocated its own memory and we had to jump through
 641      * hooks to make phys_ram_base point to this.  Modern versions of KVM
 642      * just use a user allocated buffer so we can use regular pages
 643      * unmodified.  Make sure we have a sufficiently modern version of KVM.
 644      */
 645     if (!kvm_check_extension(s, KVM_CAP_USER_MEMORY)) {
 646         ret = -EINVAL;
 647         fprintf(stderr, "kvm does not support KVM_CAP_USER_MEMORY\n%s",
 648                 upgrade_note);
 649         goto err;
 650     }
 651
 652     /* There was a nasty bug in < kvm-80 that prevents memory slots from being
 653      * destroyed properly.  Since we rely on this capability, refuse to work
 654      * with any kernel without this capability. */
 655     if (!kvm_check_extension(s, KVM_CAP_DESTROY_MEMORY_REGION_WORKS)) {
 656         ret = -EINVAL;
 657
 658         fprintf(stderr,
 659                 "KVM kernel module broken (DESTROY_MEMORY_REGION).\n%s",
 660                 upgrade_note);
 661         goto err;
 662     }
 663
 664     s->coalesced_mmio = 0;
 665 #ifdef KVM_CAP_COALESCED_MMIO
 666     s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
 667     s->coalesced_mmio_ring = NULL;
 668 #endif
 669
 670     s->broken_set_mem_region = 1;
 671 #ifdef KVM_CAP_JOIN_MEMORY_REGIONS_WORKS
 672     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
 673     if (ret > 0) {
 674         s->broken_set_mem_region = 0;
 675     }
 676 #endif
 677
 678     s->vcpu_events = 0;
 679 #ifdef KVM_CAP_VCPU_EVENTS
 680     s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
 681 #endif
 682
 683     s->robust_singlestep = 0;
 684 #ifdef KVM_CAP_X86_ROBUST_SINGLESTEP
 685     s->robust_singlestep =
 686         kvm_check_extension(s, KVM_CAP_X86_ROBUST_SINGLESTEP);
 687 #endif
 688
 689     s->debugregs = 0;
 690 #ifdef KVM_CAP_DEBUGREGS
 691     s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS);
 692 #endif
 693
 694     ret = kvm_arch_init(s, smp_cpus);
 695     if (ret < 0)
 696         goto err;
 697
 698     kvm_state = s;
 699     cpu_register_phys_memory_client(&kvm_cpu_phys_memory_client);
 700
 701     return 0;
 702
 703 err:
 704     if (s) {
 705         if (s->vmfd != -1)
 706             close(s->vmfd);
 707         if (s->fd != -1)
 708             close(s->fd);
 709     }
 710     qemu_free(s);
 711
 712     return ret;
 713 }
 714
 715 static int kvm_handle_io(uint16_t port, void *data, int direction, int size,
 716                          uint32_t count)
 717 {
 718     int i;
 719     uint8_t *ptr = data;
 720
 721     for (i = 0; i < count; i++) {
 722         if (direction == KVM_EXIT_IO_IN) {
 723             switch (size) {
 724             case 1:
 725                 stb_p(ptr, cpu_inb(port));
 726                 break;
 727             case 2:
 728                 stw_p(ptr, cpu_inw(port));
 729                 break;
 730             case 4:
 731                 stl_p(ptr, cpu_inl(port));
 732                 break;
 733             }
 734         } else {
 735             switch (size) {
 736             case 1:
 737                 cpu_outb(port, ldub_p(ptr));
 738                 break;
 739             case 2:
 740                 cpu_outw(port, lduw_p(ptr));
 741                 break;
 742             case 4:
 743                 cpu_outl(port, ldl_p(ptr));
 744                 break;
 745             }
 746         }
 747
 748         ptr += size;
 749     }
 750
 751     return 1;
 752 }
 753
 754 #ifdef KVM_CAP_INTERNAL_ERROR_DATA
 755 static void kvm_handle_internal_error(CPUState *env, struct kvm_run *run)
 756 {
 757
 758     if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) {
 759         int i;
 760
 761         fprintf(stderr, "KVM internal error. Suberror: %d\n",
 762                 run->internal.suberror);
 763
 764         for (i = 0; i < run->internal.ndata; ++i) {
 765             fprintf(stderr, "extra data[%d]: %"PRIx64"\n",
 766                     i, (uint64_t)run->internal.data[i]);
 767         }
 768     }
 769     cpu_dump_state(env, stderr, fprintf, 0);
 770     if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) {
 771         fprintf(stderr, "emulation failure\n");
 772     }
 773     /* FIXME: Should trigger a qmp message to let management know
 774      * something went wrong.
 775      */
 776     vm_stop(0);
 777 }
 778 #endif
 779
 780 void kvm_flush_coalesced_mmio_buffer(void)
 781 {
 782 #ifdef KVM_CAP_COALESCED_MMIO
 783     KVMState *s = kvm_state;
 784     if (s->coalesced_mmio_ring) {
 785         struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
 786         while (ring->first != ring->last) {
 787             struct kvm_coalesced_mmio *ent;
 788
 789             ent = &ring->coalesced_mmio[ring->first];
 790
 791             cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
 792             smp_wmb();
 793             ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
 794         }
 795     }
 796 #endif
 797 }
 798
 799 void kvm_cpu_synchronize_state(CPUState *env)
 800 {
 801     if (!env->kvm_vcpu_dirty) {
 802         kvm_arch_get_registers(env);
 803         env->kvm_vcpu_dirty = 1;
 804     }
 805 }
 806
 807 void kvm_cpu_synchronize_post_reset(CPUState *env)
 808 {
 809     kvm_arch_put_registers(env, KVM_PUT_RESET_STATE);
 810     env->kvm_vcpu_dirty = 0;
 811 }
 812
 813 void kvm_cpu_synchronize_post_init(CPUState *env)
 814 {
 815     kvm_arch_put_registers(env, KVM_PUT_FULL_STATE);
 816     env->kvm_vcpu_dirty = 0;
 817 }
 818
 819 int kvm_cpu_exec(CPUState *env)
 820 {
 821     struct kvm_run *run = env->kvm_run;
 822     int ret;
 823
 824     DPRINTF("kvm_cpu_exec()\n");
 825
 826     do {
 827 #ifndef CONFIG_IOTHREAD
 828         if (env->exit_request) {
 829             DPRINTF("interrupt exit requested\n");
 830             ret = 0;
 831             break;
 832         }
 833 #endif
 834
 835         if (env->kvm_vcpu_dirty) {
 836             kvm_arch_put_registers(env, KVM_PUT_RUNTIME_STATE);
 837             env->kvm_vcpu_dirty = 0;
 838         }
 839
 840         kvm_arch_pre_run(env, run);
 841         qemu_mutex_unlock_iothread();
 842         ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);
 843         qemu_mutex_lock_iothread();
 844         kvm_arch_post_run(env, run);
 845
 846         if (ret == -EINTR || ret == -EAGAIN) {
 847             cpu_exit(env);
 848             DPRINTF("io window exit\n");
 849             ret = 0;
 850             break;
 851         }
 852
 853         if (ret < 0) {
 854             DPRINTF("kvm run failed %s\n", strerror(-ret));
 855             abort();
 856         }
 857
 858         kvm_flush_coalesced_mmio_buffer();
 859
 860         ret = 0; /* exit loop */
 861         switch (run->exit_reason) {
 862         case KVM_EXIT_IO:
 863             DPRINTF("handle_io\n");
 864             ret = kvm_handle_io(run->io.port,
 865                                 (uint8_t *)run + run->io.data_offset,
 866                                 run->io.direction,
 867                                 run->io.size,
 868                                 run->io.count);
 869             break;
 870         case KVM_EXIT_MMIO:
 871             DPRINTF("handle_mmio\n");
 872             cpu_physical_memory_rw(run->mmio.phys_addr,
 873                                    run->mmio.data,
 874                                    run->mmio.len,
 875                                    run->mmio.is_write);
 876             ret = 1;
 877             break;
 878         case KVM_EXIT_IRQ_WINDOW_OPEN:
 879             DPRINTF("irq_window_open\n");
 880             break;
 881         case KVM_EXIT_SHUTDOWN:
 882             DPRINTF("shutdown\n");
 883             qemu_system_reset_request();
 884             ret = 1;
 885             break;
 886         case KVM_EXIT_UNKNOWN:
 887             DPRINTF("kvm_exit_unknown\n");
 888             break;
 889         case KVM_EXIT_FAIL_ENTRY:
 890             DPRINTF("kvm_exit_fail_entry\n");
 891             break;
 892         case KVM_EXIT_EXCEPTION:
 893             DPRINTF("kvm_exit_exception\n");
 894             break;
 895 #ifdef KVM_CAP_INTERNAL_ERROR_DATA
 896         case KVM_EXIT_INTERNAL_ERROR:
 897             kvm_handle_internal_error(env, run);
 898             break;
 899 #endif
 900         case KVM_EXIT_DEBUG:
 901             DPRINTF("kvm_exit_debug\n");
 902 #ifdef KVM_CAP_SET_GUEST_DEBUG
 903             if (kvm_arch_debug(&run->debug.arch)) {
 904                 gdb_set_stop_cpu(env);
 905                 vm_stop(EXCP_DEBUG);
 906                 env->exception_index = EXCP_DEBUG;
 907                 return 0;
 908             }
 909             /* re-enter, this exception was guest-internal */
 910             ret = 1;
 911 #endif /* KVM_CAP_SET_GUEST_DEBUG */
 912             break;
 913         default:
 914             DPRINTF("kvm_arch_handle_exit\n");
 915             ret = kvm_arch_handle_exit(env, run);
 916             break;
 917         }
 918     } while (ret > 0);
 919
 920     if (env->exit_request) {
 921         env->exit_request = 0;
 922         env->exception_index = EXCP_INTERRUPT;
 923     }
 924
 925     return ret;
 926 }
 927
 928 int kvm_ioctl(KVMState *s, int type, ...)
 929 {
 930     int ret;
 931     void *arg;
 932     va_list ap;
 933
 934     va_start(ap, type);
 935     arg = va_arg(ap, void *);
 936     va_end(ap);
 937
 938     ret = ioctl(s->fd, type, arg);
 939     if (ret == -1)
 940         ret = -errno;
 941
 942     return ret;
 943 }
 944
 945 int kvm_vm_ioctl(KVMState *s, int type, ...)
 946 {
 947     int ret;
 948     void *arg;
 949     va_list ap;
 950
 951     va_start(ap, type);
 952     arg = va_arg(ap, void *);
 953     va_end(ap);
 954
 955     ret = ioctl(s->vmfd, type, arg);
 956     if (ret == -1)
 957         ret = -errno;
 958
 959     return ret;
 960 }
 961
 962 int kvm_vcpu_ioctl(CPUState *env, int type, ...)
 963 {
 964     int ret;
 965     void *arg;
 966     va_list ap;
 967
 968     va_start(ap, type);
 969     arg = va_arg(ap, void *);
 970     va_end(ap);
 971
 972     ret = ioctl(env->kvm_fd, type, arg);
 973     if (ret == -1)
 974         ret = -errno;
 975
 976     return ret;
 977 }
 978
 979 int kvm_has_sync_mmu(void)
 980 {
 981 #ifdef KVM_CAP_SYNC_MMU
 982     KVMState *s = kvm_state;
 983
 984     return kvm_check_extension(s, KVM_CAP_SYNC_MMU);
 985 #else
 986     return 0;
 987 #endif
 988 }
 989
 990 int kvm_has_vcpu_events(void)
 991 {
 992     return kvm_state->vcpu_events;
 993 }
 994
 995 int kvm_has_robust_singlestep(void)
 996 {
 997     return kvm_state->robust_singlestep;
 998 }
 999
1000 int kvm_has_debugregs(void)
1001 {
1002     return kvm_state->debugregs;
1003 }
1004
1005 void kvm_setup_guest_memory(void *start, size_t size)
1006 {
1007     if (!kvm_has_sync_mmu()) {
1008 #ifdef MADV_DONTFORK
1009         int ret = madvise(start, size, MADV_DONTFORK);
1010
1011         if (ret) {
1012             perror("madvice");
1013             exit(1);
1014         }
1015 #else
1016         fprintf(stderr,
1017                 "Need MADV_DONTFORK in absence of synchronous KVM MMU\n");
1018         exit(1);
1019 #endif
1020     }
1021 }
1022
1023 #ifdef KVM_CAP_SET_GUEST_DEBUG
1024 static void on_vcpu(CPUState *env, void (*func)(void *data), void *data)
1025 {
1026 #ifdef CONFIG_IOTHREAD
1027     if (env != cpu_single_env) {
1028         abort();
1029     }
1030 #endif
1031     func(data);
1032 }
1033
1034 struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *env,
1035                                                  target_ulong pc)
1036 {
1037     struct kvm_sw_breakpoint *bp;
1038
1039     QTAILQ_FOREACH(bp, &env->kvm_state->kvm_sw_breakpoints, entry) {
1040         if (bp->pc == pc)
1041             return bp;
1042     }
1043     return NULL;
1044 }
1045
1046 int kvm_sw_breakpoints_active(CPUState *env)
1047 {
1048     return !QTAILQ_EMPTY(&env->kvm_state->kvm_sw_breakpoints);
1049 }
1050
1051 struct kvm_set_guest_debug_data {
1052     struct kvm_guest_debug dbg;
1053     CPUState *env;
1054     int err;
1055 };
1056
1057 static void kvm_invoke_set_guest_debug(void *data)
1058 {
1059     struct kvm_set_guest_debug_data *dbg_data = data;
1060     CPUState *env = dbg_data->env;
1061
1062     dbg_data->err = kvm_vcpu_ioctl(env, KVM_SET_GUEST_DEBUG, &dbg_data->dbg);
1063 }
1064
1065 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1066 {
1067     struct kvm_set_guest_debug_data data;
1068
1069     data.dbg.control = reinject_trap;
1070
1071     if (env->singlestep_enabled) {
1072         data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
1073     }
1074     kvm_arch_update_guest_debug(env, &data.dbg);
1075     data.env = env;
1076
1077     on_vcpu(env, kvm_invoke_set_guest_debug, &data);
1078     return data.err;
1079 }
1080
1081 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1082                           target_ulong len, int type)
1083 {
1084     struct kvm_sw_breakpoint *bp;
1085     CPUState *env;
1086     int err;
1087
1088     if (type == GDB_BREAKPOINT_SW) {
1089         bp = kvm_find_sw_breakpoint(current_env, addr);
1090         if (bp) {
1091             bp->use_count++;
1092             return 0;
1093         }
1094
1095         bp = qemu_malloc(sizeof(struct kvm_sw_breakpoint));
1096         if (!bp)
1097             return -ENOMEM;
1098
1099         bp->pc = addr;
1100         bp->use_count = 1;
1101         err = kvm_arch_insert_sw_breakpoint(current_env, bp);
1102         if (err) {
1103             free(bp);
1104             return err;
1105         }
1106
1107         QTAILQ_INSERT_HEAD(&current_env->kvm_state->kvm_sw_breakpoints,
1108                           bp, entry);
1109     } else {
1110         err = kvm_arch_insert_hw_breakpoint(addr, len, type);
1111         if (err)
1112             return err;
1113     }
1114
1115     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1116         err = kvm_update_guest_debug(env, 0);
1117         if (err)
1118             return err;
1119     }
1120     return 0;
1121 }
1122
1123 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1124                           target_ulong len, int type)
1125 {
1126     struct kvm_sw_breakpoint *bp;
1127     CPUState *env;
1128     int err;
1129
1130     if (type == GDB_BREAKPOINT_SW) {
1131         bp = kvm_find_sw_breakpoint(current_env, addr);
1132         if (!bp)
1133             return -ENOENT;
1134
1135         if (bp->use_count > 1) {
1136             bp->use_count--;
1137             return 0;
1138         }
1139
1140         err = kvm_arch_remove_sw_breakpoint(current_env, bp);
1141         if (err)
1142             return err;
1143
1144         QTAILQ_REMOVE(&current_env->kvm_state->kvm_sw_breakpoints, bp, entry);
1145         qemu_free(bp);
1146     } else {
1147         err = kvm_arch_remove_hw_breakpoint(addr, len, type);
1148         if (err)
1149             return err;
1150     }
1151
1152     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1153         err = kvm_update_guest_debug(env, 0);
1154         if (err)
1155             return err;
1156     }
1157     return 0;
1158 }
1159
1160 void kvm_remove_all_breakpoints(CPUState *current_env)
1161 {
1162     struct kvm_sw_breakpoint *bp, *next;
1163     KVMState *s = current_env->kvm_state;
1164     CPUState *env;
1165
1166     QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
1167         if (kvm_arch_remove_sw_breakpoint(current_env, bp) != 0) {
1168             /* Try harder to find a CPU that currently sees the breakpoint. */
1169             for (env = first_cpu; env != NULL; env = env->next_cpu) {
1170                 if (kvm_arch_remove_sw_breakpoint(env, bp) == 0)
1171                     break;
1172             }
1173         }
1174     }
1175     kvm_arch_remove_all_hw_breakpoints();
1176
1177     for (env = first_cpu; env != NULL; env = env->next_cpu)
1178         kvm_update_guest_debug(env, 0);
1179 }
1180
1181 #else /* !KVM_CAP_SET_GUEST_DEBUG */
1182
1183 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1184 {
1185     return -EINVAL;
1186 }
1187
1188 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1189                           target_ulong len, int type)
1190 {
1191     return -EINVAL;
1192 }
1193
1194 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1195                           target_ulong len, int type)
1196 {
1197     return -EINVAL;
1198 }
1199
1200 void kvm_remove_all_breakpoints(CPUState *current_env)
1201 {
1202 }
1203 #endif /* !KVM_CAP_SET_GUEST_DEBUG */
1204
1205 int kvm_set_signal_mask(CPUState *env, const sigset_t *sigset)
1206 {
1207     struct kvm_signal_mask *sigmask;
1208     int r;
1209
1210     if (!sigset)
1211         return kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, NULL);
1212
1213     sigmask = qemu_malloc(sizeof(*sigmask) + sizeof(*sigset));
1214
1215     sigmask->len = 8;
1216     memcpy(sigmask->sigset, sigset, sizeof(*sigset));
1217     r = kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, sigmask);
1218     free(sigmask);
1219
1220     return r;
1221 }
1222
1223 int kvm_set_ioeventfd_pio_word(int fd, uint16_t addr, uint16_t val, bool assign)
1224 {
1225 #ifdef KVM_IOEVENTFD
1226     struct kvm_ioeventfd kick = {
1227         .datamatch = val,
1228         .addr = addr,
1229         .len = 2,
1230         .flags = KVM_IOEVENTFD_FLAG_DATAMATCH | KVM_IOEVENTFD_FLAG_PIO,
1231         .fd = fd,
1232     };
1233     int r;
1234     if (!kvm_enabled())
1235         return -ENOSYS;
1236     if (!assign)
1237         kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
1238     r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
1239     if (r < 0)
1240         return r;
1241     return 0;
1242 #else
1243     return -ENOSYS;
1244 #endif
1245 }