kvm-all.c

   1 /*
   2  * QEMU KVM support
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *           Red Hat, Inc. 2008
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *  Glauber Costa     <gcosta@redhat.com>
  10  *
  11  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  12  * See the COPYING file in the top-level directory.
  13  *
  14  */
  15
  16 #include <sys/types.h>
  17 #include <sys/ioctl.h>
  18 #include <sys/mman.h>
  19 #include <stdarg.h>
  20
  21 #include <linux/kvm.h>
  22
  23 #include "qemu-common.h"
  24 #include "qemu-barrier.h"
  25 #include "sysemu.h"
  26 #include "hw/hw.h"
  27 #include "gdbstub.h"
  28 #include "kvm.h"
  29 #include "bswap.h"
  30
  31 /* KVM uses PAGE_SIZE in it's definition of COALESCED_MMIO_MAX */
  32 #define PAGE_SIZE TARGET_PAGE_SIZE
  33
  34 //#define DEBUG_KVM
  35
  36 #ifdef DEBUG_KVM
  37 #define DPRINTF(fmt, ...) \
  38     do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
  39 #else
  40 #define DPRINTF(fmt, ...) \
  41     do { } while (0)
  42 #endif
  43
  44 #ifdef KVM_UPSTREAM
  45
  46 typedef struct KVMSlot
  47 {
  48     target_phys_addr_t start_addr;
  49     ram_addr_t memory_size;
  50     ram_addr_t phys_offset;
  51     int slot;
  52     int flags;
  53 } KVMSlot;
  54
  55 typedef struct kvm_dirty_log KVMDirtyLog;
  56
  57 struct KVMState
  58 {
  59     KVMSlot slots[32];
  60     int fd;
  61     int vmfd;
  62     int coalesced_mmio;
  63 #ifdef KVM_CAP_COALESCED_MMIO
  64     struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
  65 #endif
  66     int broken_set_mem_region;
  67     int migration_log;
  68     int vcpu_events;
  69     int robust_singlestep;
  70     int debugregs;
  71 #ifdef KVM_CAP_SET_GUEST_DEBUG
  72     struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
  73 #endif
  74     int irqchip_in_kernel;
  75     int pit_in_kernel;
  76     int xsave, xcrs;
  77 };
  78
  79 static KVMState *kvm_state;
  80
  81 #endif
  82
  83 static KVMSlot *kvm_alloc_slot(KVMState *s)
  84 {
  85     int i;
  86
  87     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
  88         /* KVM private memory slots */
  89         if (i >= 8 && i < 12)
  90             continue;
  91         if (s->slots[i].memory_size == 0)
  92             return &s->slots[i];
  93     }
  94
  95     fprintf(stderr, "%s: no free slot available\n", __func__);
  96     abort();
  97 }
  98
  99 static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
 100                                          target_phys_addr_t start_addr,
 101                                          target_phys_addr_t end_addr)
 102 {
 103     int i;
 104
 105     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 106         KVMSlot *mem = &s->slots[i];
 107
 108         if (start_addr == mem->start_addr &&
 109             end_addr == mem->start_addr + mem->memory_size) {
 110             return mem;
 111         }
 112     }
 113
 114     return NULL;
 115 }
 116
 117 /*
 118  * Find overlapping slot with lowest start address
 119  */
 120 static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
 121                                             target_phys_addr_t start_addr,
 122                                             target_phys_addr_t end_addr)
 123 {
 124     KVMSlot *found = NULL;
 125     int i;
 126
 127     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 128         KVMSlot *mem = &s->slots[i];
 129
 130         if (mem->memory_size == 0 ||
 131             (found && found->start_addr < mem->start_addr)) {
 132             continue;
 133         }
 134
 135         if (end_addr > mem->start_addr &&
 136             start_addr < mem->start_addr + mem->memory_size) {
 137             found = mem;
 138         }
 139     }
 140
 141     return found;
 142 }
 143
 144 static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
 145 {
 146     struct kvm_userspace_memory_region mem;
 147
 148     mem.slot = slot->slot;
 149     mem.guest_phys_addr = slot->start_addr;
 150     mem.memory_size = slot->memory_size;
 151     mem.userspace_addr = (unsigned long)qemu_get_ram_ptr(slot->phys_offset);
 152     mem.flags = slot->flags;
 153     if (s->migration_log) {
 154         mem.flags |= KVM_MEM_LOG_DIRTY_PAGES;
 155     }
 156     return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
 157 }
 158
 159 #ifdef KVM_UPSTREAM
 160 static void kvm_reset_vcpu(void *opaque)
 161 {
 162     CPUState *env = opaque;
 163
 164     kvm_arch_reset_vcpu(env);
 165 }
 166 #endif
 167
 168 int kvm_irqchip_in_kernel(void)
 169 {
 170     return kvm_state->irqchip_in_kernel;
 171 }
 172
 173 int kvm_pit_in_kernel(void)
 174 {
 175     return kvm_state->pit_in_kernel;
 176 }
 177
 178
 179 #ifdef KVM_UPSTREAM
 180 int kvm_init_vcpu(CPUState *env)
 181 {
 182     KVMState *s = kvm_state;
 183     long mmap_size;
 184     int ret;
 185
 186     DPRINTF("kvm_init_vcpu\n");
 187
 188     ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, env->cpu_index);
 189     if (ret < 0) {
 190         DPRINTF("kvm_create_vcpu failed\n");
 191         goto err;
 192     }
 193
 194     env->kvm_fd = ret;
 195     env->kvm_state = s;
 196
 197     mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
 198     if (mmap_size < 0) {
 199         DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
 200         goto err;
 201     }
 202
 203     env->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
 204                         env->kvm_fd, 0);
 205     if (env->kvm_run == MAP_FAILED) {
 206         ret = -errno;
 207         DPRINTF("mmap'ing vcpu state failed\n");
 208         goto err;
 209     }
 210
 211 #ifdef KVM_CAP_COALESCED_MMIO
 212     if (s->coalesced_mmio && !s->coalesced_mmio_ring)
 213         s->coalesced_mmio_ring = (void *) env->kvm_run +
 214                 s->coalesced_mmio * PAGE_SIZE;
 215 #endif
 216
 217     ret = kvm_arch_init_vcpu(env);
 218     if (ret == 0) {
 219         qemu_register_reset(kvm_reset_vcpu, env);
 220         kvm_arch_reset_vcpu(env);
 221     }
 222 err:
 223     return ret;
 224 }
 225 #endif
 226
 227 /*
 228  * dirty pages logging control
 229  */
 230 static int kvm_dirty_pages_log_change(target_phys_addr_t phys_addr,
 231                                       ram_addr_t size, int flags, int mask)
 232 {
 233     KVMState *s = kvm_state;
 234     KVMSlot *mem = kvm_lookup_matching_slot(s, phys_addr, phys_addr + size);
 235     int old_flags;
 236
 237     if (mem == NULL)  {
 238             fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
 239                     TARGET_FMT_plx "\n", __func__, phys_addr,
 240                     (target_phys_addr_t)(phys_addr + size - 1));
 241             return -EINVAL;
 242     }
 243
 244     old_flags = mem->flags;
 245
 246     flags = (mem->flags & ~mask) | flags;
 247     mem->flags = flags;
 248
 249     /* If nothing changed effectively, no need to issue ioctl */
 250     if (s->migration_log) {
 251         flags |= KVM_MEM_LOG_DIRTY_PAGES;
 252     }
 253     if (flags == old_flags) {
 254             return 0;
 255     }
 256
 257     return kvm_set_user_memory_region(s, mem);
 258 }
 259
 260 int kvm_log_start(target_phys_addr_t phys_addr, ram_addr_t size)
 261 {
 262         return kvm_dirty_pages_log_change(phys_addr, size,
 263                                           KVM_MEM_LOG_DIRTY_PAGES,
 264                                           KVM_MEM_LOG_DIRTY_PAGES);
 265 }
 266
 267 int kvm_log_stop(target_phys_addr_t phys_addr, ram_addr_t size)
 268 {
 269         return kvm_dirty_pages_log_change(phys_addr, size,
 270                                           0,
 271                                           KVM_MEM_LOG_DIRTY_PAGES);
 272 }
 273
 274 static int kvm_set_migration_log(int enable)
 275 {
 276     KVMState *s = kvm_state;
 277     KVMSlot *mem;
 278     int i, err;
 279
 280     s->migration_log = enable;
 281
 282     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 283         mem = &s->slots[i];
 284
 285         if (!mem->memory_size) {
 286             continue;
 287         }
 288         if (!!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) == enable) {
 289             continue;
 290         }
 291         err = kvm_set_user_memory_region(s, mem);
 292         if (err) {
 293             return err;
 294         }
 295     }
 296     return 0;
 297 }
 298
 299 /* get kvm's dirty pages bitmap and update qemu's */
 300 static int kvm_get_dirty_pages_log_range(unsigned long start_addr,
 301                                          unsigned long *bitmap,
 302                                          unsigned long offset,
 303                                          unsigned long mem_size)
 304 {
 305     unsigned int i, j;
 306     unsigned long page_number, addr, addr1, c;
 307     ram_addr_t ram_addr;
 308     unsigned int len = ((mem_size / TARGET_PAGE_SIZE) + HOST_LONG_BITS - 1) /
 309         HOST_LONG_BITS;
 310
 311     /*
 312      * bitmap-traveling is faster than memory-traveling (for addr...)
 313      * especially when most of the memory is not dirty.
 314      */
 315     for (i = 0; i < len; i++) {
 316         if (bitmap[i] != 0) {
 317             c = leul_to_cpu(bitmap[i]);
 318             do {
 319                 j = ffsl(c) - 1;
 320                 c &= ~(1ul << j);
 321                 page_number = i * HOST_LONG_BITS + j;
 322                 addr1 = page_number * TARGET_PAGE_SIZE;
 323                 addr = offset + addr1;
 324                 ram_addr = cpu_get_physical_page_desc(addr);
 325                 cpu_physical_memory_set_dirty(ram_addr);
 326             } while (c != 0);
 327         }
 328     }
 329     return 0;
 330 }
 331
 332 #define ALIGN(x, y)  (((x)+(y)-1) & ~((y)-1))
 333
 334 /**
 335  * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
 336  * This function updates qemu's dirty bitmap using cpu_physical_memory_set_dirty().
 337  * This means all bits are set to dirty.
 338  *
 339  * @start_add: start of logged region.
 340  * @end_addr: end of logged region.
 341  */
 342 static int kvm_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
 343                                           target_phys_addr_t end_addr)
 344 {
 345     KVMState *s = kvm_state;
 346     unsigned long size, allocated_size = 0;
 347     KVMDirtyLog d;
 348     KVMSlot *mem;
 349     int ret = 0;
 350
 351     d.dirty_bitmap = NULL;
 352     while (start_addr < end_addr) {
 353         mem = kvm_lookup_overlapping_slot(s, start_addr, end_addr);
 354         if (mem == NULL) {
 355             break;
 356         }
 357
 358         size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS), HOST_LONG_BITS) / 8;
 359         if (!d.dirty_bitmap) {
 360             d.dirty_bitmap = qemu_malloc(size);
 361         } else if (size > allocated_size) {
 362             d.dirty_bitmap = qemu_realloc(d.dirty_bitmap, size);
 363         }
 364         allocated_size = size;
 365         memset(d.dirty_bitmap, 0, allocated_size);
 366
 367         d.slot = mem->slot;
 368
 369         if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
 370             DPRINTF("ioctl failed %d\n", errno);
 371             ret = -1;
 372             break;
 373         }
 374
 375         kvm_get_dirty_pages_log_range(mem->start_addr, d.dirty_bitmap,
 376                                       mem->start_addr, mem->memory_size);
 377         start_addr = mem->start_addr + mem->memory_size;
 378     }
 379     qemu_free(d.dirty_bitmap);
 380
 381     return ret;
 382 }
 383
 384 int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
 385 {
 386     int ret = -ENOSYS;
 387 #ifdef KVM_CAP_COALESCED_MMIO
 388     KVMState *s = kvm_state;
 389
 390     if (s->coalesced_mmio) {
 391         struct kvm_coalesced_mmio_zone zone;
 392
 393         zone.addr = start;
 394         zone.size = size;
 395
 396         ret = kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
 397     }
 398 #endif
 399
 400     return ret;
 401 }
 402
 403 int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
 404 {
 405     int ret = -ENOSYS;
 406 #ifdef KVM_CAP_COALESCED_MMIO
 407     KVMState *s = kvm_state;
 408
 409     if (s->coalesced_mmio) {
 410         struct kvm_coalesced_mmio_zone zone;
 411
 412         zone.addr = start;
 413         zone.size = size;
 414
 415         ret = kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
 416     }
 417 #endif
 418
 419     return ret;
 420 }
 421
 422 int kvm_check_extension(KVMState *s, unsigned int extension)
 423 {
 424     int ret;
 425
 426     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
 427     if (ret < 0) {
 428         ret = 0;
 429     }
 430
 431     return ret;
 432 }
 433
 434 static void kvm_set_phys_mem(target_phys_addr_t start_addr,
 435                              ram_addr_t size,
 436                              ram_addr_t phys_offset)
 437 {
 438     KVMState *s = kvm_state;
 439     ram_addr_t flags = phys_offset & ~TARGET_PAGE_MASK;
 440     KVMSlot *mem, old;
 441     int err;
 442
 443     /* kvm works in page size chunks, but the function may be called
 444        with sub-page size and unaligned start address. */
 445     size = TARGET_PAGE_ALIGN(size);
 446     start_addr = TARGET_PAGE_ALIGN(start_addr);
 447
 448     /* KVM does not support read-only slots */
 449     phys_offset &= ~IO_MEM_ROM;
 450
 451     while (1) {
 452         mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size);
 453         if (!mem) {
 454             break;
 455         }
 456
 457         if (flags < IO_MEM_UNASSIGNED && start_addr >= mem->start_addr &&
 458             (start_addr + size <= mem->start_addr + mem->memory_size) &&
 459             (phys_offset - start_addr == mem->phys_offset - mem->start_addr)) {
 460             /* The new slot fits into the existing one and comes with
 461              * identical parameters - nothing to be done. */
 462             return;
 463         }
 464
 465         old = *mem;
 466
 467         /* unregister the overlapping slot */
 468         mem->memory_size = 0;
 469         err = kvm_set_user_memory_region(s, mem);
 470         if (err) {
 471             fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
 472                     __func__, strerror(-err));
 473             abort();
 474         }
 475
 476         /* Workaround for older KVM versions: we can't join slots, even not by
 477          * unregistering the previous ones and then registering the larger
 478          * slot. We have to maintain the existing fragmentation. Sigh.
 479          *
 480          * This workaround assumes that the new slot starts at the same
 481          * address as the first existing one. If not or if some overlapping
 482          * slot comes around later, we will fail (not seen in practice so far)
 483          * - and actually require a recent KVM version. */
 484         if (s->broken_set_mem_region &&
 485             old.start_addr == start_addr && old.memory_size < size &&
 486             flags < IO_MEM_UNASSIGNED) {
 487             mem = kvm_alloc_slot(s);
 488             mem->memory_size = old.memory_size;
 489             mem->start_addr = old.start_addr;
 490             mem->phys_offset = old.phys_offset;
 491             mem->flags = 0;
 492
 493             err = kvm_set_user_memory_region(s, mem);
 494             if (err) {
 495                 fprintf(stderr, "%s: error updating slot: %s\n", __func__,
 496                         strerror(-err));
 497                 abort();
 498             }
 499
 500             start_addr += old.memory_size;
 501             phys_offset += old.memory_size;
 502             size -= old.memory_size;
 503             continue;
 504         }
 505
 506         /* register prefix slot */
 507         if (old.start_addr < start_addr) {
 508             mem = kvm_alloc_slot(s);
 509             mem->memory_size = start_addr - old.start_addr;
 510             mem->start_addr = old.start_addr;
 511             mem->phys_offset = old.phys_offset;
 512             mem->flags = 0;
 513
 514             err = kvm_set_user_memory_region(s, mem);
 515             if (err) {
 516                 fprintf(stderr, "%s: error registering prefix slot: %s\n",
 517                         __func__, strerror(-err));
 518                 abort();
 519             }
 520         }
 521
 522         /* register suffix slot */
 523         if (old.start_addr + old.memory_size > start_addr + size) {
 524             ram_addr_t size_delta;
 525
 526             mem = kvm_alloc_slot(s);
 527             mem->start_addr = start_addr + size;
 528             size_delta = mem->start_addr - old.start_addr;
 529             mem->memory_size = old.memory_size - size_delta;
 530             mem->phys_offset = old.phys_offset + size_delta;
 531             mem->flags = 0;
 532
 533             err = kvm_set_user_memory_region(s, mem);
 534             if (err) {
 535                 fprintf(stderr, "%s: error registering suffix slot: %s\n",
 536                         __func__, strerror(-err));
 537                 abort();
 538             }
 539         }
 540     }
 541
 542     /* in case the KVM bug workaround already "consumed" the new slot */
 543     if (!size)
 544         return;
 545
 546     /* KVM does not need to know about this memory */
 547     if (flags >= IO_MEM_UNASSIGNED)
 548         return;
 549
 550     mem = kvm_alloc_slot(s);
 551     mem->memory_size = size;
 552     mem->start_addr = start_addr;
 553     mem->phys_offset = phys_offset;
 554     mem->flags = 0;
 555
 556     err = kvm_set_user_memory_region(s, mem);
 557     if (err) {
 558         fprintf(stderr, "%s: error registering slot: %s\n", __func__,
 559                 strerror(-err));
 560         abort();
 561     }
 562 }
 563
 564 static void kvm_client_set_memory(struct CPUPhysMemoryClient *client,
 565                                   target_phys_addr_t start_addr,
 566                                   ram_addr_t size,
 567                                   ram_addr_t phys_offset)
 568 {
 569         kvm_set_phys_mem(start_addr, size, phys_offset);
 570 }
 571
 572 static int kvm_client_sync_dirty_bitmap(struct CPUPhysMemoryClient *client,
 573                                         target_phys_addr_t start_addr,
 574                                         target_phys_addr_t end_addr)
 575 {
 576         return kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
 577 }
 578
 579 static int kvm_client_migration_log(struct CPUPhysMemoryClient *client,
 580                                     int enable)
 581 {
 582         return kvm_set_migration_log(enable);
 583 }
 584
 585 static CPUPhysMemoryClient kvm_cpu_phys_memory_client = {
 586         .set_memory = kvm_client_set_memory,
 587         .sync_dirty_bitmap = kvm_client_sync_dirty_bitmap,
 588         .migration_log = kvm_client_migration_log,
 589 };
 590
 591
 592 void kvm_cpu_register_phys_memory_client(void)
 593 {
 594     cpu_register_phys_memory_client(&kvm_cpu_phys_memory_client);
 595 }
 596
 597 #ifdef KVM_UPSTREAM
 598
 599 int kvm_init(int smp_cpus)
 600 {
 601     static const char upgrade_note[] =
 602         "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
 603         "(see http://sourceforge.net/projects/kvm).\n";
 604     KVMState *s;
 605     int ret;
 606     int i;
 607
 608     s = qemu_mallocz(sizeof(KVMState));
 609
 610 #ifdef KVM_CAP_SET_GUEST_DEBUG
 611     QTAILQ_INIT(&s->kvm_sw_breakpoints);
 612 #endif
 613     for (i = 0; i < ARRAY_SIZE(s->slots); i++)
 614         s->slots[i].slot = i;
 615
 616     s->vmfd = -1;
 617     s->fd = qemu_open("/dev/kvm", O_RDWR);
 618     if (s->fd == -1) {
 619         fprintf(stderr, "Could not access KVM kernel module: %m\n");
 620         ret = -errno;
 621         goto err;
 622     }
 623
 624     ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
 625     if (ret < KVM_API_VERSION) {
 626         if (ret > 0)
 627             ret = -EINVAL;
 628         fprintf(stderr, "kvm version too old\n");
 629         goto err;
 630     }
 631
 632     if (ret > KVM_API_VERSION) {
 633         ret = -EINVAL;
 634         fprintf(stderr, "kvm version not supported\n");
 635         goto err;
 636     }
 637
 638     s->vmfd = kvm_ioctl(s, KVM_CREATE_VM, 0);
 639     if (s->vmfd < 0) {
 640 #ifdef TARGET_S390X
 641         fprintf(stderr, "Please add the 'switch_amode' kernel parameter to "
 642                         "your host kernel command line\n");
 643 #endif
 644         goto err;
 645     }
 646
 647     /* initially, KVM allocated its own memory and we had to jump through
 648      * hooks to make phys_ram_base point to this.  Modern versions of KVM
 649      * just use a user allocated buffer so we can use regular pages
 650      * unmodified.  Make sure we have a sufficiently modern version of KVM.
 651      */
 652     if (!kvm_check_extension(s, KVM_CAP_USER_MEMORY)) {
 653         ret = -EINVAL;
 654         fprintf(stderr, "kvm does not support KVM_CAP_USER_MEMORY\n%s",
 655                 upgrade_note);
 656         goto err;
 657     }
 658
 659     /* There was a nasty bug in < kvm-80 that prevents memory slots from being
 660      * destroyed properly.  Since we rely on this capability, refuse to work
 661      * with any kernel without this capability. */
 662     if (!kvm_check_extension(s, KVM_CAP_DESTROY_MEMORY_REGION_WORKS)) {
 663         ret = -EINVAL;
 664
 665         fprintf(stderr,
 666                 "KVM kernel module broken (DESTROY_MEMORY_REGION).\n%s",
 667                 upgrade_note);
 668         goto err;
 669     }
 670
 671     s->coalesced_mmio = 0;
 672 #ifdef KVM_CAP_COALESCED_MMIO
 673     s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
 674     s->coalesced_mmio_ring = NULL;
 675 #endif
 676
 677     s->broken_set_mem_region = 1;
 678 #ifdef KVM_CAP_JOIN_MEMORY_REGIONS_WORKS
 679     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
 680     if (ret > 0) {
 681         s->broken_set_mem_region = 0;
 682     }
 683 #endif
 684
 685     s->vcpu_events = 0;
 686 #ifdef KVM_CAP_VCPU_EVENTS
 687     s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
 688 #endif
 689
 690     s->robust_singlestep = 0;
 691 #ifdef KVM_CAP_X86_ROBUST_SINGLESTEP
 692     s->robust_singlestep =
 693         kvm_check_extension(s, KVM_CAP_X86_ROBUST_SINGLESTEP);
 694 #endif
 695
 696     s->debugregs = 0;
 697 #ifdef KVM_CAP_DEBUGREGS
 698     s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS);
 699 #endif
 700
 701     s->xsave = 0;
 702 #ifdef KVM_CAP_XSAVE
 703     s->xsave = kvm_check_extension(s, KVM_CAP_XSAVE);
 704 #endif
 705
 706     s->xcrs = 0;
 707 #ifdef KVM_CAP_XCRS
 708     s->xcrs = kvm_check_extension(s, KVM_CAP_XCRS);
 709 #endif
 710
 711     ret = kvm_arch_init(s, smp_cpus);
 712     if (ret < 0)
 713         goto err;
 714
 715     kvm_state = s;
 716     cpu_register_phys_memory_client(&kvm_cpu_phys_memory_client);
 717
 718     return 0;
 719
 720 err:
 721     if (s) {
 722         if (s->vmfd != -1)
 723             close(s->vmfd);
 724         if (s->fd != -1)
 725             close(s->fd);
 726     }
 727     qemu_free(s);
 728
 729     return ret;
 730 }
 731 #endif
 732
 733 static int kvm_handle_io(uint16_t port, void *data, int direction, int size,
 734                          uint32_t count)
 735 {
 736     int i;
 737     uint8_t *ptr = data;
 738
 739     for (i = 0; i < count; i++) {
 740         if (direction == KVM_EXIT_IO_IN) {
 741             switch (size) {
 742             case 1:
 743                 stb_p(ptr, cpu_inb(port));
 744                 break;
 745             case 2:
 746                 stw_p(ptr, cpu_inw(port));
 747                 break;
 748             case 4:
 749                 stl_p(ptr, cpu_inl(port));
 750                 break;
 751             }
 752         } else {
 753             switch (size) {
 754             case 1:
 755                 cpu_outb(port, ldub_p(ptr));
 756                 break;
 757             case 2:
 758                 cpu_outw(port, lduw_p(ptr));
 759                 break;
 760             case 4:
 761                 cpu_outl(port, ldl_p(ptr));
 762                 break;
 763             }
 764         }
 765
 766         ptr += size;
 767     }
 768
 769     return 1;
 770 }
 771
 772 #ifdef KVM_CAP_INTERNAL_ERROR_DATA
 773 static void kvm_handle_internal_error(CPUState *env, struct kvm_run *run)
 774 {
 775
 776     if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) {
 777         int i;
 778
 779         fprintf(stderr, "KVM internal error. Suberror: %d\n",
 780                 run->internal.suberror);
 781
 782         for (i = 0; i < run->internal.ndata; ++i) {
 783             fprintf(stderr, "extra data[%d]: %"PRIx64"\n",
 784                     i, (uint64_t)run->internal.data[i]);
 785         }
 786     }
 787     cpu_dump_state(env, stderr, fprintf, 0);
 788     if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) {
 789         fprintf(stderr, "emulation failure\n");
 790         if (!kvm_arch_stop_on_emulation_error(env))
 791                 return;
 792     }
 793     /* FIXME: Should trigger a qmp message to let management know
 794      * something went wrong.
 795      */
 796     vm_stop(0);
 797 }
 798 #endif
 799
 800 void kvm_flush_coalesced_mmio_buffer(void)
 801 {
 802 #ifdef KVM_CAP_COALESCED_MMIO
 803     KVMState *s = kvm_state;
 804     if (s->coalesced_mmio_ring) {
 805         struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
 806         while (ring->first != ring->last) {
 807             struct kvm_coalesced_mmio *ent;
 808
 809             ent = &ring->coalesced_mmio[ring->first];
 810
 811             cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
 812             smp_wmb();
 813             ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
 814         }
 815     }
 816 #endif
 817 }
 818
 819 #ifdef KVM_UPSTREAM
 820
 821 static void do_kvm_cpu_synchronize_state(void *_env)
 822 {
 823     CPUState *env = _env;
 824
 825     if (!env->kvm_vcpu_dirty) {
 826         kvm_arch_get_registers(env);
 827         env->kvm_vcpu_dirty = 1;
 828     }
 829 }
 830
 831 void kvm_cpu_synchronize_state(CPUState *env)
 832 {
 833     if (!env->kvm_vcpu_dirty)
 834         run_on_cpu(env, do_kvm_cpu_synchronize_state, env);
 835 }
 836
 837 void kvm_cpu_synchronize_post_reset(CPUState *env)
 838 {
 839     kvm_arch_put_registers(env, KVM_PUT_RESET_STATE);
 840     env->kvm_vcpu_dirty = 0;
 841 }
 842
 843 void kvm_cpu_synchronize_post_init(CPUState *env)
 844 {
 845     kvm_arch_put_registers(env, KVM_PUT_FULL_STATE);
 846     env->kvm_vcpu_dirty = 0;
 847 }
 848
 849 int kvm_cpu_exec(CPUState *env)
 850 {
 851     struct kvm_run *run = env->kvm_run;
 852     int ret;
 853
 854     DPRINTF("kvm_cpu_exec()\n");
 855
 856     do {
 857 #ifndef CONFIG_IOTHREAD
 858         if (env->exit_request) {
 859             DPRINTF("interrupt exit requested\n");
 860             ret = 0;
 861             break;
 862         }
 863 #endif
 864
 865         if (kvm_arch_process_irqchip_events(env)) {
 866             ret = 0;
 867             break;
 868         }
 869
 870         if (env->kvm_vcpu_dirty) {
 871             kvm_arch_put_registers(env, KVM_PUT_RUNTIME_STATE);
 872             env->kvm_vcpu_dirty = 0;
 873         }
 874
 875         kvm_arch_pre_run(env, run);
 876         cpu_single_env = NULL;
 877         qemu_mutex_unlock_iothread();
 878         ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);
 879         qemu_mutex_lock_iothread();
 880         cpu_single_env = env;
 881         kvm_arch_post_run(env, run);
 882
 883         if (ret == -EINTR || ret == -EAGAIN) {
 884             cpu_exit(env);
 885             DPRINTF("io window exit\n");
 886             ret = 0;
 887             break;
 888         }
 889
 890         if (ret < 0) {
 891             DPRINTF("kvm run failed %s\n", strerror(-ret));
 892             abort();
 893         }
 894
 895         kvm_flush_coalesced_mmio_buffer();
 896
 897         ret = 0; /* exit loop */
 898         switch (run->exit_reason) {
 899         case KVM_EXIT_IO:
 900             DPRINTF("handle_io\n");
 901             ret = kvm_handle_io(run->io.port,
 902                                 (uint8_t *)run + run->io.data_offset,
 903                                 run->io.direction,
 904                                 run->io.size,
 905                                 run->io.count);
 906             break;
 907         case KVM_EXIT_MMIO:
 908             DPRINTF("handle_mmio\n");
 909             cpu_physical_memory_rw(run->mmio.phys_addr,
 910                                    run->mmio.data,
 911                                    run->mmio.len,
 912                                    run->mmio.is_write);
 913             ret = 1;
 914             break;
 915         case KVM_EXIT_IRQ_WINDOW_OPEN:
 916             DPRINTF("irq_window_open\n");
 917             break;
 918         case KVM_EXIT_SHUTDOWN:
 919             DPRINTF("shutdown\n");
 920             qemu_system_reset_request();
 921             ret = 1;
 922             break;
 923         case KVM_EXIT_UNKNOWN:
 924             DPRINTF("kvm_exit_unknown\n");
 925             break;
 926         case KVM_EXIT_FAIL_ENTRY:
 927             DPRINTF("kvm_exit_fail_entry\n");
 928             break;
 929         case KVM_EXIT_EXCEPTION:
 930             DPRINTF("kvm_exit_exception\n");
 931             break;
 932 #ifdef KVM_CAP_INTERNAL_ERROR_DATA
 933         case KVM_EXIT_INTERNAL_ERROR:
 934             kvm_handle_internal_error(env, run);
 935             break;
 936 #endif
 937         case KVM_EXIT_DEBUG:
 938             DPRINTF("kvm_exit_debug\n");
 939 #ifdef KVM_CAP_SET_GUEST_DEBUG
 940             if (kvm_arch_debug(&run->debug.arch)) {
 941                 env->exception_index = EXCP_DEBUG;
 942                 return 0;
 943             }
 944             /* re-enter, this exception was guest-internal */
 945             ret = 1;
 946 #endif /* KVM_CAP_SET_GUEST_DEBUG */
 947             break;
 948         default:
 949             DPRINTF("kvm_arch_handle_exit\n");
 950             ret = kvm_arch_handle_exit(env, run);
 951             break;
 952         }
 953     } while (ret > 0);
 954
 955     if (env->exit_request) {
 956         env->exit_request = 0;
 957         env->exception_index = EXCP_INTERRUPT;
 958     }
 959
 960     return ret;
 961 }
 962
 963 #endif
 964 int kvm_ioctl(KVMState *s, int type, ...)
 965 {
 966     int ret;
 967     void *arg;
 968     va_list ap;
 969
 970     va_start(ap, type);
 971     arg = va_arg(ap, void *);
 972     va_end(ap);
 973
 974     ret = ioctl(s->fd, type, arg);
 975     if (ret == -1)
 976         ret = -errno;
 977
 978     return ret;
 979 }
 980
 981 int kvm_vm_ioctl(KVMState *s, int type, ...)
 982 {
 983     int ret;
 984     void *arg;
 985     va_list ap;
 986
 987     va_start(ap, type);
 988     arg = va_arg(ap, void *);
 989     va_end(ap);
 990
 991     ret = ioctl(s->vmfd, type, arg);
 992     if (ret == -1)
 993         ret = -errno;
 994
 995     return ret;
 996 }
 997
 998 int kvm_vcpu_ioctl(CPUState *env, int type, ...)
 999 {
1000     int ret;
1001     void *arg;
1002     va_list ap;
1003
1004     va_start(ap, type);
1005     arg = va_arg(ap, void *);
1006     va_end(ap);
1007
1008     ret = ioctl(env->kvm_fd, type, arg);
1009     if (ret == -1)
1010         ret = -errno;
1011
1012     return ret;
1013 }
1014
1015 int kvm_has_sync_mmu(void)
1016 {
1017 #ifdef KVM_CAP_SYNC_MMU
1018     KVMState *s = kvm_state;
1019
1020     return kvm_check_extension(s, KVM_CAP_SYNC_MMU);
1021 #else
1022     return 0;
1023 #endif
1024 }
1025
1026 int kvm_has_vcpu_events(void)
1027 {
1028     return kvm_state->vcpu_events;
1029 }
1030
1031 int kvm_has_robust_singlestep(void)
1032 {
1033     return kvm_state->robust_singlestep;
1034 }
1035
1036 int kvm_has_debugregs(void)
1037 {
1038     return kvm_state->debugregs;
1039 }
1040
1041 #ifdef KVM_UPSTREAM
1042 int kvm_has_xsave(void)
1043 {
1044     return kvm_state->xsave;
1045 }
1046
1047 int kvm_has_xcrs(void)
1048 {
1049     return kvm_state->xcrs;
1050 }
1051 #endif
1052
1053 void kvm_setup_guest_memory(void *start, size_t size)
1054 {
1055     if (!kvm_has_sync_mmu()) {
1056 #ifdef MADV_DONTFORK
1057         int ret = madvise(start, size, MADV_DONTFORK);
1058
1059         if (ret) {
1060             perror("madvice");
1061             exit(1);
1062         }
1063 #else
1064         fprintf(stderr,
1065                 "Need MADV_DONTFORK in absence of synchronous KVM MMU\n");
1066         exit(1);
1067 #endif
1068     }
1069 }
1070
1071 #ifdef KVM_CAP_SET_GUEST_DEBUG
1072 #ifndef KVM_UPSTREAM
1073 #define run_on_cpu on_vcpu
1074 static void on_vcpu(CPUState *env, void (*func)(void *data), void *data);
1075 #endif /* !KVM_UPSTREAM */
1076
1077 struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *env,
1078                                                  target_ulong pc)
1079 {
1080     struct kvm_sw_breakpoint *bp;
1081
1082     QTAILQ_FOREACH(bp, &env->kvm_state->kvm_sw_breakpoints, entry) {
1083         if (bp->pc == pc)
1084             return bp;
1085     }
1086     return NULL;
1087 }
1088
1089 int kvm_sw_breakpoints_active(CPUState *env)
1090 {
1091     return !QTAILQ_EMPTY(&env->kvm_state->kvm_sw_breakpoints);
1092 }
1093
1094 struct kvm_set_guest_debug_data {
1095     struct kvm_guest_debug dbg;
1096     CPUState *env;
1097     int err;
1098 };
1099
1100 static void kvm_invoke_set_guest_debug(void *data)
1101 {
1102     struct kvm_set_guest_debug_data *dbg_data = data;
1103     CPUState *env = dbg_data->env;
1104
1105     dbg_data->err = kvm_vcpu_ioctl(env, KVM_SET_GUEST_DEBUG, &dbg_data->dbg);
1106 }
1107
1108 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1109 {
1110     struct kvm_set_guest_debug_data data;
1111
1112     data.dbg.control = reinject_trap;
1113
1114     if (env->singlestep_enabled) {
1115         data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
1116     }
1117     kvm_arch_update_guest_debug(env, &data.dbg);
1118     data.env = env;
1119
1120     run_on_cpu(env, kvm_invoke_set_guest_debug, &data);
1121     return data.err;
1122 }
1123
1124 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1125                           target_ulong len, int type)
1126 {
1127     struct kvm_sw_breakpoint *bp;
1128     CPUState *env;
1129     int err;
1130
1131     if (type == GDB_BREAKPOINT_SW) {
1132         bp = kvm_find_sw_breakpoint(current_env, addr);
1133         if (bp) {
1134             bp->use_count++;
1135             return 0;
1136         }
1137
1138         bp = qemu_malloc(sizeof(struct kvm_sw_breakpoint));
1139         if (!bp)
1140             return -ENOMEM;
1141
1142         bp->pc = addr;
1143         bp->use_count = 1;
1144         err = kvm_arch_insert_sw_breakpoint(current_env, bp);
1145         if (err) {
1146             free(bp);
1147             return err;
1148         }
1149
1150         QTAILQ_INSERT_HEAD(&current_env->kvm_state->kvm_sw_breakpoints,
1151                           bp, entry);
1152     } else {
1153         err = kvm_arch_insert_hw_breakpoint(addr, len, type);
1154         if (err)
1155             return err;
1156     }
1157
1158     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1159         err = kvm_update_guest_debug(env, 0);
1160         if (err)
1161             return err;
1162     }
1163     return 0;
1164 }
1165
1166 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1167                           target_ulong len, int type)
1168 {
1169     struct kvm_sw_breakpoint *bp;
1170     CPUState *env;
1171     int err;
1172
1173     if (type == GDB_BREAKPOINT_SW) {
1174         bp = kvm_find_sw_breakpoint(current_env, addr);
1175         if (!bp)
1176             return -ENOENT;
1177
1178         if (bp->use_count > 1) {
1179             bp->use_count--;
1180             return 0;
1181         }
1182
1183         err = kvm_arch_remove_sw_breakpoint(current_env, bp);
1184         if (err)
1185             return err;
1186
1187         QTAILQ_REMOVE(&current_env->kvm_state->kvm_sw_breakpoints, bp, entry);
1188         qemu_free(bp);
1189     } else {
1190         err = kvm_arch_remove_hw_breakpoint(addr, len, type);
1191         if (err)
1192             return err;
1193     }
1194
1195     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1196         err = kvm_update_guest_debug(env, 0);
1197         if (err)
1198             return err;
1199     }
1200     return 0;
1201 }
1202
1203 void kvm_remove_all_breakpoints(CPUState *current_env)
1204 {
1205     struct kvm_sw_breakpoint *bp, *next;
1206     KVMState *s = current_env->kvm_state;
1207     CPUState *env;
1208
1209     QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
1210         if (kvm_arch_remove_sw_breakpoint(current_env, bp) != 0) {
1211             /* Try harder to find a CPU that currently sees the breakpoint. */
1212             for (env = first_cpu; env != NULL; env = env->next_cpu) {
1213                 if (kvm_arch_remove_sw_breakpoint(env, bp) == 0)
1214                     break;
1215             }
1216         }
1217     }
1218     kvm_arch_remove_all_hw_breakpoints();
1219
1220     for (env = first_cpu; env != NULL; env = env->next_cpu)
1221         kvm_update_guest_debug(env, 0);
1222 }
1223
1224 #else /* !KVM_CAP_SET_GUEST_DEBUG */
1225
1226 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1227 {
1228     return -EINVAL;
1229 }
1230
1231 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1232                           target_ulong len, int type)
1233 {
1234     return -EINVAL;
1235 }
1236
1237 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1238                           target_ulong len, int type)
1239 {
1240     return -EINVAL;
1241 }
1242
1243 void kvm_remove_all_breakpoints(CPUState *current_env)
1244 {
1245 }
1246 #endif /* !KVM_CAP_SET_GUEST_DEBUG */
1247
1248 int kvm_set_signal_mask(CPUState *env, const sigset_t *sigset)
1249 {
1250     struct kvm_signal_mask *sigmask;
1251     int r;
1252
1253     if (!sigset)
1254         return kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, NULL);
1255
1256     sigmask = qemu_malloc(sizeof(*sigmask) + sizeof(*sigset));
1257
1258     sigmask->len = 8;
1259     memcpy(sigmask->sigset, sigset, sizeof(*sigset));
1260     r = kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, sigmask);
1261     free(sigmask);
1262
1263     return r;
1264 }
1265
1266 int kvm_set_ioeventfd_mmio_long(int fd, uint32_t addr, uint32_t val, bool assign)
1267 {
1268 #ifdef KVM_IOEVENTFD
1269     int ret;
1270     struct kvm_ioeventfd iofd;
1271
1272     iofd.datamatch = val;
1273     iofd.addr = addr;
1274     iofd.len = 4;
1275     iofd.flags = KVM_IOEVENTFD_FLAG_DATAMATCH;
1276     iofd.fd = fd;
1277
1278     if (!kvm_enabled()) {
1279         return -ENOSYS;
1280     }
1281
1282     if (!assign) {
1283         iofd.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
1284     }
1285
1286     ret = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &iofd);
1287
1288     if (ret < 0) {
1289         return -errno;
1290     }
1291
1292     return 0;
1293 #else
1294     return -ENOSYS;
1295 #endif
1296 }
1297
1298 int kvm_set_ioeventfd_pio_word(int fd, uint16_t addr, uint16_t val, bool assign)
1299 {
1300 #ifdef KVM_IOEVENTFD
1301     struct kvm_ioeventfd kick = {
1302         .datamatch = val,
1303         .addr = addr,
1304         .len = 2,
1305         .flags = KVM_IOEVENTFD_FLAG_DATAMATCH | KVM_IOEVENTFD_FLAG_PIO,
1306         .fd = fd,
1307     };
1308     int r;
1309     if (!kvm_enabled())
1310         return -ENOSYS;
1311     if (!assign)
1312         kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
1313     r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
1314     if (r < 0)
1315         return r;
1316     return 0;
1317 #else
1318     return -ENOSYS;
1319 #endif
1320 }
1321
1322 #if defined(KVM_IRQFD)
1323 int kvm_set_irqfd(int gsi, int fd, bool assigned)
1324 {
1325     struct kvm_irqfd irqfd = {
1326         .fd = fd,
1327         .gsi = gsi,
1328         .flags = assigned ? 0 : KVM_IRQFD_FLAG_DEASSIGN,
1329     };
1330     int r;
1331     if (!kvm_enabled() || !kvm_irqchip_in_kernel())
1332         return -ENOSYS;
1333
1334     r = kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd);
1335     if (r < 0)
1336         return r;
1337     return 0;
1338 }
1339 #endif
1340
1341 #undef PAGE_SIZE
1342 #include "qemu-kvm.c"