kvm-all.c

   1 /*
   2  * QEMU KVM support
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *           Red Hat, Inc. 2008
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *  Glauber Costa     <gcosta@redhat.com>
  10  *
  11  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  12  * See the COPYING file in the top-level directory.
  13  *
  14  */
  15
  16 #include <sys/types.h>
  17 #include <sys/ioctl.h>
  18 #include <sys/mman.h>
  19 #include <stdarg.h>
  20
  21 #include <linux/kvm.h>
  22
  23 #include "qemu-common.h"
  24 #include "qemu-barrier.h"
  25 #include "sysemu.h"
  26 #include "hw/hw.h"
  27 #include "gdbstub.h"
  28 #include "kvm.h"
  29 #include "bswap.h"
  30
  31 /* KVM uses PAGE_SIZE in it's definition of COALESCED_MMIO_MAX */
  32 #define PAGE_SIZE TARGET_PAGE_SIZE
  33
  34 //#define DEBUG_KVM
  35
  36 #ifdef DEBUG_KVM
  37 #define DPRINTF(fmt, ...) \
  38     do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
  39 #else
  40 #define DPRINTF(fmt, ...) \
  41     do { } while (0)
  42 #endif
  43
  44 #ifdef KVM_UPSTREAM
  45
  46 typedef struct KVMSlot
  47 {
  48     target_phys_addr_t start_addr;
  49     ram_addr_t memory_size;
  50     ram_addr_t phys_offset;
  51     int slot;
  52     int flags;
  53 } KVMSlot;
  54
  55 typedef struct kvm_dirty_log KVMDirtyLog;
  56
  57 struct KVMState
  58 {
  59     KVMSlot slots[32];
  60     int fd;
  61     int vmfd;
  62     int coalesced_mmio;
  63 #ifdef KVM_CAP_COALESCED_MMIO
  64     struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
  65 #endif
  66     int broken_set_mem_region;
  67     int migration_log;
  68     int vcpu_events;
  69     int robust_singlestep;
  70     int debugregs;
  71 #ifdef KVM_CAP_SET_GUEST_DEBUG
  72     struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
  73 #endif
  74     int irqchip_in_kernel;
  75     int pit_in_kernel;
  76 };
  77
  78 static KVMState *kvm_state;
  79
  80 #endif
  81
  82 static KVMSlot *kvm_alloc_slot(KVMState *s)
  83 {
  84     int i;
  85
  86     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
  87         /* KVM private memory slots */
  88         if (i >= 8 && i < 12)
  89             continue;
  90         if (s->slots[i].memory_size == 0)
  91             return &s->slots[i];
  92     }
  93
  94     fprintf(stderr, "%s: no free slot available\n", __func__);
  95     abort();
  96 }
  97
  98 static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
  99                                          target_phys_addr_t start_addr,
 100                                          target_phys_addr_t end_addr)
 101 {
 102     int i;
 103
 104     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 105         KVMSlot *mem = &s->slots[i];
 106
 107         if (start_addr == mem->start_addr &&
 108             end_addr == mem->start_addr + mem->memory_size) {
 109             return mem;
 110         }
 111     }
 112
 113     return NULL;
 114 }
 115
 116 /*
 117  * Find overlapping slot with lowest start address
 118  */
 119 static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
 120                                             target_phys_addr_t start_addr,
 121                                             target_phys_addr_t end_addr)
 122 {
 123     KVMSlot *found = NULL;
 124     int i;
 125
 126     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 127         KVMSlot *mem = &s->slots[i];
 128
 129         if (mem->memory_size == 0 ||
 130             (found && found->start_addr < mem->start_addr)) {
 131             continue;
 132         }
 133
 134         if (end_addr > mem->start_addr &&
 135             start_addr < mem->start_addr + mem->memory_size) {
 136             found = mem;
 137         }
 138     }
 139
 140     return found;
 141 }
 142
 143 static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
 144 {
 145     struct kvm_userspace_memory_region mem;
 146
 147     mem.slot = slot->slot;
 148     mem.guest_phys_addr = slot->start_addr;
 149     mem.memory_size = slot->memory_size;
 150     mem.userspace_addr = (unsigned long)qemu_get_ram_ptr(slot->phys_offset);
 151     mem.flags = slot->flags;
 152     if (s->migration_log) {
 153         mem.flags |= KVM_MEM_LOG_DIRTY_PAGES;
 154     }
 155     return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
 156 }
 157
 158 #ifdef KVM_UPSTREAM
 159 static void kvm_reset_vcpu(void *opaque)
 160 {
 161     CPUState *env = opaque;
 162
 163     kvm_arch_reset_vcpu(env);
 164 }
 165 #endif
 166
 167 int kvm_irqchip_in_kernel(void)
 168 {
 169     return kvm_state->irqchip_in_kernel;
 170 }
 171
 172 int kvm_pit_in_kernel(void)
 173 {
 174     return kvm_state->pit_in_kernel;
 175 }
 176
 177
 178 #ifdef KVM_UPSTREAM
 179 int kvm_init_vcpu(CPUState *env)
 180 {
 181     KVMState *s = kvm_state;
 182     long mmap_size;
 183     int ret;
 184
 185     DPRINTF("kvm_init_vcpu\n");
 186
 187     ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, env->cpu_index);
 188     if (ret < 0) {
 189         DPRINTF("kvm_create_vcpu failed\n");
 190         goto err;
 191     }
 192
 193     env->kvm_fd = ret;
 194     env->kvm_state = s;
 195
 196     mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
 197     if (mmap_size < 0) {
 198         DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
 199         goto err;
 200     }
 201
 202     env->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
 203                         env->kvm_fd, 0);
 204     if (env->kvm_run == MAP_FAILED) {
 205         ret = -errno;
 206         DPRINTF("mmap'ing vcpu state failed\n");
 207         goto err;
 208     }
 209
 210 #ifdef KVM_CAP_COALESCED_MMIO
 211     if (s->coalesced_mmio && !s->coalesced_mmio_ring)
 212         s->coalesced_mmio_ring = (void *) env->kvm_run +
 213                 s->coalesced_mmio * PAGE_SIZE;
 214 #endif
 215
 216     ret = kvm_arch_init_vcpu(env);
 217     if (ret == 0) {
 218         qemu_register_reset(kvm_reset_vcpu, env);
 219         kvm_arch_reset_vcpu(env);
 220     }
 221 err:
 222     return ret;
 223 }
 224 #endif
 225
 226 /*
 227  * dirty pages logging control
 228  */
 229 static int kvm_dirty_pages_log_change(target_phys_addr_t phys_addr,
 230                                       ram_addr_t size, int flags, int mask)
 231 {
 232     KVMState *s = kvm_state;
 233     KVMSlot *mem = kvm_lookup_matching_slot(s, phys_addr, phys_addr + size);
 234     int old_flags;
 235
 236     if (mem == NULL)  {
 237             fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
 238                     TARGET_FMT_plx "\n", __func__, phys_addr,
 239                     (target_phys_addr_t)(phys_addr + size - 1));
 240             return -EINVAL;
 241     }
 242
 243     old_flags = mem->flags;
 244
 245     flags = (mem->flags & ~mask) | flags;
 246     mem->flags = flags;
 247
 248     /* If nothing changed effectively, no need to issue ioctl */
 249     if (s->migration_log) {
 250         flags |= KVM_MEM_LOG_DIRTY_PAGES;
 251     }
 252     if (flags == old_flags) {
 253             return 0;
 254     }
 255
 256     return kvm_set_user_memory_region(s, mem);
 257 }
 258
 259 int kvm_log_start(target_phys_addr_t phys_addr, ram_addr_t size)
 260 {
 261         return kvm_dirty_pages_log_change(phys_addr, size,
 262                                           KVM_MEM_LOG_DIRTY_PAGES,
 263                                           KVM_MEM_LOG_DIRTY_PAGES);
 264 }
 265
 266 int kvm_log_stop(target_phys_addr_t phys_addr, ram_addr_t size)
 267 {
 268         return kvm_dirty_pages_log_change(phys_addr, size,
 269                                           0,
 270                                           KVM_MEM_LOG_DIRTY_PAGES);
 271 }
 272
 273 static int kvm_set_migration_log(int enable)
 274 {
 275     KVMState *s = kvm_state;
 276     KVMSlot *mem;
 277     int i, err;
 278
 279     s->migration_log = enable;
 280
 281     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 282         mem = &s->slots[i];
 283
 284         if (!!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) == enable) {
 285             continue;
 286         }
 287         err = kvm_set_user_memory_region(s, mem);
 288         if (err) {
 289             return err;
 290         }
 291     }
 292     return 0;
 293 }
 294
 295 /* get kvm's dirty pages bitmap and update qemu's */
 296 static int kvm_get_dirty_pages_log_range(unsigned long start_addr,
 297                                          unsigned long *bitmap,
 298                                          unsigned long offset,
 299                                          unsigned long mem_size)
 300 {
 301     unsigned int i, j;
 302     unsigned long page_number, addr, addr1, c;
 303     ram_addr_t ram_addr;
 304     unsigned int len = ((mem_size / TARGET_PAGE_SIZE) + HOST_LONG_BITS - 1) /
 305         HOST_LONG_BITS;
 306
 307     /*
 308      * bitmap-traveling is faster than memory-traveling (for addr...)
 309      * especially when most of the memory is not dirty.
 310      */
 311     for (i = 0; i < len; i++) {
 312         if (bitmap[i] != 0) {
 313             c = leul_to_cpu(bitmap[i]);
 314             do {
 315                 j = ffsl(c) - 1;
 316                 c &= ~(1ul << j);
 317                 page_number = i * HOST_LONG_BITS + j;
 318                 addr1 = page_number * TARGET_PAGE_SIZE;
 319                 addr = offset + addr1;
 320                 ram_addr = cpu_get_physical_page_desc(addr);
 321                 cpu_physical_memory_set_dirty(ram_addr);
 322             } while (c != 0);
 323         }
 324     }
 325     return 0;
 326 }
 327
 328 #define ALIGN(x, y)  (((x)+(y)-1) & ~((y)-1))
 329
 330 /**
 331  * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
 332  * This function updates qemu's dirty bitmap using cpu_physical_memory_set_dirty().
 333  * This means all bits are set to dirty.
 334  *
 335  * @start_add: start of logged region.
 336  * @end_addr: end of logged region.
 337  */
 338 static int kvm_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
 339                                           target_phys_addr_t end_addr)
 340 {
 341     KVMState *s = kvm_state;
 342     unsigned long size, allocated_size = 0;
 343     KVMDirtyLog d;
 344     KVMSlot *mem;
 345     int ret = 0;
 346
 347     d.dirty_bitmap = NULL;
 348     while (start_addr < end_addr) {
 349         mem = kvm_lookup_overlapping_slot(s, start_addr, end_addr);
 350         if (mem == NULL) {
 351             break;
 352         }
 353
 354         size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS), HOST_LONG_BITS) / 8;
 355         if (!d.dirty_bitmap) {
 356             d.dirty_bitmap = qemu_malloc(size);
 357         } else if (size > allocated_size) {
 358             d.dirty_bitmap = qemu_realloc(d.dirty_bitmap, size);
 359         }
 360         allocated_size = size;
 361         memset(d.dirty_bitmap, 0, allocated_size);
 362
 363         d.slot = mem->slot;
 364
 365         if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
 366             DPRINTF("ioctl failed %d\n", errno);
 367             ret = -1;
 368             break;
 369         }
 370
 371         kvm_get_dirty_pages_log_range(mem->start_addr, d.dirty_bitmap,
 372                                       mem->start_addr, mem->memory_size);
 373         start_addr = mem->start_addr + mem->memory_size;
 374     }
 375     qemu_free(d.dirty_bitmap);
 376
 377     return ret;
 378 }
 379
 380 int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
 381 {
 382     int ret = -ENOSYS;
 383 #ifdef KVM_CAP_COALESCED_MMIO
 384     KVMState *s = kvm_state;
 385
 386     if (s->coalesced_mmio) {
 387         struct kvm_coalesced_mmio_zone zone;
 388
 389         zone.addr = start;
 390         zone.size = size;
 391
 392         ret = kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
 393     }
 394 #endif
 395
 396     return ret;
 397 }
 398
 399 int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
 400 {
 401     int ret = -ENOSYS;
 402 #ifdef KVM_CAP_COALESCED_MMIO
 403     KVMState *s = kvm_state;
 404
 405     if (s->coalesced_mmio) {
 406         struct kvm_coalesced_mmio_zone zone;
 407
 408         zone.addr = start;
 409         zone.size = size;
 410
 411         ret = kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
 412     }
 413 #endif
 414
 415     return ret;
 416 }
 417
 418 int kvm_check_extension(KVMState *s, unsigned int extension)
 419 {
 420     int ret;
 421
 422     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
 423     if (ret < 0) {
 424         ret = 0;
 425     }
 426
 427     return ret;
 428 }
 429
 430 static void kvm_set_phys_mem(target_phys_addr_t start_addr,
 431                              ram_addr_t size,
 432                              ram_addr_t phys_offset)
 433 {
 434     KVMState *s = kvm_state;
 435     ram_addr_t flags = phys_offset & ~TARGET_PAGE_MASK;
 436     KVMSlot *mem, old;
 437     int err;
 438
 439     if (start_addr & ~TARGET_PAGE_MASK) {
 440         if (flags >= IO_MEM_UNASSIGNED) {
 441             if (!kvm_lookup_overlapping_slot(s, start_addr,
 442                                              start_addr + size)) {
 443                 return;
 444             }
 445             fprintf(stderr, "Unaligned split of a KVM memory slot\n");
 446         } else {
 447             fprintf(stderr, "Only page-aligned memory slots supported\n");
 448         }
 449         abort();
 450     }
 451
 452     /* KVM does not support read-only slots */
 453     phys_offset &= ~IO_MEM_ROM;
 454
 455     while (1) {
 456         mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size);
 457         if (!mem) {
 458             break;
 459         }
 460
 461         if (flags < IO_MEM_UNASSIGNED && start_addr >= mem->start_addr &&
 462             (start_addr + size <= mem->start_addr + mem->memory_size) &&
 463             (phys_offset - start_addr == mem->phys_offset - mem->start_addr)) {
 464             /* The new slot fits into the existing one and comes with
 465              * identical parameters - nothing to be done. */
 466             return;
 467         }
 468
 469         old = *mem;
 470
 471         /* unregister the overlapping slot */
 472         mem->memory_size = 0;
 473         err = kvm_set_user_memory_region(s, mem);
 474         if (err) {
 475             fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
 476                     __func__, strerror(-err));
 477             abort();
 478         }
 479
 480         /* Workaround for older KVM versions: we can't join slots, even not by
 481          * unregistering the previous ones and then registering the larger
 482          * slot. We have to maintain the existing fragmentation. Sigh.
 483          *
 484          * This workaround assumes that the new slot starts at the same
 485          * address as the first existing one. If not or if some overlapping
 486          * slot comes around later, we will fail (not seen in practice so far)
 487          * - and actually require a recent KVM version. */
 488         if (s->broken_set_mem_region &&
 489             old.start_addr == start_addr && old.memory_size < size &&
 490             flags < IO_MEM_UNASSIGNED) {
 491             mem = kvm_alloc_slot(s);
 492             mem->memory_size = old.memory_size;
 493             mem->start_addr = old.start_addr;
 494             mem->phys_offset = old.phys_offset;
 495             mem->flags = 0;
 496
 497             err = kvm_set_user_memory_region(s, mem);
 498             if (err) {
 499                 fprintf(stderr, "%s: error updating slot: %s\n", __func__,
 500                         strerror(-err));
 501                 abort();
 502             }
 503
 504             start_addr += old.memory_size;
 505             phys_offset += old.memory_size;
 506             size -= old.memory_size;
 507             continue;
 508         }
 509
 510         /* register prefix slot */
 511         if (old.start_addr < start_addr) {
 512             mem = kvm_alloc_slot(s);
 513             mem->memory_size = start_addr - old.start_addr;
 514             mem->start_addr = old.start_addr;
 515             mem->phys_offset = old.phys_offset;
 516             mem->flags = 0;
 517
 518             err = kvm_set_user_memory_region(s, mem);
 519             if (err) {
 520                 fprintf(stderr, "%s: error registering prefix slot: %s\n",
 521                         __func__, strerror(-err));
 522                 abort();
 523             }
 524         }
 525
 526         /* register suffix slot */
 527         if (old.start_addr + old.memory_size > start_addr + size) {
 528             ram_addr_t size_delta;
 529
 530             mem = kvm_alloc_slot(s);
 531             mem->start_addr = start_addr + size;
 532             size_delta = mem->start_addr - old.start_addr;
 533             mem->memory_size = old.memory_size - size_delta;
 534             mem->phys_offset = old.phys_offset + size_delta;
 535             mem->flags = 0;
 536
 537             err = kvm_set_user_memory_region(s, mem);
 538             if (err) {
 539                 fprintf(stderr, "%s: error registering suffix slot: %s\n",
 540                         __func__, strerror(-err));
 541                 abort();
 542             }
 543         }
 544     }
 545
 546     /* in case the KVM bug workaround already "consumed" the new slot */
 547     if (!size)
 548         return;
 549
 550     /* KVM does not need to know about this memory */
 551     if (flags >= IO_MEM_UNASSIGNED)
 552         return;
 553
 554     mem = kvm_alloc_slot(s);
 555     mem->memory_size = size;
 556     mem->start_addr = start_addr;
 557     mem->phys_offset = phys_offset;
 558     mem->flags = 0;
 559
 560     err = kvm_set_user_memory_region(s, mem);
 561     if (err) {
 562         fprintf(stderr, "%s: error registering slot: %s\n", __func__,
 563                 strerror(-err));
 564         abort();
 565     }
 566 }
 567
 568 static void kvm_client_set_memory(struct CPUPhysMemoryClient *client,
 569                                   target_phys_addr_t start_addr,
 570                                   ram_addr_t size,
 571                                   ram_addr_t phys_offset)
 572 {
 573         kvm_set_phys_mem(start_addr, size, phys_offset);
 574 }
 575
 576 static int kvm_client_sync_dirty_bitmap(struct CPUPhysMemoryClient *client,
 577                                         target_phys_addr_t start_addr,
 578                                         target_phys_addr_t end_addr)
 579 {
 580         return kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
 581 }
 582
 583 static int kvm_client_migration_log(struct CPUPhysMemoryClient *client,
 584                                     int enable)
 585 {
 586         return kvm_set_migration_log(enable);
 587 }
 588
 589 static CPUPhysMemoryClient kvm_cpu_phys_memory_client = {
 590         .set_memory = kvm_client_set_memory,
 591         .sync_dirty_bitmap = kvm_client_sync_dirty_bitmap,
 592         .migration_log = kvm_client_migration_log,
 593 };
 594
 595
 596 void kvm_cpu_register_phys_memory_client(void)
 597 {
 598     cpu_register_phys_memory_client(&kvm_cpu_phys_memory_client);
 599 }
 600
 601 #ifdef KVM_UPSTREAM
 602
 603 int kvm_init(int smp_cpus)
 604 {
 605     static const char upgrade_note[] =
 606         "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
 607         "(see http://sourceforge.net/projects/kvm).\n";
 608     KVMState *s;
 609     int ret;
 610     int i;
 611
 612     s = qemu_mallocz(sizeof(KVMState));
 613
 614 #ifdef KVM_CAP_SET_GUEST_DEBUG
 615     QTAILQ_INIT(&s->kvm_sw_breakpoints);
 616 #endif
 617     for (i = 0; i < ARRAY_SIZE(s->slots); i++)
 618         s->slots[i].slot = i;
 619
 620     s->vmfd = -1;
 621     s->fd = qemu_open("/dev/kvm", O_RDWR);
 622     if (s->fd == -1) {
 623         fprintf(stderr, "Could not access KVM kernel module: %m\n");
 624         ret = -errno;
 625         goto err;
 626     }
 627
 628     ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
 629     if (ret < KVM_API_VERSION) {
 630         if (ret > 0)
 631             ret = -EINVAL;
 632         fprintf(stderr, "kvm version too old\n");
 633         goto err;
 634     }
 635
 636     if (ret > KVM_API_VERSION) {
 637         ret = -EINVAL;
 638         fprintf(stderr, "kvm version not supported\n");
 639         goto err;
 640     }
 641
 642     s->vmfd = kvm_ioctl(s, KVM_CREATE_VM, 0);
 643     if (s->vmfd < 0) {
 644 #ifdef TARGET_S390X
 645         fprintf(stderr, "Please add the 'switch_amode' kernel parameter to "
 646                         "your host kernel command line\n");
 647 #endif
 648         goto err;
 649     }
 650
 651     /* initially, KVM allocated its own memory and we had to jump through
 652      * hooks to make phys_ram_base point to this.  Modern versions of KVM
 653      * just use a user allocated buffer so we can use regular pages
 654      * unmodified.  Make sure we have a sufficiently modern version of KVM.
 655      */
 656     if (!kvm_check_extension(s, KVM_CAP_USER_MEMORY)) {
 657         ret = -EINVAL;
 658         fprintf(stderr, "kvm does not support KVM_CAP_USER_MEMORY\n%s",
 659                 upgrade_note);
 660         goto err;
 661     }
 662
 663     /* There was a nasty bug in < kvm-80 that prevents memory slots from being
 664      * destroyed properly.  Since we rely on this capability, refuse to work
 665      * with any kernel without this capability. */
 666     if (!kvm_check_extension(s, KVM_CAP_DESTROY_MEMORY_REGION_WORKS)) {
 667         ret = -EINVAL;
 668
 669         fprintf(stderr,
 670                 "KVM kernel module broken (DESTROY_MEMORY_REGION).\n%s",
 671                 upgrade_note);
 672         goto err;
 673     }
 674
 675     s->coalesced_mmio = 0;
 676 #ifdef KVM_CAP_COALESCED_MMIO
 677     s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
 678     s->coalesced_mmio_ring = NULL;
 679 #endif
 680
 681     s->broken_set_mem_region = 1;
 682 #ifdef KVM_CAP_JOIN_MEMORY_REGIONS_WORKS
 683     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
 684     if (ret > 0) {
 685         s->broken_set_mem_region = 0;
 686     }
 687 #endif
 688
 689     s->vcpu_events = 0;
 690 #ifdef KVM_CAP_VCPU_EVENTS
 691     s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
 692 #endif
 693
 694     s->robust_singlestep = 0;
 695 #ifdef KVM_CAP_X86_ROBUST_SINGLESTEP
 696     s->robust_singlestep =
 697         kvm_check_extension(s, KVM_CAP_X86_ROBUST_SINGLESTEP);
 698 #endif
 699
 700     s->debugregs = 0;
 701 #ifdef KVM_CAP_DEBUGREGS
 702     s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS);
 703 #endif
 704
 705     ret = kvm_arch_init(s, smp_cpus);
 706     if (ret < 0)
 707         goto err;
 708
 709     kvm_state = s;
 710     cpu_register_phys_memory_client(&kvm_cpu_phys_memory_client);
 711
 712     return 0;
 713
 714 err:
 715     if (s) {
 716         if (s->vmfd != -1)
 717             close(s->vmfd);
 718         if (s->fd != -1)
 719             close(s->fd);
 720     }
 721     qemu_free(s);
 722
 723     return ret;
 724 }
 725 #endif
 726
 727 static int kvm_handle_io(uint16_t port, void *data, int direction, int size,
 728                          uint32_t count)
 729 {
 730     int i;
 731     uint8_t *ptr = data;
 732
 733     for (i = 0; i < count; i++) {
 734         if (direction == KVM_EXIT_IO_IN) {
 735             switch (size) {
 736             case 1:
 737                 stb_p(ptr, cpu_inb(port));
 738                 break;
 739             case 2:
 740                 stw_p(ptr, cpu_inw(port));
 741                 break;
 742             case 4:
 743                 stl_p(ptr, cpu_inl(port));
 744                 break;
 745             }
 746         } else {
 747             switch (size) {
 748             case 1:
 749                 cpu_outb(port, ldub_p(ptr));
 750                 break;
 751             case 2:
 752                 cpu_outw(port, lduw_p(ptr));
 753                 break;
 754             case 4:
 755                 cpu_outl(port, ldl_p(ptr));
 756                 break;
 757             }
 758         }
 759
 760         ptr += size;
 761     }
 762
 763     return 1;
 764 }
 765
 766 #ifdef KVM_CAP_INTERNAL_ERROR_DATA
 767 static void kvm_handle_internal_error(CPUState *env, struct kvm_run *run)
 768 {
 769
 770     if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) {
 771         int i;
 772
 773         fprintf(stderr, "KVM internal error. Suberror: %d\n",
 774                 run->internal.suberror);
 775
 776         for (i = 0; i < run->internal.ndata; ++i) {
 777             fprintf(stderr, "extra data[%d]: %"PRIx64"\n",
 778                     i, (uint64_t)run->internal.data[i]);
 779         }
 780     }
 781     cpu_dump_state(env, stderr, fprintf, 0);
 782     if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) {
 783         fprintf(stderr, "emulation failure\n");
 784         if (!kvm_arch_stop_on_emulation_error(env))
 785                 return;
 786     }
 787     /* FIXME: Should trigger a qmp message to let management know
 788      * something went wrong.
 789      */
 790     vm_stop(0);
 791 }
 792 #endif
 793
 794 void kvm_flush_coalesced_mmio_buffer(void)
 795 {
 796 #ifdef KVM_CAP_COALESCED_MMIO
 797     KVMState *s = kvm_state;
 798     if (s->coalesced_mmio_ring) {
 799         struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
 800         while (ring->first != ring->last) {
 801             struct kvm_coalesced_mmio *ent;
 802
 803             ent = &ring->coalesced_mmio[ring->first];
 804
 805             cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
 806             smp_wmb();
 807             ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
 808         }
 809     }
 810 #endif
 811 }
 812
 813 #ifdef KVM_UPSTREAM
 814
 815 static void do_kvm_cpu_synchronize_state(void *_env)
 816 {
 817     CPUState *env = _env;
 818
 819     if (!env->kvm_vcpu_dirty) {
 820         kvm_arch_get_registers(env);
 821         env->kvm_vcpu_dirty = 1;
 822     }
 823 }
 824
 825 void kvm_cpu_synchronize_state(CPUState *env)
 826 {
 827     if (!env->kvm_vcpu_dirty)
 828         run_on_cpu(env, do_kvm_cpu_synchronize_state, env);
 829 }
 830
 831 void kvm_cpu_synchronize_post_reset(CPUState *env)
 832 {
 833     kvm_arch_put_registers(env, KVM_PUT_RESET_STATE);
 834     env->kvm_vcpu_dirty = 0;
 835 }
 836
 837 void kvm_cpu_synchronize_post_init(CPUState *env)
 838 {
 839     kvm_arch_put_registers(env, KVM_PUT_FULL_STATE);
 840     env->kvm_vcpu_dirty = 0;
 841 }
 842
 843 int kvm_cpu_exec(CPUState *env)
 844 {
 845     struct kvm_run *run = env->kvm_run;
 846     int ret;
 847
 848     DPRINTF("kvm_cpu_exec()\n");
 849
 850     do {
 851 #ifndef CONFIG_IOTHREAD
 852         if (env->exit_request) {
 853             DPRINTF("interrupt exit requested\n");
 854             ret = 0;
 855             break;
 856         }
 857 #endif
 858
 859         if (kvm_arch_process_irqchip_events(env)) {
 860             ret = 0;
 861             break;
 862         }
 863
 864         if (env->kvm_vcpu_dirty) {
 865             kvm_arch_put_registers(env, KVM_PUT_RUNTIME_STATE);
 866             env->kvm_vcpu_dirty = 0;
 867         }
 868
 869         kvm_arch_pre_run(env, run);
 870         cpu_single_env = NULL;
 871         qemu_mutex_unlock_iothread();
 872         ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);
 873         qemu_mutex_lock_iothread();
 874         cpu_single_env = env;
 875         kvm_arch_post_run(env, run);
 876
 877         if (ret == -EINTR || ret == -EAGAIN) {
 878             cpu_exit(env);
 879             DPRINTF("io window exit\n");
 880             ret = 0;
 881             break;
 882         }
 883
 884         if (ret < 0) {
 885             DPRINTF("kvm run failed %s\n", strerror(-ret));
 886             abort();
 887         }
 888
 889         kvm_flush_coalesced_mmio_buffer();
 890
 891         ret = 0; /* exit loop */
 892         switch (run->exit_reason) {
 893         case KVM_EXIT_IO:
 894             DPRINTF("handle_io\n");
 895             ret = kvm_handle_io(run->io.port,
 896                                 (uint8_t *)run + run->io.data_offset,
 897                                 run->io.direction,
 898                                 run->io.size,
 899                                 run->io.count);
 900             break;
 901         case KVM_EXIT_MMIO:
 902             DPRINTF("handle_mmio\n");
 903             cpu_physical_memory_rw(run->mmio.phys_addr,
 904                                    run->mmio.data,
 905                                    run->mmio.len,
 906                                    run->mmio.is_write);
 907             ret = 1;
 908             break;
 909         case KVM_EXIT_IRQ_WINDOW_OPEN:
 910             DPRINTF("irq_window_open\n");
 911             break;
 912         case KVM_EXIT_SHUTDOWN:
 913             DPRINTF("shutdown\n");
 914             qemu_system_reset_request();
 915             ret = 1;
 916             break;
 917         case KVM_EXIT_UNKNOWN:
 918             DPRINTF("kvm_exit_unknown\n");
 919             break;
 920         case KVM_EXIT_FAIL_ENTRY:
 921             DPRINTF("kvm_exit_fail_entry\n");
 922             break;
 923         case KVM_EXIT_EXCEPTION:
 924             DPRINTF("kvm_exit_exception\n");
 925             break;
 926 #ifdef KVM_CAP_INTERNAL_ERROR_DATA
 927         case KVM_EXIT_INTERNAL_ERROR:
 928             kvm_handle_internal_error(env, run);
 929             break;
 930 #endif
 931         case KVM_EXIT_DEBUG:
 932             DPRINTF("kvm_exit_debug\n");
 933 #ifdef KVM_CAP_SET_GUEST_DEBUG
 934             if (kvm_arch_debug(&run->debug.arch)) {
 935                 gdb_set_stop_cpu(env);
 936                 vm_stop(EXCP_DEBUG);
 937                 env->exception_index = EXCP_DEBUG;
 938                 return 0;
 939             }
 940             /* re-enter, this exception was guest-internal */
 941             ret = 1;
 942 #endif /* KVM_CAP_SET_GUEST_DEBUG */
 943             break;
 944         default:
 945             DPRINTF("kvm_arch_handle_exit\n");
 946             ret = kvm_arch_handle_exit(env, run);
 947             break;
 948         }
 949     } while (ret > 0);
 950
 951     if (env->exit_request) {
 952         env->exit_request = 0;
 953         env->exception_index = EXCP_INTERRUPT;
 954     }
 955
 956     return ret;
 957 }
 958
 959 #endif
 960 int kvm_ioctl(KVMState *s, int type, ...)
 961 {
 962     int ret;
 963     void *arg;
 964     va_list ap;
 965
 966     va_start(ap, type);
 967     arg = va_arg(ap, void *);
 968     va_end(ap);
 969
 970     ret = ioctl(s->fd, type, arg);
 971     if (ret == -1)
 972         ret = -errno;
 973
 974     return ret;
 975 }
 976
 977 int kvm_vm_ioctl(KVMState *s, int type, ...)
 978 {
 979     int ret;
 980     void *arg;
 981     va_list ap;
 982
 983     va_start(ap, type);
 984     arg = va_arg(ap, void *);
 985     va_end(ap);
 986
 987     ret = ioctl(s->vmfd, type, arg);
 988     if (ret == -1)
 989         ret = -errno;
 990
 991     return ret;
 992 }
 993
 994 int kvm_vcpu_ioctl(CPUState *env, int type, ...)
 995 {
 996     int ret;
 997     void *arg;
 998     va_list ap;
 999
1000     va_start(ap, type);
1001     arg = va_arg(ap, void *);
1002     va_end(ap);
1003
1004     ret = ioctl(env->kvm_fd, type, arg);
1005     if (ret == -1)
1006         ret = -errno;
1007
1008     return ret;
1009 }
1010
1011 int kvm_has_sync_mmu(void)
1012 {
1013 #ifdef KVM_CAP_SYNC_MMU
1014     KVMState *s = kvm_state;
1015
1016     return kvm_check_extension(s, KVM_CAP_SYNC_MMU);
1017 #else
1018     return 0;
1019 #endif
1020 }
1021
1022 int kvm_has_vcpu_events(void)
1023 {
1024     return kvm_state->vcpu_events;
1025 }
1026
1027 int kvm_has_robust_singlestep(void)
1028 {
1029     return kvm_state->robust_singlestep;
1030 }
1031
1032 int kvm_has_debugregs(void)
1033 {
1034     return kvm_state->debugregs;
1035 }
1036
1037 void kvm_setup_guest_memory(void *start, size_t size)
1038 {
1039     if (!kvm_has_sync_mmu()) {
1040 #ifdef MADV_DONTFORK
1041         int ret = madvise(start, size, MADV_DONTFORK);
1042
1043         if (ret) {
1044             perror("madvice");
1045             exit(1);
1046         }
1047 #else
1048         fprintf(stderr,
1049                 "Need MADV_DONTFORK in absence of synchronous KVM MMU\n");
1050         exit(1);
1051 #endif
1052     }
1053 }
1054
1055 #ifdef KVM_CAP_SET_GUEST_DEBUG
1056
1057 #ifdef KVM_UPSTREAM
1058 static void on_vcpu(CPUState *env, void (*func)(void *data), void *data)
1059 {
1060 #ifdef CONFIG_IOTHREAD
1061     if (env != cpu_single_env) {
1062         abort();
1063     }
1064 #endif
1065     func(data);
1066 }
1067 #else /* !KVM_UPSTREAM */
1068 static void on_vcpu(CPUState *env, void (*func)(void *data), void *data);
1069 #endif /* !KVM_UPSTREAM */
1070
1071 struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *env,
1072                                                  target_ulong pc)
1073 {
1074     struct kvm_sw_breakpoint *bp;
1075
1076     QTAILQ_FOREACH(bp, &env->kvm_state->kvm_sw_breakpoints, entry) {
1077         if (bp->pc == pc)
1078             return bp;
1079     }
1080     return NULL;
1081 }
1082
1083 int kvm_sw_breakpoints_active(CPUState *env)
1084 {
1085     return !QTAILQ_EMPTY(&env->kvm_state->kvm_sw_breakpoints);
1086 }
1087
1088 struct kvm_set_guest_debug_data {
1089     struct kvm_guest_debug dbg;
1090     CPUState *env;
1091     int err;
1092 };
1093
1094 static void kvm_invoke_set_guest_debug(void *data)
1095 {
1096     struct kvm_set_guest_debug_data *dbg_data = data;
1097     CPUState *env = dbg_data->env;
1098
1099     dbg_data->err = kvm_vcpu_ioctl(env, KVM_SET_GUEST_DEBUG, &dbg_data->dbg);
1100 }
1101
1102 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1103 {
1104     struct kvm_set_guest_debug_data data;
1105
1106     data.dbg.control = reinject_trap;
1107
1108     if (env->singlestep_enabled) {
1109         data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
1110     }
1111     kvm_arch_update_guest_debug(env, &data.dbg);
1112     data.env = env;
1113
1114     on_vcpu(env, kvm_invoke_set_guest_debug, &data);
1115     return data.err;
1116 }
1117
1118 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1119                           target_ulong len, int type)
1120 {
1121     struct kvm_sw_breakpoint *bp;
1122     CPUState *env;
1123     int err;
1124
1125     if (type == GDB_BREAKPOINT_SW) {
1126         bp = kvm_find_sw_breakpoint(current_env, addr);
1127         if (bp) {
1128             bp->use_count++;
1129             return 0;
1130         }
1131
1132         bp = qemu_malloc(sizeof(struct kvm_sw_breakpoint));
1133         if (!bp)
1134             return -ENOMEM;
1135
1136         bp->pc = addr;
1137         bp->use_count = 1;
1138         err = kvm_arch_insert_sw_breakpoint(current_env, bp);
1139         if (err) {
1140             free(bp);
1141             return err;
1142         }
1143
1144         QTAILQ_INSERT_HEAD(&current_env->kvm_state->kvm_sw_breakpoints,
1145                           bp, entry);
1146     } else {
1147         err = kvm_arch_insert_hw_breakpoint(addr, len, type);
1148         if (err)
1149             return err;
1150     }
1151
1152     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1153         err = kvm_update_guest_debug(env, 0);
1154         if (err)
1155             return err;
1156     }
1157     return 0;
1158 }
1159
1160 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1161                           target_ulong len, int type)
1162 {
1163     struct kvm_sw_breakpoint *bp;
1164     CPUState *env;
1165     int err;
1166
1167     if (type == GDB_BREAKPOINT_SW) {
1168         bp = kvm_find_sw_breakpoint(current_env, addr);
1169         if (!bp)
1170             return -ENOENT;
1171
1172         if (bp->use_count > 1) {
1173             bp->use_count--;
1174             return 0;
1175         }
1176
1177         err = kvm_arch_remove_sw_breakpoint(current_env, bp);
1178         if (err)
1179             return err;
1180
1181         QTAILQ_REMOVE(&current_env->kvm_state->kvm_sw_breakpoints, bp, entry);
1182         qemu_free(bp);
1183     } else {
1184         err = kvm_arch_remove_hw_breakpoint(addr, len, type);
1185         if (err)
1186             return err;
1187     }
1188
1189     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1190         err = kvm_update_guest_debug(env, 0);
1191         if (err)
1192             return err;
1193     }
1194     return 0;
1195 }
1196
1197 void kvm_remove_all_breakpoints(CPUState *current_env)
1198 {
1199     struct kvm_sw_breakpoint *bp, *next;
1200     KVMState *s = current_env->kvm_state;
1201     CPUState *env;
1202
1203     QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
1204         if (kvm_arch_remove_sw_breakpoint(current_env, bp) != 0) {
1205             /* Try harder to find a CPU that currently sees the breakpoint. */
1206             for (env = first_cpu; env != NULL; env = env->next_cpu) {
1207                 if (kvm_arch_remove_sw_breakpoint(env, bp) == 0)
1208                     break;
1209             }
1210         }
1211     }
1212     kvm_arch_remove_all_hw_breakpoints();
1213
1214     for (env = first_cpu; env != NULL; env = env->next_cpu)
1215         kvm_update_guest_debug(env, 0);
1216 }
1217
1218 #else /* !KVM_CAP_SET_GUEST_DEBUG */
1219
1220 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1221 {
1222     return -EINVAL;
1223 }
1224
1225 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1226                           target_ulong len, int type)
1227 {
1228     return -EINVAL;
1229 }
1230
1231 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1232                           target_ulong len, int type)
1233 {
1234     return -EINVAL;
1235 }
1236
1237 void kvm_remove_all_breakpoints(CPUState *current_env)
1238 {
1239 }
1240 #endif /* !KVM_CAP_SET_GUEST_DEBUG */
1241
1242 int kvm_set_signal_mask(CPUState *env, const sigset_t *sigset)
1243 {
1244     struct kvm_signal_mask *sigmask;
1245     int r;
1246
1247     if (!sigset)
1248         return kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, NULL);
1249
1250     sigmask = qemu_malloc(sizeof(*sigmask) + sizeof(*sigset));
1251
1252     sigmask->len = 8;
1253     memcpy(sigmask->sigset, sigset, sizeof(*sigset));
1254     r = kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, sigmask);
1255     free(sigmask);
1256
1257     return r;
1258 }
1259
1260 int kvm_set_ioeventfd_pio_word(int fd, uint16_t addr, uint16_t val, bool assign)
1261 {
1262 #ifdef KVM_IOEVENTFD
1263     struct kvm_ioeventfd kick = {
1264         .datamatch = val,
1265         .addr = addr,
1266         .len = 2,
1267         .flags = KVM_IOEVENTFD_FLAG_DATAMATCH | KVM_IOEVENTFD_FLAG_PIO,
1268         .fd = fd,
1269     };
1270     int r;
1271     if (!kvm_enabled())
1272         return -ENOSYS;
1273     if (!assign)
1274         kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
1275     r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
1276     if (r < 0)
1277         return r;
1278     return 0;
1279 #else
1280     return -ENOSYS;
1281 #endif
1282 }
1283
1284 #if defined(KVM_IRQFD)
1285 int kvm_set_irqfd(int gsi, int fd, bool assigned)
1286 {
1287     struct kvm_irqfd irqfd = {
1288         .fd = fd,
1289         .gsi = gsi,
1290         .flags = assigned ? 0 : KVM_IRQFD_FLAG_DEASSIGN,
1291     };
1292     int r;
1293     if (!kvm_enabled() || !kvm_irqchip_in_kernel())
1294         return -ENOSYS;
1295
1296     r = kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd);
1297     if (r < 0)
1298         return r;
1299     return 0;
1300 }
1301 #endif
1302
1303 #undef PAGE_SIZE
1304 #include "qemu-kvm.c"