kvm-all.c

   1 /*
   2  * QEMU KVM support
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *           Red Hat, Inc. 2008
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *  Glauber Costa     <gcosta@redhat.com>
  10  *
  11  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  12  * See the COPYING file in the top-level directory.
  13  *
  14  */
  15
  16 #include <sys/types.h>
  17 #include <sys/ioctl.h>
  18 #include <sys/mman.h>
  19 #include <stdarg.h>
  20
  21 #include <linux/kvm.h>
  22
  23 #include "qemu-common.h"
  24 #include "qemu-barrier.h"
  25 #include "sysemu.h"
  26 #include "hw/hw.h"
  27 #include "gdbstub.h"
  28 #include "kvm.h"
  29 #include "bswap.h"
  30
  31 /* KVM uses PAGE_SIZE in it's definition of COALESCED_MMIO_MAX */
  32 #define PAGE_SIZE TARGET_PAGE_SIZE
  33
  34 //#define DEBUG_KVM
  35
  36 #ifdef DEBUG_KVM
  37 #define DPRINTF(fmt, ...) \
  38     do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
  39 #else
  40 #define DPRINTF(fmt, ...) \
  41     do { } while (0)
  42 #endif
  43
  44 #ifdef KVM_UPSTREAM
  45
  46 typedef struct KVMSlot
  47 {
  48     target_phys_addr_t start_addr;
  49     ram_addr_t memory_size;
  50     ram_addr_t phys_offset;
  51     int slot;
  52     int flags;
  53 } KVMSlot;
  54
  55 typedef struct kvm_dirty_log KVMDirtyLog;
  56
  57 struct KVMState
  58 {
  59     KVMSlot slots[32];
  60     int fd;
  61     int vmfd;
  62     int coalesced_mmio;
  63 #ifdef KVM_CAP_COALESCED_MMIO
  64     struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
  65 #endif
  66     int broken_set_mem_region;
  67     int migration_log;
  68     int vcpu_events;
  69     int robust_singlestep;
  70     int debugregs;
  71 #ifdef KVM_CAP_SET_GUEST_DEBUG
  72     struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
  73 #endif
  74     int irqchip_in_kernel;
  75     int pit_in_kernel;
  76     int xsave, xcrs;
  77 };
  78
  79 static KVMState *kvm_state;
  80
  81 #endif
  82
  83 static KVMSlot *kvm_alloc_slot(KVMState *s)
  84 {
  85     int i;
  86
  87     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
  88         /* KVM private memory slots */
  89         if (i >= 8 && i < 12)
  90             continue;
  91         if (s->slots[i].memory_size == 0)
  92             return &s->slots[i];
  93     }
  94
  95     fprintf(stderr, "%s: no free slot available\n", __func__);
  96     abort();
  97 }
  98
  99 static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
 100                                          target_phys_addr_t start_addr,
 101                                          target_phys_addr_t end_addr)
 102 {
 103     int i;
 104
 105     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 106         KVMSlot *mem = &s->slots[i];
 107
 108         if (start_addr == mem->start_addr &&
 109             end_addr == mem->start_addr + mem->memory_size) {
 110             return mem;
 111         }
 112     }
 113
 114     return NULL;
 115 }
 116
 117 /*
 118  * Find overlapping slot with lowest start address
 119  */
 120 static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
 121                                             target_phys_addr_t start_addr,
 122                                             target_phys_addr_t end_addr)
 123 {
 124     KVMSlot *found = NULL;
 125     int i;
 126
 127     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 128         KVMSlot *mem = &s->slots[i];
 129
 130         if (mem->memory_size == 0 ||
 131             (found && found->start_addr < mem->start_addr)) {
 132             continue;
 133         }
 134
 135         if (end_addr > mem->start_addr &&
 136             start_addr < mem->start_addr + mem->memory_size) {
 137             found = mem;
 138         }
 139     }
 140
 141     return found;
 142 }
 143
 144 static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
 145 {
 146     struct kvm_userspace_memory_region mem;
 147
 148     mem.slot = slot->slot;
 149     mem.guest_phys_addr = slot->start_addr;
 150     mem.memory_size = slot->memory_size;
 151     mem.userspace_addr = (unsigned long)qemu_get_ram_ptr(slot->phys_offset);
 152     mem.flags = slot->flags;
 153     if (s->migration_log) {
 154         mem.flags |= KVM_MEM_LOG_DIRTY_PAGES;
 155     }
 156     return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
 157 }
 158
 159 #ifdef KVM_UPSTREAM
 160 static void kvm_reset_vcpu(void *opaque)
 161 {
 162     CPUState *env = opaque;
 163
 164     kvm_arch_reset_vcpu(env);
 165 }
 166 #endif
 167
 168 int kvm_irqchip_in_kernel(void)
 169 {
 170     return kvm_state->irqchip_in_kernel;
 171 }
 172
 173 int kvm_pit_in_kernel(void)
 174 {
 175     return kvm_state->pit_in_kernel;
 176 }
 177
 178
 179 #ifdef KVM_UPSTREAM
 180 int kvm_init_vcpu(CPUState *env)
 181 {
 182     KVMState *s = kvm_state;
 183     long mmap_size;
 184     int ret;
 185
 186     DPRINTF("kvm_init_vcpu\n");
 187
 188     ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, env->cpu_index);
 189     if (ret < 0) {
 190         DPRINTF("kvm_create_vcpu failed\n");
 191         goto err;
 192     }
 193
 194     env->kvm_fd = ret;
 195     env->kvm_state = s;
 196
 197     mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
 198     if (mmap_size < 0) {
 199         DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
 200         goto err;
 201     }
 202
 203     env->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
 204                         env->kvm_fd, 0);
 205     if (env->kvm_run == MAP_FAILED) {
 206         ret = -errno;
 207         DPRINTF("mmap'ing vcpu state failed\n");
 208         goto err;
 209     }
 210
 211 #ifdef KVM_CAP_COALESCED_MMIO
 212     if (s->coalesced_mmio && !s->coalesced_mmio_ring)
 213         s->coalesced_mmio_ring = (void *) env->kvm_run +
 214                 s->coalesced_mmio * PAGE_SIZE;
 215 #endif
 216
 217     ret = kvm_arch_init_vcpu(env);
 218     if (ret == 0) {
 219         qemu_register_reset(kvm_reset_vcpu, env);
 220         kvm_arch_reset_vcpu(env);
 221     }
 222 err:
 223     return ret;
 224 }
 225 #endif
 226
 227 /*
 228  * dirty pages logging control
 229  */
 230 static int kvm_dirty_pages_log_change(target_phys_addr_t phys_addr,
 231                                       ram_addr_t size, int flags, int mask)
 232 {
 233     KVMState *s = kvm_state;
 234     KVMSlot *mem = kvm_lookup_matching_slot(s, phys_addr, phys_addr + size);
 235     int old_flags;
 236
 237     if (mem == NULL)  {
 238             fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
 239                     TARGET_FMT_plx "\n", __func__, phys_addr,
 240                     (target_phys_addr_t)(phys_addr + size - 1));
 241             return -EINVAL;
 242     }
 243
 244     old_flags = mem->flags;
 245
 246     flags = (mem->flags & ~mask) | flags;
 247     mem->flags = flags;
 248
 249     /* If nothing changed effectively, no need to issue ioctl */
 250     if (s->migration_log) {
 251         flags |= KVM_MEM_LOG_DIRTY_PAGES;
 252     }
 253     if (flags == old_flags) {
 254             return 0;
 255     }
 256
 257     return kvm_set_user_memory_region(s, mem);
 258 }
 259
 260 int kvm_log_start(target_phys_addr_t phys_addr, ram_addr_t size)
 261 {
 262         return kvm_dirty_pages_log_change(phys_addr, size,
 263                                           KVM_MEM_LOG_DIRTY_PAGES,
 264                                           KVM_MEM_LOG_DIRTY_PAGES);
 265 }
 266
 267 int kvm_log_stop(target_phys_addr_t phys_addr, ram_addr_t size)
 268 {
 269         return kvm_dirty_pages_log_change(phys_addr, size,
 270                                           0,
 271                                           KVM_MEM_LOG_DIRTY_PAGES);
 272 }
 273
 274 static int kvm_set_migration_log(int enable)
 275 {
 276     KVMState *s = kvm_state;
 277     KVMSlot *mem;
 278     int i, err;
 279
 280     s->migration_log = enable;
 281
 282     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 283         mem = &s->slots[i];
 284
 285         if (!!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) == enable) {
 286             continue;
 287         }
 288         err = kvm_set_user_memory_region(s, mem);
 289         if (err) {
 290             return err;
 291         }
 292     }
 293     return 0;
 294 }
 295
 296 /* get kvm's dirty pages bitmap and update qemu's */
 297 static int kvm_get_dirty_pages_log_range(unsigned long start_addr,
 298                                          unsigned long *bitmap,
 299                                          unsigned long offset,
 300                                          unsigned long mem_size)
 301 {
 302     unsigned int i, j;
 303     unsigned long page_number, addr, addr1, c;
 304     ram_addr_t ram_addr;
 305     unsigned int len = ((mem_size / TARGET_PAGE_SIZE) + HOST_LONG_BITS - 1) /
 306         HOST_LONG_BITS;
 307
 308     /*
 309      * bitmap-traveling is faster than memory-traveling (for addr...)
 310      * especially when most of the memory is not dirty.
 311      */
 312     for (i = 0; i < len; i++) {
 313         if (bitmap[i] != 0) {
 314             c = leul_to_cpu(bitmap[i]);
 315             do {
 316                 j = ffsl(c) - 1;
 317                 c &= ~(1ul << j);
 318                 page_number = i * HOST_LONG_BITS + j;
 319                 addr1 = page_number * TARGET_PAGE_SIZE;
 320                 addr = offset + addr1;
 321                 ram_addr = cpu_get_physical_page_desc(addr);
 322                 cpu_physical_memory_set_dirty(ram_addr);
 323             } while (c != 0);
 324         }
 325     }
 326     return 0;
 327 }
 328
 329 #define ALIGN(x, y)  (((x)+(y)-1) & ~((y)-1))
 330
 331 /**
 332  * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
 333  * This function updates qemu's dirty bitmap using cpu_physical_memory_set_dirty().
 334  * This means all bits are set to dirty.
 335  *
 336  * @start_add: start of logged region.
 337  * @end_addr: end of logged region.
 338  */
 339 static int kvm_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
 340                                           target_phys_addr_t end_addr)
 341 {
 342     KVMState *s = kvm_state;
 343     unsigned long size, allocated_size = 0;
 344     KVMDirtyLog d;
 345     KVMSlot *mem;
 346     int ret = 0;
 347
 348     d.dirty_bitmap = NULL;
 349     while (start_addr < end_addr) {
 350         mem = kvm_lookup_overlapping_slot(s, start_addr, end_addr);
 351         if (mem == NULL) {
 352             break;
 353         }
 354
 355         size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS), HOST_LONG_BITS) / 8;
 356         if (!d.dirty_bitmap) {
 357             d.dirty_bitmap = qemu_malloc(size);
 358         } else if (size > allocated_size) {
 359             d.dirty_bitmap = qemu_realloc(d.dirty_bitmap, size);
 360         }
 361         allocated_size = size;
 362         memset(d.dirty_bitmap, 0, allocated_size);
 363
 364         d.slot = mem->slot;
 365
 366         if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
 367             DPRINTF("ioctl failed %d\n", errno);
 368             ret = -1;
 369             break;
 370         }
 371
 372         kvm_get_dirty_pages_log_range(mem->start_addr, d.dirty_bitmap,
 373                                       mem->start_addr, mem->memory_size);
 374         start_addr = mem->start_addr + mem->memory_size;
 375     }
 376     qemu_free(d.dirty_bitmap);
 377
 378     return ret;
 379 }
 380
 381 int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
 382 {
 383     int ret = -ENOSYS;
 384 #ifdef KVM_CAP_COALESCED_MMIO
 385     KVMState *s = kvm_state;
 386
 387     if (s->coalesced_mmio) {
 388         struct kvm_coalesced_mmio_zone zone;
 389
 390         zone.addr = start;
 391         zone.size = size;
 392
 393         ret = kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
 394     }
 395 #endif
 396
 397     return ret;
 398 }
 399
 400 int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
 401 {
 402     int ret = -ENOSYS;
 403 #ifdef KVM_CAP_COALESCED_MMIO
 404     KVMState *s = kvm_state;
 405
 406     if (s->coalesced_mmio) {
 407         struct kvm_coalesced_mmio_zone zone;
 408
 409         zone.addr = start;
 410         zone.size = size;
 411
 412         ret = kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
 413     }
 414 #endif
 415
 416     return ret;
 417 }
 418
 419 int kvm_check_extension(KVMState *s, unsigned int extension)
 420 {
 421     int ret;
 422
 423     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
 424     if (ret < 0) {
 425         ret = 0;
 426     }
 427
 428     return ret;
 429 }
 430
 431 static void kvm_set_phys_mem(target_phys_addr_t start_addr,
 432                              ram_addr_t size,
 433                              ram_addr_t phys_offset)
 434 {
 435     KVMState *s = kvm_state;
 436     ram_addr_t flags = phys_offset & ~TARGET_PAGE_MASK;
 437     KVMSlot *mem, old;
 438     int err;
 439
 440     if (start_addr & ~TARGET_PAGE_MASK) {
 441         if (flags >= IO_MEM_UNASSIGNED) {
 442             if (!kvm_lookup_overlapping_slot(s, start_addr,
 443                                              start_addr + size)) {
 444                 return;
 445             }
 446             fprintf(stderr, "Unaligned split of a KVM memory slot\n");
 447         } else {
 448             fprintf(stderr, "Only page-aligned memory slots supported\n");
 449         }
 450         abort();
 451     }
 452
 453     /* KVM does not support read-only slots */
 454     phys_offset &= ~IO_MEM_ROM;
 455
 456     while (1) {
 457         mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size);
 458         if (!mem) {
 459             break;
 460         }
 461
 462         if (flags < IO_MEM_UNASSIGNED && start_addr >= mem->start_addr &&
 463             (start_addr + size <= mem->start_addr + mem->memory_size) &&
 464             (phys_offset - start_addr == mem->phys_offset - mem->start_addr)) {
 465             /* The new slot fits into the existing one and comes with
 466              * identical parameters - nothing to be done. */
 467             return;
 468         }
 469
 470         old = *mem;
 471
 472         /* unregister the overlapping slot */
 473         mem->memory_size = 0;
 474         err = kvm_set_user_memory_region(s, mem);
 475         if (err) {
 476             fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
 477                     __func__, strerror(-err));
 478             abort();
 479         }
 480
 481         /* Workaround for older KVM versions: we can't join slots, even not by
 482          * unregistering the previous ones and then registering the larger
 483          * slot. We have to maintain the existing fragmentation. Sigh.
 484          *
 485          * This workaround assumes that the new slot starts at the same
 486          * address as the first existing one. If not or if some overlapping
 487          * slot comes around later, we will fail (not seen in practice so far)
 488          * - and actually require a recent KVM version. */
 489         if (s->broken_set_mem_region &&
 490             old.start_addr == start_addr && old.memory_size < size &&
 491             flags < IO_MEM_UNASSIGNED) {
 492             mem = kvm_alloc_slot(s);
 493             mem->memory_size = old.memory_size;
 494             mem->start_addr = old.start_addr;
 495             mem->phys_offset = old.phys_offset;
 496             mem->flags = 0;
 497
 498             err = kvm_set_user_memory_region(s, mem);
 499             if (err) {
 500                 fprintf(stderr, "%s: error updating slot: %s\n", __func__,
 501                         strerror(-err));
 502                 abort();
 503             }
 504
 505             start_addr += old.memory_size;
 506             phys_offset += old.memory_size;
 507             size -= old.memory_size;
 508             continue;
 509         }
 510
 511         /* register prefix slot */
 512         if (old.start_addr < start_addr) {
 513             mem = kvm_alloc_slot(s);
 514             mem->memory_size = start_addr - old.start_addr;
 515             mem->start_addr = old.start_addr;
 516             mem->phys_offset = old.phys_offset;
 517             mem->flags = 0;
 518
 519             err = kvm_set_user_memory_region(s, mem);
 520             if (err) {
 521                 fprintf(stderr, "%s: error registering prefix slot: %s\n",
 522                         __func__, strerror(-err));
 523                 abort();
 524             }
 525         }
 526
 527         /* register suffix slot */
 528         if (old.start_addr + old.memory_size > start_addr + size) {
 529             ram_addr_t size_delta;
 530
 531             mem = kvm_alloc_slot(s);
 532             mem->start_addr = start_addr + size;
 533             size_delta = mem->start_addr - old.start_addr;
 534             mem->memory_size = old.memory_size - size_delta;
 535             mem->phys_offset = old.phys_offset + size_delta;
 536             mem->flags = 0;
 537
 538             err = kvm_set_user_memory_region(s, mem);
 539             if (err) {
 540                 fprintf(stderr, "%s: error registering suffix slot: %s\n",
 541                         __func__, strerror(-err));
 542                 abort();
 543             }
 544         }
 545     }
 546
 547     /* in case the KVM bug workaround already "consumed" the new slot */
 548     if (!size)
 549         return;
 550
 551     /* KVM does not need to know about this memory */
 552     if (flags >= IO_MEM_UNASSIGNED)
 553         return;
 554
 555     mem = kvm_alloc_slot(s);
 556     mem->memory_size = size;
 557     mem->start_addr = start_addr;
 558     mem->phys_offset = phys_offset;
 559     mem->flags = 0;
 560
 561     err = kvm_set_user_memory_region(s, mem);
 562     if (err) {
 563         fprintf(stderr, "%s: error registering slot: %s\n", __func__,
 564                 strerror(-err));
 565         abort();
 566     }
 567 }
 568
 569 static void kvm_client_set_memory(struct CPUPhysMemoryClient *client,
 570                                   target_phys_addr_t start_addr,
 571                                   ram_addr_t size,
 572                                   ram_addr_t phys_offset)
 573 {
 574         kvm_set_phys_mem(start_addr, size, phys_offset);
 575 }
 576
 577 static int kvm_client_sync_dirty_bitmap(struct CPUPhysMemoryClient *client,
 578                                         target_phys_addr_t start_addr,
 579                                         target_phys_addr_t end_addr)
 580 {
 581         return kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
 582 }
 583
 584 static int kvm_client_migration_log(struct CPUPhysMemoryClient *client,
 585                                     int enable)
 586 {
 587         return kvm_set_migration_log(enable);
 588 }
 589
 590 static CPUPhysMemoryClient kvm_cpu_phys_memory_client = {
 591         .set_memory = kvm_client_set_memory,
 592         .sync_dirty_bitmap = kvm_client_sync_dirty_bitmap,
 593         .migration_log = kvm_client_migration_log,
 594 };
 595
 596
 597 void kvm_cpu_register_phys_memory_client(void)
 598 {
 599     cpu_register_phys_memory_client(&kvm_cpu_phys_memory_client);
 600 }
 601
 602 #ifdef KVM_UPSTREAM
 603
 604 int kvm_init(int smp_cpus)
 605 {
 606     static const char upgrade_note[] =
 607         "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
 608         "(see http://sourceforge.net/projects/kvm).\n";
 609     KVMState *s;
 610     int ret;
 611     int i;
 612
 613     s = qemu_mallocz(sizeof(KVMState));
 614
 615 #ifdef KVM_CAP_SET_GUEST_DEBUG
 616     QTAILQ_INIT(&s->kvm_sw_breakpoints);
 617 #endif
 618     for (i = 0; i < ARRAY_SIZE(s->slots); i++)
 619         s->slots[i].slot = i;
 620
 621     s->vmfd = -1;
 622     s->fd = qemu_open("/dev/kvm", O_RDWR);
 623     if (s->fd == -1) {
 624         fprintf(stderr, "Could not access KVM kernel module: %m\n");
 625         ret = -errno;
 626         goto err;
 627     }
 628
 629     ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
 630     if (ret < KVM_API_VERSION) {
 631         if (ret > 0)
 632             ret = -EINVAL;
 633         fprintf(stderr, "kvm version too old\n");
 634         goto err;
 635     }
 636
 637     if (ret > KVM_API_VERSION) {
 638         ret = -EINVAL;
 639         fprintf(stderr, "kvm version not supported\n");
 640         goto err;
 641     }
 642
 643     s->vmfd = kvm_ioctl(s, KVM_CREATE_VM, 0);
 644     if (s->vmfd < 0) {
 645 #ifdef TARGET_S390X
 646         fprintf(stderr, "Please add the 'switch_amode' kernel parameter to "
 647                         "your host kernel command line\n");
 648 #endif
 649         goto err;
 650     }
 651
 652     /* initially, KVM allocated its own memory and we had to jump through
 653      * hooks to make phys_ram_base point to this.  Modern versions of KVM
 654      * just use a user allocated buffer so we can use regular pages
 655      * unmodified.  Make sure we have a sufficiently modern version of KVM.
 656      */
 657     if (!kvm_check_extension(s, KVM_CAP_USER_MEMORY)) {
 658         ret = -EINVAL;
 659         fprintf(stderr, "kvm does not support KVM_CAP_USER_MEMORY\n%s",
 660                 upgrade_note);
 661         goto err;
 662     }
 663
 664     /* There was a nasty bug in < kvm-80 that prevents memory slots from being
 665      * destroyed properly.  Since we rely on this capability, refuse to work
 666      * with any kernel without this capability. */
 667     if (!kvm_check_extension(s, KVM_CAP_DESTROY_MEMORY_REGION_WORKS)) {
 668         ret = -EINVAL;
 669
 670         fprintf(stderr,
 671                 "KVM kernel module broken (DESTROY_MEMORY_REGION).\n%s",
 672                 upgrade_note);
 673         goto err;
 674     }
 675
 676     s->coalesced_mmio = 0;
 677 #ifdef KVM_CAP_COALESCED_MMIO
 678     s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
 679     s->coalesced_mmio_ring = NULL;
 680 #endif
 681
 682     s->broken_set_mem_region = 1;
 683 #ifdef KVM_CAP_JOIN_MEMORY_REGIONS_WORKS
 684     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
 685     if (ret > 0) {
 686         s->broken_set_mem_region = 0;
 687     }
 688 #endif
 689
 690     s->vcpu_events = 0;
 691 #ifdef KVM_CAP_VCPU_EVENTS
 692     s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
 693 #endif
 694
 695     s->robust_singlestep = 0;
 696 #ifdef KVM_CAP_X86_ROBUST_SINGLESTEP
 697     s->robust_singlestep =
 698         kvm_check_extension(s, KVM_CAP_X86_ROBUST_SINGLESTEP);
 699 #endif
 700
 701     s->debugregs = 0;
 702 #ifdef KVM_CAP_DEBUGREGS
 703     s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS);
 704 #endif
 705
 706     s->xsave = 0;
 707 #ifdef KVM_CAP_XSAVE
 708     s->xsave = kvm_check_extension(s, KVM_CAP_XSAVE);
 709 #endif
 710
 711     s->xcrs = 0;
 712 #ifdef KVM_CAP_XCRS
 713     s->xcrs = kvm_check_extension(s, KVM_CAP_XCRS);
 714 #endif
 715
 716     ret = kvm_arch_init(s, smp_cpus);
 717     if (ret < 0)
 718         goto err;
 719
 720     kvm_state = s;
 721     cpu_register_phys_memory_client(&kvm_cpu_phys_memory_client);
 722
 723     return 0;
 724
 725 err:
 726     if (s) {
 727         if (s->vmfd != -1)
 728             close(s->vmfd);
 729         if (s->fd != -1)
 730             close(s->fd);
 731     }
 732     qemu_free(s);
 733
 734     return ret;
 735 }
 736 #endif
 737
 738 static int kvm_handle_io(uint16_t port, void *data, int direction, int size,
 739                          uint32_t count)
 740 {
 741     int i;
 742     uint8_t *ptr = data;
 743
 744     for (i = 0; i < count; i++) {
 745         if (direction == KVM_EXIT_IO_IN) {
 746             switch (size) {
 747             case 1:
 748                 stb_p(ptr, cpu_inb(port));
 749                 break;
 750             case 2:
 751                 stw_p(ptr, cpu_inw(port));
 752                 break;
 753             case 4:
 754                 stl_p(ptr, cpu_inl(port));
 755                 break;
 756             }
 757         } else {
 758             switch (size) {
 759             case 1:
 760                 cpu_outb(port, ldub_p(ptr));
 761                 break;
 762             case 2:
 763                 cpu_outw(port, lduw_p(ptr));
 764                 break;
 765             case 4:
 766                 cpu_outl(port, ldl_p(ptr));
 767                 break;
 768             }
 769         }
 770
 771         ptr += size;
 772     }
 773
 774     return 1;
 775 }
 776
 777 #ifdef KVM_CAP_INTERNAL_ERROR_DATA
 778 static void kvm_handle_internal_error(CPUState *env, struct kvm_run *run)
 779 {
 780
 781     if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) {
 782         int i;
 783
 784         fprintf(stderr, "KVM internal error. Suberror: %d\n",
 785                 run->internal.suberror);
 786
 787         for (i = 0; i < run->internal.ndata; ++i) {
 788             fprintf(stderr, "extra data[%d]: %"PRIx64"\n",
 789                     i, (uint64_t)run->internal.data[i]);
 790         }
 791     }
 792     cpu_dump_state(env, stderr, fprintf, 0);
 793     if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) {
 794         fprintf(stderr, "emulation failure\n");
 795         if (!kvm_arch_stop_on_emulation_error(env))
 796                 return;
 797     }
 798     /* FIXME: Should trigger a qmp message to let management know
 799      * something went wrong.
 800      */
 801     vm_stop(0);
 802 }
 803 #endif
 804
 805 void kvm_flush_coalesced_mmio_buffer(void)
 806 {
 807 #ifdef KVM_CAP_COALESCED_MMIO
 808     KVMState *s = kvm_state;
 809     if (s->coalesced_mmio_ring) {
 810         struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
 811         while (ring->first != ring->last) {
 812             struct kvm_coalesced_mmio *ent;
 813
 814             ent = &ring->coalesced_mmio[ring->first];
 815
 816             cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
 817             smp_wmb();
 818             ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
 819         }
 820     }
 821 #endif
 822 }
 823
 824 #ifdef KVM_UPSTREAM
 825
 826 static void do_kvm_cpu_synchronize_state(void *_env)
 827 {
 828     CPUState *env = _env;
 829
 830     if (!env->kvm_vcpu_dirty) {
 831         kvm_arch_get_registers(env);
 832         env->kvm_vcpu_dirty = 1;
 833     }
 834 }
 835
 836 void kvm_cpu_synchronize_state(CPUState *env)
 837 {
 838     if (!env->kvm_vcpu_dirty)
 839         run_on_cpu(env, do_kvm_cpu_synchronize_state, env);
 840 }
 841
 842 void kvm_cpu_synchronize_post_reset(CPUState *env)
 843 {
 844     kvm_arch_put_registers(env, KVM_PUT_RESET_STATE);
 845     env->kvm_vcpu_dirty = 0;
 846 }
 847
 848 void kvm_cpu_synchronize_post_init(CPUState *env)
 849 {
 850     kvm_arch_put_registers(env, KVM_PUT_FULL_STATE);
 851     env->kvm_vcpu_dirty = 0;
 852 }
 853
 854 int kvm_cpu_exec(CPUState *env)
 855 {
 856     struct kvm_run *run = env->kvm_run;
 857     int ret;
 858
 859     DPRINTF("kvm_cpu_exec()\n");
 860
 861     do {
 862 #ifndef CONFIG_IOTHREAD
 863         if (env->exit_request) {
 864             DPRINTF("interrupt exit requested\n");
 865             ret = 0;
 866             break;
 867         }
 868 #endif
 869
 870         if (kvm_arch_process_irqchip_events(env)) {
 871             ret = 0;
 872             break;
 873         }
 874
 875         if (env->kvm_vcpu_dirty) {
 876             kvm_arch_put_registers(env, KVM_PUT_RUNTIME_STATE);
 877             env->kvm_vcpu_dirty = 0;
 878         }
 879
 880         kvm_arch_pre_run(env, run);
 881         cpu_single_env = NULL;
 882         qemu_mutex_unlock_iothread();
 883         ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);
 884         qemu_mutex_lock_iothread();
 885         cpu_single_env = env;
 886         kvm_arch_post_run(env, run);
 887
 888         if (ret == -EINTR || ret == -EAGAIN) {
 889             cpu_exit(env);
 890             DPRINTF("io window exit\n");
 891             ret = 0;
 892             break;
 893         }
 894
 895         if (ret < 0) {
 896             DPRINTF("kvm run failed %s\n", strerror(-ret));
 897             abort();
 898         }
 899
 900         kvm_flush_coalesced_mmio_buffer();
 901
 902         ret = 0; /* exit loop */
 903         switch (run->exit_reason) {
 904         case KVM_EXIT_IO:
 905             DPRINTF("handle_io\n");
 906             ret = kvm_handle_io(run->io.port,
 907                                 (uint8_t *)run + run->io.data_offset,
 908                                 run->io.direction,
 909                                 run->io.size,
 910                                 run->io.count);
 911             break;
 912         case KVM_EXIT_MMIO:
 913             DPRINTF("handle_mmio\n");
 914             cpu_physical_memory_rw(run->mmio.phys_addr,
 915                                    run->mmio.data,
 916                                    run->mmio.len,
 917                                    run->mmio.is_write);
 918             ret = 1;
 919             break;
 920         case KVM_EXIT_IRQ_WINDOW_OPEN:
 921             DPRINTF("irq_window_open\n");
 922             break;
 923         case KVM_EXIT_SHUTDOWN:
 924             DPRINTF("shutdown\n");
 925             qemu_system_reset_request();
 926             ret = 1;
 927             break;
 928         case KVM_EXIT_UNKNOWN:
 929             DPRINTF("kvm_exit_unknown\n");
 930             break;
 931         case KVM_EXIT_FAIL_ENTRY:
 932             DPRINTF("kvm_exit_fail_entry\n");
 933             break;
 934         case KVM_EXIT_EXCEPTION:
 935             DPRINTF("kvm_exit_exception\n");
 936             break;
 937 #ifdef KVM_CAP_INTERNAL_ERROR_DATA
 938         case KVM_EXIT_INTERNAL_ERROR:
 939             kvm_handle_internal_error(env, run);
 940             break;
 941 #endif
 942         case KVM_EXIT_DEBUG:
 943             DPRINTF("kvm_exit_debug\n");
 944 #ifdef KVM_CAP_SET_GUEST_DEBUG
 945             if (kvm_arch_debug(&run->debug.arch)) {
 946                 env->exception_index = EXCP_DEBUG;
 947                 return 0;
 948             }
 949             /* re-enter, this exception was guest-internal */
 950             ret = 1;
 951 #endif /* KVM_CAP_SET_GUEST_DEBUG */
 952             break;
 953         default:
 954             DPRINTF("kvm_arch_handle_exit\n");
 955             ret = kvm_arch_handle_exit(env, run);
 956             break;
 957         }
 958     } while (ret > 0);
 959
 960     if (env->exit_request) {
 961         env->exit_request = 0;
 962         env->exception_index = EXCP_INTERRUPT;
 963     }
 964
 965     return ret;
 966 }
 967
 968 #endif
 969 int kvm_ioctl(KVMState *s, int type, ...)
 970 {
 971     int ret;
 972     void *arg;
 973     va_list ap;
 974
 975     va_start(ap, type);
 976     arg = va_arg(ap, void *);
 977     va_end(ap);
 978
 979     ret = ioctl(s->fd, type, arg);
 980     if (ret == -1)
 981         ret = -errno;
 982
 983     return ret;
 984 }
 985
 986 int kvm_vm_ioctl(KVMState *s, int type, ...)
 987 {
 988     int ret;
 989     void *arg;
 990     va_list ap;
 991
 992     va_start(ap, type);
 993     arg = va_arg(ap, void *);
 994     va_end(ap);
 995
 996     ret = ioctl(s->vmfd, type, arg);
 997     if (ret == -1)
 998         ret = -errno;
 999
1000     return ret;
1001 }
1002
1003 int kvm_vcpu_ioctl(CPUState *env, int type, ...)
1004 {
1005     int ret;
1006     void *arg;
1007     va_list ap;
1008
1009     va_start(ap, type);
1010     arg = va_arg(ap, void *);
1011     va_end(ap);
1012
1013     ret = ioctl(env->kvm_fd, type, arg);
1014     if (ret == -1)
1015         ret = -errno;
1016
1017     return ret;
1018 }
1019
1020 int kvm_has_sync_mmu(void)
1021 {
1022 #ifdef KVM_CAP_SYNC_MMU
1023     KVMState *s = kvm_state;
1024
1025     return kvm_check_extension(s, KVM_CAP_SYNC_MMU);
1026 #else
1027     return 0;
1028 #endif
1029 }
1030
1031 int kvm_has_vcpu_events(void)
1032 {
1033     return kvm_state->vcpu_events;
1034 }
1035
1036 int kvm_has_robust_singlestep(void)
1037 {
1038     return kvm_state->robust_singlestep;
1039 }
1040
1041 int kvm_has_debugregs(void)
1042 {
1043     return kvm_state->debugregs;
1044 }
1045
1046 #ifdef KVM_UPSTREAM
1047 int kvm_has_xsave(void)
1048 {
1049     return kvm_state->xsave;
1050 }
1051
1052 int kvm_has_xcrs(void)
1053 {
1054     return kvm_state->xcrs;
1055 }
1056 #endif
1057
1058 void kvm_setup_guest_memory(void *start, size_t size)
1059 {
1060     if (!kvm_has_sync_mmu()) {
1061 #ifdef MADV_DONTFORK
1062         int ret = madvise(start, size, MADV_DONTFORK);
1063
1064         if (ret) {
1065             perror("madvice");
1066             exit(1);
1067         }
1068 #else
1069         fprintf(stderr,
1070                 "Need MADV_DONTFORK in absence of synchronous KVM MMU\n");
1071         exit(1);
1072 #endif
1073     }
1074 }
1075
1076 #ifdef KVM_CAP_SET_GUEST_DEBUG
1077 #ifndef KVM_UPSTREAM
1078 #define run_on_cpu on_vcpu
1079 static void on_vcpu(CPUState *env, void (*func)(void *data), void *data);
1080 #endif /* !KVM_UPSTREAM */
1081
1082 struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *env,
1083                                                  target_ulong pc)
1084 {
1085     struct kvm_sw_breakpoint *bp;
1086
1087     QTAILQ_FOREACH(bp, &env->kvm_state->kvm_sw_breakpoints, entry) {
1088         if (bp->pc == pc)
1089             return bp;
1090     }
1091     return NULL;
1092 }
1093
1094 int kvm_sw_breakpoints_active(CPUState *env)
1095 {
1096     return !QTAILQ_EMPTY(&env->kvm_state->kvm_sw_breakpoints);
1097 }
1098
1099 struct kvm_set_guest_debug_data {
1100     struct kvm_guest_debug dbg;
1101     CPUState *env;
1102     int err;
1103 };
1104
1105 static void kvm_invoke_set_guest_debug(void *data)
1106 {
1107     struct kvm_set_guest_debug_data *dbg_data = data;
1108     CPUState *env = dbg_data->env;
1109
1110     dbg_data->err = kvm_vcpu_ioctl(env, KVM_SET_GUEST_DEBUG, &dbg_data->dbg);
1111 }
1112
1113 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1114 {
1115     struct kvm_set_guest_debug_data data;
1116
1117     data.dbg.control = reinject_trap;
1118
1119     if (env->singlestep_enabled) {
1120         data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
1121     }
1122     kvm_arch_update_guest_debug(env, &data.dbg);
1123     data.env = env;
1124
1125     run_on_cpu(env, kvm_invoke_set_guest_debug, &data);
1126     return data.err;
1127 }
1128
1129 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1130                           target_ulong len, int type)
1131 {
1132     struct kvm_sw_breakpoint *bp;
1133     CPUState *env;
1134     int err;
1135
1136     if (type == GDB_BREAKPOINT_SW) {
1137         bp = kvm_find_sw_breakpoint(current_env, addr);
1138         if (bp) {
1139             bp->use_count++;
1140             return 0;
1141         }
1142
1143         bp = qemu_malloc(sizeof(struct kvm_sw_breakpoint));
1144         if (!bp)
1145             return -ENOMEM;
1146
1147         bp->pc = addr;
1148         bp->use_count = 1;
1149         err = kvm_arch_insert_sw_breakpoint(current_env, bp);
1150         if (err) {
1151             free(bp);
1152             return err;
1153         }
1154
1155         QTAILQ_INSERT_HEAD(&current_env->kvm_state->kvm_sw_breakpoints,
1156                           bp, entry);
1157     } else {
1158         err = kvm_arch_insert_hw_breakpoint(addr, len, type);
1159         if (err)
1160             return err;
1161     }
1162
1163     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1164         err = kvm_update_guest_debug(env, 0);
1165         if (err)
1166             return err;
1167     }
1168     return 0;
1169 }
1170
1171 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1172                           target_ulong len, int type)
1173 {
1174     struct kvm_sw_breakpoint *bp;
1175     CPUState *env;
1176     int err;
1177
1178     if (type == GDB_BREAKPOINT_SW) {
1179         bp = kvm_find_sw_breakpoint(current_env, addr);
1180         if (!bp)
1181             return -ENOENT;
1182
1183         if (bp->use_count > 1) {
1184             bp->use_count--;
1185             return 0;
1186         }
1187
1188         err = kvm_arch_remove_sw_breakpoint(current_env, bp);
1189         if (err)
1190             return err;
1191
1192         QTAILQ_REMOVE(&current_env->kvm_state->kvm_sw_breakpoints, bp, entry);
1193         qemu_free(bp);
1194     } else {
1195         err = kvm_arch_remove_hw_breakpoint(addr, len, type);
1196         if (err)
1197             return err;
1198     }
1199
1200     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1201         err = kvm_update_guest_debug(env, 0);
1202         if (err)
1203             return err;
1204     }
1205     return 0;
1206 }
1207
1208 void kvm_remove_all_breakpoints(CPUState *current_env)
1209 {
1210     struct kvm_sw_breakpoint *bp, *next;
1211     KVMState *s = current_env->kvm_state;
1212     CPUState *env;
1213
1214     QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
1215         if (kvm_arch_remove_sw_breakpoint(current_env, bp) != 0) {
1216             /* Try harder to find a CPU that currently sees the breakpoint. */
1217             for (env = first_cpu; env != NULL; env = env->next_cpu) {
1218                 if (kvm_arch_remove_sw_breakpoint(env, bp) == 0)
1219                     break;
1220             }
1221         }
1222     }
1223     kvm_arch_remove_all_hw_breakpoints();
1224
1225     for (env = first_cpu; env != NULL; env = env->next_cpu)
1226         kvm_update_guest_debug(env, 0);
1227 }
1228
1229 #else /* !KVM_CAP_SET_GUEST_DEBUG */
1230
1231 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1232 {
1233     return -EINVAL;
1234 }
1235
1236 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1237                           target_ulong len, int type)
1238 {
1239     return -EINVAL;
1240 }
1241
1242 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1243                           target_ulong len, int type)
1244 {
1245     return -EINVAL;
1246 }
1247
1248 void kvm_remove_all_breakpoints(CPUState *current_env)
1249 {
1250 }
1251 #endif /* !KVM_CAP_SET_GUEST_DEBUG */
1252
1253 int kvm_set_signal_mask(CPUState *env, const sigset_t *sigset)
1254 {
1255     struct kvm_signal_mask *sigmask;
1256     int r;
1257
1258     if (!sigset)
1259         return kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, NULL);
1260
1261     sigmask = qemu_malloc(sizeof(*sigmask) + sizeof(*sigset));
1262
1263     sigmask->len = 8;
1264     memcpy(sigmask->sigset, sigset, sizeof(*sigset));
1265     r = kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, sigmask);
1266     free(sigmask);
1267
1268     return r;
1269 }
1270
1271 int kvm_set_ioeventfd_pio_word(int fd, uint16_t addr, uint16_t val, bool assign)
1272 {
1273 #ifdef KVM_IOEVENTFD
1274     struct kvm_ioeventfd kick = {
1275         .datamatch = val,
1276         .addr = addr,
1277         .len = 2,
1278         .flags = KVM_IOEVENTFD_FLAG_DATAMATCH | KVM_IOEVENTFD_FLAG_PIO,
1279         .fd = fd,
1280     };
1281     int r;
1282     if (!kvm_enabled())
1283         return -ENOSYS;
1284     if (!assign)
1285         kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
1286     r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
1287     if (r < 0)
1288         return r;
1289     return 0;
1290 #else
1291     return -ENOSYS;
1292 #endif
1293 }
1294
1295 #if defined(KVM_IRQFD)
1296 int kvm_set_irqfd(int gsi, int fd, bool assigned)
1297 {
1298     struct kvm_irqfd irqfd = {
1299         .fd = fd,
1300         .gsi = gsi,
1301         .flags = assigned ? 0 : KVM_IRQFD_FLAG_DEASSIGN,
1302     };
1303     int r;
1304     if (!kvm_enabled() || !kvm_irqchip_in_kernel())
1305         return -ENOSYS;
1306
1307     r = kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd);
1308     if (r < 0)
1309         return r;
1310     return 0;
1311 }
1312 #endif
1313
1314 #undef PAGE_SIZE
1315 #include "qemu-kvm.c"