kvm-all.c

   1 /*
   2  * QEMU KVM support
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *           Red Hat, Inc. 2008
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *  Glauber Costa     <gcosta@redhat.com>
  10  *
  11  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  12  * See the COPYING file in the top-level directory.
  13  *
  14  */
  15
  16 #include <sys/types.h>
  17 #include <sys/ioctl.h>
  18 #include <sys/mman.h>
  19 #include <stdarg.h>
  20
  21 #include <linux/kvm.h>
  22
  23 #include "qemu-common.h"
  24 #include "qemu-barrier.h"
  25 #include "sysemu.h"
  26 #include "hw/hw.h"
  27 #include "gdbstub.h"
  28 #include "kvm.h"
  29 #include "bswap.h"
  30
  31 /* KVM uses PAGE_SIZE in it's definition of COALESCED_MMIO_MAX */
  32 #define PAGE_SIZE TARGET_PAGE_SIZE
  33
  34 //#define DEBUG_KVM
  35
  36 #ifdef DEBUG_KVM
  37 #define DPRINTF(fmt, ...) \
  38     do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
  39 #else
  40 #define DPRINTF(fmt, ...) \
  41     do { } while (0)
  42 #endif
  43
  44 #ifdef KVM_UPSTREAM
  45
  46 typedef struct KVMSlot
  47 {
  48     target_phys_addr_t start_addr;
  49     ram_addr_t memory_size;
  50     ram_addr_t phys_offset;
  51     int slot;
  52     int flags;
  53 } KVMSlot;
  54
  55 typedef struct kvm_dirty_log KVMDirtyLog;
  56
  57 struct KVMState
  58 {
  59     KVMSlot slots[32];
  60     int fd;
  61     int vmfd;
  62     int coalesced_mmio;
  63 #ifdef KVM_CAP_COALESCED_MMIO
  64     struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
  65 #endif
  66     int broken_set_mem_region;
  67     int migration_log;
  68     int vcpu_events;
  69     int robust_singlestep;
  70     int debugregs;
  71 #ifdef KVM_CAP_SET_GUEST_DEBUG
  72     struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
  73 #endif
  74     int irqchip_in_kernel;
  75     int pit_in_kernel;
  76     int xsave, xcrs;
  77 };
  78
  79 static KVMState *kvm_state;
  80
  81 #endif
  82
  83 static KVMSlot *kvm_alloc_slot(KVMState *s)
  84 {
  85     int i;
  86
  87     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
  88         /* KVM private memory slots */
  89         if (i >= 8 && i < 12)
  90             continue;
  91         if (s->slots[i].memory_size == 0)
  92             return &s->slots[i];
  93     }
  94
  95     fprintf(stderr, "%s: no free slot available\n", __func__);
  96     abort();
  97 }
  98
  99 static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
 100                                          target_phys_addr_t start_addr,
 101                                          target_phys_addr_t end_addr)
 102 {
 103     int i;
 104
 105     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 106         KVMSlot *mem = &s->slots[i];
 107
 108         if (start_addr == mem->start_addr &&
 109             end_addr == mem->start_addr + mem->memory_size) {
 110             return mem;
 111         }
 112     }
 113
 114     return NULL;
 115 }
 116
 117 /*
 118  * Find overlapping slot with lowest start address
 119  */
 120 static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
 121                                             target_phys_addr_t start_addr,
 122                                             target_phys_addr_t end_addr)
 123 {
 124     KVMSlot *found = NULL;
 125     int i;
 126
 127     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 128         KVMSlot *mem = &s->slots[i];
 129
 130         if (mem->memory_size == 0 ||
 131             (found && found->start_addr < mem->start_addr)) {
 132             continue;
 133         }
 134
 135         if (end_addr > mem->start_addr &&
 136             start_addr < mem->start_addr + mem->memory_size) {
 137             found = mem;
 138         }
 139     }
 140
 141     return found;
 142 }
 143
 144 static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
 145 {
 146     struct kvm_userspace_memory_region mem;
 147
 148     mem.slot = slot->slot;
 149     mem.guest_phys_addr = slot->start_addr;
 150     mem.memory_size = slot->memory_size;
 151     mem.userspace_addr = (unsigned long)qemu_get_ram_ptr(slot->phys_offset);
 152     mem.flags = slot->flags;
 153     if (s->migration_log) {
 154         mem.flags |= KVM_MEM_LOG_DIRTY_PAGES;
 155     }
 156     return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
 157 }
 158
 159 #ifdef KVM_UPSTREAM
 160 static void kvm_reset_vcpu(void *opaque)
 161 {
 162     CPUState *env = opaque;
 163
 164     kvm_arch_reset_vcpu(env);
 165 }
 166 #endif
 167
 168 int kvm_irqchip_in_kernel(void)
 169 {
 170     return kvm_state->irqchip_in_kernel;
 171 }
 172
 173 int kvm_pit_in_kernel(void)
 174 {
 175     return kvm_state->pit_in_kernel;
 176 }
 177
 178
 179 #ifdef KVM_UPSTREAM
 180 int kvm_init_vcpu(CPUState *env)
 181 {
 182     KVMState *s = kvm_state;
 183     long mmap_size;
 184     int ret;
 185
 186     DPRINTF("kvm_init_vcpu\n");
 187
 188     ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, env->cpu_index);
 189     if (ret < 0) {
 190         DPRINTF("kvm_create_vcpu failed\n");
 191         goto err;
 192     }
 193
 194     env->kvm_fd = ret;
 195     env->kvm_state = s;
 196
 197     mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
 198     if (mmap_size < 0) {
 199         DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
 200         goto err;
 201     }
 202
 203     env->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
 204                         env->kvm_fd, 0);
 205     if (env->kvm_run == MAP_FAILED) {
 206         ret = -errno;
 207         DPRINTF("mmap'ing vcpu state failed\n");
 208         goto err;
 209     }
 210
 211 #ifdef KVM_CAP_COALESCED_MMIO
 212     if (s->coalesced_mmio && !s->coalesced_mmio_ring)
 213         s->coalesced_mmio_ring = (void *) env->kvm_run +
 214                 s->coalesced_mmio * PAGE_SIZE;
 215 #endif
 216
 217     ret = kvm_arch_init_vcpu(env);
 218     if (ret == 0) {
 219         qemu_register_reset(kvm_reset_vcpu, env);
 220         kvm_arch_reset_vcpu(env);
 221     }
 222 err:
 223     return ret;
 224 }
 225 #endif
 226
 227 /*
 228  * dirty pages logging control
 229  */
 230 static int kvm_dirty_pages_log_change(target_phys_addr_t phys_addr,
 231                                       ram_addr_t size, int flags, int mask)
 232 {
 233     KVMState *s = kvm_state;
 234     KVMSlot *mem = kvm_lookup_matching_slot(s, phys_addr, phys_addr + size);
 235     int old_flags;
 236
 237     if (mem == NULL)  {
 238             fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
 239                     TARGET_FMT_plx "\n", __func__, phys_addr,
 240                     (target_phys_addr_t)(phys_addr + size - 1));
 241             return -EINVAL;
 242     }
 243
 244     old_flags = mem->flags;
 245
 246     flags = (mem->flags & ~mask) | flags;
 247     mem->flags = flags;
 248
 249     /* If nothing changed effectively, no need to issue ioctl */
 250     if (s->migration_log) {
 251         flags |= KVM_MEM_LOG_DIRTY_PAGES;
 252     }
 253     if (flags == old_flags) {
 254             return 0;
 255     }
 256
 257     return kvm_set_user_memory_region(s, mem);
 258 }
 259
 260 int kvm_log_start(target_phys_addr_t phys_addr, ram_addr_t size)
 261 {
 262         return kvm_dirty_pages_log_change(phys_addr, size,
 263                                           KVM_MEM_LOG_DIRTY_PAGES,
 264                                           KVM_MEM_LOG_DIRTY_PAGES);
 265 }
 266
 267 int kvm_log_stop(target_phys_addr_t phys_addr, ram_addr_t size)
 268 {
 269         return kvm_dirty_pages_log_change(phys_addr, size,
 270                                           0,
 271                                           KVM_MEM_LOG_DIRTY_PAGES);
 272 }
 273
 274 static int kvm_set_migration_log(int enable)
 275 {
 276     KVMState *s = kvm_state;
 277     KVMSlot *mem;
 278     int i, err;
 279
 280     s->migration_log = enable;
 281
 282     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 283         mem = &s->slots[i];
 284
 285         if (!!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) == enable) {
 286             continue;
 287         }
 288         err = kvm_set_user_memory_region(s, mem);
 289         if (err) {
 290             return err;
 291         }
 292     }
 293     return 0;
 294 }
 295
 296 /* get kvm's dirty pages bitmap and update qemu's */
 297 static int kvm_get_dirty_pages_log_range(unsigned long start_addr,
 298                                          unsigned long *bitmap,
 299                                          unsigned long offset,
 300                                          unsigned long mem_size)
 301 {
 302     unsigned int i, j;
 303     unsigned long page_number, addr, addr1, c;
 304     ram_addr_t ram_addr;
 305     unsigned int len = ((mem_size / TARGET_PAGE_SIZE) + HOST_LONG_BITS - 1) /
 306         HOST_LONG_BITS;
 307
 308     /*
 309      * bitmap-traveling is faster than memory-traveling (for addr...)
 310      * especially when most of the memory is not dirty.
 311      */
 312     for (i = 0; i < len; i++) {
 313         if (bitmap[i] != 0) {
 314             c = leul_to_cpu(bitmap[i]);
 315             do {
 316                 j = ffsl(c) - 1;
 317                 c &= ~(1ul << j);
 318                 page_number = i * HOST_LONG_BITS + j;
 319                 addr1 = page_number * TARGET_PAGE_SIZE;
 320                 addr = offset + addr1;
 321                 ram_addr = cpu_get_physical_page_desc(addr);
 322                 cpu_physical_memory_set_dirty(ram_addr);
 323             } while (c != 0);
 324         }
 325     }
 326     return 0;
 327 }
 328
 329 #define ALIGN(x, y)  (((x)+(y)-1) & ~((y)-1))
 330
 331 /**
 332  * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
 333  * This function updates qemu's dirty bitmap using cpu_physical_memory_set_dirty().
 334  * This means all bits are set to dirty.
 335  *
 336  * @start_add: start of logged region.
 337  * @end_addr: end of logged region.
 338  */
 339 static int kvm_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
 340                                           target_phys_addr_t end_addr)
 341 {
 342     KVMState *s = kvm_state;
 343     unsigned long size, allocated_size = 0;
 344     KVMDirtyLog d;
 345     KVMSlot *mem;
 346     int ret = 0;
 347
 348     d.dirty_bitmap = NULL;
 349     while (start_addr < end_addr) {
 350         mem = kvm_lookup_overlapping_slot(s, start_addr, end_addr);
 351         if (mem == NULL) {
 352             break;
 353         }
 354
 355         size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS), HOST_LONG_BITS) / 8;
 356         if (!d.dirty_bitmap) {
 357             d.dirty_bitmap = qemu_malloc(size);
 358         } else if (size > allocated_size) {
 359             d.dirty_bitmap = qemu_realloc(d.dirty_bitmap, size);
 360         }
 361         allocated_size = size;
 362         memset(d.dirty_bitmap, 0, allocated_size);
 363
 364         d.slot = mem->slot;
 365
 366         if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
 367             DPRINTF("ioctl failed %d\n", errno);
 368             ret = -1;
 369             break;
 370         }
 371
 372         kvm_get_dirty_pages_log_range(mem->start_addr, d.dirty_bitmap,
 373                                       mem->start_addr, mem->memory_size);
 374         start_addr = mem->start_addr + mem->memory_size;
 375     }
 376     qemu_free(d.dirty_bitmap);
 377
 378     return ret;
 379 }
 380
 381 int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
 382 {
 383     int ret = -ENOSYS;
 384 #ifdef KVM_CAP_COALESCED_MMIO
 385     KVMState *s = kvm_state;
 386
 387     if (s->coalesced_mmio) {
 388         struct kvm_coalesced_mmio_zone zone;
 389
 390         zone.addr = start;
 391         zone.size = size;
 392
 393         ret = kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
 394     }
 395 #endif
 396
 397     return ret;
 398 }
 399
 400 int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
 401 {
 402     int ret = -ENOSYS;
 403 #ifdef KVM_CAP_COALESCED_MMIO
 404     KVMState *s = kvm_state;
 405
 406     if (s->coalesced_mmio) {
 407         struct kvm_coalesced_mmio_zone zone;
 408
 409         zone.addr = start;
 410         zone.size = size;
 411
 412         ret = kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
 413     }
 414 #endif
 415
 416     return ret;
 417 }
 418
 419 int kvm_check_extension(KVMState *s, unsigned int extension)
 420 {
 421     int ret;
 422
 423     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
 424     if (ret < 0) {
 425         ret = 0;
 426     }
 427
 428     return ret;
 429 }
 430
 431 static void kvm_set_phys_mem(target_phys_addr_t start_addr,
 432                              ram_addr_t size,
 433                              ram_addr_t phys_offset)
 434 {
 435     KVMState *s = kvm_state;
 436     ram_addr_t flags = phys_offset & ~TARGET_PAGE_MASK;
 437     KVMSlot *mem, old;
 438     int err;
 439
 440     if (start_addr & ~TARGET_PAGE_MASK) {
 441         if (flags >= IO_MEM_UNASSIGNED) {
 442             if (!kvm_lookup_overlapping_slot(s, start_addr,
 443                                              start_addr + size)) {
 444                 return;
 445             }
 446             fprintf(stderr, "Unaligned split of a KVM memory slot\n");
 447         } else {
 448             fprintf(stderr, "Only page-aligned memory slots supported\n");
 449         }
 450         abort();
 451     }
 452
 453     /* KVM does not support read-only slots */
 454     phys_offset &= ~IO_MEM_ROM;
 455
 456     while (1) {
 457         mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size);
 458         if (!mem) {
 459             break;
 460         }
 461
 462         if (flags < IO_MEM_UNASSIGNED && start_addr >= mem->start_addr &&
 463             (start_addr + size <= mem->start_addr + mem->memory_size) &&
 464             (phys_offset - start_addr == mem->phys_offset - mem->start_addr)) {
 465             /* The new slot fits into the existing one and comes with
 466              * identical parameters - nothing to be done. */
 467             return;
 468         }
 469
 470         old = *mem;
 471
 472         /* unregister the overlapping slot */
 473         mem->memory_size = 0;
 474         err = kvm_set_user_memory_region(s, mem);
 475         if (err) {
 476             fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
 477                     __func__, strerror(-err));
 478             abort();
 479         }
 480
 481         /* Workaround for older KVM versions: we can't join slots, even not by
 482          * unregistering the previous ones and then registering the larger
 483          * slot. We have to maintain the existing fragmentation. Sigh.
 484          *
 485          * This workaround assumes that the new slot starts at the same
 486          * address as the first existing one. If not or if some overlapping
 487          * slot comes around later, we will fail (not seen in practice so far)
 488          * - and actually require a recent KVM version. */
 489         if (s->broken_set_mem_region &&
 490             old.start_addr == start_addr && old.memory_size < size &&
 491             flags < IO_MEM_UNASSIGNED) {
 492             mem = kvm_alloc_slot(s);
 493             mem->memory_size = old.memory_size;
 494             mem->start_addr = old.start_addr;
 495             mem->phys_offset = old.phys_offset;
 496             mem->flags = 0;
 497
 498             err = kvm_set_user_memory_region(s, mem);
 499             if (err) {
 500                 fprintf(stderr, "%s: error updating slot: %s\n", __func__,
 501                         strerror(-err));
 502                 abort();
 503             }
 504
 505             start_addr += old.memory_size;
 506             phys_offset += old.memory_size;
 507             size -= old.memory_size;
 508             continue;
 509         }
 510
 511         /* register prefix slot */
 512         if (old.start_addr < start_addr) {
 513             mem = kvm_alloc_slot(s);
 514             mem->memory_size = start_addr - old.start_addr;
 515             mem->start_addr = old.start_addr;
 516             mem->phys_offset = old.phys_offset;
 517             mem->flags = 0;
 518
 519             err = kvm_set_user_memory_region(s, mem);
 520             if (err) {
 521                 fprintf(stderr, "%s: error registering prefix slot: %s\n",
 522                         __func__, strerror(-err));
 523                 abort();
 524             }
 525         }
 526
 527         /* register suffix slot */
 528         if (old.start_addr + old.memory_size > start_addr + size) {
 529             ram_addr_t size_delta;
 530
 531             mem = kvm_alloc_slot(s);
 532             mem->start_addr = start_addr + size;
 533             size_delta = mem->start_addr - old.start_addr;
 534             mem->memory_size = old.memory_size - size_delta;
 535             mem->phys_offset = old.phys_offset + size_delta;
 536             mem->flags = 0;
 537
 538             err = kvm_set_user_memory_region(s, mem);
 539             if (err) {
 540                 fprintf(stderr, "%s: error registering suffix slot: %s\n",
 541                         __func__, strerror(-err));
 542                 abort();
 543             }
 544         }
 545     }
 546
 547     /* in case the KVM bug workaround already "consumed" the new slot */
 548     if (!size)
 549         return;
 550
 551     /* KVM does not need to know about this memory */
 552     if (flags >= IO_MEM_UNASSIGNED)
 553         return;
 554
 555     mem = kvm_alloc_slot(s);
 556     mem->memory_size = size;
 557     mem->start_addr = start_addr;
 558     mem->phys_offset = phys_offset;
 559     mem->flags = 0;
 560
 561     err = kvm_set_user_memory_region(s, mem);
 562     if (err) {
 563         fprintf(stderr, "%s: error registering slot: %s\n", __func__,
 564                 strerror(-err));
 565         abort();
 566     }
 567 }
 568
 569 static void kvm_client_set_memory(struct CPUPhysMemoryClient *client,
 570                                   target_phys_addr_t start_addr,
 571                                   ram_addr_t size,
 572                                   ram_addr_t phys_offset)
 573 {
 574         kvm_set_phys_mem(start_addr, size, phys_offset);
 575 }
 576
 577 static int kvm_client_sync_dirty_bitmap(struct CPUPhysMemoryClient *client,
 578                                         target_phys_addr_t start_addr,
 579                                         target_phys_addr_t end_addr)
 580 {
 581         return kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
 582 }
 583
 584 static int kvm_client_migration_log(struct CPUPhysMemoryClient *client,
 585                                     int enable)
 586 {
 587         return kvm_set_migration_log(enable);
 588 }
 589
 590 static CPUPhysMemoryClient kvm_cpu_phys_memory_client = {
 591         .set_memory = kvm_client_set_memory,
 592         .sync_dirty_bitmap = kvm_client_sync_dirty_bitmap,
 593         .migration_log = kvm_client_migration_log,
 594 };
 595
 596
 597 void kvm_cpu_register_phys_memory_client(void)
 598 {
 599     cpu_register_phys_memory_client(&kvm_cpu_phys_memory_client);
 600 }
 601
 602 #ifdef KVM_UPSTREAM
 603
 604 int kvm_init(int smp_cpus)
 605 {
 606     static const char upgrade_note[] =
 607         "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
 608         "(see http://sourceforge.net/projects/kvm).\n";
 609     KVMState *s;
 610     int ret;
 611     int i;
 612
 613     s = qemu_mallocz(sizeof(KVMState));
 614
 615 #ifdef KVM_CAP_SET_GUEST_DEBUG
 616     QTAILQ_INIT(&s->kvm_sw_breakpoints);
 617 #endif
 618     for (i = 0; i < ARRAY_SIZE(s->slots); i++)
 619         s->slots[i].slot = i;
 620
 621     s->vmfd = -1;
 622     s->fd = qemu_open("/dev/kvm", O_RDWR);
 623     if (s->fd == -1) {
 624         fprintf(stderr, "Could not access KVM kernel module: %m\n");
 625         ret = -errno;
 626         goto err;
 627     }
 628
 629     ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
 630     if (ret < KVM_API_VERSION) {
 631         if (ret > 0)
 632             ret = -EINVAL;
 633         fprintf(stderr, "kvm version too old\n");
 634         goto err;
 635     }
 636
 637     if (ret > KVM_API_VERSION) {
 638         ret = -EINVAL;
 639         fprintf(stderr, "kvm version not supported\n");
 640         goto err;
 641     }
 642
 643     s->vmfd = kvm_ioctl(s, KVM_CREATE_VM, 0);
 644     if (s->vmfd < 0) {
 645 #ifdef TARGET_S390X
 646         fprintf(stderr, "Please add the 'switch_amode' kernel parameter to "
 647                         "your host kernel command line\n");
 648 #endif
 649         goto err;
 650     }
 651
 652     /* initially, KVM allocated its own memory and we had to jump through
 653      * hooks to make phys_ram_base point to this.  Modern versions of KVM
 654      * just use a user allocated buffer so we can use regular pages
 655      * unmodified.  Make sure we have a sufficiently modern version of KVM.
 656      */
 657     if (!kvm_check_extension(s, KVM_CAP_USER_MEMORY)) {
 658         ret = -EINVAL;
 659         fprintf(stderr, "kvm does not support KVM_CAP_USER_MEMORY\n%s",
 660                 upgrade_note);
 661         goto err;
 662     }
 663
 664     /* There was a nasty bug in < kvm-80 that prevents memory slots from being
 665      * destroyed properly.  Since we rely on this capability, refuse to work
 666      * with any kernel without this capability. */
 667     if (!kvm_check_extension(s, KVM_CAP_DESTROY_MEMORY_REGION_WORKS)) {
 668         ret = -EINVAL;
 669
 670         fprintf(stderr,
 671                 "KVM kernel module broken (DESTROY_MEMORY_REGION).\n%s",
 672                 upgrade_note);
 673         goto err;
 674     }
 675
 676     s->coalesced_mmio = 0;
 677 #ifdef KVM_CAP_COALESCED_MMIO
 678     s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
 679     s->coalesced_mmio_ring = NULL;
 680 #endif
 681
 682     s->broken_set_mem_region = 1;
 683 #ifdef KVM_CAP_JOIN_MEMORY_REGIONS_WORKS
 684     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
 685     if (ret > 0) {
 686         s->broken_set_mem_region = 0;
 687     }
 688 #endif
 689
 690     s->vcpu_events = 0;
 691 #ifdef KVM_CAP_VCPU_EVENTS
 692     s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
 693 #endif
 694
 695     s->robust_singlestep = 0;
 696 #ifdef KVM_CAP_X86_ROBUST_SINGLESTEP
 697     s->robust_singlestep =
 698         kvm_check_extension(s, KVM_CAP_X86_ROBUST_SINGLESTEP);
 699 #endif
 700
 701     s->debugregs = 0;
 702 #ifdef KVM_CAP_DEBUGREGS
 703     s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS);
 704 #endif
 705
 706     s->xsave = 0;
 707 #ifdef KVM_CAP_XSAVE
 708     s->xsave = kvm_check_extension(s, KVM_CAP_XSAVE);
 709 #endif
 710
 711     s->xcrs = 0;
 712 #ifdef KVM_CAP_XCRS
 713     s->xcrs = kvm_check_extension(s, KVM_CAP_XCRS);
 714 #endif
 715
 716     ret = kvm_arch_init(s, smp_cpus);
 717     if (ret < 0)
 718         goto err;
 719
 720     kvm_state = s;
 721     cpu_register_phys_memory_client(&kvm_cpu_phys_memory_client);
 722
 723     return 0;
 724
 725 err:
 726     if (s) {
 727         if (s->vmfd != -1)
 728             close(s->vmfd);
 729         if (s->fd != -1)
 730             close(s->fd);
 731     }
 732     qemu_free(s);
 733
 734     return ret;
 735 }
 736 #endif
 737
 738 static int kvm_handle_io(uint16_t port, void *data, int direction, int size,
 739                          uint32_t count)
 740 {
 741     int i;
 742     uint8_t *ptr = data;
 743
 744     for (i = 0; i < count; i++) {
 745         if (direction == KVM_EXIT_IO_IN) {
 746             switch (size) {
 747             case 1:
 748                 stb_p(ptr, cpu_inb(port));
 749                 break;
 750             case 2:
 751                 stw_p(ptr, cpu_inw(port));
 752                 break;
 753             case 4:
 754                 stl_p(ptr, cpu_inl(port));
 755                 break;
 756             }
 757         } else {
 758             switch (size) {
 759             case 1:
 760                 cpu_outb(port, ldub_p(ptr));
 761                 break;
 762             case 2:
 763                 cpu_outw(port, lduw_p(ptr));
 764                 break;
 765             case 4:
 766                 cpu_outl(port, ldl_p(ptr));
 767                 break;
 768             }
 769         }
 770
 771         ptr += size;
 772     }
 773
 774     return 1;
 775 }
 776
 777 #ifdef KVM_CAP_INTERNAL_ERROR_DATA
 778 static void kvm_handle_internal_error(CPUState *env, struct kvm_run *run)
 779 {
 780
 781     if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) {
 782         int i;
 783
 784         fprintf(stderr, "KVM internal error. Suberror: %d\n",
 785                 run->internal.suberror);
 786
 787         for (i = 0; i < run->internal.ndata; ++i) {
 788             fprintf(stderr, "extra data[%d]: %"PRIx64"\n",
 789                     i, (uint64_t)run->internal.data[i]);
 790         }
 791     }
 792     cpu_dump_state(env, stderr, fprintf, 0);
 793     if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) {
 794         fprintf(stderr, "emulation failure\n");
 795         if (!kvm_arch_stop_on_emulation_error(env))
 796                 return;
 797     }
 798     /* FIXME: Should trigger a qmp message to let management know
 799      * something went wrong.
 800      */
 801     vm_stop(0);
 802 }
 803 #endif
 804
 805 void kvm_flush_coalesced_mmio_buffer(void)
 806 {
 807 #ifdef KVM_CAP_COALESCED_MMIO
 808     KVMState *s = kvm_state;
 809     if (s->coalesced_mmio_ring) {
 810         struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
 811         while (ring->first != ring->last) {
 812             struct kvm_coalesced_mmio *ent;
 813
 814             ent = &ring->coalesced_mmio[ring->first];
 815
 816             cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
 817             smp_wmb();
 818             ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
 819         }
 820     }
 821 #endif
 822 }
 823
 824 #ifdef KVM_UPSTREAM
 825
 826 static void do_kvm_cpu_synchronize_state(void *_env)
 827 {
 828     CPUState *env = _env;
 829
 830     if (!env->kvm_vcpu_dirty) {
 831         kvm_arch_get_registers(env);
 832         env->kvm_vcpu_dirty = 1;
 833     }
 834 }
 835
 836 void kvm_cpu_synchronize_state(CPUState *env)
 837 {
 838     if (!env->kvm_vcpu_dirty)
 839         run_on_cpu(env, do_kvm_cpu_synchronize_state, env);
 840 }
 841
 842 void kvm_cpu_synchronize_post_reset(CPUState *env)
 843 {
 844     kvm_arch_put_registers(env, KVM_PUT_RESET_STATE);
 845     env->kvm_vcpu_dirty = 0;
 846 }
 847
 848 void kvm_cpu_synchronize_post_init(CPUState *env)
 849 {
 850     kvm_arch_put_registers(env, KVM_PUT_FULL_STATE);
 851     env->kvm_vcpu_dirty = 0;
 852 }
 853
 854 int kvm_cpu_exec(CPUState *env)
 855 {
 856     struct kvm_run *run = env->kvm_run;
 857     int ret;
 858
 859     DPRINTF("kvm_cpu_exec()\n");
 860
 861     do {
 862 #ifndef CONFIG_IOTHREAD
 863         if (env->exit_request) {
 864             DPRINTF("interrupt exit requested\n");
 865             ret = 0;
 866             break;
 867         }
 868 #endif
 869
 870         if (kvm_arch_process_irqchip_events(env)) {
 871             ret = 0;
 872             break;
 873         }
 874
 875         if (env->kvm_vcpu_dirty) {
 876             kvm_arch_put_registers(env, KVM_PUT_RUNTIME_STATE);
 877             env->kvm_vcpu_dirty = 0;
 878         }
 879
 880         kvm_arch_pre_run(env, run);
 881         cpu_single_env = NULL;
 882         qemu_mutex_unlock_iothread();
 883         ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);
 884         qemu_mutex_lock_iothread();
 885         cpu_single_env = env;
 886         kvm_arch_post_run(env, run);
 887
 888         if (ret == -EINTR || ret == -EAGAIN) {
 889             cpu_exit(env);
 890             DPRINTF("io window exit\n");
 891             ret = 0;
 892             break;
 893         }
 894
 895         if (ret < 0) {
 896             DPRINTF("kvm run failed %s\n", strerror(-ret));
 897             abort();
 898         }
 899
 900         kvm_flush_coalesced_mmio_buffer();
 901
 902         ret = 0; /* exit loop */
 903         switch (run->exit_reason) {
 904         case KVM_EXIT_IO:
 905             DPRINTF("handle_io\n");
 906             ret = kvm_handle_io(run->io.port,
 907                                 (uint8_t *)run + run->io.data_offset,
 908                                 run->io.direction,
 909                                 run->io.size,
 910                                 run->io.count);
 911             break;
 912         case KVM_EXIT_MMIO:
 913             DPRINTF("handle_mmio\n");
 914             cpu_physical_memory_rw(run->mmio.phys_addr,
 915                                    run->mmio.data,
 916                                    run->mmio.len,
 917                                    run->mmio.is_write);
 918             ret = 1;
 919             break;
 920         case KVM_EXIT_IRQ_WINDOW_OPEN:
 921             DPRINTF("irq_window_open\n");
 922             break;
 923         case KVM_EXIT_SHUTDOWN:
 924             DPRINTF("shutdown\n");
 925             qemu_system_reset_request();
 926             ret = 1;
 927             break;
 928         case KVM_EXIT_UNKNOWN:
 929             DPRINTF("kvm_exit_unknown\n");
 930             break;
 931         case KVM_EXIT_FAIL_ENTRY:
 932             DPRINTF("kvm_exit_fail_entry\n");
 933             break;
 934         case KVM_EXIT_EXCEPTION:
 935             DPRINTF("kvm_exit_exception\n");
 936             break;
 937 #ifdef KVM_CAP_INTERNAL_ERROR_DATA
 938         case KVM_EXIT_INTERNAL_ERROR:
 939             kvm_handle_internal_error(env, run);
 940             break;
 941 #endif
 942         case KVM_EXIT_DEBUG:
 943             DPRINTF("kvm_exit_debug\n");
 944 #ifdef KVM_CAP_SET_GUEST_DEBUG
 945             if (kvm_arch_debug(&run->debug.arch)) {
 946                 gdb_set_stop_cpu(env);
 947                 vm_stop(EXCP_DEBUG);
 948                 env->exception_index = EXCP_DEBUG;
 949                 return 0;
 950             }
 951             /* re-enter, this exception was guest-internal */
 952             ret = 1;
 953 #endif /* KVM_CAP_SET_GUEST_DEBUG */
 954             break;
 955         default:
 956             DPRINTF("kvm_arch_handle_exit\n");
 957             ret = kvm_arch_handle_exit(env, run);
 958             break;
 959         }
 960     } while (ret > 0);
 961
 962     if (env->exit_request) {
 963         env->exit_request = 0;
 964         env->exception_index = EXCP_INTERRUPT;
 965     }
 966
 967     return ret;
 968 }
 969
 970 #endif
 971 int kvm_ioctl(KVMState *s, int type, ...)
 972 {
 973     int ret;
 974     void *arg;
 975     va_list ap;
 976
 977     va_start(ap, type);
 978     arg = va_arg(ap, void *);
 979     va_end(ap);
 980
 981     ret = ioctl(s->fd, type, arg);
 982     if (ret == -1)
 983         ret = -errno;
 984
 985     return ret;
 986 }
 987
 988 int kvm_vm_ioctl(KVMState *s, int type, ...)
 989 {
 990     int ret;
 991     void *arg;
 992     va_list ap;
 993
 994     va_start(ap, type);
 995     arg = va_arg(ap, void *);
 996     va_end(ap);
 997
 998     ret = ioctl(s->vmfd, type, arg);
 999     if (ret == -1)
1000         ret = -errno;
1001
1002     return ret;
1003 }
1004
1005 int kvm_vcpu_ioctl(CPUState *env, int type, ...)
1006 {
1007     int ret;
1008     void *arg;
1009     va_list ap;
1010
1011     va_start(ap, type);
1012     arg = va_arg(ap, void *);
1013     va_end(ap);
1014
1015     ret = ioctl(env->kvm_fd, type, arg);
1016     if (ret == -1)
1017         ret = -errno;
1018
1019     return ret;
1020 }
1021
1022 int kvm_has_sync_mmu(void)
1023 {
1024 #ifdef KVM_CAP_SYNC_MMU
1025     KVMState *s = kvm_state;
1026
1027     return kvm_check_extension(s, KVM_CAP_SYNC_MMU);
1028 #else
1029     return 0;
1030 #endif
1031 }
1032
1033 int kvm_has_vcpu_events(void)
1034 {
1035     return kvm_state->vcpu_events;
1036 }
1037
1038 int kvm_has_robust_singlestep(void)
1039 {
1040     return kvm_state->robust_singlestep;
1041 }
1042
1043 int kvm_has_debugregs(void)
1044 {
1045     return kvm_state->debugregs;
1046 }
1047
1048 #ifdef KVM_UPSTREAM
1049 int kvm_has_xsave(void)
1050 {
1051     return kvm_state->xsave;
1052 }
1053
1054 int kvm_has_xcrs(void)
1055 {
1056     return kvm_state->xcrs;
1057 }
1058 #endif
1059
1060 void kvm_setup_guest_memory(void *start, size_t size)
1061 {
1062     if (!kvm_has_sync_mmu()) {
1063 #ifdef MADV_DONTFORK
1064         int ret = madvise(start, size, MADV_DONTFORK);
1065
1066         if (ret) {
1067             perror("madvice");
1068             exit(1);
1069         }
1070 #else
1071         fprintf(stderr,
1072                 "Need MADV_DONTFORK in absence of synchronous KVM MMU\n");
1073         exit(1);
1074 #endif
1075     }
1076 }
1077
1078 #ifdef KVM_CAP_SET_GUEST_DEBUG
1079 #ifndef KVM_UPSTREAM
1080 #define run_on_cpu on_vcpu
1081 static void on_vcpu(CPUState *env, void (*func)(void *data), void *data);
1082 #endif /* !KVM_UPSTREAM */
1083
1084 struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *env,
1085                                                  target_ulong pc)
1086 {
1087     struct kvm_sw_breakpoint *bp;
1088
1089     QTAILQ_FOREACH(bp, &env->kvm_state->kvm_sw_breakpoints, entry) {
1090         if (bp->pc == pc)
1091             return bp;
1092     }
1093     return NULL;
1094 }
1095
1096 int kvm_sw_breakpoints_active(CPUState *env)
1097 {
1098     return !QTAILQ_EMPTY(&env->kvm_state->kvm_sw_breakpoints);
1099 }
1100
1101 struct kvm_set_guest_debug_data {
1102     struct kvm_guest_debug dbg;
1103     CPUState *env;
1104     int err;
1105 };
1106
1107 static void kvm_invoke_set_guest_debug(void *data)
1108 {
1109     struct kvm_set_guest_debug_data *dbg_data = data;
1110     CPUState *env = dbg_data->env;
1111
1112     dbg_data->err = kvm_vcpu_ioctl(env, KVM_SET_GUEST_DEBUG, &dbg_data->dbg);
1113 }
1114
1115 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1116 {
1117     struct kvm_set_guest_debug_data data;
1118
1119     data.dbg.control = reinject_trap;
1120
1121     if (env->singlestep_enabled) {
1122         data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
1123     }
1124     kvm_arch_update_guest_debug(env, &data.dbg);
1125     data.env = env;
1126
1127     run_on_cpu(env, kvm_invoke_set_guest_debug, &data);
1128     return data.err;
1129 }
1130
1131 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1132                           target_ulong len, int type)
1133 {
1134     struct kvm_sw_breakpoint *bp;
1135     CPUState *env;
1136     int err;
1137
1138     if (type == GDB_BREAKPOINT_SW) {
1139         bp = kvm_find_sw_breakpoint(current_env, addr);
1140         if (bp) {
1141             bp->use_count++;
1142             return 0;
1143         }
1144
1145         bp = qemu_malloc(sizeof(struct kvm_sw_breakpoint));
1146         if (!bp)
1147             return -ENOMEM;
1148
1149         bp->pc = addr;
1150         bp->use_count = 1;
1151         err = kvm_arch_insert_sw_breakpoint(current_env, bp);
1152         if (err) {
1153             free(bp);
1154             return err;
1155         }
1156
1157         QTAILQ_INSERT_HEAD(&current_env->kvm_state->kvm_sw_breakpoints,
1158                           bp, entry);
1159     } else {
1160         err = kvm_arch_insert_hw_breakpoint(addr, len, type);
1161         if (err)
1162             return err;
1163     }
1164
1165     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1166         err = kvm_update_guest_debug(env, 0);
1167         if (err)
1168             return err;
1169     }
1170     return 0;
1171 }
1172
1173 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1174                           target_ulong len, int type)
1175 {
1176     struct kvm_sw_breakpoint *bp;
1177     CPUState *env;
1178     int err;
1179
1180     if (type == GDB_BREAKPOINT_SW) {
1181         bp = kvm_find_sw_breakpoint(current_env, addr);
1182         if (!bp)
1183             return -ENOENT;
1184
1185         if (bp->use_count > 1) {
1186             bp->use_count--;
1187             return 0;
1188         }
1189
1190         err = kvm_arch_remove_sw_breakpoint(current_env, bp);
1191         if (err)
1192             return err;
1193
1194         QTAILQ_REMOVE(&current_env->kvm_state->kvm_sw_breakpoints, bp, entry);
1195         qemu_free(bp);
1196     } else {
1197         err = kvm_arch_remove_hw_breakpoint(addr, len, type);
1198         if (err)
1199             return err;
1200     }
1201
1202     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1203         err = kvm_update_guest_debug(env, 0);
1204         if (err)
1205             return err;
1206     }
1207     return 0;
1208 }
1209
1210 void kvm_remove_all_breakpoints(CPUState *current_env)
1211 {
1212     struct kvm_sw_breakpoint *bp, *next;
1213     KVMState *s = current_env->kvm_state;
1214     CPUState *env;
1215
1216     QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
1217         if (kvm_arch_remove_sw_breakpoint(current_env, bp) != 0) {
1218             /* Try harder to find a CPU that currently sees the breakpoint. */
1219             for (env = first_cpu; env != NULL; env = env->next_cpu) {
1220                 if (kvm_arch_remove_sw_breakpoint(env, bp) == 0)
1221                     break;
1222             }
1223         }
1224     }
1225     kvm_arch_remove_all_hw_breakpoints();
1226
1227     for (env = first_cpu; env != NULL; env = env->next_cpu)
1228         kvm_update_guest_debug(env, 0);
1229 }
1230
1231 #else /* !KVM_CAP_SET_GUEST_DEBUG */
1232
1233 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1234 {
1235     return -EINVAL;
1236 }
1237
1238 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1239                           target_ulong len, int type)
1240 {
1241     return -EINVAL;
1242 }
1243
1244 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1245                           target_ulong len, int type)
1246 {
1247     return -EINVAL;
1248 }
1249
1250 void kvm_remove_all_breakpoints(CPUState *current_env)
1251 {
1252 }
1253 #endif /* !KVM_CAP_SET_GUEST_DEBUG */
1254
1255 int kvm_set_signal_mask(CPUState *env, const sigset_t *sigset)
1256 {
1257     struct kvm_signal_mask *sigmask;
1258     int r;
1259
1260     if (!sigset)
1261         return kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, NULL);
1262
1263     sigmask = qemu_malloc(sizeof(*sigmask) + sizeof(*sigset));
1264
1265     sigmask->len = 8;
1266     memcpy(sigmask->sigset, sigset, sizeof(*sigset));
1267     r = kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, sigmask);
1268     free(sigmask);
1269
1270     return r;
1271 }
1272
1273 int kvm_set_ioeventfd_pio_word(int fd, uint16_t addr, uint16_t val, bool assign)
1274 {
1275 #ifdef KVM_IOEVENTFD
1276     struct kvm_ioeventfd kick = {
1277         .datamatch = val,
1278         .addr = addr,
1279         .len = 2,
1280         .flags = KVM_IOEVENTFD_FLAG_DATAMATCH | KVM_IOEVENTFD_FLAG_PIO,
1281         .fd = fd,
1282     };
1283     int r;
1284     if (!kvm_enabled())
1285         return -ENOSYS;
1286     if (!assign)
1287         kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
1288     r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
1289     if (r < 0)
1290         return r;
1291     return 0;
1292 #else
1293     return -ENOSYS;
1294 #endif
1295 }
1296
1297 #if defined(KVM_IRQFD)
1298 int kvm_set_irqfd(int gsi, int fd, bool assigned)
1299 {
1300     struct kvm_irqfd irqfd = {
1301         .fd = fd,
1302         .gsi = gsi,
1303         .flags = assigned ? 0 : KVM_IRQFD_FLAG_DEASSIGN,
1304     };
1305     int r;
1306     if (!kvm_enabled() || !kvm_irqchip_in_kernel())
1307         return -ENOSYS;
1308
1309     r = kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd);
1310     if (r < 0)
1311         return r;
1312     return 0;
1313 }
1314 #endif
1315
1316 #undef PAGE_SIZE
1317 #include "qemu-kvm.c"