kvm-all.c

   1 /*
   2  * QEMU KVM support
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *           Red Hat, Inc. 2008
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *  Glauber Costa     <gcosta@redhat.com>
  10  *
  11  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  12  * See the COPYING file in the top-level directory.
  13  *
  14  */
  15
  16 #include <sys/types.h>
  17 #include <sys/ioctl.h>
  18 #include <sys/mman.h>
  19 #include <stdarg.h>
  20
  21 #include <linux/kvm.h>
  22
  23 #include "qemu-common.h"
  24 #include "qemu-barrier.h"
  25 #include "sysemu.h"
  26 #include "hw/hw.h"
  27 #include "gdbstub.h"
  28 #include "kvm.h"
  29 #include "bswap.h"
  30
  31 /* KVM uses PAGE_SIZE in it's definition of COALESCED_MMIO_MAX */
  32 #define PAGE_SIZE TARGET_PAGE_SIZE
  33
  34 //#define DEBUG_KVM
  35
  36 #ifdef DEBUG_KVM
  37 #define DPRINTF(fmt, ...) \
  38     do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
  39 #else
  40 #define DPRINTF(fmt, ...) \
  41     do { } while (0)
  42 #endif
  43
  44 #ifdef OBSOLETE_KVM_IMPL
  45
  46 typedef struct KVMSlot
  47 {
  48     target_phys_addr_t start_addr;
  49     ram_addr_t memory_size;
  50     ram_addr_t phys_offset;
  51     int slot;
  52     int flags;
  53 } KVMSlot;
  54
  55 typedef struct kvm_dirty_log KVMDirtyLog;
  56
  57 struct KVMState
  58 {
  59     KVMSlot slots[32];
  60     int fd;
  61     int vmfd;
  62     int coalesced_mmio;
  63 #ifdef KVM_CAP_COALESCED_MMIO
  64     struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
  65 #endif
  66     int broken_set_mem_region;
  67     int migration_log;
  68     int vcpu_events;
  69     int robust_singlestep;
  70     int debugregs;
  71 #ifdef KVM_CAP_SET_GUEST_DEBUG
  72     struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
  73 #endif
  74     int irqchip_in_kernel;
  75     int pit_in_kernel;
  76     int xsave, xcrs;
  77 };
  78
  79 static KVMState *kvm_state;
  80
  81 #endif
  82
  83 static KVMSlot *kvm_alloc_slot(KVMState *s)
  84 {
  85     int i;
  86
  87     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
  88         /* KVM private memory slots */
  89         if (i >= 8 && i < 12)
  90             continue;
  91         if (s->slots[i].memory_size == 0)
  92             return &s->slots[i];
  93     }
  94
  95     fprintf(stderr, "%s: no free slot available\n", __func__);
  96     abort();
  97 }
  98
  99 static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
 100                                          target_phys_addr_t start_addr,
 101                                          target_phys_addr_t end_addr)
 102 {
 103     int i;
 104
 105     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 106         KVMSlot *mem = &s->slots[i];
 107
 108         if (start_addr == mem->start_addr &&
 109             end_addr == mem->start_addr + mem->memory_size) {
 110             return mem;
 111         }
 112     }
 113
 114     return NULL;
 115 }
 116
 117 /*
 118  * Find overlapping slot with lowest start address
 119  */
 120 static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
 121                                             target_phys_addr_t start_addr,
 122                                             target_phys_addr_t end_addr)
 123 {
 124     KVMSlot *found = NULL;
 125     int i;
 126
 127     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 128         KVMSlot *mem = &s->slots[i];
 129
 130         if (mem->memory_size == 0 ||
 131             (found && found->start_addr < mem->start_addr)) {
 132             continue;
 133         }
 134
 135         if (end_addr > mem->start_addr &&
 136             start_addr < mem->start_addr + mem->memory_size) {
 137             found = mem;
 138         }
 139     }
 140
 141     return found;
 142 }
 143
 144 int kvm_physical_memory_addr_from_ram(KVMState *s, ram_addr_t ram_addr,
 145                                       target_phys_addr_t *phys_addr)
 146 {
 147     int i;
 148
 149     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 150         KVMSlot *mem = &s->slots[i];
 151
 152         if (ram_addr >= mem->phys_offset &&
 153             ram_addr < mem->phys_offset + mem->memory_size) {
 154             *phys_addr = mem->start_addr + (ram_addr - mem->phys_offset);
 155             return 1;
 156         }
 157     }
 158
 159     return 0;
 160 }
 161
 162 static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
 163 {
 164     struct kvm_userspace_memory_region mem;
 165
 166     mem.slot = slot->slot;
 167     mem.guest_phys_addr = slot->start_addr;
 168     mem.memory_size = slot->memory_size;
 169     mem.userspace_addr = (unsigned long)qemu_get_ram_ptr(slot->phys_offset);
 170     mem.flags = slot->flags;
 171     if (s->migration_log) {
 172         mem.flags |= KVM_MEM_LOG_DIRTY_PAGES;
 173     }
 174     return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
 175 }
 176
 177 #ifdef OBSOLETE_KVM_IMPL
 178 static void kvm_reset_vcpu(void *opaque)
 179 {
 180     CPUState *env = opaque;
 181
 182     kvm_arch_reset_vcpu(env);
 183 }
 184 #endif
 185
 186 int kvm_irqchip_in_kernel(void)
 187 {
 188     return kvm_state->irqchip_in_kernel;
 189 }
 190
 191 int kvm_pit_in_kernel(void)
 192 {
 193     return kvm_state->pit_in_kernel;
 194 }
 195
 196
 197 #ifdef OBSOLETE_KVM_IMPL
 198 int kvm_init_vcpu(CPUState *env)
 199 {
 200     KVMState *s = kvm_state;
 201     long mmap_size;
 202     int ret;
 203
 204     DPRINTF("kvm_init_vcpu\n");
 205
 206     ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, env->cpu_index);
 207     if (ret < 0) {
 208         DPRINTF("kvm_create_vcpu failed\n");
 209         goto err;
 210     }
 211
 212     env->kvm_fd = ret;
 213     env->kvm_state = s;
 214
 215     mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
 216     if (mmap_size < 0) {
 217         DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
 218         goto err;
 219     }
 220
 221     env->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
 222                         env->kvm_fd, 0);
 223     if (env->kvm_run == MAP_FAILED) {
 224         ret = -errno;
 225         DPRINTF("mmap'ing vcpu state failed\n");
 226         goto err;
 227     }
 228
 229 #ifdef KVM_CAP_COALESCED_MMIO
 230     if (s->coalesced_mmio && !s->coalesced_mmio_ring)
 231         s->coalesced_mmio_ring = (void *) env->kvm_run +
 232                 s->coalesced_mmio * PAGE_SIZE;
 233 #endif
 234
 235     ret = kvm_arch_init_vcpu(env);
 236     if (ret == 0) {
 237         qemu_register_reset(kvm_reset_vcpu, env);
 238         kvm_arch_reset_vcpu(env);
 239     }
 240 err:
 241     return ret;
 242 }
 243 #endif
 244
 245 /*
 246  * dirty pages logging control
 247  */
 248 static int kvm_dirty_pages_log_change(target_phys_addr_t phys_addr,
 249                                       ram_addr_t size, int flags, int mask)
 250 {
 251     KVMState *s = kvm_state;
 252     KVMSlot *mem = kvm_lookup_matching_slot(s, phys_addr, phys_addr + size);
 253     int old_flags;
 254
 255     if (mem == NULL)  {
 256             fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
 257                     TARGET_FMT_plx "\n", __func__, phys_addr,
 258                     (target_phys_addr_t)(phys_addr + size - 1));
 259             return -EINVAL;
 260     }
 261
 262     old_flags = mem->flags;
 263
 264     flags = (mem->flags & ~mask) | flags;
 265     mem->flags = flags;
 266
 267     /* If nothing changed effectively, no need to issue ioctl */
 268     if (s->migration_log) {
 269         flags |= KVM_MEM_LOG_DIRTY_PAGES;
 270     }
 271     if (flags == old_flags) {
 272             return 0;
 273     }
 274
 275     return kvm_set_user_memory_region(s, mem);
 276 }
 277
 278 int kvm_log_start(target_phys_addr_t phys_addr, ram_addr_t size)
 279 {
 280         return kvm_dirty_pages_log_change(phys_addr, size,
 281                                           KVM_MEM_LOG_DIRTY_PAGES,
 282                                           KVM_MEM_LOG_DIRTY_PAGES);
 283 }
 284
 285 int kvm_log_stop(target_phys_addr_t phys_addr, ram_addr_t size)
 286 {
 287         return kvm_dirty_pages_log_change(phys_addr, size,
 288                                           0,
 289                                           KVM_MEM_LOG_DIRTY_PAGES);
 290 }
 291
 292 static int kvm_set_migration_log(int enable)
 293 {
 294     KVMState *s = kvm_state;
 295     KVMSlot *mem;
 296     int i, err;
 297
 298     s->migration_log = enable;
 299
 300     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 301         mem = &s->slots[i];
 302
 303         if (!mem->memory_size) {
 304             continue;
 305         }
 306         if (!!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) == enable) {
 307             continue;
 308         }
 309         err = kvm_set_user_memory_region(s, mem);
 310         if (err) {
 311             return err;
 312         }
 313     }
 314     return 0;
 315 }
 316
 317 /* get kvm's dirty pages bitmap and update qemu's */
 318 static int kvm_get_dirty_pages_log_range(unsigned long start_addr,
 319                                          unsigned long *bitmap,
 320                                          unsigned long offset,
 321                                          unsigned long mem_size)
 322 {
 323     unsigned int i, j;
 324     unsigned long page_number, addr, addr1, c;
 325     ram_addr_t ram_addr;
 326     unsigned int len = ((mem_size / TARGET_PAGE_SIZE) + HOST_LONG_BITS - 1) /
 327         HOST_LONG_BITS;
 328
 329     /*
 330      * bitmap-traveling is faster than memory-traveling (for addr...)
 331      * especially when most of the memory is not dirty.
 332      */
 333     for (i = 0; i < len; i++) {
 334         if (bitmap[i] != 0) {
 335             c = leul_to_cpu(bitmap[i]);
 336             do {
 337                 j = ffsl(c) - 1;
 338                 c &= ~(1ul << j);
 339                 page_number = i * HOST_LONG_BITS + j;
 340                 addr1 = page_number * TARGET_PAGE_SIZE;
 341                 addr = offset + addr1;
 342                 ram_addr = cpu_get_physical_page_desc(addr);
 343                 cpu_physical_memory_set_dirty(ram_addr);
 344             } while (c != 0);
 345         }
 346     }
 347     return 0;
 348 }
 349
 350 #define ALIGN(x, y)  (((x)+(y)-1) & ~((y)-1))
 351
 352 /**
 353  * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
 354  * This function updates qemu's dirty bitmap using cpu_physical_memory_set_dirty().
 355  * This means all bits are set to dirty.
 356  *
 357  * @start_add: start of logged region.
 358  * @end_addr: end of logged region.
 359  */
 360 static int kvm_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
 361                                           target_phys_addr_t end_addr)
 362 {
 363     KVMState *s = kvm_state;
 364     unsigned long size, allocated_size = 0;
 365     KVMDirtyLog d;
 366     KVMSlot *mem;
 367     int ret = 0;
 368
 369     d.dirty_bitmap = NULL;
 370     while (start_addr < end_addr) {
 371         mem = kvm_lookup_overlapping_slot(s, start_addr, end_addr);
 372         if (mem == NULL) {
 373             break;
 374         }
 375
 376         size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS), HOST_LONG_BITS) / 8;
 377         if (!d.dirty_bitmap) {
 378             d.dirty_bitmap = qemu_malloc(size);
 379         } else if (size > allocated_size) {
 380             d.dirty_bitmap = qemu_realloc(d.dirty_bitmap, size);
 381         }
 382         allocated_size = size;
 383         memset(d.dirty_bitmap, 0, allocated_size);
 384
 385         d.slot = mem->slot;
 386
 387         if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
 388             DPRINTF("ioctl failed %d\n", errno);
 389             ret = -1;
 390             break;
 391         }
 392
 393         kvm_get_dirty_pages_log_range(mem->start_addr, d.dirty_bitmap,
 394                                       mem->start_addr, mem->memory_size);
 395         start_addr = mem->start_addr + mem->memory_size;
 396     }
 397     qemu_free(d.dirty_bitmap);
 398
 399     return ret;
 400 }
 401
 402 int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
 403 {
 404     int ret = -ENOSYS;
 405 #ifdef KVM_CAP_COALESCED_MMIO
 406     KVMState *s = kvm_state;
 407
 408     if (s->coalesced_mmio) {
 409         struct kvm_coalesced_mmio_zone zone;
 410
 411         zone.addr = start;
 412         zone.size = size;
 413
 414         ret = kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
 415     }
 416 #endif
 417
 418     return ret;
 419 }
 420
 421 int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
 422 {
 423     int ret = -ENOSYS;
 424 #ifdef KVM_CAP_COALESCED_MMIO
 425     KVMState *s = kvm_state;
 426
 427     if (s->coalesced_mmio) {
 428         struct kvm_coalesced_mmio_zone zone;
 429
 430         zone.addr = start;
 431         zone.size = size;
 432
 433         ret = kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
 434     }
 435 #endif
 436
 437     return ret;
 438 }
 439
 440 int kvm_check_extension(KVMState *s, unsigned int extension)
 441 {
 442     int ret;
 443
 444     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
 445     if (ret < 0) {
 446         ret = 0;
 447     }
 448
 449     return ret;
 450 }
 451
 452 static void kvm_set_phys_mem(target_phys_addr_t start_addr,
 453                              ram_addr_t size,
 454                              ram_addr_t phys_offset)
 455 {
 456     KVMState *s = kvm_state;
 457     ram_addr_t flags = phys_offset & ~TARGET_PAGE_MASK;
 458     KVMSlot *mem, old;
 459     int err;
 460
 461     /* kvm works in page size chunks, but the function may be called
 462        with sub-page size and unaligned start address. */
 463     size = TARGET_PAGE_ALIGN(size);
 464     start_addr = TARGET_PAGE_ALIGN(start_addr);
 465
 466     /* KVM does not support read-only slots */
 467     phys_offset &= ~IO_MEM_ROM;
 468
 469     while (1) {
 470         mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size);
 471         if (!mem) {
 472             break;
 473         }
 474
 475         if (flags < IO_MEM_UNASSIGNED && start_addr >= mem->start_addr &&
 476             (start_addr + size <= mem->start_addr + mem->memory_size) &&
 477             (phys_offset - start_addr == mem->phys_offset - mem->start_addr)) {
 478             /* The new slot fits into the existing one and comes with
 479              * identical parameters - nothing to be done. */
 480             return;
 481         }
 482
 483         old = *mem;
 484
 485         /* unregister the overlapping slot */
 486         mem->memory_size = 0;
 487         err = kvm_set_user_memory_region(s, mem);
 488         if (err) {
 489             fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
 490                     __func__, strerror(-err));
 491             abort();
 492         }
 493
 494         /* Workaround for older KVM versions: we can't join slots, even not by
 495          * unregistering the previous ones and then registering the larger
 496          * slot. We have to maintain the existing fragmentation. Sigh.
 497          *
 498          * This workaround assumes that the new slot starts at the same
 499          * address as the first existing one. If not or if some overlapping
 500          * slot comes around later, we will fail (not seen in practice so far)
 501          * - and actually require a recent KVM version. */
 502         if (s->broken_set_mem_region &&
 503             old.start_addr == start_addr && old.memory_size < size &&
 504             flags < IO_MEM_UNASSIGNED) {
 505             mem = kvm_alloc_slot(s);
 506             mem->memory_size = old.memory_size;
 507             mem->start_addr = old.start_addr;
 508             mem->phys_offset = old.phys_offset;
 509             mem->flags = 0;
 510
 511             err = kvm_set_user_memory_region(s, mem);
 512             if (err) {
 513                 fprintf(stderr, "%s: error updating slot: %s\n", __func__,
 514                         strerror(-err));
 515                 abort();
 516             }
 517
 518             start_addr += old.memory_size;
 519             phys_offset += old.memory_size;
 520             size -= old.memory_size;
 521             continue;
 522         }
 523
 524         /* register prefix slot */
 525         if (old.start_addr < start_addr) {
 526             mem = kvm_alloc_slot(s);
 527             mem->memory_size = start_addr - old.start_addr;
 528             mem->start_addr = old.start_addr;
 529             mem->phys_offset = old.phys_offset;
 530             mem->flags = 0;
 531
 532             err = kvm_set_user_memory_region(s, mem);
 533             if (err) {
 534                 fprintf(stderr, "%s: error registering prefix slot: %s\n",
 535                         __func__, strerror(-err));
 536                 abort();
 537             }
 538         }
 539
 540         /* register suffix slot */
 541         if (old.start_addr + old.memory_size > start_addr + size) {
 542             ram_addr_t size_delta;
 543
 544             mem = kvm_alloc_slot(s);
 545             mem->start_addr = start_addr + size;
 546             size_delta = mem->start_addr - old.start_addr;
 547             mem->memory_size = old.memory_size - size_delta;
 548             mem->phys_offset = old.phys_offset + size_delta;
 549             mem->flags = 0;
 550
 551             err = kvm_set_user_memory_region(s, mem);
 552             if (err) {
 553                 fprintf(stderr, "%s: error registering suffix slot: %s\n",
 554                         __func__, strerror(-err));
 555                 abort();
 556             }
 557         }
 558     }
 559
 560     /* in case the KVM bug workaround already "consumed" the new slot */
 561     if (!size)
 562         return;
 563
 564     /* KVM does not need to know about this memory */
 565     if (flags >= IO_MEM_UNASSIGNED)
 566         return;
 567
 568     mem = kvm_alloc_slot(s);
 569     mem->memory_size = size;
 570     mem->start_addr = start_addr;
 571     mem->phys_offset = phys_offset;
 572     mem->flags = 0;
 573
 574     err = kvm_set_user_memory_region(s, mem);
 575     if (err) {
 576         fprintf(stderr, "%s: error registering slot: %s\n", __func__,
 577                 strerror(-err));
 578         abort();
 579     }
 580 }
 581
 582 static void kvm_client_set_memory(struct CPUPhysMemoryClient *client,
 583                                   target_phys_addr_t start_addr,
 584                                   ram_addr_t size,
 585                                   ram_addr_t phys_offset)
 586 {
 587         kvm_set_phys_mem(start_addr, size, phys_offset);
 588 }
 589
 590 static int kvm_client_sync_dirty_bitmap(struct CPUPhysMemoryClient *client,
 591                                         target_phys_addr_t start_addr,
 592                                         target_phys_addr_t end_addr)
 593 {
 594         return kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
 595 }
 596
 597 static int kvm_client_migration_log(struct CPUPhysMemoryClient *client,
 598                                     int enable)
 599 {
 600         return kvm_set_migration_log(enable);
 601 }
 602
 603 static CPUPhysMemoryClient kvm_cpu_phys_memory_client = {
 604         .set_memory = kvm_client_set_memory,
 605         .sync_dirty_bitmap = kvm_client_sync_dirty_bitmap,
 606         .migration_log = kvm_client_migration_log,
 607 };
 608
 609
 610 void kvm_cpu_register_phys_memory_client(void)
 611 {
 612     cpu_register_phys_memory_client(&kvm_cpu_phys_memory_client);
 613 }
 614
 615 #ifdef OBSOLETE_KVM_IMPL
 616
 617 int kvm_init(int smp_cpus)
 618 {
 619     static const char upgrade_note[] =
 620         "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
 621         "(see http://sourceforge.net/projects/kvm).\n";
 622     KVMState *s;
 623     int ret;
 624     int i;
 625
 626     s = qemu_mallocz(sizeof(KVMState));
 627
 628 #ifdef KVM_CAP_SET_GUEST_DEBUG
 629     QTAILQ_INIT(&s->kvm_sw_breakpoints);
 630 #endif
 631     for (i = 0; i < ARRAY_SIZE(s->slots); i++)
 632         s->slots[i].slot = i;
 633
 634     s->vmfd = -1;
 635     s->fd = qemu_open("/dev/kvm", O_RDWR);
 636     if (s->fd == -1) {
 637         fprintf(stderr, "Could not access KVM kernel module: %m\n");
 638         ret = -errno;
 639         goto err;
 640     }
 641
 642     ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
 643     if (ret < KVM_API_VERSION) {
 644         if (ret > 0)
 645             ret = -EINVAL;
 646         fprintf(stderr, "kvm version too old\n");
 647         goto err;
 648     }
 649
 650     if (ret > KVM_API_VERSION) {
 651         ret = -EINVAL;
 652         fprintf(stderr, "kvm version not supported\n");
 653         goto err;
 654     }
 655
 656     s->vmfd = kvm_ioctl(s, KVM_CREATE_VM, 0);
 657     if (s->vmfd < 0) {
 658 #ifdef TARGET_S390X
 659         fprintf(stderr, "Please add the 'switch_amode' kernel parameter to "
 660                         "your host kernel command line\n");
 661 #endif
 662         goto err;
 663     }
 664
 665     /* initially, KVM allocated its own memory and we had to jump through
 666      * hooks to make phys_ram_base point to this.  Modern versions of KVM
 667      * just use a user allocated buffer so we can use regular pages
 668      * unmodified.  Make sure we have a sufficiently modern version of KVM.
 669      */
 670     if (!kvm_check_extension(s, KVM_CAP_USER_MEMORY)) {
 671         ret = -EINVAL;
 672         fprintf(stderr, "kvm does not support KVM_CAP_USER_MEMORY\n%s",
 673                 upgrade_note);
 674         goto err;
 675     }
 676
 677     /* There was a nasty bug in < kvm-80 that prevents memory slots from being
 678      * destroyed properly.  Since we rely on this capability, refuse to work
 679      * with any kernel without this capability. */
 680     if (!kvm_check_extension(s, KVM_CAP_DESTROY_MEMORY_REGION_WORKS)) {
 681         ret = -EINVAL;
 682
 683         fprintf(stderr,
 684                 "KVM kernel module broken (DESTROY_MEMORY_REGION).\n%s",
 685                 upgrade_note);
 686         goto err;
 687     }
 688
 689     s->coalesced_mmio = 0;
 690 #ifdef KVM_CAP_COALESCED_MMIO
 691     s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
 692     s->coalesced_mmio_ring = NULL;
 693 #endif
 694
 695     s->broken_set_mem_region = 1;
 696 #ifdef KVM_CAP_JOIN_MEMORY_REGIONS_WORKS
 697     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
 698     if (ret > 0) {
 699         s->broken_set_mem_region = 0;
 700     }
 701 #endif
 702
 703     s->vcpu_events = 0;
 704 #ifdef KVM_CAP_VCPU_EVENTS
 705     s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
 706 #endif
 707
 708     s->robust_singlestep = 0;
 709 #ifdef KVM_CAP_X86_ROBUST_SINGLESTEP
 710     s->robust_singlestep =
 711         kvm_check_extension(s, KVM_CAP_X86_ROBUST_SINGLESTEP);
 712 #endif
 713
 714     s->debugregs = 0;
 715 #ifdef KVM_CAP_DEBUGREGS
 716     s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS);
 717 #endif
 718
 719     s->xsave = 0;
 720 #ifdef KVM_CAP_XSAVE
 721     s->xsave = kvm_check_extension(s, KVM_CAP_XSAVE);
 722 #endif
 723
 724     s->xcrs = 0;
 725 #ifdef KVM_CAP_XCRS
 726     s->xcrs = kvm_check_extension(s, KVM_CAP_XCRS);
 727 #endif
 728
 729     ret = kvm_arch_init(s, smp_cpus);
 730     if (ret < 0)
 731         goto err;
 732
 733     kvm_state = s;
 734     cpu_register_phys_memory_client(&kvm_cpu_phys_memory_client);
 735
 736     return 0;
 737
 738 err:
 739     if (s) {
 740         if (s->vmfd != -1)
 741             close(s->vmfd);
 742         if (s->fd != -1)
 743             close(s->fd);
 744     }
 745     qemu_free(s);
 746
 747     return ret;
 748 }
 749 #endif
 750
 751 static int kvm_handle_io(uint16_t port, void *data, int direction, int size,
 752                          uint32_t count)
 753 {
 754     int i;
 755     uint8_t *ptr = data;
 756
 757     for (i = 0; i < count; i++) {
 758         if (direction == KVM_EXIT_IO_IN) {
 759             switch (size) {
 760             case 1:
 761                 stb_p(ptr, cpu_inb(port));
 762                 break;
 763             case 2:
 764                 stw_p(ptr, cpu_inw(port));
 765                 break;
 766             case 4:
 767                 stl_p(ptr, cpu_inl(port));
 768                 break;
 769             }
 770         } else {
 771             switch (size) {
 772             case 1:
 773                 cpu_outb(port, ldub_p(ptr));
 774                 break;
 775             case 2:
 776                 cpu_outw(port, lduw_p(ptr));
 777                 break;
 778             case 4:
 779                 cpu_outl(port, ldl_p(ptr));
 780                 break;
 781             }
 782         }
 783
 784         ptr += size;
 785     }
 786
 787     return 1;
 788 }
 789
 790 #ifdef KVM_CAP_INTERNAL_ERROR_DATA
 791 static void kvm_handle_internal_error(CPUState *env, struct kvm_run *run)
 792 {
 793
 794     if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) {
 795         int i;
 796
 797         fprintf(stderr, "KVM internal error. Suberror: %d\n",
 798                 run->internal.suberror);
 799
 800         for (i = 0; i < run->internal.ndata; ++i) {
 801             fprintf(stderr, "extra data[%d]: %"PRIx64"\n",
 802                     i, (uint64_t)run->internal.data[i]);
 803         }
 804     }
 805     cpu_dump_state(env, stderr, fprintf, 0);
 806     if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) {
 807         fprintf(stderr, "emulation failure\n");
 808         if (!kvm_arch_stop_on_emulation_error(env))
 809                 return;
 810     }
 811     /* FIXME: Should trigger a qmp message to let management know
 812      * something went wrong.
 813      */
 814     vm_stop(0);
 815 }
 816 #endif
 817
 818 void kvm_flush_coalesced_mmio_buffer(void)
 819 {
 820 #ifdef KVM_CAP_COALESCED_MMIO
 821     KVMState *s = kvm_state;
 822     if (s->coalesced_mmio_ring) {
 823         struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
 824         while (ring->first != ring->last) {
 825             struct kvm_coalesced_mmio *ent;
 826
 827             ent = &ring->coalesced_mmio[ring->first];
 828
 829             cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
 830             smp_wmb();
 831             ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
 832         }
 833     }
 834 #endif
 835 }
 836
 837 #ifdef OBSOLETE_KVM_IMPL
 838
 839 static void do_kvm_cpu_synchronize_state(void *_env)
 840 {
 841     CPUState *env = _env;
 842
 843     if (!env->kvm_vcpu_dirty) {
 844         kvm_arch_get_registers(env);
 845         env->kvm_vcpu_dirty = 1;
 846     }
 847 }
 848
 849 void kvm_cpu_synchronize_state(CPUState *env)
 850 {
 851     if (!env->kvm_vcpu_dirty)
 852         run_on_cpu(env, do_kvm_cpu_synchronize_state, env);
 853 }
 854
 855 void kvm_cpu_synchronize_post_reset(CPUState *env)
 856 {
 857     kvm_arch_put_registers(env, KVM_PUT_RESET_STATE);
 858     env->kvm_vcpu_dirty = 0;
 859 }
 860
 861 void kvm_cpu_synchronize_post_init(CPUState *env)
 862 {
 863     kvm_arch_put_registers(env, KVM_PUT_FULL_STATE);
 864     env->kvm_vcpu_dirty = 0;
 865 }
 866
 867 int kvm_cpu_exec(CPUState *env)
 868 {
 869     struct kvm_run *run = env->kvm_run;
 870     int ret;
 871
 872     DPRINTF("kvm_cpu_exec()\n");
 873
 874     do {
 875 #ifndef CONFIG_IOTHREAD
 876         if (env->exit_request) {
 877             DPRINTF("interrupt exit requested\n");
 878             ret = 0;
 879             break;
 880         }
 881 #endif
 882
 883         if (kvm_arch_process_irqchip_events(env)) {
 884             ret = 0;
 885             break;
 886         }
 887
 888         if (env->kvm_vcpu_dirty) {
 889             kvm_arch_put_registers(env, KVM_PUT_RUNTIME_STATE);
 890             env->kvm_vcpu_dirty = 0;
 891         }
 892
 893         kvm_arch_pre_run(env, run);
 894         cpu_single_env = NULL;
 895         qemu_mutex_unlock_iothread();
 896         ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);
 897         qemu_mutex_lock_iothread();
 898         cpu_single_env = env;
 899         kvm_arch_post_run(env, run);
 900
 901         if (ret == -EINTR || ret == -EAGAIN) {
 902             cpu_exit(env);
 903             DPRINTF("io window exit\n");
 904             ret = 0;
 905             break;
 906         }
 907
 908         if (ret < 0) {
 909             DPRINTF("kvm run failed %s\n", strerror(-ret));
 910             abort();
 911         }
 912
 913         kvm_flush_coalesced_mmio_buffer();
 914
 915         ret = 0; /* exit loop */
 916         switch (run->exit_reason) {
 917         case KVM_EXIT_IO:
 918             DPRINTF("handle_io\n");
 919             ret = kvm_handle_io(run->io.port,
 920                                 (uint8_t *)run + run->io.data_offset,
 921                                 run->io.direction,
 922                                 run->io.size,
 923                                 run->io.count);
 924             break;
 925         case KVM_EXIT_MMIO:
 926             DPRINTF("handle_mmio\n");
 927             cpu_physical_memory_rw(run->mmio.phys_addr,
 928                                    run->mmio.data,
 929                                    run->mmio.len,
 930                                    run->mmio.is_write);
 931             ret = 1;
 932             break;
 933         case KVM_EXIT_IRQ_WINDOW_OPEN:
 934             DPRINTF("irq_window_open\n");
 935             break;
 936         case KVM_EXIT_SHUTDOWN:
 937             DPRINTF("shutdown\n");
 938             qemu_system_reset_request();
 939             ret = 1;
 940             break;
 941         case KVM_EXIT_UNKNOWN:
 942             DPRINTF("kvm_exit_unknown\n");
 943             break;
 944         case KVM_EXIT_FAIL_ENTRY:
 945             DPRINTF("kvm_exit_fail_entry\n");
 946             break;
 947         case KVM_EXIT_EXCEPTION:
 948             DPRINTF("kvm_exit_exception\n");
 949             break;
 950 #ifdef KVM_CAP_INTERNAL_ERROR_DATA
 951         case KVM_EXIT_INTERNAL_ERROR:
 952             kvm_handle_internal_error(env, run);
 953             break;
 954 #endif
 955         case KVM_EXIT_DEBUG:
 956             DPRINTF("kvm_exit_debug\n");
 957 #ifdef KVM_CAP_SET_GUEST_DEBUG
 958             if (kvm_arch_debug(&run->debug.arch)) {
 959                 env->exception_index = EXCP_DEBUG;
 960                 return 0;
 961             }
 962             /* re-enter, this exception was guest-internal */
 963             ret = 1;
 964 #endif /* KVM_CAP_SET_GUEST_DEBUG */
 965             break;
 966         default:
 967             DPRINTF("kvm_arch_handle_exit\n");
 968             ret = kvm_arch_handle_exit(env, run);
 969             break;
 970         }
 971     } while (ret > 0);
 972
 973     if (env->exit_request) {
 974         env->exit_request = 0;
 975         env->exception_index = EXCP_INTERRUPT;
 976     }
 977
 978     return ret;
 979 }
 980
 981 #endif
 982 int kvm_ioctl(KVMState *s, int type, ...)
 983 {
 984     int ret;
 985     void *arg;
 986     va_list ap;
 987
 988     va_start(ap, type);
 989     arg = va_arg(ap, void *);
 990     va_end(ap);
 991
 992     ret = ioctl(s->fd, type, arg);
 993     if (ret == -1)
 994         ret = -errno;
 995
 996     return ret;
 997 }
 998
 999 int kvm_vm_ioctl(KVMState *s, int type, ...)
1000 {
1001     int ret;
1002     void *arg;
1003     va_list ap;
1004
1005     va_start(ap, type);
1006     arg = va_arg(ap, void *);
1007     va_end(ap);
1008
1009     ret = ioctl(s->vmfd, type, arg);
1010     if (ret == -1)
1011         ret = -errno;
1012
1013     return ret;
1014 }
1015
1016 int kvm_vcpu_ioctl(CPUState *env, int type, ...)
1017 {
1018     int ret;
1019     void *arg;
1020     va_list ap;
1021
1022     va_start(ap, type);
1023     arg = va_arg(ap, void *);
1024     va_end(ap);
1025
1026     ret = ioctl(env->kvm_fd, type, arg);
1027     if (ret == -1)
1028         ret = -errno;
1029
1030     return ret;
1031 }
1032
1033 int kvm_has_sync_mmu(void)
1034 {
1035 #ifdef KVM_CAP_SYNC_MMU
1036     KVMState *s = kvm_state;
1037
1038     return kvm_check_extension(s, KVM_CAP_SYNC_MMU);
1039 #else
1040     return 0;
1041 #endif
1042 }
1043
1044 int kvm_has_vcpu_events(void)
1045 {
1046     return kvm_state->vcpu_events;
1047 }
1048
1049 int kvm_has_robust_singlestep(void)
1050 {
1051     return kvm_state->robust_singlestep;
1052 }
1053
1054 int kvm_has_debugregs(void)
1055 {
1056     return kvm_state->debugregs;
1057 }
1058
1059 int kvm_has_xsave(void)
1060 {
1061     return kvm_state->xsave;
1062 }
1063
1064 int kvm_has_xcrs(void)
1065 {
1066     return kvm_state->xcrs;
1067 }
1068
1069 void kvm_setup_guest_memory(void *start, size_t size)
1070 {
1071     if (!kvm_has_sync_mmu()) {
1072         int ret = qemu_madvise(start, size, QEMU_MADV_DONTFORK);
1073
1074         if (ret) {
1075             perror("qemu_madvise");
1076             fprintf(stderr,
1077                     "Need MADV_DONTFORK in absence of synchronous KVM MMU\n");
1078             exit(1);
1079         }
1080     }
1081 }
1082
1083 #ifdef KVM_CAP_SET_GUEST_DEBUG
1084 #ifndef OBSOLETE_KVM_IMPL
1085 #define run_on_cpu on_vcpu
1086 #endif /* !OBSOLETE_KVM_IMPL */
1087
1088 struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *env,
1089                                                  target_ulong pc)
1090 {
1091     struct kvm_sw_breakpoint *bp;
1092
1093     QTAILQ_FOREACH(bp, &env->kvm_state->kvm_sw_breakpoints, entry) {
1094         if (bp->pc == pc)
1095             return bp;
1096     }
1097     return NULL;
1098 }
1099
1100 int kvm_sw_breakpoints_active(CPUState *env)
1101 {
1102     return !QTAILQ_EMPTY(&env->kvm_state->kvm_sw_breakpoints);
1103 }
1104
1105 struct kvm_set_guest_debug_data {
1106     struct kvm_guest_debug dbg;
1107     CPUState *env;
1108     int err;
1109 };
1110
1111 static void kvm_invoke_set_guest_debug(void *data)
1112 {
1113     struct kvm_set_guest_debug_data *dbg_data = data;
1114     CPUState *env = dbg_data->env;
1115
1116     dbg_data->err = kvm_vcpu_ioctl(env, KVM_SET_GUEST_DEBUG, &dbg_data->dbg);
1117 }
1118
1119 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1120 {
1121     struct kvm_set_guest_debug_data data;
1122
1123     data.dbg.control = reinject_trap;
1124
1125     if (env->singlestep_enabled) {
1126         data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
1127     }
1128     kvm_arch_update_guest_debug(env, &data.dbg);
1129     data.env = env;
1130
1131     run_on_cpu(env, kvm_invoke_set_guest_debug, &data);
1132     return data.err;
1133 }
1134
1135 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1136                           target_ulong len, int type)
1137 {
1138     struct kvm_sw_breakpoint *bp;
1139     CPUState *env;
1140     int err;
1141
1142     if (type == GDB_BREAKPOINT_SW) {
1143         bp = kvm_find_sw_breakpoint(current_env, addr);
1144         if (bp) {
1145             bp->use_count++;
1146             return 0;
1147         }
1148
1149         bp = qemu_malloc(sizeof(struct kvm_sw_breakpoint));
1150         if (!bp)
1151             return -ENOMEM;
1152
1153         bp->pc = addr;
1154         bp->use_count = 1;
1155         err = kvm_arch_insert_sw_breakpoint(current_env, bp);
1156         if (err) {
1157             free(bp);
1158             return err;
1159         }
1160
1161         QTAILQ_INSERT_HEAD(&current_env->kvm_state->kvm_sw_breakpoints,
1162                           bp, entry);
1163     } else {
1164         err = kvm_arch_insert_hw_breakpoint(addr, len, type);
1165         if (err)
1166             return err;
1167     }
1168
1169     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1170         err = kvm_update_guest_debug(env, 0);
1171         if (err)
1172             return err;
1173     }
1174     return 0;
1175 }
1176
1177 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1178                           target_ulong len, int type)
1179 {
1180     struct kvm_sw_breakpoint *bp;
1181     CPUState *env;
1182     int err;
1183
1184     if (type == GDB_BREAKPOINT_SW) {
1185         bp = kvm_find_sw_breakpoint(current_env, addr);
1186         if (!bp)
1187             return -ENOENT;
1188
1189         if (bp->use_count > 1) {
1190             bp->use_count--;
1191             return 0;
1192         }
1193
1194         err = kvm_arch_remove_sw_breakpoint(current_env, bp);
1195         if (err)
1196             return err;
1197
1198         QTAILQ_REMOVE(&current_env->kvm_state->kvm_sw_breakpoints, bp, entry);
1199         qemu_free(bp);
1200     } else {
1201         err = kvm_arch_remove_hw_breakpoint(addr, len, type);
1202         if (err)
1203             return err;
1204     }
1205
1206     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1207         err = kvm_update_guest_debug(env, 0);
1208         if (err)
1209             return err;
1210     }
1211     return 0;
1212 }
1213
1214 void kvm_remove_all_breakpoints(CPUState *current_env)
1215 {
1216     struct kvm_sw_breakpoint *bp, *next;
1217     KVMState *s = current_env->kvm_state;
1218     CPUState *env;
1219
1220     QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
1221         if (kvm_arch_remove_sw_breakpoint(current_env, bp) != 0) {
1222             /* Try harder to find a CPU that currently sees the breakpoint. */
1223             for (env = first_cpu; env != NULL; env = env->next_cpu) {
1224                 if (kvm_arch_remove_sw_breakpoint(env, bp) == 0)
1225                     break;
1226             }
1227         }
1228     }
1229     kvm_arch_remove_all_hw_breakpoints();
1230
1231     for (env = first_cpu; env != NULL; env = env->next_cpu)
1232         kvm_update_guest_debug(env, 0);
1233 }
1234
1235 #else /* !KVM_CAP_SET_GUEST_DEBUG */
1236
1237 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1238 {
1239     return -EINVAL;
1240 }
1241
1242 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1243                           target_ulong len, int type)
1244 {
1245     return -EINVAL;
1246 }
1247
1248 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1249                           target_ulong len, int type)
1250 {
1251     return -EINVAL;
1252 }
1253
1254 void kvm_remove_all_breakpoints(CPUState *current_env)
1255 {
1256 }
1257 #endif /* !KVM_CAP_SET_GUEST_DEBUG */
1258
1259 int kvm_set_signal_mask(CPUState *env, const sigset_t *sigset)
1260 {
1261     struct kvm_signal_mask *sigmask;
1262     int r;
1263
1264     if (!sigset)
1265         return kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, NULL);
1266
1267     sigmask = qemu_malloc(sizeof(*sigmask) + sizeof(*sigset));
1268
1269     sigmask->len = 8;
1270     memcpy(sigmask->sigset, sigset, sizeof(*sigset));
1271     r = kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, sigmask);
1272     free(sigmask);
1273
1274     return r;
1275 }
1276
1277 int kvm_set_ioeventfd_mmio_long(int fd, uint32_t addr, uint32_t val, bool assign)
1278 {
1279 #ifdef KVM_IOEVENTFD
1280     int ret;
1281     struct kvm_ioeventfd iofd;
1282
1283     iofd.datamatch = val;
1284     iofd.addr = addr;
1285     iofd.len = 4;
1286     iofd.flags = KVM_IOEVENTFD_FLAG_DATAMATCH;
1287     iofd.fd = fd;
1288
1289     if (!kvm_enabled()) {
1290         return -ENOSYS;
1291     }
1292
1293     if (!assign) {
1294         iofd.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
1295     }
1296
1297     ret = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &iofd);
1298
1299     if (ret < 0) {
1300         return -errno;
1301     }
1302
1303     return 0;
1304 #else
1305     return -ENOSYS;
1306 #endif
1307 }
1308
1309 int kvm_set_ioeventfd_pio_word(int fd, uint16_t addr, uint16_t val, bool assign)
1310 {
1311 #ifdef KVM_IOEVENTFD
1312     struct kvm_ioeventfd kick = {
1313         .datamatch = val,
1314         .addr = addr,
1315         .len = 2,
1316         .flags = KVM_IOEVENTFD_FLAG_DATAMATCH | KVM_IOEVENTFD_FLAG_PIO,
1317         .fd = fd,
1318     };
1319     int r;
1320     if (!kvm_enabled())
1321         return -ENOSYS;
1322     if (!assign)
1323         kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
1324     r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
1325     if (r < 0)
1326         return r;
1327     return 0;
1328 #else
1329     return -ENOSYS;
1330 #endif
1331 }
1332
1333 #if defined(KVM_IRQFD)
1334 int kvm_set_irqfd(int gsi, int fd, bool assigned)
1335 {
1336     struct kvm_irqfd irqfd = {
1337         .fd = fd,
1338         .gsi = gsi,
1339         .flags = assigned ? 0 : KVM_IRQFD_FLAG_DEASSIGN,
1340     };
1341     int r;
1342     if (!kvm_enabled() || !kvm_irqchip_in_kernel())
1343         return -ENOSYS;
1344
1345     r = kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd);
1346     if (r < 0)
1347         return r;
1348     return 0;
1349 }
1350 #endif
1351
1352 #undef PAGE_SIZE
1353 #include "qemu-kvm.c"