kvm-all.c

   1 /*
   2  * QEMU KVM support
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *           Red Hat, Inc. 2008
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *  Glauber Costa     <gcosta@redhat.com>
  10  *
  11  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  12  * See the COPYING file in the top-level directory.
  13  *
  14  */
  15
  16 #include <sys/types.h>
  17 #include <sys/ioctl.h>
  18 #include <sys/mman.h>
  19 #include <stdarg.h>
  20
  21 #include <linux/kvm.h>
  22
  23 #include "qemu-common.h"
  24 #include "qemu-barrier.h"
  25 #include "qemu-option.h"
  26 #include "qemu-config.h"
  27 #include "sysemu.h"
  28 #include "hw/hw.h"
  29 #include "hw/msi.h"
  30 #include "gdbstub.h"
  31 #include "kvm.h"
  32 #include "bswap.h"
  33 #include "memory.h"
  34 #include "exec-memory.h"
  35 #include "event_notifier.h"
  36
  37 /* This check must be after config-host.h is included */
  38 #ifdef CONFIG_EVENTFD
  39 #include <sys/eventfd.h>
  40 #endif
  41
  42 #ifdef CONFIG_VALGRIND_H
  43 #include <valgrind/memcheck.h>
  44 #endif
  45
  46 /* KVM uses PAGE_SIZE in its definition of COALESCED_MMIO_MAX */
  47 #define PAGE_SIZE TARGET_PAGE_SIZE
  48
  49 //#define DEBUG_KVM
  50
  51 #ifdef DEBUG_KVM
  52 #define DPRINTF(fmt, ...) \
  53     do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
  54 #else
  55 #define DPRINTF(fmt, ...) \
  56     do { } while (0)
  57 #endif
  58
  59 #define KVM_MSI_HASHTAB_SIZE    256
  60
  61 typedef struct KVMSlot
  62 {
  63     target_phys_addr_t start_addr;
  64     ram_addr_t memory_size;
  65     void *ram;
  66     int slot;
  67     int flags;
  68 } KVMSlot;
  69
  70 typedef struct kvm_dirty_log KVMDirtyLog;
  71
  72 struct KVMState
  73 {
  74     KVMSlot slots[32];
  75     int fd;
  76     int vmfd;
  77     int coalesced_mmio;
  78     struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
  79     bool coalesced_flush_in_progress;
  80     int broken_set_mem_region;
  81     int migration_log;
  82     int vcpu_events;
  83     int robust_singlestep;
  84     int debugregs;
  85 #ifdef KVM_CAP_SET_GUEST_DEBUG
  86     struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
  87 #endif
  88     int pit_state2;
  89     int xsave, xcrs;
  90     int many_ioeventfds;
  91     int intx_set_mask;
  92     /* The man page (and posix) say ioctl numbers are signed int, but
  93      * they're not.  Linux, glibc and *BSD all treat ioctl numbers as
  94      * unsigned, and treating them as signed here can break things */
  95     unsigned irq_set_ioctl;
  96 #ifdef KVM_CAP_IRQ_ROUTING
  97     struct kvm_irq_routing *irq_routes;
  98     int nr_allocated_irq_routes;
  99     uint32_t *used_gsi_bitmap;
 100     unsigned int gsi_count;
 101     QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
 102     bool direct_msi;
 103 #endif
 104 };
 105
 106 KVMState *kvm_state;
 107 bool kvm_kernel_irqchip;
 108 bool kvm_async_interrupts_allowed;
 109 bool kvm_irqfds_allowed;
 110 bool kvm_msi_via_irqfd_allowed;
 111 bool kvm_gsi_routing_allowed;
 112
 113 static const KVMCapabilityInfo kvm_required_capabilites[] = {
 114     KVM_CAP_INFO(USER_MEMORY),
 115     KVM_CAP_INFO(DESTROY_MEMORY_REGION_WORKS),
 116     KVM_CAP_LAST_INFO
 117 };
 118
 119 static KVMSlot *kvm_alloc_slot(KVMState *s)
 120 {
 121     int i;
 122
 123     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 124         if (s->slots[i].memory_size == 0) {
 125             return &s->slots[i];
 126         }
 127     }
 128
 129     fprintf(stderr, "%s: no free slot available\n", __func__);
 130     abort();
 131 }
 132
 133 static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
 134                                          target_phys_addr_t start_addr,
 135                                          target_phys_addr_t end_addr)
 136 {
 137     int i;
 138
 139     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 140         KVMSlot *mem = &s->slots[i];
 141
 142         if (start_addr == mem->start_addr &&
 143             end_addr == mem->start_addr + mem->memory_size) {
 144             return mem;
 145         }
 146     }
 147
 148     return NULL;
 149 }
 150
 151 /*
 152  * Find overlapping slot with lowest start address
 153  */
 154 static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
 155                                             target_phys_addr_t start_addr,
 156                                             target_phys_addr_t end_addr)
 157 {
 158     KVMSlot *found = NULL;
 159     int i;
 160
 161     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 162         KVMSlot *mem = &s->slots[i];
 163
 164         if (mem->memory_size == 0 ||
 165             (found && found->start_addr < mem->start_addr)) {
 166             continue;
 167         }
 168
 169         if (end_addr > mem->start_addr &&
 170             start_addr < mem->start_addr + mem->memory_size) {
 171             found = mem;
 172         }
 173     }
 174
 175     return found;
 176 }
 177
 178 int kvm_physical_memory_addr_from_host(KVMState *s, void *ram,
 179                                        target_phys_addr_t *phys_addr)
 180 {
 181     int i;
 182
 183     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 184         KVMSlot *mem = &s->slots[i];
 185
 186         if (ram >= mem->ram && ram < mem->ram + mem->memory_size) {
 187             *phys_addr = mem->start_addr + (ram - mem->ram);
 188             return 1;
 189         }
 190     }
 191
 192     return 0;
 193 }
 194
 195 static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
 196 {
 197     struct kvm_userspace_memory_region mem;
 198
 199     mem.slot = slot->slot;
 200     mem.guest_phys_addr = slot->start_addr;
 201     mem.memory_size = slot->memory_size;
 202     mem.userspace_addr = (unsigned long)slot->ram;
 203     mem.flags = slot->flags;
 204     if (s->migration_log) {
 205         mem.flags |= KVM_MEM_LOG_DIRTY_PAGES;
 206     }
 207     return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
 208 }
 209
 210 static void kvm_reset_vcpu(void *opaque)
 211 {
 212     CPUArchState *env = opaque;
 213
 214     kvm_arch_reset_vcpu(env);
 215 }
 216
 217 int kvm_init_vcpu(CPUArchState *env)
 218 {
 219     KVMState *s = kvm_state;
 220     long mmap_size;
 221     int ret;
 222
 223     DPRINTF("kvm_init_vcpu\n");
 224
 225     ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, env->cpu_index);
 226     if (ret < 0) {
 227         DPRINTF("kvm_create_vcpu failed\n");
 228         goto err;
 229     }
 230
 231     env->kvm_fd = ret;
 232     env->kvm_state = s;
 233     env->kvm_vcpu_dirty = 1;
 234
 235     mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
 236     if (mmap_size < 0) {
 237         ret = mmap_size;
 238         DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
 239         goto err;
 240     }
 241
 242     env->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
 243                         env->kvm_fd, 0);
 244     if (env->kvm_run == MAP_FAILED) {
 245         ret = -errno;
 246         DPRINTF("mmap'ing vcpu state failed\n");
 247         goto err;
 248     }
 249
 250     if (s->coalesced_mmio && !s->coalesced_mmio_ring) {
 251         s->coalesced_mmio_ring =
 252             (void *)env->kvm_run + s->coalesced_mmio * PAGE_SIZE;
 253     }
 254
 255     ret = kvm_arch_init_vcpu(env);
 256     if (ret == 0) {
 257         qemu_register_reset(kvm_reset_vcpu, env);
 258         kvm_arch_reset_vcpu(env);
 259     }
 260 err:
 261     return ret;
 262 }
 263
 264 /*
 265  * dirty pages logging control
 266  */
 267
 268 static int kvm_mem_flags(KVMState *s, bool log_dirty)
 269 {
 270     return log_dirty ? KVM_MEM_LOG_DIRTY_PAGES : 0;
 271 }
 272
 273 static int kvm_slot_dirty_pages_log_change(KVMSlot *mem, bool log_dirty)
 274 {
 275     KVMState *s = kvm_state;
 276     int flags, mask = KVM_MEM_LOG_DIRTY_PAGES;
 277     int old_flags;
 278
 279     old_flags = mem->flags;
 280
 281     flags = (mem->flags & ~mask) | kvm_mem_flags(s, log_dirty);
 282     mem->flags = flags;
 283
 284     /* If nothing changed effectively, no need to issue ioctl */
 285     if (s->migration_log) {
 286         flags |= KVM_MEM_LOG_DIRTY_PAGES;
 287     }
 288
 289     if (flags == old_flags) {
 290         return 0;
 291     }
 292
 293     return kvm_set_user_memory_region(s, mem);
 294 }
 295
 296 static int kvm_dirty_pages_log_change(target_phys_addr_t phys_addr,
 297                                       ram_addr_t size, bool log_dirty)
 298 {
 299     KVMState *s = kvm_state;
 300     KVMSlot *mem = kvm_lookup_matching_slot(s, phys_addr, phys_addr + size);
 301
 302     if (mem == NULL)  {
 303         fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
 304                 TARGET_FMT_plx "\n", __func__, phys_addr,
 305                 (target_phys_addr_t)(phys_addr + size - 1));
 306         return -EINVAL;
 307     }
 308     return kvm_slot_dirty_pages_log_change(mem, log_dirty);
 309 }
 310
 311 static void kvm_log_start(MemoryListener *listener,
 312                           MemoryRegionSection *section)
 313 {
 314     int r;
 315
 316     r = kvm_dirty_pages_log_change(section->offset_within_address_space,
 317                                    section->size, true);
 318     if (r < 0) {
 319         abort();
 320     }
 321 }
 322
 323 static void kvm_log_stop(MemoryListener *listener,
 324                           MemoryRegionSection *section)
 325 {
 326     int r;
 327
 328     r = kvm_dirty_pages_log_change(section->offset_within_address_space,
 329                                    section->size, false);
 330     if (r < 0) {
 331         abort();
 332     }
 333 }
 334
 335 static int kvm_set_migration_log(int enable)
 336 {
 337     KVMState *s = kvm_state;
 338     KVMSlot *mem;
 339     int i, err;
 340
 341     s->migration_log = enable;
 342
 343     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 344         mem = &s->slots[i];
 345
 346         if (!mem->memory_size) {
 347             continue;
 348         }
 349         if (!!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) == enable) {
 350             continue;
 351         }
 352         err = kvm_set_user_memory_region(s, mem);
 353         if (err) {
 354             return err;
 355         }
 356     }
 357     return 0;
 358 }
 359
 360 /* get kvm's dirty pages bitmap and update qemu's */
 361 static int kvm_get_dirty_pages_log_range(MemoryRegionSection *section,
 362                                          unsigned long *bitmap)
 363 {
 364     unsigned int i, j;
 365     unsigned long page_number, c;
 366     target_phys_addr_t addr, addr1;
 367     unsigned int len = ((section->size / TARGET_PAGE_SIZE) + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
 368     unsigned long hpratio = getpagesize() / TARGET_PAGE_SIZE;
 369
 370     /*
 371      * bitmap-traveling is faster than memory-traveling (for addr...)
 372      * especially when most of the memory is not dirty.
 373      */
 374     for (i = 0; i < len; i++) {
 375         if (bitmap[i] != 0) {
 376             c = leul_to_cpu(bitmap[i]);
 377             do {
 378                 j = ffsl(c) - 1;
 379                 c &= ~(1ul << j);
 380                 page_number = (i * HOST_LONG_BITS + j) * hpratio;
 381                 addr1 = page_number * TARGET_PAGE_SIZE;
 382                 addr = section->offset_within_region + addr1;
 383                 memory_region_set_dirty(section->mr, addr,
 384                                         TARGET_PAGE_SIZE * hpratio);
 385             } while (c != 0);
 386         }
 387     }
 388     return 0;
 389 }
 390
 391 #define ALIGN(x, y)  (((x)+(y)-1) & ~((y)-1))
 392
 393 /**
 394  * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
 395  * This function updates qemu's dirty bitmap using
 396  * memory_region_set_dirty().  This means all bits are set
 397  * to dirty.
 398  *
 399  * @start_add: start of logged region.
 400  * @end_addr: end of logged region.
 401  */
 402 static int kvm_physical_sync_dirty_bitmap(MemoryRegionSection *section)
 403 {
 404     KVMState *s = kvm_state;
 405     unsigned long size, allocated_size = 0;
 406     KVMDirtyLog d;
 407     KVMSlot *mem;
 408     int ret = 0;
 409     target_phys_addr_t start_addr = section->offset_within_address_space;
 410     target_phys_addr_t end_addr = start_addr + section->size;
 411
 412     d.dirty_bitmap = NULL;
 413     while (start_addr < end_addr) {
 414         mem = kvm_lookup_overlapping_slot(s, start_addr, end_addr);
 415         if (mem == NULL) {
 416             break;
 417         }
 418
 419         /* XXX bad kernel interface alert
 420          * For dirty bitmap, kernel allocates array of size aligned to
 421          * bits-per-long.  But for case when the kernel is 64bits and
 422          * the userspace is 32bits, userspace can't align to the same
 423          * bits-per-long, since sizeof(long) is different between kernel
 424          * and user space.  This way, userspace will provide buffer which
 425          * may be 4 bytes less than the kernel will use, resulting in
 426          * userspace memory corruption (which is not detectable by valgrind
 427          * too, in most cases).
 428          * So for now, let's align to 64 instead of HOST_LONG_BITS here, in
 429          * a hope that sizeof(long) wont become >8 any time soon.
 430          */
 431         size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS),
 432                      /*HOST_LONG_BITS*/ 64) / 8;
 433         if (!d.dirty_bitmap) {
 434             d.dirty_bitmap = g_malloc(size);
 435         } else if (size > allocated_size) {
 436             d.dirty_bitmap = g_realloc(d.dirty_bitmap, size);
 437         }
 438         allocated_size = size;
 439         memset(d.dirty_bitmap, 0, allocated_size);
 440
 441         d.slot = mem->slot;
 442
 443         if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
 444             DPRINTF("ioctl failed %d\n", errno);
 445             ret = -1;
 446             break;
 447         }
 448
 449         kvm_get_dirty_pages_log_range(section, d.dirty_bitmap);
 450         start_addr = mem->start_addr + mem->memory_size;
 451     }
 452     g_free(d.dirty_bitmap);
 453
 454     return ret;
 455 }
 456
 457 int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
 458 {
 459     int ret = -ENOSYS;
 460     KVMState *s = kvm_state;
 461
 462     if (s->coalesced_mmio) {
 463         struct kvm_coalesced_mmio_zone zone;
 464
 465         zone.addr = start;
 466         zone.size = size;
 467         zone.pad = 0;
 468
 469         ret = kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
 470     }
 471
 472     return ret;
 473 }
 474
 475 int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
 476 {
 477     int ret = -ENOSYS;
 478     KVMState *s = kvm_state;
 479
 480     if (s->coalesced_mmio) {
 481         struct kvm_coalesced_mmio_zone zone;
 482
 483         zone.addr = start;
 484         zone.size = size;
 485         zone.pad = 0;
 486
 487         ret = kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
 488     }
 489
 490     return ret;
 491 }
 492
 493 int kvm_check_extension(KVMState *s, unsigned int extension)
 494 {
 495     int ret;
 496
 497     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
 498     if (ret < 0) {
 499         ret = 0;
 500     }
 501
 502     return ret;
 503 }
 504
 505 static int kvm_check_many_ioeventfds(void)
 506 {
 507     /* Userspace can use ioeventfd for io notification.  This requires a host
 508      * that supports eventfd(2) and an I/O thread; since eventfd does not
 509      * support SIGIO it cannot interrupt the vcpu.
 510      *
 511      * Older kernels have a 6 device limit on the KVM io bus.  Find out so we
 512      * can avoid creating too many ioeventfds.
 513      */
 514 #if defined(CONFIG_EVENTFD)
 515     int ioeventfds[7];
 516     int i, ret = 0;
 517     for (i = 0; i < ARRAY_SIZE(ioeventfds); i++) {
 518         ioeventfds[i] = eventfd(0, EFD_CLOEXEC);
 519         if (ioeventfds[i] < 0) {
 520             break;
 521         }
 522         ret = kvm_set_ioeventfd_pio_word(ioeventfds[i], 0, i, true);
 523         if (ret < 0) {
 524             close(ioeventfds[i]);
 525             break;
 526         }
 527     }
 528
 529     /* Decide whether many devices are supported or not */
 530     ret = i == ARRAY_SIZE(ioeventfds);
 531
 532     while (i-- > 0) {
 533         kvm_set_ioeventfd_pio_word(ioeventfds[i], 0, i, false);
 534         close(ioeventfds[i]);
 535     }
 536     return ret;
 537 #else
 538     return 0;
 539 #endif
 540 }
 541
 542 static const KVMCapabilityInfo *
 543 kvm_check_extension_list(KVMState *s, const KVMCapabilityInfo *list)
 544 {
 545     while (list->name) {
 546         if (!kvm_check_extension(s, list->value)) {
 547             return list;
 548         }
 549         list++;
 550     }
 551     return NULL;
 552 }
 553
 554 static void kvm_set_phys_mem(MemoryRegionSection *section, bool add)
 555 {
 556     KVMState *s = kvm_state;
 557     KVMSlot *mem, old;
 558     int err;
 559     MemoryRegion *mr = section->mr;
 560     bool log_dirty = memory_region_is_logging(mr);
 561     target_phys_addr_t start_addr = section->offset_within_address_space;
 562     ram_addr_t size = section->size;
 563     void *ram = NULL;
 564     unsigned delta;
 565
 566     /* kvm works in page size chunks, but the function may be called
 567        with sub-page size and unaligned start address. */
 568     delta = TARGET_PAGE_ALIGN(size) - size;
 569     if (delta > size) {
 570         return;
 571     }
 572     start_addr += delta;
 573     size -= delta;
 574     size &= TARGET_PAGE_MASK;
 575     if (!size || (start_addr & ~TARGET_PAGE_MASK)) {
 576         return;
 577     }
 578
 579     if (!memory_region_is_ram(mr)) {
 580         return;
 581     }
 582
 583     ram = memory_region_get_ram_ptr(mr) + section->offset_within_region + delta;
 584
 585     while (1) {
 586         mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size);
 587         if (!mem) {
 588             break;
 589         }
 590
 591         if (add && start_addr >= mem->start_addr &&
 592             (start_addr + size <= mem->start_addr + mem->memory_size) &&
 593             (ram - start_addr == mem->ram - mem->start_addr)) {
 594             /* The new slot fits into the existing one and comes with
 595              * identical parameters - update flags and done. */
 596             kvm_slot_dirty_pages_log_change(mem, log_dirty);
 597             return;
 598         }
 599
 600         old = *mem;
 601
 602         if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) {
 603             kvm_physical_sync_dirty_bitmap(section);
 604         }
 605
 606         /* unregister the overlapping slot */
 607         mem->memory_size = 0;
 608         err = kvm_set_user_memory_region(s, mem);
 609         if (err) {
 610             fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
 611                     __func__, strerror(-err));
 612             abort();
 613         }
 614
 615         /* Workaround for older KVM versions: we can't join slots, even not by
 616          * unregistering the previous ones and then registering the larger
 617          * slot. We have to maintain the existing fragmentation. Sigh.
 618          *
 619          * This workaround assumes that the new slot starts at the same
 620          * address as the first existing one. If not or if some overlapping
 621          * slot comes around later, we will fail (not seen in practice so far)
 622          * - and actually require a recent KVM version. */
 623         if (s->broken_set_mem_region &&
 624             old.start_addr == start_addr && old.memory_size < size && add) {
 625             mem = kvm_alloc_slot(s);
 626             mem->memory_size = old.memory_size;
 627             mem->start_addr = old.start_addr;
 628             mem->ram = old.ram;
 629             mem->flags = kvm_mem_flags(s, log_dirty);
 630
 631             err = kvm_set_user_memory_region(s, mem);
 632             if (err) {
 633                 fprintf(stderr, "%s: error updating slot: %s\n", __func__,
 634                         strerror(-err));
 635                 abort();
 636             }
 637
 638             start_addr += old.memory_size;
 639             ram += old.memory_size;
 640             size -= old.memory_size;
 641             continue;
 642         }
 643
 644         /* register prefix slot */
 645         if (old.start_addr < start_addr) {
 646             mem = kvm_alloc_slot(s);
 647             mem->memory_size = start_addr - old.start_addr;
 648             mem->start_addr = old.start_addr;
 649             mem->ram = old.ram;
 650             mem->flags =  kvm_mem_flags(s, log_dirty);
 651
 652             err = kvm_set_user_memory_region(s, mem);
 653             if (err) {
 654                 fprintf(stderr, "%s: error registering prefix slot: %s\n",
 655                         __func__, strerror(-err));
 656 #ifdef TARGET_PPC
 657                 fprintf(stderr, "%s: This is probably because your kernel's " \
 658                                 "PAGE_SIZE is too big. Please try to use 4k " \
 659                                 "PAGE_SIZE!\n", __func__);
 660 #endif
 661                 abort();
 662             }
 663         }
 664
 665         /* register suffix slot */
 666         if (old.start_addr + old.memory_size > start_addr + size) {
 667             ram_addr_t size_delta;
 668
 669             mem = kvm_alloc_slot(s);
 670             mem->start_addr = start_addr + size;
 671             size_delta = mem->start_addr - old.start_addr;
 672             mem->memory_size = old.memory_size - size_delta;
 673             mem->ram = old.ram + size_delta;
 674             mem->flags = kvm_mem_flags(s, log_dirty);
 675
 676             err = kvm_set_user_memory_region(s, mem);
 677             if (err) {
 678                 fprintf(stderr, "%s: error registering suffix slot: %s\n",
 679                         __func__, strerror(-err));
 680                 abort();
 681             }
 682         }
 683     }
 684
 685     /* in case the KVM bug workaround already "consumed" the new slot */
 686     if (!size) {
 687         return;
 688     }
 689     if (!add) {
 690         return;
 691     }
 692     mem = kvm_alloc_slot(s);
 693     mem->memory_size = size;
 694     mem->start_addr = start_addr;
 695     mem->ram = ram;
 696     mem->flags = kvm_mem_flags(s, log_dirty);
 697
 698     err = kvm_set_user_memory_region(s, mem);
 699     if (err) {
 700         fprintf(stderr, "%s: error registering slot: %s\n", __func__,
 701                 strerror(-err));
 702         abort();
 703     }
 704 }
 705
 706 static void kvm_begin(MemoryListener *listener)
 707 {
 708 }
 709
 710 static void kvm_commit(MemoryListener *listener)
 711 {
 712 }
 713
 714 static void kvm_region_add(MemoryListener *listener,
 715                            MemoryRegionSection *section)
 716 {
 717     kvm_set_phys_mem(section, true);
 718 }
 719
 720 static void kvm_region_del(MemoryListener *listener,
 721                            MemoryRegionSection *section)
 722 {
 723     kvm_set_phys_mem(section, false);
 724 }
 725
 726 static void kvm_region_nop(MemoryListener *listener,
 727                            MemoryRegionSection *section)
 728 {
 729 }
 730
 731 static void kvm_log_sync(MemoryListener *listener,
 732                          MemoryRegionSection *section)
 733 {
 734     int r;
 735
 736     r = kvm_physical_sync_dirty_bitmap(section);
 737     if (r < 0) {
 738         abort();
 739     }
 740 }
 741
 742 static void kvm_log_global_start(struct MemoryListener *listener)
 743 {
 744     int r;
 745
 746     r = kvm_set_migration_log(1);
 747     assert(r >= 0);
 748 }
 749
 750 static void kvm_log_global_stop(struct MemoryListener *listener)
 751 {
 752     int r;
 753
 754     r = kvm_set_migration_log(0);
 755     assert(r >= 0);
 756 }
 757
 758 static void kvm_mem_ioeventfd_add(MemoryRegionSection *section,
 759                                   bool match_data, uint64_t data, int fd)
 760 {
 761     int r;
 762
 763     assert(match_data && section->size <= 8);
 764
 765     r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
 766                                data, true, section->size);
 767     if (r < 0) {
 768         abort();
 769     }
 770 }
 771
 772 static void kvm_mem_ioeventfd_del(MemoryRegionSection *section,
 773                                   bool match_data, uint64_t data, int fd)
 774 {
 775     int r;
 776
 777     r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
 778                                data, false, section->size);
 779     if (r < 0) {
 780         abort();
 781     }
 782 }
 783
 784 static void kvm_io_ioeventfd_add(MemoryRegionSection *section,
 785                                  bool match_data, uint64_t data, int fd)
 786 {
 787     int r;
 788
 789     assert(match_data && section->size == 2);
 790
 791     r = kvm_set_ioeventfd_pio_word(fd, section->offset_within_address_space,
 792                                    data, true);
 793     if (r < 0) {
 794         abort();
 795     }
 796 }
 797
 798 static void kvm_io_ioeventfd_del(MemoryRegionSection *section,
 799                                  bool match_data, uint64_t data, int fd)
 800
 801 {
 802     int r;
 803
 804     r = kvm_set_ioeventfd_pio_word(fd, section->offset_within_address_space,
 805                                    data, false);
 806     if (r < 0) {
 807         abort();
 808     }
 809 }
 810
 811 static void kvm_eventfd_add(MemoryListener *listener,
 812                             MemoryRegionSection *section,
 813                             bool match_data, uint64_t data,
 814                             EventNotifier *e)
 815 {
 816     if (section->address_space == get_system_memory()) {
 817         kvm_mem_ioeventfd_add(section, match_data, data,
 818                               event_notifier_get_fd(e));
 819     } else {
 820         kvm_io_ioeventfd_add(section, match_data, data,
 821                              event_notifier_get_fd(e));
 822     }
 823 }
 824
 825 static void kvm_eventfd_del(MemoryListener *listener,
 826                             MemoryRegionSection *section,
 827                             bool match_data, uint64_t data,
 828                             EventNotifier *e)
 829 {
 830     if (section->address_space == get_system_memory()) {
 831         kvm_mem_ioeventfd_del(section, match_data, data,
 832                               event_notifier_get_fd(e));
 833     } else {
 834         kvm_io_ioeventfd_del(section, match_data, data,
 835                              event_notifier_get_fd(e));
 836     }
 837 }
 838
 839 static MemoryListener kvm_memory_listener = {
 840     .begin = kvm_begin,
 841     .commit = kvm_commit,
 842     .region_add = kvm_region_add,
 843     .region_del = kvm_region_del,
 844     .region_nop = kvm_region_nop,
 845     .log_start = kvm_log_start,
 846     .log_stop = kvm_log_stop,
 847     .log_sync = kvm_log_sync,
 848     .log_global_start = kvm_log_global_start,
 849     .log_global_stop = kvm_log_global_stop,
 850     .eventfd_add = kvm_eventfd_add,
 851     .eventfd_del = kvm_eventfd_del,
 852     .priority = 10,
 853 };
 854
 855 static void kvm_handle_interrupt(CPUArchState *env, int mask)
 856 {
 857     env->interrupt_request |= mask;
 858
 859     if (!qemu_cpu_is_self(env)) {
 860         qemu_cpu_kick(env);
 861     }
 862 }
 863
 864 int kvm_set_irq(KVMState *s, int irq, int level)
 865 {
 866     struct kvm_irq_level event;
 867     int ret;
 868
 869     assert(kvm_async_interrupts_enabled());
 870
 871     event.level = level;
 872     event.irq = irq;
 873     ret = kvm_vm_ioctl(s, s->irq_set_ioctl, &event);
 874     if (ret < 0) {
 875         perror("kvm_set_irq");
 876         abort();
 877     }
 878
 879     return (s->irq_set_ioctl == KVM_IRQ_LINE) ? 1 : event.status;
 880 }
 881
 882 #ifdef KVM_CAP_IRQ_ROUTING
 883 typedef struct KVMMSIRoute {
 884     struct kvm_irq_routing_entry kroute;
 885     QTAILQ_ENTRY(KVMMSIRoute) entry;
 886 } KVMMSIRoute;
 887
 888 static void set_gsi(KVMState *s, unsigned int gsi)
 889 {
 890     s->used_gsi_bitmap[gsi / 32] |= 1U << (gsi % 32);
 891 }
 892
 893 static void clear_gsi(KVMState *s, unsigned int gsi)
 894 {
 895     s->used_gsi_bitmap[gsi / 32] &= ~(1U << (gsi % 32));
 896 }
 897
 898 static void kvm_init_irq_routing(KVMState *s)
 899 {
 900     int gsi_count, i;
 901
 902     gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING);
 903     if (gsi_count > 0) {
 904         unsigned int gsi_bits, i;
 905
 906         /* Round up so we can search ints using ffs */
 907         gsi_bits = ALIGN(gsi_count, 32);
 908         s->used_gsi_bitmap = g_malloc0(gsi_bits / 8);
 909         s->gsi_count = gsi_count;
 910
 911         /* Mark any over-allocated bits as already in use */
 912         for (i = gsi_count; i < gsi_bits; i++) {
 913             set_gsi(s, i);
 914         }
 915     }
 916
 917     s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
 918     s->nr_allocated_irq_routes = 0;
 919
 920     if (!s->direct_msi) {
 921         for (i = 0; i < KVM_MSI_HASHTAB_SIZE; i++) {
 922             QTAILQ_INIT(&s->msi_hashtab[i]);
 923         }
 924     }
 925
 926     kvm_arch_init_irq_routing(s);
 927 }
 928
 929 static void kvm_irqchip_commit_routes(KVMState *s)
 930 {
 931     int ret;
 932
 933     s->irq_routes->flags = 0;
 934     ret = kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes);
 935     assert(ret == 0);
 936 }
 937
 938 static void kvm_add_routing_entry(KVMState *s,
 939                                   struct kvm_irq_routing_entry *entry)
 940 {
 941     struct kvm_irq_routing_entry *new;
 942     int n, size;
 943
 944     if (s->irq_routes->nr == s->nr_allocated_irq_routes) {
 945         n = s->nr_allocated_irq_routes * 2;
 946         if (n < 64) {
 947             n = 64;
 948         }
 949         size = sizeof(struct kvm_irq_routing);
 950         size += n * sizeof(*new);
 951         s->irq_routes = g_realloc(s->irq_routes, size);
 952         s->nr_allocated_irq_routes = n;
 953     }
 954     n = s->irq_routes->nr++;
 955     new = &s->irq_routes->entries[n];
 956     memset(new, 0, sizeof(*new));
 957     new->gsi = entry->gsi;
 958     new->type = entry->type;
 959     new->flags = entry->flags;
 960     new->u = entry->u;
 961
 962     set_gsi(s, entry->gsi);
 963
 964     kvm_irqchip_commit_routes(s);
 965 }
 966
 967 static int kvm_update_routing_entry(KVMState *s,
 968                                     struct kvm_irq_routing_entry *new_entry)
 969 {
 970     struct kvm_irq_routing_entry *entry;
 971     int n;
 972
 973     for (n = 0; n < s->irq_routes->nr; n++) {
 974         entry = &s->irq_routes->entries[n];
 975         if (entry->gsi != new_entry->gsi) {
 976             continue;
 977         }
 978
 979         entry->type = new_entry->type;
 980         entry->flags = new_entry->flags;
 981         entry->u = new_entry->u;
 982
 983         kvm_irqchip_commit_routes(s);
 984
 985         return 0;
 986     }
 987
 988     return -ESRCH;
 989 }
 990
 991 void kvm_irqchip_add_irq_route(KVMState *s, int irq, int irqchip, int pin)
 992 {
 993     struct kvm_irq_routing_entry e;
 994
 995     assert(pin < s->gsi_count);
 996
 997     e.gsi = irq;
 998     e.type = KVM_IRQ_ROUTING_IRQCHIP;
 999     e.flags = 0;
1000     e.u.irqchip.irqchip = irqchip;
1001     e.u.irqchip.pin = pin;
1002     kvm_add_routing_entry(s, &e);
1003 }
1004
1005 void kvm_irqchip_release_virq(KVMState *s, int virq)
1006 {
1007     struct kvm_irq_routing_entry *e;
1008     int i;
1009
1010     for (i = 0; i < s->irq_routes->nr; i++) {
1011         e = &s->irq_routes->entries[i];
1012         if (e->gsi == virq) {
1013             s->irq_routes->nr--;
1014             *e = s->irq_routes->entries[s->irq_routes->nr];
1015         }
1016     }
1017     clear_gsi(s, virq);
1018
1019     kvm_irqchip_commit_routes(s);
1020 }
1021
1022 static unsigned int kvm_hash_msi(uint32_t data)
1023 {
1024     /* This is optimized for IA32 MSI layout. However, no other arch shall
1025      * repeat the mistake of not providing a direct MSI injection API. */
1026     return data & 0xff;
1027 }
1028
1029 static void kvm_flush_dynamic_msi_routes(KVMState *s)
1030 {
1031     KVMMSIRoute *route, *next;
1032     unsigned int hash;
1033
1034     for (hash = 0; hash < KVM_MSI_HASHTAB_SIZE; hash++) {
1035         QTAILQ_FOREACH_SAFE(route, &s->msi_hashtab[hash], entry, next) {
1036             kvm_irqchip_release_virq(s, route->kroute.gsi);
1037             QTAILQ_REMOVE(&s->msi_hashtab[hash], route, entry);
1038             g_free(route);
1039         }
1040     }
1041 }
1042
1043 static int kvm_irqchip_get_virq(KVMState *s)
1044 {
1045     uint32_t *word = s->used_gsi_bitmap;
1046     int max_words = ALIGN(s->gsi_count, 32) / 32;
1047     int i, bit;
1048     bool retry = true;
1049
1050 again:
1051     /* Return the lowest unused GSI in the bitmap */
1052     for (i = 0; i < max_words; i++) {
1053         bit = ffs(~word[i]);
1054         if (!bit) {
1055             continue;
1056         }
1057
1058         return bit - 1 + i * 32;
1059     }
1060     if (!s->direct_msi && retry) {
1061         retry = false;
1062         kvm_flush_dynamic_msi_routes(s);
1063         goto again;
1064     }
1065     return -ENOSPC;
1066
1067 }
1068
1069 static KVMMSIRoute *kvm_lookup_msi_route(KVMState *s, MSIMessage msg)
1070 {
1071     unsigned int hash = kvm_hash_msi(msg.data);
1072     KVMMSIRoute *route;
1073
1074     QTAILQ_FOREACH(route, &s->msi_hashtab[hash], entry) {
1075         if (route->kroute.u.msi.address_lo == (uint32_t)msg.address &&
1076             route->kroute.u.msi.address_hi == (msg.address >> 32) &&
1077             route->kroute.u.msi.data == msg.data) {
1078             return route;
1079         }
1080     }
1081     return NULL;
1082 }
1083
1084 int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
1085 {
1086     struct kvm_msi msi;
1087     KVMMSIRoute *route;
1088
1089     if (s->direct_msi) {
1090         msi.address_lo = (uint32_t)msg.address;
1091         msi.address_hi = msg.address >> 32;
1092         msi.data = msg.data;
1093         msi.flags = 0;
1094         memset(msi.pad, 0, sizeof(msi.pad));
1095
1096         return kvm_vm_ioctl(s, KVM_SIGNAL_MSI, &msi);
1097     }
1098
1099     route = kvm_lookup_msi_route(s, msg);
1100     if (!route) {
1101         int virq;
1102
1103         virq = kvm_irqchip_get_virq(s);
1104         if (virq < 0) {
1105             return virq;
1106         }
1107
1108         route = g_malloc(sizeof(KVMMSIRoute));
1109         route->kroute.gsi = virq;
1110         route->kroute.type = KVM_IRQ_ROUTING_MSI;
1111         route->kroute.flags = 0;
1112         route->kroute.u.msi.address_lo = (uint32_t)msg.address;
1113         route->kroute.u.msi.address_hi = msg.address >> 32;
1114         route->kroute.u.msi.data = msg.data;
1115
1116         kvm_add_routing_entry(s, &route->kroute);
1117
1118         QTAILQ_INSERT_TAIL(&s->msi_hashtab[kvm_hash_msi(msg.data)], route,
1119                            entry);
1120     }
1121
1122     assert(route->kroute.type == KVM_IRQ_ROUTING_MSI);
1123
1124     return kvm_set_irq(s, route->kroute.gsi, 1);
1125 }
1126
1127 int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
1128 {
1129     struct kvm_irq_routing_entry kroute;
1130     int virq;
1131
1132     if (!kvm_gsi_routing_enabled()) {
1133         return -ENOSYS;
1134     }
1135
1136     virq = kvm_irqchip_get_virq(s);
1137     if (virq < 0) {
1138         return virq;
1139     }
1140
1141     kroute.gsi = virq;
1142     kroute.type = KVM_IRQ_ROUTING_MSI;
1143     kroute.flags = 0;
1144     kroute.u.msi.address_lo = (uint32_t)msg.address;
1145     kroute.u.msi.address_hi = msg.address >> 32;
1146     kroute.u.msi.data = msg.data;
1147
1148     kvm_add_routing_entry(s, &kroute);
1149
1150     return virq;
1151 }
1152
1153 int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
1154 {
1155     struct kvm_irq_routing_entry kroute;
1156
1157     if (!kvm_irqchip_in_kernel()) {
1158         return -ENOSYS;
1159     }
1160
1161     kroute.gsi = virq;
1162     kroute.type = KVM_IRQ_ROUTING_MSI;
1163     kroute.flags = 0;
1164     kroute.u.msi.address_lo = (uint32_t)msg.address;
1165     kroute.u.msi.address_hi = msg.address >> 32;
1166     kroute.u.msi.data = msg.data;
1167
1168     return kvm_update_routing_entry(s, &kroute);
1169 }
1170
1171 static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
1172 {
1173     struct kvm_irqfd irqfd = {
1174         .fd = fd,
1175         .gsi = virq,
1176         .flags = assign ? 0 : KVM_IRQFD_FLAG_DEASSIGN,
1177     };
1178
1179     if (!kvm_irqfds_enabled()) {
1180         return -ENOSYS;
1181     }
1182
1183     return kvm_vm_ioctl(s, KVM_IRQFD, &irqfd);
1184 }
1185
1186 #else /* !KVM_CAP_IRQ_ROUTING */
1187
1188 static void kvm_init_irq_routing(KVMState *s)
1189 {
1190 }
1191
1192 void kvm_irqchip_release_virq(KVMState *s, int virq)
1193 {
1194 }
1195
1196 int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
1197 {
1198     abort();
1199 }
1200
1201 int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
1202 {
1203     return -ENOSYS;
1204 }
1205
1206 static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
1207 {
1208     abort();
1209 }
1210 #endif /* !KVM_CAP_IRQ_ROUTING */
1211
1212 int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n, int virq)
1213 {
1214     return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n), virq, true);
1215 }
1216
1217 int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n, int virq)
1218 {
1219     return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n), virq, false);
1220 }
1221
1222 static int kvm_irqchip_create(KVMState *s)
1223 {
1224     QemuOptsList *list = qemu_find_opts("machine");
1225     int ret;
1226
1227     if (QTAILQ_EMPTY(&list->head) ||
1228         !qemu_opt_get_bool(QTAILQ_FIRST(&list->head),
1229                            "kernel_irqchip", true) ||
1230         !kvm_check_extension(s, KVM_CAP_IRQCHIP)) {
1231         return 0;
1232     }
1233
1234     ret = kvm_vm_ioctl(s, KVM_CREATE_IRQCHIP);
1235     if (ret < 0) {
1236         fprintf(stderr, "Create kernel irqchip failed\n");
1237         return ret;
1238     }
1239
1240     kvm_kernel_irqchip = true;
1241     /* If we have an in-kernel IRQ chip then we must have asynchronous
1242      * interrupt delivery (though the reverse is not necessarily true)
1243      */
1244     kvm_async_interrupts_allowed = true;
1245
1246     kvm_init_irq_routing(s);
1247
1248     return 0;
1249 }
1250
1251 static int kvm_max_vcpus(KVMState *s)
1252 {
1253     int ret;
1254
1255     /* Find number of supported CPUs using the recommended
1256      * procedure from the kernel API documentation to cope with
1257      * older kernels that may be missing capabilities.
1258      */
1259     ret = kvm_check_extension(s, KVM_CAP_MAX_VCPUS);
1260     if (ret) {
1261         return ret;
1262     }
1263     ret = kvm_check_extension(s, KVM_CAP_NR_VCPUS);
1264     if (ret) {
1265         return ret;
1266     }
1267
1268     return 4;
1269 }
1270
1271 int kvm_init(void)
1272 {
1273     static const char upgrade_note[] =
1274         "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
1275         "(see http://sourceforge.net/projects/kvm).\n";
1276     KVMState *s;
1277     const KVMCapabilityInfo *missing_cap;
1278     int ret;
1279     int i;
1280     int max_vcpus;
1281
1282     s = g_malloc0(sizeof(KVMState));
1283
1284     /*
1285      * On systems where the kernel can support different base page
1286      * sizes, host page size may be different from TARGET_PAGE_SIZE,
1287      * even with KVM.  TARGET_PAGE_SIZE is assumed to be the minimum
1288      * page size for the system though.
1289      */
1290     assert(TARGET_PAGE_SIZE <= getpagesize());
1291
1292 #ifdef KVM_CAP_SET_GUEST_DEBUG
1293     QTAILQ_INIT(&s->kvm_sw_breakpoints);
1294 #endif
1295     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
1296         s->slots[i].slot = i;
1297     }
1298     s->vmfd = -1;
1299     s->fd = qemu_open("/dev/kvm", O_RDWR);
1300     if (s->fd == -1) {
1301         fprintf(stderr, "Could not access KVM kernel module: %m\n");
1302         ret = -errno;
1303         goto err;
1304     }
1305
1306     ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
1307     if (ret < KVM_API_VERSION) {
1308         if (ret > 0) {
1309             ret = -EINVAL;
1310         }
1311         fprintf(stderr, "kvm version too old\n");
1312         goto err;
1313     }
1314
1315     if (ret > KVM_API_VERSION) {
1316         ret = -EINVAL;
1317         fprintf(stderr, "kvm version not supported\n");
1318         goto err;
1319     }
1320
1321     max_vcpus = kvm_max_vcpus(s);
1322     if (smp_cpus > max_vcpus) {
1323         ret = -EINVAL;
1324         fprintf(stderr, "Number of SMP cpus requested (%d) exceeds max cpus "
1325                 "supported by KVM (%d)\n", smp_cpus, max_vcpus);
1326         goto err;
1327     }
1328
1329     s->vmfd = kvm_ioctl(s, KVM_CREATE_VM, 0);
1330     if (s->vmfd < 0) {
1331 #ifdef TARGET_S390X
1332         fprintf(stderr, "Please add the 'switch_amode' kernel parameter to "
1333                         "your host kernel command line\n");
1334 #endif
1335         ret = s->vmfd;
1336         goto err;
1337     }
1338
1339     missing_cap = kvm_check_extension_list(s, kvm_required_capabilites);
1340     if (!missing_cap) {
1341         missing_cap =
1342             kvm_check_extension_list(s, kvm_arch_required_capabilities);
1343     }
1344     if (missing_cap) {
1345         ret = -EINVAL;
1346         fprintf(stderr, "kvm does not support %s\n%s",
1347                 missing_cap->name, upgrade_note);
1348         goto err;
1349     }
1350
1351     s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
1352
1353     s->broken_set_mem_region = 1;
1354     ret = kvm_check_extension(s, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
1355     if (ret > 0) {
1356         s->broken_set_mem_region = 0;
1357     }
1358
1359 #ifdef KVM_CAP_VCPU_EVENTS
1360     s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
1361 #endif
1362
1363     s->robust_singlestep =
1364         kvm_check_extension(s, KVM_CAP_X86_ROBUST_SINGLESTEP);
1365
1366 #ifdef KVM_CAP_DEBUGREGS
1367     s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS);
1368 #endif
1369
1370 #ifdef KVM_CAP_XSAVE
1371     s->xsave = kvm_check_extension(s, KVM_CAP_XSAVE);
1372 #endif
1373
1374 #ifdef KVM_CAP_XCRS
1375     s->xcrs = kvm_check_extension(s, KVM_CAP_XCRS);
1376 #endif
1377
1378 #ifdef KVM_CAP_PIT_STATE2
1379     s->pit_state2 = kvm_check_extension(s, KVM_CAP_PIT_STATE2);
1380 #endif
1381
1382 #ifdef KVM_CAP_IRQ_ROUTING
1383     s->direct_msi = (kvm_check_extension(s, KVM_CAP_SIGNAL_MSI) > 0);
1384 #endif
1385
1386     s->intx_set_mask = kvm_check_extension(s, KVM_CAP_PCI_2_3);
1387
1388     s->irq_set_ioctl = KVM_IRQ_LINE;
1389     if (kvm_check_extension(s, KVM_CAP_IRQ_INJECT_STATUS)) {
1390         s->irq_set_ioctl = KVM_IRQ_LINE_STATUS;
1391     }
1392
1393     ret = kvm_arch_init(s);
1394     if (ret < 0) {
1395         goto err;
1396     }
1397
1398     ret = kvm_irqchip_create(s);
1399     if (ret < 0) {
1400         goto err;
1401     }
1402
1403     kvm_state = s;
1404     memory_listener_register(&kvm_memory_listener, NULL);
1405
1406     s->many_ioeventfds = kvm_check_many_ioeventfds();
1407
1408     cpu_interrupt_handler = kvm_handle_interrupt;
1409
1410     return 0;
1411
1412 err:
1413     if (s->vmfd >= 0) {
1414         close(s->vmfd);
1415     }
1416     if (s->fd != -1) {
1417         close(s->fd);
1418     }
1419     g_free(s);
1420
1421     return ret;
1422 }
1423
1424 static void kvm_handle_io(uint16_t port, void *data, int direction, int size,
1425                           uint32_t count)
1426 {
1427     int i;
1428     uint8_t *ptr = data;
1429
1430     for (i = 0; i < count; i++) {
1431         if (direction == KVM_EXIT_IO_IN) {
1432             switch (size) {
1433             case 1:
1434                 stb_p(ptr, cpu_inb(port));
1435                 break;
1436             case 2:
1437                 stw_p(ptr, cpu_inw(port));
1438                 break;
1439             case 4:
1440                 stl_p(ptr, cpu_inl(port));
1441                 break;
1442             }
1443         } else {
1444             switch (size) {
1445             case 1:
1446                 cpu_outb(port, ldub_p(ptr));
1447                 break;
1448             case 2:
1449                 cpu_outw(port, lduw_p(ptr));
1450                 break;
1451             case 4:
1452                 cpu_outl(port, ldl_p(ptr));
1453                 break;
1454             }
1455         }
1456
1457         ptr += size;
1458     }
1459 }
1460
1461 static int kvm_handle_internal_error(CPUArchState *env, struct kvm_run *run)
1462 {
1463     fprintf(stderr, "KVM internal error.");
1464     if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) {
1465         int i;
1466
1467         fprintf(stderr, " Suberror: %d\n", run->internal.suberror);
1468         for (i = 0; i < run->internal.ndata; ++i) {
1469             fprintf(stderr, "extra data[%d]: %"PRIx64"\n",
1470                     i, (uint64_t)run->internal.data[i]);
1471         }
1472     } else {
1473         fprintf(stderr, "\n");
1474     }
1475     if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) {
1476         fprintf(stderr, "emulation failure\n");
1477         if (!kvm_arch_stop_on_emulation_error(env)) {
1478             cpu_dump_state(env, stderr, fprintf, CPU_DUMP_CODE);
1479             return EXCP_INTERRUPT;
1480         }
1481     }
1482     /* FIXME: Should trigger a qmp message to let management know
1483      * something went wrong.
1484      */
1485     return -1;
1486 }
1487
1488 void kvm_flush_coalesced_mmio_buffer(void)
1489 {
1490     KVMState *s = kvm_state;
1491
1492     if (s->coalesced_flush_in_progress) {
1493         return;
1494     }
1495
1496     s->coalesced_flush_in_progress = true;
1497
1498     if (s->coalesced_mmio_ring) {
1499         struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
1500         while (ring->first != ring->last) {
1501             struct kvm_coalesced_mmio *ent;
1502
1503             ent = &ring->coalesced_mmio[ring->first];
1504
1505             cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
1506             smp_wmb();
1507             ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
1508         }
1509     }
1510
1511     s->coalesced_flush_in_progress = false;
1512 }
1513
1514 static void do_kvm_cpu_synchronize_state(void *_env)
1515 {
1516     CPUArchState *env = _env;
1517
1518     if (!env->kvm_vcpu_dirty) {
1519         kvm_arch_get_registers(env);
1520         env->kvm_vcpu_dirty = 1;
1521     }
1522 }
1523
1524 void kvm_cpu_synchronize_state(CPUArchState *env)
1525 {
1526     if (!env->kvm_vcpu_dirty) {
1527         run_on_cpu(env, do_kvm_cpu_synchronize_state, env);
1528     }
1529 }
1530
1531 void kvm_cpu_synchronize_post_reset(CPUArchState *env)
1532 {
1533     kvm_arch_put_registers(env, KVM_PUT_RESET_STATE);
1534     env->kvm_vcpu_dirty = 0;
1535 }
1536
1537 void kvm_cpu_synchronize_post_init(CPUArchState *env)
1538 {
1539     kvm_arch_put_registers(env, KVM_PUT_FULL_STATE);
1540     env->kvm_vcpu_dirty = 0;
1541 }
1542
1543 int kvm_cpu_exec(CPUArchState *env)
1544 {
1545     struct kvm_run *run = env->kvm_run;
1546     int ret, run_ret;
1547
1548     DPRINTF("kvm_cpu_exec()\n");
1549
1550     if (kvm_arch_process_async_events(env)) {
1551         env->exit_request = 0;
1552         return EXCP_HLT;
1553     }
1554
1555     do {
1556         if (env->kvm_vcpu_dirty) {
1557             kvm_arch_put_registers(env, KVM_PUT_RUNTIME_STATE);
1558             env->kvm_vcpu_dirty = 0;
1559         }
1560
1561         kvm_arch_pre_run(env, run);
1562         if (env->exit_request) {
1563             DPRINTF("interrupt exit requested\n");
1564             /*
1565              * KVM requires us to reenter the kernel after IO exits to complete
1566              * instruction emulation. This self-signal will ensure that we
1567              * leave ASAP again.
1568              */
1569             qemu_cpu_kick_self();
1570         }
1571         qemu_mutex_unlock_iothread();
1572
1573         run_ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);
1574
1575         qemu_mutex_lock_iothread();
1576         kvm_arch_post_run(env, run);
1577
1578         if (run_ret < 0) {
1579             if (run_ret == -EINTR || run_ret == -EAGAIN) {
1580                 DPRINTF("io window exit\n");
1581                 ret = EXCP_INTERRUPT;
1582                 break;
1583             }
1584             fprintf(stderr, "error: kvm run failed %s\n",
1585                     strerror(-run_ret));
1586             abort();
1587         }
1588
1589         switch (run->exit_reason) {
1590         case KVM_EXIT_IO:
1591             DPRINTF("handle_io\n");
1592             kvm_handle_io(run->io.port,
1593                           (uint8_t *)run + run->io.data_offset,
1594                           run->io.direction,
1595                           run->io.size,
1596                           run->io.count);
1597             ret = 0;
1598             break;
1599         case KVM_EXIT_MMIO:
1600             DPRINTF("handle_mmio\n");
1601             cpu_physical_memory_rw(run->mmio.phys_addr,
1602                                    run->mmio.data,
1603                                    run->mmio.len,
1604                                    run->mmio.is_write);
1605             ret = 0;
1606             break;
1607         case KVM_EXIT_IRQ_WINDOW_OPEN:
1608             DPRINTF("irq_window_open\n");
1609             ret = EXCP_INTERRUPT;
1610             break;
1611         case KVM_EXIT_SHUTDOWN:
1612             DPRINTF("shutdown\n");
1613             qemu_system_reset_request();
1614             ret = EXCP_INTERRUPT;
1615             break;
1616         case KVM_EXIT_UNKNOWN:
1617             fprintf(stderr, "KVM: unknown exit, hardware reason %" PRIx64 "\n",
1618                     (uint64_t)run->hw.hardware_exit_reason);
1619             ret = -1;
1620             break;
1621         case KVM_EXIT_INTERNAL_ERROR:
1622             ret = kvm_handle_internal_error(env, run);
1623             break;
1624         default:
1625             DPRINTF("kvm_arch_handle_exit\n");
1626             ret = kvm_arch_handle_exit(env, run);
1627             break;
1628         }
1629     } while (ret == 0);
1630
1631     if (ret < 0) {
1632         cpu_dump_state(env, stderr, fprintf, CPU_DUMP_CODE);
1633         vm_stop(RUN_STATE_INTERNAL_ERROR);
1634     }
1635
1636     env->exit_request = 0;
1637     return ret;
1638 }
1639
1640 int kvm_ioctl(KVMState *s, int type, ...)
1641 {
1642     int ret;
1643     void *arg;
1644     va_list ap;
1645
1646     va_start(ap, type);
1647     arg = va_arg(ap, void *);
1648     va_end(ap);
1649
1650     ret = ioctl(s->fd, type, arg);
1651     if (ret == -1) {
1652         ret = -errno;
1653     }
1654     return ret;
1655 }
1656
1657 int kvm_vm_ioctl(KVMState *s, int type, ...)
1658 {
1659     int ret;
1660     void *arg;
1661     va_list ap;
1662
1663     va_start(ap, type);
1664     arg = va_arg(ap, void *);
1665     va_end(ap);
1666
1667     ret = ioctl(s->vmfd, type, arg);
1668     if (ret == -1) {
1669         ret = -errno;
1670     }
1671     return ret;
1672 }
1673
1674 int kvm_vcpu_ioctl(CPUArchState *env, int type, ...)
1675 {
1676     int ret;
1677     void *arg;
1678     va_list ap;
1679
1680     va_start(ap, type);
1681     arg = va_arg(ap, void *);
1682     va_end(ap);
1683
1684     ret = ioctl(env->kvm_fd, type, arg);
1685     if (ret == -1) {
1686         ret = -errno;
1687     }
1688     return ret;
1689 }
1690
1691 int kvm_has_sync_mmu(void)
1692 {
1693     return kvm_check_extension(kvm_state, KVM_CAP_SYNC_MMU);
1694 }
1695
1696 int kvm_has_vcpu_events(void)
1697 {
1698     return kvm_state->vcpu_events;
1699 }
1700
1701 int kvm_has_robust_singlestep(void)
1702 {
1703     return kvm_state->robust_singlestep;
1704 }
1705
1706 int kvm_has_debugregs(void)
1707 {
1708     return kvm_state->debugregs;
1709 }
1710
1711 int kvm_has_xsave(void)
1712 {
1713     return kvm_state->xsave;
1714 }
1715
1716 int kvm_has_xcrs(void)
1717 {
1718     return kvm_state->xcrs;
1719 }
1720
1721 int kvm_has_pit_state2(void)
1722 {
1723     return kvm_state->pit_state2;
1724 }
1725
1726 int kvm_has_many_ioeventfds(void)
1727 {
1728     if (!kvm_enabled()) {
1729         return 0;
1730     }
1731     return kvm_state->many_ioeventfds;
1732 }
1733
1734 int kvm_has_gsi_routing(void)
1735 {
1736 #ifdef KVM_CAP_IRQ_ROUTING
1737     return kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING);
1738 #else
1739     return false;
1740 #endif
1741 }
1742
1743 int kvm_has_intx_set_mask(void)
1744 {
1745     return kvm_state->intx_set_mask;
1746 }
1747
1748 void *kvm_vmalloc(ram_addr_t size)
1749 {
1750 #ifdef TARGET_S390X
1751     void *mem;
1752
1753     mem = kvm_arch_vmalloc(size);
1754     if (mem) {
1755         return mem;
1756     }
1757 #endif
1758     return qemu_vmalloc(size);
1759 }
1760
1761 void kvm_setup_guest_memory(void *start, size_t size)
1762 {
1763 #ifdef CONFIG_VALGRIND_H
1764     VALGRIND_MAKE_MEM_DEFINED(start, size);
1765 #endif
1766     if (!kvm_has_sync_mmu()) {
1767         int ret = qemu_madvise(start, size, QEMU_MADV_DONTFORK);
1768
1769         if (ret) {
1770             perror("qemu_madvise");
1771             fprintf(stderr,
1772                     "Need MADV_DONTFORK in absence of synchronous KVM MMU\n");
1773             exit(1);
1774         }
1775     }
1776 }
1777
1778 #ifdef KVM_CAP_SET_GUEST_DEBUG
1779 struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUArchState *env,
1780                                                  target_ulong pc)
1781 {
1782     struct kvm_sw_breakpoint *bp;
1783
1784     QTAILQ_FOREACH(bp, &env->kvm_state->kvm_sw_breakpoints, entry) {
1785         if (bp->pc == pc) {
1786             return bp;
1787         }
1788     }
1789     return NULL;
1790 }
1791
1792 int kvm_sw_breakpoints_active(CPUArchState *env)
1793 {
1794     return !QTAILQ_EMPTY(&env->kvm_state->kvm_sw_breakpoints);
1795 }
1796
1797 struct kvm_set_guest_debug_data {
1798     struct kvm_guest_debug dbg;
1799     CPUArchState *env;
1800     int err;
1801 };
1802
1803 static void kvm_invoke_set_guest_debug(void *data)
1804 {
1805     struct kvm_set_guest_debug_data *dbg_data = data;
1806     CPUArchState *env = dbg_data->env;
1807
1808     dbg_data->err = kvm_vcpu_ioctl(env, KVM_SET_GUEST_DEBUG, &dbg_data->dbg);
1809 }
1810
1811 int kvm_update_guest_debug(CPUArchState *env, unsigned long reinject_trap)
1812 {
1813     struct kvm_set_guest_debug_data data;
1814
1815     data.dbg.control = reinject_trap;
1816
1817     if (env->singlestep_enabled) {
1818         data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
1819     }
1820     kvm_arch_update_guest_debug(env, &data.dbg);
1821     data.env = env;
1822
1823     run_on_cpu(env, kvm_invoke_set_guest_debug, &data);
1824     return data.err;
1825 }
1826
1827 int kvm_insert_breakpoint(CPUArchState *current_env, target_ulong addr,
1828                           target_ulong len, int type)
1829 {
1830     struct kvm_sw_breakpoint *bp;
1831     CPUArchState *env;
1832     int err;
1833
1834     if (type == GDB_BREAKPOINT_SW) {
1835         bp = kvm_find_sw_breakpoint(current_env, addr);
1836         if (bp) {
1837             bp->use_count++;
1838             return 0;
1839         }
1840
1841         bp = g_malloc(sizeof(struct kvm_sw_breakpoint));
1842         if (!bp) {
1843             return -ENOMEM;
1844         }
1845
1846         bp->pc = addr;
1847         bp->use_count = 1;
1848         err = kvm_arch_insert_sw_breakpoint(current_env, bp);
1849         if (err) {
1850             g_free(bp);
1851             return err;
1852         }
1853
1854         QTAILQ_INSERT_HEAD(&current_env->kvm_state->kvm_sw_breakpoints,
1855                           bp, entry);
1856     } else {
1857         err = kvm_arch_insert_hw_breakpoint(addr, len, type);
1858         if (err) {
1859             return err;
1860         }
1861     }
1862
1863     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1864         err = kvm_update_guest_debug(env, 0);
1865         if (err) {
1866             return err;
1867         }
1868     }
1869     return 0;
1870 }
1871
1872 int kvm_remove_breakpoint(CPUArchState *current_env, target_ulong addr,
1873                           target_ulong len, int type)
1874 {
1875     struct kvm_sw_breakpoint *bp;
1876     CPUArchState *env;
1877     int err;
1878
1879     if (type == GDB_BREAKPOINT_SW) {
1880         bp = kvm_find_sw_breakpoint(current_env, addr);
1881         if (!bp) {
1882             return -ENOENT;
1883         }
1884
1885         if (bp->use_count > 1) {
1886             bp->use_count--;
1887             return 0;
1888         }
1889
1890         err = kvm_arch_remove_sw_breakpoint(current_env, bp);
1891         if (err) {
1892             return err;
1893         }
1894
1895         QTAILQ_REMOVE(&current_env->kvm_state->kvm_sw_breakpoints, bp, entry);
1896         g_free(bp);
1897     } else {
1898         err = kvm_arch_remove_hw_breakpoint(addr, len, type);
1899         if (err) {
1900             return err;
1901         }
1902     }
1903
1904     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1905         err = kvm_update_guest_debug(env, 0);
1906         if (err) {
1907             return err;
1908         }
1909     }
1910     return 0;
1911 }
1912
1913 void kvm_remove_all_breakpoints(CPUArchState *current_env)
1914 {
1915     struct kvm_sw_breakpoint *bp, *next;
1916     KVMState *s = current_env->kvm_state;
1917     CPUArchState *env;
1918
1919     QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
1920         if (kvm_arch_remove_sw_breakpoint(current_env, bp) != 0) {
1921             /* Try harder to find a CPU that currently sees the breakpoint. */
1922             for (env = first_cpu; env != NULL; env = env->next_cpu) {
1923                 if (kvm_arch_remove_sw_breakpoint(env, bp) == 0) {
1924                     break;
1925                 }
1926             }
1927         }
1928     }
1929     kvm_arch_remove_all_hw_breakpoints();
1930
1931     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1932         kvm_update_guest_debug(env, 0);
1933     }
1934 }
1935
1936 #else /* !KVM_CAP_SET_GUEST_DEBUG */
1937
1938 int kvm_update_guest_debug(CPUArchState *env, unsigned long reinject_trap)
1939 {
1940     return -EINVAL;
1941 }
1942
1943 int kvm_insert_breakpoint(CPUArchState *current_env, target_ulong addr,
1944                           target_ulong len, int type)
1945 {
1946     return -EINVAL;
1947 }
1948
1949 int kvm_remove_breakpoint(CPUArchState *current_env, target_ulong addr,
1950                           target_ulong len, int type)
1951 {
1952     return -EINVAL;
1953 }
1954
1955 void kvm_remove_all_breakpoints(CPUArchState *current_env)
1956 {
1957 }
1958 #endif /* !KVM_CAP_SET_GUEST_DEBUG */
1959
1960 int kvm_set_signal_mask(CPUArchState *env, const sigset_t *sigset)
1961 {
1962     struct kvm_signal_mask *sigmask;
1963     int r;
1964
1965     if (!sigset) {
1966         return kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, NULL);
1967     }
1968
1969     sigmask = g_malloc(sizeof(*sigmask) + sizeof(*sigset));
1970
1971     sigmask->len = 8;
1972     memcpy(sigmask->sigset, sigset, sizeof(*sigset));
1973     r = kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, sigmask);
1974     g_free(sigmask);
1975
1976     return r;
1977 }
1978
1979 int kvm_set_ioeventfd_mmio(int fd, uint32_t addr, uint32_t val, bool assign,
1980                            uint32_t size)
1981 {
1982     int ret;
1983     struct kvm_ioeventfd iofd;
1984
1985     iofd.datamatch = val;
1986     iofd.addr = addr;
1987     iofd.len = size;
1988     iofd.flags = KVM_IOEVENTFD_FLAG_DATAMATCH;
1989     iofd.fd = fd;
1990
1991     if (!kvm_enabled()) {
1992         return -ENOSYS;
1993     }
1994
1995     if (!assign) {
1996         iofd.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
1997     }
1998
1999     ret = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &iofd);
2000
2001     if (ret < 0) {
2002         return -errno;
2003     }
2004
2005     return 0;
2006 }
2007
2008 int kvm_set_ioeventfd_pio_word(int fd, uint16_t addr, uint16_t val, bool assign)
2009 {
2010     struct kvm_ioeventfd kick = {
2011         .datamatch = val,
2012         .addr = addr,
2013         .len = 2,
2014         .flags = KVM_IOEVENTFD_FLAG_DATAMATCH | KVM_IOEVENTFD_FLAG_PIO,
2015         .fd = fd,
2016     };
2017     int r;
2018     if (!kvm_enabled()) {
2019         return -ENOSYS;
2020     }
2021     if (!assign) {
2022         kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
2023     }
2024     r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
2025     if (r < 0) {
2026         return r;
2027     }
2028     return 0;
2029 }
2030
2031 int kvm_on_sigbus_vcpu(CPUArchState *env, int code, void *addr)
2032 {
2033     return kvm_arch_on_sigbus_vcpu(env, code, addr);
2034 }
2035
2036 int kvm_on_sigbus(int code, void *addr)
2037 {
2038     return kvm_arch_on_sigbus(code, addr);
2039 }