accel/kvm/kvm-all.c

   1 /*
   2  * QEMU KVM support
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *           Red Hat, Inc. 2008
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *  Glauber Costa     <gcosta@redhat.com>
  10  *
  11  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  12  * See the COPYING file in the top-level directory.
  13  *
  14  */
  15
  16 #include "qemu/osdep.h"
  17 #include <sys/ioctl.h>
  18
  19 #include <linux/kvm.h>
  20
  21 #include "qemu-common.h"
  22 #include "qemu/atomic.h"
  23 #include "qemu/option.h"
  24 #include "qemu/config-file.h"
  25 #include "qemu/error-report.h"
  26 #include "qapi/error.h"
  27 #include "hw/hw.h"
  28 #include "hw/pci/msi.h"
  29 #include "hw/pci/msix.h"
  30 #include "hw/s390x/adapter.h"
  31 #include "exec/gdbstub.h"
  32 #include "sysemu/kvm_int.h"
  33 #include "sysemu/cpus.h"
  34 #include "qemu/bswap.h"
  35 #include "exec/memory.h"
  36 #include "exec/ram_addr.h"
  37 #include "exec/address-spaces.h"
  38 #include "qemu/event_notifier.h"
  39 #include "trace.h"
  40 #include "hw/irq.h"
  41
  42 #include "hw/boards.h"
  43
  44 /* This check must be after config-host.h is included */
  45 #ifdef CONFIG_EVENTFD
  46 #include <sys/eventfd.h>
  47 #endif
  48
  49 /* KVM uses PAGE_SIZE in its definition of KVM_COALESCED_MMIO_MAX. We
  50  * need to use the real host PAGE_SIZE, as that's what KVM will use.
  51  */
  52 #define PAGE_SIZE getpagesize()
  53
  54 //#define DEBUG_KVM
  55
  56 #ifdef DEBUG_KVM
  57 #define DPRINTF(fmt, ...) \
  58     do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
  59 #else
  60 #define DPRINTF(fmt, ...) \
  61     do { } while (0)
  62 #endif
  63
  64 #define KVM_MSI_HASHTAB_SIZE    256
  65
  66 struct KVMParkedVcpu {
  67     unsigned long vcpu_id;
  68     int kvm_fd;
  69     QLIST_ENTRY(KVMParkedVcpu) node;
  70 };
  71
  72 struct KVMState
  73 {
  74     AccelState parent_obj;
  75
  76     int nr_slots;
  77     int fd;
  78     int vmfd;
  79     int coalesced_mmio;
  80     struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
  81     bool coalesced_flush_in_progress;
  82     int broken_set_mem_region;
  83     int vcpu_events;
  84     int robust_singlestep;
  85     int debugregs;
  86 #ifdef KVM_CAP_SET_GUEST_DEBUG
  87     struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
  88 #endif
  89     int many_ioeventfds;
  90     int intx_set_mask;
  91     /* The man page (and posix) say ioctl numbers are signed int, but
  92      * they're not.  Linux, glibc and *BSD all treat ioctl numbers as
  93      * unsigned, and treating them as signed here can break things */
  94     unsigned irq_set_ioctl;
  95     unsigned int sigmask_len;
  96     GHashTable *gsimap;
  97 #ifdef KVM_CAP_IRQ_ROUTING
  98     struct kvm_irq_routing *irq_routes;
  99     int nr_allocated_irq_routes;
 100     unsigned long *used_gsi_bitmap;
 101     unsigned int gsi_count;
 102     QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
 103 #endif
 104     KVMMemoryListener memory_listener;
 105     QLIST_HEAD(, KVMParkedVcpu) kvm_parked_vcpus;
 106 };
 107
 108 KVMState *kvm_state;
 109 bool kvm_kernel_irqchip;
 110 bool kvm_split_irqchip;
 111 bool kvm_async_interrupts_allowed;
 112 bool kvm_halt_in_kernel_allowed;
 113 bool kvm_eventfds_allowed;
 114 bool kvm_irqfds_allowed;
 115 bool kvm_resamplefds_allowed;
 116 bool kvm_msi_via_irqfd_allowed;
 117 bool kvm_gsi_routing_allowed;
 118 bool kvm_gsi_direct_mapping;
 119 bool kvm_allowed;
 120 bool kvm_readonly_mem_allowed;
 121 bool kvm_vm_attributes_allowed;
 122 bool kvm_direct_msi_allowed;
 123 bool kvm_ioeventfd_any_length_allowed;
 124 bool kvm_msi_use_devid;
 125 static bool kvm_immediate_exit;
 126
 127 static const KVMCapabilityInfo kvm_required_capabilites[] = {
 128     KVM_CAP_INFO(USER_MEMORY),
 129     KVM_CAP_INFO(DESTROY_MEMORY_REGION_WORKS),
 130     KVM_CAP_LAST_INFO
 131 };
 132
 133 int kvm_get_max_memslots(void)
 134 {
 135     KVMState *s = KVM_STATE(current_machine->accelerator);
 136
 137     return s->nr_slots;
 138 }
 139
 140 static KVMSlot *kvm_get_free_slot(KVMMemoryListener *kml)
 141 {
 142     KVMState *s = kvm_state;
 143     int i;
 144
 145     for (i = 0; i < s->nr_slots; i++) {
 146         if (kml->slots[i].memory_size == 0) {
 147             return &kml->slots[i];
 148         }
 149     }
 150
 151     return NULL;
 152 }
 153
 154 bool kvm_has_free_slot(MachineState *ms)
 155 {
 156     KVMState *s = KVM_STATE(ms->accelerator);
 157
 158     return kvm_get_free_slot(&s->memory_listener);
 159 }
 160
 161 static KVMSlot *kvm_alloc_slot(KVMMemoryListener *kml)
 162 {
 163     KVMSlot *slot = kvm_get_free_slot(kml);
 164
 165     if (slot) {
 166         return slot;
 167     }
 168
 169     fprintf(stderr, "%s: no free slot available\n", __func__);
 170     abort();
 171 }
 172
 173 static KVMSlot *kvm_lookup_matching_slot(KVMMemoryListener *kml,
 174                                          hwaddr start_addr,
 175                                          hwaddr end_addr)
 176 {
 177     KVMState *s = kvm_state;
 178     int i;
 179
 180     for (i = 0; i < s->nr_slots; i++) {
 181         KVMSlot *mem = &kml->slots[i];
 182
 183         if (start_addr == mem->start_addr &&
 184             end_addr == mem->start_addr + mem->memory_size) {
 185             return mem;
 186         }
 187     }
 188
 189     return NULL;
 190 }
 191
 192 /*
 193  * Find overlapping slot with lowest start address
 194  */
 195 static KVMSlot *kvm_lookup_overlapping_slot(KVMMemoryListener *kml,
 196                                             hwaddr start_addr,
 197                                             hwaddr end_addr)
 198 {
 199     KVMState *s = kvm_state;
 200     KVMSlot *found = NULL;
 201     int i;
 202
 203     for (i = 0; i < s->nr_slots; i++) {
 204         KVMSlot *mem = &kml->slots[i];
 205
 206         if (mem->memory_size == 0 ||
 207             (found && found->start_addr < mem->start_addr)) {
 208             continue;
 209         }
 210
 211         if (end_addr > mem->start_addr &&
 212             start_addr < mem->start_addr + mem->memory_size) {
 213             found = mem;
 214         }
 215     }
 216
 217     return found;
 218 }
 219
 220 int kvm_physical_memory_addr_from_host(KVMState *s, void *ram,
 221                                        hwaddr *phys_addr)
 222 {
 223     KVMMemoryListener *kml = &s->memory_listener;
 224     int i;
 225
 226     for (i = 0; i < s->nr_slots; i++) {
 227         KVMSlot *mem = &kml->slots[i];
 228
 229         if (ram >= mem->ram && ram < mem->ram + mem->memory_size) {
 230             *phys_addr = mem->start_addr + (ram - mem->ram);
 231             return 1;
 232         }
 233     }
 234
 235     return 0;
 236 }
 237
 238 static int kvm_set_user_memory_region(KVMMemoryListener *kml, KVMSlot *slot)
 239 {
 240     KVMState *s = kvm_state;
 241     struct kvm_userspace_memory_region mem;
 242
 243     mem.slot = slot->slot | (kml->as_id << 16);
 244     mem.guest_phys_addr = slot->start_addr;
 245     mem.userspace_addr = (unsigned long)slot->ram;
 246     mem.flags = slot->flags;
 247
 248     if (slot->memory_size && mem.flags & KVM_MEM_READONLY) {
 249         /* Set the slot size to 0 before setting the slot to the desired
 250          * value. This is needed based on KVM commit 75d61fbc. */
 251         mem.memory_size = 0;
 252         kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
 253     }
 254     mem.memory_size = slot->memory_size;
 255     return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
 256 }
 257
 258 int kvm_destroy_vcpu(CPUState *cpu)
 259 {
 260     KVMState *s = kvm_state;
 261     long mmap_size;
 262     struct KVMParkedVcpu *vcpu = NULL;
 263     int ret = 0;
 264
 265     DPRINTF("kvm_destroy_vcpu\n");
 266
 267     mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
 268     if (mmap_size < 0) {
 269         ret = mmap_size;
 270         DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
 271         goto err;
 272     }
 273
 274     ret = munmap(cpu->kvm_run, mmap_size);
 275     if (ret < 0) {
 276         goto err;
 277     }
 278
 279     vcpu = g_malloc0(sizeof(*vcpu));
 280     vcpu->vcpu_id = kvm_arch_vcpu_id(cpu);
 281     vcpu->kvm_fd = cpu->kvm_fd;
 282     QLIST_INSERT_HEAD(&kvm_state->kvm_parked_vcpus, vcpu, node);
 283 err:
 284     return ret;
 285 }
 286
 287 static int kvm_get_vcpu(KVMState *s, unsigned long vcpu_id)
 288 {
 289     struct KVMParkedVcpu *cpu;
 290
 291     QLIST_FOREACH(cpu, &s->kvm_parked_vcpus, node) {
 292         if (cpu->vcpu_id == vcpu_id) {
 293             int kvm_fd;
 294
 295             QLIST_REMOVE(cpu, node);
 296             kvm_fd = cpu->kvm_fd;
 297             g_free(cpu);
 298             return kvm_fd;
 299         }
 300     }
 301
 302     return kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)vcpu_id);
 303 }
 304
 305 int kvm_init_vcpu(CPUState *cpu)
 306 {
 307     KVMState *s = kvm_state;
 308     long mmap_size;
 309     int ret;
 310
 311     DPRINTF("kvm_init_vcpu\n");
 312
 313     ret = kvm_get_vcpu(s, kvm_arch_vcpu_id(cpu));
 314     if (ret < 0) {
 315         DPRINTF("kvm_create_vcpu failed\n");
 316         goto err;
 317     }
 318
 319     cpu->kvm_fd = ret;
 320     cpu->kvm_state = s;
 321     cpu->vcpu_dirty = true;
 322
 323     mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
 324     if (mmap_size < 0) {
 325         ret = mmap_size;
 326         DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
 327         goto err;
 328     }
 329
 330     cpu->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
 331                         cpu->kvm_fd, 0);
 332     if (cpu->kvm_run == MAP_FAILED) {
 333         ret = -errno;
 334         DPRINTF("mmap'ing vcpu state failed\n");
 335         goto err;
 336     }
 337
 338     if (s->coalesced_mmio && !s->coalesced_mmio_ring) {
 339         s->coalesced_mmio_ring =
 340             (void *)cpu->kvm_run + s->coalesced_mmio * PAGE_SIZE;
 341     }
 342
 343     ret = kvm_arch_init_vcpu(cpu);
 344 err:
 345     return ret;
 346 }
 347
 348 /*
 349  * dirty pages logging control
 350  */
 351
 352 static int kvm_mem_flags(MemoryRegion *mr)
 353 {
 354     bool readonly = mr->readonly || memory_region_is_romd(mr);
 355     int flags = 0;
 356
 357     if (memory_region_get_dirty_log_mask(mr) != 0) {
 358         flags |= KVM_MEM_LOG_DIRTY_PAGES;
 359     }
 360     if (readonly && kvm_readonly_mem_allowed) {
 361         flags |= KVM_MEM_READONLY;
 362     }
 363     return flags;
 364 }
 365
 366 static int kvm_slot_update_flags(KVMMemoryListener *kml, KVMSlot *mem,
 367                                  MemoryRegion *mr)
 368 {
 369     int old_flags;
 370
 371     old_flags = mem->flags;
 372     mem->flags = kvm_mem_flags(mr);
 373
 374     /* If nothing changed effectively, no need to issue ioctl */
 375     if (mem->flags == old_flags) {
 376         return 0;
 377     }
 378
 379     return kvm_set_user_memory_region(kml, mem);
 380 }
 381
 382 static int kvm_section_update_flags(KVMMemoryListener *kml,
 383                                     MemoryRegionSection *section)
 384 {
 385     hwaddr phys_addr = section->offset_within_address_space;
 386     ram_addr_t size = int128_get64(section->size);
 387     KVMSlot *mem = kvm_lookup_matching_slot(kml, phys_addr, phys_addr + size);
 388
 389     if (mem == NULL)  {
 390         return 0;
 391     } else {
 392         return kvm_slot_update_flags(kml, mem, section->mr);
 393     }
 394 }
 395
 396 static void kvm_log_start(MemoryListener *listener,
 397                           MemoryRegionSection *section,
 398                           int old, int new)
 399 {
 400     KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
 401     int r;
 402
 403     if (old != 0) {
 404         return;
 405     }
 406
 407     r = kvm_section_update_flags(kml, section);
 408     if (r < 0) {
 409         abort();
 410     }
 411 }
 412
 413 static void kvm_log_stop(MemoryListener *listener,
 414                           MemoryRegionSection *section,
 415                           int old, int new)
 416 {
 417     KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
 418     int r;
 419
 420     if (new != 0) {
 421         return;
 422     }
 423
 424     r = kvm_section_update_flags(kml, section);
 425     if (r < 0) {
 426         abort();
 427     }
 428 }
 429
 430 /* get kvm's dirty pages bitmap and update qemu's */
 431 static int kvm_get_dirty_pages_log_range(MemoryRegionSection *section,
 432                                          unsigned long *bitmap)
 433 {
 434     ram_addr_t start = section->offset_within_region +
 435                        memory_region_get_ram_addr(section->mr);
 436     ram_addr_t pages = int128_get64(section->size) / getpagesize();
 437
 438     cpu_physical_memory_set_dirty_lebitmap(bitmap, start, pages);
 439     return 0;
 440 }
 441
 442 #define ALIGN(x, y)  (((x)+(y)-1) & ~((y)-1))
 443
 444 /**
 445  * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
 446  * This function updates qemu's dirty bitmap using
 447  * memory_region_set_dirty().  This means all bits are set
 448  * to dirty.
 449  *
 450  * @start_add: start of logged region.
 451  * @end_addr: end of logged region.
 452  */
 453 static int kvm_physical_sync_dirty_bitmap(KVMMemoryListener *kml,
 454                                           MemoryRegionSection *section)
 455 {
 456     KVMState *s = kvm_state;
 457     unsigned long size, allocated_size = 0;
 458     struct kvm_dirty_log d = {};
 459     KVMSlot *mem;
 460     int ret = 0;
 461     hwaddr start_addr = section->offset_within_address_space;
 462     hwaddr end_addr = start_addr + int128_get64(section->size);
 463
 464     d.dirty_bitmap = NULL;
 465     while (start_addr < end_addr) {
 466         mem = kvm_lookup_overlapping_slot(kml, start_addr, end_addr);
 467         if (mem == NULL) {
 468             break;
 469         }
 470
 471         /* XXX bad kernel interface alert
 472          * For dirty bitmap, kernel allocates array of size aligned to
 473          * bits-per-long.  But for case when the kernel is 64bits and
 474          * the userspace is 32bits, userspace can't align to the same
 475          * bits-per-long, since sizeof(long) is different between kernel
 476          * and user space.  This way, userspace will provide buffer which
 477          * may be 4 bytes less than the kernel will use, resulting in
 478          * userspace memory corruption (which is not detectable by valgrind
 479          * too, in most cases).
 480          * So for now, let's align to 64 instead of HOST_LONG_BITS here, in
 481          * a hope that sizeof(long) won't become >8 any time soon.
 482          */
 483         size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS),
 484                      /*HOST_LONG_BITS*/ 64) / 8;
 485         if (!d.dirty_bitmap) {
 486             d.dirty_bitmap = g_malloc(size);
 487         } else if (size > allocated_size) {
 488             d.dirty_bitmap = g_realloc(d.dirty_bitmap, size);
 489         }
 490         allocated_size = size;
 491         memset(d.dirty_bitmap, 0, allocated_size);
 492
 493         d.slot = mem->slot | (kml->as_id << 16);
 494         if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
 495             DPRINTF("ioctl failed %d\n", errno);
 496             ret = -1;
 497             break;
 498         }
 499
 500         kvm_get_dirty_pages_log_range(section, d.dirty_bitmap);
 501         start_addr = mem->start_addr + mem->memory_size;
 502     }
 503     g_free(d.dirty_bitmap);
 504
 505     return ret;
 506 }
 507
 508 static void kvm_coalesce_mmio_region(MemoryListener *listener,
 509                                      MemoryRegionSection *secion,
 510                                      hwaddr start, hwaddr size)
 511 {
 512     KVMState *s = kvm_state;
 513
 514     if (s->coalesced_mmio) {
 515         struct kvm_coalesced_mmio_zone zone;
 516
 517         zone.addr = start;
 518         zone.size = size;
 519         zone.pad = 0;
 520
 521         (void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
 522     }
 523 }
 524
 525 static void kvm_uncoalesce_mmio_region(MemoryListener *listener,
 526                                        MemoryRegionSection *secion,
 527                                        hwaddr start, hwaddr size)
 528 {
 529     KVMState *s = kvm_state;
 530
 531     if (s->coalesced_mmio) {
 532         struct kvm_coalesced_mmio_zone zone;
 533
 534         zone.addr = start;
 535         zone.size = size;
 536         zone.pad = 0;
 537
 538         (void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
 539     }
 540 }
 541
 542 int kvm_check_extension(KVMState *s, unsigned int extension)
 543 {
 544     int ret;
 545
 546     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
 547     if (ret < 0) {
 548         ret = 0;
 549     }
 550
 551     return ret;
 552 }
 553
 554 int kvm_vm_check_extension(KVMState *s, unsigned int extension)
 555 {
 556     int ret;
 557
 558     ret = kvm_vm_ioctl(s, KVM_CHECK_EXTENSION, extension);
 559     if (ret < 0) {
 560         /* VM wide version not implemented, use global one instead */
 561         ret = kvm_check_extension(s, extension);
 562     }
 563
 564     return ret;
 565 }
 566
 567 static uint32_t adjust_ioeventfd_endianness(uint32_t val, uint32_t size)
 568 {
 569 #if defined(HOST_WORDS_BIGENDIAN) != defined(TARGET_WORDS_BIGENDIAN)
 570     /* The kernel expects ioeventfd values in HOST_WORDS_BIGENDIAN
 571      * endianness, but the memory core hands them in target endianness.
 572      * For example, PPC is always treated as big-endian even if running
 573      * on KVM and on PPC64LE.  Correct here.
 574      */
 575     switch (size) {
 576     case 2:
 577         val = bswap16(val);
 578         break;
 579     case 4:
 580         val = bswap32(val);
 581         break;
 582     }
 583 #endif
 584     return val;
 585 }
 586
 587 static int kvm_set_ioeventfd_mmio(int fd, hwaddr addr, uint32_t val,
 588                                   bool assign, uint32_t size, bool datamatch)
 589 {
 590     int ret;
 591     struct kvm_ioeventfd iofd = {
 592         .datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0,
 593         .addr = addr,
 594         .len = size,
 595         .flags = 0,
 596         .fd = fd,
 597     };
 598
 599     if (!kvm_enabled()) {
 600         return -ENOSYS;
 601     }
 602
 603     if (datamatch) {
 604         iofd.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH;
 605     }
 606     if (!assign) {
 607         iofd.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
 608     }
 609
 610     ret = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &iofd);
 611
 612     if (ret < 0) {
 613         return -errno;
 614     }
 615
 616     return 0;
 617 }
 618
 619 static int kvm_set_ioeventfd_pio(int fd, uint16_t addr, uint16_t val,
 620                                  bool assign, uint32_t size, bool datamatch)
 621 {
 622     struct kvm_ioeventfd kick = {
 623         .datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0,
 624         .addr = addr,
 625         .flags = KVM_IOEVENTFD_FLAG_PIO,
 626         .len = size,
 627         .fd = fd,
 628     };
 629     int r;
 630     if (!kvm_enabled()) {
 631         return -ENOSYS;
 632     }
 633     if (datamatch) {
 634         kick.flags |= KVM_IOEVENTFD_FLAG_DATAMATCH;
 635     }
 636     if (!assign) {
 637         kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
 638     }
 639     r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
 640     if (r < 0) {
 641         return r;
 642     }
 643     return 0;
 644 }
 645
 646
 647 static int kvm_check_many_ioeventfds(void)
 648 {
 649     /* Userspace can use ioeventfd for io notification.  This requires a host
 650      * that supports eventfd(2) and an I/O thread; since eventfd does not
 651      * support SIGIO it cannot interrupt the vcpu.
 652      *
 653      * Older kernels have a 6 device limit on the KVM io bus.  Find out so we
 654      * can avoid creating too many ioeventfds.
 655      */
 656 #if defined(CONFIG_EVENTFD)
 657     int ioeventfds[7];
 658     int i, ret = 0;
 659     for (i = 0; i < ARRAY_SIZE(ioeventfds); i++) {
 660         ioeventfds[i] = eventfd(0, EFD_CLOEXEC);
 661         if (ioeventfds[i] < 0) {
 662             break;
 663         }
 664         ret = kvm_set_ioeventfd_pio(ioeventfds[i], 0, i, true, 2, true);
 665         if (ret < 0) {
 666             close(ioeventfds[i]);
 667             break;
 668         }
 669     }
 670
 671     /* Decide whether many devices are supported or not */
 672     ret = i == ARRAY_SIZE(ioeventfds);
 673
 674     while (i-- > 0) {
 675         kvm_set_ioeventfd_pio(ioeventfds[i], 0, i, false, 2, true);
 676         close(ioeventfds[i]);
 677     }
 678     return ret;
 679 #else
 680     return 0;
 681 #endif
 682 }
 683
 684 static const KVMCapabilityInfo *
 685 kvm_check_extension_list(KVMState *s, const KVMCapabilityInfo *list)
 686 {
 687     while (list->name) {
 688         if (!kvm_check_extension(s, list->value)) {
 689             return list;
 690         }
 691         list++;
 692     }
 693     return NULL;
 694 }
 695
 696 static void kvm_set_phys_mem(KVMMemoryListener *kml,
 697                              MemoryRegionSection *section, bool add)
 698 {
 699     KVMState *s = kvm_state;
 700     KVMSlot *mem, old;
 701     int err;
 702     MemoryRegion *mr = section->mr;
 703     bool writeable = !mr->readonly && !mr->rom_device;
 704     hwaddr start_addr = section->offset_within_address_space;
 705     ram_addr_t size = int128_get64(section->size);
 706     void *ram = NULL;
 707     unsigned delta;
 708
 709     /* kvm works in page size chunks, but the function may be called
 710        with sub-page size and unaligned start address. Pad the start
 711        address to next and truncate size to previous page boundary. */
 712     delta = qemu_real_host_page_size - (start_addr & ~qemu_real_host_page_mask);
 713     delta &= ~qemu_real_host_page_mask;
 714     if (delta > size) {
 715         return;
 716     }
 717     start_addr += delta;
 718     size -= delta;
 719     size &= qemu_real_host_page_mask;
 720     if (!size || (start_addr & ~qemu_real_host_page_mask)) {
 721         return;
 722     }
 723
 724     if (!memory_region_is_ram(mr)) {
 725         if (writeable || !kvm_readonly_mem_allowed) {
 726             return;
 727         } else if (!mr->romd_mode) {
 728             /* If the memory device is not in romd_mode, then we actually want
 729              * to remove the kvm memory slot so all accesses will trap. */
 730             add = false;
 731         }
 732     }
 733
 734     ram = memory_region_get_ram_ptr(mr) + section->offset_within_region + delta;
 735
 736     while (1) {
 737         mem = kvm_lookup_overlapping_slot(kml, start_addr, start_addr + size);
 738         if (!mem) {
 739             break;
 740         }
 741
 742         if (add && start_addr >= mem->start_addr &&
 743             (start_addr + size <= mem->start_addr + mem->memory_size) &&
 744             (ram - start_addr == mem->ram - mem->start_addr)) {
 745             /* The new slot fits into the existing one and comes with
 746              * identical parameters - update flags and done. */
 747             kvm_slot_update_flags(kml, mem, mr);
 748             return;
 749         }
 750
 751         old = *mem;
 752
 753         if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) {
 754             kvm_physical_sync_dirty_bitmap(kml, section);
 755         }
 756
 757         /* unregister the overlapping slot */
 758         mem->memory_size = 0;
 759         err = kvm_set_user_memory_region(kml, mem);
 760         if (err) {
 761             fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
 762                     __func__, strerror(-err));
 763             abort();
 764         }
 765
 766         /* Workaround for older KVM versions: we can't join slots, even not by
 767          * unregistering the previous ones and then registering the larger
 768          * slot. We have to maintain the existing fragmentation. Sigh.
 769          *
 770          * This workaround assumes that the new slot starts at the same
 771          * address as the first existing one. If not or if some overlapping
 772          * slot comes around later, we will fail (not seen in practice so far)
 773          * - and actually require a recent KVM version. */
 774         if (s->broken_set_mem_region &&
 775             old.start_addr == start_addr && old.memory_size < size && add) {
 776             mem = kvm_alloc_slot(kml);
 777             mem->memory_size = old.memory_size;
 778             mem->start_addr = old.start_addr;
 779             mem->ram = old.ram;
 780             mem->flags = kvm_mem_flags(mr);
 781
 782             err = kvm_set_user_memory_region(kml, mem);
 783             if (err) {
 784                 fprintf(stderr, "%s: error updating slot: %s\n", __func__,
 785                         strerror(-err));
 786                 abort();
 787             }
 788
 789             start_addr += old.memory_size;
 790             ram += old.memory_size;
 791             size -= old.memory_size;
 792             continue;
 793         }
 794
 795         /* register prefix slot */
 796         if (old.start_addr < start_addr) {
 797             mem = kvm_alloc_slot(kml);
 798             mem->memory_size = start_addr - old.start_addr;
 799             mem->start_addr = old.start_addr;
 800             mem->ram = old.ram;
 801             mem->flags =  kvm_mem_flags(mr);
 802
 803             err = kvm_set_user_memory_region(kml, mem);
 804             if (err) {
 805                 fprintf(stderr, "%s: error registering prefix slot: %s\n",
 806                         __func__, strerror(-err));
 807 #ifdef TARGET_PPC
 808                 fprintf(stderr, "%s: This is probably because your kernel's " \
 809                                 "PAGE_SIZE is too big. Please try to use 4k " \
 810                                 "PAGE_SIZE!\n", __func__);
 811 #endif
 812                 abort();
 813             }
 814         }
 815
 816         /* register suffix slot */
 817         if (old.start_addr + old.memory_size > start_addr + size) {
 818             ram_addr_t size_delta;
 819
 820             mem = kvm_alloc_slot(kml);
 821             mem->start_addr = start_addr + size;
 822             size_delta = mem->start_addr - old.start_addr;
 823             mem->memory_size = old.memory_size - size_delta;
 824             mem->ram = old.ram + size_delta;
 825             mem->flags = kvm_mem_flags(mr);
 826
 827             err = kvm_set_user_memory_region(kml, mem);
 828             if (err) {
 829                 fprintf(stderr, "%s: error registering suffix slot: %s\n",
 830                         __func__, strerror(-err));
 831                 abort();
 832             }
 833         }
 834     }
 835
 836     /* in case the KVM bug workaround already "consumed" the new slot */
 837     if (!size) {
 838         return;
 839     }
 840     if (!add) {
 841         return;
 842     }
 843     mem = kvm_alloc_slot(kml);
 844     mem->memory_size = size;
 845     mem->start_addr = start_addr;
 846     mem->ram = ram;
 847     mem->flags = kvm_mem_flags(mr);
 848
 849     err = kvm_set_user_memory_region(kml, mem);
 850     if (err) {
 851         fprintf(stderr, "%s: error registering slot: %s\n", __func__,
 852                 strerror(-err));
 853         abort();
 854     }
 855 }
 856
 857 static void kvm_region_add(MemoryListener *listener,
 858                            MemoryRegionSection *section)
 859 {
 860     KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
 861
 862     memory_region_ref(section->mr);
 863     kvm_set_phys_mem(kml, section, true);
 864 }
 865
 866 static void kvm_region_del(MemoryListener *listener,
 867                            MemoryRegionSection *section)
 868 {
 869     KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
 870
 871     kvm_set_phys_mem(kml, section, false);
 872     memory_region_unref(section->mr);
 873 }
 874
 875 static void kvm_log_sync(MemoryListener *listener,
 876                          MemoryRegionSection *section)
 877 {
 878     KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
 879     int r;
 880
 881     r = kvm_physical_sync_dirty_bitmap(kml, section);
 882     if (r < 0) {
 883         abort();
 884     }
 885 }
 886
 887 static void kvm_mem_ioeventfd_add(MemoryListener *listener,
 888                                   MemoryRegionSection *section,
 889                                   bool match_data, uint64_t data,
 890                                   EventNotifier *e)
 891 {
 892     int fd = event_notifier_get_fd(e);
 893     int r;
 894
 895     r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
 896                                data, true, int128_get64(section->size),
 897                                match_data);
 898     if (r < 0) {
 899         fprintf(stderr, "%s: error adding ioeventfd: %s\n",
 900                 __func__, strerror(-r));
 901         abort();
 902     }
 903 }
 904
 905 static void kvm_mem_ioeventfd_del(MemoryListener *listener,
 906                                   MemoryRegionSection *section,
 907                                   bool match_data, uint64_t data,
 908                                   EventNotifier *e)
 909 {
 910     int fd = event_notifier_get_fd(e);
 911     int r;
 912
 913     r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
 914                                data, false, int128_get64(section->size),
 915                                match_data);
 916     if (r < 0) {
 917         abort();
 918     }
 919 }
 920
 921 static void kvm_io_ioeventfd_add(MemoryListener *listener,
 922                                  MemoryRegionSection *section,
 923                                  bool match_data, uint64_t data,
 924                                  EventNotifier *e)
 925 {
 926     int fd = event_notifier_get_fd(e);
 927     int r;
 928
 929     r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space,
 930                               data, true, int128_get64(section->size),
 931                               match_data);
 932     if (r < 0) {
 933         fprintf(stderr, "%s: error adding ioeventfd: %s\n",
 934                 __func__, strerror(-r));
 935         abort();
 936     }
 937 }
 938
 939 static void kvm_io_ioeventfd_del(MemoryListener *listener,
 940                                  MemoryRegionSection *section,
 941                                  bool match_data, uint64_t data,
 942                                  EventNotifier *e)
 943
 944 {
 945     int fd = event_notifier_get_fd(e);
 946     int r;
 947
 948     r = kvm_set_ioeventfd_pio(fd, section->offset_within_address_space,
 949                               data, false, int128_get64(section->size),
 950                               match_data);
 951     if (r < 0) {
 952         abort();
 953     }
 954 }
 955
 956 void kvm_memory_listener_register(KVMState *s, KVMMemoryListener *kml,
 957                                   AddressSpace *as, int as_id)
 958 {
 959     int i;
 960
 961     kml->slots = g_malloc0(s->nr_slots * sizeof(KVMSlot));
 962     kml->as_id = as_id;
 963
 964     for (i = 0; i < s->nr_slots; i++) {
 965         kml->slots[i].slot = i;
 966     }
 967
 968     kml->listener.region_add = kvm_region_add;
 969     kml->listener.region_del = kvm_region_del;
 970     kml->listener.log_start = kvm_log_start;
 971     kml->listener.log_stop = kvm_log_stop;
 972     kml->listener.log_sync = kvm_log_sync;
 973     kml->listener.priority = 10;
 974
 975     memory_listener_register(&kml->listener, as);
 976 }
 977
 978 static MemoryListener kvm_io_listener = {
 979     .eventfd_add = kvm_io_ioeventfd_add,
 980     .eventfd_del = kvm_io_ioeventfd_del,
 981     .priority = 10,
 982 };
 983
 984 int kvm_set_irq(KVMState *s, int irq, int level)
 985 {
 986     struct kvm_irq_level event;
 987     int ret;
 988
 989     assert(kvm_async_interrupts_enabled());
 990
 991     event.level = level;
 992     event.irq = irq;
 993     ret = kvm_vm_ioctl(s, s->irq_set_ioctl, &event);
 994     if (ret < 0) {
 995         perror("kvm_set_irq");
 996         abort();
 997     }
 998
 999     return (s->irq_set_ioctl == KVM_IRQ_LINE) ? 1 : event.status;
1000 }
1001
1002 #ifdef KVM_CAP_IRQ_ROUTING
1003 typedef struct KVMMSIRoute {
1004     struct kvm_irq_routing_entry kroute;
1005     QTAILQ_ENTRY(KVMMSIRoute) entry;
1006 } KVMMSIRoute;
1007
1008 static void set_gsi(KVMState *s, unsigned int gsi)
1009 {
1010     set_bit(gsi, s->used_gsi_bitmap);
1011 }
1012
1013 static void clear_gsi(KVMState *s, unsigned int gsi)
1014 {
1015     clear_bit(gsi, s->used_gsi_bitmap);
1016 }
1017
1018 void kvm_init_irq_routing(KVMState *s)
1019 {
1020     int gsi_count, i;
1021
1022     gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING) - 1;
1023     if (gsi_count > 0) {
1024         /* Round up so we can search ints using ffs */
1025         s->used_gsi_bitmap = bitmap_new(gsi_count);
1026         s->gsi_count = gsi_count;
1027     }
1028
1029     s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
1030     s->nr_allocated_irq_routes = 0;
1031
1032     if (!kvm_direct_msi_allowed) {
1033         for (i = 0; i < KVM_MSI_HASHTAB_SIZE; i++) {
1034             QTAILQ_INIT(&s->msi_hashtab[i]);
1035         }
1036     }
1037
1038     kvm_arch_init_irq_routing(s);
1039 }
1040
1041 void kvm_irqchip_commit_routes(KVMState *s)
1042 {
1043     int ret;
1044
1045     if (kvm_gsi_direct_mapping()) {
1046         return;
1047     }
1048
1049     if (!kvm_gsi_routing_enabled()) {
1050         return;
1051     }
1052
1053     s->irq_routes->flags = 0;
1054     trace_kvm_irqchip_commit_routes();
1055     ret = kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes);
1056     assert(ret == 0);
1057 }
1058
1059 static void kvm_add_routing_entry(KVMState *s,
1060                                   struct kvm_irq_routing_entry *entry)
1061 {
1062     struct kvm_irq_routing_entry *new;
1063     int n, size;
1064
1065     if (s->irq_routes->nr == s->nr_allocated_irq_routes) {
1066         n = s->nr_allocated_irq_routes * 2;
1067         if (n < 64) {
1068             n = 64;
1069         }
1070         size = sizeof(struct kvm_irq_routing);
1071         size += n * sizeof(*new);
1072         s->irq_routes = g_realloc(s->irq_routes, size);
1073         s->nr_allocated_irq_routes = n;
1074     }
1075     n = s->irq_routes->nr++;
1076     new = &s->irq_routes->entries[n];
1077
1078     *new = *entry;
1079
1080     set_gsi(s, entry->gsi);
1081 }
1082
1083 static int kvm_update_routing_entry(KVMState *s,
1084                                     struct kvm_irq_routing_entry *new_entry)
1085 {
1086     struct kvm_irq_routing_entry *entry;
1087     int n;
1088
1089     for (n = 0; n < s->irq_routes->nr; n++) {
1090         entry = &s->irq_routes->entries[n];
1091         if (entry->gsi != new_entry->gsi) {
1092             continue;
1093         }
1094
1095         if(!memcmp(entry, new_entry, sizeof *entry)) {
1096             return 0;
1097         }
1098
1099         *entry = *new_entry;
1100
1101         return 0;
1102     }
1103
1104     return -ESRCH;
1105 }
1106
1107 void kvm_irqchip_add_irq_route(KVMState *s, int irq, int irqchip, int pin)
1108 {
1109     struct kvm_irq_routing_entry e = {};
1110
1111     assert(pin < s->gsi_count);
1112
1113     e.gsi = irq;
1114     e.type = KVM_IRQ_ROUTING_IRQCHIP;
1115     e.flags = 0;
1116     e.u.irqchip.irqchip = irqchip;
1117     e.u.irqchip.pin = pin;
1118     kvm_add_routing_entry(s, &e);
1119 }
1120
1121 void kvm_irqchip_release_virq(KVMState *s, int virq)
1122 {
1123     struct kvm_irq_routing_entry *e;
1124     int i;
1125
1126     if (kvm_gsi_direct_mapping()) {
1127         return;
1128     }
1129
1130     for (i = 0; i < s->irq_routes->nr; i++) {
1131         e = &s->irq_routes->entries[i];
1132         if (e->gsi == virq) {
1133             s->irq_routes->nr--;
1134             *e = s->irq_routes->entries[s->irq_routes->nr];
1135         }
1136     }
1137     clear_gsi(s, virq);
1138     kvm_arch_release_virq_post(virq);
1139     trace_kvm_irqchip_release_virq(virq);
1140 }
1141
1142 static unsigned int kvm_hash_msi(uint32_t data)
1143 {
1144     /* This is optimized for IA32 MSI layout. However, no other arch shall
1145      * repeat the mistake of not providing a direct MSI injection API. */
1146     return data & 0xff;
1147 }
1148
1149 static void kvm_flush_dynamic_msi_routes(KVMState *s)
1150 {
1151     KVMMSIRoute *route, *next;
1152     unsigned int hash;
1153
1154     for (hash = 0; hash < KVM_MSI_HASHTAB_SIZE; hash++) {
1155         QTAILQ_FOREACH_SAFE(route, &s->msi_hashtab[hash], entry, next) {
1156             kvm_irqchip_release_virq(s, route->kroute.gsi);
1157             QTAILQ_REMOVE(&s->msi_hashtab[hash], route, entry);
1158             g_free(route);
1159         }
1160     }
1161 }
1162
1163 static int kvm_irqchip_get_virq(KVMState *s)
1164 {
1165     int next_virq;
1166
1167     /*
1168      * PIC and IOAPIC share the first 16 GSI numbers, thus the available
1169      * GSI numbers are more than the number of IRQ route. Allocating a GSI
1170      * number can succeed even though a new route entry cannot be added.
1171      * When this happens, flush dynamic MSI entries to free IRQ route entries.
1172      */
1173     if (!kvm_direct_msi_allowed && s->irq_routes->nr == s->gsi_count) {
1174         kvm_flush_dynamic_msi_routes(s);
1175     }
1176
1177     /* Return the lowest unused GSI in the bitmap */
1178     next_virq = find_first_zero_bit(s->used_gsi_bitmap, s->gsi_count);
1179     if (next_virq >= s->gsi_count) {
1180         return -ENOSPC;
1181     } else {
1182         return next_virq;
1183     }
1184 }
1185
1186 static KVMMSIRoute *kvm_lookup_msi_route(KVMState *s, MSIMessage msg)
1187 {
1188     unsigned int hash = kvm_hash_msi(msg.data);
1189     KVMMSIRoute *route;
1190
1191     QTAILQ_FOREACH(route, &s->msi_hashtab[hash], entry) {
1192         if (route->kroute.u.msi.address_lo == (uint32_t)msg.address &&
1193             route->kroute.u.msi.address_hi == (msg.address >> 32) &&
1194             route->kroute.u.msi.data == le32_to_cpu(msg.data)) {
1195             return route;
1196         }
1197     }
1198     return NULL;
1199 }
1200
1201 int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
1202 {
1203     struct kvm_msi msi;
1204     KVMMSIRoute *route;
1205
1206     if (kvm_direct_msi_allowed) {
1207         msi.address_lo = (uint32_t)msg.address;
1208         msi.address_hi = msg.address >> 32;
1209         msi.data = le32_to_cpu(msg.data);
1210         msi.flags = 0;
1211         memset(msi.pad, 0, sizeof(msi.pad));
1212
1213         return kvm_vm_ioctl(s, KVM_SIGNAL_MSI, &msi);
1214     }
1215
1216     route = kvm_lookup_msi_route(s, msg);
1217     if (!route) {
1218         int virq;
1219
1220         virq = kvm_irqchip_get_virq(s);
1221         if (virq < 0) {
1222             return virq;
1223         }
1224
1225         route = g_malloc0(sizeof(KVMMSIRoute));
1226         route->kroute.gsi = virq;
1227         route->kroute.type = KVM_IRQ_ROUTING_MSI;
1228         route->kroute.flags = 0;
1229         route->kroute.u.msi.address_lo = (uint32_t)msg.address;
1230         route->kroute.u.msi.address_hi = msg.address >> 32;
1231         route->kroute.u.msi.data = le32_to_cpu(msg.data);
1232
1233         kvm_add_routing_entry(s, &route->kroute);
1234         kvm_irqchip_commit_routes(s);
1235
1236         QTAILQ_INSERT_TAIL(&s->msi_hashtab[kvm_hash_msi(msg.data)], route,
1237                            entry);
1238     }
1239
1240     assert(route->kroute.type == KVM_IRQ_ROUTING_MSI);
1241
1242     return kvm_set_irq(s, route->kroute.gsi, 1);
1243 }
1244
1245 int kvm_irqchip_add_msi_route(KVMState *s, int vector, PCIDevice *dev)
1246 {
1247     struct kvm_irq_routing_entry kroute = {};
1248     int virq;
1249     MSIMessage msg = {0, 0};
1250
1251     if (pci_available && dev) {
1252         msg = pci_get_msi_message(dev, vector);
1253     }
1254
1255     if (kvm_gsi_direct_mapping()) {
1256         return kvm_arch_msi_data_to_gsi(msg.data);
1257     }
1258
1259     if (!kvm_gsi_routing_enabled()) {
1260         return -ENOSYS;
1261     }
1262
1263     virq = kvm_irqchip_get_virq(s);
1264     if (virq < 0) {
1265         return virq;
1266     }
1267
1268     kroute.gsi = virq;
1269     kroute.type = KVM_IRQ_ROUTING_MSI;
1270     kroute.flags = 0;
1271     kroute.u.msi.address_lo = (uint32_t)msg.address;
1272     kroute.u.msi.address_hi = msg.address >> 32;
1273     kroute.u.msi.data = le32_to_cpu(msg.data);
1274     if (pci_available && kvm_msi_devid_required()) {
1275         kroute.flags = KVM_MSI_VALID_DEVID;
1276         kroute.u.msi.devid = pci_requester_id(dev);
1277     }
1278     if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) {
1279         kvm_irqchip_release_virq(s, virq);
1280         return -EINVAL;
1281     }
1282
1283     trace_kvm_irqchip_add_msi_route(dev ? dev->name : (char *)"N/A",
1284                                     vector, virq);
1285
1286     kvm_add_routing_entry(s, &kroute);
1287     kvm_arch_add_msi_route_post(&kroute, vector, dev);
1288     kvm_irqchip_commit_routes(s);
1289
1290     return virq;
1291 }
1292
1293 int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg,
1294                                  PCIDevice *dev)
1295 {
1296     struct kvm_irq_routing_entry kroute = {};
1297
1298     if (kvm_gsi_direct_mapping()) {
1299         return 0;
1300     }
1301
1302     if (!kvm_irqchip_in_kernel()) {
1303         return -ENOSYS;
1304     }
1305
1306     kroute.gsi = virq;
1307     kroute.type = KVM_IRQ_ROUTING_MSI;
1308     kroute.flags = 0;
1309     kroute.u.msi.address_lo = (uint32_t)msg.address;
1310     kroute.u.msi.address_hi = msg.address >> 32;
1311     kroute.u.msi.data = le32_to_cpu(msg.data);
1312     if (pci_available && kvm_msi_devid_required()) {
1313         kroute.flags = KVM_MSI_VALID_DEVID;
1314         kroute.u.msi.devid = pci_requester_id(dev);
1315     }
1316     if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) {
1317         return -EINVAL;
1318     }
1319
1320     trace_kvm_irqchip_update_msi_route(virq);
1321
1322     return kvm_update_routing_entry(s, &kroute);
1323 }
1324
1325 static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int rfd, int virq,
1326                                     bool assign)
1327 {
1328     struct kvm_irqfd irqfd = {
1329         .fd = fd,
1330         .gsi = virq,
1331         .flags = assign ? 0 : KVM_IRQFD_FLAG_DEASSIGN,
1332     };
1333
1334     if (rfd != -1) {
1335         irqfd.flags |= KVM_IRQFD_FLAG_RESAMPLE;
1336         irqfd.resamplefd = rfd;
1337     }
1338
1339     if (!kvm_irqfds_enabled()) {
1340         return -ENOSYS;
1341     }
1342
1343     return kvm_vm_ioctl(s, KVM_IRQFD, &irqfd);
1344 }
1345
1346 int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
1347 {
1348     struct kvm_irq_routing_entry kroute = {};
1349     int virq;
1350
1351     if (!kvm_gsi_routing_enabled()) {
1352         return -ENOSYS;
1353     }
1354
1355     virq = kvm_irqchip_get_virq(s);
1356     if (virq < 0) {
1357         return virq;
1358     }
1359
1360     kroute.gsi = virq;
1361     kroute.type = KVM_IRQ_ROUTING_S390_ADAPTER;
1362     kroute.flags = 0;
1363     kroute.u.adapter.summary_addr = adapter->summary_addr;
1364     kroute.u.adapter.ind_addr = adapter->ind_addr;
1365     kroute.u.adapter.summary_offset = adapter->summary_offset;
1366     kroute.u.adapter.ind_offset = adapter->ind_offset;
1367     kroute.u.adapter.adapter_id = adapter->adapter_id;
1368
1369     kvm_add_routing_entry(s, &kroute);
1370
1371     return virq;
1372 }
1373
1374 int kvm_irqchip_add_hv_sint_route(KVMState *s, uint32_t vcpu, uint32_t sint)
1375 {
1376     struct kvm_irq_routing_entry kroute = {};
1377     int virq;
1378
1379     if (!kvm_gsi_routing_enabled()) {
1380         return -ENOSYS;
1381     }
1382     if (!kvm_check_extension(s, KVM_CAP_HYPERV_SYNIC)) {
1383         return -ENOSYS;
1384     }
1385     virq = kvm_irqchip_get_virq(s);
1386     if (virq < 0) {
1387         return virq;
1388     }
1389
1390     kroute.gsi = virq;
1391     kroute.type = KVM_IRQ_ROUTING_HV_SINT;
1392     kroute.flags = 0;
1393     kroute.u.hv_sint.vcpu = vcpu;
1394     kroute.u.hv_sint.sint = sint;
1395
1396     kvm_add_routing_entry(s, &kroute);
1397     kvm_irqchip_commit_routes(s);
1398
1399     return virq;
1400 }
1401
1402 #else /* !KVM_CAP_IRQ_ROUTING */
1403
1404 void kvm_init_irq_routing(KVMState *s)
1405 {
1406 }
1407
1408 void kvm_irqchip_release_virq(KVMState *s, int virq)
1409 {
1410 }
1411
1412 int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
1413 {
1414     abort();
1415 }
1416
1417 int kvm_irqchip_add_msi_route(KVMState *s, int vector, PCIDevice *dev)
1418 {
1419     return -ENOSYS;
1420 }
1421
1422 int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
1423 {
1424     return -ENOSYS;
1425 }
1426
1427 int kvm_irqchip_add_hv_sint_route(KVMState *s, uint32_t vcpu, uint32_t sint)
1428 {
1429     return -ENOSYS;
1430 }
1431
1432 static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
1433 {
1434     abort();
1435 }
1436
1437 int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
1438 {
1439     return -ENOSYS;
1440 }
1441 #endif /* !KVM_CAP_IRQ_ROUTING */
1442
1443 int kvm_irqchip_add_irqfd_notifier_gsi(KVMState *s, EventNotifier *n,
1444                                        EventNotifier *rn, int virq)
1445 {
1446     return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n),
1447            rn ? event_notifier_get_fd(rn) : -1, virq, true);
1448 }
1449
1450 int kvm_irqchip_remove_irqfd_notifier_gsi(KVMState *s, EventNotifier *n,
1451                                           int virq)
1452 {
1453     return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n), -1, virq,
1454            false);
1455 }
1456
1457 int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n,
1458                                    EventNotifier *rn, qemu_irq irq)
1459 {
1460     gpointer key, gsi;
1461     gboolean found = g_hash_table_lookup_extended(s->gsimap, irq, &key, &gsi);
1462
1463     if (!found) {
1464         return -ENXIO;
1465     }
1466     return kvm_irqchip_add_irqfd_notifier_gsi(s, n, rn, GPOINTER_TO_INT(gsi));
1467 }
1468
1469 int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n,
1470                                       qemu_irq irq)
1471 {
1472     gpointer key, gsi;
1473     gboolean found = g_hash_table_lookup_extended(s->gsimap, irq, &key, &gsi);
1474
1475     if (!found) {
1476         return -ENXIO;
1477     }
1478     return kvm_irqchip_remove_irqfd_notifier_gsi(s, n, GPOINTER_TO_INT(gsi));
1479 }
1480
1481 void kvm_irqchip_set_qemuirq_gsi(KVMState *s, qemu_irq irq, int gsi)
1482 {
1483     g_hash_table_insert(s->gsimap, irq, GINT_TO_POINTER(gsi));
1484 }
1485
1486 static void kvm_irqchip_create(MachineState *machine, KVMState *s)
1487 {
1488     int ret;
1489
1490     if (kvm_check_extension(s, KVM_CAP_IRQCHIP)) {
1491         ;
1492     } else if (kvm_check_extension(s, KVM_CAP_S390_IRQCHIP)) {
1493         ret = kvm_vm_enable_cap(s, KVM_CAP_S390_IRQCHIP, 0);
1494         if (ret < 0) {
1495             fprintf(stderr, "Enable kernel irqchip failed: %s\n", strerror(-ret));
1496             exit(1);
1497         }
1498     } else {
1499         return;
1500     }
1501
1502     /* First probe and see if there's a arch-specific hook to create the
1503      * in-kernel irqchip for us */
1504     ret = kvm_arch_irqchip_create(machine, s);
1505     if (ret == 0) {
1506         if (machine_kernel_irqchip_split(machine)) {
1507             perror("Split IRQ chip mode not supported.");
1508             exit(1);
1509         } else {
1510             ret = kvm_vm_ioctl(s, KVM_CREATE_IRQCHIP);
1511         }
1512     }
1513     if (ret < 0) {
1514         fprintf(stderr, "Create kernel irqchip failed: %s\n", strerror(-ret));
1515         exit(1);
1516     }
1517
1518     kvm_kernel_irqchip = true;
1519     /* If we have an in-kernel IRQ chip then we must have asynchronous
1520      * interrupt delivery (though the reverse is not necessarily true)
1521      */
1522     kvm_async_interrupts_allowed = true;
1523     kvm_halt_in_kernel_allowed = true;
1524
1525     kvm_init_irq_routing(s);
1526
1527     s->gsimap = g_hash_table_new(g_direct_hash, g_direct_equal);
1528 }
1529
1530 /* Find number of supported CPUs using the recommended
1531  * procedure from the kernel API documentation to cope with
1532  * older kernels that may be missing capabilities.
1533  */
1534 static int kvm_recommended_vcpus(KVMState *s)
1535 {
1536     int ret = kvm_check_extension(s, KVM_CAP_NR_VCPUS);
1537     return (ret) ? ret : 4;
1538 }
1539
1540 static int kvm_max_vcpus(KVMState *s)
1541 {
1542     int ret = kvm_check_extension(s, KVM_CAP_MAX_VCPUS);
1543     return (ret) ? ret : kvm_recommended_vcpus(s);
1544 }
1545
1546 static int kvm_max_vcpu_id(KVMState *s)
1547 {
1548     int ret = kvm_check_extension(s, KVM_CAP_MAX_VCPU_ID);
1549     return (ret) ? ret : kvm_max_vcpus(s);
1550 }
1551
1552 bool kvm_vcpu_id_is_valid(int vcpu_id)
1553 {
1554     KVMState *s = KVM_STATE(current_machine->accelerator);
1555     return vcpu_id >= 0 && vcpu_id < kvm_max_vcpu_id(s);
1556 }
1557
1558 static int kvm_init(MachineState *ms)
1559 {
1560     MachineClass *mc = MACHINE_GET_CLASS(ms);
1561     static const char upgrade_note[] =
1562         "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
1563         "(see http://sourceforge.net/projects/kvm).\n";
1564     struct {
1565         const char *name;
1566         int num;
1567     } num_cpus[] = {
1568         { "SMP",          smp_cpus },
1569         { "hotpluggable", max_cpus },
1570         { NULL, }
1571     }, *nc = num_cpus;
1572     int soft_vcpus_limit, hard_vcpus_limit;
1573     KVMState *s;
1574     const KVMCapabilityInfo *missing_cap;
1575     int ret;
1576     int type = 0;
1577     const char *kvm_type;
1578
1579     s = KVM_STATE(ms->accelerator);
1580
1581     /*
1582      * On systems where the kernel can support different base page
1583      * sizes, host page size may be different from TARGET_PAGE_SIZE,
1584      * even with KVM.  TARGET_PAGE_SIZE is assumed to be the minimum
1585      * page size for the system though.
1586      */
1587     assert(TARGET_PAGE_SIZE <= getpagesize());
1588
1589     s->sigmask_len = 8;
1590
1591 #ifdef KVM_CAP_SET_GUEST_DEBUG
1592     QTAILQ_INIT(&s->kvm_sw_breakpoints);
1593 #endif
1594     QLIST_INIT(&s->kvm_parked_vcpus);
1595     s->vmfd = -1;
1596     s->fd = qemu_open("/dev/kvm", O_RDWR);
1597     if (s->fd == -1) {
1598         fprintf(stderr, "Could not access KVM kernel module: %m\n");
1599         ret = -errno;
1600         goto err;
1601     }
1602
1603     ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
1604     if (ret < KVM_API_VERSION) {
1605         if (ret >= 0) {
1606             ret = -EINVAL;
1607         }
1608         fprintf(stderr, "kvm version too old\n");
1609         goto err;
1610     }
1611
1612     if (ret > KVM_API_VERSION) {
1613         ret = -EINVAL;
1614         fprintf(stderr, "kvm version not supported\n");
1615         goto err;
1616     }
1617
1618     kvm_immediate_exit = kvm_check_extension(s, KVM_CAP_IMMEDIATE_EXIT);
1619     s->nr_slots = kvm_check_extension(s, KVM_CAP_NR_MEMSLOTS);
1620
1621     /* If unspecified, use the default value */
1622     if (!s->nr_slots) {
1623         s->nr_slots = 32;
1624     }
1625
1626     /* check the vcpu limits */
1627     soft_vcpus_limit = kvm_recommended_vcpus(s);
1628     hard_vcpus_limit = kvm_max_vcpus(s);
1629
1630     while (nc->name) {
1631         if (nc->num > soft_vcpus_limit) {
1632             fprintf(stderr,
1633                     "Warning: Number of %s cpus requested (%d) exceeds "
1634                     "the recommended cpus supported by KVM (%d)\n",
1635                     nc->name, nc->num, soft_vcpus_limit);
1636
1637             if (nc->num > hard_vcpus_limit) {
1638                 fprintf(stderr, "Number of %s cpus requested (%d) exceeds "
1639                         "the maximum cpus supported by KVM (%d)\n",
1640                         nc->name, nc->num, hard_vcpus_limit);
1641                 exit(1);
1642             }
1643         }
1644         nc++;
1645     }
1646
1647     kvm_type = qemu_opt_get(qemu_get_machine_opts(), "kvm-type");
1648     if (mc->kvm_type) {
1649         type = mc->kvm_type(kvm_type);
1650     } else if (kvm_type) {
1651         ret = -EINVAL;
1652         fprintf(stderr, "Invalid argument kvm-type=%s\n", kvm_type);
1653         goto err;
1654     }
1655
1656     do {
1657         ret = kvm_ioctl(s, KVM_CREATE_VM, type);
1658     } while (ret == -EINTR);
1659
1660     if (ret < 0) {
1661         fprintf(stderr, "ioctl(KVM_CREATE_VM) failed: %d %s\n", -ret,
1662                 strerror(-ret));
1663
1664 #ifdef TARGET_S390X
1665         if (ret == -EINVAL) {
1666             fprintf(stderr,
1667                     "Host kernel setup problem detected. Please verify:\n");
1668             fprintf(stderr, "- for kernels supporting the switch_amode or"
1669                     " user_mode parameters, whether\n");
1670             fprintf(stderr,
1671                     "  user space is running in primary address space\n");
1672             fprintf(stderr,
1673                     "- for kernels supporting the vm.allocate_pgste sysctl, "
1674                     "whether it is enabled\n");
1675         }
1676 #endif
1677         goto err;
1678     }
1679
1680     s->vmfd = ret;
1681     missing_cap = kvm_check_extension_list(s, kvm_required_capabilites);
1682     if (!missing_cap) {
1683         missing_cap =
1684             kvm_check_extension_list(s, kvm_arch_required_capabilities);
1685     }
1686     if (missing_cap) {
1687         ret = -EINVAL;
1688         fprintf(stderr, "kvm does not support %s\n%s",
1689                 missing_cap->name, upgrade_note);
1690         goto err;
1691     }
1692
1693     s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
1694
1695     s->broken_set_mem_region = 1;
1696     ret = kvm_check_extension(s, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
1697     if (ret > 0) {
1698         s->broken_set_mem_region = 0;
1699     }
1700
1701 #ifdef KVM_CAP_VCPU_EVENTS
1702     s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
1703 #endif
1704
1705     s->robust_singlestep =
1706         kvm_check_extension(s, KVM_CAP_X86_ROBUST_SINGLESTEP);
1707
1708 #ifdef KVM_CAP_DEBUGREGS
1709     s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS);
1710 #endif
1711
1712 #ifdef KVM_CAP_IRQ_ROUTING
1713     kvm_direct_msi_allowed = (kvm_check_extension(s, KVM_CAP_SIGNAL_MSI) > 0);
1714 #endif
1715
1716     s->intx_set_mask = kvm_check_extension(s, KVM_CAP_PCI_2_3);
1717
1718     s->irq_set_ioctl = KVM_IRQ_LINE;
1719     if (kvm_check_extension(s, KVM_CAP_IRQ_INJECT_STATUS)) {
1720         s->irq_set_ioctl = KVM_IRQ_LINE_STATUS;
1721     }
1722
1723 #ifdef KVM_CAP_READONLY_MEM
1724     kvm_readonly_mem_allowed =
1725         (kvm_check_extension(s, KVM_CAP_READONLY_MEM) > 0);
1726 #endif
1727
1728     kvm_eventfds_allowed =
1729         (kvm_check_extension(s, KVM_CAP_IOEVENTFD) > 0);
1730
1731     kvm_irqfds_allowed =
1732         (kvm_check_extension(s, KVM_CAP_IRQFD) > 0);
1733
1734     kvm_resamplefds_allowed =
1735         (kvm_check_extension(s, KVM_CAP_IRQFD_RESAMPLE) > 0);
1736
1737     kvm_vm_attributes_allowed =
1738         (kvm_check_extension(s, KVM_CAP_VM_ATTRIBUTES) > 0);
1739
1740     kvm_ioeventfd_any_length_allowed =
1741         (kvm_check_extension(s, KVM_CAP_IOEVENTFD_ANY_LENGTH) > 0);
1742
1743     kvm_state = s;
1744
1745     ret = kvm_arch_init(ms, s);
1746     if (ret < 0) {
1747         goto err;
1748     }
1749
1750     if (machine_kernel_irqchip_allowed(ms)) {
1751         kvm_irqchip_create(ms, s);
1752     }
1753
1754     if (kvm_eventfds_allowed) {
1755         s->memory_listener.listener.eventfd_add = kvm_mem_ioeventfd_add;
1756         s->memory_listener.listener.eventfd_del = kvm_mem_ioeventfd_del;
1757     }
1758     s->memory_listener.listener.coalesced_mmio_add = kvm_coalesce_mmio_region;
1759     s->memory_listener.listener.coalesced_mmio_del = kvm_uncoalesce_mmio_region;
1760
1761     kvm_memory_listener_register(s, &s->memory_listener,
1762                                  &address_space_memory, 0);
1763     memory_listener_register(&kvm_io_listener,
1764                              &address_space_io);
1765
1766     s->many_ioeventfds = kvm_check_many_ioeventfds();
1767
1768     return 0;
1769
1770 err:
1771     assert(ret < 0);
1772     if (s->vmfd >= 0) {
1773         close(s->vmfd);
1774     }
1775     if (s->fd != -1) {
1776         close(s->fd);
1777     }
1778     g_free(s->memory_listener.slots);
1779
1780     return ret;
1781 }
1782
1783 void kvm_set_sigmask_len(KVMState *s, unsigned int sigmask_len)
1784 {
1785     s->sigmask_len = sigmask_len;
1786 }
1787
1788 static void kvm_handle_io(uint16_t port, MemTxAttrs attrs, void *data, int direction,
1789                           int size, uint32_t count)
1790 {
1791     int i;
1792     uint8_t *ptr = data;
1793
1794     for (i = 0; i < count; i++) {
1795         address_space_rw(&address_space_io, port, attrs,
1796                          ptr, size,
1797                          direction == KVM_EXIT_IO_OUT);
1798         ptr += size;
1799     }
1800 }
1801
1802 static int kvm_handle_internal_error(CPUState *cpu, struct kvm_run *run)
1803 {
1804     fprintf(stderr, "KVM internal error. Suberror: %d\n",
1805             run->internal.suberror);
1806
1807     if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) {
1808         int i;
1809
1810         for (i = 0; i < run->internal.ndata; ++i) {
1811             fprintf(stderr, "extra data[%d]: %"PRIx64"\n",
1812                     i, (uint64_t)run->internal.data[i]);
1813         }
1814     }
1815     if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) {
1816         fprintf(stderr, "emulation failure\n");
1817         if (!kvm_arch_stop_on_emulation_error(cpu)) {
1818             cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_CODE);
1819             return EXCP_INTERRUPT;
1820         }
1821     }
1822     /* FIXME: Should trigger a qmp message to let management know
1823      * something went wrong.
1824      */
1825     return -1;
1826 }
1827
1828 void kvm_flush_coalesced_mmio_buffer(void)
1829 {
1830     KVMState *s = kvm_state;
1831
1832     if (s->coalesced_flush_in_progress) {
1833         return;
1834     }
1835
1836     s->coalesced_flush_in_progress = true;
1837
1838     if (s->coalesced_mmio_ring) {
1839         struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
1840         while (ring->first != ring->last) {
1841             struct kvm_coalesced_mmio *ent;
1842
1843             ent = &ring->coalesced_mmio[ring->first];
1844
1845             cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
1846             smp_wmb();
1847             ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
1848         }
1849     }
1850
1851     s->coalesced_flush_in_progress = false;
1852 }
1853
1854 static void do_kvm_cpu_synchronize_state(CPUState *cpu, run_on_cpu_data arg)
1855 {
1856     if (!cpu->vcpu_dirty) {
1857         kvm_arch_get_registers(cpu);
1858         cpu->vcpu_dirty = true;
1859     }
1860 }
1861
1862 void kvm_cpu_synchronize_state(CPUState *cpu)
1863 {
1864     if (!cpu->vcpu_dirty) {
1865         run_on_cpu(cpu, do_kvm_cpu_synchronize_state, RUN_ON_CPU_NULL);
1866     }
1867 }
1868
1869 static void do_kvm_cpu_synchronize_post_reset(CPUState *cpu, run_on_cpu_data arg)
1870 {
1871     kvm_arch_put_registers(cpu, KVM_PUT_RESET_STATE);
1872     cpu->vcpu_dirty = false;
1873 }
1874
1875 void kvm_cpu_synchronize_post_reset(CPUState *cpu)
1876 {
1877     run_on_cpu(cpu, do_kvm_cpu_synchronize_post_reset, RUN_ON_CPU_NULL);
1878 }
1879
1880 static void do_kvm_cpu_synchronize_post_init(CPUState *cpu, run_on_cpu_data arg)
1881 {
1882     kvm_arch_put_registers(cpu, KVM_PUT_FULL_STATE);
1883     cpu->vcpu_dirty = false;
1884 }
1885
1886 void kvm_cpu_synchronize_post_init(CPUState *cpu)
1887 {
1888     run_on_cpu(cpu, do_kvm_cpu_synchronize_post_init, RUN_ON_CPU_NULL);
1889 }
1890
1891 static void do_kvm_cpu_synchronize_pre_loadvm(CPUState *cpu, run_on_cpu_data arg)
1892 {
1893     cpu->vcpu_dirty = true;
1894 }
1895
1896 void kvm_cpu_synchronize_pre_loadvm(CPUState *cpu)
1897 {
1898     run_on_cpu(cpu, do_kvm_cpu_synchronize_pre_loadvm, RUN_ON_CPU_NULL);
1899 }
1900
1901 #ifdef KVM_HAVE_MCE_INJECTION
1902 static __thread void *pending_sigbus_addr;
1903 static __thread int pending_sigbus_code;
1904 static __thread bool have_sigbus_pending;
1905 #endif
1906
1907 static void kvm_cpu_kick(CPUState *cpu)
1908 {
1909     atomic_set(&cpu->kvm_run->immediate_exit, 1);
1910 }
1911
1912 static void kvm_cpu_kick_self(void)
1913 {
1914     if (kvm_immediate_exit) {
1915         kvm_cpu_kick(current_cpu);
1916     } else {
1917         qemu_cpu_kick_self();
1918     }
1919 }
1920
1921 static void kvm_eat_signals(CPUState *cpu)
1922 {
1923     struct timespec ts = { 0, 0 };
1924     siginfo_t siginfo;
1925     sigset_t waitset;
1926     sigset_t chkset;
1927     int r;
1928
1929     if (kvm_immediate_exit) {
1930         atomic_set(&cpu->kvm_run->immediate_exit, 0);
1931         /* Write kvm_run->immediate_exit before the cpu->exit_request
1932          * write in kvm_cpu_exec.
1933          */
1934         smp_wmb();
1935         return;
1936     }
1937
1938     sigemptyset(&waitset);
1939     sigaddset(&waitset, SIG_IPI);
1940
1941     do {
1942         r = sigtimedwait(&waitset, &siginfo, &ts);
1943         if (r == -1 && !(errno == EAGAIN || errno == EINTR)) {
1944             perror("sigtimedwait");
1945             exit(1);
1946         }
1947
1948         r = sigpending(&chkset);
1949         if (r == -1) {
1950             perror("sigpending");
1951             exit(1);
1952         }
1953     } while (sigismember(&chkset, SIG_IPI));
1954 }
1955
1956 int kvm_cpu_exec(CPUState *cpu)
1957 {
1958     struct kvm_run *run = cpu->kvm_run;
1959     int ret, run_ret;
1960
1961     DPRINTF("kvm_cpu_exec()\n");
1962
1963     if (kvm_arch_process_async_events(cpu)) {
1964         atomic_set(&cpu->exit_request, 0);
1965         return EXCP_HLT;
1966     }
1967
1968     qemu_mutex_unlock_iothread();
1969     cpu_exec_start(cpu);
1970
1971     do {
1972         MemTxAttrs attrs;
1973
1974         if (cpu->vcpu_dirty) {
1975             kvm_arch_put_registers(cpu, KVM_PUT_RUNTIME_STATE);
1976             cpu->vcpu_dirty = false;
1977         }
1978
1979         kvm_arch_pre_run(cpu, run);
1980         if (atomic_read(&cpu->exit_request)) {
1981             DPRINTF("interrupt exit requested\n");
1982             /*
1983              * KVM requires us to reenter the kernel after IO exits to complete
1984              * instruction emulation. This self-signal will ensure that we
1985              * leave ASAP again.
1986              */
1987             kvm_cpu_kick_self();
1988         }
1989
1990         /* Read cpu->exit_request before KVM_RUN reads run->immediate_exit.
1991          * Matching barrier in kvm_eat_signals.
1992          */
1993         smp_rmb();
1994
1995         run_ret = kvm_vcpu_ioctl(cpu, KVM_RUN, 0);
1996
1997         attrs = kvm_arch_post_run(cpu, run);
1998
1999 #ifdef KVM_HAVE_MCE_INJECTION
2000         if (unlikely(have_sigbus_pending)) {
2001             qemu_mutex_lock_iothread();
2002             kvm_arch_on_sigbus_vcpu(cpu, pending_sigbus_code,
2003                                     pending_sigbus_addr);
2004             have_sigbus_pending = false;
2005             qemu_mutex_unlock_iothread();
2006         }
2007 #endif
2008
2009         if (run_ret < 0) {
2010             if (run_ret == -EINTR || run_ret == -EAGAIN) {
2011                 DPRINTF("io window exit\n");
2012                 kvm_eat_signals(cpu);
2013                 ret = EXCP_INTERRUPT;
2014                 break;
2015             }
2016             fprintf(stderr, "error: kvm run failed %s\n",
2017                     strerror(-run_ret));
2018 #ifdef TARGET_PPC
2019             if (run_ret == -EBUSY) {
2020                 fprintf(stderr,
2021                         "This is probably because your SMT is enabled.\n"
2022                         "VCPU can only run on primary threads with all "
2023                         "secondary threads offline.\n");
2024             }
2025 #endif
2026             ret = -1;
2027             break;
2028         }
2029
2030         trace_kvm_run_exit(cpu->cpu_index, run->exit_reason);
2031         switch (run->exit_reason) {
2032         case KVM_EXIT_IO:
2033             DPRINTF("handle_io\n");
2034             /* Called outside BQL */
2035             kvm_handle_io(run->io.port, attrs,
2036                           (uint8_t *)run + run->io.data_offset,
2037                           run->io.direction,
2038                           run->io.size,
2039                           run->io.count);
2040             ret = 0;
2041             break;
2042         case KVM_EXIT_MMIO:
2043             DPRINTF("handle_mmio\n");
2044             /* Called outside BQL */
2045             address_space_rw(&address_space_memory,
2046                              run->mmio.phys_addr, attrs,
2047                              run->mmio.data,
2048                              run->mmio.len,
2049                              run->mmio.is_write);
2050             ret = 0;
2051             break;
2052         case KVM_EXIT_IRQ_WINDOW_OPEN:
2053             DPRINTF("irq_window_open\n");
2054             ret = EXCP_INTERRUPT;
2055             break;
2056         case KVM_EXIT_SHUTDOWN:
2057             DPRINTF("shutdown\n");
2058             qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET);
2059             ret = EXCP_INTERRUPT;
2060             break;
2061         case KVM_EXIT_UNKNOWN:
2062             fprintf(stderr, "KVM: unknown exit, hardware reason %" PRIx64 "\n",
2063                     (uint64_t)run->hw.hardware_exit_reason);
2064             ret = -1;
2065             break;
2066         case KVM_EXIT_INTERNAL_ERROR:
2067             ret = kvm_handle_internal_error(cpu, run);
2068             break;
2069         case KVM_EXIT_SYSTEM_EVENT:
2070             switch (run->system_event.type) {
2071             case KVM_SYSTEM_EVENT_SHUTDOWN:
2072                 qemu_system_shutdown_request(SHUTDOWN_CAUSE_GUEST_SHUTDOWN);
2073                 ret = EXCP_INTERRUPT;
2074                 break;
2075             case KVM_SYSTEM_EVENT_RESET:
2076                 qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET);
2077                 ret = EXCP_INTERRUPT;
2078                 break;
2079             case KVM_SYSTEM_EVENT_CRASH:
2080                 kvm_cpu_synchronize_state(cpu);
2081                 qemu_mutex_lock_iothread();
2082                 qemu_system_guest_panicked(cpu_get_crash_info(cpu));
2083                 qemu_mutex_unlock_iothread();
2084                 ret = 0;
2085                 break;
2086             default:
2087                 DPRINTF("kvm_arch_handle_exit\n");
2088                 ret = kvm_arch_handle_exit(cpu, run);
2089                 break;
2090             }
2091             break;
2092         default:
2093             DPRINTF("kvm_arch_handle_exit\n");
2094             ret = kvm_arch_handle_exit(cpu, run);
2095             break;
2096         }
2097     } while (ret == 0);
2098
2099     cpu_exec_end(cpu);
2100     qemu_mutex_lock_iothread();
2101
2102     if (ret < 0) {
2103         cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_CODE);
2104         vm_stop(RUN_STATE_INTERNAL_ERROR);
2105     }
2106
2107     atomic_set(&cpu->exit_request, 0);
2108     return ret;
2109 }
2110
2111 int kvm_ioctl(KVMState *s, int type, ...)
2112 {
2113     int ret;
2114     void *arg;
2115     va_list ap;
2116
2117     va_start(ap, type);
2118     arg = va_arg(ap, void *);
2119     va_end(ap);
2120
2121     trace_kvm_ioctl(type, arg);
2122     ret = ioctl(s->fd, type, arg);
2123     if (ret == -1) {
2124         ret = -errno;
2125     }
2126     return ret;
2127 }
2128
2129 int kvm_vm_ioctl(KVMState *s, int type, ...)
2130 {
2131     int ret;
2132     void *arg;
2133     va_list ap;
2134
2135     va_start(ap, type);
2136     arg = va_arg(ap, void *);
2137     va_end(ap);
2138
2139     trace_kvm_vm_ioctl(type, arg);
2140     ret = ioctl(s->vmfd, type, arg);
2141     if (ret == -1) {
2142         ret = -errno;
2143     }
2144     return ret;
2145 }
2146
2147 int kvm_vcpu_ioctl(CPUState *cpu, int type, ...)
2148 {
2149     int ret;
2150     void *arg;
2151     va_list ap;
2152
2153     va_start(ap, type);
2154     arg = va_arg(ap, void *);
2155     va_end(ap);
2156
2157     trace_kvm_vcpu_ioctl(cpu->cpu_index, type, arg);
2158     ret = ioctl(cpu->kvm_fd, type, arg);
2159     if (ret == -1) {
2160         ret = -errno;
2161     }
2162     return ret;
2163 }
2164
2165 int kvm_device_ioctl(int fd, int type, ...)
2166 {
2167     int ret;
2168     void *arg;
2169     va_list ap;
2170
2171     va_start(ap, type);
2172     arg = va_arg(ap, void *);
2173     va_end(ap);
2174
2175     trace_kvm_device_ioctl(fd, type, arg);
2176     ret = ioctl(fd, type, arg);
2177     if (ret == -1) {
2178         ret = -errno;
2179     }
2180     return ret;
2181 }
2182
2183 int kvm_vm_check_attr(KVMState *s, uint32_t group, uint64_t attr)
2184 {
2185     int ret;
2186     struct kvm_device_attr attribute = {
2187         .group = group,
2188         .attr = attr,
2189     };
2190
2191     if (!kvm_vm_attributes_allowed) {
2192         return 0;
2193     }
2194
2195     ret = kvm_vm_ioctl(s, KVM_HAS_DEVICE_ATTR, &attribute);
2196     /* kvm returns 0 on success for HAS_DEVICE_ATTR */
2197     return ret ? 0 : 1;
2198 }
2199
2200 int kvm_device_check_attr(int dev_fd, uint32_t group, uint64_t attr)
2201 {
2202     struct kvm_device_attr attribute = {
2203         .group = group,
2204         .attr = attr,
2205         .flags = 0,
2206     };
2207
2208     return kvm_device_ioctl(dev_fd, KVM_HAS_DEVICE_ATTR, &attribute) ? 0 : 1;
2209 }
2210
2211 int kvm_device_access(int fd, int group, uint64_t attr,
2212                       void *val, bool write, Error **errp)
2213 {
2214     struct kvm_device_attr kvmattr;
2215     int err;
2216
2217     kvmattr.flags = 0;
2218     kvmattr.group = group;
2219     kvmattr.attr = attr;
2220     kvmattr.addr = (uintptr_t)val;
2221
2222     err = kvm_device_ioctl(fd,
2223                            write ? KVM_SET_DEVICE_ATTR : KVM_GET_DEVICE_ATTR,
2224                            &kvmattr);
2225     if (err < 0) {
2226         error_setg_errno(errp, -err,
2227                          "KVM_%s_DEVICE_ATTR failed: Group %d "
2228                          "attr 0x%016" PRIx64,
2229                          write ? "SET" : "GET", group, attr);
2230     }
2231     return err;
2232 }
2233
2234 /* Return 1 on success, 0 on failure */
2235 int kvm_has_sync_mmu(void)
2236 {
2237     return kvm_check_extension(kvm_state, KVM_CAP_SYNC_MMU);
2238 }
2239
2240 int kvm_has_vcpu_events(void)
2241 {
2242     return kvm_state->vcpu_events;
2243 }
2244
2245 int kvm_has_robust_singlestep(void)
2246 {
2247     return kvm_state->robust_singlestep;
2248 }
2249
2250 int kvm_has_debugregs(void)
2251 {
2252     return kvm_state->debugregs;
2253 }
2254
2255 int kvm_has_many_ioeventfds(void)
2256 {
2257     if (!kvm_enabled()) {
2258         return 0;
2259     }
2260     return kvm_state->many_ioeventfds;
2261 }
2262
2263 int kvm_has_gsi_routing(void)
2264 {
2265 #ifdef KVM_CAP_IRQ_ROUTING
2266     return kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING);
2267 #else
2268     return false;
2269 #endif
2270 }
2271
2272 int kvm_has_intx_set_mask(void)
2273 {
2274     return kvm_state->intx_set_mask;
2275 }
2276
2277 bool kvm_arm_supports_user_irq(void)
2278 {
2279     return kvm_check_extension(kvm_state, KVM_CAP_ARM_USER_IRQ);
2280 }
2281
2282 #ifdef KVM_CAP_SET_GUEST_DEBUG
2283 struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *cpu,
2284                                                  target_ulong pc)
2285 {
2286     struct kvm_sw_breakpoint *bp;
2287
2288     QTAILQ_FOREACH(bp, &cpu->kvm_state->kvm_sw_breakpoints, entry) {
2289         if (bp->pc == pc) {
2290             return bp;
2291         }
2292     }
2293     return NULL;
2294 }
2295
2296 int kvm_sw_breakpoints_active(CPUState *cpu)
2297 {
2298     return !QTAILQ_EMPTY(&cpu->kvm_state->kvm_sw_breakpoints);
2299 }
2300
2301 struct kvm_set_guest_debug_data {
2302     struct kvm_guest_debug dbg;
2303     int err;
2304 };
2305
2306 static void kvm_invoke_set_guest_debug(CPUState *cpu, run_on_cpu_data data)
2307 {
2308     struct kvm_set_guest_debug_data *dbg_data =
2309         (struct kvm_set_guest_debug_data *) data.host_ptr;
2310
2311     dbg_data->err = kvm_vcpu_ioctl(cpu, KVM_SET_GUEST_DEBUG,
2312                                    &dbg_data->dbg);
2313 }
2314
2315 int kvm_update_guest_debug(CPUState *cpu, unsigned long reinject_trap)
2316 {
2317     struct kvm_set_guest_debug_data data;
2318
2319     data.dbg.control = reinject_trap;
2320
2321     if (cpu->singlestep_enabled) {
2322         data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
2323     }
2324     kvm_arch_update_guest_debug(cpu, &data.dbg);
2325
2326     run_on_cpu(cpu, kvm_invoke_set_guest_debug,
2327                RUN_ON_CPU_HOST_PTR(&data));
2328     return data.err;
2329 }
2330
2331 int kvm_insert_breakpoint(CPUState *cpu, target_ulong addr,
2332                           target_ulong len, int type)
2333 {
2334     struct kvm_sw_breakpoint *bp;
2335     int err;
2336
2337     if (type == GDB_BREAKPOINT_SW) {
2338         bp = kvm_find_sw_breakpoint(cpu, addr);
2339         if (bp) {
2340             bp->use_count++;
2341             return 0;
2342         }
2343
2344         bp = g_malloc(sizeof(struct kvm_sw_breakpoint));
2345         bp->pc = addr;
2346         bp->use_count = 1;
2347         err = kvm_arch_insert_sw_breakpoint(cpu, bp);
2348         if (err) {
2349             g_free(bp);
2350             return err;
2351         }
2352
2353         QTAILQ_INSERT_HEAD(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry);
2354     } else {
2355         err = kvm_arch_insert_hw_breakpoint(addr, len, type);
2356         if (err) {
2357             return err;
2358         }
2359     }
2360
2361     CPU_FOREACH(cpu) {
2362         err = kvm_update_guest_debug(cpu, 0);
2363         if (err) {
2364             return err;
2365         }
2366     }
2367     return 0;
2368 }
2369
2370 int kvm_remove_breakpoint(CPUState *cpu, target_ulong addr,
2371                           target_ulong len, int type)
2372 {
2373     struct kvm_sw_breakpoint *bp;
2374     int err;
2375
2376     if (type == GDB_BREAKPOINT_SW) {
2377         bp = kvm_find_sw_breakpoint(cpu, addr);
2378         if (!bp) {
2379             return -ENOENT;
2380         }
2381
2382         if (bp->use_count > 1) {
2383             bp->use_count--;
2384             return 0;
2385         }
2386
2387         err = kvm_arch_remove_sw_breakpoint(cpu, bp);
2388         if (err) {
2389             return err;
2390         }
2391
2392         QTAILQ_REMOVE(&cpu->kvm_state->kvm_sw_breakpoints, bp, entry);
2393         g_free(bp);
2394     } else {
2395         err = kvm_arch_remove_hw_breakpoint(addr, len, type);
2396         if (err) {
2397             return err;
2398         }
2399     }
2400
2401     CPU_FOREACH(cpu) {
2402         err = kvm_update_guest_debug(cpu, 0);
2403         if (err) {
2404             return err;
2405         }
2406     }
2407     return 0;
2408 }
2409
2410 void kvm_remove_all_breakpoints(CPUState *cpu)
2411 {
2412     struct kvm_sw_breakpoint *bp, *next;
2413     KVMState *s = cpu->kvm_state;
2414     CPUState *tmpcpu;
2415
2416     QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
2417         if (kvm_arch_remove_sw_breakpoint(cpu, bp) != 0) {
2418             /* Try harder to find a CPU that currently sees the breakpoint. */
2419             CPU_FOREACH(tmpcpu) {
2420                 if (kvm_arch_remove_sw_breakpoint(tmpcpu, bp) == 0) {
2421                     break;
2422                 }
2423             }
2424         }
2425         QTAILQ_REMOVE(&s->kvm_sw_breakpoints, bp, entry);
2426         g_free(bp);
2427     }
2428     kvm_arch_remove_all_hw_breakpoints();
2429
2430     CPU_FOREACH(cpu) {
2431         kvm_update_guest_debug(cpu, 0);
2432     }
2433 }
2434
2435 #else /* !KVM_CAP_SET_GUEST_DEBUG */
2436
2437 int kvm_update_guest_debug(CPUState *cpu, unsigned long reinject_trap)
2438 {
2439     return -EINVAL;
2440 }
2441
2442 int kvm_insert_breakpoint(CPUState *cpu, target_ulong addr,
2443                           target_ulong len, int type)
2444 {
2445     return -EINVAL;
2446 }
2447
2448 int kvm_remove_breakpoint(CPUState *cpu, target_ulong addr,
2449                           target_ulong len, int type)
2450 {
2451     return -EINVAL;
2452 }
2453
2454 void kvm_remove_all_breakpoints(CPUState *cpu)
2455 {
2456 }
2457 #endif /* !KVM_CAP_SET_GUEST_DEBUG */
2458
2459 static int kvm_set_signal_mask(CPUState *cpu, const sigset_t *sigset)
2460 {
2461     KVMState *s = kvm_state;
2462     struct kvm_signal_mask *sigmask;
2463     int r;
2464
2465     sigmask = g_malloc(sizeof(*sigmask) + sizeof(*sigset));
2466
2467     sigmask->len = s->sigmask_len;
2468     memcpy(sigmask->sigset, sigset, sizeof(*sigset));
2469     r = kvm_vcpu_ioctl(cpu, KVM_SET_SIGNAL_MASK, sigmask);
2470     g_free(sigmask);
2471
2472     return r;
2473 }
2474
2475 static void kvm_ipi_signal(int sig)
2476 {
2477     if (current_cpu) {
2478         assert(kvm_immediate_exit);
2479         kvm_cpu_kick(current_cpu);
2480     }
2481 }
2482
2483 void kvm_init_cpu_signals(CPUState *cpu)
2484 {
2485     int r;
2486     sigset_t set;
2487     struct sigaction sigact;
2488
2489     memset(&sigact, 0, sizeof(sigact));
2490     sigact.sa_handler = kvm_ipi_signal;
2491     sigaction(SIG_IPI, &sigact, NULL);
2492
2493     pthread_sigmask(SIG_BLOCK, NULL, &set);
2494 #if defined KVM_HAVE_MCE_INJECTION
2495     sigdelset(&set, SIGBUS);
2496     pthread_sigmask(SIG_SETMASK, &set, NULL);
2497 #endif
2498     sigdelset(&set, SIG_IPI);
2499     if (kvm_immediate_exit) {
2500         r = pthread_sigmask(SIG_SETMASK, &set, NULL);
2501     } else {
2502         r = kvm_set_signal_mask(cpu, &set);
2503     }
2504     if (r) {
2505         fprintf(stderr, "kvm_set_signal_mask: %s\n", strerror(-r));
2506         exit(1);
2507     }
2508 }
2509
2510 /* Called asynchronously in VCPU thread.  */
2511 int kvm_on_sigbus_vcpu(CPUState *cpu, int code, void *addr)
2512 {
2513 #ifdef KVM_HAVE_MCE_INJECTION
2514     if (have_sigbus_pending) {
2515         return 1;
2516     }
2517     have_sigbus_pending = true;
2518     pending_sigbus_addr = addr;
2519     pending_sigbus_code = code;
2520     atomic_set(&cpu->exit_request, 1);
2521     return 0;
2522 #else
2523     return 1;
2524 #endif
2525 }
2526
2527 /* Called synchronously (via signalfd) in main thread.  */
2528 int kvm_on_sigbus(int code, void *addr)
2529 {
2530 #ifdef KVM_HAVE_MCE_INJECTION
2531     /* Action required MCE kills the process if SIGBUS is blocked.  Because
2532      * that's what happens in the I/O thread, where we handle MCE via signalfd,
2533      * we can only get action optional here.
2534      */
2535     assert(code != BUS_MCEERR_AR);
2536     kvm_arch_on_sigbus_vcpu(first_cpu, code, addr);
2537     return 0;
2538 #else
2539     return 1;
2540 #endif
2541 }
2542
2543 int kvm_create_device(KVMState *s, uint64_t type, bool test)
2544 {
2545     int ret;
2546     struct kvm_create_device create_dev;
2547
2548     create_dev.type = type;
2549     create_dev.fd = -1;
2550     create_dev.flags = test ? KVM_CREATE_DEVICE_TEST : 0;
2551
2552     if (!kvm_check_extension(s, KVM_CAP_DEVICE_CTRL)) {
2553         return -ENOTSUP;
2554     }
2555
2556     ret = kvm_vm_ioctl(s, KVM_CREATE_DEVICE, &create_dev);
2557     if (ret) {
2558         return ret;
2559     }
2560
2561     return test ? 0 : create_dev.fd;
2562 }
2563
2564 bool kvm_device_supported(int vmfd, uint64_t type)
2565 {
2566     struct kvm_create_device create_dev = {
2567         .type = type,
2568         .fd = -1,
2569         .flags = KVM_CREATE_DEVICE_TEST,
2570     };
2571
2572     if (ioctl(vmfd, KVM_CHECK_EXTENSION, KVM_CAP_DEVICE_CTRL) <= 0) {
2573         return false;
2574     }
2575
2576     return (ioctl(vmfd, KVM_CREATE_DEVICE, &create_dev) >= 0);
2577 }
2578
2579 int kvm_set_one_reg(CPUState *cs, uint64_t id, void *source)
2580 {
2581     struct kvm_one_reg reg;
2582     int r;
2583
2584     reg.id = id;
2585     reg.addr = (uintptr_t) source;
2586     r = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
2587     if (r) {
2588         trace_kvm_failed_reg_set(id, strerror(-r));
2589     }
2590     return r;
2591 }
2592
2593 int kvm_get_one_reg(CPUState *cs, uint64_t id, void *target)
2594 {
2595     struct kvm_one_reg reg;
2596     int r;
2597
2598     reg.id = id;
2599     reg.addr = (uintptr_t) target;
2600     r = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
2601     if (r) {
2602         trace_kvm_failed_reg_get(id, strerror(-r));
2603     }
2604     return r;
2605 }
2606
2607 static void kvm_accel_class_init(ObjectClass *oc, void *data)
2608 {
2609     AccelClass *ac = ACCEL_CLASS(oc);
2610     ac->name = "KVM";
2611     ac->init_machine = kvm_init;
2612     ac->allowed = &kvm_allowed;
2613 }
2614
2615 static const TypeInfo kvm_accel_type = {
2616     .name = TYPE_KVM_ACCEL,
2617     .parent = TYPE_ACCEL,
2618     .class_init = kvm_accel_class_init,
2619     .instance_size = sizeof(KVMState),
2620 };
2621
2622 static void kvm_type_init(void)
2623 {
2624     type_register_static(&kvm_accel_type);
2625 }
2626
2627 type_init(kvm_type_init);