kvm-all.c

   1 /*
   2  * QEMU KVM support
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *           Red Hat, Inc. 2008
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *  Glauber Costa     <gcosta@redhat.com>
  10  *
  11  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  12  * See the COPYING file in the top-level directory.
  13  *
  14  */
  15
  16 #include <sys/types.h>
  17 #include <sys/ioctl.h>
  18 #include <sys/mman.h>
  19 #include <stdarg.h>
  20
  21 #include <linux/kvm.h>
  22
  23 #include "qemu-common.h"
  24 #include "qemu-barrier.h"
  25 #include "sysemu.h"
  26 #include "hw/hw.h"
  27 #include "gdbstub.h"
  28 #include "kvm.h"
  29 #include "bswap.h"
  30
  31 /* KVM uses PAGE_SIZE in it's definition of COALESCED_MMIO_MAX */
  32 #define PAGE_SIZE TARGET_PAGE_SIZE
  33 #ifdef KVM_UPSTREAM
  34
  35 //#define DEBUG_KVM
  36
  37 #ifdef DEBUG_KVM
  38 #define DPRINTF(fmt, ...) \
  39     do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
  40 #else
  41 #define DPRINTF(fmt, ...) \
  42     do { } while (0)
  43 #endif
  44
  45 typedef struct KVMSlot
  46 {
  47     target_phys_addr_t start_addr;
  48     ram_addr_t memory_size;
  49     ram_addr_t phys_offset;
  50     int slot;
  51     int flags;
  52 } KVMSlot;
  53
  54 typedef struct kvm_dirty_log KVMDirtyLog;
  55
  56 struct KVMState
  57 {
  58     KVMSlot slots[32];
  59     int fd;
  60     int vmfd;
  61     int coalesced_mmio;
  62 #ifdef KVM_CAP_COALESCED_MMIO
  63     struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
  64 #endif
  65     int broken_set_mem_region;
  66     int migration_log;
  67     int vcpu_events;
  68     int robust_singlestep;
  69     int debugregs;
  70 #ifdef KVM_CAP_SET_GUEST_DEBUG
  71     struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
  72 #endif
  73     int irqchip_in_kernel;
  74     int pit_in_kernel;
  75 };
  76
  77 static KVMState *kvm_state;
  78
  79 static KVMSlot *kvm_alloc_slot(KVMState *s)
  80 {
  81     int i;
  82
  83     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
  84         /* KVM private memory slots */
  85         if (i >= 8 && i < 12)
  86             continue;
  87         if (s->slots[i].memory_size == 0)
  88             return &s->slots[i];
  89     }
  90
  91     fprintf(stderr, "%s: no free slot available\n", __func__);
  92     abort();
  93 }
  94
  95 static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
  96                                          target_phys_addr_t start_addr,
  97                                          target_phys_addr_t end_addr)
  98 {
  99     int i;
 100
 101     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 102         KVMSlot *mem = &s->slots[i];
 103
 104         if (start_addr == mem->start_addr &&
 105             end_addr == mem->start_addr + mem->memory_size) {
 106             return mem;
 107         }
 108     }
 109
 110     return NULL;
 111 }
 112
 113 /*
 114  * Find overlapping slot with lowest start address
 115  */
 116 static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
 117                                             target_phys_addr_t start_addr,
 118                                             target_phys_addr_t end_addr)
 119 {
 120     KVMSlot *found = NULL;
 121     int i;
 122
 123     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 124         KVMSlot *mem = &s->slots[i];
 125
 126         if (mem->memory_size == 0 ||
 127             (found && found->start_addr < mem->start_addr)) {
 128             continue;
 129         }
 130
 131         if (end_addr > mem->start_addr &&
 132             start_addr < mem->start_addr + mem->memory_size) {
 133             found = mem;
 134         }
 135     }
 136
 137     return found;
 138 }
 139
 140 static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
 141 {
 142     struct kvm_userspace_memory_region mem;
 143
 144     mem.slot = slot->slot;
 145     mem.guest_phys_addr = slot->start_addr;
 146     mem.memory_size = slot->memory_size;
 147     mem.userspace_addr = (unsigned long)qemu_get_ram_ptr(slot->phys_offset);
 148     mem.flags = slot->flags;
 149     if (s->migration_log) {
 150         mem.flags |= KVM_MEM_LOG_DIRTY_PAGES;
 151     }
 152     return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
 153 }
 154
 155 static void kvm_reset_vcpu(void *opaque)
 156 {
 157     CPUState *env = opaque;
 158
 159     kvm_arch_reset_vcpu(env);
 160 }
 161 #endif
 162
 163 int kvm_irqchip_in_kernel(void)
 164 {
 165     return kvm_state->irqchip_in_kernel;
 166 }
 167
 168 int kvm_pit_in_kernel(void)
 169 {
 170     return kvm_state->pit_in_kernel;
 171 }
 172
 173
 174 #ifdef KVM_UPSTREAM
 175 int kvm_init_vcpu(CPUState *env)
 176 {
 177     KVMState *s = kvm_state;
 178     long mmap_size;
 179     int ret;
 180
 181     DPRINTF("kvm_init_vcpu\n");
 182
 183     ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, env->cpu_index);
 184     if (ret < 0) {
 185         DPRINTF("kvm_create_vcpu failed\n");
 186         goto err;
 187     }
 188
 189     env->kvm_fd = ret;
 190     env->kvm_state = s;
 191
 192     mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
 193     if (mmap_size < 0) {
 194         DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
 195         goto err;
 196     }
 197
 198     env->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
 199                         env->kvm_fd, 0);
 200     if (env->kvm_run == MAP_FAILED) {
 201         ret = -errno;
 202         DPRINTF("mmap'ing vcpu state failed\n");
 203         goto err;
 204     }
 205
 206 #ifdef KVM_CAP_COALESCED_MMIO
 207     if (s->coalesced_mmio && !s->coalesced_mmio_ring)
 208         s->coalesced_mmio_ring = (void *) env->kvm_run +
 209                 s->coalesced_mmio * PAGE_SIZE;
 210 #endif
 211
 212     ret = kvm_arch_init_vcpu(env);
 213     if (ret == 0) {
 214         qemu_register_reset(kvm_reset_vcpu, env);
 215         kvm_arch_reset_vcpu(env);
 216     }
 217 err:
 218     return ret;
 219 }
 220
 221 /*
 222  * dirty pages logging control
 223  */
 224 static int kvm_dirty_pages_log_change(target_phys_addr_t phys_addr,
 225                                       ram_addr_t size, int flags, int mask)
 226 {
 227     KVMState *s = kvm_state;
 228     KVMSlot *mem = kvm_lookup_matching_slot(s, phys_addr, phys_addr + size);
 229     int old_flags;
 230
 231     if (mem == NULL)  {
 232             fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
 233                     TARGET_FMT_plx "\n", __func__, phys_addr,
 234                     (target_phys_addr_t)(phys_addr + size - 1));
 235             return -EINVAL;
 236     }
 237
 238     old_flags = mem->flags;
 239
 240     flags = (mem->flags & ~mask) | flags;
 241     mem->flags = flags;
 242
 243     /* If nothing changed effectively, no need to issue ioctl */
 244     if (s->migration_log) {
 245         flags |= KVM_MEM_LOG_DIRTY_PAGES;
 246     }
 247     if (flags == old_flags) {
 248             return 0;
 249     }
 250
 251     return kvm_set_user_memory_region(s, mem);
 252 }
 253
 254 int kvm_log_start(target_phys_addr_t phys_addr, ram_addr_t size)
 255 {
 256         return kvm_dirty_pages_log_change(phys_addr, size,
 257                                           KVM_MEM_LOG_DIRTY_PAGES,
 258                                           KVM_MEM_LOG_DIRTY_PAGES);
 259 }
 260
 261 int kvm_log_stop(target_phys_addr_t phys_addr, ram_addr_t size)
 262 {
 263         return kvm_dirty_pages_log_change(phys_addr, size,
 264                                           0,
 265                                           KVM_MEM_LOG_DIRTY_PAGES);
 266 }
 267
 268 static int kvm_set_migration_log(int enable)
 269 {
 270     KVMState *s = kvm_state;
 271     KVMSlot *mem;
 272     int i, err;
 273
 274     s->migration_log = enable;
 275
 276     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 277         mem = &s->slots[i];
 278
 279         if (!!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) == enable) {
 280             continue;
 281         }
 282         err = kvm_set_user_memory_region(s, mem);
 283         if (err) {
 284             return err;
 285         }
 286     }
 287     return 0;
 288 }
 289
 290 /* get kvm's dirty pages bitmap and update qemu's */
 291 static int kvm_get_dirty_pages_log_range(unsigned long start_addr,
 292                                          unsigned long *bitmap,
 293                                          unsigned long offset,
 294                                          unsigned long mem_size)
 295 {
 296     unsigned int i, j;
 297     unsigned long page_number, addr, addr1, c;
 298     ram_addr_t ram_addr;
 299     unsigned int len = ((mem_size / TARGET_PAGE_SIZE) + HOST_LONG_BITS - 1) /
 300         HOST_LONG_BITS;
 301
 302     /*
 303      * bitmap-traveling is faster than memory-traveling (for addr...)
 304      * especially when most of the memory is not dirty.
 305      */
 306     for (i = 0; i < len; i++) {
 307         if (bitmap[i] != 0) {
 308             c = leul_to_cpu(bitmap[i]);
 309             do {
 310                 j = ffsl(c) - 1;
 311                 c &= ~(1ul << j);
 312                 page_number = i * HOST_LONG_BITS + j;
 313                 addr1 = page_number * TARGET_PAGE_SIZE;
 314                 addr = offset + addr1;
 315                 ram_addr = cpu_get_physical_page_desc(addr);
 316                 cpu_physical_memory_set_dirty(ram_addr);
 317             } while (c != 0);
 318         }
 319     }
 320     return 0;
 321 }
 322
 323 #define ALIGN(x, y)  (((x)+(y)-1) & ~((y)-1))
 324
 325 /**
 326  * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
 327  * This function updates qemu's dirty bitmap using cpu_physical_memory_set_dirty().
 328  * This means all bits are set to dirty.
 329  *
 330  * @start_add: start of logged region.
 331  * @end_addr: end of logged region.
 332  */
 333 static int kvm_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
 334                                           target_phys_addr_t end_addr)
 335 {
 336     KVMState *s = kvm_state;
 337     unsigned long size, allocated_size = 0;
 338     KVMDirtyLog d;
 339     KVMSlot *mem;
 340     int ret = 0;
 341
 342     d.dirty_bitmap = NULL;
 343     while (start_addr < end_addr) {
 344         mem = kvm_lookup_overlapping_slot(s, start_addr, end_addr);
 345         if (mem == NULL) {
 346             break;
 347         }
 348
 349         size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS), HOST_LONG_BITS) / 8;
 350         if (!d.dirty_bitmap) {
 351             d.dirty_bitmap = qemu_malloc(size);
 352         } else if (size > allocated_size) {
 353             d.dirty_bitmap = qemu_realloc(d.dirty_bitmap, size);
 354         }
 355         allocated_size = size;
 356         memset(d.dirty_bitmap, 0, allocated_size);
 357
 358         d.slot = mem->slot;
 359
 360         if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
 361             DPRINTF("ioctl failed %d\n", errno);
 362             ret = -1;
 363             break;
 364         }
 365
 366         kvm_get_dirty_pages_log_range(mem->start_addr, d.dirty_bitmap,
 367                                       mem->start_addr, mem->memory_size);
 368         start_addr = mem->start_addr + mem->memory_size;
 369     }
 370     qemu_free(d.dirty_bitmap);
 371
 372     return ret;
 373 }
 374 #endif
 375
 376 int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
 377 {
 378     int ret = -ENOSYS;
 379 #ifdef KVM_CAP_COALESCED_MMIO
 380     KVMState *s = kvm_state;
 381
 382     if (s->coalesced_mmio) {
 383         struct kvm_coalesced_mmio_zone zone;
 384
 385         zone.addr = start;
 386         zone.size = size;
 387
 388         ret = kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
 389     }
 390 #endif
 391
 392     return ret;
 393 }
 394
 395 int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
 396 {
 397     int ret = -ENOSYS;
 398 #ifdef KVM_CAP_COALESCED_MMIO
 399     KVMState *s = kvm_state;
 400
 401     if (s->coalesced_mmio) {
 402         struct kvm_coalesced_mmio_zone zone;
 403
 404         zone.addr = start;
 405         zone.size = size;
 406
 407         ret = kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
 408     }
 409 #endif
 410
 411     return ret;
 412 }
 413
 414 int kvm_check_extension(KVMState *s, unsigned int extension)
 415 {
 416     int ret;
 417
 418     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
 419     if (ret < 0) {
 420         ret = 0;
 421     }
 422
 423     return ret;
 424 }
 425 #ifdef KVM_UPSTREAM
 426
 427 static void kvm_set_phys_mem(target_phys_addr_t start_addr,
 428                              ram_addr_t size,
 429                              ram_addr_t phys_offset)
 430 {
 431     KVMState *s = kvm_state;
 432     ram_addr_t flags = phys_offset & ~TARGET_PAGE_MASK;
 433     KVMSlot *mem, old;
 434     int err;
 435
 436     if (start_addr & ~TARGET_PAGE_MASK) {
 437         if (flags >= IO_MEM_UNASSIGNED) {
 438             if (!kvm_lookup_overlapping_slot(s, start_addr,
 439                                              start_addr + size)) {
 440                 return;
 441             }
 442             fprintf(stderr, "Unaligned split of a KVM memory slot\n");
 443         } else {
 444             fprintf(stderr, "Only page-aligned memory slots supported\n");
 445         }
 446         abort();
 447     }
 448
 449     /* KVM does not support read-only slots */
 450     phys_offset &= ~IO_MEM_ROM;
 451
 452     while (1) {
 453         mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size);
 454         if (!mem) {
 455             break;
 456         }
 457
 458         if (flags < IO_MEM_UNASSIGNED && start_addr >= mem->start_addr &&
 459             (start_addr + size <= mem->start_addr + mem->memory_size) &&
 460             (phys_offset - start_addr == mem->phys_offset - mem->start_addr)) {
 461             /* The new slot fits into the existing one and comes with
 462              * identical parameters - nothing to be done. */
 463             return;
 464         }
 465
 466         old = *mem;
 467
 468         /* unregister the overlapping slot */
 469         mem->memory_size = 0;
 470         err = kvm_set_user_memory_region(s, mem);
 471         if (err) {
 472             fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
 473                     __func__, strerror(-err));
 474             abort();
 475         }
 476
 477         /* Workaround for older KVM versions: we can't join slots, even not by
 478          * unregistering the previous ones and then registering the larger
 479          * slot. We have to maintain the existing fragmentation. Sigh.
 480          *
 481          * This workaround assumes that the new slot starts at the same
 482          * address as the first existing one. If not or if some overlapping
 483          * slot comes around later, we will fail (not seen in practice so far)
 484          * - and actually require a recent KVM version. */
 485         if (s->broken_set_mem_region &&
 486             old.start_addr == start_addr && old.memory_size < size &&
 487             flags < IO_MEM_UNASSIGNED) {
 488             mem = kvm_alloc_slot(s);
 489             mem->memory_size = old.memory_size;
 490             mem->start_addr = old.start_addr;
 491             mem->phys_offset = old.phys_offset;
 492             mem->flags = 0;
 493
 494             err = kvm_set_user_memory_region(s, mem);
 495             if (err) {
 496                 fprintf(stderr, "%s: error updating slot: %s\n", __func__,
 497                         strerror(-err));
 498                 abort();
 499             }
 500
 501             start_addr += old.memory_size;
 502             phys_offset += old.memory_size;
 503             size -= old.memory_size;
 504             continue;
 505         }
 506
 507         /* register prefix slot */
 508         if (old.start_addr < start_addr) {
 509             mem = kvm_alloc_slot(s);
 510             mem->memory_size = start_addr - old.start_addr;
 511             mem->start_addr = old.start_addr;
 512             mem->phys_offset = old.phys_offset;
 513             mem->flags = 0;
 514
 515             err = kvm_set_user_memory_region(s, mem);
 516             if (err) {
 517                 fprintf(stderr, "%s: error registering prefix slot: %s\n",
 518                         __func__, strerror(-err));
 519                 abort();
 520             }
 521         }
 522
 523         /* register suffix slot */
 524         if (old.start_addr + old.memory_size > start_addr + size) {
 525             ram_addr_t size_delta;
 526
 527             mem = kvm_alloc_slot(s);
 528             mem->start_addr = start_addr + size;
 529             size_delta = mem->start_addr - old.start_addr;
 530             mem->memory_size = old.memory_size - size_delta;
 531             mem->phys_offset = old.phys_offset + size_delta;
 532             mem->flags = 0;
 533
 534             err = kvm_set_user_memory_region(s, mem);
 535             if (err) {
 536                 fprintf(stderr, "%s: error registering suffix slot: %s\n",
 537                         __func__, strerror(-err));
 538                 abort();
 539             }
 540         }
 541     }
 542
 543     /* in case the KVM bug workaround already "consumed" the new slot */
 544     if (!size)
 545         return;
 546
 547     /* KVM does not need to know about this memory */
 548     if (flags >= IO_MEM_UNASSIGNED)
 549         return;
 550
 551     mem = kvm_alloc_slot(s);
 552     mem->memory_size = size;
 553     mem->start_addr = start_addr;
 554     mem->phys_offset = phys_offset;
 555     mem->flags = 0;
 556
 557     err = kvm_set_user_memory_region(s, mem);
 558     if (err) {
 559         fprintf(stderr, "%s: error registering slot: %s\n", __func__,
 560                 strerror(-err));
 561         abort();
 562     }
 563 }
 564
 565 #endif
 566
 567 static void kvm_client_set_memory(struct CPUPhysMemoryClient *client,
 568                                   target_phys_addr_t start_addr,
 569                                   ram_addr_t size,
 570                                   ram_addr_t phys_offset)
 571 {
 572         kvm_set_phys_mem(start_addr, size, phys_offset);
 573 }
 574
 575 static int kvm_client_sync_dirty_bitmap(struct CPUPhysMemoryClient *client,
 576                                         target_phys_addr_t start_addr,
 577                                         target_phys_addr_t end_addr)
 578 {
 579         return kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
 580 }
 581
 582 static int kvm_client_migration_log(struct CPUPhysMemoryClient *client,
 583                                     int enable)
 584 {
 585         return kvm_set_migration_log(enable);
 586 }
 587
 588 static CPUPhysMemoryClient kvm_cpu_phys_memory_client = {
 589         .set_memory = kvm_client_set_memory,
 590         .sync_dirty_bitmap = kvm_client_sync_dirty_bitmap,
 591         .migration_log = kvm_client_migration_log,
 592 };
 593
 594
 595 void kvm_cpu_register_phys_memory_client(void)
 596 {
 597     cpu_register_phys_memory_client(&kvm_cpu_phys_memory_client);
 598 }
 599
 600 #ifdef KVM_UPSTREAM
 601
 602 int kvm_init(int smp_cpus)
 603 {
 604     static const char upgrade_note[] =
 605         "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
 606         "(see http://sourceforge.net/projects/kvm).\n";
 607     KVMState *s;
 608     int ret;
 609     int i;
 610
 611     if (smp_cpus > 1) {
 612         fprintf(stderr, "No SMP KVM support, use '-smp 1'\n");
 613         return -EINVAL;
 614     }
 615
 616     s = qemu_mallocz(sizeof(KVMState));
 617
 618 #ifdef KVM_CAP_SET_GUEST_DEBUG
 619     QTAILQ_INIT(&s->kvm_sw_breakpoints);
 620 #endif
 621     for (i = 0; i < ARRAY_SIZE(s->slots); i++)
 622         s->slots[i].slot = i;
 623
 624     s->vmfd = -1;
 625     s->fd = qemu_open("/dev/kvm", O_RDWR);
 626     if (s->fd == -1) {
 627         fprintf(stderr, "Could not access KVM kernel module: %m\n");
 628         ret = -errno;
 629         goto err;
 630     }
 631
 632     ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
 633     if (ret < KVM_API_VERSION) {
 634         if (ret > 0)
 635             ret = -EINVAL;
 636         fprintf(stderr, "kvm version too old\n");
 637         goto err;
 638     }
 639
 640     if (ret > KVM_API_VERSION) {
 641         ret = -EINVAL;
 642         fprintf(stderr, "kvm version not supported\n");
 643         goto err;
 644     }
 645
 646     s->vmfd = kvm_ioctl(s, KVM_CREATE_VM, 0);
 647     if (s->vmfd < 0) {
 648 #ifdef TARGET_S390X
 649         fprintf(stderr, "Please add the 'switch_amode' kernel parameter to "
 650                         "your host kernel command line\n");
 651 #endif
 652         goto err;
 653     }
 654
 655     /* initially, KVM allocated its own memory and we had to jump through
 656      * hooks to make phys_ram_base point to this.  Modern versions of KVM
 657      * just use a user allocated buffer so we can use regular pages
 658      * unmodified.  Make sure we have a sufficiently modern version of KVM.
 659      */
 660     if (!kvm_check_extension(s, KVM_CAP_USER_MEMORY)) {
 661         ret = -EINVAL;
 662         fprintf(stderr, "kvm does not support KVM_CAP_USER_MEMORY\n%s",
 663                 upgrade_note);
 664         goto err;
 665     }
 666
 667     /* There was a nasty bug in < kvm-80 that prevents memory slots from being
 668      * destroyed properly.  Since we rely on this capability, refuse to work
 669      * with any kernel without this capability. */
 670     if (!kvm_check_extension(s, KVM_CAP_DESTROY_MEMORY_REGION_WORKS)) {
 671         ret = -EINVAL;
 672
 673         fprintf(stderr,
 674                 "KVM kernel module broken (DESTROY_MEMORY_REGION).\n%s",
 675                 upgrade_note);
 676         goto err;
 677     }
 678
 679     s->coalesced_mmio = 0;
 680 #ifdef KVM_CAP_COALESCED_MMIO
 681     s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
 682     s->coalesced_mmio_ring = NULL;
 683 #endif
 684
 685     s->broken_set_mem_region = 1;
 686 #ifdef KVM_CAP_JOIN_MEMORY_REGIONS_WORKS
 687     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
 688     if (ret > 0) {
 689         s->broken_set_mem_region = 0;
 690     }
 691 #endif
 692
 693     s->vcpu_events = 0;
 694 #ifdef KVM_CAP_VCPU_EVENTS
 695     s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
 696 #endif
 697
 698     s->robust_singlestep = 0;
 699 #ifdef KVM_CAP_X86_ROBUST_SINGLESTEP
 700     s->robust_singlestep =
 701         kvm_check_extension(s, KVM_CAP_X86_ROBUST_SINGLESTEP);
 702 #endif
 703
 704     s->debugregs = 0;
 705 #ifdef KVM_CAP_DEBUGREGS
 706     s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS);
 707 #endif
 708
 709     ret = kvm_arch_init(s, smp_cpus);
 710     if (ret < 0)
 711         goto err;
 712
 713     kvm_state = s;
 714     cpu_register_phys_memory_client(&kvm_cpu_phys_memory_client);
 715
 716     return 0;
 717
 718 err:
 719     if (s) {
 720         if (s->vmfd != -1)
 721             close(s->vmfd);
 722         if (s->fd != -1)
 723             close(s->fd);
 724     }
 725     qemu_free(s);
 726
 727     return ret;
 728 }
 729 #endif
 730
 731 static int kvm_handle_io(uint16_t port, void *data, int direction, int size,
 732                          uint32_t count)
 733 {
 734     int i;
 735     uint8_t *ptr = data;
 736
 737     for (i = 0; i < count; i++) {
 738         if (direction == KVM_EXIT_IO_IN) {
 739             switch (size) {
 740             case 1:
 741                 stb_p(ptr, cpu_inb(port));
 742                 break;
 743             case 2:
 744                 stw_p(ptr, cpu_inw(port));
 745                 break;
 746             case 4:
 747                 stl_p(ptr, cpu_inl(port));
 748                 break;
 749             }
 750         } else {
 751             switch (size) {
 752             case 1:
 753                 cpu_outb(port, ldub_p(ptr));
 754                 break;
 755             case 2:
 756                 cpu_outw(port, lduw_p(ptr));
 757                 break;
 758             case 4:
 759                 cpu_outl(port, ldl_p(ptr));
 760                 break;
 761             }
 762         }
 763
 764         ptr += size;
 765     }
 766
 767     return 1;
 768 }
 769
 770 #ifdef KVM_CAP_INTERNAL_ERROR_DATA
 771 static void kvm_handle_internal_error(CPUState *env, struct kvm_run *run)
 772 {
 773
 774     if (kvm_check_extension(kvm_state, KVM_CAP_INTERNAL_ERROR_DATA)) {
 775         int i;
 776
 777         fprintf(stderr, "KVM internal error. Suberror: %d\n",
 778                 run->internal.suberror);
 779
 780         for (i = 0; i < run->internal.ndata; ++i) {
 781             fprintf(stderr, "extra data[%d]: %"PRIx64"\n",
 782                     i, (uint64_t)run->internal.data[i]);
 783         }
 784     }
 785     cpu_dump_state(env, stderr, fprintf, 0);
 786     if (run->internal.suberror == KVM_INTERNAL_ERROR_EMULATION) {
 787         fprintf(stderr, "emulation failure\n");
 788     }
 789     /* FIXME: Should trigger a qmp message to let management know
 790      * something went wrong.
 791      */
 792     vm_stop(0);
 793 }
 794 #endif
 795
 796 void kvm_flush_coalesced_mmio_buffer(void)
 797 {
 798 #ifdef KVM_CAP_COALESCED_MMIO
 799     KVMState *s = kvm_state;
 800     if (s->coalesced_mmio_ring) {
 801         struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
 802         while (ring->first != ring->last) {
 803             struct kvm_coalesced_mmio *ent;
 804
 805             ent = &ring->coalesced_mmio[ring->first];
 806
 807             cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
 808             smp_wmb();
 809             ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
 810         }
 811     }
 812 #endif
 813 }
 814
 815 #ifdef KVM_UPSTREAM
 816
 817 void kvm_cpu_synchronize_state(CPUState *env)
 818 {
 819     if (!env->kvm_vcpu_dirty) {
 820         kvm_arch_get_registers(env);
 821         env->kvm_vcpu_dirty = 1;
 822     }
 823 }
 824
 825 void kvm_cpu_synchronize_post_reset(CPUState *env)
 826 {
 827     kvm_arch_put_registers(env, KVM_PUT_RESET_STATE);
 828     env->kvm_vcpu_dirty = 0;
 829 }
 830
 831 void kvm_cpu_synchronize_post_init(CPUState *env)
 832 {
 833     kvm_arch_put_registers(env, KVM_PUT_FULL_STATE);
 834     env->kvm_vcpu_dirty = 0;
 835 }
 836
 837 int kvm_cpu_exec(CPUState *env)
 838 {
 839     struct kvm_run *run = env->kvm_run;
 840     int ret;
 841
 842     DPRINTF("kvm_cpu_exec()\n");
 843
 844     do {
 845 #ifndef CONFIG_IOTHREAD
 846         if (env->exit_request) {
 847             DPRINTF("interrupt exit requested\n");
 848             ret = 0;
 849             break;
 850         }
 851 #endif
 852
 853         if (env->kvm_vcpu_dirty) {
 854             kvm_arch_put_registers(env, KVM_PUT_RUNTIME_STATE);
 855             env->kvm_vcpu_dirty = 0;
 856         }
 857
 858         kvm_arch_pre_run(env, run);
 859         qemu_mutex_unlock_iothread();
 860         ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);
 861         qemu_mutex_lock_iothread();
 862         kvm_arch_post_run(env, run);
 863
 864         if (ret == -EINTR || ret == -EAGAIN) {
 865             cpu_exit(env);
 866             DPRINTF("io window exit\n");
 867             ret = 0;
 868             break;
 869         }
 870
 871         if (ret < 0) {
 872             DPRINTF("kvm run failed %s\n", strerror(-ret));
 873             abort();
 874         }
 875
 876         kvm_flush_coalesced_mmio_buffer();
 877
 878         ret = 0; /* exit loop */
 879         switch (run->exit_reason) {
 880         case KVM_EXIT_IO:
 881             DPRINTF("handle_io\n");
 882             ret = kvm_handle_io(run->io.port,
 883                                 (uint8_t *)run + run->io.data_offset,
 884                                 run->io.direction,
 885                                 run->io.size,
 886                                 run->io.count);
 887             break;
 888         case KVM_EXIT_MMIO:
 889             DPRINTF("handle_mmio\n");
 890             cpu_physical_memory_rw(run->mmio.phys_addr,
 891                                    run->mmio.data,
 892                                    run->mmio.len,
 893                                    run->mmio.is_write);
 894             ret = 1;
 895             break;
 896         case KVM_EXIT_IRQ_WINDOW_OPEN:
 897             DPRINTF("irq_window_open\n");
 898             break;
 899         case KVM_EXIT_SHUTDOWN:
 900             DPRINTF("shutdown\n");
 901             qemu_system_reset_request();
 902             ret = 1;
 903             break;
 904         case KVM_EXIT_UNKNOWN:
 905             DPRINTF("kvm_exit_unknown\n");
 906             break;
 907         case KVM_EXIT_FAIL_ENTRY:
 908             DPRINTF("kvm_exit_fail_entry\n");
 909             break;
 910         case KVM_EXIT_EXCEPTION:
 911             DPRINTF("kvm_exit_exception\n");
 912             break;
 913 #ifdef KVM_CAP_INTERNAL_ERROR_DATA
 914         case KVM_EXIT_INTERNAL_ERROR:
 915             kvm_handle_internal_error(env, run);
 916             break;
 917 #endif
 918         case KVM_EXIT_DEBUG:
 919             DPRINTF("kvm_exit_debug\n");
 920 #ifdef KVM_CAP_SET_GUEST_DEBUG
 921             if (kvm_arch_debug(&run->debug.arch)) {
 922                 gdb_set_stop_cpu(env);
 923                 vm_stop(EXCP_DEBUG);
 924                 env->exception_index = EXCP_DEBUG;
 925                 return 0;
 926             }
 927             /* re-enter, this exception was guest-internal */
 928             ret = 1;
 929 #endif /* KVM_CAP_SET_GUEST_DEBUG */
 930             break;
 931         default:
 932             DPRINTF("kvm_arch_handle_exit\n");
 933             ret = kvm_arch_handle_exit(env, run);
 934             break;
 935         }
 936     } while (ret > 0);
 937
 938     if (env->exit_request) {
 939         env->exit_request = 0;
 940         env->exception_index = EXCP_INTERRUPT;
 941     }
 942
 943     return ret;
 944 }
 945
 946 #endif
 947 int kvm_ioctl(KVMState *s, int type, ...)
 948 {
 949     int ret;
 950     void *arg;
 951     va_list ap;
 952
 953     va_start(ap, type);
 954     arg = va_arg(ap, void *);
 955     va_end(ap);
 956
 957     ret = ioctl(s->fd, type, arg);
 958     if (ret == -1)
 959         ret = -errno;
 960
 961     return ret;
 962 }
 963
 964 int kvm_vm_ioctl(KVMState *s, int type, ...)
 965 {
 966     int ret;
 967     void *arg;
 968     va_list ap;
 969
 970     va_start(ap, type);
 971     arg = va_arg(ap, void *);
 972     va_end(ap);
 973
 974     ret = ioctl(s->vmfd, type, arg);
 975     if (ret == -1)
 976         ret = -errno;
 977
 978     return ret;
 979 }
 980
 981 int kvm_vcpu_ioctl(CPUState *env, int type, ...)
 982 {
 983     int ret;
 984     void *arg;
 985     va_list ap;
 986
 987     va_start(ap, type);
 988     arg = va_arg(ap, void *);
 989     va_end(ap);
 990
 991     ret = ioctl(env->kvm_fd, type, arg);
 992     if (ret == -1)
 993         ret = -errno;
 994
 995     return ret;
 996 }
 997
 998 int kvm_has_sync_mmu(void)
 999 {
1000 #ifdef KVM_CAP_SYNC_MMU
1001     KVMState *s = kvm_state;
1002
1003     return kvm_check_extension(s, KVM_CAP_SYNC_MMU);
1004 #else
1005     return 0;
1006 #endif
1007 }
1008
1009 int kvm_has_vcpu_events(void)
1010 {
1011     return kvm_state->vcpu_events;
1012 }
1013
1014 int kvm_has_robust_singlestep(void)
1015 {
1016     return kvm_state->robust_singlestep;
1017 }
1018
1019 int kvm_has_debugregs(void)
1020 {
1021     return kvm_state->debugregs;
1022 }
1023
1024 void kvm_setup_guest_memory(void *start, size_t size)
1025 {
1026     if (!kvm_has_sync_mmu()) {
1027 #ifdef MADV_DONTFORK
1028         int ret = madvise(start, size, MADV_DONTFORK);
1029
1030         if (ret) {
1031             perror("madvice");
1032             exit(1);
1033         }
1034 #else
1035         fprintf(stderr,
1036                 "Need MADV_DONTFORK in absence of synchronous KVM MMU\n");
1037         exit(1);
1038 #endif
1039     }
1040 }
1041
1042 #ifdef KVM_CAP_SET_GUEST_DEBUG
1043
1044 #ifdef KVM_UPSTREAM
1045 static void on_vcpu(CPUState *env, void (*func)(void *data), void *data)
1046 {
1047 #ifdef CONFIG_IOTHREAD
1048     if (env != cpu_single_env) {
1049         abort();
1050     }
1051 #endif
1052     func(data);
1053 }
1054 #else /* !KVM_UPSTREAM */
1055 static void on_vcpu(CPUState *env, void (*func)(void *data), void *data);
1056 #endif /* !KVM_UPSTREAM */
1057
1058 struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *env,
1059                                                  target_ulong pc)
1060 {
1061     struct kvm_sw_breakpoint *bp;
1062
1063     QTAILQ_FOREACH(bp, &env->kvm_state->kvm_sw_breakpoints, entry) {
1064         if (bp->pc == pc)
1065             return bp;
1066     }
1067     return NULL;
1068 }
1069
1070 int kvm_sw_breakpoints_active(CPUState *env)
1071 {
1072     return !QTAILQ_EMPTY(&env->kvm_state->kvm_sw_breakpoints);
1073 }
1074
1075 struct kvm_set_guest_debug_data {
1076     struct kvm_guest_debug dbg;
1077     CPUState *env;
1078     int err;
1079 };
1080
1081 static void kvm_invoke_set_guest_debug(void *data)
1082 {
1083     struct kvm_set_guest_debug_data *dbg_data = data;
1084     CPUState *env = dbg_data->env;
1085
1086     dbg_data->err = kvm_vcpu_ioctl(env, KVM_SET_GUEST_DEBUG, &dbg_data->dbg);
1087 }
1088
1089 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1090 {
1091     struct kvm_set_guest_debug_data data;
1092
1093     data.dbg.control = reinject_trap;
1094
1095     if (env->singlestep_enabled) {
1096         data.dbg.control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
1097     }
1098     kvm_arch_update_guest_debug(env, &data.dbg);
1099     data.env = env;
1100
1101     on_vcpu(env, kvm_invoke_set_guest_debug, &data);
1102     return data.err;
1103 }
1104
1105 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1106                           target_ulong len, int type)
1107 {
1108     struct kvm_sw_breakpoint *bp;
1109     CPUState *env;
1110     int err;
1111
1112     if (type == GDB_BREAKPOINT_SW) {
1113         bp = kvm_find_sw_breakpoint(current_env, addr);
1114         if (bp) {
1115             bp->use_count++;
1116             return 0;
1117         }
1118
1119         bp = qemu_malloc(sizeof(struct kvm_sw_breakpoint));
1120         if (!bp)
1121             return -ENOMEM;
1122
1123         bp->pc = addr;
1124         bp->use_count = 1;
1125         err = kvm_arch_insert_sw_breakpoint(current_env, bp);
1126         if (err) {
1127             free(bp);
1128             return err;
1129         }
1130
1131         QTAILQ_INSERT_HEAD(&current_env->kvm_state->kvm_sw_breakpoints,
1132                           bp, entry);
1133     } else {
1134         err = kvm_arch_insert_hw_breakpoint(addr, len, type);
1135         if (err)
1136             return err;
1137     }
1138
1139     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1140         err = kvm_update_guest_debug(env, 0);
1141         if (err)
1142             return err;
1143     }
1144     return 0;
1145 }
1146
1147 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1148                           target_ulong len, int type)
1149 {
1150     struct kvm_sw_breakpoint *bp;
1151     CPUState *env;
1152     int err;
1153
1154     if (type == GDB_BREAKPOINT_SW) {
1155         bp = kvm_find_sw_breakpoint(current_env, addr);
1156         if (!bp)
1157             return -ENOENT;
1158
1159         if (bp->use_count > 1) {
1160             bp->use_count--;
1161             return 0;
1162         }
1163
1164         err = kvm_arch_remove_sw_breakpoint(current_env, bp);
1165         if (err)
1166             return err;
1167
1168         QTAILQ_REMOVE(&current_env->kvm_state->kvm_sw_breakpoints, bp, entry);
1169         qemu_free(bp);
1170     } else {
1171         err = kvm_arch_remove_hw_breakpoint(addr, len, type);
1172         if (err)
1173             return err;
1174     }
1175
1176     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1177         err = kvm_update_guest_debug(env, 0);
1178         if (err)
1179             return err;
1180     }
1181     return 0;
1182 }
1183
1184 void kvm_remove_all_breakpoints(CPUState *current_env)
1185 {
1186     struct kvm_sw_breakpoint *bp, *next;
1187     KVMState *s = current_env->kvm_state;
1188     CPUState *env;
1189
1190     QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
1191         if (kvm_arch_remove_sw_breakpoint(current_env, bp) != 0) {
1192             /* Try harder to find a CPU that currently sees the breakpoint. */
1193             for (env = first_cpu; env != NULL; env = env->next_cpu) {
1194                 if (kvm_arch_remove_sw_breakpoint(env, bp) == 0)
1195                     break;
1196             }
1197         }
1198     }
1199     kvm_arch_remove_all_hw_breakpoints();
1200
1201     for (env = first_cpu; env != NULL; env = env->next_cpu)
1202         kvm_update_guest_debug(env, 0);
1203 }
1204
1205 #else /* !KVM_CAP_SET_GUEST_DEBUG */
1206
1207 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1208 {
1209     return -EINVAL;
1210 }
1211
1212 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1213                           target_ulong len, int type)
1214 {
1215     return -EINVAL;
1216 }
1217
1218 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1219                           target_ulong len, int type)
1220 {
1221     return -EINVAL;
1222 }
1223
1224 void kvm_remove_all_breakpoints(CPUState *current_env)
1225 {
1226 }
1227 #endif /* !KVM_CAP_SET_GUEST_DEBUG */
1228
1229 int kvm_set_signal_mask(CPUState *env, const sigset_t *sigset)
1230 {
1231     struct kvm_signal_mask *sigmask;
1232     int r;
1233
1234     if (!sigset)
1235         return kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, NULL);
1236
1237     sigmask = qemu_malloc(sizeof(*sigmask) + sizeof(*sigset));
1238
1239     sigmask->len = 8;
1240     memcpy(sigmask->sigset, sigset, sizeof(*sigset));
1241     r = kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, sigmask);
1242     free(sigmask);
1243
1244     return r;
1245 }
1246
1247 int kvm_set_ioeventfd_pio_word(int fd, uint16_t addr, uint16_t val, bool assign)
1248 {
1249 #ifdef KVM_IOEVENTFD
1250     struct kvm_ioeventfd kick = {
1251         .datamatch = val,
1252         .addr = addr,
1253         .len = 2,
1254         .flags = KVM_IOEVENTFD_FLAG_DATAMATCH | KVM_IOEVENTFD_FLAG_PIO,
1255         .fd = fd,
1256     };
1257     int r;
1258     if (!kvm_enabled())
1259         return -ENOSYS;
1260     if (!assign)
1261         kick.flags |= KVM_IOEVENTFD_FLAG_DEASSIGN;
1262     r = kvm_vm_ioctl(kvm_state, KVM_IOEVENTFD, &kick);
1263     if (r < 0)
1264         return r;
1265     return 0;
1266 #else
1267     return -ENOSYS;
1268 #endif
1269 }
1270
1271 #if defined(KVM_IRQFD)
1272 int kvm_set_irqfd(int gsi, int fd, bool assigned)
1273 {
1274     struct kvm_irqfd irqfd = {
1275         .fd = fd,
1276         .gsi = gsi,
1277         .flags = assigned ? 0 : KVM_IRQFD_FLAG_DEASSIGN,
1278     };
1279     int r;
1280     if (!kvm_enabled() || !kvm_irqchip_in_kernel())
1281         return -ENOSYS;
1282
1283     r = kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd);
1284     if (r < 0)
1285         return r;
1286     return 0;
1287 }
1288 #endif
1289
1290 #undef PAGE_SIZE
1291 #include "qemu-kvm.c"