kvm-all.c

   1 /*
   2  * QEMU KVM support
   3  *
   4  * Copyright IBM, Corp. 2008
   5  *           Red Hat, Inc. 2008
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *  Glauber Costa     <gcosta@redhat.com>
  10  *
  11  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  12  * See the COPYING file in the top-level directory.
  13  *
  14  */
  15
  16 #include <sys/types.h>
  17 #include <sys/ioctl.h>
  18 #include <sys/mman.h>
  19 #include <stdarg.h>
  20
  21 #include <linux/kvm.h>
  22
  23 #include "qemu-common.h"
  24 #include "qemu-barrier.h"
  25 #include "sysemu.h"
  26 #include "hw/hw.h"
  27 #include "gdbstub.h"
  28 #include "kvm.h"
  29
  30 /* KVM uses PAGE_SIZE in it's definition of COALESCED_MMIO_MAX */
  31 #define PAGE_SIZE TARGET_PAGE_SIZE
  32
  33 //#define DEBUG_KVM
  34
  35 #ifdef DEBUG_KVM
  36 #define dprintf(fmt, ...) \
  37     do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
  38 #else
  39 #define dprintf(fmt, ...) \
  40     do { } while (0)
  41 #endif
  42
  43 typedef struct KVMSlot
  44 {
  45     target_phys_addr_t start_addr;
  46     ram_addr_t memory_size;
  47     ram_addr_t phys_offset;
  48     int slot;
  49     int flags;
  50 } KVMSlot;
  51
  52 typedef struct kvm_dirty_log KVMDirtyLog;
  53
  54 int kvm_allowed = 0;
  55
  56 struct KVMState
  57 {
  58     KVMSlot slots[32];
  59     int fd;
  60     int vmfd;
  61     int coalesced_mmio;
  62 #ifdef KVM_CAP_COALESCED_MMIO
  63     struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
  64 #endif
  65     int broken_set_mem_region;
  66     int migration_log;
  67     int vcpu_events;
  68 #ifdef KVM_CAP_SET_GUEST_DEBUG
  69     struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
  70 #endif
  71     int irqchip_in_kernel;
  72     int pit_in_kernel;
  73 };
  74
  75 static KVMState *kvm_state;
  76
  77 static KVMSlot *kvm_alloc_slot(KVMState *s)
  78 {
  79     int i;
  80
  81     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
  82         /* KVM private memory slots */
  83         if (i >= 8 && i < 12)
  84             continue;
  85         if (s->slots[i].memory_size == 0)
  86             return &s->slots[i];
  87     }
  88
  89     fprintf(stderr, "%s: no free slot available\n", __func__);
  90     abort();
  91 }
  92
  93 static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
  94                                          target_phys_addr_t start_addr,
  95                                          target_phys_addr_t end_addr)
  96 {
  97     int i;
  98
  99     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 100         KVMSlot *mem = &s->slots[i];
 101
 102         if (start_addr == mem->start_addr &&
 103             end_addr == mem->start_addr + mem->memory_size) {
 104             return mem;
 105         }
 106     }
 107
 108     return NULL;
 109 }
 110
 111 /*
 112  * Find overlapping slot with lowest start address
 113  */
 114 static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
 115                                             target_phys_addr_t start_addr,
 116                                             target_phys_addr_t end_addr)
 117 {
 118     KVMSlot *found = NULL;
 119     int i;
 120
 121     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 122         KVMSlot *mem = &s->slots[i];
 123
 124         if (mem->memory_size == 0 ||
 125             (found && found->start_addr < mem->start_addr)) {
 126             continue;
 127         }
 128
 129         if (end_addr > mem->start_addr &&
 130             start_addr < mem->start_addr + mem->memory_size) {
 131             found = mem;
 132         }
 133     }
 134
 135     return found;
 136 }
 137
 138 static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
 139 {
 140     struct kvm_userspace_memory_region mem;
 141
 142     mem.slot = slot->slot;
 143     mem.guest_phys_addr = slot->start_addr;
 144     mem.memory_size = slot->memory_size;
 145     mem.userspace_addr = (unsigned long)qemu_get_ram_ptr(slot->phys_offset);
 146     mem.flags = slot->flags;
 147     if (s->migration_log) {
 148         mem.flags |= KVM_MEM_LOG_DIRTY_PAGES;
 149     }
 150     return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
 151 }
 152
 153 static void kvm_reset_vcpu(void *opaque)
 154 {
 155     CPUState *env = opaque;
 156
 157     kvm_arch_reset_vcpu(env);
 158     if (kvm_arch_put_registers(env)) {
 159         fprintf(stderr, "Fatal: kvm vcpu reset failed\n");
 160         abort();
 161     }
 162 }
 163
 164 int kvm_irqchip_in_kernel(void)
 165 {
 166     return kvm_state->irqchip_in_kernel;
 167 }
 168
 169 int kvm_pit_in_kernel(void)
 170 {
 171     return kvm_state->pit_in_kernel;
 172 }
 173
 174
 175 int kvm_init_vcpu(CPUState *env)
 176 {
 177     KVMState *s = kvm_state;
 178     long mmap_size;
 179     int ret;
 180
 181     dprintf("kvm_init_vcpu\n");
 182
 183     ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, env->cpu_index);
 184     if (ret < 0) {
 185         dprintf("kvm_create_vcpu failed\n");
 186         goto err;
 187     }
 188
 189     env->kvm_fd = ret;
 190     env->kvm_state = s;
 191
 192     mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
 193     if (mmap_size < 0) {
 194         dprintf("KVM_GET_VCPU_MMAP_SIZE failed\n");
 195         goto err;
 196     }
 197
 198     env->kvm_run = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED,
 199                         env->kvm_fd, 0);
 200     if (env->kvm_run == MAP_FAILED) {
 201         ret = -errno;
 202         dprintf("mmap'ing vcpu state failed\n");
 203         goto err;
 204     }
 205
 206 #ifdef KVM_CAP_COALESCED_MMIO
 207     if (s->coalesced_mmio && !s->coalesced_mmio_ring)
 208         s->coalesced_mmio_ring = (void *) env->kvm_run +
 209                 s->coalesced_mmio * PAGE_SIZE;
 210 #endif
 211
 212     ret = kvm_arch_init_vcpu(env);
 213     if (ret == 0) {
 214         qemu_register_reset(kvm_reset_vcpu, env);
 215         kvm_arch_reset_vcpu(env);
 216         ret = kvm_arch_put_registers(env);
 217     }
 218 err:
 219     return ret;
 220 }
 221
 222 /*
 223  * dirty pages logging control
 224  */
 225 static int kvm_dirty_pages_log_change(target_phys_addr_t phys_addr,
 226                                       ram_addr_t size, int flags, int mask)
 227 {
 228     KVMState *s = kvm_state;
 229     KVMSlot *mem = kvm_lookup_matching_slot(s, phys_addr, phys_addr + size);
 230     int old_flags;
 231
 232     if (mem == NULL)  {
 233             fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
 234                     TARGET_FMT_plx "\n", __func__, phys_addr,
 235                     (target_phys_addr_t)(phys_addr + size - 1));
 236             return -EINVAL;
 237     }
 238
 239     old_flags = mem->flags;
 240
 241     flags = (mem->flags & ~mask) | flags;
 242     mem->flags = flags;
 243
 244     /* If nothing changed effectively, no need to issue ioctl */
 245     if (s->migration_log) {
 246         flags |= KVM_MEM_LOG_DIRTY_PAGES;
 247     }
 248     if (flags == old_flags) {
 249             return 0;
 250     }
 251
 252     return kvm_set_user_memory_region(s, mem);
 253 }
 254
 255 int kvm_log_start(target_phys_addr_t phys_addr, ram_addr_t size)
 256 {
 257         return kvm_dirty_pages_log_change(phys_addr, size,
 258                                           KVM_MEM_LOG_DIRTY_PAGES,
 259                                           KVM_MEM_LOG_DIRTY_PAGES);
 260 }
 261
 262 int kvm_log_stop(target_phys_addr_t phys_addr, ram_addr_t size)
 263 {
 264         return kvm_dirty_pages_log_change(phys_addr, size,
 265                                           0,
 266                                           KVM_MEM_LOG_DIRTY_PAGES);
 267 }
 268
 269 static int kvm_set_migration_log(int enable)
 270 {
 271     KVMState *s = kvm_state;
 272     KVMSlot *mem;
 273     int i, err;
 274
 275     s->migration_log = enable;
 276
 277     for (i = 0; i < ARRAY_SIZE(s->slots); i++) {
 278         mem = &s->slots[i];
 279
 280         if (!!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) == enable) {
 281             continue;
 282         }
 283         err = kvm_set_user_memory_region(s, mem);
 284         if (err) {
 285             return err;
 286         }
 287     }
 288     return 0;
 289 }
 290
 291 static int test_le_bit(unsigned long nr, unsigned char *addr)
 292 {
 293     return (addr[nr >> 3] >> (nr & 7)) & 1;
 294 }
 295
 296 /**
 297  * kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
 298  * This function updates qemu's dirty bitmap using cpu_physical_memory_set_dirty().
 299  * This means all bits are set to dirty.
 300  *
 301  * @start_add: start of logged region.
 302  * @end_addr: end of logged region.
 303  */
 304 static int kvm_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
 305                                           target_phys_addr_t end_addr)
 306 {
 307     KVMState *s = kvm_state;
 308     unsigned long size, allocated_size = 0;
 309     target_phys_addr_t phys_addr;
 310     ram_addr_t addr;
 311     KVMDirtyLog d;
 312     KVMSlot *mem;
 313     int ret = 0;
 314
 315     d.dirty_bitmap = NULL;
 316     while (start_addr < end_addr) {
 317         mem = kvm_lookup_overlapping_slot(s, start_addr, end_addr);
 318         if (mem == NULL) {
 319             break;
 320         }
 321
 322         size = ((mem->memory_size >> TARGET_PAGE_BITS) + 7) / 8;
 323         if (!d.dirty_bitmap) {
 324             d.dirty_bitmap = qemu_malloc(size);
 325         } else if (size > allocated_size) {
 326             d.dirty_bitmap = qemu_realloc(d.dirty_bitmap, size);
 327         }
 328         allocated_size = size;
 329         memset(d.dirty_bitmap, 0, allocated_size);
 330
 331         d.slot = mem->slot;
 332
 333         if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
 334             dprintf("ioctl failed %d\n", errno);
 335             ret = -1;
 336             break;
 337         }
 338
 339         for (phys_addr = mem->start_addr, addr = mem->phys_offset;
 340              phys_addr < mem->start_addr + mem->memory_size;
 341              phys_addr += TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
 342             unsigned char *bitmap = (unsigned char *)d.dirty_bitmap;
 343             unsigned nr = (phys_addr - mem->start_addr) >> TARGET_PAGE_BITS;
 344
 345             if (test_le_bit(nr, bitmap)) {
 346                 cpu_physical_memory_set_dirty(addr);
 347             }
 348         }
 349         start_addr = phys_addr;
 350     }
 351     qemu_free(d.dirty_bitmap);
 352
 353     return ret;
 354 }
 355
 356 int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
 357 {
 358     int ret = -ENOSYS;
 359 #ifdef KVM_CAP_COALESCED_MMIO
 360     KVMState *s = kvm_state;
 361
 362     if (s->coalesced_mmio) {
 363         struct kvm_coalesced_mmio_zone zone;
 364
 365         zone.addr = start;
 366         zone.size = size;
 367
 368         ret = kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
 369     }
 370 #endif
 371
 372     return ret;
 373 }
 374
 375 int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
 376 {
 377     int ret = -ENOSYS;
 378 #ifdef KVM_CAP_COALESCED_MMIO
 379     KVMState *s = kvm_state;
 380
 381     if (s->coalesced_mmio) {
 382         struct kvm_coalesced_mmio_zone zone;
 383
 384         zone.addr = start;
 385         zone.size = size;
 386
 387         ret = kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
 388     }
 389 #endif
 390
 391     return ret;
 392 }
 393
 394 int kvm_check_extension(KVMState *s, unsigned int extension)
 395 {
 396     int ret;
 397
 398     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, extension);
 399     if (ret < 0) {
 400         ret = 0;
 401     }
 402
 403     return ret;
 404 }
 405
 406 static void kvm_set_phys_mem(target_phys_addr_t start_addr,
 407                              ram_addr_t size,
 408                              ram_addr_t phys_offset)
 409 {
 410     KVMState *s = kvm_state;
 411     ram_addr_t flags = phys_offset & ~TARGET_PAGE_MASK;
 412     KVMSlot *mem, old;
 413     int err;
 414
 415     if (start_addr & ~TARGET_PAGE_MASK) {
 416         if (flags >= IO_MEM_UNASSIGNED) {
 417             if (!kvm_lookup_overlapping_slot(s, start_addr,
 418                                              start_addr + size)) {
 419                 return;
 420             }
 421             fprintf(stderr, "Unaligned split of a KVM memory slot\n");
 422         } else {
 423             fprintf(stderr, "Only page-aligned memory slots supported\n");
 424         }
 425         abort();
 426     }
 427
 428     /* KVM does not support read-only slots */
 429     phys_offset &= ~IO_MEM_ROM;
 430
 431     while (1) {
 432         mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size);
 433         if (!mem) {
 434             break;
 435         }
 436
 437         if (flags < IO_MEM_UNASSIGNED && start_addr >= mem->start_addr &&
 438             (start_addr + size <= mem->start_addr + mem->memory_size) &&
 439             (phys_offset - start_addr == mem->phys_offset - mem->start_addr)) {
 440             /* The new slot fits into the existing one and comes with
 441              * identical parameters - nothing to be done. */
 442             return;
 443         }
 444
 445         old = *mem;
 446
 447         /* unregister the overlapping slot */
 448         mem->memory_size = 0;
 449         err = kvm_set_user_memory_region(s, mem);
 450         if (err) {
 451             fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
 452                     __func__, strerror(-err));
 453             abort();
 454         }
 455
 456         /* Workaround for older KVM versions: we can't join slots, even not by
 457          * unregistering the previous ones and then registering the larger
 458          * slot. We have to maintain the existing fragmentation. Sigh.
 459          *
 460          * This workaround assumes that the new slot starts at the same
 461          * address as the first existing one. If not or if some overlapping
 462          * slot comes around later, we will fail (not seen in practice so far)
 463          * - and actually require a recent KVM version. */
 464         if (s->broken_set_mem_region &&
 465             old.start_addr == start_addr && old.memory_size < size &&
 466             flags < IO_MEM_UNASSIGNED) {
 467             mem = kvm_alloc_slot(s);
 468             mem->memory_size = old.memory_size;
 469             mem->start_addr = old.start_addr;
 470             mem->phys_offset = old.phys_offset;
 471             mem->flags = 0;
 472
 473             err = kvm_set_user_memory_region(s, mem);
 474             if (err) {
 475                 fprintf(stderr, "%s: error updating slot: %s\n", __func__,
 476                         strerror(-err));
 477                 abort();
 478             }
 479
 480             start_addr += old.memory_size;
 481             phys_offset += old.memory_size;
 482             size -= old.memory_size;
 483             continue;
 484         }
 485
 486         /* register prefix slot */
 487         if (old.start_addr < start_addr) {
 488             mem = kvm_alloc_slot(s);
 489             mem->memory_size = start_addr - old.start_addr;
 490             mem->start_addr = old.start_addr;
 491             mem->phys_offset = old.phys_offset;
 492             mem->flags = 0;
 493
 494             err = kvm_set_user_memory_region(s, mem);
 495             if (err) {
 496                 fprintf(stderr, "%s: error registering prefix slot: %s\n",
 497                         __func__, strerror(-err));
 498                 abort();
 499             }
 500         }
 501
 502         /* register suffix slot */
 503         if (old.start_addr + old.memory_size > start_addr + size) {
 504             ram_addr_t size_delta;
 505
 506             mem = kvm_alloc_slot(s);
 507             mem->start_addr = start_addr + size;
 508             size_delta = mem->start_addr - old.start_addr;
 509             mem->memory_size = old.memory_size - size_delta;
 510             mem->phys_offset = old.phys_offset + size_delta;
 511             mem->flags = 0;
 512
 513             err = kvm_set_user_memory_region(s, mem);
 514             if (err) {
 515                 fprintf(stderr, "%s: error registering suffix slot: %s\n",
 516                         __func__, strerror(-err));
 517                 abort();
 518             }
 519         }
 520     }
 521
 522     /* in case the KVM bug workaround already "consumed" the new slot */
 523     if (!size)
 524         return;
 525
 526     /* KVM does not need to know about this memory */
 527     if (flags >= IO_MEM_UNASSIGNED)
 528         return;
 529
 530     mem = kvm_alloc_slot(s);
 531     mem->memory_size = size;
 532     mem->start_addr = start_addr;
 533     mem->phys_offset = phys_offset;
 534     mem->flags = 0;
 535
 536     err = kvm_set_user_memory_region(s, mem);
 537     if (err) {
 538         fprintf(stderr, "%s: error registering slot: %s\n", __func__,
 539                 strerror(-err));
 540         abort();
 541     }
 542 }
 543
 544 static void kvm_client_set_memory(struct CPUPhysMemoryClient *client,
 545                                   target_phys_addr_t start_addr,
 546                                   ram_addr_t size,
 547                                   ram_addr_t phys_offset)
 548 {
 549         kvm_set_phys_mem(start_addr, size, phys_offset);
 550 }
 551
 552 static int kvm_client_sync_dirty_bitmap(struct CPUPhysMemoryClient *client,
 553                                         target_phys_addr_t start_addr,
 554                                         target_phys_addr_t end_addr)
 555 {
 556         return kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
 557 }
 558
 559 static int kvm_client_migration_log(struct CPUPhysMemoryClient *client,
 560                                     int enable)
 561 {
 562         return kvm_set_migration_log(enable);
 563 }
 564
 565 static CPUPhysMemoryClient kvm_cpu_phys_memory_client = {
 566         .set_memory = kvm_client_set_memory,
 567         .sync_dirty_bitmap = kvm_client_sync_dirty_bitmap,
 568         .migration_log = kvm_client_migration_log,
 569 };
 570
 571 int kvm_init(int smp_cpus)
 572 {
 573     static const char upgrade_note[] =
 574         "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
 575         "(see http://sourceforge.net/projects/kvm).\n";
 576     KVMState *s;
 577     int ret;
 578     int i;
 579
 580     if (smp_cpus > 1) {
 581         fprintf(stderr, "No SMP KVM support, use '-smp 1'\n");
 582         return -EINVAL;
 583     }
 584
 585     s = qemu_mallocz(sizeof(KVMState));
 586
 587 #ifdef KVM_CAP_SET_GUEST_DEBUG
 588     QTAILQ_INIT(&s->kvm_sw_breakpoints);
 589 #endif
 590     for (i = 0; i < ARRAY_SIZE(s->slots); i++)
 591         s->slots[i].slot = i;
 592
 593     s->vmfd = -1;
 594     s->fd = qemu_open("/dev/kvm", O_RDWR);
 595     if (s->fd == -1) {
 596         fprintf(stderr, "Could not access KVM kernel module: %m\n");
 597         ret = -errno;
 598         goto err;
 599     }
 600
 601     ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
 602     if (ret < KVM_API_VERSION) {
 603         if (ret > 0)
 604             ret = -EINVAL;
 605         fprintf(stderr, "kvm version too old\n");
 606         goto err;
 607     }
 608
 609     if (ret > KVM_API_VERSION) {
 610         ret = -EINVAL;
 611         fprintf(stderr, "kvm version not supported\n");
 612         goto err;
 613     }
 614
 615     s->vmfd = kvm_ioctl(s, KVM_CREATE_VM, 0);
 616     if (s->vmfd < 0)
 617         goto err;
 618
 619     /* initially, KVM allocated its own memory and we had to jump through
 620      * hooks to make phys_ram_base point to this.  Modern versions of KVM
 621      * just use a user allocated buffer so we can use regular pages
 622      * unmodified.  Make sure we have a sufficiently modern version of KVM.
 623      */
 624     if (!kvm_check_extension(s, KVM_CAP_USER_MEMORY)) {
 625         ret = -EINVAL;
 626         fprintf(stderr, "kvm does not support KVM_CAP_USER_MEMORY\n%s",
 627                 upgrade_note);
 628         goto err;
 629     }
 630
 631     /* There was a nasty bug in < kvm-80 that prevents memory slots from being
 632      * destroyed properly.  Since we rely on this capability, refuse to work
 633      * with any kernel without this capability. */
 634     if (!kvm_check_extension(s, KVM_CAP_DESTROY_MEMORY_REGION_WORKS)) {
 635         ret = -EINVAL;
 636
 637         fprintf(stderr,
 638                 "KVM kernel module broken (DESTROY_MEMORY_REGION).\n%s",
 639                 upgrade_note);
 640         goto err;
 641     }
 642
 643     s->coalesced_mmio = 0;
 644 #ifdef KVM_CAP_COALESCED_MMIO
 645     s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
 646     s->coalesced_mmio_ring = NULL;
 647 #endif
 648
 649     s->broken_set_mem_region = 1;
 650 #ifdef KVM_CAP_JOIN_MEMORY_REGIONS_WORKS
 651     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, KVM_CAP_JOIN_MEMORY_REGIONS_WORKS);
 652     if (ret > 0) {
 653         s->broken_set_mem_region = 0;
 654     }
 655 #endif
 656
 657     s->vcpu_events = 0;
 658 #ifdef KVM_CAP_VCPU_EVENTS
 659     s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
 660 #endif
 661
 662     ret = kvm_arch_init(s, smp_cpus);
 663     if (ret < 0)
 664         goto err;
 665
 666     kvm_state = s;
 667     cpu_register_phys_memory_client(&kvm_cpu_phys_memory_client);
 668
 669     return 0;
 670
 671 err:
 672     if (s) {
 673         if (s->vmfd != -1)
 674             close(s->vmfd);
 675         if (s->fd != -1)
 676             close(s->fd);
 677     }
 678     qemu_free(s);
 679
 680     return ret;
 681 }
 682
 683 static int kvm_handle_io(uint16_t port, void *data, int direction, int size,
 684                          uint32_t count)
 685 {
 686     int i;
 687     uint8_t *ptr = data;
 688
 689     for (i = 0; i < count; i++) {
 690         if (direction == KVM_EXIT_IO_IN) {
 691             switch (size) {
 692             case 1:
 693                 stb_p(ptr, cpu_inb(port));
 694                 break;
 695             case 2:
 696                 stw_p(ptr, cpu_inw(port));
 697                 break;
 698             case 4:
 699                 stl_p(ptr, cpu_inl(port));
 700                 break;
 701             }
 702         } else {
 703             switch (size) {
 704             case 1:
 705                 cpu_outb(port, ldub_p(ptr));
 706                 break;
 707             case 2:
 708                 cpu_outw(port, lduw_p(ptr));
 709                 break;
 710             case 4:
 711                 cpu_outl(port, ldl_p(ptr));
 712                 break;
 713             }
 714         }
 715
 716         ptr += size;
 717     }
 718
 719     return 1;
 720 }
 721
 722 void kvm_flush_coalesced_mmio_buffer(void)
 723 {
 724 #ifdef KVM_CAP_COALESCED_MMIO
 725     KVMState *s = kvm_state;
 726     if (s->coalesced_mmio_ring) {
 727         struct kvm_coalesced_mmio_ring *ring = s->coalesced_mmio_ring;
 728         while (ring->first != ring->last) {
 729             struct kvm_coalesced_mmio *ent;
 730
 731             ent = &ring->coalesced_mmio[ring->first];
 732
 733             cpu_physical_memory_write(ent->phys_addr, ent->data, ent->len);
 734             smp_wmb();
 735             ring->first = (ring->first + 1) % KVM_COALESCED_MMIO_MAX;
 736         }
 737     }
 738 #endif
 739 }
 740
 741 void kvm_cpu_synchronize_state(CPUState *env)
 742 {
 743     if (!env->kvm_vcpu_dirty) {
 744         kvm_arch_get_registers(env);
 745         env->kvm_vcpu_dirty = 1;
 746     }
 747 }
 748
 749 int kvm_cpu_exec(CPUState *env)
 750 {
 751     struct kvm_run *run = env->kvm_run;
 752     int ret;
 753
 754     dprintf("kvm_cpu_exec()\n");
 755
 756     do {
 757 #ifndef CONFIG_IOTHREAD
 758         if (env->exit_request) {
 759             dprintf("interrupt exit requested\n");
 760             ret = 0;
 761             break;
 762         }
 763 #endif
 764
 765         if (env->kvm_vcpu_dirty) {
 766             kvm_arch_put_registers(env);
 767             env->kvm_vcpu_dirty = 0;
 768         }
 769
 770         kvm_arch_pre_run(env, run);
 771         qemu_mutex_unlock_iothread();
 772         ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);
 773         qemu_mutex_lock_iothread();
 774         kvm_arch_post_run(env, run);
 775
 776         if (ret == -EINTR || ret == -EAGAIN) {
 777             cpu_exit(env);
 778             dprintf("io window exit\n");
 779             ret = 0;
 780             break;
 781         }
 782
 783         if (ret < 0) {
 784             dprintf("kvm run failed %s\n", strerror(-ret));
 785             abort();
 786         }
 787
 788         kvm_flush_coalesced_mmio_buffer();
 789
 790         ret = 0; /* exit loop */
 791         switch (run->exit_reason) {
 792         case KVM_EXIT_IO:
 793             dprintf("handle_io\n");
 794             ret = kvm_handle_io(run->io.port,
 795                                 (uint8_t *)run + run->io.data_offset,
 796                                 run->io.direction,
 797                                 run->io.size,
 798                                 run->io.count);
 799             break;
 800         case KVM_EXIT_MMIO:
 801             dprintf("handle_mmio\n");
 802             cpu_physical_memory_rw(run->mmio.phys_addr,
 803                                    run->mmio.data,
 804                                    run->mmio.len,
 805                                    run->mmio.is_write);
 806             ret = 1;
 807             break;
 808         case KVM_EXIT_IRQ_WINDOW_OPEN:
 809             dprintf("irq_window_open\n");
 810             break;
 811         case KVM_EXIT_SHUTDOWN:
 812             dprintf("shutdown\n");
 813             qemu_system_reset_request();
 814             ret = 1;
 815             break;
 816         case KVM_EXIT_UNKNOWN:
 817             dprintf("kvm_exit_unknown\n");
 818             break;
 819         case KVM_EXIT_FAIL_ENTRY:
 820             dprintf("kvm_exit_fail_entry\n");
 821             break;
 822         case KVM_EXIT_EXCEPTION:
 823             dprintf("kvm_exit_exception\n");
 824             break;
 825         case KVM_EXIT_DEBUG:
 826             dprintf("kvm_exit_debug\n");
 827 #ifdef KVM_CAP_SET_GUEST_DEBUG
 828             if (kvm_arch_debug(&run->debug.arch)) {
 829                 gdb_set_stop_cpu(env);
 830                 vm_stop(EXCP_DEBUG);
 831                 env->exception_index = EXCP_DEBUG;
 832                 return 0;
 833             }
 834             /* re-enter, this exception was guest-internal */
 835             ret = 1;
 836 #endif /* KVM_CAP_SET_GUEST_DEBUG */
 837             break;
 838         default:
 839             dprintf("kvm_arch_handle_exit\n");
 840             ret = kvm_arch_handle_exit(env, run);
 841             break;
 842         }
 843     } while (ret > 0);
 844
 845     if (env->exit_request) {
 846         env->exit_request = 0;
 847         env->exception_index = EXCP_INTERRUPT;
 848     }
 849
 850     return ret;
 851 }
 852
 853 int kvm_ioctl(KVMState *s, int type, ...)
 854 {
 855     int ret;
 856     void *arg;
 857     va_list ap;
 858
 859     va_start(ap, type);
 860     arg = va_arg(ap, void *);
 861     va_end(ap);
 862
 863     ret = ioctl(s->fd, type, arg);
 864     if (ret == -1)
 865         ret = -errno;
 866
 867     return ret;
 868 }
 869
 870 int kvm_vm_ioctl(KVMState *s, int type, ...)
 871 {
 872     int ret;
 873     void *arg;
 874     va_list ap;
 875
 876     va_start(ap, type);
 877     arg = va_arg(ap, void *);
 878     va_end(ap);
 879
 880     ret = ioctl(s->vmfd, type, arg);
 881     if (ret == -1)
 882         ret = -errno;
 883
 884     return ret;
 885 }
 886
 887 int kvm_vcpu_ioctl(CPUState *env, int type, ...)
 888 {
 889     int ret;
 890     void *arg;
 891     va_list ap;
 892
 893     va_start(ap, type);
 894     arg = va_arg(ap, void *);
 895     va_end(ap);
 896
 897     ret = ioctl(env->kvm_fd, type, arg);
 898     if (ret == -1)
 899         ret = -errno;
 900
 901     return ret;
 902 }
 903
 904 int kvm_has_sync_mmu(void)
 905 {
 906 #ifdef KVM_CAP_SYNC_MMU
 907     KVMState *s = kvm_state;
 908
 909     return kvm_check_extension(s, KVM_CAP_SYNC_MMU);
 910 #else
 911     return 0;
 912 #endif
 913 }
 914
 915 int kvm_has_vcpu_events(void)
 916 {
 917     return kvm_state->vcpu_events;
 918 }
 919
 920 void kvm_setup_guest_memory(void *start, size_t size)
 921 {
 922     if (!kvm_has_sync_mmu()) {
 923 #ifdef MADV_DONTFORK
 924         int ret = madvise(start, size, MADV_DONTFORK);
 925
 926         if (ret) {
 927             perror("madvice");
 928             exit(1);
 929         }
 930 #else
 931         fprintf(stderr,
 932                 "Need MADV_DONTFORK in absence of synchronous KVM MMU\n");
 933         exit(1);
 934 #endif
 935     }
 936 }
 937
 938 #ifdef KVM_CAP_SET_GUEST_DEBUG
 939 static void on_vcpu(CPUState *env, void (*func)(void *data), void *data)
 940 {
 941 #ifdef CONFIG_IOTHREAD
 942     if (env != cpu_single_env) {
 943         abort();
 944     }
 945 #endif
 946     func(data);
 947 }
 948
 949 struct kvm_sw_breakpoint *kvm_find_sw_breakpoint(CPUState *env,
 950                                                  target_ulong pc)
 951 {
 952     struct kvm_sw_breakpoint *bp;
 953
 954     QTAILQ_FOREACH(bp, &env->kvm_state->kvm_sw_breakpoints, entry) {
 955         if (bp->pc == pc)
 956             return bp;
 957     }
 958     return NULL;
 959 }
 960
 961 int kvm_sw_breakpoints_active(CPUState *env)
 962 {
 963     return !QTAILQ_EMPTY(&env->kvm_state->kvm_sw_breakpoints);
 964 }
 965
 966 struct kvm_set_guest_debug_data {
 967     struct kvm_guest_debug dbg;
 968     CPUState *env;
 969     int err;
 970 };
 971
 972 static void kvm_invoke_set_guest_debug(void *data)
 973 {
 974     struct kvm_set_guest_debug_data *dbg_data = data;
 975     CPUState *env = dbg_data->env;
 976
 977     if (env->kvm_vcpu_dirty) {
 978         kvm_arch_put_registers(env);
 979         env->kvm_vcpu_dirty = 0;
 980     }
 981     dbg_data->err = kvm_vcpu_ioctl(env, KVM_SET_GUEST_DEBUG, &dbg_data->dbg);
 982 }
 983
 984 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
 985 {
 986     struct kvm_set_guest_debug_data data;
 987
 988     data.dbg.control = 0;
 989     if (env->singlestep_enabled)
 990         data.dbg.control = KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP;
 991
 992     kvm_arch_update_guest_debug(env, &data.dbg);
 993     data.dbg.control |= reinject_trap;
 994     data.env = env;
 995
 996     on_vcpu(env, kvm_invoke_set_guest_debug, &data);
 997     return data.err;
 998 }
 999
1000 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1001                           target_ulong len, int type)
1002 {
1003     struct kvm_sw_breakpoint *bp;
1004     CPUState *env;
1005     int err;
1006
1007     if (type == GDB_BREAKPOINT_SW) {
1008         bp = kvm_find_sw_breakpoint(current_env, addr);
1009         if (bp) {
1010             bp->use_count++;
1011             return 0;
1012         }
1013
1014         bp = qemu_malloc(sizeof(struct kvm_sw_breakpoint));
1015         if (!bp)
1016             return -ENOMEM;
1017
1018         bp->pc = addr;
1019         bp->use_count = 1;
1020         err = kvm_arch_insert_sw_breakpoint(current_env, bp);
1021         if (err) {
1022             free(bp);
1023             return err;
1024         }
1025
1026         QTAILQ_INSERT_HEAD(&current_env->kvm_state->kvm_sw_breakpoints,
1027                           bp, entry);
1028     } else {
1029         err = kvm_arch_insert_hw_breakpoint(addr, len, type);
1030         if (err)
1031             return err;
1032     }
1033
1034     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1035         err = kvm_update_guest_debug(env, 0);
1036         if (err)
1037             return err;
1038     }
1039     return 0;
1040 }
1041
1042 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1043                           target_ulong len, int type)
1044 {
1045     struct kvm_sw_breakpoint *bp;
1046     CPUState *env;
1047     int err;
1048
1049     if (type == GDB_BREAKPOINT_SW) {
1050         bp = kvm_find_sw_breakpoint(current_env, addr);
1051         if (!bp)
1052             return -ENOENT;
1053
1054         if (bp->use_count > 1) {
1055             bp->use_count--;
1056             return 0;
1057         }
1058
1059         err = kvm_arch_remove_sw_breakpoint(current_env, bp);
1060         if (err)
1061             return err;
1062
1063         QTAILQ_REMOVE(&current_env->kvm_state->kvm_sw_breakpoints, bp, entry);
1064         qemu_free(bp);
1065     } else {
1066         err = kvm_arch_remove_hw_breakpoint(addr, len, type);
1067         if (err)
1068             return err;
1069     }
1070
1071     for (env = first_cpu; env != NULL; env = env->next_cpu) {
1072         err = kvm_update_guest_debug(env, 0);
1073         if (err)
1074             return err;
1075     }
1076     return 0;
1077 }
1078
1079 void kvm_remove_all_breakpoints(CPUState *current_env)
1080 {
1081     struct kvm_sw_breakpoint *bp, *next;
1082     KVMState *s = current_env->kvm_state;
1083     CPUState *env;
1084
1085     QTAILQ_FOREACH_SAFE(bp, &s->kvm_sw_breakpoints, entry, next) {
1086         if (kvm_arch_remove_sw_breakpoint(current_env, bp) != 0) {
1087             /* Try harder to find a CPU that currently sees the breakpoint. */
1088             for (env = first_cpu; env != NULL; env = env->next_cpu) {
1089                 if (kvm_arch_remove_sw_breakpoint(env, bp) == 0)
1090                     break;
1091             }
1092         }
1093     }
1094     kvm_arch_remove_all_hw_breakpoints();
1095
1096     for (env = first_cpu; env != NULL; env = env->next_cpu)
1097         kvm_update_guest_debug(env, 0);
1098 }
1099
1100 #else /* !KVM_CAP_SET_GUEST_DEBUG */
1101
1102 int kvm_update_guest_debug(CPUState *env, unsigned long reinject_trap)
1103 {
1104     return -EINVAL;
1105 }
1106
1107 int kvm_insert_breakpoint(CPUState *current_env, target_ulong addr,
1108                           target_ulong len, int type)
1109 {
1110     return -EINVAL;
1111 }
1112
1113 int kvm_remove_breakpoint(CPUState *current_env, target_ulong addr,
1114                           target_ulong len, int type)
1115 {
1116     return -EINVAL;
1117 }
1118
1119 void kvm_remove_all_breakpoints(CPUState *current_env)
1120 {
1121 }
1122 #endif /* !KVM_CAP_SET_GUEST_DEBUG */
1123
1124 int kvm_set_signal_mask(CPUState *env, const sigset_t *sigset)
1125 {
1126     struct kvm_signal_mask *sigmask;
1127     int r;
1128
1129     if (!sigset)
1130         return kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, NULL);
1131
1132     sigmask = qemu_malloc(sizeof(*sigmask) + sizeof(*sigset));
1133
1134     sigmask->len = 8;
1135     memcpy(sigmask->sigset, sigset, sizeof(*sigset));
1136     r = kvm_vcpu_ioctl(env, KVM_SET_SIGNAL_MASK, sigmask);
1137     free(sigmask);
1138
1139     return r;
1140 }