kvm/libkvm/libkvm.c

   1 /*
   2  * Kernel-based Virtual Machine control library
   3  *
   4  * This library provides an API to control the kvm hardware virtualization
   5  * module.
   6  *
   7  * Copyright (C) 2006 Qumranet
   8  *
   9  * Authors:
  10  *
  11  *  Avi Kivity <avi@qumranet.com>
  12  *  Yaniv Kamay <yaniv@qumranet.com>
  13  *
  14  * This work is licensed under the GNU LGPL license, version 2.
  15  */
  16
  17 #ifndef __user
  18 #define __user /* temporary, until installed via make headers_install */
  19 #endif
  20
  21 #include <linux/kvm.h>
  22
  23 #define EXPECTED_KVM_API_VERSION 12
  24
  25 #if EXPECTED_KVM_API_VERSION != KVM_API_VERSION
  26 #error libkvm: userspace and kernel version mismatch
  27 #endif
  28
  29 #include <unistd.h>
  30 #include <fcntl.h>
  31 #include <stdio.h>
  32 #include <stdlib.h>
  33 #include <sys/mman.h>
  34 #include <string.h>
  35 #include <errno.h>
  36 #include <sys/ioctl.h>
  37 #include <inttypes.h>
  38 #include "libkvm.h"
  39
  40 #if defined(__x86_64__) || defined(__i386__)
  41 #include "kvm-x86.h"
  42 #endif
  43
  44 #if defined(__ia64__)
  45 #include "kvm-ia64.h"
  46 #endif
  47
  48 #if defined(__powerpc__)
  49 #include "kvm-powerpc.h"
  50 #endif
  51
  52 #if defined(__s390__)
  53 #include "kvm-s390.h"
  54 #endif
  55
  56 //#define DEBUG_MEMREG
  57 #ifdef  DEBUG_MEMREG
  58 #define DPRINTF(fmt, args...) \
  59         do { fprintf(stderr, "%s:%d " fmt , __func__, __LINE__, ##args); } while (0)
  60 #else
  61 #define DPRINTF(fmt, args...) do {} while (0)
  62 #endif
  63
  64
  65 int kvm_abi = EXPECTED_KVM_API_VERSION;
  66 int kvm_page_size;
  67
  68 struct slot_info {
  69         unsigned long phys_addr;
  70         unsigned long len;
  71         unsigned long userspace_addr;
  72         unsigned flags;
  73         int logging_count;
  74 };
  75
  76 struct slot_info slots[KVM_MAX_NUM_MEM_REGIONS];
  77
  78 static void init_slots(void)
  79 {
  80         int i;
  81
  82         for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS; ++i)
  83                 slots[i].len = 0;
  84 }
  85
  86 static int get_free_slot(kvm_context_t kvm)
  87 {
  88         int i;
  89         int tss_ext;
  90
  91 #if defined(KVM_CAP_SET_TSS_ADDR) && !defined(__s390__)
  92         tss_ext = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_SET_TSS_ADDR);
  93 #else
  94         tss_ext = 0;
  95 #endif
  96
  97         /*
  98          * on older kernels where the set tss ioctl is not supprted we must save
  99          * slot 0 to hold the extended memory, as the vmx will use the last 3
 100          * pages of this slot.
 101          */
 102         if (tss_ext > 0)
 103                 i = 0;
 104         else
 105                 i = 1;
 106
 107         for (; i < KVM_MAX_NUM_MEM_REGIONS; ++i)
 108                 if (!slots[i].len)
 109                         return i;
 110         return -1;
 111 }
 112
 113 static void register_slot(int slot, unsigned long phys_addr, unsigned long len,
 114                    unsigned long userspace_addr, unsigned flags)
 115 {
 116         slots[slot].phys_addr = phys_addr;
 117         slots[slot].len = len;
 118         slots[slot].userspace_addr = userspace_addr;
 119         slots[slot].flags = flags;
 120 }
 121
 122 static void free_slot(int slot)
 123 {
 124         slots[slot].len = 0;
 125         slots[slot].logging_count = 0;
 126 }
 127
 128 static int get_slot(unsigned long phys_addr)
 129 {
 130         int i;
 131
 132         for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS ; ++i) {
 133                 if (slots[i].len && slots[i].phys_addr <= phys_addr &&
 134                     (slots[i].phys_addr + slots[i].len-1) >= phys_addr)
 135                         return i;
 136         }
 137         return -1;
 138 }
 139
 140 /* Returns -1 if this slot is not totally contained on any other,
 141  * and the number of the slot otherwise */
 142 static int get_container_slot(uint64_t phys_addr, unsigned long size)
 143 {
 144         int i;
 145
 146         for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS ; ++i)
 147                 if (slots[i].len && slots[i].phys_addr <= phys_addr &&
 148                     (slots[i].phys_addr + slots[i].len) >= phys_addr + size)
 149                         return i;
 150         return -1;
 151 }
 152
 153 int kvm_is_containing_region(kvm_context_t kvm, unsigned long phys_addr, unsigned long size)
 154 {
 155         int slot = get_container_slot(phys_addr, size);
 156         if (slot == -1)
 157                 return 0;
 158         return 1;
 159 }
 160
 161 /*
 162  * dirty pages logging control
 163  */
 164 static int kvm_dirty_pages_log_change(kvm_context_t kvm,
 165                                       unsigned long phys_addr,
 166                                       unsigned flags,
 167                                       unsigned mask)
 168 {
 169         int r = -1;
 170         int slot = get_slot(phys_addr);
 171
 172         if (slot == -1) {
 173                 fprintf(stderr, "BUG: %s: invalid parameters\n", __FUNCTION__);
 174                 return 1;
 175         }
 176
 177         flags = (slots[slot].flags & ~mask) | flags;
 178         if (flags == slots[slot].flags)
 179                 return 0;
 180         slots[slot].flags = flags;
 181
 182         {
 183                 struct kvm_userspace_memory_region mem = {
 184                         .slot = slot,
 185                         .memory_size = slots[slot].len,
 186                         .guest_phys_addr = slots[slot].phys_addr,
 187                         .userspace_addr = slots[slot].userspace_addr,
 188                         .flags = slots[slot].flags,
 189                 };
 190
 191
 192                 DPRINTF("slot %d start %llx len %llx flags %x\n",
 193                         mem.slot,
 194                         mem.guest_phys_addr,
 195                         mem.memory_size,
 196                         mem.flags);
 197                 r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &mem);
 198                 if (r == -1)
 199                         fprintf(stderr, "%s: %m\n", __FUNCTION__);
 200         }
 201         return r;
 202 }
 203
 204 static int kvm_dirty_pages_log_change_all(kvm_context_t kvm,
 205                                           int (*change)(kvm_context_t kvm,
 206                                                         uint64_t start,
 207                                                         uint64_t len))
 208 {
 209         int i, r;
 210
 211         for (i=r=0; i<KVM_MAX_NUM_MEM_REGIONS && r==0; i++) {
 212                 if (slots[i].len)
 213                         r = change(kvm, slots[i].phys_addr, slots[i].len);
 214         }
 215         return r;
 216 }
 217
 218 int kvm_dirty_pages_log_enable_slot(kvm_context_t kvm,
 219                                     uint64_t phys_addr,
 220                                     uint64_t len)
 221 {
 222         int slot = get_slot(phys_addr);
 223
 224         DPRINTF("start %"PRIx64" len %"PRIx64"\n", phys_addr, len);
 225         if (slot == -1) {
 226                 fprintf(stderr, "BUG: %s: invalid parameters\n", __func__);
 227                 return -EINVAL;
 228         }
 229
 230         if (slots[slot].logging_count++)
 231                 return 0;
 232
 233         return kvm_dirty_pages_log_change(kvm, slots[slot].phys_addr,
 234                                           KVM_MEM_LOG_DIRTY_PAGES,
 235                                           KVM_MEM_LOG_DIRTY_PAGES);
 236 }
 237
 238 int kvm_dirty_pages_log_disable_slot(kvm_context_t kvm,
 239                                      uint64_t phys_addr,
 240                                      uint64_t len)
 241 {
 242         int slot = get_slot(phys_addr);
 243
 244         if (slot == -1) {
 245                 fprintf(stderr, "BUG: %s: invalid parameters\n", __func__);
 246                 return -EINVAL;
 247         }
 248
 249         if (--slots[slot].logging_count)
 250                 return 0;
 251
 252         return kvm_dirty_pages_log_change(kvm, slots[slot].phys_addr,
 253                                           0,
 254                                           KVM_MEM_LOG_DIRTY_PAGES);
 255 }
 256
 257 /**
 258  * Enable dirty page logging for all memory regions
 259  */
 260 int kvm_dirty_pages_log_enable_all(kvm_context_t kvm)
 261 {
 262         if (kvm->dirty_pages_log_all)
 263                 return 0;
 264         kvm->dirty_pages_log_all = 1;
 265         return kvm_dirty_pages_log_change_all(kvm,
 266                                               kvm_dirty_pages_log_enable_slot);
 267 }
 268
 269 /**
 270  * Enable dirty page logging only for memory regions that were created with
 271  *     dirty logging enabled (disable for all other memory regions).
 272  */
 273 int kvm_dirty_pages_log_reset(kvm_context_t kvm)
 274 {
 275         if (!kvm->dirty_pages_log_all)
 276                 return 0;
 277         kvm->dirty_pages_log_all = 0;
 278         return kvm_dirty_pages_log_change_all(kvm,
 279                                               kvm_dirty_pages_log_disable_slot);
 280 }
 281
 282
 283 kvm_context_t kvm_init(struct kvm_callbacks *callbacks,
 284                        void *opaque)
 285 {
 286         int fd;
 287         kvm_context_t kvm;
 288         int r;
 289
 290         fd = open("/dev/kvm", O_RDWR);
 291         if (fd == -1) {
 292                 perror("open /dev/kvm");
 293                 return NULL;
 294         }
 295         r = ioctl(fd, KVM_GET_API_VERSION, 0);
 296         if (r == -1) {
 297             fprintf(stderr, "kvm kernel version too old: "
 298                     "KVM_GET_API_VERSION ioctl not supported\n");
 299             goto out_close;
 300         }
 301         if (r < EXPECTED_KVM_API_VERSION) {
 302                 fprintf(stderr, "kvm kernel version too old: "
 303                         "We expect API version %d or newer, but got "
 304                         "version %d\n",
 305                         EXPECTED_KVM_API_VERSION, r);
 306             goto out_close;
 307         }
 308         if (r > EXPECTED_KVM_API_VERSION) {
 309             fprintf(stderr, "kvm userspace version too old\n");
 310             goto out_close;
 311         }
 312         kvm_abi = r;
 313         kvm_page_size = getpagesize();
 314         kvm = malloc(sizeof(*kvm));
 315         if (kvm == NULL)
 316                 goto out_close;
 317         memset(kvm, 0, sizeof(*kvm));
 318         kvm->fd = fd;
 319         kvm->vm_fd = -1;
 320         kvm->callbacks = callbacks;
 321         kvm->opaque = opaque;
 322         kvm->dirty_pages_log_all = 0;
 323         kvm->no_irqchip_creation = 0;
 324         kvm->no_pit_creation = 0;
 325
 326         return kvm;
 327  out_close:
 328         close(fd);
 329         return NULL;
 330 }
 331
 332 void kvm_finalize(kvm_context_t kvm)
 333 {
 334         if (kvm->vcpu_fd[0] != -1)
 335                 close(kvm->vcpu_fd[0]);
 336         if (kvm->vm_fd != -1)
 337                 close(kvm->vm_fd);
 338         close(kvm->fd);
 339         free(kvm);
 340 }
 341
 342 void kvm_disable_irqchip_creation(kvm_context_t kvm)
 343 {
 344         kvm->no_irqchip_creation = 1;
 345 }
 346
 347 void kvm_disable_pit_creation(kvm_context_t kvm)
 348 {
 349         kvm->no_pit_creation = 1;
 350 }
 351
 352 int kvm_create_vcpu(kvm_context_t kvm, int slot)
 353 {
 354         long mmap_size;
 355         int r;
 356
 357         r = ioctl(kvm->vm_fd, KVM_CREATE_VCPU, slot);
 358         if (r == -1) {
 359                 r = -errno;
 360                 fprintf(stderr, "kvm_create_vcpu: %m\n");
 361                 return r;
 362         }
 363         kvm->vcpu_fd[slot] = r;
 364         mmap_size = ioctl(kvm->fd, KVM_GET_VCPU_MMAP_SIZE, 0);
 365         if (mmap_size == -1) {
 366                 r = -errno;
 367                 fprintf(stderr, "get vcpu mmap size: %m\n");
 368                 return r;
 369         }
 370         kvm->run[slot] = mmap(NULL, mmap_size, PROT_READ|PROT_WRITE, MAP_SHARED,
 371                               kvm->vcpu_fd[slot], 0);
 372         if (kvm->run[slot] == MAP_FAILED) {
 373                 r = -errno;
 374                 fprintf(stderr, "mmap vcpu area: %m\n");
 375                 return r;
 376         }
 377         return 0;
 378 }
 379
 380 int kvm_create_vm(kvm_context_t kvm)
 381 {
 382         int fd = kvm->fd;
 383
 384 #ifdef KVM_CAP_IRQ_ROUTING
 385         kvm->irq_routes = malloc(sizeof(*kvm->irq_routes));
 386         if (!kvm->irq_routes)
 387                 return -ENOMEM;
 388         memset(kvm->irq_routes, 0, sizeof(*kvm->irq_routes));
 389         kvm->nr_allocated_irq_routes = 0;
 390 #endif
 391
 392         kvm->vcpu_fd[0] = -1;
 393
 394         fd = ioctl(fd, KVM_CREATE_VM, 0);
 395         if (fd == -1) {
 396                 fprintf(stderr, "kvm_create_vm: %m\n");
 397                 return -1;
 398         }
 399         kvm->vm_fd = fd;
 400         return 0;
 401 }
 402
 403 static int kvm_create_default_phys_mem(kvm_context_t kvm,
 404                                        unsigned long phys_mem_bytes,
 405                                        void **vm_mem)
 406 {
 407 #ifdef KVM_CAP_USER_MEMORY
 408         int r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_USER_MEMORY);
 409         if (r > 0)
 410                 return 0;
 411         fprintf(stderr, "Hypervisor too old: KVM_CAP_USER_MEMORY extension not supported\n");
 412 #else
 413 #error Hypervisor too old: KVM_CAP_USER_MEMORY extension not supported
 414 #endif
 415         return -1;
 416 }
 417
 418 int kvm_check_extension(kvm_context_t kvm, int ext)
 419 {
 420         int ret;
 421
 422         ret = ioctl(kvm->fd, KVM_CHECK_EXTENSION, ext);
 423         if (ret > 0)
 424                 return 1;
 425         return 0;
 426 }
 427
 428 void kvm_create_irqchip(kvm_context_t kvm)
 429 {
 430         int r;
 431
 432         kvm->irqchip_in_kernel = 0;
 433 #ifdef KVM_CAP_IRQCHIP
 434         if (!kvm->no_irqchip_creation) {
 435                 r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_IRQCHIP);
 436                 if (r > 0) {    /* kernel irqchip supported */
 437                         r = ioctl(kvm->vm_fd, KVM_CREATE_IRQCHIP);
 438                         if (r >= 0) {
 439                                 kvm->irqchip_inject_ioctl = KVM_IRQ_LINE;
 440 #if defined(KVM_CAP_IRQ_INJECT_STATUS) && defined(KVM_IRQ_LINE_STATUS)
 441                                 r = ioctl(kvm->fd, KVM_CHECK_EXTENSION,
 442                           KVM_CAP_IRQ_INJECT_STATUS);
 443                                 if (r > 0)
 444                                         kvm->irqchip_inject_ioctl = KVM_IRQ_LINE_STATUS;
 445 #endif
 446                                 kvm->irqchip_in_kernel = 1;
 447                         }
 448                         else
 449                                 fprintf(stderr, "Create kernel PIC irqchip failed\n");
 450                 }
 451         }
 452 #endif
 453 }
 454
 455 int kvm_create(kvm_context_t kvm, unsigned long phys_mem_bytes, void **vm_mem)
 456 {
 457         int r;
 458
 459         r = kvm_create_vm(kvm);
 460         if (r < 0)
 461                 return r;
 462         r = kvm_arch_create(kvm, phys_mem_bytes, vm_mem);
 463         if (r < 0)
 464                 return r;
 465         init_slots();
 466         r = kvm_create_default_phys_mem(kvm, phys_mem_bytes, vm_mem);
 467         if (r < 0)
 468                 return r;
 469         kvm_create_irqchip(kvm);
 470
 471         return 0;
 472 }
 473
 474
 475 void *kvm_create_phys_mem(kvm_context_t kvm, unsigned long phys_start,
 476                           unsigned long len, int log, int writable)
 477 {
 478         int r;
 479         int prot = PROT_READ;
 480         void *ptr;
 481         struct kvm_userspace_memory_region memory = {
 482                 .memory_size = len,
 483                 .guest_phys_addr = phys_start,
 484                 .flags = log ? KVM_MEM_LOG_DIRTY_PAGES : 0,
 485         };
 486
 487         if (writable)
 488                 prot |= PROT_WRITE;
 489
 490 #if !defined(__s390__)
 491         ptr = mmap(NULL, len, prot, MAP_ANONYMOUS | MAP_SHARED, -1, 0);
 492 #else
 493         ptr = mmap(LIBKVM_S390_ORIGIN, len, prot | PROT_EXEC,
 494                 MAP_FIXED | MAP_SHARED | MAP_ANONYMOUS, -1, 0);
 495 #endif
 496         if (ptr == MAP_FAILED) {
 497                 fprintf(stderr, "%s: %s", __func__, strerror(errno));
 498                 return 0;
 499         }
 500
 501         memset(ptr, 0, len);
 502
 503         memory.userspace_addr = (unsigned long)ptr;
 504         memory.slot = get_free_slot(kvm);
 505         DPRINTF("slot %d start %llx len %llx flags %x\n",
 506                 memory.slot,
 507                 memory.guest_phys_addr,
 508                 memory.memory_size,
 509                 memory.flags);
 510         r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &memory);
 511         if (r == -1) {
 512                 fprintf(stderr, "%s: %s", __func__, strerror(errno));
 513                 return 0;
 514         }
 515         register_slot(memory.slot, memory.guest_phys_addr, memory.memory_size,
 516                       memory.userspace_addr, memory.flags);
 517
 518         return ptr;
 519 }
 520
 521 int kvm_register_phys_mem(kvm_context_t kvm,
 522                           unsigned long phys_start, void *userspace_addr,
 523                           unsigned long len, int log)
 524 {
 525
 526         struct kvm_userspace_memory_region memory = {
 527                 .memory_size = len,
 528                 .guest_phys_addr = phys_start,
 529                 .userspace_addr = (unsigned long)(intptr_t)userspace_addr,
 530                 .flags = log ? KVM_MEM_LOG_DIRTY_PAGES : 0,
 531         };
 532         int r;
 533
 534         memory.slot = get_free_slot(kvm);
 535         DPRINTF("memory: gpa: %llx, size: %llx, uaddr: %llx, slot: %x, flags: %lx\n",
 536                 memory.guest_phys_addr, memory.memory_size,
 537                 memory.userspace_addr, memory.slot, memory.flags);
 538         r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &memory);
 539         if (r == -1) {
 540                 fprintf(stderr, "create_userspace_phys_mem: %s\n", strerror(errno));
 541                 return -1;
 542         }
 543         register_slot(memory.slot, memory.guest_phys_addr, memory.memory_size,
 544                       memory.userspace_addr, memory.flags);
 545         return 0;
 546 }
 547
 548
 549 /* destroy/free a whole slot.
 550  * phys_start, len and slot are the params passed to kvm_create_phys_mem()
 551  */
 552 void kvm_destroy_phys_mem(kvm_context_t kvm, unsigned long phys_start,
 553                           unsigned long len)
 554 {
 555         int slot;
 556         int r;
 557         struct kvm_userspace_memory_region memory = {
 558                 .memory_size = 0,
 559                 .guest_phys_addr = phys_start,
 560                 .userspace_addr = 0,
 561                 .flags = 0,
 562         };
 563
 564         slot = get_slot(phys_start);
 565
 566         if ((slot >= KVM_MAX_NUM_MEM_REGIONS) || (slot == -1)) {
 567                 fprintf(stderr, "BUG: %s: invalid parameters (slot=%d)\n",
 568                         __FUNCTION__, slot);
 569                 return;
 570         }
 571         if (phys_start != slots[slot].phys_addr) {
 572                 fprintf(stderr,
 573                         "WARNING: %s: phys_start is 0x%lx expecting 0x%lx\n",
 574                         __FUNCTION__, phys_start, slots[slot].phys_addr);
 575                 phys_start = slots[slot].phys_addr;
 576         }
 577
 578         memory.slot = slot;
 579         DPRINTF("slot %d start %llx len %llx flags %x\n",
 580                 memory.slot,
 581                 memory.guest_phys_addr,
 582                 memory.memory_size,
 583                 memory.flags);
 584         r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &memory);
 585         if (r == -1) {
 586                 fprintf(stderr, "destroy_userspace_phys_mem: %s",
 587                         strerror(errno));
 588                 return;
 589         }
 590
 591         free_slot(memory.slot);
 592 }
 593
 594 void kvm_unregister_memory_area(kvm_context_t kvm, uint64_t phys_addr, unsigned long size)
 595 {
 596
 597         int slot = get_container_slot(phys_addr, size);
 598
 599         if (slot != -1) {
 600                 DPRINTF("Unregistering memory region %llx (%lx)\n", phys_addr, size);
 601                 kvm_destroy_phys_mem(kvm, phys_addr, size);
 602                 return;
 603         }
 604 }
 605
 606 static int kvm_get_map(kvm_context_t kvm, int ioctl_num, int slot, void *buf)
 607 {
 608         int r;
 609         struct kvm_dirty_log log = {
 610                 .slot = slot,
 611         };
 612
 613         log.dirty_bitmap = buf;
 614
 615         r = ioctl(kvm->vm_fd, ioctl_num, &log);
 616         if (r == -1)
 617                 return -errno;
 618         return 0;
 619 }
 620
 621 int kvm_get_dirty_pages(kvm_context_t kvm, unsigned long phys_addr, void *buf)
 622 {
 623         int slot;
 624
 625         slot = get_slot(phys_addr);
 626         return kvm_get_map(kvm, KVM_GET_DIRTY_LOG, slot, buf);
 627 }
 628
 629 #define ALIGN(x, y)  (((x)+(y)-1) & ~((y)-1))
 630 #define BITMAP_SIZE(m) (ALIGN(((m)/PAGE_SIZE), sizeof(long) * 8) / 8)
 631
 632 int kvm_get_dirty_pages_range(kvm_context_t kvm, unsigned long phys_addr,
 633                               unsigned long len, void *buf, void *opaque,
 634                               int (*cb)(unsigned long start, unsigned long len,
 635                                         void*bitmap, void *opaque))
 636 {
 637         int i;
 638         int r;
 639         unsigned long end_addr = phys_addr + len;
 640
 641         for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS; ++i) {
 642                 if ((slots[i].len && (uint64_t)slots[i].phys_addr >= phys_addr)
 643                     && ((uint64_t)slots[i].phys_addr + slots[i].len  <= end_addr)) {
 644                         r = kvm_get_map(kvm, KVM_GET_DIRTY_LOG, i, buf);
 645                         if (r)
 646                                 return r;
 647                         r = cb(slots[i].phys_addr, slots[i].len, buf, opaque);
 648                         if (r)
 649                                 return r;
 650                 }
 651         }
 652         return 0;
 653 }
 654
 655 #ifdef KVM_CAP_IRQCHIP
 656
 657 int kvm_set_irq_level(kvm_context_t kvm, int irq, int level, int *status)
 658 {
 659         struct kvm_irq_level event;
 660         int r;
 661
 662         if (!kvm->irqchip_in_kernel)
 663                 return 0;
 664         event.level = level;
 665         event.irq = irq;
 666         r = ioctl(kvm->vm_fd, kvm->irqchip_inject_ioctl, &event);
 667         if (r == -1)
 668                 perror("kvm_set_irq_level");
 669
 670         if (status)
 671                 *status = (kvm->irqchip_inject_ioctl == KVM_IRQ_LINE) ?
 672                         1 : event.status;
 673
 674         return 1;
 675 }
 676
 677 int kvm_get_irqchip(kvm_context_t kvm, struct kvm_irqchip *chip)
 678 {
 679         int r;
 680
 681         if (!kvm->irqchip_in_kernel)
 682                 return 0;
 683         r = ioctl(kvm->vm_fd, KVM_GET_IRQCHIP, chip);
 684         if (r == -1) {
 685                 r = -errno;
 686                 perror("kvm_get_irqchip\n");
 687         }
 688         return r;
 689 }
 690
 691 int kvm_set_irqchip(kvm_context_t kvm, struct kvm_irqchip *chip)
 692 {
 693         int r;
 694
 695         if (!kvm->irqchip_in_kernel)
 696                 return 0;
 697         r = ioctl(kvm->vm_fd, KVM_SET_IRQCHIP, chip);
 698         if (r == -1) {
 699                 r = -errno;
 700                 perror("kvm_set_irqchip\n");
 701         }
 702         return r;
 703 }
 704
 705 #endif
 706
 707 static int handle_io(kvm_context_t kvm, struct kvm_run *run, int vcpu)
 708 {
 709         uint16_t addr = run->io.port;
 710         int r;
 711         int i;
 712         void *p = (void *)run + run->io.data_offset;
 713
 714         for (i = 0; i < run->io.count; ++i) {
 715                 switch (run->io.direction) {
 716                 case KVM_EXIT_IO_IN:
 717                         switch (run->io.size) {
 718                         case 1:
 719                                 r = kvm->callbacks->inb(kvm->opaque, addr, p);
 720                                 break;
 721                         case 2:
 722                                 r = kvm->callbacks->inw(kvm->opaque, addr, p);
 723                                 break;
 724                         case 4:
 725                                 r = kvm->callbacks->inl(kvm->opaque, addr, p);
 726                                 break;
 727                         default:
 728                                 fprintf(stderr, "bad I/O size %d\n", run->io.size);
 729                                 return -EMSGSIZE;
 730                         }
 731                         break;
 732                 case KVM_EXIT_IO_OUT:
 733                         switch (run->io.size) {
 734                         case 1:
 735                                 r = kvm->callbacks->outb(kvm->opaque, addr,
 736                                                      *(uint8_t *)p);
 737                                 break;
 738                         case 2:
 739                                 r = kvm->callbacks->outw(kvm->opaque, addr,
 740                                                      *(uint16_t *)p);
 741                                 break;
 742                         case 4:
 743                                 r = kvm->callbacks->outl(kvm->opaque, addr,
 744                                                      *(uint32_t *)p);
 745                                 break;
 746                         default:
 747                                 fprintf(stderr, "bad I/O size %d\n", run->io.size);
 748                                 return -EMSGSIZE;
 749                         }
 750                         break;
 751                 default:
 752                         fprintf(stderr, "bad I/O direction %d\n", run->io.direction);
 753                         return -EPROTO;
 754                 }
 755
 756                 p += run->io.size;
 757         }
 758
 759         return 0;
 760 }
 761
 762 int handle_debug(kvm_context_t kvm, int vcpu, void *env)
 763 {
 764 #ifdef KVM_CAP_SET_GUEST_DEBUG
 765     struct kvm_run *run = kvm->run[vcpu];
 766
 767     return kvm->callbacks->debug(kvm->opaque, env, &run->debug.arch);
 768 #else
 769     return 0;
 770 #endif
 771 }
 772
 773 int kvm_get_regs(kvm_context_t kvm, int vcpu, struct kvm_regs *regs)
 774 {
 775     return ioctl(kvm->vcpu_fd[vcpu], KVM_GET_REGS, regs);
 776 }
 777
 778 int kvm_set_regs(kvm_context_t kvm, int vcpu, struct kvm_regs *regs)
 779 {
 780     return ioctl(kvm->vcpu_fd[vcpu], KVM_SET_REGS, regs);
 781 }
 782
 783 int kvm_get_fpu(kvm_context_t kvm, int vcpu, struct kvm_fpu *fpu)
 784 {
 785     return ioctl(kvm->vcpu_fd[vcpu], KVM_GET_FPU, fpu);
 786 }
 787
 788 int kvm_set_fpu(kvm_context_t kvm, int vcpu, struct kvm_fpu *fpu)
 789 {
 790     return ioctl(kvm->vcpu_fd[vcpu], KVM_SET_FPU, fpu);
 791 }
 792
 793 int kvm_get_sregs(kvm_context_t kvm, int vcpu, struct kvm_sregs *sregs)
 794 {
 795     return ioctl(kvm->vcpu_fd[vcpu], KVM_GET_SREGS, sregs);
 796 }
 797
 798 int kvm_set_sregs(kvm_context_t kvm, int vcpu, struct kvm_sregs *sregs)
 799 {
 800     return ioctl(kvm->vcpu_fd[vcpu], KVM_SET_SREGS, sregs);
 801 }
 802
 803 #ifdef KVM_CAP_MP_STATE
 804 int kvm_get_mpstate(kvm_context_t kvm, int vcpu, struct kvm_mp_state *mp_state)
 805 {
 806     int r;
 807
 808     r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_MP_STATE);
 809     if (r > 0)
 810         return ioctl(kvm->vcpu_fd[vcpu], KVM_GET_MP_STATE, mp_state);
 811     return -ENOSYS;
 812 }
 813
 814 int kvm_set_mpstate(kvm_context_t kvm, int vcpu, struct kvm_mp_state *mp_state)
 815 {
 816     int r;
 817
 818     r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_MP_STATE);
 819     if (r > 0)
 820         return ioctl(kvm->vcpu_fd[vcpu], KVM_SET_MP_STATE, mp_state);
 821     return -ENOSYS;
 822 }
 823 #endif
 824
 825 static int handle_mmio(kvm_context_t kvm, struct kvm_run *kvm_run)
 826 {
 827         unsigned long addr = kvm_run->mmio.phys_addr;
 828         void *data = kvm_run->mmio.data;
 829
 830         /* hack: Red Hat 7.1 generates these weird accesses. */
 831         if ((addr > 0xa0000-4 && addr <= 0xa0000) && kvm_run->mmio.len == 3)
 832             return 0;
 833
 834         if (kvm_run->mmio.is_write)
 835                 return kvm->callbacks->mmio_write(kvm->opaque, addr, data,
 836                                         kvm_run->mmio.len);
 837         else
 838                 return kvm->callbacks->mmio_read(kvm->opaque, addr, data,
 839                                         kvm_run->mmio.len);
 840 }
 841
 842 int handle_io_window(kvm_context_t kvm)
 843 {
 844         return kvm->callbacks->io_window(kvm->opaque);
 845 }
 846
 847 int handle_halt(kvm_context_t kvm, int vcpu)
 848 {
 849         return kvm->callbacks->halt(kvm->opaque, vcpu);
 850 }
 851
 852 int handle_shutdown(kvm_context_t kvm, void *env)
 853 {
 854         return kvm->callbacks->shutdown(kvm->opaque, env);
 855 }
 856
 857 int try_push_interrupts(kvm_context_t kvm)
 858 {
 859         return kvm->callbacks->try_push_interrupts(kvm->opaque);
 860 }
 861
 862 static inline void push_nmi(kvm_context_t kvm)
 863 {
 864 #ifdef KVM_CAP_USER_NMI
 865         kvm->callbacks->push_nmi(kvm->opaque);
 866 #endif /* KVM_CAP_USER_NMI */
 867 }
 868
 869 void post_kvm_run(kvm_context_t kvm, void *env)
 870 {
 871         kvm->callbacks->post_kvm_run(kvm->opaque, env);
 872 }
 873
 874 int pre_kvm_run(kvm_context_t kvm, void *env)
 875 {
 876         return kvm->callbacks->pre_kvm_run(kvm->opaque, env);
 877 }
 878
 879 int kvm_get_interrupt_flag(kvm_context_t kvm, int vcpu)
 880 {
 881         struct kvm_run *run = kvm->run[vcpu];
 882
 883         return run->if_flag;
 884 }
 885
 886 int kvm_is_ready_for_interrupt_injection(kvm_context_t kvm, int vcpu)
 887 {
 888         struct kvm_run *run = kvm->run[vcpu];
 889
 890         return run->ready_for_interrupt_injection;
 891 }
 892
 893 int kvm_run(kvm_context_t kvm, int vcpu, void *env)
 894 {
 895         int r;
 896         int fd = kvm->vcpu_fd[vcpu];
 897         struct kvm_run *run = kvm->run[vcpu];
 898
 899 again:
 900         push_nmi(kvm);
 901 #if !defined(__s390__)
 902         if (!kvm->irqchip_in_kernel)
 903                 run->request_interrupt_window = try_push_interrupts(kvm);
 904 #endif
 905         r = pre_kvm_run(kvm, env);
 906         if (r)
 907             return r;
 908         r = ioctl(fd, KVM_RUN, 0);
 909
 910         if (r == -1 && errno != EINTR && errno != EAGAIN) {
 911                 r = -errno;
 912                 post_kvm_run(kvm, env);
 913                 fprintf(stderr, "kvm_run: %s\n", strerror(-r));
 914                 return r;
 915         }
 916
 917         post_kvm_run(kvm, env);
 918
 919 #if defined(KVM_CAP_COALESCED_MMIO)
 920         if (kvm->coalesced_mmio) {
 921                 struct kvm_coalesced_mmio_ring *ring = (void *)run +
 922                                                 kvm->coalesced_mmio * PAGE_SIZE;
 923                 while (ring->first != ring->last) {
 924                         kvm->callbacks->mmio_write(kvm->opaque,
 925                                  ring->coalesced_mmio[ring->first].phys_addr,
 926                                 &ring->coalesced_mmio[ring->first].data[0],
 927                                  ring->coalesced_mmio[ring->first].len);
 928                         smp_wmb();
 929                         ring->first = (ring->first + 1) %
 930                                                         KVM_COALESCED_MMIO_MAX;
 931                 }
 932         }
 933 #endif
 934
 935 #if !defined(__s390__)
 936         if (r == -1) {
 937                 r = handle_io_window(kvm);
 938                 goto more;
 939         }
 940 #endif
 941         if (1) {
 942                 switch (run->exit_reason) {
 943                 case KVM_EXIT_UNKNOWN:
 944                         fprintf(stderr, "unhandled vm exit: 0x%x vcpu_id %d\n",
 945                                 (unsigned)run->hw.hardware_exit_reason, vcpu);
 946                         kvm_show_regs(kvm, vcpu);
 947                         abort();
 948                         break;
 949                 case KVM_EXIT_FAIL_ENTRY:
 950                         fprintf(stderr, "kvm_run: failed entry, reason %u\n",
 951                                 (unsigned)run->fail_entry.hardware_entry_failure_reason & 0xffff);
 952                         kvm_show_regs(kvm, vcpu);
 953                         return -ENOEXEC;
 954                         break;
 955                 case KVM_EXIT_EXCEPTION:
 956                         fprintf(stderr, "exception %d (%x)\n",
 957                                run->ex.exception,
 958                                run->ex.error_code);
 959                         kvm_show_regs(kvm, vcpu);
 960                         kvm_show_code(kvm, vcpu);
 961                         abort();
 962                         break;
 963                 case KVM_EXIT_IO:
 964                         r = handle_io(kvm, run, vcpu);
 965                         break;
 966                 case KVM_EXIT_DEBUG:
 967                         r = handle_debug(kvm, vcpu, env);
 968                         break;
 969                 case KVM_EXIT_MMIO:
 970                         r = handle_mmio(kvm, run);
 971                         break;
 972                 case KVM_EXIT_HLT:
 973                         r = handle_halt(kvm, vcpu);
 974                         break;
 975                 case KVM_EXIT_IRQ_WINDOW_OPEN:
 976                         break;
 977                 case KVM_EXIT_SHUTDOWN:
 978                         r = handle_shutdown(kvm, env);
 979                         break;
 980 #if defined(__s390__)
 981                 case KVM_EXIT_S390_SIEIC:
 982                         r = kvm->callbacks->s390_handle_intercept(kvm, vcpu,
 983                                 run);
 984                         break;
 985                 case KVM_EXIT_S390_RESET:
 986                         r = kvm->callbacks->s390_handle_reset(kvm, vcpu, run);
 987                         break;
 988 #endif
 989                 default:
 990                         if (kvm_arch_run(run, kvm, vcpu)) {
 991                                 fprintf(stderr, "unhandled vm exit: 0x%x\n",
 992                                                         run->exit_reason);
 993                                 kvm_show_regs(kvm, vcpu);
 994                                 abort();
 995                         }
 996                         break;
 997                 }
 998         }
 999 more:
1000         if (!r)
1001                 goto again;
1002         return r;
1003 }
1004
1005 int kvm_inject_irq(kvm_context_t kvm, int vcpu, unsigned irq)
1006 {
1007         struct kvm_interrupt intr;
1008
1009         intr.irq = irq;
1010         return ioctl(kvm->vcpu_fd[vcpu], KVM_INTERRUPT, &intr);
1011 }
1012
1013 #ifdef KVM_CAP_SET_GUEST_DEBUG
1014 int kvm_set_guest_debug(kvm_context_t kvm, int vcpu, struct kvm_guest_debug *dbg)
1015 {
1016         return ioctl(kvm->vcpu_fd[vcpu], KVM_SET_GUEST_DEBUG, dbg);
1017 }
1018 #endif
1019
1020 int kvm_set_signal_mask(kvm_context_t kvm, int vcpu, const sigset_t *sigset)
1021 {
1022         struct kvm_signal_mask *sigmask;
1023         int r;
1024
1025         if (!sigset) {
1026                 r = ioctl(kvm->vcpu_fd[vcpu], KVM_SET_SIGNAL_MASK, NULL);
1027                 if (r == -1)
1028                         r = -errno;
1029                 return r;
1030         }
1031         sigmask = malloc(sizeof(*sigmask) + sizeof(*sigset));
1032         if (!sigmask)
1033                 return -ENOMEM;
1034
1035         sigmask->len = 8;
1036         memcpy(sigmask->sigset, sigset, sizeof(*sigset));
1037         r = ioctl(kvm->vcpu_fd[vcpu], KVM_SET_SIGNAL_MASK, sigmask);
1038         if (r == -1)
1039                 r = -errno;
1040         free(sigmask);
1041         return r;
1042 }
1043
1044 int kvm_irqchip_in_kernel(kvm_context_t kvm)
1045 {
1046         return kvm->irqchip_in_kernel;
1047 }
1048
1049 int kvm_pit_in_kernel(kvm_context_t kvm)
1050 {
1051         return kvm->pit_in_kernel;
1052 }
1053
1054 int kvm_has_sync_mmu(kvm_context_t kvm)
1055 {
1056         int r = 0;
1057 #ifdef KVM_CAP_SYNC_MMU
1058         r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_SYNC_MMU);
1059 #endif
1060         return r;
1061 }
1062
1063 int kvm_inject_nmi(kvm_context_t kvm, int vcpu)
1064 {
1065 #ifdef KVM_CAP_USER_NMI
1066         return ioctl(kvm->vcpu_fd[vcpu], KVM_NMI);
1067 #else
1068         return -ENOSYS;
1069 #endif
1070 }
1071
1072 int kvm_init_coalesced_mmio(kvm_context_t kvm)
1073 {
1074         int r = 0;
1075         kvm->coalesced_mmio = 0;
1076 #ifdef KVM_CAP_COALESCED_MMIO
1077         r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_COALESCED_MMIO);
1078         if (r > 0) {
1079                 kvm->coalesced_mmio = r;
1080                 return 0;
1081         }
1082 #endif
1083         return r;
1084 }
1085
1086 int kvm_register_coalesced_mmio(kvm_context_t kvm, uint64_t addr, uint32_t size)
1087 {
1088 #ifdef KVM_CAP_COALESCED_MMIO
1089         struct kvm_coalesced_mmio_zone zone;
1090         int r;
1091
1092         if (kvm->coalesced_mmio) {
1093
1094                 zone.addr = addr;
1095                 zone.size = size;
1096
1097                 r = ioctl(kvm->vm_fd, KVM_REGISTER_COALESCED_MMIO, &zone);
1098                 if (r == -1) {
1099                         perror("kvm_register_coalesced_mmio_zone");
1100                         return -errno;
1101                 }
1102                 return 0;
1103         }
1104 #endif
1105         return -ENOSYS;
1106 }
1107
1108 int kvm_unregister_coalesced_mmio(kvm_context_t kvm, uint64_t addr, uint32_t size)
1109 {
1110 #ifdef KVM_CAP_COALESCED_MMIO
1111         struct kvm_coalesced_mmio_zone zone;
1112         int r;
1113
1114         if (kvm->coalesced_mmio) {
1115
1116                 zone.addr = addr;
1117                 zone.size = size;
1118
1119                 r = ioctl(kvm->vm_fd, KVM_UNREGISTER_COALESCED_MMIO, &zone);
1120                 if (r == -1) {
1121                         perror("kvm_unregister_coalesced_mmio_zone");
1122                         return -errno;
1123                 }
1124                 DPRINTF("Unregistered coalesced mmio region for %llx (%lx)\n", addr, size);
1125                 return 0;
1126         }
1127 #endif
1128         return -ENOSYS;
1129 }
1130
1131 #ifdef KVM_CAP_DEVICE_ASSIGNMENT
1132 int kvm_assign_pci_device(kvm_context_t kvm,
1133                           struct kvm_assigned_pci_dev *assigned_dev)
1134 {
1135         int ret;
1136
1137         ret = ioctl(kvm->vm_fd, KVM_ASSIGN_PCI_DEVICE, assigned_dev);
1138         if (ret < 0)
1139                 return -errno;
1140
1141         return ret;
1142 }
1143
1144 int kvm_assign_irq(kvm_context_t kvm,
1145                    struct kvm_assigned_irq *assigned_irq)
1146 {
1147         int ret;
1148
1149         ret = ioctl(kvm->vm_fd, KVM_ASSIGN_IRQ, assigned_irq);
1150         if (ret < 0)
1151                 return -errno;
1152
1153         return ret;
1154 }
1155 #endif
1156
1157 #ifdef KVM_CAP_DEVICE_DEASSIGNMENT
1158 int kvm_deassign_pci_device(kvm_context_t kvm,
1159                             struct kvm_assigned_pci_dev *assigned_dev)
1160 {
1161         int ret;
1162
1163         ret = ioctl(kvm->vm_fd, KVM_DEASSIGN_PCI_DEVICE, assigned_dev);
1164         if (ret < 0)
1165                 return -errno;
1166
1167         return ret;
1168 }
1169 #endif
1170
1171 int kvm_destroy_memory_region_works(kvm_context_t kvm)
1172 {
1173         int ret = 0;
1174
1175 #ifdef KVM_CAP_DESTROY_MEMORY_REGION_WORKS
1176         ret = ioctl(kvm->fd, KVM_CHECK_EXTENSION,
1177                     KVM_CAP_DESTROY_MEMORY_REGION_WORKS);
1178         if (ret <= 0)
1179                 ret = 0;
1180 #endif
1181         return ret;
1182 }
1183
1184 int kvm_reinject_control(kvm_context_t kvm, int pit_reinject)
1185 {
1186 #ifdef KVM_CAP_REINJECT_CONTROL
1187         int r;
1188         struct kvm_reinject_control control;
1189
1190         control.pit_reinject = pit_reinject;
1191
1192         r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_REINJECT_CONTROL);
1193         if (r > 0) {
1194                 r = ioctl(kvm->vm_fd, KVM_REINJECT_CONTROL, &control);
1195                 if (r == -1)
1196                         return -errno;
1197                 return r;
1198         }
1199 #endif
1200         return -ENOSYS;
1201 }
1202
1203 int kvm_has_gsi_routing(kvm_context_t kvm)
1204 {
1205     int r = 0;
1206
1207 #ifdef KVM_CAP_IRQ_ROUTING
1208     r = kvm_check_extension(kvm, KVM_CAP_IRQ_ROUTING);
1209 #endif
1210     return r;
1211 }
1212
1213 int kvm_get_gsi_count(kvm_context_t kvm)
1214 {
1215 #ifdef KVM_CAP_IRQ_ROUTING
1216         return kvm_check_extension(kvm, KVM_CAP_IRQ_ROUTING);
1217 #else
1218         return -EINVAL;
1219 #endif
1220 }
1221
1222 int kvm_clear_gsi_routes(kvm_context_t kvm)
1223 {
1224 #ifdef KVM_CAP_IRQ_ROUTING
1225         kvm->irq_routes->nr = 0;
1226         return 0;
1227 #else
1228         return -EINVAL;
1229 #endif
1230 }
1231
1232 int kvm_add_irq_route(kvm_context_t kvm, int gsi, int irqchip, int pin)
1233 {
1234 #ifdef KVM_CAP_IRQ_ROUTING
1235         struct kvm_irq_routing *z;
1236         struct kvm_irq_routing_entry *e;
1237         int n, size;
1238
1239         if (kvm->irq_routes->nr == kvm->nr_allocated_irq_routes) {
1240                 n = kvm->nr_allocated_irq_routes * 2;
1241                 if (n < 64)
1242                         n = 64;
1243                 size = sizeof(struct kvm_irq_routing);
1244                 size += n * sizeof(*e);
1245                 z = realloc(kvm->irq_routes, size);
1246                 if (!z)
1247                         return -ENOMEM;
1248                 kvm->nr_allocated_irq_routes = n;
1249                 kvm->irq_routes = z;
1250         }
1251         n = kvm->irq_routes->nr++;
1252         e = &kvm->irq_routes->entries[n];
1253         memset(e, 0, sizeof(*e));
1254         e->gsi = gsi;
1255         e->type = KVM_IRQ_ROUTING_IRQCHIP;
1256         e->flags = 0;
1257         e->u.irqchip.irqchip = irqchip;
1258         e->u.irqchip.pin = pin;
1259         return 0;
1260 #else
1261         return -ENOSYS;
1262 #endif
1263 }
1264
1265 int kvm_del_irq_route(kvm_context_t kvm, int gsi, int irqchip, int pin)
1266 {
1267 #ifdef KVM_CAP_IRQ_ROUTING
1268         struct kvm_irq_routing_entry *e, *p;
1269         int i;
1270
1271         for (i = 0; i < kvm->irq_routes->nr; ++i) {
1272                 e = &kvm->irq_routes->entries[i];
1273                 if (e->type == KVM_IRQ_ROUTING_IRQCHIP
1274                     && e->gsi == gsi
1275                     && e->u.irqchip.irqchip == irqchip
1276                     && e->u.irqchip.pin == pin) {
1277                         p = &kvm->irq_routes->entries[--kvm->irq_routes->nr];
1278                         *e = *p;
1279                         return 0;
1280                 }
1281         }
1282         return -ESRCH;
1283 #else
1284         return -ENOSYS;
1285 #endif
1286 }
1287
1288 int kvm_commit_irq_routes(kvm_context_t kvm)
1289 {
1290 #ifdef KVM_CAP_IRQ_ROUTING
1291         int r;
1292
1293         kvm->irq_routes->flags = 0;
1294         r = ioctl(kvm->vm_fd, KVM_SET_GSI_ROUTING, kvm->irq_routes);
1295         if (r == -1)
1296                 r = -errno;
1297         return r;
1298 #else
1299         return -ENOSYS;
1300 #endif
1301 }