kvm/libkvm/libkvm.c

   1 /*
   2  * Kernel-based Virtual Machine control library
   3  *
   4  * This library provides an API to control the kvm hardware virtualization
   5  * module.
   6  *
   7  * Copyright (C) 2006 Qumranet
   8  *
   9  * Authors:
  10  *
  11  *  Avi Kivity <avi@qumranet.com>
  12  *  Yaniv Kamay <yaniv@qumranet.com>
  13  *
  14  * This work is licensed under the GNU LGPL license, version 2.
  15  */
  16
  17 #ifndef __user
  18 #define __user /* temporary, until installed via make headers_install */
  19 #endif
  20
  21 #include <linux/kvm.h>
  22
  23 #define EXPECTED_KVM_API_VERSION 12
  24
  25 #if EXPECTED_KVM_API_VERSION != KVM_API_VERSION
  26 #error libkvm: userspace and kernel version mismatch
  27 #endif
  28
  29 #include <unistd.h>
  30 #include <fcntl.h>
  31 #include <stdio.h>
  32 #include <stdlib.h>
  33 #include <sys/mman.h>
  34 #include <string.h>
  35 #include <errno.h>
  36 #include <sys/ioctl.h>
  37 #include "libkvm.h"
  38
  39 #if defined(__x86_64__) || defined(__i386__)
  40 #include "kvm-x86.h"
  41 #endif
  42
  43 #if defined(__ia64__)
  44 #include "kvm-ia64.h"
  45 #endif
  46
  47 #if defined(__powerpc__)
  48 #include "kvm-powerpc.h"
  49 #endif
  50
  51 int kvm_abi = EXPECTED_KVM_API_VERSION;
  52 int kvm_page_size;
  53
  54 struct slot_info {
  55         unsigned long phys_addr;
  56         unsigned long len;
  57         int user_alloc;
  58         unsigned long userspace_addr;
  59         unsigned flags;
  60 };
  61
  62 struct slot_info slots[KVM_MAX_NUM_MEM_REGIONS];
  63
  64 void init_slots(void)
  65 {
  66         int i;
  67
  68         for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS; ++i)
  69                 slots[i].len = 0;
  70 }
  71
  72 int get_free_slot(kvm_context_t kvm)
  73 {
  74         int i;
  75         int tss_ext;
  76
  77 #ifdef KVM_CAP_SET_TSS_ADDR
  78         tss_ext = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_SET_TSS_ADDR);
  79 #else
  80         tss_ext = 0;
  81 #endif
  82
  83         /*
  84          * on older kernels where the set tss ioctl is not supprted we must save
  85          * slot 0 to hold the extended memory, as the vmx will use the last 3
  86          * pages of this slot.
  87          */
  88         if (tss_ext > 0)
  89                 i = 0;
  90         else
  91                 i = 1;
  92
  93         for (; i < KVM_MAX_NUM_MEM_REGIONS; ++i)
  94                 if (!slots[i].len)
  95                         return i;
  96         return -1;
  97 }
  98
  99 void register_slot(int slot, unsigned long phys_addr, unsigned long len,
 100                    int user_alloc, unsigned long userspace_addr, unsigned flags)
 101 {
 102         slots[slot].phys_addr = phys_addr;
 103         slots[slot].len = len;
 104         slots[slot].user_alloc = user_alloc;
 105         slots[slot].userspace_addr = userspace_addr;
 106         slots[slot].flags = flags;
 107 }
 108
 109 void free_slot(int slot)
 110 {
 111         slots[slot].len = 0;
 112 }
 113
 114 int get_slot(unsigned long phys_addr)
 115 {
 116         int i;
 117
 118         for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS ; ++i) {
 119                 if (slots[i].len && slots[i].phys_addr <= phys_addr &&
 120                     (slots[i].phys_addr + slots[i].len-1) >= phys_addr)
 121                         return i;
 122         }
 123         return -1;
 124 }
 125
 126 int get_intersecting_slot(unsigned long phys_addr)
 127 {
 128         int i;
 129
 130         for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS ; ++i)
 131                 if (slots[i].len && slots[i].phys_addr < phys_addr &&
 132                     (slots[i].phys_addr + slots[i].len) > phys_addr)
 133                         return i;
 134         return -1;
 135 }
 136
 137 /*
 138  * dirty pages logging control
 139  */
 140 static int kvm_dirty_pages_log_change(kvm_context_t kvm, unsigned long phys_addr
 141                                       , __u32 flag)
 142 {
 143         int r;
 144         int slot;
 145
 146         slot = get_slot(phys_addr);
 147         if (slot == -1) {
 148                 fprintf(stderr, "BUG: %s: invalid parameters\n", __FUNCTION__);
 149                 return 1;
 150         }
 151         flag |= slots[slot].flags;
 152 #ifdef KVM_CAP_USER_MEMORY
 153         if (slots[slot].user_alloc) {
 154                 struct kvm_userspace_memory_region mem = {
 155                         .slot = slot,
 156                         .memory_size = slots[slot].len,
 157                         .guest_phys_addr = slots[slot].phys_addr,
 158                         .userspace_addr = slots[slot].userspace_addr,
 159                         .flags = flag,
 160                 };
 161                 r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &mem);
 162         }
 163 #endif
 164         if (!slots[slot].user_alloc) {
 165                 struct kvm_memory_region mem = {
 166                         .slot = slot,
 167                         .memory_size = slots[slot].len,
 168                         .guest_phys_addr = slots[slot].phys_addr,
 169                         .flags = flag,
 170                 };
 171                 r = ioctl(kvm->vm_fd, KVM_SET_MEMORY_REGION, &mem);
 172         }
 173         if (r == -1)
 174                 fprintf(stderr, "%s: %m\n", __FUNCTION__);
 175         return r;
 176 }
 177
 178 static int kvm_dirty_pages_log_change_all(kvm_context_t kvm, __u32 flag)
 179 {
 180         int i, r;
 181
 182         for (i=r=0; i<KVM_MAX_NUM_MEM_REGIONS && r==0; i++) {
 183                 if (slots[i].len)
 184                         r = kvm_dirty_pages_log_change(kvm, slots[i].phys_addr,
 185                                                        flag);
 186         }
 187         return r;
 188 }
 189
 190 /**
 191  * Enable dirty page logging for all memory regions
 192  */
 193 int kvm_dirty_pages_log_enable_all(kvm_context_t kvm)
 194 {
 195         if (kvm->dirty_pages_log_all)
 196                 return 0;
 197         kvm->dirty_pages_log_all = 1;
 198         return kvm_dirty_pages_log_change_all(kvm, KVM_MEM_LOG_DIRTY_PAGES);
 199 }
 200
 201 /**
 202  * Enable dirty page logging only for memory regions that were created with
 203  *     dirty logging enabled (disable for all other memory regions).
 204  */
 205 int kvm_dirty_pages_log_reset(kvm_context_t kvm)
 206 {
 207         if (!kvm->dirty_pages_log_all)
 208                 return 0;
 209         kvm->dirty_pages_log_all = 0;
 210         return kvm_dirty_pages_log_change_all(kvm, 0);
 211 }
 212
 213
 214 kvm_context_t kvm_init(struct kvm_callbacks *callbacks,
 215                        void *opaque)
 216 {
 217         int fd;
 218         kvm_context_t kvm;
 219         int r;
 220
 221         fd = open("/dev/kvm", O_RDWR);
 222         if (fd == -1) {
 223                 perror("open /dev/kvm");
 224                 return NULL;
 225         }
 226         r = ioctl(fd, KVM_GET_API_VERSION, 0);
 227         if (r == -1) {
 228             fprintf(stderr, "kvm kernel version too old: "
 229                     "KVM_GET_API_VERSION ioctl not supported\n");
 230             goto out_close;
 231         }
 232         if (r < EXPECTED_KVM_API_VERSION) {
 233                 fprintf(stderr, "kvm kernel version too old: "
 234                         "We expect API version %d or newer, but got "
 235                         "version %d\n",
 236                         EXPECTED_KVM_API_VERSION, r);
 237             goto out_close;
 238         }
 239         if (r > EXPECTED_KVM_API_VERSION) {
 240             fprintf(stderr, "kvm userspace version too old\n");
 241             goto out_close;
 242         }
 243         kvm_abi = r;
 244         kvm_page_size = getpagesize();
 245         kvm = malloc(sizeof(*kvm));
 246         kvm->fd = fd;
 247         kvm->vm_fd = -1;
 248         kvm->callbacks = callbacks;
 249         kvm->opaque = opaque;
 250         kvm->dirty_pages_log_all = 0;
 251         kvm->no_irqchip_creation = 0;
 252
 253         return kvm;
 254  out_close:
 255         close(fd);
 256         return NULL;
 257 }
 258
 259 void kvm_finalize(kvm_context_t kvm)
 260 {
 261         if (kvm->vcpu_fd[0] != -1)
 262                 close(kvm->vcpu_fd[0]);
 263         if (kvm->vm_fd != -1)
 264                 close(kvm->vm_fd);
 265         close(kvm->fd);
 266         free(kvm);
 267 }
 268
 269 void kvm_disable_irqchip_creation(kvm_context_t kvm)
 270 {
 271         kvm->no_irqchip_creation = 1;
 272 }
 273
 274 void kvm_disable_pit_creation(kvm_context_t kvm)
 275 {
 276         kvm->no_pit_creation = 1;
 277 }
 278
 279 int kvm_create_vcpu(kvm_context_t kvm, int slot)
 280 {
 281         long mmap_size;
 282         int r;
 283
 284         r = ioctl(kvm->vm_fd, KVM_CREATE_VCPU, slot);
 285         if (r == -1) {
 286                 r = -errno;
 287                 fprintf(stderr, "kvm_create_vcpu: %m\n");
 288                 return r;
 289         }
 290         kvm->vcpu_fd[slot] = r;
 291         mmap_size = ioctl(kvm->fd, KVM_GET_VCPU_MMAP_SIZE, 0);
 292         if (mmap_size == -1) {
 293                 r = -errno;
 294                 fprintf(stderr, "get vcpu mmap size: %m\n");
 295                 return r;
 296         }
 297         kvm->run[slot] = mmap(NULL, mmap_size, PROT_READ|PROT_WRITE, MAP_SHARED,
 298                               kvm->vcpu_fd[slot], 0);
 299         if (kvm->run[slot] == MAP_FAILED) {
 300                 r = -errno;
 301                 fprintf(stderr, "mmap vcpu area: %m\n");
 302                 return r;
 303         }
 304         return 0;
 305 }
 306
 307 int kvm_create_vm(kvm_context_t kvm)
 308 {
 309         int fd = kvm->fd;
 310
 311         kvm->vcpu_fd[0] = -1;
 312
 313         fd = ioctl(fd, KVM_CREATE_VM, 0);
 314         if (fd == -1) {
 315                 fprintf(stderr, "kvm_create_vm: %m\n");
 316                 return -1;
 317         }
 318         kvm->vm_fd = fd;
 319         return 0;
 320 }
 321
 322 static int kvm_create_default_phys_mem(kvm_context_t kvm,
 323                                        unsigned long phys_mem_bytes,
 324                                        void **vm_mem)
 325 {
 326         unsigned long memory = (phys_mem_bytes + PAGE_SIZE - 1) & PAGE_MASK;
 327         int r;
 328
 329 #ifdef KVM_CAP_USER_MEMORY
 330         r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_USER_MEMORY);
 331         if (r > 0)
 332                 return 0;
 333         else
 334 #endif
 335                 r = kvm_alloc_kernel_memory(kvm, memory, vm_mem);
 336         if (r < 0)
 337                 return r;
 338
 339         r = kvm_arch_create_default_phys_mem(kvm, phys_mem_bytes, vm_mem);
 340         if (r < 0)
 341                 return r;
 342
 343         kvm->physical_memory = *vm_mem;
 344         return 0;
 345 }
 346
 347 int kvm_check_extension(kvm_context_t kvm, int ext)
 348 {
 349         int ret;
 350
 351         ret = ioctl(kvm->fd, KVM_CHECK_EXTENSION, ext);
 352         if (ret > 0)
 353                 return 1;
 354         return 0;
 355 }
 356
 357 void kvm_create_irqchip(kvm_context_t kvm)
 358 {
 359         int r;
 360
 361         kvm->irqchip_in_kernel = 0;
 362 #ifdef KVM_CAP_IRQCHIP
 363         if (!kvm->no_irqchip_creation) {
 364                 r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_IRQCHIP);
 365                 if (r > 0) {    /* kernel irqchip supported */
 366                         r = ioctl(kvm->vm_fd, KVM_CREATE_IRQCHIP);
 367                         if (r >= 0)
 368                                 kvm->irqchip_in_kernel = 1;
 369                         else
 370                                 printf("Create kernel PIC irqchip failed\n");
 371                 }
 372         }
 373 #endif
 374 }
 375
 376 int kvm_create(kvm_context_t kvm, unsigned long phys_mem_bytes, void **vm_mem)
 377 {
 378         int r;
 379
 380         r = kvm_create_vm(kvm);
 381         if (r < 0)
 382                 return r;
 383         r = kvm_arch_create(kvm, phys_mem_bytes, vm_mem);
 384         if (r < 0)
 385                 return r;
 386         init_slots();
 387         r = kvm_create_default_phys_mem(kvm, phys_mem_bytes, vm_mem);
 388         if (r < 0)
 389                 return r;
 390         kvm_create_irqchip(kvm);
 391
 392         return 0;
 393 }
 394
 395
 396 #ifdef KVM_CAP_USER_MEMORY
 397
 398 void *kvm_create_userspace_phys_mem(kvm_context_t kvm, unsigned long phys_start,
 399                         unsigned long len, int log, int writable)
 400 {
 401         int r;
 402         int prot = PROT_READ;
 403         void *ptr;
 404         struct kvm_userspace_memory_region memory = {
 405                 .memory_size = len,
 406                 .guest_phys_addr = phys_start,
 407                 .flags = log ? KVM_MEM_LOG_DIRTY_PAGES : 0,
 408         };
 409
 410         if (writable)
 411                 prot |= PROT_WRITE;
 412
 413         ptr = mmap(NULL, len, prot, MAP_ANONYMOUS | MAP_SHARED, -1, 0);
 414         if (ptr == MAP_FAILED) {
 415                 fprintf(stderr, "create_userspace_phys_mem: %s", strerror(errno));
 416                 return 0;
 417         }
 418
 419         memset(ptr, 0, len);
 420
 421         memory.userspace_addr = (unsigned long)ptr;
 422         memory.slot = get_free_slot(kvm);
 423         r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &memory);
 424         if (r == -1) {
 425                 fprintf(stderr, "create_userspace_phys_mem: %s", strerror(errno));
 426                 return 0;
 427         }
 428         register_slot(memory.slot, memory.guest_phys_addr, memory.memory_size,
 429                       1, memory.userspace_addr, memory.flags);
 430
 431         return ptr;
 432 }
 433
 434 #endif
 435
 436 void *kvm_create_phys_mem(kvm_context_t kvm, unsigned long phys_start,
 437                           unsigned long len, int log, int writable)
 438 {
 439 #ifdef KVM_CAP_USER_MEMORY
 440         int r;
 441
 442         r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_USER_MEMORY);
 443         if (r > 0)
 444                 return kvm_create_userspace_phys_mem(kvm, phys_start, len,
 445                                                                 log, writable);
 446         else
 447 #endif
 448                 return kvm_create_kernel_phys_mem(kvm, phys_start, len,
 449                                                                 log, writable);
 450 }
 451
 452 int kvm_is_intersecting_mem(kvm_context_t kvm, unsigned long phys_start)
 453 {
 454         return get_intersecting_slot(phys_start) != -1;
 455 }
 456
 457 int kvm_is_allocated_mem(kvm_context_t kvm, unsigned long phys_start,
 458                          unsigned long len)
 459 {
 460         int slot;
 461
 462         slot = get_slot(phys_start);
 463         if (slot == -1)
 464                 return 0;
 465         if (slots[slot].len == len)
 466                 return 1;
 467         return 0;
 468 }
 469
 470 int kvm_create_mem_hole(kvm_context_t kvm, unsigned long phys_start,
 471                         unsigned long len)
 472 {
 473 #ifdef KVM_CAP_USER_MEMORY
 474         int slot;
 475         int r;
 476         struct kvm_userspace_memory_region rmslot;
 477         struct kvm_userspace_memory_region newslot1;
 478         struct kvm_userspace_memory_region newslot2;
 479
 480         len = (len + PAGE_SIZE - 1) & PAGE_MASK;
 481
 482         slot = get_intersecting_slot(phys_start);
 483         /* no need to create hole, as there is already hole */
 484         if (slot == -1)
 485                 return 0;
 486
 487         memset(&rmslot, 0, sizeof(struct kvm_userspace_memory_region));
 488         memset(&newslot1, 0, sizeof(struct kvm_userspace_memory_region));
 489         memset(&newslot2, 0, sizeof(struct kvm_userspace_memory_region));
 490
 491         rmslot.guest_phys_addr = slots[slot].phys_addr;
 492         rmslot.slot = slot;
 493
 494         newslot1.guest_phys_addr = slots[slot].phys_addr;
 495         newslot1.memory_size = phys_start - slots[slot].phys_addr;
 496         newslot1.slot = slot;
 497         newslot1.userspace_addr = slots[slot].userspace_addr;
 498         newslot1.flags = slots[slot].flags;
 499
 500         newslot2.guest_phys_addr = newslot1.guest_phys_addr +
 501                                    newslot1.memory_size + len;
 502         newslot2.memory_size = slots[slot].phys_addr +
 503                                slots[slot].len - newslot2.guest_phys_addr;
 504         newslot2.userspace_addr = newslot1.userspace_addr +
 505                                   newslot1.memory_size;
 506         newslot2.slot = get_free_slot(kvm);
 507         newslot2.flags = newslot1.flags;
 508
 509         r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &rmslot);
 510         if (r == -1) {
 511                 fprintf(stderr, "kvm_create_mem_hole: %s\n", strerror(errno));
 512                 return -1;
 513         }
 514         free_slot(slot);
 515
 516         r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &newslot1);
 517         if (r == -1) {
 518                 fprintf(stderr, "kvm_create_mem_hole: %s\n", strerror(errno));
 519                 return -1;
 520         }
 521         register_slot(newslot1.slot, newslot1.guest_phys_addr,
 522                       newslot1.memory_size, 1, newslot1.userspace_addr,
 523                       newslot1.flags);
 524
 525         r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &newslot2);
 526         if (r == -1) {
 527                 fprintf(stderr, "kvm_create_mem_hole: %s\n", strerror(errno));
 528                 return -1;
 529         }
 530         register_slot(newslot2.slot, newslot2.guest_phys_addr,
 531                       newslot2.memory_size, 1, newslot2.userspace_addr,
 532                       newslot2.flags);
 533 #endif
 534         return 0;
 535 }
 536
 537 int kvm_register_userspace_phys_mem(kvm_context_t kvm,
 538                         unsigned long phys_start, void *userspace_addr,
 539                         unsigned long len, int log)
 540 {
 541
 542 #ifdef KVM_CAP_USER_MEMORY
 543         struct kvm_userspace_memory_region memory = {
 544                 .memory_size = len,
 545                 .guest_phys_addr = phys_start,
 546                 .userspace_addr = (unsigned long)(intptr_t)userspace_addr,
 547                 .flags = log ? KVM_MEM_LOG_DIRTY_PAGES : 0,
 548         };
 549         int r;
 550
 551         if (!kvm->physical_memory)
 552                 kvm->physical_memory = userspace_addr - phys_start;
 553
 554         memory.slot = get_free_slot(kvm);
 555         r = ioctl(kvm->vm_fd, KVM_SET_USER_MEMORY_REGION, &memory);
 556         if (r == -1) {
 557                 fprintf(stderr, "create_userspace_phys_mem: %s\n", strerror(errno));
 558                 return -1;
 559         }
 560         register_slot(memory.slot, memory.guest_phys_addr, memory.memory_size,
 561                       1, memory.userspace_addr, memory.flags);
 562         return 0;
 563 #else
 564         return -ENOSYS;
 565 #endif
 566 }
 567
 568
 569 /* destroy/free a whole slot.
 570  * phys_start, len and slot are the params passed to kvm_create_phys_mem()
 571  */
 572 void kvm_destroy_phys_mem(kvm_context_t kvm, unsigned long phys_start,
 573                           unsigned long len)
 574 {
 575         int slot;
 576
 577         slot = get_slot(phys_start);
 578
 579         if (slot >= KVM_MAX_NUM_MEM_REGIONS) {
 580                 fprintf(stderr, "BUG: %s: invalid parameters (slot=%d)\n",
 581                         __FUNCTION__, slot);
 582                 return;
 583         }
 584         if (phys_start != slots[slot].phys_addr) {
 585                 fprintf(stderr,
 586                         "WARNING: %s: phys_start is 0x%lx expecting 0x%lx\n",
 587                         __FUNCTION__, phys_start, slots[slot].phys_addr);
 588                 phys_start = slots[slot].phys_addr;
 589         }
 590         kvm_create_phys_mem(kvm, phys_start, 0, 0, 0);
 591 }
 592
 593 static int kvm_get_map(kvm_context_t kvm, int ioctl_num, int slot, void *buf)
 594 {
 595         int r;
 596         struct kvm_dirty_log log = {
 597                 .slot = slot,
 598         };
 599
 600         log.dirty_bitmap = buf;
 601
 602         r = ioctl(kvm->vm_fd, ioctl_num, &log);
 603         if (r == -1)
 604                 return -errno;
 605         return 0;
 606 }
 607
 608 int kvm_get_dirty_pages(kvm_context_t kvm, unsigned long phys_addr, void *buf)
 609 {
 610         int slot;
 611
 612         slot = get_slot(phys_addr);
 613         return kvm_get_map(kvm, KVM_GET_DIRTY_LOG, slot, buf);
 614 }
 615
 616 #define ALIGN(x, y)  (((x)+(y)-1) & ~((y)-1))
 617 #define BITMAP_SIZE(m) (ALIGN(((m)/PAGE_SIZE), sizeof(long) * 8) / 8)
 618
 619 int kvm_get_dirty_pages_range(kvm_context_t kvm, unsigned long phys_addr,
 620                               unsigned long len, void *buf, void *opaque,
 621                               int (*cb)(unsigned long start, unsigned long len,
 622                                         void*bitmap, void *opaque))
 623 {
 624         int i;
 625         int r;
 626         unsigned long end_addr = phys_addr + len;
 627
 628         for (i = 0; i < KVM_MAX_NUM_MEM_REGIONS; ++i) {
 629                 if ((slots[i].len && slots[i].phys_addr >= phys_addr) &&
 630                     (slots[i].phys_addr + slots[i].len  <= end_addr)) {
 631                         r = kvm_get_map(kvm, KVM_GET_DIRTY_LOG, i, buf);
 632                         if (r)
 633                                 return r;
 634                         r = cb(slots[i].phys_addr, slots[i].len, buf, opaque);
 635                         if (r)
 636                                 return r;
 637                 }
 638         }
 639         return 0;
 640 }
 641
 642 #ifdef KVM_CAP_IRQCHIP
 643
 644 int kvm_set_irq_level(kvm_context_t kvm, int irq, int level)
 645 {
 646         struct kvm_irq_level event;
 647         int r;
 648
 649         if (!kvm->irqchip_in_kernel)
 650                 return 0;
 651         event.level = level;
 652         event.irq = irq;
 653         r = ioctl(kvm->vm_fd, KVM_IRQ_LINE, &event);
 654         if (r == -1)
 655                 perror("kvm_set_irq_level");
 656         return 1;
 657 }
 658
 659 int kvm_get_irqchip(kvm_context_t kvm, struct kvm_irqchip *chip)
 660 {
 661         int r;
 662
 663         if (!kvm->irqchip_in_kernel)
 664                 return 0;
 665         r = ioctl(kvm->vm_fd, KVM_GET_IRQCHIP, chip);
 666         if (r == -1) {
 667                 r = -errno;
 668                 perror("kvm_get_irqchip\n");
 669         }
 670         return r;
 671 }
 672
 673 int kvm_set_irqchip(kvm_context_t kvm, struct kvm_irqchip *chip)
 674 {
 675         int r;
 676
 677         if (!kvm->irqchip_in_kernel)
 678                 return 0;
 679         r = ioctl(kvm->vm_fd, KVM_SET_IRQCHIP, chip);
 680         if (r == -1) {
 681                 r = -errno;
 682                 perror("kvm_set_irqchip\n");
 683         }
 684         return r;
 685 }
 686
 687 #endif
 688
 689 static int handle_io(kvm_context_t kvm, struct kvm_run *run, int vcpu)
 690 {
 691         uint16_t addr = run->io.port;
 692         int r;
 693         int i;
 694         void *p = (void *)run + run->io.data_offset;
 695
 696         for (i = 0; i < run->io.count; ++i) {
 697                 switch (run->io.direction) {
 698                 case KVM_EXIT_IO_IN:
 699                         switch (run->io.size) {
 700                         case 1:
 701                                 r = kvm->callbacks->inb(kvm->opaque, addr, p);
 702                                 break;
 703                         case 2:
 704                                 r = kvm->callbacks->inw(kvm->opaque, addr, p);
 705                                 break;
 706                         case 4:
 707                                 r = kvm->callbacks->inl(kvm->opaque, addr, p);
 708                                 break;
 709                         default:
 710                                 fprintf(stderr, "bad I/O size %d\n", run->io.size);
 711                                 return -EMSGSIZE;
 712                         }
 713                         break;
 714                 case KVM_EXIT_IO_OUT:
 715                         switch (run->io.size) {
 716                         case 1:
 717                                 r = kvm->callbacks->outb(kvm->opaque, addr,
 718                                                      *(uint8_t *)p);
 719                                 break;
 720                         case 2:
 721                                 r = kvm->callbacks->outw(kvm->opaque, addr,
 722                                                      *(uint16_t *)p);
 723                                 break;
 724                         case 4:
 725                                 r = kvm->callbacks->outl(kvm->opaque, addr,
 726                                                      *(uint32_t *)p);
 727                                 break;
 728                         default:
 729                                 fprintf(stderr, "bad I/O size %d\n", run->io.size);
 730                                 return -EMSGSIZE;
 731                         }
 732                         break;
 733                 default:
 734                         fprintf(stderr, "bad I/O direction %d\n", run->io.direction);
 735                         return -EPROTO;
 736                 }
 737
 738                 p += run->io.size;
 739         }
 740
 741         return 0;
 742 }
 743
 744 int handle_debug(kvm_context_t kvm, int vcpu)
 745 {
 746         return kvm->callbacks->debug(kvm->opaque, vcpu);
 747 }
 748
 749 int kvm_get_regs(kvm_context_t kvm, int vcpu, struct kvm_regs *regs)
 750 {
 751     return ioctl(kvm->vcpu_fd[vcpu], KVM_GET_REGS, regs);
 752 }
 753
 754 int kvm_set_regs(kvm_context_t kvm, int vcpu, struct kvm_regs *regs)
 755 {
 756     return ioctl(kvm->vcpu_fd[vcpu], KVM_SET_REGS, regs);
 757 }
 758
 759 int kvm_get_fpu(kvm_context_t kvm, int vcpu, struct kvm_fpu *fpu)
 760 {
 761     return ioctl(kvm->vcpu_fd[vcpu], KVM_GET_FPU, fpu);
 762 }
 763
 764 int kvm_set_fpu(kvm_context_t kvm, int vcpu, struct kvm_fpu *fpu)
 765 {
 766     return ioctl(kvm->vcpu_fd[vcpu], KVM_SET_FPU, fpu);
 767 }
 768
 769 int kvm_get_sregs(kvm_context_t kvm, int vcpu, struct kvm_sregs *sregs)
 770 {
 771     return ioctl(kvm->vcpu_fd[vcpu], KVM_GET_SREGS, sregs);
 772 }
 773
 774 int kvm_set_sregs(kvm_context_t kvm, int vcpu, struct kvm_sregs *sregs)
 775 {
 776     return ioctl(kvm->vcpu_fd[vcpu], KVM_SET_SREGS, sregs);
 777 }
 778
 779 #ifdef KVM_CAP_MP_STATE
 780 int kvm_get_mpstate(kvm_context_t kvm, int vcpu, struct kvm_mp_state *mp_state)
 781 {
 782     int r;
 783
 784     r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_MP_STATE);
 785     if (r > 0)
 786         return ioctl(kvm->vcpu_fd[vcpu], KVM_GET_MP_STATE, mp_state);
 787     return -ENOSYS;
 788 }
 789
 790 int kvm_set_mpstate(kvm_context_t kvm, int vcpu, struct kvm_mp_state *mp_state)
 791 {
 792     int r;
 793
 794     r = ioctl(kvm->fd, KVM_CHECK_EXTENSION, KVM_CAP_MP_STATE);
 795     if (r > 0)
 796         return ioctl(kvm->vcpu_fd[vcpu], KVM_SET_MP_STATE, mp_state);
 797     return -ENOSYS;
 798 }
 799 #endif
 800
 801 static int handle_mmio(kvm_context_t kvm, struct kvm_run *kvm_run)
 802 {
 803         unsigned long addr = kvm_run->mmio.phys_addr;
 804         void *data = kvm_run->mmio.data;
 805
 806         /* hack: Red Hat 7.1 generates these weird accesses. */
 807         if ((addr > 0xa0000-4 && addr <= 0xa0000) && kvm_run->mmio.len == 3)
 808             return 0;
 809
 810         if (kvm_run->mmio.is_write)
 811                 return kvm->callbacks->mmio_write(kvm->opaque, addr, data,
 812                                         kvm_run->mmio.len);
 813         else
 814                 return kvm->callbacks->mmio_read(kvm->opaque, addr, data,
 815                                         kvm_run->mmio.len);
 816 }
 817
 818 int handle_io_window(kvm_context_t kvm)
 819 {
 820         return kvm->callbacks->io_window(kvm->opaque);
 821 }
 822
 823 int handle_halt(kvm_context_t kvm, int vcpu)
 824 {
 825         return kvm->callbacks->halt(kvm->opaque, vcpu);
 826 }
 827
 828 int handle_shutdown(kvm_context_t kvm, int vcpu)
 829 {
 830         return kvm->callbacks->shutdown(kvm->opaque, vcpu);
 831 }
 832
 833 int try_push_interrupts(kvm_context_t kvm)
 834 {
 835         return kvm->callbacks->try_push_interrupts(kvm->opaque);
 836 }
 837
 838 void post_kvm_run(kvm_context_t kvm, int vcpu)
 839 {
 840         kvm->callbacks->post_kvm_run(kvm->opaque, vcpu);
 841 }
 842
 843 int pre_kvm_run(kvm_context_t kvm, int vcpu)
 844 {
 845         return kvm->callbacks->pre_kvm_run(kvm->opaque, vcpu);
 846 }
 847
 848 int kvm_get_interrupt_flag(kvm_context_t kvm, int vcpu)
 849 {
 850         struct kvm_run *run = kvm->run[vcpu];
 851
 852         return run->if_flag;
 853 }
 854
 855 int kvm_is_ready_for_interrupt_injection(kvm_context_t kvm, int vcpu)
 856 {
 857         struct kvm_run *run = kvm->run[vcpu];
 858
 859         return run->ready_for_interrupt_injection;
 860 }
 861
 862 int kvm_run(kvm_context_t kvm, int vcpu)
 863 {
 864         int r;
 865         int fd = kvm->vcpu_fd[vcpu];
 866         struct kvm_run *run = kvm->run[vcpu];
 867
 868 again:
 869         if (!kvm->irqchip_in_kernel)
 870                 run->request_interrupt_window = try_push_interrupts(kvm);
 871         r = pre_kvm_run(kvm, vcpu);
 872         if (r)
 873             return r;
 874         r = ioctl(fd, KVM_RUN, 0);
 875
 876         if (r == -1 && errno != EINTR && errno != EAGAIN) {
 877                 r = -errno;
 878                 post_kvm_run(kvm, vcpu);
 879                 printf("kvm_run: %s\n", strerror(-r));
 880                 return r;
 881         }
 882
 883         post_kvm_run(kvm, vcpu);
 884
 885         if (r == -1) {
 886                 r = handle_io_window(kvm);
 887                 goto more;
 888         }
 889         if (1) {
 890                 switch (run->exit_reason) {
 891                 case KVM_EXIT_UNKNOWN:
 892                         fprintf(stderr, "unhandled vm exit: 0x%x vcpu_id %d\n",
 893                                 (unsigned)run->hw.hardware_exit_reason, vcpu);
 894                         kvm_show_regs(kvm, vcpu);
 895                         abort();
 896                         break;
 897                 case KVM_EXIT_FAIL_ENTRY:
 898                         fprintf(stderr, "kvm_run: failed entry, reason %u\n",
 899                                 (unsigned)run->fail_entry.hardware_entry_failure_reason & 0xffff);
 900                         return -ENOEXEC;
 901                         break;
 902                 case KVM_EXIT_EXCEPTION:
 903                         fprintf(stderr, "exception %d (%x)\n",
 904                                run->ex.exception,
 905                                run->ex.error_code);
 906                         kvm_show_regs(kvm, vcpu);
 907                         kvm_show_code(kvm, vcpu);
 908                         abort();
 909                         break;
 910                 case KVM_EXIT_IO:
 911                         r = handle_io(kvm, run, vcpu);
 912                         break;
 913                 case KVM_EXIT_DEBUG:
 914                         r = handle_debug(kvm, vcpu);
 915                         break;
 916                 case KVM_EXIT_MMIO:
 917                         r = handle_mmio(kvm, run);
 918                         break;
 919                 case KVM_EXIT_HLT:
 920                         r = handle_halt(kvm, vcpu);
 921                         break;
 922                 case KVM_EXIT_IRQ_WINDOW_OPEN:
 923                         break;
 924                 case KVM_EXIT_SHUTDOWN:
 925                         r = handle_shutdown(kvm, vcpu);
 926                         break;
 927                 default:
 928                         if (kvm_arch_run(run, kvm, vcpu)) {
 929                                 fprintf(stderr, "unhandled vm exit: 0x%x\n",
 930                                                         run->exit_reason);
 931                                 kvm_show_regs(kvm, vcpu);
 932                                 abort();
 933                         }
 934                         break;
 935                 }
 936         }
 937 more:
 938         if (!r)
 939                 goto again;
 940         return r;
 941 }
 942
 943 int kvm_inject_irq(kvm_context_t kvm, int vcpu, unsigned irq)
 944 {
 945         struct kvm_interrupt intr;
 946
 947         intr.irq = irq;
 948         return ioctl(kvm->vcpu_fd[vcpu], KVM_INTERRUPT, &intr);
 949 }
 950
 951 int kvm_guest_debug(kvm_context_t kvm, int vcpu, struct kvm_debug_guest *dbg)
 952 {
 953         return ioctl(kvm->vcpu_fd[vcpu], KVM_DEBUG_GUEST, dbg);
 954 }
 955
 956 int kvm_set_signal_mask(kvm_context_t kvm, int vcpu, const sigset_t *sigset)
 957 {
 958         struct kvm_signal_mask *sigmask;
 959         int r;
 960
 961         if (!sigset) {
 962                 r = ioctl(kvm->vcpu_fd[vcpu], KVM_SET_SIGNAL_MASK, NULL);
 963                 if (r == -1)
 964                         r = -errno;
 965                 return r;
 966         }
 967         sigmask = malloc(sizeof(*sigmask) + sizeof(*sigset));
 968         if (!sigmask)
 969                 return -ENOMEM;
 970
 971         sigmask->len = 8;
 972         memcpy(sigmask->sigset, sigset, sizeof(*sigset));
 973         r = ioctl(kvm->vcpu_fd[vcpu], KVM_SET_SIGNAL_MASK, sigmask);
 974         if (r == -1)
 975                 r = -errno;
 976         free(sigmask);
 977         return r;
 978 }
 979
 980 int kvm_irqchip_in_kernel(kvm_context_t kvm)
 981 {
 982     return kvm->irqchip_in_kernel;
 983 }
 984
 985 int kvm_pit_in_kernel(kvm_context_t kvm)
 986 {
 987         return kvm->pit_in_kernel;
 988 }