target-i386/kvm.c

   1 /*
   2  * QEMU KVM support
   3  *
   4  * Copyright (C) 2006-2008 Qumranet Technologies
   5  * Copyright IBM, Corp. 2008
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *
  10  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  11  * See the COPYING file in the top-level directory.
  12  *
  13  */
  14
  15 #include <sys/types.h>
  16 #include <sys/ioctl.h>
  17 #include <sys/mman.h>
  18 #include <sys/utsname.h>
  19
  20 #include <linux/kvm.h>
  21
  22 #include "qemu-common.h"
  23 #include "sysemu.h"
  24 #include "kvm.h"
  25 #include "cpu.h"
  26 #include "gdbstub.h"
  27 #include "host-utils.h"
  28 #include "hw/pc.h"
  29 #include "hw/apic.h"
  30 #include "ioport.h"
  31 #include "kvm_x86.h"
  32
  33 #ifdef CONFIG_KVM_PARA
  34 #include <linux/kvm_para.h>
  35 #endif
  36 //
  37 //#define DEBUG_KVM
  38
  39 #ifdef DEBUG_KVM
  40 #define DPRINTF(fmt, ...) \
  41     do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
  42 #else
  43 #define DPRINTF(fmt, ...) \
  44     do { } while (0)
  45 #endif
  46
  47 #define MSR_KVM_WALL_CLOCK  0x11
  48 #define MSR_KVM_SYSTEM_TIME 0x12
  49
  50 #ifndef BUS_MCEERR_AR
  51 #define BUS_MCEERR_AR 4
  52 #endif
  53 #ifndef BUS_MCEERR_AO
  54 #define BUS_MCEERR_AO 5
  55 #endif
  56
  57 static int lm_capable_kernel;
  58
  59 #ifdef KVM_CAP_EXT_CPUID
  60
  61 static struct kvm_cpuid2 *try_get_cpuid(KVMState *s, int max)
  62 {
  63     struct kvm_cpuid2 *cpuid;
  64     int r, size;
  65
  66     size = sizeof(*cpuid) + max * sizeof(*cpuid->entries);
  67     cpuid = (struct kvm_cpuid2 *)qemu_mallocz(size);
  68     cpuid->nent = max;
  69     r = kvm_ioctl(s, KVM_GET_SUPPORTED_CPUID, cpuid);
  70     if (r == 0 && cpuid->nent >= max) {
  71         r = -E2BIG;
  72     }
  73     if (r < 0) {
  74         if (r == -E2BIG) {
  75             qemu_free(cpuid);
  76             return NULL;
  77         } else {
  78             fprintf(stderr, "KVM_GET_SUPPORTED_CPUID failed: %s\n",
  79                     strerror(-r));
  80             exit(1);
  81         }
  82     }
  83     return cpuid;
  84 }
  85
  86 uint32_t kvm_arch_get_supported_cpuid(CPUState *env, uint32_t function,
  87                                       uint32_t index, int reg)
  88 {
  89     struct kvm_cpuid2 *cpuid;
  90     int i, max;
  91     uint32_t ret = 0;
  92     uint32_t cpuid_1_edx;
  93
  94     if (!kvm_check_extension(env->kvm_state, KVM_CAP_EXT_CPUID)) {
  95         return -1U;
  96     }
  97
  98     max = 1;
  99     while ((cpuid = try_get_cpuid(env->kvm_state, max)) == NULL) {
 100         max *= 2;
 101     }
 102
 103     for (i = 0; i < cpuid->nent; ++i) {
 104         if (cpuid->entries[i].function == function &&
 105             cpuid->entries[i].index == index) {
 106             switch (reg) {
 107             case R_EAX:
 108                 ret = cpuid->entries[i].eax;
 109                 break;
 110             case R_EBX:
 111                 ret = cpuid->entries[i].ebx;
 112                 break;
 113             case R_ECX:
 114                 ret = cpuid->entries[i].ecx;
 115                 break;
 116             case R_EDX:
 117                 ret = cpuid->entries[i].edx;
 118                 switch (function) {
 119                 case 1:
 120                     /* KVM before 2.6.30 misreports the following features */
 121                     ret |= CPUID_MTRR | CPUID_PAT | CPUID_MCE | CPUID_MCA;
 122                     break;
 123                 case 0x80000001:
 124                     /* On Intel, kvm returns cpuid according to the Intel spec,
 125                      * so add missing bits according to the AMD spec:
 126                      */
 127                     cpuid_1_edx = kvm_arch_get_supported_cpuid(env, 1, 0, R_EDX);
 128                     ret |= cpuid_1_edx & 0x183f7ff;
 129                     break;
 130                 }
 131                 break;
 132             }
 133         }
 134     }
 135
 136     qemu_free(cpuid);
 137
 138     return ret;
 139 }
 140
 141 #else
 142
 143 uint32_t kvm_arch_get_supported_cpuid(CPUState *env, uint32_t function,
 144                                       uint32_t index, int reg)
 145 {
 146     return -1U;
 147 }
 148
 149 #endif
 150
 151 #ifdef CONFIG_KVM_PARA
 152 struct kvm_para_features {
 153         int cap;
 154         int feature;
 155 } para_features[] = {
 156 #ifdef KVM_CAP_CLOCKSOURCE
 157         { KVM_CAP_CLOCKSOURCE, KVM_FEATURE_CLOCKSOURCE },
 158 #endif
 159 #ifdef KVM_CAP_NOP_IO_DELAY
 160         { KVM_CAP_NOP_IO_DELAY, KVM_FEATURE_NOP_IO_DELAY },
 161 #endif
 162 #ifdef KVM_CAP_PV_MMU
 163         { KVM_CAP_PV_MMU, KVM_FEATURE_MMU_OP },
 164 #endif
 165         { -1, -1 }
 166 };
 167
 168 static int get_para_features(CPUState *env)
 169 {
 170         int i, features = 0;
 171
 172         for (i = 0; i < ARRAY_SIZE(para_features) - 1; i++) {
 173                 if (kvm_check_extension(env->kvm_state, para_features[i].cap))
 174                         features |= (1 << para_features[i].feature);
 175         }
 176
 177         return features;
 178 }
 179 #endif
 180
 181 #ifdef KVM_CAP_MCE
 182 static int kvm_get_mce_cap_supported(KVMState *s, uint64_t *mce_cap,
 183                                      int *max_banks)
 184 {
 185     int r;
 186
 187     r = kvm_ioctl(s, KVM_CHECK_EXTENSION, KVM_CAP_MCE);
 188     if (r > 0) {
 189         *max_banks = r;
 190         return kvm_ioctl(s, KVM_X86_GET_MCE_CAP_SUPPORTED, mce_cap);
 191     }
 192     return -ENOSYS;
 193 }
 194
 195 static int kvm_setup_mce(CPUState *env, uint64_t *mcg_cap)
 196 {
 197     return kvm_vcpu_ioctl(env, KVM_X86_SETUP_MCE, mcg_cap);
 198 }
 199
 200 static int kvm_set_mce(CPUState *env, struct kvm_x86_mce *m)
 201 {
 202     return kvm_vcpu_ioctl(env, KVM_X86_SET_MCE, m);
 203 }
 204
 205 static int kvm_get_msr(CPUState *env, struct kvm_msr_entry *msrs, int n)
 206 {
 207     struct kvm_msrs *kmsrs = qemu_malloc(sizeof *kmsrs + n * sizeof *msrs);
 208     int r;
 209
 210     kmsrs->nmsrs = n;
 211     memcpy(kmsrs->entries, msrs, n * sizeof *msrs);
 212     r = kvm_vcpu_ioctl(env, KVM_GET_MSRS, kmsrs);
 213     memcpy(msrs, kmsrs->entries, n * sizeof *msrs);
 214     free(kmsrs);
 215     return r;
 216 }
 217
 218 /* FIXME: kill this and kvm_get_msr, use env->mcg_status instead */
 219 static int kvm_mce_in_exception(CPUState *env)
 220 {
 221     struct kvm_msr_entry msr_mcg_status = {
 222         .index = MSR_MCG_STATUS,
 223     };
 224     int r;
 225
 226     r = kvm_get_msr(env, &msr_mcg_status, 1);
 227     if (r == -1 || r == 0) {
 228         return -1;
 229     }
 230     return !!(msr_mcg_status.data & MCG_STATUS_MCIP);
 231 }
 232
 233 struct kvm_x86_mce_data
 234 {
 235     CPUState *env;
 236     struct kvm_x86_mce *mce;
 237     int abort_on_error;
 238 };
 239
 240 static void kvm_do_inject_x86_mce(void *_data)
 241 {
 242     struct kvm_x86_mce_data *data = _data;
 243     int r;
 244
 245     /* If there is an MCE exception being processed, ignore this SRAO MCE */
 246     if ((data->env->mcg_cap & MCG_SER_P) &&
 247         !(data->mce->status & MCI_STATUS_AR)) {
 248         r = kvm_mce_in_exception(data->env);
 249         if (r == -1) {
 250             fprintf(stderr, "Failed to get MCE status\n");
 251         } else if (r) {
 252             return;
 253         }
 254     }
 255
 256     r = kvm_set_mce(data->env, data->mce);
 257     if (r < 0) {
 258         perror("kvm_set_mce FAILED");
 259         if (data->abort_on_error) {
 260             abort();
 261         }
 262     }
 263 }
 264 #endif
 265
 266 void kvm_inject_x86_mce(CPUState *cenv, int bank, uint64_t status,
 267                         uint64_t mcg_status, uint64_t addr, uint64_t misc,
 268                         int abort_on_error)
 269 {
 270 #ifdef KVM_CAP_MCE
 271     struct kvm_x86_mce mce = {
 272         .bank = bank,
 273         .status = status,
 274         .mcg_status = mcg_status,
 275         .addr = addr,
 276         .misc = misc,
 277     };
 278     struct kvm_x86_mce_data data = {
 279             .env = cenv,
 280             .mce = &mce,
 281     };
 282
 283     if (!cenv->mcg_cap) {
 284         fprintf(stderr, "MCE support is not enabled!\n");
 285         return;
 286     }
 287
 288     run_on_cpu(cenv, kvm_do_inject_x86_mce, &data);
 289 #else
 290     if (abort_on_error)
 291         abort();
 292 #endif
 293 }
 294
 295 int kvm_arch_init_vcpu(CPUState *env)
 296 {
 297     struct {
 298         struct kvm_cpuid2 cpuid;
 299         struct kvm_cpuid_entry2 entries[100];
 300     } __attribute__((packed)) cpuid_data;
 301     uint32_t limit, i, j, cpuid_i;
 302     uint32_t unused;
 303     struct kvm_cpuid_entry2 *c;
 304 #ifdef KVM_CPUID_SIGNATURE
 305     uint32_t signature[3];
 306 #endif
 307
 308     env->mp_state = KVM_MP_STATE_RUNNABLE;
 309
 310     env->cpuid_features &= kvm_arch_get_supported_cpuid(env, 1, 0, R_EDX);
 311
 312     i = env->cpuid_ext_features & CPUID_EXT_HYPERVISOR;
 313     env->cpuid_ext_features &= kvm_arch_get_supported_cpuid(env, 1, 0, R_ECX);
 314     env->cpuid_ext_features |= i;
 315
 316     env->cpuid_ext2_features &= kvm_arch_get_supported_cpuid(env, 0x80000001,
 317                                                              0, R_EDX);
 318     env->cpuid_ext3_features &= kvm_arch_get_supported_cpuid(env, 0x80000001,
 319                                                              0, R_ECX);
 320     env->cpuid_svm_features  &= kvm_arch_get_supported_cpuid(env, 0x8000000A,
 321                                                              0, R_EDX);
 322
 323
 324     cpuid_i = 0;
 325
 326 #ifdef CONFIG_KVM_PARA
 327     /* Paravirtualization CPUIDs */
 328     memcpy(signature, "KVMKVMKVM\0\0\0", 12);
 329     c = &cpuid_data.entries[cpuid_i++];
 330     memset(c, 0, sizeof(*c));
 331     c->function = KVM_CPUID_SIGNATURE;
 332     c->eax = 0;
 333     c->ebx = signature[0];
 334     c->ecx = signature[1];
 335     c->edx = signature[2];
 336
 337     c = &cpuid_data.entries[cpuid_i++];
 338     memset(c, 0, sizeof(*c));
 339     c->function = KVM_CPUID_FEATURES;
 340     c->eax = env->cpuid_kvm_features & get_para_features(env);
 341 #endif
 342
 343     cpu_x86_cpuid(env, 0, 0, &limit, &unused, &unused, &unused);
 344
 345     for (i = 0; i <= limit; i++) {
 346         c = &cpuid_data.entries[cpuid_i++];
 347
 348         switch (i) {
 349         case 2: {
 350             /* Keep reading function 2 till all the input is received */
 351             int times;
 352
 353             c->function = i;
 354             c->flags = KVM_CPUID_FLAG_STATEFUL_FUNC |
 355                        KVM_CPUID_FLAG_STATE_READ_NEXT;
 356             cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 357             times = c->eax & 0xff;
 358
 359             for (j = 1; j < times; ++j) {
 360                 c = &cpuid_data.entries[cpuid_i++];
 361                 c->function = i;
 362                 c->flags = KVM_CPUID_FLAG_STATEFUL_FUNC;
 363                 cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 364             }
 365             break;
 366         }
 367         case 4:
 368         case 0xb:
 369         case 0xd:
 370             for (j = 0; ; j++) {
 371                 c->function = i;
 372                 c->flags = KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
 373                 c->index = j;
 374                 cpu_x86_cpuid(env, i, j, &c->eax, &c->ebx, &c->ecx, &c->edx);
 375
 376                 if (i == 4 && c->eax == 0)
 377                     break;
 378                 if (i == 0xb && !(c->ecx & 0xff00))
 379                     break;
 380                 if (i == 0xd && c->eax == 0)
 381                     break;
 382
 383                 c = &cpuid_data.entries[cpuid_i++];
 384             }
 385             break;
 386         default:
 387             c->function = i;
 388             c->flags = 0;
 389             cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 390             break;
 391         }
 392     }
 393     cpu_x86_cpuid(env, 0x80000000, 0, &limit, &unused, &unused, &unused);
 394
 395     for (i = 0x80000000; i <= limit; i++) {
 396         c = &cpuid_data.entries[cpuid_i++];
 397
 398         c->function = i;
 399         c->flags = 0;
 400         cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 401     }
 402
 403     cpuid_data.cpuid.nent = cpuid_i;
 404
 405 #ifdef KVM_CAP_MCE
 406     if (((env->cpuid_version >> 8)&0xF) >= 6
 407         && (env->cpuid_features&(CPUID_MCE|CPUID_MCA)) == (CPUID_MCE|CPUID_MCA)
 408         && kvm_check_extension(env->kvm_state, KVM_CAP_MCE) > 0) {
 409         uint64_t mcg_cap;
 410         int banks;
 411
 412         if (kvm_get_mce_cap_supported(env->kvm_state, &mcg_cap, &banks))
 413             perror("kvm_get_mce_cap_supported FAILED");
 414         else {
 415             if (banks > MCE_BANKS_DEF)
 416                 banks = MCE_BANKS_DEF;
 417             mcg_cap &= MCE_CAP_DEF;
 418             mcg_cap |= banks;
 419             if (kvm_setup_mce(env, &mcg_cap))
 420                 perror("kvm_setup_mce FAILED");
 421             else
 422                 env->mcg_cap = mcg_cap;
 423         }
 424     }
 425 #endif
 426
 427     return kvm_vcpu_ioctl(env, KVM_SET_CPUID2, &cpuid_data);
 428 }
 429
 430 void kvm_arch_reset_vcpu(CPUState *env)
 431 {
 432     env->exception_injected = -1;
 433     env->interrupt_injected = -1;
 434     env->nmi_injected = 0;
 435     env->nmi_pending = 0;
 436     if (kvm_irqchip_in_kernel()) {
 437         env->mp_state = cpu_is_bsp(env) ? KVM_MP_STATE_RUNNABLE :
 438                                           KVM_MP_STATE_UNINITIALIZED;
 439     } else {
 440         env->mp_state = KVM_MP_STATE_RUNNABLE;
 441     }
 442 }
 443
 444 int has_msr_star;
 445 int has_msr_hsave_pa;
 446
 447 static void kvm_supported_msrs(CPUState *env)
 448 {
 449     static int kvm_supported_msrs;
 450     int ret;
 451
 452     /* first time */
 453     if (kvm_supported_msrs == 0) {
 454         struct kvm_msr_list msr_list, *kvm_msr_list;
 455
 456         kvm_supported_msrs = -1;
 457
 458         /* Obtain MSR list from KVM.  These are the MSRs that we must
 459          * save/restore */
 460         msr_list.nmsrs = 0;
 461         ret = kvm_ioctl(env->kvm_state, KVM_GET_MSR_INDEX_LIST, &msr_list);
 462         if (ret < 0 && ret != -E2BIG) {
 463             return;
 464         }
 465         /* Old kernel modules had a bug and could write beyond the provided
 466            memory. Allocate at least a safe amount of 1K. */
 467         kvm_msr_list = qemu_mallocz(MAX(1024, sizeof(msr_list) +
 468                                               msr_list.nmsrs *
 469                                               sizeof(msr_list.indices[0])));
 470
 471         kvm_msr_list->nmsrs = msr_list.nmsrs;
 472         ret = kvm_ioctl(env->kvm_state, KVM_GET_MSR_INDEX_LIST, kvm_msr_list);
 473         if (ret >= 0) {
 474             int i;
 475
 476             for (i = 0; i < kvm_msr_list->nmsrs; i++) {
 477                 if (kvm_msr_list->indices[i] == MSR_STAR) {
 478                     has_msr_star = 1;
 479                     continue;
 480                 }
 481                 if (kvm_msr_list->indices[i] == MSR_VM_HSAVE_PA) {
 482                     has_msr_hsave_pa = 1;
 483                     continue;
 484                 }
 485             }
 486         }
 487
 488         free(kvm_msr_list);
 489     }
 490
 491     return;
 492 }
 493
 494 static int kvm_has_msr_hsave_pa(CPUState *env)
 495 {
 496     kvm_supported_msrs(env);
 497     return has_msr_hsave_pa;
 498 }
 499
 500 static int kvm_has_msr_star(CPUState *env)
 501 {
 502     kvm_supported_msrs(env);
 503     return has_msr_star;
 504 }
 505
 506 static int kvm_init_identity_map_page(KVMState *s)
 507 {
 508 #ifdef KVM_CAP_SET_IDENTITY_MAP_ADDR
 509     int ret;
 510     uint64_t addr = 0xfffbc000;
 511
 512     if (!kvm_check_extension(s, KVM_CAP_SET_IDENTITY_MAP_ADDR)) {
 513         return 0;
 514     }
 515
 516     ret = kvm_vm_ioctl(s, KVM_SET_IDENTITY_MAP_ADDR, &addr);
 517     if (ret < 0) {
 518         fprintf(stderr, "kvm_set_identity_map_addr: %s\n", strerror(ret));
 519         return ret;
 520     }
 521 #endif
 522     return 0;
 523 }
 524
 525 int kvm_arch_init(KVMState *s, int smp_cpus)
 526 {
 527     int ret;
 528
 529     struct utsname utsname;
 530
 531     uname(&utsname);
 532     lm_capable_kernel = strcmp(utsname.machine, "x86_64") == 0;
 533
 534     /* create vm86 tss.  KVM uses vm86 mode to emulate 16-bit code
 535      * directly.  In order to use vm86 mode, a TSS is needed.  Since this
 536      * must be part of guest physical memory, we need to allocate it.  Older
 537      * versions of KVM just assumed that it would be at the end of physical
 538      * memory but that doesn't work with more than 4GB of memory.  We simply
 539      * refuse to work with those older versions of KVM. */
 540     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, KVM_CAP_SET_TSS_ADDR);
 541     if (ret <= 0) {
 542         fprintf(stderr, "kvm does not support KVM_CAP_SET_TSS_ADDR\n");
 543         return ret;
 544     }
 545
 546     /* this address is 3 pages before the bios, and the bios should present
 547      * as unavaible memory.  FIXME, need to ensure the e820 map deals with
 548      * this?
 549      */
 550     /*
 551      * Tell fw_cfg to notify the BIOS to reserve the range.
 552      */
 553     if (e820_add_entry(0xfffbc000, 0x4000, E820_RESERVED) < 0) {
 554         perror("e820_add_entry() table is full");
 555         exit(1);
 556     }
 557     ret = kvm_vm_ioctl(s, KVM_SET_TSS_ADDR, 0xfffbd000);
 558     if (ret < 0) {
 559         return ret;
 560     }
 561
 562     return kvm_init_identity_map_page(s);
 563 }
 564
 565 static void set_v8086_seg(struct kvm_segment *lhs, const SegmentCache *rhs)
 566 {
 567     lhs->selector = rhs->selector;
 568     lhs->base = rhs->base;
 569     lhs->limit = rhs->limit;
 570     lhs->type = 3;
 571     lhs->present = 1;
 572     lhs->dpl = 3;
 573     lhs->db = 0;
 574     lhs->s = 1;
 575     lhs->l = 0;
 576     lhs->g = 0;
 577     lhs->avl = 0;
 578     lhs->unusable = 0;
 579 }
 580
 581 static void set_seg(struct kvm_segment *lhs, const SegmentCache *rhs)
 582 {
 583     unsigned flags = rhs->flags;
 584     lhs->selector = rhs->selector;
 585     lhs->base = rhs->base;
 586     lhs->limit = rhs->limit;
 587     lhs->type = (flags >> DESC_TYPE_SHIFT) & 15;
 588     lhs->present = (flags & DESC_P_MASK) != 0;
 589     lhs->dpl = rhs->selector & 3;
 590     lhs->db = (flags >> DESC_B_SHIFT) & 1;
 591     lhs->s = (flags & DESC_S_MASK) != 0;
 592     lhs->l = (flags >> DESC_L_SHIFT) & 1;
 593     lhs->g = (flags & DESC_G_MASK) != 0;
 594     lhs->avl = (flags & DESC_AVL_MASK) != 0;
 595     lhs->unusable = 0;
 596 }
 597
 598 static void get_seg(SegmentCache *lhs, const struct kvm_segment *rhs)
 599 {
 600     lhs->selector = rhs->selector;
 601     lhs->base = rhs->base;
 602     lhs->limit = rhs->limit;
 603     lhs->flags =
 604         (rhs->type << DESC_TYPE_SHIFT)
 605         | (rhs->present * DESC_P_MASK)
 606         | (rhs->dpl << DESC_DPL_SHIFT)
 607         | (rhs->db << DESC_B_SHIFT)
 608         | (rhs->s * DESC_S_MASK)
 609         | (rhs->l << DESC_L_SHIFT)
 610         | (rhs->g * DESC_G_MASK)
 611         | (rhs->avl * DESC_AVL_MASK);
 612 }
 613
 614 static void kvm_getput_reg(__u64 *kvm_reg, target_ulong *qemu_reg, int set)
 615 {
 616     if (set)
 617         *kvm_reg = *qemu_reg;
 618     else
 619         *qemu_reg = *kvm_reg;
 620 }
 621
 622 static int kvm_getput_regs(CPUState *env, int set)
 623 {
 624     struct kvm_regs regs;
 625     int ret = 0;
 626
 627     if (!set) {
 628         ret = kvm_vcpu_ioctl(env, KVM_GET_REGS, &regs);
 629         if (ret < 0)
 630             return ret;
 631     }
 632
 633     kvm_getput_reg(&regs.rax, &env->regs[R_EAX], set);
 634     kvm_getput_reg(&regs.rbx, &env->regs[R_EBX], set);
 635     kvm_getput_reg(&regs.rcx, &env->regs[R_ECX], set);
 636     kvm_getput_reg(&regs.rdx, &env->regs[R_EDX], set);
 637     kvm_getput_reg(&regs.rsi, &env->regs[R_ESI], set);
 638     kvm_getput_reg(&regs.rdi, &env->regs[R_EDI], set);
 639     kvm_getput_reg(&regs.rsp, &env->regs[R_ESP], set);
 640     kvm_getput_reg(&regs.rbp, &env->regs[R_EBP], set);
 641 #ifdef TARGET_X86_64
 642     kvm_getput_reg(&regs.r8, &env->regs[8], set);
 643     kvm_getput_reg(&regs.r9, &env->regs[9], set);
 644     kvm_getput_reg(&regs.r10, &env->regs[10], set);
 645     kvm_getput_reg(&regs.r11, &env->regs[11], set);
 646     kvm_getput_reg(&regs.r12, &env->regs[12], set);
 647     kvm_getput_reg(&regs.r13, &env->regs[13], set);
 648     kvm_getput_reg(&regs.r14, &env->regs[14], set);
 649     kvm_getput_reg(&regs.r15, &env->regs[15], set);
 650 #endif
 651
 652     kvm_getput_reg(&regs.rflags, &env->eflags, set);
 653     kvm_getput_reg(&regs.rip, &env->eip, set);
 654
 655     if (set)
 656         ret = kvm_vcpu_ioctl(env, KVM_SET_REGS, &regs);
 657
 658     return ret;
 659 }
 660
 661 static int kvm_put_fpu(CPUState *env)
 662 {
 663     struct kvm_fpu fpu;
 664     int i;
 665
 666     memset(&fpu, 0, sizeof fpu);
 667     fpu.fsw = env->fpus & ~(7 << 11);
 668     fpu.fsw |= (env->fpstt & 7) << 11;
 669     fpu.fcw = env->fpuc;
 670     for (i = 0; i < 8; ++i)
 671         fpu.ftwx |= (!env->fptags[i]) << i;
 672     memcpy(fpu.fpr, env->fpregs, sizeof env->fpregs);
 673     memcpy(fpu.xmm, env->xmm_regs, sizeof env->xmm_regs);
 674     fpu.mxcsr = env->mxcsr;
 675
 676     return kvm_vcpu_ioctl(env, KVM_SET_FPU, &fpu);
 677 }
 678
 679 #ifdef KVM_CAP_XSAVE
 680 #define XSAVE_CWD_RIP     2
 681 #define XSAVE_CWD_RDP     4
 682 #define XSAVE_MXCSR       6
 683 #define XSAVE_ST_SPACE    8
 684 #define XSAVE_XMM_SPACE   40
 685 #define XSAVE_XSTATE_BV   128
 686 #define XSAVE_YMMH_SPACE  144
 687 #endif
 688
 689 static int kvm_put_xsave(CPUState *env)
 690 {
 691 #ifdef KVM_CAP_XSAVE
 692     int i, r;
 693     struct kvm_xsave* xsave;
 694     uint16_t cwd, swd, twd, fop;
 695
 696     if (!kvm_has_xsave())
 697         return kvm_put_fpu(env);
 698
 699     xsave = qemu_memalign(4096, sizeof(struct kvm_xsave));
 700     memset(xsave, 0, sizeof(struct kvm_xsave));
 701     cwd = swd = twd = fop = 0;
 702     swd = env->fpus & ~(7 << 11);
 703     swd |= (env->fpstt & 7) << 11;
 704     cwd = env->fpuc;
 705     for (i = 0; i < 8; ++i)
 706         twd |= (!env->fptags[i]) << i;
 707     xsave->region[0] = (uint32_t)(swd << 16) + cwd;
 708     xsave->region[1] = (uint32_t)(fop << 16) + twd;
 709     memcpy(&xsave->region[XSAVE_ST_SPACE], env->fpregs,
 710             sizeof env->fpregs);
 711     memcpy(&xsave->region[XSAVE_XMM_SPACE], env->xmm_regs,
 712             sizeof env->xmm_regs);
 713     xsave->region[XSAVE_MXCSR] = env->mxcsr;
 714     *(uint64_t *)&xsave->region[XSAVE_XSTATE_BV] = env->xstate_bv;
 715     memcpy(&xsave->region[XSAVE_YMMH_SPACE], env->ymmh_regs,
 716             sizeof env->ymmh_regs);
 717     r = kvm_vcpu_ioctl(env, KVM_SET_XSAVE, xsave);
 718     qemu_free(xsave);
 719     return r;
 720 #else
 721     return kvm_put_fpu(env);
 722 #endif
 723 }
 724
 725 static int kvm_put_xcrs(CPUState *env)
 726 {
 727 #ifdef KVM_CAP_XCRS
 728     struct kvm_xcrs xcrs;
 729
 730     if (!kvm_has_xcrs())
 731         return 0;
 732
 733     xcrs.nr_xcrs = 1;
 734     xcrs.flags = 0;
 735     xcrs.xcrs[0].xcr = 0;
 736     xcrs.xcrs[0].value = env->xcr0;
 737     return kvm_vcpu_ioctl(env, KVM_SET_XCRS, &xcrs);
 738 #else
 739     return 0;
 740 #endif
 741 }
 742
 743 static int kvm_put_sregs(CPUState *env)
 744 {
 745     struct kvm_sregs sregs;
 746
 747     memset(sregs.interrupt_bitmap, 0, sizeof(sregs.interrupt_bitmap));
 748     if (env->interrupt_injected >= 0) {
 749         sregs.interrupt_bitmap[env->interrupt_injected / 64] |=
 750                 (uint64_t)1 << (env->interrupt_injected % 64);
 751     }
 752
 753     if ((env->eflags & VM_MASK)) {
 754             set_v8086_seg(&sregs.cs, &env->segs[R_CS]);
 755             set_v8086_seg(&sregs.ds, &env->segs[R_DS]);
 756             set_v8086_seg(&sregs.es, &env->segs[R_ES]);
 757             set_v8086_seg(&sregs.fs, &env->segs[R_FS]);
 758             set_v8086_seg(&sregs.gs, &env->segs[R_GS]);
 759             set_v8086_seg(&sregs.ss, &env->segs[R_SS]);
 760     } else {
 761             set_seg(&sregs.cs, &env->segs[R_CS]);
 762             set_seg(&sregs.ds, &env->segs[R_DS]);
 763             set_seg(&sregs.es, &env->segs[R_ES]);
 764             set_seg(&sregs.fs, &env->segs[R_FS]);
 765             set_seg(&sregs.gs, &env->segs[R_GS]);
 766             set_seg(&sregs.ss, &env->segs[R_SS]);
 767
 768             if (env->cr[0] & CR0_PE_MASK) {
 769                 /* force ss cpl to cs cpl */
 770                 sregs.ss.selector = (sregs.ss.selector & ~3) |
 771                         (sregs.cs.selector & 3);
 772                 sregs.ss.dpl = sregs.ss.selector & 3;
 773             }
 774     }
 775
 776     set_seg(&sregs.tr, &env->tr);
 777     set_seg(&sregs.ldt, &env->ldt);
 778
 779     sregs.idt.limit = env->idt.limit;
 780     sregs.idt.base = env->idt.base;
 781     sregs.gdt.limit = env->gdt.limit;
 782     sregs.gdt.base = env->gdt.base;
 783
 784     sregs.cr0 = env->cr[0];
 785     sregs.cr2 = env->cr[2];
 786     sregs.cr3 = env->cr[3];
 787     sregs.cr4 = env->cr[4];
 788
 789     sregs.cr8 = cpu_get_apic_tpr(env->apic_state);
 790     sregs.apic_base = cpu_get_apic_base(env->apic_state);
 791
 792     sregs.efer = env->efer;
 793
 794     return kvm_vcpu_ioctl(env, KVM_SET_SREGS, &sregs);
 795 }
 796
 797 static void kvm_msr_entry_set(struct kvm_msr_entry *entry,
 798                               uint32_t index, uint64_t value)
 799 {
 800     entry->index = index;
 801     entry->data = value;
 802 }
 803
 804 static int kvm_put_msrs(CPUState *env, int level)
 805 {
 806     struct {
 807         struct kvm_msrs info;
 808         struct kvm_msr_entry entries[100];
 809     } msr_data;
 810     struct kvm_msr_entry *msrs = msr_data.entries;
 811     int n = 0;
 812
 813     kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_CS, env->sysenter_cs);
 814     kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_ESP, env->sysenter_esp);
 815     kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_EIP, env->sysenter_eip);
 816     if (kvm_has_msr_star(env))
 817         kvm_msr_entry_set(&msrs[n++], MSR_STAR, env->star);
 818     if (kvm_has_msr_hsave_pa(env))
 819         kvm_msr_entry_set(&msrs[n++], MSR_VM_HSAVE_PA, env->vm_hsave);
 820 #ifdef TARGET_X86_64
 821     if (lm_capable_kernel) {
 822         kvm_msr_entry_set(&msrs[n++], MSR_CSTAR, env->cstar);
 823         kvm_msr_entry_set(&msrs[n++], MSR_KERNELGSBASE, env->kernelgsbase);
 824         kvm_msr_entry_set(&msrs[n++], MSR_FMASK, env->fmask);
 825         kvm_msr_entry_set(&msrs[n++], MSR_LSTAR, env->lstar);
 826     }
 827 #endif
 828     if (level == KVM_PUT_FULL_STATE) {
 829         /*
 830          * KVM is yet unable to synchronize TSC values of multiple VCPUs on
 831          * writeback. Until this is fixed, we only write the offset to SMP
 832          * guests after migration, desynchronizing the VCPUs, but avoiding
 833          * huge jump-backs that would occur without any writeback at all.
 834          */
 835         if (smp_cpus == 1 || env->tsc != 0) {
 836             kvm_msr_entry_set(&msrs[n++], MSR_IA32_TSC, env->tsc);
 837         }
 838         kvm_msr_entry_set(&msrs[n++], MSR_KVM_SYSTEM_TIME,
 839                           env->system_time_msr);
 840         kvm_msr_entry_set(&msrs[n++], MSR_KVM_WALL_CLOCK, env->wall_clock_msr);
 841     }
 842 #ifdef KVM_CAP_MCE
 843     if (env->mcg_cap) {
 844         int i;
 845         if (level == KVM_PUT_RESET_STATE)
 846             kvm_msr_entry_set(&msrs[n++], MSR_MCG_STATUS, env->mcg_status);
 847         else if (level == KVM_PUT_FULL_STATE) {
 848             kvm_msr_entry_set(&msrs[n++], MSR_MCG_STATUS, env->mcg_status);
 849             kvm_msr_entry_set(&msrs[n++], MSR_MCG_CTL, env->mcg_ctl);
 850             for (i = 0; i < (env->mcg_cap & 0xff) * 4; i++)
 851                 kvm_msr_entry_set(&msrs[n++], MSR_MC0_CTL + i, env->mce_banks[i]);
 852         }
 853     }
 854 #endif
 855
 856     msr_data.info.nmsrs = n;
 857
 858     return kvm_vcpu_ioctl(env, KVM_SET_MSRS, &msr_data);
 859
 860 }
 861
 862
 863 static int kvm_get_fpu(CPUState *env)
 864 {
 865     struct kvm_fpu fpu;
 866     int i, ret;
 867
 868     ret = kvm_vcpu_ioctl(env, KVM_GET_FPU, &fpu);
 869     if (ret < 0)
 870         return ret;
 871
 872     env->fpstt = (fpu.fsw >> 11) & 7;
 873     env->fpus = fpu.fsw;
 874     env->fpuc = fpu.fcw;
 875     for (i = 0; i < 8; ++i)
 876         env->fptags[i] = !((fpu.ftwx >> i) & 1);
 877     memcpy(env->fpregs, fpu.fpr, sizeof env->fpregs);
 878     memcpy(env->xmm_regs, fpu.xmm, sizeof env->xmm_regs);
 879     env->mxcsr = fpu.mxcsr;
 880
 881     return 0;
 882 }
 883
 884 static int kvm_get_xsave(CPUState *env)
 885 {
 886 #ifdef KVM_CAP_XSAVE
 887     struct kvm_xsave* xsave;
 888     int ret, i;
 889     uint16_t cwd, swd, twd, fop;
 890
 891     if (!kvm_has_xsave())
 892         return kvm_get_fpu(env);
 893
 894     xsave = qemu_memalign(4096, sizeof(struct kvm_xsave));
 895     ret = kvm_vcpu_ioctl(env, KVM_GET_XSAVE, xsave);
 896     if (ret < 0) {
 897         qemu_free(xsave);
 898         return ret;
 899     }
 900
 901     cwd = (uint16_t)xsave->region[0];
 902     swd = (uint16_t)(xsave->region[0] >> 16);
 903     twd = (uint16_t)xsave->region[1];
 904     fop = (uint16_t)(xsave->region[1] >> 16);
 905     env->fpstt = (swd >> 11) & 7;
 906     env->fpus = swd;
 907     env->fpuc = cwd;
 908     for (i = 0; i < 8; ++i)
 909         env->fptags[i] = !((twd >> i) & 1);
 910     env->mxcsr = xsave->region[XSAVE_MXCSR];
 911     memcpy(env->fpregs, &xsave->region[XSAVE_ST_SPACE],
 912             sizeof env->fpregs);
 913     memcpy(env->xmm_regs, &xsave->region[XSAVE_XMM_SPACE],
 914             sizeof env->xmm_regs);
 915     env->xstate_bv = *(uint64_t *)&xsave->region[XSAVE_XSTATE_BV];
 916     memcpy(env->ymmh_regs, &xsave->region[XSAVE_YMMH_SPACE],
 917             sizeof env->ymmh_regs);
 918     qemu_free(xsave);
 919     return 0;
 920 #else
 921     return kvm_get_fpu(env);
 922 #endif
 923 }
 924
 925 static int kvm_get_xcrs(CPUState *env)
 926 {
 927 #ifdef KVM_CAP_XCRS
 928     int i, ret;
 929     struct kvm_xcrs xcrs;
 930
 931     if (!kvm_has_xcrs())
 932         return 0;
 933
 934     ret = kvm_vcpu_ioctl(env, KVM_GET_XCRS, &xcrs);
 935     if (ret < 0)
 936         return ret;
 937
 938     for (i = 0; i < xcrs.nr_xcrs; i++)
 939         /* Only support xcr0 now */
 940         if (xcrs.xcrs[0].xcr == 0) {
 941             env->xcr0 = xcrs.xcrs[0].value;
 942             break;
 943         }
 944     return 0;
 945 #else
 946     return 0;
 947 #endif
 948 }
 949
 950 static int kvm_get_sregs(CPUState *env)
 951 {
 952     struct kvm_sregs sregs;
 953     uint32_t hflags;
 954     int bit, i, ret;
 955
 956     ret = kvm_vcpu_ioctl(env, KVM_GET_SREGS, &sregs);
 957     if (ret < 0)
 958         return ret;
 959
 960     /* There can only be one pending IRQ set in the bitmap at a time, so try
 961        to find it and save its number instead (-1 for none). */
 962     env->interrupt_injected = -1;
 963     for (i = 0; i < ARRAY_SIZE(sregs.interrupt_bitmap); i++) {
 964         if (sregs.interrupt_bitmap[i]) {
 965             bit = ctz64(sregs.interrupt_bitmap[i]);
 966             env->interrupt_injected = i * 64 + bit;
 967             break;
 968         }
 969     }
 970
 971     get_seg(&env->segs[R_CS], &sregs.cs);
 972     get_seg(&env->segs[R_DS], &sregs.ds);
 973     get_seg(&env->segs[R_ES], &sregs.es);
 974     get_seg(&env->segs[R_FS], &sregs.fs);
 975     get_seg(&env->segs[R_GS], &sregs.gs);
 976     get_seg(&env->segs[R_SS], &sregs.ss);
 977
 978     get_seg(&env->tr, &sregs.tr);
 979     get_seg(&env->ldt, &sregs.ldt);
 980
 981     env->idt.limit = sregs.idt.limit;
 982     env->idt.base = sregs.idt.base;
 983     env->gdt.limit = sregs.gdt.limit;
 984     env->gdt.base = sregs.gdt.base;
 985
 986     env->cr[0] = sregs.cr0;
 987     env->cr[2] = sregs.cr2;
 988     env->cr[3] = sregs.cr3;
 989     env->cr[4] = sregs.cr4;
 990
 991     cpu_set_apic_base(env->apic_state, sregs.apic_base);
 992
 993     env->efer = sregs.efer;
 994     //cpu_set_apic_tpr(env->apic_state, sregs.cr8);
 995
 996 #define HFLAG_COPY_MASK ~( \
 997                         HF_CPL_MASK | HF_PE_MASK | HF_MP_MASK | HF_EM_MASK | \
 998                         HF_TS_MASK | HF_TF_MASK | HF_VM_MASK | HF_IOPL_MASK | \
 999                         HF_OSFXSR_MASK | HF_LMA_MASK | HF_CS32_MASK | \
1000                         HF_SS32_MASK | HF_CS64_MASK | HF_ADDSEG_MASK)
1001
1002
1003
1004     hflags = (env->segs[R_CS].flags >> DESC_DPL_SHIFT) & HF_CPL_MASK;
1005     hflags |= (env->cr[0] & CR0_PE_MASK) << (HF_PE_SHIFT - CR0_PE_SHIFT);
1006     hflags |= (env->cr[0] << (HF_MP_SHIFT - CR0_MP_SHIFT)) &
1007             (HF_MP_MASK | HF_EM_MASK | HF_TS_MASK);
1008     hflags |= (env->eflags & (HF_TF_MASK | HF_VM_MASK | HF_IOPL_MASK));
1009     hflags |= (env->cr[4] & CR4_OSFXSR_MASK) <<
1010             (HF_OSFXSR_SHIFT - CR4_OSFXSR_SHIFT);
1011
1012     if (env->efer & MSR_EFER_LMA) {
1013         hflags |= HF_LMA_MASK;
1014     }
1015
1016     if ((hflags & HF_LMA_MASK) && (env->segs[R_CS].flags & DESC_L_MASK)) {
1017         hflags |= HF_CS32_MASK | HF_SS32_MASK | HF_CS64_MASK;
1018     } else {
1019         hflags |= (env->segs[R_CS].flags & DESC_B_MASK) >>
1020                 (DESC_B_SHIFT - HF_CS32_SHIFT);
1021         hflags |= (env->segs[R_SS].flags & DESC_B_MASK) >>
1022                 (DESC_B_SHIFT - HF_SS32_SHIFT);
1023         if (!(env->cr[0] & CR0_PE_MASK) ||
1024                    (env->eflags & VM_MASK) ||
1025                    !(hflags & HF_CS32_MASK)) {
1026                 hflags |= HF_ADDSEG_MASK;
1027             } else {
1028                 hflags |= ((env->segs[R_DS].base |
1029                                 env->segs[R_ES].base |
1030                                 env->segs[R_SS].base) != 0) <<
1031                     HF_ADDSEG_SHIFT;
1032             }
1033     }
1034     env->hflags = (env->hflags & HFLAG_COPY_MASK) | hflags;
1035
1036     return 0;
1037 }
1038
1039 static int kvm_get_msrs(CPUState *env)
1040 {
1041     struct {
1042         struct kvm_msrs info;
1043         struct kvm_msr_entry entries[100];
1044     } msr_data;
1045     struct kvm_msr_entry *msrs = msr_data.entries;
1046     int ret, i, n;
1047
1048     n = 0;
1049     msrs[n++].index = MSR_IA32_SYSENTER_CS;
1050     msrs[n++].index = MSR_IA32_SYSENTER_ESP;
1051     msrs[n++].index = MSR_IA32_SYSENTER_EIP;
1052     if (kvm_has_msr_star(env))
1053         msrs[n++].index = MSR_STAR;
1054     if (kvm_has_msr_hsave_pa(env))
1055         msrs[n++].index = MSR_VM_HSAVE_PA;
1056     msrs[n++].index = MSR_IA32_TSC;
1057 #ifdef TARGET_X86_64
1058     if (lm_capable_kernel) {
1059         msrs[n++].index = MSR_CSTAR;
1060         msrs[n++].index = MSR_KERNELGSBASE;
1061         msrs[n++].index = MSR_FMASK;
1062         msrs[n++].index = MSR_LSTAR;
1063     }
1064 #endif
1065     msrs[n++].index = MSR_KVM_SYSTEM_TIME;
1066     msrs[n++].index = MSR_KVM_WALL_CLOCK;
1067
1068 #ifdef KVM_CAP_MCE
1069     if (env->mcg_cap) {
1070         msrs[n++].index = MSR_MCG_STATUS;
1071         msrs[n++].index = MSR_MCG_CTL;
1072         for (i = 0; i < (env->mcg_cap & 0xff) * 4; i++)
1073             msrs[n++].index = MSR_MC0_CTL + i;
1074     }
1075 #endif
1076
1077     msr_data.info.nmsrs = n;
1078     ret = kvm_vcpu_ioctl(env, KVM_GET_MSRS, &msr_data);
1079     if (ret < 0)
1080         return ret;
1081
1082     for (i = 0; i < ret; i++) {
1083         switch (msrs[i].index) {
1084         case MSR_IA32_SYSENTER_CS:
1085             env->sysenter_cs = msrs[i].data;
1086             break;
1087         case MSR_IA32_SYSENTER_ESP:
1088             env->sysenter_esp = msrs[i].data;
1089             break;
1090         case MSR_IA32_SYSENTER_EIP:
1091             env->sysenter_eip = msrs[i].data;
1092             break;
1093         case MSR_STAR:
1094             env->star = msrs[i].data;
1095             break;
1096 #ifdef TARGET_X86_64
1097         case MSR_CSTAR:
1098             env->cstar = msrs[i].data;
1099             break;
1100         case MSR_KERNELGSBASE:
1101             env->kernelgsbase = msrs[i].data;
1102             break;
1103         case MSR_FMASK:
1104             env->fmask = msrs[i].data;
1105             break;
1106         case MSR_LSTAR:
1107             env->lstar = msrs[i].data;
1108             break;
1109 #endif
1110         case MSR_IA32_TSC:
1111             env->tsc = msrs[i].data;
1112             break;
1113         case MSR_VM_HSAVE_PA:
1114             env->vm_hsave = msrs[i].data;
1115             break;
1116         case MSR_KVM_SYSTEM_TIME:
1117             env->system_time_msr = msrs[i].data;
1118             break;
1119         case MSR_KVM_WALL_CLOCK:
1120             env->wall_clock_msr = msrs[i].data;
1121             break;
1122 #ifdef KVM_CAP_MCE
1123         case MSR_MCG_STATUS:
1124             env->mcg_status = msrs[i].data;
1125             break;
1126         case MSR_MCG_CTL:
1127             env->mcg_ctl = msrs[i].data;
1128             break;
1129 #endif
1130         default:
1131 #ifdef KVM_CAP_MCE
1132             if (msrs[i].index >= MSR_MC0_CTL &&
1133                 msrs[i].index < MSR_MC0_CTL + (env->mcg_cap & 0xff) * 4) {
1134                 env->mce_banks[msrs[i].index - MSR_MC0_CTL] = msrs[i].data;
1135             }
1136 #endif
1137             break;
1138         }
1139     }
1140
1141     return 0;
1142 }
1143
1144 static int kvm_put_mp_state(CPUState *env)
1145 {
1146     struct kvm_mp_state mp_state = { .mp_state = env->mp_state };
1147
1148     return kvm_vcpu_ioctl(env, KVM_SET_MP_STATE, &mp_state);
1149 }
1150
1151 static int kvm_get_mp_state(CPUState *env)
1152 {
1153     struct kvm_mp_state mp_state;
1154     int ret;
1155
1156     ret = kvm_vcpu_ioctl(env, KVM_GET_MP_STATE, &mp_state);
1157     if (ret < 0) {
1158         return ret;
1159     }
1160     env->mp_state = mp_state.mp_state;
1161     return 0;
1162 }
1163
1164 static int kvm_put_vcpu_events(CPUState *env, int level)
1165 {
1166 #ifdef KVM_CAP_VCPU_EVENTS
1167     struct kvm_vcpu_events events;
1168
1169     if (!kvm_has_vcpu_events()) {
1170         return 0;
1171     }
1172
1173     events.exception.injected = (env->exception_injected >= 0);
1174     events.exception.nr = env->exception_injected;
1175     events.exception.has_error_code = env->has_error_code;
1176     events.exception.error_code = env->error_code;
1177
1178     events.interrupt.injected = (env->interrupt_injected >= 0);
1179     events.interrupt.nr = env->interrupt_injected;
1180     events.interrupt.soft = env->soft_interrupt;
1181
1182     events.nmi.injected = env->nmi_injected;
1183     events.nmi.pending = env->nmi_pending;
1184     events.nmi.masked = !!(env->hflags2 & HF2_NMI_MASK);
1185
1186     events.sipi_vector = env->sipi_vector;
1187
1188     events.flags = 0;
1189     if (level >= KVM_PUT_RESET_STATE) {
1190         events.flags |=
1191             KVM_VCPUEVENT_VALID_NMI_PENDING | KVM_VCPUEVENT_VALID_SIPI_VECTOR;
1192     }
1193
1194     return kvm_vcpu_ioctl(env, KVM_SET_VCPU_EVENTS, &events);
1195 #else
1196     return 0;
1197 #endif
1198 }
1199
1200 static int kvm_get_vcpu_events(CPUState *env)
1201 {
1202 #ifdef KVM_CAP_VCPU_EVENTS
1203     struct kvm_vcpu_events events;
1204     int ret;
1205
1206     if (!kvm_has_vcpu_events()) {
1207         return 0;
1208     }
1209
1210     ret = kvm_vcpu_ioctl(env, KVM_GET_VCPU_EVENTS, &events);
1211     if (ret < 0) {
1212        return ret;
1213     }
1214     env->exception_injected =
1215        events.exception.injected ? events.exception.nr : -1;
1216     env->has_error_code = events.exception.has_error_code;
1217     env->error_code = events.exception.error_code;
1218
1219     env->interrupt_injected =
1220         events.interrupt.injected ? events.interrupt.nr : -1;
1221     env->soft_interrupt = events.interrupt.soft;
1222
1223     env->nmi_injected = events.nmi.injected;
1224     env->nmi_pending = events.nmi.pending;
1225     if (events.nmi.masked) {
1226         env->hflags2 |= HF2_NMI_MASK;
1227     } else {
1228         env->hflags2 &= ~HF2_NMI_MASK;
1229     }
1230
1231     env->sipi_vector = events.sipi_vector;
1232 #endif
1233
1234     return 0;
1235 }
1236
1237 static int kvm_guest_debug_workarounds(CPUState *env)
1238 {
1239     int ret = 0;
1240 #ifdef KVM_CAP_SET_GUEST_DEBUG
1241     unsigned long reinject_trap = 0;
1242
1243     if (!kvm_has_vcpu_events()) {
1244         if (env->exception_injected == 1) {
1245             reinject_trap = KVM_GUESTDBG_INJECT_DB;
1246         } else if (env->exception_injected == 3) {
1247             reinject_trap = KVM_GUESTDBG_INJECT_BP;
1248         }
1249         env->exception_injected = -1;
1250     }
1251
1252     /*
1253      * Kernels before KVM_CAP_X86_ROBUST_SINGLESTEP overwrote flags.TF
1254      * injected via SET_GUEST_DEBUG while updating GP regs. Work around this
1255      * by updating the debug state once again if single-stepping is on.
1256      * Another reason to call kvm_update_guest_debug here is a pending debug
1257      * trap raise by the guest. On kernels without SET_VCPU_EVENTS we have to
1258      * reinject them via SET_GUEST_DEBUG.
1259      */
1260     if (reinject_trap ||
1261         (!kvm_has_robust_singlestep() && env->singlestep_enabled)) {
1262         ret = kvm_update_guest_debug(env, reinject_trap);
1263     }
1264 #endif /* KVM_CAP_SET_GUEST_DEBUG */
1265     return ret;
1266 }
1267
1268 static int kvm_put_debugregs(CPUState *env)
1269 {
1270 #ifdef KVM_CAP_DEBUGREGS
1271     struct kvm_debugregs dbgregs;
1272     int i;
1273
1274     if (!kvm_has_debugregs()) {
1275         return 0;
1276     }
1277
1278     for (i = 0; i < 4; i++) {
1279         dbgregs.db[i] = env->dr[i];
1280     }
1281     dbgregs.dr6 = env->dr[6];
1282     dbgregs.dr7 = env->dr[7];
1283     dbgregs.flags = 0;
1284
1285     return kvm_vcpu_ioctl(env, KVM_SET_DEBUGREGS, &dbgregs);
1286 #else
1287     return 0;
1288 #endif
1289 }
1290
1291 static int kvm_get_debugregs(CPUState *env)
1292 {
1293 #ifdef KVM_CAP_DEBUGREGS
1294     struct kvm_debugregs dbgregs;
1295     int i, ret;
1296
1297     if (!kvm_has_debugregs()) {
1298         return 0;
1299     }
1300
1301     ret = kvm_vcpu_ioctl(env, KVM_GET_DEBUGREGS, &dbgregs);
1302     if (ret < 0) {
1303        return ret;
1304     }
1305     for (i = 0; i < 4; i++) {
1306         env->dr[i] = dbgregs.db[i];
1307     }
1308     env->dr[4] = env->dr[6] = dbgregs.dr6;
1309     env->dr[5] = env->dr[7] = dbgregs.dr7;
1310 #endif
1311
1312     return 0;
1313 }
1314
1315 int kvm_arch_put_registers(CPUState *env, int level)
1316 {
1317     int ret;
1318
1319     assert(cpu_is_stopped(env) || qemu_cpu_self(env));
1320
1321     ret = kvm_getput_regs(env, 1);
1322     if (ret < 0)
1323         return ret;
1324
1325     ret = kvm_put_xsave(env);
1326     if (ret < 0)
1327         return ret;
1328
1329     ret = kvm_put_xcrs(env);
1330     if (ret < 0)
1331         return ret;
1332
1333     ret = kvm_put_sregs(env);
1334     if (ret < 0)
1335         return ret;
1336
1337     ret = kvm_put_msrs(env, level);
1338     if (ret < 0)
1339         return ret;
1340
1341     if (level >= KVM_PUT_RESET_STATE) {
1342         ret = kvm_put_mp_state(env);
1343         if (ret < 0)
1344             return ret;
1345     }
1346
1347     ret = kvm_put_vcpu_events(env, level);
1348     if (ret < 0)
1349         return ret;
1350
1351     /* must be last */
1352     ret = kvm_guest_debug_workarounds(env);
1353     if (ret < 0)
1354         return ret;
1355
1356     ret = kvm_put_debugregs(env);
1357     if (ret < 0)
1358         return ret;
1359
1360     return 0;
1361 }
1362
1363 int kvm_arch_get_registers(CPUState *env)
1364 {
1365     int ret;
1366
1367     assert(cpu_is_stopped(env) || qemu_cpu_self(env));
1368
1369     ret = kvm_getput_regs(env, 0);
1370     if (ret < 0)
1371         return ret;
1372
1373     ret = kvm_get_xsave(env);
1374     if (ret < 0)
1375         return ret;
1376
1377     ret = kvm_get_xcrs(env);
1378     if (ret < 0)
1379         return ret;
1380
1381     ret = kvm_get_sregs(env);
1382     if (ret < 0)
1383         return ret;
1384
1385     ret = kvm_get_msrs(env);
1386     if (ret < 0)
1387         return ret;
1388
1389     ret = kvm_get_mp_state(env);
1390     if (ret < 0)
1391         return ret;
1392
1393     ret = kvm_get_vcpu_events(env);
1394     if (ret < 0)
1395         return ret;
1396
1397     ret = kvm_get_debugregs(env);
1398     if (ret < 0)
1399         return ret;
1400
1401     return 0;
1402 }
1403
1404 int kvm_arch_pre_run(CPUState *env, struct kvm_run *run)
1405 {
1406     /* Try to inject an interrupt if the guest can accept it */
1407     if (run->ready_for_interrupt_injection &&
1408         (env->interrupt_request & CPU_INTERRUPT_HARD) &&
1409         (env->eflags & IF_MASK)) {
1410         int irq;
1411
1412         env->interrupt_request &= ~CPU_INTERRUPT_HARD;
1413         irq = cpu_get_pic_interrupt(env);
1414         if (irq >= 0) {
1415             struct kvm_interrupt intr;
1416             intr.irq = irq;
1417             /* FIXME: errors */
1418             DPRINTF("injected interrupt %d\n", irq);
1419             kvm_vcpu_ioctl(env, KVM_INTERRUPT, &intr);
1420         }
1421     }
1422
1423     /* If we have an interrupt but the guest is not ready to receive an
1424      * interrupt, request an interrupt window exit.  This will
1425      * cause a return to userspace as soon as the guest is ready to
1426      * receive interrupts. */
1427     if ((env->interrupt_request & CPU_INTERRUPT_HARD))
1428         run->request_interrupt_window = 1;
1429     else
1430         run->request_interrupt_window = 0;
1431
1432     DPRINTF("setting tpr\n");
1433     run->cr8 = cpu_get_apic_tpr(env->apic_state);
1434
1435     return 0;
1436 }
1437
1438 int kvm_arch_post_run(CPUState *env, struct kvm_run *run)
1439 {
1440     if (run->if_flag)
1441         env->eflags |= IF_MASK;
1442     else
1443         env->eflags &= ~IF_MASK;
1444
1445     cpu_set_apic_tpr(env->apic_state, run->cr8);
1446     cpu_set_apic_base(env->apic_state, run->apic_base);
1447
1448     return 0;
1449 }
1450
1451 int kvm_arch_process_irqchip_events(CPUState *env)
1452 {
1453     if (env->interrupt_request & CPU_INTERRUPT_INIT) {
1454         kvm_cpu_synchronize_state(env);
1455         do_cpu_init(env);
1456         env->exception_index = EXCP_HALTED;
1457     }
1458
1459     if (env->interrupt_request & CPU_INTERRUPT_SIPI) {
1460         kvm_cpu_synchronize_state(env);
1461         do_cpu_sipi(env);
1462     }
1463
1464     return env->halted;
1465 }
1466
1467 static int kvm_handle_halt(CPUState *env)
1468 {
1469     if (!((env->interrupt_request & CPU_INTERRUPT_HARD) &&
1470           (env->eflags & IF_MASK)) &&
1471         !(env->interrupt_request & CPU_INTERRUPT_NMI)) {
1472         env->halted = 1;
1473         env->exception_index = EXCP_HLT;
1474         return 0;
1475     }
1476
1477     return 1;
1478 }
1479
1480 int kvm_arch_handle_exit(CPUState *env, struct kvm_run *run)
1481 {
1482     int ret = 0;
1483
1484     switch (run->exit_reason) {
1485     case KVM_EXIT_HLT:
1486         DPRINTF("handle_hlt\n");
1487         ret = kvm_handle_halt(env);
1488         break;
1489     }
1490
1491     return ret;
1492 }
1493
1494 #ifdef KVM_CAP_SET_GUEST_DEBUG
1495 int kvm_arch_insert_sw_breakpoint(CPUState *env, struct kvm_sw_breakpoint *bp)
1496 {
1497     static const uint8_t int3 = 0xcc;
1498
1499     if (cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&bp->saved_insn, 1, 0) ||
1500         cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&int3, 1, 1))
1501         return -EINVAL;
1502     return 0;
1503 }
1504
1505 int kvm_arch_remove_sw_breakpoint(CPUState *env, struct kvm_sw_breakpoint *bp)
1506 {
1507     uint8_t int3;
1508
1509     if (cpu_memory_rw_debug(env, bp->pc, &int3, 1, 0) || int3 != 0xcc ||
1510         cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&bp->saved_insn, 1, 1))
1511         return -EINVAL;
1512     return 0;
1513 }
1514
1515 static struct {
1516     target_ulong addr;
1517     int len;
1518     int type;
1519 } hw_breakpoint[4];
1520
1521 static int nb_hw_breakpoint;
1522
1523 static int find_hw_breakpoint(target_ulong addr, int len, int type)
1524 {
1525     int n;
1526
1527     for (n = 0; n < nb_hw_breakpoint; n++)
1528         if (hw_breakpoint[n].addr == addr && hw_breakpoint[n].type == type &&
1529             (hw_breakpoint[n].len == len || len == -1))
1530             return n;
1531     return -1;
1532 }
1533
1534 int kvm_arch_insert_hw_breakpoint(target_ulong addr,
1535                                   target_ulong len, int type)
1536 {
1537     switch (type) {
1538     case GDB_BREAKPOINT_HW:
1539         len = 1;
1540         break;
1541     case GDB_WATCHPOINT_WRITE:
1542     case GDB_WATCHPOINT_ACCESS:
1543         switch (len) {
1544         case 1:
1545             break;
1546         case 2:
1547         case 4:
1548         case 8:
1549             if (addr & (len - 1))
1550                 return -EINVAL;
1551             break;
1552         default:
1553             return -EINVAL;
1554         }
1555         break;
1556     default:
1557         return -ENOSYS;
1558     }
1559
1560     if (nb_hw_breakpoint == 4)
1561         return -ENOBUFS;
1562
1563     if (find_hw_breakpoint(addr, len, type) >= 0)
1564         return -EEXIST;
1565
1566     hw_breakpoint[nb_hw_breakpoint].addr = addr;
1567     hw_breakpoint[nb_hw_breakpoint].len = len;
1568     hw_breakpoint[nb_hw_breakpoint].type = type;
1569     nb_hw_breakpoint++;
1570
1571     return 0;
1572 }
1573
1574 int kvm_arch_remove_hw_breakpoint(target_ulong addr,
1575                                   target_ulong len, int type)
1576 {
1577     int n;
1578
1579     n = find_hw_breakpoint(addr, (type == GDB_BREAKPOINT_HW) ? 1 : len, type);
1580     if (n < 0)
1581         return -ENOENT;
1582
1583     nb_hw_breakpoint--;
1584     hw_breakpoint[n] = hw_breakpoint[nb_hw_breakpoint];
1585
1586     return 0;
1587 }
1588
1589 void kvm_arch_remove_all_hw_breakpoints(void)
1590 {
1591     nb_hw_breakpoint = 0;
1592 }
1593
1594 static CPUWatchpoint hw_watchpoint;
1595
1596 int kvm_arch_debug(struct kvm_debug_exit_arch *arch_info)
1597 {
1598     int handle = 0;
1599     int n;
1600
1601     if (arch_info->exception == 1) {
1602         if (arch_info->dr6 & (1 << 14)) {
1603             if (cpu_single_env->singlestep_enabled)
1604                 handle = 1;
1605         } else {
1606             for (n = 0; n < 4; n++)
1607                 if (arch_info->dr6 & (1 << n))
1608                     switch ((arch_info->dr7 >> (16 + n*4)) & 0x3) {
1609                     case 0x0:
1610                         handle = 1;
1611                         break;
1612                     case 0x1:
1613                         handle = 1;
1614                         cpu_single_env->watchpoint_hit = &hw_watchpoint;
1615                         hw_watchpoint.vaddr = hw_breakpoint[n].addr;
1616                         hw_watchpoint.flags = BP_MEM_WRITE;
1617                         break;
1618                     case 0x3:
1619                         handle = 1;
1620                         cpu_single_env->watchpoint_hit = &hw_watchpoint;
1621                         hw_watchpoint.vaddr = hw_breakpoint[n].addr;
1622                         hw_watchpoint.flags = BP_MEM_ACCESS;
1623                         break;
1624                     }
1625         }
1626     } else if (kvm_find_sw_breakpoint(cpu_single_env, arch_info->pc))
1627         handle = 1;
1628
1629     if (!handle) {
1630         cpu_synchronize_state(cpu_single_env);
1631         assert(cpu_single_env->exception_injected == -1);
1632
1633         cpu_single_env->exception_injected = arch_info->exception;
1634         cpu_single_env->has_error_code = 0;
1635     }
1636
1637     return handle;
1638 }
1639
1640 void kvm_arch_update_guest_debug(CPUState *env, struct kvm_guest_debug *dbg)
1641 {
1642     const uint8_t type_code[] = {
1643         [GDB_BREAKPOINT_HW] = 0x0,
1644         [GDB_WATCHPOINT_WRITE] = 0x1,
1645         [GDB_WATCHPOINT_ACCESS] = 0x3
1646     };
1647     const uint8_t len_code[] = {
1648         [1] = 0x0, [2] = 0x1, [4] = 0x3, [8] = 0x2
1649     };
1650     int n;
1651
1652     if (kvm_sw_breakpoints_active(env))
1653         dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP;
1654
1655     if (nb_hw_breakpoint > 0) {
1656         dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW_BP;
1657         dbg->arch.debugreg[7] = 0x0600;
1658         for (n = 0; n < nb_hw_breakpoint; n++) {
1659             dbg->arch.debugreg[n] = hw_breakpoint[n].addr;
1660             dbg->arch.debugreg[7] |= (2 << (n * 2)) |
1661                 (type_code[hw_breakpoint[n].type] << (16 + n*4)) |
1662                 (len_code[hw_breakpoint[n].len] << (18 + n*4));
1663         }
1664     }
1665     /* Legal xcr0 for loading */
1666     env->xcr0 = 1;
1667 }
1668 #endif /* KVM_CAP_SET_GUEST_DEBUG */
1669
1670 bool kvm_arch_stop_on_emulation_error(CPUState *env)
1671 {
1672       return !(env->cr[0] & CR0_PE_MASK) ||
1673               ((env->segs[R_CS].selector  & 3) != 3);
1674 }
1675
1676 static void hardware_memory_error(void)
1677 {
1678     fprintf(stderr, "Hardware memory error!\n");
1679     exit(1);
1680 }
1681
1682 #ifdef KVM_CAP_MCE
1683 static void kvm_mce_broadcast_rest(CPUState *env)
1684 {
1685     CPUState *cenv;
1686     int family, model, cpuver = env->cpuid_version;
1687
1688     family = (cpuver >> 8) & 0xf;
1689     model = ((cpuver >> 12) & 0xf0) + ((cpuver >> 4) & 0xf);
1690
1691     /* Broadcast MCA signal for processor version 06H_EH and above */
1692     if ((family == 6 && model >= 14) || family > 6) {
1693         for (cenv = first_cpu; cenv != NULL; cenv = cenv->next_cpu) {
1694             if (cenv == env) {
1695                 continue;
1696             }
1697             kvm_inject_x86_mce(cenv, 1, MCI_STATUS_VAL | MCI_STATUS_UC,
1698                                MCG_STATUS_MCIP | MCG_STATUS_RIPV, 0, 0, 1);
1699         }
1700     }
1701 }
1702 #endif
1703
1704 int kvm_on_sigbus_vcpu(CPUState *env, int code, void *addr)
1705 {
1706 #if defined(KVM_CAP_MCE)
1707     struct kvm_x86_mce mce = {
1708             .bank = 9,
1709     };
1710     void *vaddr;
1711     ram_addr_t ram_addr;
1712     target_phys_addr_t paddr;
1713     int r;
1714
1715     if ((env->mcg_cap & MCG_SER_P) && addr
1716         && (code == BUS_MCEERR_AR
1717             || code == BUS_MCEERR_AO)) {
1718         if (code == BUS_MCEERR_AR) {
1719             /* Fake an Intel architectural Data Load SRAR UCR */
1720             mce.status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN
1721                 | MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S
1722                 | MCI_STATUS_AR | 0x134;
1723             mce.misc = (MCM_ADDR_PHYS << 6) | 0xc;
1724             mce.mcg_status = MCG_STATUS_MCIP | MCG_STATUS_EIPV;
1725         } else {
1726             /*
1727              * If there is an MCE excpetion being processed, ignore
1728              * this SRAO MCE
1729              */
1730             r = kvm_mce_in_exception(env);
1731             if (r == -1) {
1732                 fprintf(stderr, "Failed to get MCE status\n");
1733             } else if (r) {
1734                 return 0;
1735             }
1736             /* Fake an Intel architectural Memory scrubbing UCR */
1737             mce.status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN
1738                 | MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S
1739                 | 0xc0;
1740             mce.misc = (MCM_ADDR_PHYS << 6) | 0xc;
1741             mce.mcg_status = MCG_STATUS_MCIP | MCG_STATUS_RIPV;
1742         }
1743         vaddr = (void *)addr;
1744         if (qemu_ram_addr_from_host(vaddr, &ram_addr) ||
1745             !kvm_physical_memory_addr_from_ram(env->kvm_state, ram_addr, &paddr)) {
1746             fprintf(stderr, "Hardware memory error for memory used by "
1747                     "QEMU itself instead of guest system!\n");
1748             /* Hope we are lucky for AO MCE */
1749             if (code == BUS_MCEERR_AO) {
1750                 return 0;
1751             } else {
1752                 hardware_memory_error();
1753             }
1754         }
1755         mce.addr = paddr;
1756         r = kvm_set_mce(env, &mce);
1757         if (r < 0) {
1758             fprintf(stderr, "kvm_set_mce: %s\n", strerror(errno));
1759             abort();
1760         }
1761         kvm_mce_broadcast_rest(env);
1762     } else
1763 #endif
1764     {
1765         if (code == BUS_MCEERR_AO) {
1766             return 0;
1767         } else if (code == BUS_MCEERR_AR) {
1768             hardware_memory_error();
1769         } else {
1770             return 1;
1771         }
1772     }
1773     return 0;
1774 }
1775
1776 int kvm_on_sigbus(int code, void *addr)
1777 {
1778 #if defined(KVM_CAP_MCE)
1779     if ((first_cpu->mcg_cap & MCG_SER_P) && addr && code == BUS_MCEERR_AO) {
1780         uint64_t status;
1781         void *vaddr;
1782         ram_addr_t ram_addr;
1783         target_phys_addr_t paddr;
1784
1785         /* Hope we are lucky for AO MCE */
1786         vaddr = addr;
1787         if (qemu_ram_addr_from_host(vaddr, &ram_addr) ||
1788             !kvm_physical_memory_addr_from_ram(first_cpu->kvm_state, ram_addr, &paddr)) {
1789             fprintf(stderr, "Hardware memory error for memory used by "
1790                     "QEMU itself instead of guest system!: %p\n", addr);
1791             return 0;
1792         }
1793         status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN
1794             | MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S
1795             | 0xc0;
1796         kvm_inject_x86_mce(first_cpu, 9, status,
1797                            MCG_STATUS_MCIP | MCG_STATUS_RIPV, paddr,
1798                            (MCM_ADDR_PHYS << 6) | 0xc, 1);
1799         kvm_mce_broadcast_rest(first_cpu);
1800     } else
1801 #endif
1802     {
1803         if (code == BUS_MCEERR_AO) {
1804             return 0;
1805         } else if (code == BUS_MCEERR_AR) {
1806             hardware_memory_error();
1807         } else {
1808             return 1;
1809         }
1810     }
1811     return 0;
1812 }