target-i386/kvm.c

   1 /*
   2  * QEMU KVM support
   3  *
   4  * Copyright (C) 2006-2008 Qumranet Technologies
   5  * Copyright IBM, Corp. 2008
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *
  10  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  11  * See the COPYING file in the top-level directory.
  12  *
  13  */
  14
  15 #include <sys/types.h>
  16 #include <sys/ioctl.h>
  17 #include <sys/mman.h>
  18
  19 #include <linux/kvm.h>
  20
  21 #include "qemu-common.h"
  22 #include "sysemu.h"
  23 #include "kvm.h"
  24 #include "cpu.h"
  25 #include "gdbstub.h"
  26 #include "host-utils.h"
  27 #include "hw/pc.h"
  28 #include "hw/apic.h"
  29 #include "ioport.h"
  30 #include "kvm_x86.h"
  31
  32 #ifdef CONFIG_KVM_PARA
  33 #include <linux/kvm_para.h>
  34 #endif
  35 //
  36 //#define DEBUG_KVM
  37
  38 #ifdef DEBUG_KVM
  39 #define DPRINTF(fmt, ...) \
  40     do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
  41 #else
  42 #define DPRINTF(fmt, ...) \
  43     do { } while (0)
  44 #endif
  45
  46 #define MSR_KVM_WALL_CLOCK  0x11
  47 #define MSR_KVM_SYSTEM_TIME 0x12
  48
  49 #ifndef BUS_MCEERR_AR
  50 #define BUS_MCEERR_AR 4
  51 #endif
  52 #ifndef BUS_MCEERR_AO
  53 #define BUS_MCEERR_AO 5
  54 #endif
  55
  56 #ifdef KVM_CAP_EXT_CPUID
  57
  58 static struct kvm_cpuid2 *try_get_cpuid(KVMState *s, int max)
  59 {
  60     struct kvm_cpuid2 *cpuid;
  61     int r, size;
  62
  63     size = sizeof(*cpuid) + max * sizeof(*cpuid->entries);
  64     cpuid = (struct kvm_cpuid2 *)qemu_mallocz(size);
  65     cpuid->nent = max;
  66     r = kvm_ioctl(s, KVM_GET_SUPPORTED_CPUID, cpuid);
  67     if (r == 0 && cpuid->nent >= max) {
  68         r = -E2BIG;
  69     }
  70     if (r < 0) {
  71         if (r == -E2BIG) {
  72             qemu_free(cpuid);
  73             return NULL;
  74         } else {
  75             fprintf(stderr, "KVM_GET_SUPPORTED_CPUID failed: %s\n",
  76                     strerror(-r));
  77             exit(1);
  78         }
  79     }
  80     return cpuid;
  81 }
  82
  83 uint32_t kvm_arch_get_supported_cpuid(CPUState *env, uint32_t function,
  84                                       uint32_t index, int reg)
  85 {
  86     struct kvm_cpuid2 *cpuid;
  87     int i, max;
  88     uint32_t ret = 0;
  89     uint32_t cpuid_1_edx;
  90
  91     if (!kvm_check_extension(env->kvm_state, KVM_CAP_EXT_CPUID)) {
  92         return -1U;
  93     }
  94
  95     max = 1;
  96     while ((cpuid = try_get_cpuid(env->kvm_state, max)) == NULL) {
  97         max *= 2;
  98     }
  99
 100     for (i = 0; i < cpuid->nent; ++i) {
 101         if (cpuid->entries[i].function == function &&
 102             cpuid->entries[i].index == index) {
 103             switch (reg) {
 104             case R_EAX:
 105                 ret = cpuid->entries[i].eax;
 106                 break;
 107             case R_EBX:
 108                 ret = cpuid->entries[i].ebx;
 109                 break;
 110             case R_ECX:
 111                 ret = cpuid->entries[i].ecx;
 112                 break;
 113             case R_EDX:
 114                 ret = cpuid->entries[i].edx;
 115                 switch (function) {
 116                 case 1:
 117                     /* KVM before 2.6.30 misreports the following features */
 118                     ret |= CPUID_MTRR | CPUID_PAT | CPUID_MCE | CPUID_MCA;
 119                     break;
 120                 case 0x80000001:
 121                     /* On Intel, kvm returns cpuid according to the Intel spec,
 122                      * so add missing bits according to the AMD spec:
 123                      */
 124                     cpuid_1_edx = kvm_arch_get_supported_cpuid(env, 1, 0, R_EDX);
 125                     ret |= cpuid_1_edx & 0x183f7ff;
 126                     break;
 127                 }
 128                 break;
 129             }
 130         }
 131     }
 132
 133     qemu_free(cpuid);
 134
 135     return ret;
 136 }
 137
 138 #else
 139
 140 uint32_t kvm_arch_get_supported_cpuid(CPUState *env, uint32_t function,
 141                                       uint32_t index, int reg)
 142 {
 143     return -1U;
 144 }
 145
 146 #endif
 147
 148 #ifdef CONFIG_KVM_PARA
 149 struct kvm_para_features {
 150         int cap;
 151         int feature;
 152 } para_features[] = {
 153 #ifdef KVM_CAP_CLOCKSOURCE
 154         { KVM_CAP_CLOCKSOURCE, KVM_FEATURE_CLOCKSOURCE },
 155 #endif
 156 #ifdef KVM_CAP_NOP_IO_DELAY
 157         { KVM_CAP_NOP_IO_DELAY, KVM_FEATURE_NOP_IO_DELAY },
 158 #endif
 159 #ifdef KVM_CAP_PV_MMU
 160         { KVM_CAP_PV_MMU, KVM_FEATURE_MMU_OP },
 161 #endif
 162         { -1, -1 }
 163 };
 164
 165 static int get_para_features(CPUState *env)
 166 {
 167         int i, features = 0;
 168
 169         for (i = 0; i < ARRAY_SIZE(para_features) - 1; i++) {
 170                 if (kvm_check_extension(env->kvm_state, para_features[i].cap))
 171                         features |= (1 << para_features[i].feature);
 172         }
 173
 174         return features;
 175 }
 176 #endif
 177
 178 #ifdef KVM_CAP_MCE
 179 static int kvm_get_mce_cap_supported(KVMState *s, uint64_t *mce_cap,
 180                                      int *max_banks)
 181 {
 182     int r;
 183
 184     r = kvm_ioctl(s, KVM_CHECK_EXTENSION, KVM_CAP_MCE);
 185     if (r > 0) {
 186         *max_banks = r;
 187         return kvm_ioctl(s, KVM_X86_GET_MCE_CAP_SUPPORTED, mce_cap);
 188     }
 189     return -ENOSYS;
 190 }
 191
 192 static int kvm_setup_mce(CPUState *env, uint64_t *mcg_cap)
 193 {
 194     return kvm_vcpu_ioctl(env, KVM_X86_SETUP_MCE, mcg_cap);
 195 }
 196
 197 static int kvm_set_mce(CPUState *env, struct kvm_x86_mce *m)
 198 {
 199     return kvm_vcpu_ioctl(env, KVM_X86_SET_MCE, m);
 200 }
 201
 202 static int kvm_get_msr(CPUState *env, struct kvm_msr_entry *msrs, int n)
 203 {
 204     struct kvm_msrs *kmsrs = qemu_malloc(sizeof *kmsrs + n * sizeof *msrs);
 205     int r;
 206
 207     kmsrs->nmsrs = n;
 208     memcpy(kmsrs->entries, msrs, n * sizeof *msrs);
 209     r = kvm_vcpu_ioctl(env, KVM_GET_MSRS, kmsrs);
 210     memcpy(msrs, kmsrs->entries, n * sizeof *msrs);
 211     free(kmsrs);
 212     return r;
 213 }
 214
 215 /* FIXME: kill this and kvm_get_msr, use env->mcg_status instead */
 216 static int kvm_mce_in_exception(CPUState *env)
 217 {
 218     struct kvm_msr_entry msr_mcg_status = {
 219         .index = MSR_MCG_STATUS,
 220     };
 221     int r;
 222
 223     r = kvm_get_msr(env, &msr_mcg_status, 1);
 224     if (r == -1 || r == 0) {
 225         return -1;
 226     }
 227     return !!(msr_mcg_status.data & MCG_STATUS_MCIP);
 228 }
 229
 230 struct kvm_x86_mce_data
 231 {
 232     CPUState *env;
 233     struct kvm_x86_mce *mce;
 234     int abort_on_error;
 235 };
 236
 237 static void kvm_do_inject_x86_mce(void *_data)
 238 {
 239     struct kvm_x86_mce_data *data = _data;
 240     int r;
 241
 242     /* If there is an MCE excpetion being processed, ignore this SRAO MCE */
 243     r = kvm_mce_in_exception(data->env);
 244     if (r == -1)
 245         fprintf(stderr, "Failed to get MCE status\n");
 246     else if (r && !(data->mce->status & MCI_STATUS_AR))
 247         return;
 248
 249     r = kvm_set_mce(data->env, data->mce);
 250     if (r < 0) {
 251         perror("kvm_set_mce FAILED");
 252         if (data->abort_on_error) {
 253             abort();
 254         }
 255     }
 256 }
 257 #endif
 258
 259 void kvm_inject_x86_mce(CPUState *cenv, int bank, uint64_t status,
 260                         uint64_t mcg_status, uint64_t addr, uint64_t misc,
 261                         int abort_on_error)
 262 {
 263 #ifdef KVM_CAP_MCE
 264     struct kvm_x86_mce mce = {
 265         .bank = bank,
 266         .status = status,
 267         .mcg_status = mcg_status,
 268         .addr = addr,
 269         .misc = misc,
 270     };
 271     struct kvm_x86_mce_data data = {
 272             .env = cenv,
 273             .mce = &mce,
 274     };
 275
 276     if (!cenv->mcg_cap) {
 277         fprintf(stderr, "MCE support is not enabled!\n");
 278         return;
 279     }
 280
 281     on_vcpu(cenv, kvm_do_inject_x86_mce, &data);
 282 #else
 283     if (abort_on_error)
 284         abort();
 285 #endif
 286 }
 287
 288 static int _kvm_arch_init_vcpu(CPUState *env);
 289
 290 int kvm_arch_init_vcpu(CPUState *env)
 291 {
 292     int r;
 293     struct {
 294         struct kvm_cpuid2 cpuid;
 295         struct kvm_cpuid_entry2 entries[100];
 296     } __attribute__((packed)) cpuid_data;
 297     uint32_t limit, i, j, cpuid_i;
 298     uint32_t unused;
 299     struct kvm_cpuid_entry2 *c;
 300 #ifdef KVM_CPUID_SIGNATURE
 301     uint32_t signature[3];
 302 #endif
 303
 304     r = _kvm_arch_init_vcpu(env);
 305     if (r < 0) {
 306         return r;
 307     }
 308
 309 #ifdef OBSOLETE_KVM_IMPL
 310
 311     env->mp_state = KVM_MP_STATE_RUNNABLE;
 312
 313 #endif
 314
 315     env->cpuid_features &= kvm_arch_get_supported_cpuid(env, 1, 0, R_EDX);
 316
 317     i = env->cpuid_ext_features & CPUID_EXT_HYPERVISOR;
 318     env->cpuid_ext_features &= kvm_arch_get_supported_cpuid(env, 1, 0, R_ECX);
 319     env->cpuid_ext_features |= i;
 320
 321     env->cpuid_ext2_features &= kvm_arch_get_supported_cpuid(env, 0x80000001,
 322                                                              0, R_EDX);
 323     env->cpuid_ext3_features &= kvm_arch_get_supported_cpuid(env, 0x80000001,
 324                                                              0, R_ECX);
 325     env->cpuid_svm_features  &= kvm_arch_get_supported_cpuid(env, 0x8000000A,
 326                                                              0, R_EDX);
 327
 328
 329     cpuid_i = 0;
 330
 331 #ifdef CONFIG_KVM_PARA
 332     /* Paravirtualization CPUIDs */
 333     memcpy(signature, "KVMKVMKVM\0\0\0", 12);
 334     c = &cpuid_data.entries[cpuid_i++];
 335     memset(c, 0, sizeof(*c));
 336     c->function = KVM_CPUID_SIGNATURE;
 337     c->eax = 0;
 338     c->ebx = signature[0];
 339     c->ecx = signature[1];
 340     c->edx = signature[2];
 341
 342     c = &cpuid_data.entries[cpuid_i++];
 343     memset(c, 0, sizeof(*c));
 344     c->function = KVM_CPUID_FEATURES;
 345     c->eax = env->cpuid_kvm_features & get_para_features(env);
 346 #endif
 347
 348     cpu_x86_cpuid(env, 0, 0, &limit, &unused, &unused, &unused);
 349
 350     for (i = 0; i <= limit; i++) {
 351         c = &cpuid_data.entries[cpuid_i++];
 352
 353         switch (i) {
 354         case 2: {
 355             /* Keep reading function 2 till all the input is received */
 356             int times;
 357
 358             c->function = i;
 359             c->flags = KVM_CPUID_FLAG_STATEFUL_FUNC |
 360                        KVM_CPUID_FLAG_STATE_READ_NEXT;
 361             cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 362             times = c->eax & 0xff;
 363
 364             for (j = 1; j < times; ++j) {
 365                 c = &cpuid_data.entries[cpuid_i++];
 366                 c->function = i;
 367                 c->flags = KVM_CPUID_FLAG_STATEFUL_FUNC;
 368                 cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 369             }
 370             break;
 371         }
 372         case 4:
 373         case 0xb:
 374         case 0xd:
 375             for (j = 0; ; j++) {
 376                 c->function = i;
 377                 c->flags = KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
 378                 c->index = j;
 379                 cpu_x86_cpuid(env, i, j, &c->eax, &c->ebx, &c->ecx, &c->edx);
 380
 381                 if (i == 4 && c->eax == 0)
 382                     break;
 383                 if (i == 0xb && !(c->ecx & 0xff00))
 384                     break;
 385                 if (i == 0xd && c->eax == 0)
 386                     break;
 387
 388                 c = &cpuid_data.entries[cpuid_i++];
 389             }
 390             break;
 391         default:
 392             c->function = i;
 393             c->flags = 0;
 394             cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 395             break;
 396         }
 397     }
 398     cpu_x86_cpuid(env, 0x80000000, 0, &limit, &unused, &unused, &unused);
 399
 400     for (i = 0x80000000; i <= limit; i++) {
 401         c = &cpuid_data.entries[cpuid_i++];
 402
 403         c->function = i;
 404         c->flags = 0;
 405         cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 406     }
 407
 408     cpuid_data.cpuid.nent = cpuid_i;
 409
 410 #ifdef KVM_CAP_MCE
 411     if (((env->cpuid_version >> 8)&0xF) >= 6
 412         && (env->cpuid_features&(CPUID_MCE|CPUID_MCA)) == (CPUID_MCE|CPUID_MCA)
 413         && kvm_check_extension(env->kvm_state, KVM_CAP_MCE) > 0) {
 414         uint64_t mcg_cap;
 415         int banks;
 416
 417         if (kvm_get_mce_cap_supported(env->kvm_state, &mcg_cap, &banks))
 418             perror("kvm_get_mce_cap_supported FAILED");
 419         else {
 420             if (banks > MCE_BANKS_DEF)
 421                 banks = MCE_BANKS_DEF;
 422             mcg_cap &= MCE_CAP_DEF;
 423             mcg_cap |= banks;
 424             if (kvm_setup_mce(env, &mcg_cap))
 425                 perror("kvm_setup_mce FAILED");
 426             else
 427                 env->mcg_cap = mcg_cap;
 428         }
 429     }
 430 #endif
 431
 432     return kvm_vcpu_ioctl(env, KVM_SET_CPUID2, &cpuid_data);
 433 }
 434
 435 void kvm_arch_reset_vcpu(CPUState *env)
 436 {
 437     env->exception_injected = -1;
 438     env->interrupt_injected = -1;
 439     env->nmi_injected = 0;
 440     env->nmi_pending = 0;
 441     /* Legal xcr0 for loading */
 442     env->xcr0 = 1;
 443     if (kvm_irqchip_in_kernel()) {
 444         env->mp_state = cpu_is_bsp(env) ? KVM_MP_STATE_RUNNABLE :
 445                                           KVM_MP_STATE_UNINITIALIZED;
 446     } else {
 447         env->mp_state = KVM_MP_STATE_RUNNABLE;
 448     }
 449 }
 450 #ifdef OBSOLETE_KVM_IMPL
 451
 452 static int kvm_has_msr_star(CPUState *env)
 453 {
 454     static int has_msr_star;
 455     int ret;
 456
 457     /* first time */
 458     if (has_msr_star == 0) {
 459         struct kvm_msr_list msr_list, *kvm_msr_list;
 460
 461         has_msr_star = -1;
 462
 463         /* Obtain MSR list from KVM.  These are the MSRs that we must
 464          * save/restore */
 465         msr_list.nmsrs = 0;
 466         ret = kvm_ioctl(env->kvm_state, KVM_GET_MSR_INDEX_LIST, &msr_list);
 467         if (ret < 0 && ret != -E2BIG) {
 468             return 0;
 469         }
 470         /* Old kernel modules had a bug and could write beyond the provided
 471            memory. Allocate at least a safe amount of 1K. */
 472         kvm_msr_list = qemu_mallocz(MAX(1024, sizeof(msr_list) +
 473                                               msr_list.nmsrs *
 474                                               sizeof(msr_list.indices[0])));
 475
 476         kvm_msr_list->nmsrs = msr_list.nmsrs;
 477         ret = kvm_ioctl(env->kvm_state, KVM_GET_MSR_INDEX_LIST, kvm_msr_list);
 478         if (ret >= 0) {
 479             int i;
 480
 481             for (i = 0; i < kvm_msr_list->nmsrs; i++) {
 482                 if (kvm_msr_list->indices[i] == MSR_STAR) {
 483                     has_msr_star = 1;
 484                     break;
 485                 }
 486             }
 487         }
 488
 489         free(kvm_msr_list);
 490     }
 491
 492     if (has_msr_star == 1)
 493         return 1;
 494     return 0;
 495 }
 496
 497 static int kvm_init_identity_map_page(KVMState *s)
 498 {
 499 #ifdef KVM_CAP_SET_IDENTITY_MAP_ADDR
 500     int ret;
 501     uint64_t addr = 0xfffbc000;
 502
 503     if (!kvm_check_extension(s, KVM_CAP_SET_IDENTITY_MAP_ADDR)) {
 504         return 0;
 505     }
 506
 507     ret = kvm_vm_ioctl(s, KVM_SET_IDENTITY_MAP_ADDR, &addr);
 508     if (ret < 0) {
 509         fprintf(stderr, "kvm_set_identity_map_addr: %s\n", strerror(ret));
 510         return ret;
 511     }
 512 #endif
 513     return 0;
 514 }
 515
 516 int kvm_arch_init(KVMState *s, int smp_cpus)
 517 {
 518     int ret;
 519
 520     /* create vm86 tss.  KVM uses vm86 mode to emulate 16-bit code
 521      * directly.  In order to use vm86 mode, a TSS is needed.  Since this
 522      * must be part of guest physical memory, we need to allocate it.  Older
 523      * versions of KVM just assumed that it would be at the end of physical
 524      * memory but that doesn't work with more than 4GB of memory.  We simply
 525      * refuse to work with those older versions of KVM. */
 526     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, KVM_CAP_SET_TSS_ADDR);
 527     if (ret <= 0) {
 528         fprintf(stderr, "kvm does not support KVM_CAP_SET_TSS_ADDR\n");
 529         return ret;
 530     }
 531
 532     /* this address is 3 pages before the bios, and the bios should present
 533      * as unavaible memory.  FIXME, need to ensure the e820 map deals with
 534      * this?
 535      */
 536     /*
 537      * Tell fw_cfg to notify the BIOS to reserve the range.
 538      */
 539     if (e820_add_entry(0xfffbc000, 0x4000, E820_RESERVED) < 0) {
 540         perror("e820_add_entry() table is full");
 541         exit(1);
 542     }
 543     ret = kvm_vm_ioctl(s, KVM_SET_TSS_ADDR, 0xfffbd000);
 544     if (ret < 0) {
 545         return ret;
 546     }
 547
 548     return kvm_init_identity_map_page(s);
 549 }
 550
 551 static void set_v8086_seg(struct kvm_segment *lhs, const SegmentCache *rhs)
 552 {
 553     lhs->selector = rhs->selector;
 554     lhs->base = rhs->base;
 555     lhs->limit = rhs->limit;
 556     lhs->type = 3;
 557     lhs->present = 1;
 558     lhs->dpl = 3;
 559     lhs->db = 0;
 560     lhs->s = 1;
 561     lhs->l = 0;
 562     lhs->g = 0;
 563     lhs->avl = 0;
 564     lhs->unusable = 0;
 565 }
 566
 567 static void set_seg(struct kvm_segment *lhs, const SegmentCache *rhs)
 568 {
 569     unsigned flags = rhs->flags;
 570     lhs->selector = rhs->selector;
 571     lhs->base = rhs->base;
 572     lhs->limit = rhs->limit;
 573     lhs->type = (flags >> DESC_TYPE_SHIFT) & 15;
 574     lhs->present = (flags & DESC_P_MASK) != 0;
 575     lhs->dpl = rhs->selector & 3;
 576     lhs->db = (flags >> DESC_B_SHIFT) & 1;
 577     lhs->s = (flags & DESC_S_MASK) != 0;
 578     lhs->l = (flags >> DESC_L_SHIFT) & 1;
 579     lhs->g = (flags & DESC_G_MASK) != 0;
 580     lhs->avl = (flags & DESC_AVL_MASK) != 0;
 581     lhs->unusable = 0;
 582 }
 583
 584 static void get_seg(SegmentCache *lhs, const struct kvm_segment *rhs)
 585 {
 586     lhs->selector = rhs->selector;
 587     lhs->base = rhs->base;
 588     lhs->limit = rhs->limit;
 589     lhs->flags =
 590         (rhs->type << DESC_TYPE_SHIFT)
 591         | (rhs->present * DESC_P_MASK)
 592         | (rhs->dpl << DESC_DPL_SHIFT)
 593         | (rhs->db << DESC_B_SHIFT)
 594         | (rhs->s * DESC_S_MASK)
 595         | (rhs->l << DESC_L_SHIFT)
 596         | (rhs->g * DESC_G_MASK)
 597         | (rhs->avl * DESC_AVL_MASK);
 598 }
 599
 600 static void kvm_getput_reg(__u64 *kvm_reg, target_ulong *qemu_reg, int set)
 601 {
 602     if (set)
 603         *kvm_reg = *qemu_reg;
 604     else
 605         *qemu_reg = *kvm_reg;
 606 }
 607
 608 static int kvm_getput_regs(CPUState *env, int set)
 609 {
 610     struct kvm_regs regs;
 611     int ret = 0;
 612
 613     if (!set) {
 614         ret = kvm_vcpu_ioctl(env, KVM_GET_REGS, &regs);
 615         if (ret < 0)
 616             return ret;
 617     }
 618
 619     kvm_getput_reg(&regs.rax, &env->regs[R_EAX], set);
 620     kvm_getput_reg(&regs.rbx, &env->regs[R_EBX], set);
 621     kvm_getput_reg(&regs.rcx, &env->regs[R_ECX], set);
 622     kvm_getput_reg(&regs.rdx, &env->regs[R_EDX], set);
 623     kvm_getput_reg(&regs.rsi, &env->regs[R_ESI], set);
 624     kvm_getput_reg(&regs.rdi, &env->regs[R_EDI], set);
 625     kvm_getput_reg(&regs.rsp, &env->regs[R_ESP], set);
 626     kvm_getput_reg(&regs.rbp, &env->regs[R_EBP], set);
 627 #ifdef TARGET_X86_64
 628     kvm_getput_reg(&regs.r8, &env->regs[8], set);
 629     kvm_getput_reg(&regs.r9, &env->regs[9], set);
 630     kvm_getput_reg(&regs.r10, &env->regs[10], set);
 631     kvm_getput_reg(&regs.r11, &env->regs[11], set);
 632     kvm_getput_reg(&regs.r12, &env->regs[12], set);
 633     kvm_getput_reg(&regs.r13, &env->regs[13], set);
 634     kvm_getput_reg(&regs.r14, &env->regs[14], set);
 635     kvm_getput_reg(&regs.r15, &env->regs[15], set);
 636 #endif
 637
 638     kvm_getput_reg(&regs.rflags, &env->eflags, set);
 639     kvm_getput_reg(&regs.rip, &env->eip, set);
 640
 641     if (set)
 642         ret = kvm_vcpu_ioctl(env, KVM_SET_REGS, &regs);
 643
 644     return ret;
 645 }
 646
 647 static int kvm_put_fpu(CPUState *env)
 648 {
 649     struct kvm_fpu fpu;
 650     int i;
 651
 652     memset(&fpu, 0, sizeof fpu);
 653     fpu.fsw = env->fpus & ~(7 << 11);
 654     fpu.fsw |= (env->fpstt & 7) << 11;
 655     fpu.fcw = env->fpuc;
 656     for (i = 0; i < 8; ++i)
 657         fpu.ftwx |= (!env->fptags[i]) << i;
 658     memcpy(fpu.fpr, env->fpregs, sizeof env->fpregs);
 659     memcpy(fpu.xmm, env->xmm_regs, sizeof env->xmm_regs);
 660     fpu.mxcsr = env->mxcsr;
 661
 662     return kvm_vcpu_ioctl(env, KVM_SET_FPU, &fpu);
 663 }
 664
 665 #ifdef KVM_CAP_XSAVE
 666 #define XSAVE_CWD_RIP     2
 667 #define XSAVE_CWD_RDP     4
 668 #define XSAVE_MXCSR       6
 669 #define XSAVE_ST_SPACE    8
 670 #define XSAVE_XMM_SPACE   40
 671 #define XSAVE_XSTATE_BV   128
 672 #define XSAVE_YMMH_SPACE  144
 673 #endif
 674
 675 static int kvm_put_xsave(CPUState *env)
 676 {
 677 #ifdef KVM_CAP_XSAVE
 678     int i;
 679     struct kvm_xsave* xsave;
 680     uint16_t cwd, swd, twd, fop;
 681
 682     if (!kvm_has_xsave())
 683         return kvm_put_fpu(env);
 684
 685     xsave = qemu_memalign(4096, sizeof(struct kvm_xsave));
 686     memset(xsave, 0, sizeof(struct kvm_xsave));
 687     cwd = swd = twd = fop = 0;
 688     swd = env->fpus & ~(7 << 11);
 689     swd |= (env->fpstt & 7) << 11;
 690     cwd = env->fpuc;
 691     for (i = 0; i < 8; ++i)
 692         twd |= (!env->fptags[i]) << i;
 693     xsave->region[0] = (uint32_t)(swd << 16) + cwd;
 694     xsave->region[1] = (uint32_t)(fop << 16) + twd;
 695     memcpy(&xsave->region[XSAVE_ST_SPACE], env->fpregs,
 696             sizeof env->fpregs);
 697     memcpy(&xsave->region[XSAVE_XMM_SPACE], env->xmm_regs,
 698             sizeof env->xmm_regs);
 699     xsave->region[XSAVE_MXCSR] = env->mxcsr;
 700     *(uint64_t *)&xsave->region[XSAVE_XSTATE_BV] = env->xstate_bv;
 701     memcpy(&xsave->region[XSAVE_YMMH_SPACE], env->ymmh_regs,
 702             sizeof env->ymmh_regs);
 703     return kvm_vcpu_ioctl(env, KVM_SET_XSAVE, xsave);
 704 #else
 705     return kvm_put_fpu(env);
 706 #endif
 707 }
 708
 709 static int kvm_put_xcrs(CPUState *env)
 710 {
 711 #ifdef KVM_CAP_XCRS
 712     struct kvm_xcrs xcrs;
 713
 714     if (!kvm_has_xcrs())
 715         return 0;
 716
 717     xcrs.nr_xcrs = 1;
 718     xcrs.flags = 0;
 719     xcrs.xcrs[0].xcr = 0;
 720     xcrs.xcrs[0].value = env->xcr0;
 721     return kvm_vcpu_ioctl(env, KVM_SET_XCRS, &xcrs);
 722 #else
 723     return 0;
 724 #endif
 725 }
 726
 727 static int kvm_put_sregs(CPUState *env)
 728 {
 729     struct kvm_sregs sregs;
 730
 731     memset(sregs.interrupt_bitmap, 0, sizeof(sregs.interrupt_bitmap));
 732     if (env->interrupt_injected >= 0) {
 733         sregs.interrupt_bitmap[env->interrupt_injected / 64] |=
 734                 (uint64_t)1 << (env->interrupt_injected % 64);
 735     }
 736
 737     if ((env->eflags & VM_MASK)) {
 738             set_v8086_seg(&sregs.cs, &env->segs[R_CS]);
 739             set_v8086_seg(&sregs.ds, &env->segs[R_DS]);
 740             set_v8086_seg(&sregs.es, &env->segs[R_ES]);
 741             set_v8086_seg(&sregs.fs, &env->segs[R_FS]);
 742             set_v8086_seg(&sregs.gs, &env->segs[R_GS]);
 743             set_v8086_seg(&sregs.ss, &env->segs[R_SS]);
 744     } else {
 745             set_seg(&sregs.cs, &env->segs[R_CS]);
 746             set_seg(&sregs.ds, &env->segs[R_DS]);
 747             set_seg(&sregs.es, &env->segs[R_ES]);
 748             set_seg(&sregs.fs, &env->segs[R_FS]);
 749             set_seg(&sregs.gs, &env->segs[R_GS]);
 750             set_seg(&sregs.ss, &env->segs[R_SS]);
 751
 752             if (env->cr[0] & CR0_PE_MASK) {
 753                 /* force ss cpl to cs cpl */
 754                 sregs.ss.selector = (sregs.ss.selector & ~3) |
 755                         (sregs.cs.selector & 3);
 756                 sregs.ss.dpl = sregs.ss.selector & 3;
 757             }
 758     }
 759
 760     set_seg(&sregs.tr, &env->tr);
 761     set_seg(&sregs.ldt, &env->ldt);
 762
 763     sregs.idt.limit = env->idt.limit;
 764     sregs.idt.base = env->idt.base;
 765     sregs.gdt.limit = env->gdt.limit;
 766     sregs.gdt.base = env->gdt.base;
 767
 768     sregs.cr0 = env->cr[0];
 769     sregs.cr2 = env->cr[2];
 770     sregs.cr3 = env->cr[3];
 771     sregs.cr4 = env->cr[4];
 772
 773     sregs.cr8 = cpu_get_apic_tpr(env->apic_state);
 774     sregs.apic_base = cpu_get_apic_base(env->apic_state);
 775
 776     sregs.efer = env->efer;
 777
 778     return kvm_vcpu_ioctl(env, KVM_SET_SREGS, &sregs);
 779 }
 780
 781 #endif
 782
 783 static void kvm_msr_entry_set(struct kvm_msr_entry *entry,
 784                               uint32_t index, uint64_t value)
 785 {
 786     entry->index = index;
 787     entry->data = value;
 788 }
 789
 790 #ifdef OBSOLETE_KVM_IMPL
 791 static int kvm_put_msrs(CPUState *env, int level)
 792 {
 793     struct {
 794         struct kvm_msrs info;
 795         struct kvm_msr_entry entries[100];
 796     } msr_data;
 797     struct kvm_msr_entry *msrs = msr_data.entries;
 798     int n = 0;
 799
 800     kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_CS, env->sysenter_cs);
 801     kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_ESP, env->sysenter_esp);
 802     kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_EIP, env->sysenter_eip);
 803     if (kvm_has_msr_star(env))
 804         kvm_msr_entry_set(&msrs[n++], MSR_STAR, env->star);
 805     kvm_msr_entry_set(&msrs[n++], MSR_VM_HSAVE_PA, env->vm_hsave);
 806 #ifdef TARGET_X86_64
 807     /* FIXME if lm capable */
 808     kvm_msr_entry_set(&msrs[n++], MSR_CSTAR, env->cstar);
 809     kvm_msr_entry_set(&msrs[n++], MSR_KERNELGSBASE, env->kernelgsbase);
 810     kvm_msr_entry_set(&msrs[n++], MSR_FMASK, env->fmask);
 811     kvm_msr_entry_set(&msrs[n++], MSR_LSTAR, env->lstar);
 812 #endif
 813     if (level == KVM_PUT_FULL_STATE) {
 814         kvm_msr_entry_set(&msrs[n++], MSR_IA32_TSC, env->tsc);
 815         kvm_msr_entry_set(&msrs[n++], MSR_KVM_SYSTEM_TIME,
 816                           env->system_time_msr);
 817         kvm_msr_entry_set(&msrs[n++], MSR_KVM_WALL_CLOCK, env->wall_clock_msr);
 818     }
 819
 820     msr_data.info.nmsrs = n;
 821
 822     return kvm_vcpu_ioctl(env, KVM_SET_MSRS, &msr_data);
 823
 824 }
 825
 826
 827 static int kvm_get_fpu(CPUState *env)
 828 {
 829     struct kvm_fpu fpu;
 830     int i, ret;
 831
 832     ret = kvm_vcpu_ioctl(env, KVM_GET_FPU, &fpu);
 833     if (ret < 0)
 834         return ret;
 835
 836     env->fpstt = (fpu.fsw >> 11) & 7;
 837     env->fpus = fpu.fsw;
 838     env->fpuc = fpu.fcw;
 839     for (i = 0; i < 8; ++i)
 840         env->fptags[i] = !((fpu.ftwx >> i) & 1);
 841     memcpy(env->fpregs, fpu.fpr, sizeof env->fpregs);
 842     memcpy(env->xmm_regs, fpu.xmm, sizeof env->xmm_regs);
 843     env->mxcsr = fpu.mxcsr;
 844
 845     return 0;
 846 }
 847
 848 static int kvm_get_xsave(CPUState *env)
 849 {
 850 #ifdef KVM_CAP_XSAVE
 851     struct kvm_xsave* xsave;
 852     int ret, i;
 853     uint16_t cwd, swd, twd, fop;
 854
 855     if (!kvm_has_xsave())
 856         return kvm_get_fpu(env);
 857
 858     xsave = qemu_memalign(4096, sizeof(struct kvm_xsave));
 859     ret = kvm_vcpu_ioctl(env, KVM_GET_XSAVE, xsave);
 860     if (ret < 0)
 861         return ret;
 862
 863     cwd = (uint16_t)xsave->region[0];
 864     swd = (uint16_t)(xsave->region[0] >> 16);
 865     twd = (uint16_t)xsave->region[1];
 866     fop = (uint16_t)(xsave->region[1] >> 16);
 867     env->fpstt = (swd >> 11) & 7;
 868     env->fpus = swd;
 869     env->fpuc = cwd;
 870     for (i = 0; i < 8; ++i)
 871         env->fptags[i] = !((twd >> i) & 1);
 872     env->mxcsr = xsave->region[XSAVE_MXCSR];
 873     memcpy(env->fpregs, &xsave->region[XSAVE_ST_SPACE],
 874             sizeof env->fpregs);
 875     memcpy(env->xmm_regs, &xsave->region[XSAVE_XMM_SPACE],
 876             sizeof env->xmm_regs);
 877     env->xstate_bv = *(uint64_t *)&xsave->region[XSAVE_XSTATE_BV];
 878     memcpy(env->ymmh_regs, &xsave->region[XSAVE_YMMH_SPACE],
 879             sizeof env->ymmh_regs);
 880     return 0;
 881 #else
 882     return kvm_get_fpu(env);
 883 #endif
 884 }
 885
 886 static int kvm_get_xcrs(CPUState *env)
 887 {
 888 #ifdef KVM_CAP_XCRS
 889     int i, ret;
 890     struct kvm_xcrs xcrs;
 891
 892     if (!kvm_has_xcrs())
 893         return 0;
 894
 895     ret = kvm_vcpu_ioctl(env, KVM_GET_XCRS, &xcrs);
 896     if (ret < 0)
 897         return ret;
 898
 899     for (i = 0; i < xcrs.nr_xcrs; i++)
 900         /* Only support xcr0 now */
 901         if (xcrs.xcrs[0].xcr == 0) {
 902             env->xcr0 = xcrs.xcrs[0].value;
 903             break;
 904         }
 905     return 0;
 906 #else
 907     return 0;
 908 #endif
 909 }
 910
 911 static int kvm_get_sregs(CPUState *env)
 912 {
 913     struct kvm_sregs sregs;
 914     uint32_t hflags;
 915     int bit, i, ret;
 916
 917     ret = kvm_vcpu_ioctl(env, KVM_GET_SREGS, &sregs);
 918     if (ret < 0)
 919         return ret;
 920
 921     /* There can only be one pending IRQ set in the bitmap at a time, so try
 922        to find it and save its number instead (-1 for none). */
 923     env->interrupt_injected = -1;
 924     for (i = 0; i < ARRAY_SIZE(sregs.interrupt_bitmap); i++) {
 925         if (sregs.interrupt_bitmap[i]) {
 926             bit = ctz64(sregs.interrupt_bitmap[i]);
 927             env->interrupt_injected = i * 64 + bit;
 928             break;
 929         }
 930     }
 931
 932     get_seg(&env->segs[R_CS], &sregs.cs);
 933     get_seg(&env->segs[R_DS], &sregs.ds);
 934     get_seg(&env->segs[R_ES], &sregs.es);
 935     get_seg(&env->segs[R_FS], &sregs.fs);
 936     get_seg(&env->segs[R_GS], &sregs.gs);
 937     get_seg(&env->segs[R_SS], &sregs.ss);
 938
 939     get_seg(&env->tr, &sregs.tr);
 940     get_seg(&env->ldt, &sregs.ldt);
 941
 942     env->idt.limit = sregs.idt.limit;
 943     env->idt.base = sregs.idt.base;
 944     env->gdt.limit = sregs.gdt.limit;
 945     env->gdt.base = sregs.gdt.base;
 946
 947     env->cr[0] = sregs.cr0;
 948     env->cr[2] = sregs.cr2;
 949     env->cr[3] = sregs.cr3;
 950     env->cr[4] = sregs.cr4;
 951
 952     cpu_set_apic_base(env->apic_state, sregs.apic_base);
 953
 954     env->efer = sregs.efer;
 955     //cpu_set_apic_tpr(env->apic_state, sregs.cr8);
 956
 957 #define HFLAG_COPY_MASK ~( \
 958                         HF_CPL_MASK | HF_PE_MASK | HF_MP_MASK | HF_EM_MASK | \
 959                         HF_TS_MASK | HF_TF_MASK | HF_VM_MASK | HF_IOPL_MASK | \
 960                         HF_OSFXSR_MASK | HF_LMA_MASK | HF_CS32_MASK | \
 961                         HF_SS32_MASK | HF_CS64_MASK | HF_ADDSEG_MASK)
 962
 963
 964
 965     hflags = (env->segs[R_CS].flags >> DESC_DPL_SHIFT) & HF_CPL_MASK;
 966     hflags |= (env->cr[0] & CR0_PE_MASK) << (HF_PE_SHIFT - CR0_PE_SHIFT);
 967     hflags |= (env->cr[0] << (HF_MP_SHIFT - CR0_MP_SHIFT)) &
 968             (HF_MP_MASK | HF_EM_MASK | HF_TS_MASK);
 969     hflags |= (env->eflags & (HF_TF_MASK | HF_VM_MASK | HF_IOPL_MASK));
 970     hflags |= (env->cr[4] & CR4_OSFXSR_MASK) <<
 971             (HF_OSFXSR_SHIFT - CR4_OSFXSR_SHIFT);
 972
 973     if (env->efer & MSR_EFER_LMA) {
 974         hflags |= HF_LMA_MASK;
 975     }
 976
 977     if ((hflags & HF_LMA_MASK) && (env->segs[R_CS].flags & DESC_L_MASK)) {
 978         hflags |= HF_CS32_MASK | HF_SS32_MASK | HF_CS64_MASK;
 979     } else {
 980         hflags |= (env->segs[R_CS].flags & DESC_B_MASK) >>
 981                 (DESC_B_SHIFT - HF_CS32_SHIFT);
 982         hflags |= (env->segs[R_SS].flags & DESC_B_MASK) >>
 983                 (DESC_B_SHIFT - HF_SS32_SHIFT);
 984         if (!(env->cr[0] & CR0_PE_MASK) ||
 985                    (env->eflags & VM_MASK) ||
 986                    !(hflags & HF_CS32_MASK)) {
 987                 hflags |= HF_ADDSEG_MASK;
 988             } else {
 989                 hflags |= ((env->segs[R_DS].base |
 990                                 env->segs[R_ES].base |
 991                                 env->segs[R_SS].base) != 0) <<
 992                     HF_ADDSEG_SHIFT;
 993             }
 994     }
 995     env->hflags = (env->hflags & HFLAG_COPY_MASK) | hflags;
 996
 997     return 0;
 998 }
 999
1000 static int kvm_get_msrs(CPUState *env)
1001 {
1002     struct {
1003         struct kvm_msrs info;
1004         struct kvm_msr_entry entries[100];
1005     } msr_data;
1006     struct kvm_msr_entry *msrs = msr_data.entries;
1007     int ret, i, n;
1008
1009     n = 0;
1010     msrs[n++].index = MSR_IA32_SYSENTER_CS;
1011     msrs[n++].index = MSR_IA32_SYSENTER_ESP;
1012     msrs[n++].index = MSR_IA32_SYSENTER_EIP;
1013     if (kvm_has_msr_star(env))
1014         msrs[n++].index = MSR_STAR;
1015     msrs[n++].index = MSR_IA32_TSC;
1016     msrs[n++].index = MSR_VM_HSAVE_PA;
1017 #ifdef TARGET_X86_64
1018     /* FIXME lm_capable_kernel */
1019     msrs[n++].index = MSR_CSTAR;
1020     msrs[n++].index = MSR_KERNELGSBASE;
1021     msrs[n++].index = MSR_FMASK;
1022     msrs[n++].index = MSR_LSTAR;
1023 #endif
1024     msrs[n++].index = MSR_KVM_SYSTEM_TIME;
1025     msrs[n++].index = MSR_KVM_WALL_CLOCK;
1026
1027     msr_data.info.nmsrs = n;
1028     ret = kvm_vcpu_ioctl(env, KVM_GET_MSRS, &msr_data);
1029     if (ret < 0)
1030         return ret;
1031
1032     for (i = 0; i < ret; i++) {
1033         switch (msrs[i].index) {
1034         case MSR_IA32_SYSENTER_CS:
1035             env->sysenter_cs = msrs[i].data;
1036             break;
1037         case MSR_IA32_SYSENTER_ESP:
1038             env->sysenter_esp = msrs[i].data;
1039             break;
1040         case MSR_IA32_SYSENTER_EIP:
1041             env->sysenter_eip = msrs[i].data;
1042             break;
1043         case MSR_STAR:
1044             env->star = msrs[i].data;
1045             break;
1046 #ifdef TARGET_X86_64
1047         case MSR_CSTAR:
1048             env->cstar = msrs[i].data;
1049             break;
1050         case MSR_KERNELGSBASE:
1051             env->kernelgsbase = msrs[i].data;
1052             break;
1053         case MSR_FMASK:
1054             env->fmask = msrs[i].data;
1055             break;
1056         case MSR_LSTAR:
1057             env->lstar = msrs[i].data;
1058             break;
1059 #endif
1060         case MSR_IA32_TSC:
1061             env->tsc = msrs[i].data;
1062             break;
1063         case MSR_KVM_SYSTEM_TIME:
1064             env->system_time_msr = msrs[i].data;
1065             break;
1066         case MSR_KVM_WALL_CLOCK:
1067             env->wall_clock_msr = msrs[i].data;
1068             break;
1069         case MSR_VM_HSAVE_PA:
1070             env->vm_hsave = msrs[i].data;
1071             break;
1072         }
1073     }
1074
1075     return 0;
1076 }
1077
1078 static int kvm_put_mp_state(CPUState *env)
1079 {
1080     struct kvm_mp_state mp_state = { .mp_state = env->mp_state };
1081
1082     return kvm_vcpu_ioctl(env, KVM_SET_MP_STATE, &mp_state);
1083 }
1084
1085 static int kvm_get_mp_state(CPUState *env)
1086 {
1087     struct kvm_mp_state mp_state;
1088     int ret;
1089
1090     ret = kvm_vcpu_ioctl(env, KVM_GET_MP_STATE, &mp_state);
1091     if (ret < 0) {
1092         return ret;
1093     }
1094     env->mp_state = mp_state.mp_state;
1095     return 0;
1096 }
1097 #endif
1098
1099 static int kvm_put_vcpu_events(CPUState *env, int level)
1100 {
1101 #ifdef KVM_CAP_VCPU_EVENTS
1102     struct kvm_vcpu_events events;
1103
1104     if (!kvm_has_vcpu_events()) {
1105         return 0;
1106     }
1107
1108     events.exception.injected = (env->exception_injected >= 0);
1109     events.exception.nr = env->exception_injected;
1110     events.exception.has_error_code = env->has_error_code;
1111     events.exception.error_code = env->error_code;
1112
1113     events.interrupt.injected = (env->interrupt_injected >= 0);
1114     events.interrupt.nr = env->interrupt_injected;
1115     events.interrupt.soft = env->soft_interrupt;
1116
1117     events.nmi.injected = env->nmi_injected;
1118     events.nmi.pending = env->nmi_pending;
1119     events.nmi.masked = !!(env->hflags2 & HF2_NMI_MASK);
1120
1121     events.sipi_vector = env->sipi_vector;
1122
1123     events.flags = 0;
1124     if (level >= KVM_PUT_RESET_STATE) {
1125         events.flags |=
1126             KVM_VCPUEVENT_VALID_NMI_PENDING | KVM_VCPUEVENT_VALID_SIPI_VECTOR;
1127     }
1128
1129     return kvm_vcpu_ioctl(env, KVM_SET_VCPU_EVENTS, &events);
1130 #else
1131     return 0;
1132 #endif
1133 }
1134
1135 static int kvm_get_vcpu_events(CPUState *env)
1136 {
1137 #ifdef KVM_CAP_VCPU_EVENTS
1138     struct kvm_vcpu_events events;
1139     int ret;
1140
1141     if (!kvm_has_vcpu_events()) {
1142         return 0;
1143     }
1144
1145     ret = kvm_vcpu_ioctl(env, KVM_GET_VCPU_EVENTS, &events);
1146     if (ret < 0) {
1147        return ret;
1148     }
1149     env->exception_injected =
1150        events.exception.injected ? events.exception.nr : -1;
1151     env->has_error_code = events.exception.has_error_code;
1152     env->error_code = events.exception.error_code;
1153
1154     env->interrupt_injected =
1155         events.interrupt.injected ? events.interrupt.nr : -1;
1156     env->soft_interrupt = events.interrupt.soft;
1157
1158     env->nmi_injected = events.nmi.injected;
1159     env->nmi_pending = events.nmi.pending;
1160     if (events.nmi.masked) {
1161         env->hflags2 |= HF2_NMI_MASK;
1162     } else {
1163         env->hflags2 &= ~HF2_NMI_MASK;
1164     }
1165
1166     env->sipi_vector = events.sipi_vector;
1167 #endif
1168
1169     return 0;
1170 }
1171
1172 static int kvm_guest_debug_workarounds(CPUState *env)
1173 {
1174     int ret = 0;
1175 #ifdef KVM_CAP_SET_GUEST_DEBUG
1176     unsigned long reinject_trap = 0;
1177
1178     if (!kvm_has_vcpu_events()) {
1179         if (env->exception_injected == 1) {
1180             reinject_trap = KVM_GUESTDBG_INJECT_DB;
1181         } else if (env->exception_injected == 3) {
1182             reinject_trap = KVM_GUESTDBG_INJECT_BP;
1183         }
1184         env->exception_injected = -1;
1185     }
1186
1187     /*
1188      * Kernels before KVM_CAP_X86_ROBUST_SINGLESTEP overwrote flags.TF
1189      * injected via SET_GUEST_DEBUG while updating GP regs. Work around this
1190      * by updating the debug state once again if single-stepping is on.
1191      * Another reason to call kvm_update_guest_debug here is a pending debug
1192      * trap raise by the guest. On kernels without SET_VCPU_EVENTS we have to
1193      * reinject them via SET_GUEST_DEBUG.
1194      */
1195     if (reinject_trap ||
1196         (!kvm_has_robust_singlestep() && env->singlestep_enabled)) {
1197         ret = kvm_update_guest_debug(env, reinject_trap);
1198     }
1199 #endif /* KVM_CAP_SET_GUEST_DEBUG */
1200     return ret;
1201 }
1202
1203 static int kvm_put_debugregs(CPUState *env)
1204 {
1205 #ifdef KVM_CAP_DEBUGREGS
1206     struct kvm_debugregs dbgregs;
1207     int i;
1208
1209     if (!kvm_has_debugregs()) {
1210         return 0;
1211     }
1212
1213     for (i = 0; i < 4; i++) {
1214         dbgregs.db[i] = env->dr[i];
1215     }
1216     dbgregs.dr6 = env->dr[6];
1217     dbgregs.dr7 = env->dr[7];
1218     dbgregs.flags = 0;
1219
1220     return kvm_vcpu_ioctl(env, KVM_SET_DEBUGREGS, &dbgregs);
1221 #else
1222     return 0;
1223 #endif
1224 }
1225
1226 static int kvm_get_debugregs(CPUState *env)
1227 {
1228 #ifdef KVM_CAP_DEBUGREGS
1229     struct kvm_debugregs dbgregs;
1230     int i, ret;
1231
1232     if (!kvm_has_debugregs()) {
1233         return 0;
1234     }
1235
1236     ret = kvm_vcpu_ioctl(env, KVM_GET_DEBUGREGS, &dbgregs);
1237     if (ret < 0) {
1238        return ret;
1239     }
1240     for (i = 0; i < 4; i++) {
1241         env->dr[i] = dbgregs.db[i];
1242     }
1243     env->dr[4] = env->dr[6] = dbgregs.dr6;
1244     env->dr[5] = env->dr[7] = dbgregs.dr7;
1245 #endif
1246
1247     return 0;
1248 }
1249
1250 #ifdef OBSOLETE_KVM_IMPL
1251 int kvm_arch_put_registers(CPUState *env, int level)
1252 {
1253     int ret;
1254
1255     assert(cpu_is_stopped(env) || qemu_cpu_self(env));
1256
1257     ret = kvm_getput_regs(env, 1);
1258     if (ret < 0)
1259         return ret;
1260
1261     ret = kvm_put_xsave(env);
1262     if (ret < 0)
1263         return ret;
1264
1265     ret = kvm_put_xcrs(env);
1266     if (ret < 0)
1267         return ret;
1268
1269     ret = kvm_put_sregs(env);
1270     if (ret < 0)
1271         return ret;
1272
1273     ret = kvm_put_msrs(env, level);
1274     if (ret < 0)
1275         return ret;
1276
1277     if (level >= KVM_PUT_RESET_STATE) {
1278         ret = kvm_put_mp_state(env);
1279         if (ret < 0)
1280             return ret;
1281     }
1282
1283     ret = kvm_put_vcpu_events(env, level);
1284     if (ret < 0)
1285         return ret;
1286
1287     /* must be last */
1288     ret = kvm_guest_debug_workarounds(env);
1289     if (ret < 0)
1290         return ret;
1291
1292     ret = kvm_put_debugregs(env);
1293     if (ret < 0)
1294         return ret;
1295
1296     return 0;
1297 }
1298
1299 int kvm_arch_get_registers(CPUState *env)
1300 {
1301     int ret;
1302
1303     assert(cpu_is_stopped(env) || qemu_cpu_self(env));
1304
1305     ret = kvm_getput_regs(env, 0);
1306     if (ret < 0)
1307         return ret;
1308
1309     ret = kvm_get_xsave(env);
1310     if (ret < 0)
1311         return ret;
1312
1313     ret = kvm_get_xcrs(env);
1314     if (ret < 0)
1315         return ret;
1316
1317     ret = kvm_get_sregs(env);
1318     if (ret < 0)
1319         return ret;
1320
1321     ret = kvm_get_msrs(env);
1322     if (ret < 0)
1323         return ret;
1324
1325     ret = kvm_get_mp_state(env);
1326     if (ret < 0)
1327         return ret;
1328
1329     ret = kvm_get_vcpu_events(env);
1330     if (ret < 0)
1331         return ret;
1332
1333     ret = kvm_get_debugregs(env);
1334     if (ret < 0)
1335         return ret;
1336
1337     return 0;
1338 }
1339
1340 int kvm_arch_pre_run(CPUState *env, struct kvm_run *run)
1341 {
1342     /* Try to inject an interrupt if the guest can accept it */
1343     if (run->ready_for_interrupt_injection &&
1344         (env->interrupt_request & CPU_INTERRUPT_HARD) &&
1345         (env->eflags & IF_MASK)) {
1346         int irq;
1347
1348         env->interrupt_request &= ~CPU_INTERRUPT_HARD;
1349         irq = cpu_get_pic_interrupt(env);
1350         if (irq >= 0) {
1351             struct kvm_interrupt intr;
1352             intr.irq = irq;
1353             /* FIXME: errors */
1354             DPRINTF("injected interrupt %d\n", irq);
1355             kvm_vcpu_ioctl(env, KVM_INTERRUPT, &intr);
1356         }
1357     }
1358
1359     /* If we have an interrupt but the guest is not ready to receive an
1360      * interrupt, request an interrupt window exit.  This will
1361      * cause a return to userspace as soon as the guest is ready to
1362      * receive interrupts. */
1363     if ((env->interrupt_request & CPU_INTERRUPT_HARD))
1364         run->request_interrupt_window = 1;
1365     else
1366         run->request_interrupt_window = 0;
1367
1368     DPRINTF("setting tpr\n");
1369     run->cr8 = cpu_get_apic_tpr(env->apic_state);
1370
1371     return 0;
1372 }
1373 #endif
1374
1375 int kvm_arch_post_run(CPUState *env, struct kvm_run *run)
1376 {
1377     if (run->if_flag)
1378         env->eflags |= IF_MASK;
1379     else
1380         env->eflags &= ~IF_MASK;
1381
1382     cpu_set_apic_tpr(env->apic_state, run->cr8);
1383     cpu_set_apic_base(env->apic_state, run->apic_base);
1384
1385     return 0;
1386 }
1387
1388 #ifdef OBSOLETE_KVM_IMPL
1389
1390 int kvm_arch_process_irqchip_events(CPUState *env)
1391 {
1392     if (env->interrupt_request & CPU_INTERRUPT_INIT) {
1393         kvm_cpu_synchronize_state(env);
1394         do_cpu_init(env);
1395         env->exception_index = EXCP_HALTED;
1396     }
1397
1398     if (env->interrupt_request & CPU_INTERRUPT_SIPI) {
1399         kvm_cpu_synchronize_state(env);
1400         do_cpu_sipi(env);
1401     }
1402
1403     return env->halted;
1404 }
1405
1406 static int kvm_handle_halt(CPUState *env)
1407 {
1408     if (!((env->interrupt_request & CPU_INTERRUPT_HARD) &&
1409           (env->eflags & IF_MASK)) &&
1410         !(env->interrupt_request & CPU_INTERRUPT_NMI)) {
1411         env->halted = 1;
1412         env->exception_index = EXCP_HLT;
1413         return 0;
1414     }
1415
1416     return 1;
1417 }
1418
1419 int kvm_arch_handle_exit(CPUState *env, struct kvm_run *run)
1420 {
1421     int ret = 0;
1422
1423     switch (run->exit_reason) {
1424     case KVM_EXIT_HLT:
1425         DPRINTF("handle_hlt\n");
1426         ret = kvm_handle_halt(env);
1427         break;
1428     }
1429
1430     return ret;
1431 }
1432 #endif
1433
1434 #ifdef KVM_CAP_SET_GUEST_DEBUG
1435 int kvm_arch_insert_sw_breakpoint(CPUState *env, struct kvm_sw_breakpoint *bp)
1436 {
1437     static const uint8_t int3 = 0xcc;
1438
1439     if (cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&bp->saved_insn, 1, 0) ||
1440         cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&int3, 1, 1))
1441         return -EINVAL;
1442     return 0;
1443 }
1444
1445 int kvm_arch_remove_sw_breakpoint(CPUState *env, struct kvm_sw_breakpoint *bp)
1446 {
1447     uint8_t int3;
1448
1449     if (cpu_memory_rw_debug(env, bp->pc, &int3, 1, 0) || int3 != 0xcc ||
1450         cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&bp->saved_insn, 1, 1))
1451         return -EINVAL;
1452     return 0;
1453 }
1454
1455 static struct {
1456     target_ulong addr;
1457     int len;
1458     int type;
1459 } hw_breakpoint[4];
1460
1461 static int nb_hw_breakpoint;
1462
1463 static int find_hw_breakpoint(target_ulong addr, int len, int type)
1464 {
1465     int n;
1466
1467     for (n = 0; n < nb_hw_breakpoint; n++)
1468         if (hw_breakpoint[n].addr == addr && hw_breakpoint[n].type == type &&
1469             (hw_breakpoint[n].len == len || len == -1))
1470             return n;
1471     return -1;
1472 }
1473
1474 int kvm_arch_insert_hw_breakpoint(target_ulong addr,
1475                                   target_ulong len, int type)
1476 {
1477     switch (type) {
1478     case GDB_BREAKPOINT_HW:
1479         len = 1;
1480         break;
1481     case GDB_WATCHPOINT_WRITE:
1482     case GDB_WATCHPOINT_ACCESS:
1483         switch (len) {
1484         case 1:
1485             break;
1486         case 2:
1487         case 4:
1488         case 8:
1489             if (addr & (len - 1))
1490                 return -EINVAL;
1491             break;
1492         default:
1493             return -EINVAL;
1494         }
1495         break;
1496     default:
1497         return -ENOSYS;
1498     }
1499
1500     if (nb_hw_breakpoint == 4)
1501         return -ENOBUFS;
1502
1503     if (find_hw_breakpoint(addr, len, type) >= 0)
1504         return -EEXIST;
1505
1506     hw_breakpoint[nb_hw_breakpoint].addr = addr;
1507     hw_breakpoint[nb_hw_breakpoint].len = len;
1508     hw_breakpoint[nb_hw_breakpoint].type = type;
1509     nb_hw_breakpoint++;
1510
1511     return 0;
1512 }
1513
1514 int kvm_arch_remove_hw_breakpoint(target_ulong addr,
1515                                   target_ulong len, int type)
1516 {
1517     int n;
1518
1519     n = find_hw_breakpoint(addr, (type == GDB_BREAKPOINT_HW) ? 1 : len, type);
1520     if (n < 0)
1521         return -ENOENT;
1522
1523     nb_hw_breakpoint--;
1524     hw_breakpoint[n] = hw_breakpoint[nb_hw_breakpoint];
1525
1526     return 0;
1527 }
1528
1529 void kvm_arch_remove_all_hw_breakpoints(void)
1530 {
1531     nb_hw_breakpoint = 0;
1532 }
1533
1534 static CPUWatchpoint hw_watchpoint;
1535
1536 int kvm_arch_debug(struct kvm_debug_exit_arch *arch_info)
1537 {
1538     int handle = 0;
1539     int n;
1540
1541     if (arch_info->exception == 1) {
1542         if (arch_info->dr6 & (1 << 14)) {
1543             if (cpu_single_env->singlestep_enabled)
1544                 handle = 1;
1545         } else {
1546             for (n = 0; n < 4; n++)
1547                 if (arch_info->dr6 & (1 << n))
1548                     switch ((arch_info->dr7 >> (16 + n*4)) & 0x3) {
1549                     case 0x0:
1550                         handle = 1;
1551                         break;
1552                     case 0x1:
1553                         handle = 1;
1554                         cpu_single_env->watchpoint_hit = &hw_watchpoint;
1555                         hw_watchpoint.vaddr = hw_breakpoint[n].addr;
1556                         hw_watchpoint.flags = BP_MEM_WRITE;
1557                         break;
1558                     case 0x3:
1559                         handle = 1;
1560                         cpu_single_env->watchpoint_hit = &hw_watchpoint;
1561                         hw_watchpoint.vaddr = hw_breakpoint[n].addr;
1562                         hw_watchpoint.flags = BP_MEM_ACCESS;
1563                         break;
1564                     }
1565         }
1566     } else if (kvm_find_sw_breakpoint(cpu_single_env, arch_info->pc))
1567         handle = 1;
1568
1569     if (!handle) {
1570         cpu_synchronize_state(cpu_single_env);
1571         assert(cpu_single_env->exception_injected == -1);
1572
1573         cpu_single_env->exception_injected = arch_info->exception;
1574         cpu_single_env->has_error_code = 0;
1575     }
1576
1577     return handle;
1578 }
1579
1580 void kvm_arch_update_guest_debug(CPUState *env, struct kvm_guest_debug *dbg)
1581 {
1582     const uint8_t type_code[] = {
1583         [GDB_BREAKPOINT_HW] = 0x0,
1584         [GDB_WATCHPOINT_WRITE] = 0x1,
1585         [GDB_WATCHPOINT_ACCESS] = 0x3
1586     };
1587     const uint8_t len_code[] = {
1588         [1] = 0x0, [2] = 0x1, [4] = 0x3, [8] = 0x2
1589     };
1590     int n;
1591
1592     if (kvm_sw_breakpoints_active(env))
1593         dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP;
1594
1595     if (nb_hw_breakpoint > 0) {
1596         dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW_BP;
1597         dbg->arch.debugreg[7] = 0x0600;
1598         for (n = 0; n < nb_hw_breakpoint; n++) {
1599             dbg->arch.debugreg[n] = hw_breakpoint[n].addr;
1600             dbg->arch.debugreg[7] |= (2 << (n * 2)) |
1601                 (type_code[hw_breakpoint[n].type] << (16 + n*4)) |
1602                 (len_code[hw_breakpoint[n].len] << (18 + n*4));
1603         }
1604     }
1605     /* Legal xcr0 for loading */
1606     env->xcr0 = 1;
1607 }
1608 #endif /* KVM_CAP_SET_GUEST_DEBUG */
1609
1610 bool kvm_arch_stop_on_emulation_error(CPUState *env)
1611 {
1612       return !(env->cr[0] & CR0_PE_MASK) ||
1613               ((env->segs[R_CS].selector  & 3) != 3);
1614 }
1615
1616 static void hardware_memory_error(void)
1617 {
1618     fprintf(stderr, "Hardware memory error!\n");
1619     exit(1);
1620 }
1621
1622 int kvm_on_sigbus_vcpu(CPUState *env, int code, void *addr)
1623 {
1624 #if defined(KVM_CAP_MCE)
1625     struct kvm_x86_mce mce = {
1626             .bank = 9,
1627     };
1628     void *vaddr;
1629     ram_addr_t ram_addr;
1630     target_phys_addr_t paddr;
1631     int r;
1632
1633     if ((env->mcg_cap & MCG_SER_P) && addr
1634         && (code == BUS_MCEERR_AR
1635             || code == BUS_MCEERR_AO)) {
1636         if (code == BUS_MCEERR_AR) {
1637             /* Fake an Intel architectural Data Load SRAR UCR */
1638             mce.status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN
1639                 | MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S
1640                 | MCI_STATUS_AR | 0x134;
1641             mce.misc = (MCM_ADDR_PHYS << 6) | 0xc;
1642             mce.mcg_status = MCG_STATUS_MCIP | MCG_STATUS_EIPV;
1643         } else {
1644             /*
1645              * If there is an MCE excpetion being processed, ignore
1646              * this SRAO MCE
1647              */
1648             r = kvm_mce_in_exception(env);
1649             if (r == -1) {
1650                 fprintf(stderr, "Failed to get MCE status\n");
1651             } else if (r) {
1652                 return 0;
1653             }
1654             /* Fake an Intel architectural Memory scrubbing UCR */
1655             mce.status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN
1656                 | MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S
1657                 | 0xc0;
1658             mce.misc = (MCM_ADDR_PHYS << 6) | 0xc;
1659             mce.mcg_status = MCG_STATUS_MCIP | MCG_STATUS_RIPV;
1660         }
1661         vaddr = (void *)addr;
1662         if (qemu_ram_addr_from_host(vaddr, &ram_addr) ||
1663             !kvm_physical_memory_addr_from_ram(env->kvm_state, ram_addr, &paddr)) {
1664             fprintf(stderr, "Hardware memory error for memory used by "
1665                     "QEMU itself instead of guest system!\n");
1666             /* Hope we are lucky for AO MCE */
1667             if (code == BUS_MCEERR_AO) {
1668                 return 0;
1669             } else {
1670                 hardware_memory_error();
1671             }
1672         }
1673         mce.addr = paddr;
1674         r = kvm_set_mce(env, &mce);
1675         if (r < 0) {
1676             fprintf(stderr, "kvm_set_mce: %s\n", strerror(errno));
1677             abort();
1678         }
1679     } else
1680 #endif
1681     {
1682         if (code == BUS_MCEERR_AO) {
1683             return 0;
1684         } else if (code == BUS_MCEERR_AR) {
1685             hardware_memory_error();
1686         } else {
1687             return 1;
1688         }
1689     }
1690     return 0;
1691 }
1692
1693 int kvm_on_sigbus(int code, void *addr)
1694 {
1695 #if defined(KVM_CAP_MCE)
1696     if ((first_cpu->mcg_cap & MCG_SER_P) && addr && code == BUS_MCEERR_AO) {
1697         uint64_t status;
1698         void *vaddr;
1699         ram_addr_t ram_addr;
1700         target_phys_addr_t paddr;
1701         CPUState *cenv;
1702
1703         /* Hope we are lucky for AO MCE */
1704         vaddr = addr;
1705         if (qemu_ram_addr_from_host(vaddr, &ram_addr) ||
1706             !kvm_physical_memory_addr_from_ram(first_cpu->kvm_state, ram_addr, &paddr)) {
1707             fprintf(stderr, "Hardware memory error for memory used by "
1708                     "QEMU itself instead of guest system!: %p\n", addr);
1709             return 0;
1710         }
1711         status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN
1712             | MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S
1713             | 0xc0;
1714         kvm_inject_x86_mce(first_cpu, 9, status,
1715                            MCG_STATUS_MCIP | MCG_STATUS_RIPV, paddr,
1716                            (MCM_ADDR_PHYS << 6) | 0xc, 1);
1717         for (cenv = first_cpu->next_cpu; cenv != NULL; cenv = cenv->next_cpu) {
1718             kvm_inject_x86_mce(cenv, 1, MCI_STATUS_VAL | MCI_STATUS_UC,
1719                                MCG_STATUS_MCIP | MCG_STATUS_RIPV, 0, 0, 1);
1720         }
1721     } else
1722 #endif
1723     {
1724         if (code == BUS_MCEERR_AO) {
1725             return 0;
1726         } else if (code == BUS_MCEERR_AR) {
1727             hardware_memory_error();
1728         } else {
1729             return 1;
1730         }
1731     }
1732     return 0;
1733 }
1734
1735 #include "qemu-kvm-x86.c"