target-i386/kvm.c

   1 /*
   2  * QEMU KVM support
   3  *
   4  * Copyright (C) 2006-2008 Qumranet Technologies
   5  * Copyright IBM, Corp. 2008
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *
  10  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  11  * See the COPYING file in the top-level directory.
  12  *
  13  */
  14
  15 #include <sys/types.h>
  16 #include <sys/ioctl.h>
  17 #include <sys/mman.h>
  18 #include <sys/utsname.h>
  19
  20 #include <linux/kvm.h>
  21
  22 #include "qemu-common.h"
  23 #include "sysemu.h"
  24 #include "kvm.h"
  25 #include "cpu.h"
  26 #include "gdbstub.h"
  27 #include "host-utils.h"
  28 #include "hw/pc.h"
  29 #include "hw/apic.h"
  30 #include "ioport.h"
  31 #include "kvm_x86.h"
  32
  33 #ifdef CONFIG_KVM_PARA
  34 #include <linux/kvm_para.h>
  35 #endif
  36 //
  37 //#define DEBUG_KVM
  38
  39 #ifdef DEBUG_KVM
  40 #define DPRINTF(fmt, ...) \
  41     do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
  42 #else
  43 #define DPRINTF(fmt, ...) \
  44     do { } while (0)
  45 #endif
  46
  47 #define MSR_KVM_WALL_CLOCK  0x11
  48 #define MSR_KVM_SYSTEM_TIME 0x12
  49
  50 #ifndef BUS_MCEERR_AR
  51 #define BUS_MCEERR_AR 4
  52 #endif
  53 #ifndef BUS_MCEERR_AO
  54 #define BUS_MCEERR_AO 5
  55 #endif
  56
  57 #ifdef OBSOLETE_KVM_IMPL
  58 static int lm_capable_kernel;
  59 #endif
  60
  61 #ifdef KVM_CAP_EXT_CPUID
  62
  63 static struct kvm_cpuid2 *try_get_cpuid(KVMState *s, int max)
  64 {
  65     struct kvm_cpuid2 *cpuid;
  66     int r, size;
  67
  68     size = sizeof(*cpuid) + max * sizeof(*cpuid->entries);
  69     cpuid = (struct kvm_cpuid2 *)qemu_mallocz(size);
  70     cpuid->nent = max;
  71     r = kvm_ioctl(s, KVM_GET_SUPPORTED_CPUID, cpuid);
  72     if (r == 0 && cpuid->nent >= max) {
  73         r = -E2BIG;
  74     }
  75     if (r < 0) {
  76         if (r == -E2BIG) {
  77             qemu_free(cpuid);
  78             return NULL;
  79         } else {
  80             fprintf(stderr, "KVM_GET_SUPPORTED_CPUID failed: %s\n",
  81                     strerror(-r));
  82             exit(1);
  83         }
  84     }
  85     return cpuid;
  86 }
  87
  88 uint32_t kvm_arch_get_supported_cpuid(CPUState *env, uint32_t function,
  89                                       uint32_t index, int reg)
  90 {
  91     struct kvm_cpuid2 *cpuid;
  92     int i, max;
  93     uint32_t ret = 0;
  94     uint32_t cpuid_1_edx;
  95
  96     if (!kvm_check_extension(env->kvm_state, KVM_CAP_EXT_CPUID)) {
  97         return -1U;
  98     }
  99
 100     max = 1;
 101     while ((cpuid = try_get_cpuid(env->kvm_state, max)) == NULL) {
 102         max *= 2;
 103     }
 104
 105     for (i = 0; i < cpuid->nent; ++i) {
 106         if (cpuid->entries[i].function == function &&
 107             cpuid->entries[i].index == index) {
 108             switch (reg) {
 109             case R_EAX:
 110                 ret = cpuid->entries[i].eax;
 111                 break;
 112             case R_EBX:
 113                 ret = cpuid->entries[i].ebx;
 114                 break;
 115             case R_ECX:
 116                 ret = cpuid->entries[i].ecx;
 117                 break;
 118             case R_EDX:
 119                 ret = cpuid->entries[i].edx;
 120                 switch (function) {
 121                 case 1:
 122                     /* KVM before 2.6.30 misreports the following features */
 123                     ret |= CPUID_MTRR | CPUID_PAT | CPUID_MCE | CPUID_MCA;
 124                     break;
 125                 case 0x80000001:
 126                     /* On Intel, kvm returns cpuid according to the Intel spec,
 127                      * so add missing bits according to the AMD spec:
 128                      */
 129                     cpuid_1_edx = kvm_arch_get_supported_cpuid(env, 1, 0, R_EDX);
 130                     ret |= cpuid_1_edx & 0x183f7ff;
 131                     break;
 132                 }
 133                 break;
 134             }
 135         }
 136     }
 137
 138     qemu_free(cpuid);
 139
 140     return ret;
 141 }
 142
 143 #else
 144
 145 uint32_t kvm_arch_get_supported_cpuid(CPUState *env, uint32_t function,
 146                                       uint32_t index, int reg)
 147 {
 148     return -1U;
 149 }
 150
 151 #endif
 152
 153 #ifdef CONFIG_KVM_PARA
 154 struct kvm_para_features {
 155         int cap;
 156         int feature;
 157 } para_features[] = {
 158 #ifdef KVM_CAP_CLOCKSOURCE
 159         { KVM_CAP_CLOCKSOURCE, KVM_FEATURE_CLOCKSOURCE },
 160 #endif
 161 #ifdef KVM_CAP_NOP_IO_DELAY
 162         { KVM_CAP_NOP_IO_DELAY, KVM_FEATURE_NOP_IO_DELAY },
 163 #endif
 164 #ifdef KVM_CAP_PV_MMU
 165         { KVM_CAP_PV_MMU, KVM_FEATURE_MMU_OP },
 166 #endif
 167         { -1, -1 }
 168 };
 169
 170 static int get_para_features(CPUState *env)
 171 {
 172         int i, features = 0;
 173
 174         for (i = 0; i < ARRAY_SIZE(para_features) - 1; i++) {
 175                 if (kvm_check_extension(env->kvm_state, para_features[i].cap))
 176                         features |= (1 << para_features[i].feature);
 177         }
 178
 179         return features;
 180 }
 181 #endif
 182
 183 #ifdef KVM_CAP_MCE
 184 static int kvm_get_mce_cap_supported(KVMState *s, uint64_t *mce_cap,
 185                                      int *max_banks)
 186 {
 187     int r;
 188
 189     r = kvm_ioctl(s, KVM_CHECK_EXTENSION, KVM_CAP_MCE);
 190     if (r > 0) {
 191         *max_banks = r;
 192         return kvm_ioctl(s, KVM_X86_GET_MCE_CAP_SUPPORTED, mce_cap);
 193     }
 194     return -ENOSYS;
 195 }
 196
 197 static int kvm_setup_mce(CPUState *env, uint64_t *mcg_cap)
 198 {
 199     return kvm_vcpu_ioctl(env, KVM_X86_SETUP_MCE, mcg_cap);
 200 }
 201
 202 static int kvm_set_mce(CPUState *env, struct kvm_x86_mce *m)
 203 {
 204     return kvm_vcpu_ioctl(env, KVM_X86_SET_MCE, m);
 205 }
 206
 207 static int kvm_get_msr(CPUState *env, struct kvm_msr_entry *msrs, int n)
 208 {
 209     struct kvm_msrs *kmsrs = qemu_malloc(sizeof *kmsrs + n * sizeof *msrs);
 210     int r;
 211
 212     kmsrs->nmsrs = n;
 213     memcpy(kmsrs->entries, msrs, n * sizeof *msrs);
 214     r = kvm_vcpu_ioctl(env, KVM_GET_MSRS, kmsrs);
 215     memcpy(msrs, kmsrs->entries, n * sizeof *msrs);
 216     free(kmsrs);
 217     return r;
 218 }
 219
 220 /* FIXME: kill this and kvm_get_msr, use env->mcg_status instead */
 221 static int kvm_mce_in_exception(CPUState *env)
 222 {
 223     struct kvm_msr_entry msr_mcg_status = {
 224         .index = MSR_MCG_STATUS,
 225     };
 226     int r;
 227
 228     r = kvm_get_msr(env, &msr_mcg_status, 1);
 229     if (r == -1 || r == 0) {
 230         return -1;
 231     }
 232     return !!(msr_mcg_status.data & MCG_STATUS_MCIP);
 233 }
 234
 235 struct kvm_x86_mce_data
 236 {
 237     CPUState *env;
 238     struct kvm_x86_mce *mce;
 239     int abort_on_error;
 240 };
 241
 242 static void kvm_do_inject_x86_mce(void *_data)
 243 {
 244     struct kvm_x86_mce_data *data = _data;
 245     int r;
 246
 247     /* If there is an MCE exception being processed, ignore this SRAO MCE */
 248     if ((data->env->mcg_cap & MCG_SER_P) &&
 249         !(data->mce->status & MCI_STATUS_AR)) {
 250         r = kvm_mce_in_exception(data->env);
 251         if (r == -1) {
 252             fprintf(stderr, "Failed to get MCE status\n");
 253         } else if (r) {
 254             return;
 255         }
 256     }
 257
 258     r = kvm_set_mce(data->env, data->mce);
 259     if (r < 0) {
 260         perror("kvm_set_mce FAILED");
 261         if (data->abort_on_error) {
 262             abort();
 263         }
 264     }
 265 }
 266 #endif
 267
 268 void kvm_inject_x86_mce(CPUState *cenv, int bank, uint64_t status,
 269                         uint64_t mcg_status, uint64_t addr, uint64_t misc,
 270                         int abort_on_error)
 271 {
 272 #ifdef KVM_CAP_MCE
 273     struct kvm_x86_mce mce = {
 274         .bank = bank,
 275         .status = status,
 276         .mcg_status = mcg_status,
 277         .addr = addr,
 278         .misc = misc,
 279     };
 280     struct kvm_x86_mce_data data = {
 281             .env = cenv,
 282             .mce = &mce,
 283     };
 284
 285     if (!cenv->mcg_cap) {
 286         fprintf(stderr, "MCE support is not enabled!\n");
 287         return;
 288     }
 289
 290     on_vcpu(cenv, kvm_do_inject_x86_mce, &data);
 291 #else
 292     if (abort_on_error)
 293         abort();
 294 #endif
 295 }
 296
 297 static int _kvm_arch_init_vcpu(CPUState *env);
 298
 299 int kvm_arch_init_vcpu(CPUState *env)
 300 {
 301     int r;
 302     struct {
 303         struct kvm_cpuid2 cpuid;
 304         struct kvm_cpuid_entry2 entries[100];
 305     } __attribute__((packed)) cpuid_data;
 306     uint32_t limit, i, j, cpuid_i;
 307     uint32_t unused;
 308     struct kvm_cpuid_entry2 *c;
 309 #ifdef KVM_CPUID_SIGNATURE
 310     uint32_t signature[3];
 311 #endif
 312
 313     r = _kvm_arch_init_vcpu(env);
 314     if (r < 0) {
 315         return r;
 316     }
 317
 318 #ifdef OBSOLETE_KVM_IMPL
 319
 320     env->mp_state = KVM_MP_STATE_RUNNABLE;
 321
 322 #endif
 323
 324     env->cpuid_features &= kvm_arch_get_supported_cpuid(env, 1, 0, R_EDX);
 325
 326     i = env->cpuid_ext_features & CPUID_EXT_HYPERVISOR;
 327     env->cpuid_ext_features &= kvm_arch_get_supported_cpuid(env, 1, 0, R_ECX);
 328     env->cpuid_ext_features |= i;
 329
 330     env->cpuid_ext2_features &= kvm_arch_get_supported_cpuid(env, 0x80000001,
 331                                                              0, R_EDX);
 332     env->cpuid_ext3_features &= kvm_arch_get_supported_cpuid(env, 0x80000001,
 333                                                              0, R_ECX);
 334     env->cpuid_svm_features  &= kvm_arch_get_supported_cpuid(env, 0x8000000A,
 335                                                              0, R_EDX);
 336
 337
 338     cpuid_i = 0;
 339
 340 #ifdef CONFIG_KVM_PARA
 341     /* Paravirtualization CPUIDs */
 342     memcpy(signature, "KVMKVMKVM\0\0\0", 12);
 343     c = &cpuid_data.entries[cpuid_i++];
 344     memset(c, 0, sizeof(*c));
 345     c->function = KVM_CPUID_SIGNATURE;
 346     c->eax = 0;
 347     c->ebx = signature[0];
 348     c->ecx = signature[1];
 349     c->edx = signature[2];
 350
 351     c = &cpuid_data.entries[cpuid_i++];
 352     memset(c, 0, sizeof(*c));
 353     c->function = KVM_CPUID_FEATURES;
 354     c->eax = env->cpuid_kvm_features & get_para_features(env);
 355 #endif
 356
 357     cpu_x86_cpuid(env, 0, 0, &limit, &unused, &unused, &unused);
 358
 359     for (i = 0; i <= limit; i++) {
 360         c = &cpuid_data.entries[cpuid_i++];
 361
 362         switch (i) {
 363         case 2: {
 364             /* Keep reading function 2 till all the input is received */
 365             int times;
 366
 367             c->function = i;
 368             c->flags = KVM_CPUID_FLAG_STATEFUL_FUNC |
 369                        KVM_CPUID_FLAG_STATE_READ_NEXT;
 370             cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 371             times = c->eax & 0xff;
 372
 373             for (j = 1; j < times; ++j) {
 374                 c = &cpuid_data.entries[cpuid_i++];
 375                 c->function = i;
 376                 c->flags = KVM_CPUID_FLAG_STATEFUL_FUNC;
 377                 cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 378             }
 379             break;
 380         }
 381         case 4:
 382         case 0xb:
 383         case 0xd:
 384             for (j = 0; ; j++) {
 385                 c->function = i;
 386                 c->flags = KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
 387                 c->index = j;
 388                 cpu_x86_cpuid(env, i, j, &c->eax, &c->ebx, &c->ecx, &c->edx);
 389
 390                 if (i == 4 && c->eax == 0)
 391                     break;
 392                 if (i == 0xb && !(c->ecx & 0xff00))
 393                     break;
 394                 if (i == 0xd && c->eax == 0)
 395                     break;
 396
 397                 c = &cpuid_data.entries[cpuid_i++];
 398             }
 399             break;
 400         default:
 401             c->function = i;
 402             c->flags = 0;
 403             cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 404             break;
 405         }
 406     }
 407     cpu_x86_cpuid(env, 0x80000000, 0, &limit, &unused, &unused, &unused);
 408
 409     for (i = 0x80000000; i <= limit; i++) {
 410         c = &cpuid_data.entries[cpuid_i++];
 411
 412         c->function = i;
 413         c->flags = 0;
 414         cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 415     }
 416
 417     cpuid_data.cpuid.nent = cpuid_i;
 418
 419 #ifdef KVM_CAP_MCE
 420     if (((env->cpuid_version >> 8)&0xF) >= 6
 421         && (env->cpuid_features&(CPUID_MCE|CPUID_MCA)) == (CPUID_MCE|CPUID_MCA)
 422         && kvm_check_extension(env->kvm_state, KVM_CAP_MCE) > 0) {
 423         uint64_t mcg_cap;
 424         int banks;
 425
 426         if (kvm_get_mce_cap_supported(env->kvm_state, &mcg_cap, &banks))
 427             perror("kvm_get_mce_cap_supported FAILED");
 428         else {
 429             if (banks > MCE_BANKS_DEF)
 430                 banks = MCE_BANKS_DEF;
 431             mcg_cap &= MCE_CAP_DEF;
 432             mcg_cap |= banks;
 433             if (kvm_setup_mce(env, &mcg_cap))
 434                 perror("kvm_setup_mce FAILED");
 435             else
 436                 env->mcg_cap = mcg_cap;
 437         }
 438     }
 439 #endif
 440
 441     return kvm_vcpu_ioctl(env, KVM_SET_CPUID2, &cpuid_data);
 442 }
 443
 444 void kvm_arch_reset_vcpu(CPUState *env)
 445 {
 446     env->exception_injected = -1;
 447     env->interrupt_injected = -1;
 448     env->nmi_injected = 0;
 449     env->nmi_pending = 0;
 450     /* Legal xcr0 for loading */
 451     env->xcr0 = 1;
 452     if (kvm_irqchip_in_kernel()) {
 453         env->mp_state = cpu_is_bsp(env) ? KVM_MP_STATE_RUNNABLE :
 454                                           KVM_MP_STATE_UNINITIALIZED;
 455     } else {
 456         env->mp_state = KVM_MP_STATE_RUNNABLE;
 457     }
 458 }
 459 #ifdef OBSOLETE_KVM_IMPL
 460
 461 int has_msr_star;
 462 int has_msr_hsave_pa;
 463
 464 static void kvm_supported_msrs(CPUState *env)
 465 {
 466     static int kvm_supported_msrs;
 467     int ret;
 468
 469     /* first time */
 470     if (kvm_supported_msrs == 0) {
 471         struct kvm_msr_list msr_list, *kvm_msr_list;
 472
 473         kvm_supported_msrs = -1;
 474
 475         /* Obtain MSR list from KVM.  These are the MSRs that we must
 476          * save/restore */
 477         msr_list.nmsrs = 0;
 478         ret = kvm_ioctl(env->kvm_state, KVM_GET_MSR_INDEX_LIST, &msr_list);
 479         if (ret < 0 && ret != -E2BIG) {
 480             return;
 481         }
 482         /* Old kernel modules had a bug and could write beyond the provided
 483            memory. Allocate at least a safe amount of 1K. */
 484         kvm_msr_list = qemu_mallocz(MAX(1024, sizeof(msr_list) +
 485                                               msr_list.nmsrs *
 486                                               sizeof(msr_list.indices[0])));
 487
 488         kvm_msr_list->nmsrs = msr_list.nmsrs;
 489         ret = kvm_ioctl(env->kvm_state, KVM_GET_MSR_INDEX_LIST, kvm_msr_list);
 490         if (ret >= 0) {
 491             int i;
 492
 493             for (i = 0; i < kvm_msr_list->nmsrs; i++) {
 494                 if (kvm_msr_list->indices[i] == MSR_STAR) {
 495                     has_msr_star = 1;
 496                     continue;
 497                 }
 498                 if (kvm_msr_list->indices[i] == MSR_VM_HSAVE_PA) {
 499                     has_msr_hsave_pa = 1;
 500                     continue;
 501                 }
 502             }
 503         }
 504
 505         free(kvm_msr_list);
 506     }
 507
 508     return;
 509 }
 510
 511 static int kvm_has_msr_hsave_pa(CPUState *env)
 512 {
 513     kvm_supported_msrs(env);
 514     return has_msr_hsave_pa;
 515 }
 516
 517 static int kvm_has_msr_star(CPUState *env)
 518 {
 519     kvm_supported_msrs(env);
 520     return has_msr_star;
 521 }
 522
 523 static int kvm_init_identity_map_page(KVMState *s)
 524 {
 525 #ifdef KVM_CAP_SET_IDENTITY_MAP_ADDR
 526     int ret;
 527     uint64_t addr = 0xfffbc000;
 528
 529     if (!kvm_check_extension(s, KVM_CAP_SET_IDENTITY_MAP_ADDR)) {
 530         return 0;
 531     }
 532
 533     ret = kvm_vm_ioctl(s, KVM_SET_IDENTITY_MAP_ADDR, &addr);
 534     if (ret < 0) {
 535         fprintf(stderr, "kvm_set_identity_map_addr: %s\n", strerror(ret));
 536         return ret;
 537     }
 538 #endif
 539     return 0;
 540 }
 541
 542 int kvm_arch_init(KVMState *s, int smp_cpus)
 543 {
 544     int ret;
 545
 546     struct utsname utsname;
 547
 548     uname(&utsname);
 549     lm_capable_kernel = strcmp(utsname.machine, "x86_64") == 0;
 550
 551     /* create vm86 tss.  KVM uses vm86 mode to emulate 16-bit code
 552      * directly.  In order to use vm86 mode, a TSS is needed.  Since this
 553      * must be part of guest physical memory, we need to allocate it.  Older
 554      * versions of KVM just assumed that it would be at the end of physical
 555      * memory but that doesn't work with more than 4GB of memory.  We simply
 556      * refuse to work with those older versions of KVM. */
 557     ret = kvm_ioctl(s, KVM_CHECK_EXTENSION, KVM_CAP_SET_TSS_ADDR);
 558     if (ret <= 0) {
 559         fprintf(stderr, "kvm does not support KVM_CAP_SET_TSS_ADDR\n");
 560         return ret;
 561     }
 562
 563     /* this address is 3 pages before the bios, and the bios should present
 564      * as unavaible memory.  FIXME, need to ensure the e820 map deals with
 565      * this?
 566      */
 567     /*
 568      * Tell fw_cfg to notify the BIOS to reserve the range.
 569      */
 570     if (e820_add_entry(0xfffbc000, 0x4000, E820_RESERVED) < 0) {
 571         perror("e820_add_entry() table is full");
 572         exit(1);
 573     }
 574     ret = kvm_vm_ioctl(s, KVM_SET_TSS_ADDR, 0xfffbd000);
 575     if (ret < 0) {
 576         return ret;
 577     }
 578
 579     return kvm_init_identity_map_page(s);
 580 }
 581
 582 #endif
 583
 584 static void set_v8086_seg(struct kvm_segment *lhs, const SegmentCache *rhs)
 585 {
 586     lhs->selector = rhs->selector;
 587     lhs->base = rhs->base;
 588     lhs->limit = rhs->limit;
 589     lhs->type = 3;
 590     lhs->present = 1;
 591     lhs->dpl = 3;
 592     lhs->db = 0;
 593     lhs->s = 1;
 594     lhs->l = 0;
 595     lhs->g = 0;
 596     lhs->avl = 0;
 597     lhs->unusable = 0;
 598 }
 599
 600 static void set_seg(struct kvm_segment *lhs, const SegmentCache *rhs)
 601 {
 602     unsigned flags = rhs->flags;
 603     lhs->selector = rhs->selector;
 604     lhs->base = rhs->base;
 605     lhs->limit = rhs->limit;
 606     lhs->type = (flags >> DESC_TYPE_SHIFT) & 15;
 607     lhs->present = (flags & DESC_P_MASK) != 0;
 608     lhs->dpl = rhs->selector & 3;
 609     lhs->db = (flags >> DESC_B_SHIFT) & 1;
 610     lhs->s = (flags & DESC_S_MASK) != 0;
 611     lhs->l = (flags >> DESC_L_SHIFT) & 1;
 612     lhs->g = (flags & DESC_G_MASK) != 0;
 613     lhs->avl = (flags & DESC_AVL_MASK) != 0;
 614     lhs->unusable = 0;
 615 }
 616
 617 static void get_seg(SegmentCache *lhs, const struct kvm_segment *rhs)
 618 {
 619     lhs->selector = rhs->selector;
 620     lhs->base = rhs->base;
 621     lhs->limit = rhs->limit;
 622     lhs->flags =
 623         (rhs->type << DESC_TYPE_SHIFT)
 624         | (rhs->present * DESC_P_MASK)
 625         | (rhs->dpl << DESC_DPL_SHIFT)
 626         | (rhs->db << DESC_B_SHIFT)
 627         | (rhs->s * DESC_S_MASK)
 628         | (rhs->l << DESC_L_SHIFT)
 629         | (rhs->g * DESC_G_MASK)
 630         | (rhs->avl * DESC_AVL_MASK);
 631 }
 632
 633 #ifdef OBSOLETE_KVM_IMPL
 634
 635 static void kvm_getput_reg(__u64 *kvm_reg, target_ulong *qemu_reg, int set)
 636 {
 637     if (set)
 638         *kvm_reg = *qemu_reg;
 639     else
 640         *qemu_reg = *kvm_reg;
 641 }
 642
 643 static int kvm_getput_regs(CPUState *env, int set)
 644 {
 645     struct kvm_regs regs;
 646     int ret = 0;
 647
 648     if (!set) {
 649         ret = kvm_vcpu_ioctl(env, KVM_GET_REGS, &regs);
 650         if (ret < 0)
 651             return ret;
 652     }
 653
 654     kvm_getput_reg(&regs.rax, &env->regs[R_EAX], set);
 655     kvm_getput_reg(&regs.rbx, &env->regs[R_EBX], set);
 656     kvm_getput_reg(&regs.rcx, &env->regs[R_ECX], set);
 657     kvm_getput_reg(&regs.rdx, &env->regs[R_EDX], set);
 658     kvm_getput_reg(&regs.rsi, &env->regs[R_ESI], set);
 659     kvm_getput_reg(&regs.rdi, &env->regs[R_EDI], set);
 660     kvm_getput_reg(&regs.rsp, &env->regs[R_ESP], set);
 661     kvm_getput_reg(&regs.rbp, &env->regs[R_EBP], set);
 662 #ifdef TARGET_X86_64
 663     kvm_getput_reg(&regs.r8, &env->regs[8], set);
 664     kvm_getput_reg(&regs.r9, &env->regs[9], set);
 665     kvm_getput_reg(&regs.r10, &env->regs[10], set);
 666     kvm_getput_reg(&regs.r11, &env->regs[11], set);
 667     kvm_getput_reg(&regs.r12, &env->regs[12], set);
 668     kvm_getput_reg(&regs.r13, &env->regs[13], set);
 669     kvm_getput_reg(&regs.r14, &env->regs[14], set);
 670     kvm_getput_reg(&regs.r15, &env->regs[15], set);
 671 #endif
 672
 673     kvm_getput_reg(&regs.rflags, &env->eflags, set);
 674     kvm_getput_reg(&regs.rip, &env->eip, set);
 675
 676     if (set)
 677         ret = kvm_vcpu_ioctl(env, KVM_SET_REGS, &regs);
 678
 679     return ret;
 680 }
 681
 682 #endif
 683
 684 static int kvm_put_fpu(CPUState *env)
 685 {
 686     struct kvm_fpu fpu;
 687     int i;
 688
 689     memset(&fpu, 0, sizeof fpu);
 690     fpu.fsw = env->fpus & ~(7 << 11);
 691     fpu.fsw |= (env->fpstt & 7) << 11;
 692     fpu.fcw = env->fpuc;
 693     for (i = 0; i < 8; ++i)
 694         fpu.ftwx |= (!env->fptags[i]) << i;
 695     memcpy(fpu.fpr, env->fpregs, sizeof env->fpregs);
 696     memcpy(fpu.xmm, env->xmm_regs, sizeof env->xmm_regs);
 697     fpu.mxcsr = env->mxcsr;
 698
 699     return kvm_vcpu_ioctl(env, KVM_SET_FPU, &fpu);
 700 }
 701
 702 #ifdef KVM_CAP_XSAVE
 703 #define XSAVE_CWD_RIP     2
 704 #define XSAVE_CWD_RDP     4
 705 #define XSAVE_MXCSR       6
 706 #define XSAVE_ST_SPACE    8
 707 #define XSAVE_XMM_SPACE   40
 708 #define XSAVE_XSTATE_BV   128
 709 #define XSAVE_YMMH_SPACE  144
 710 #endif
 711
 712 static int kvm_put_xsave(CPUState *env)
 713 {
 714 #ifdef KVM_CAP_XSAVE
 715     int i, r;
 716     struct kvm_xsave* xsave;
 717     uint16_t cwd, swd, twd, fop;
 718
 719     if (!kvm_has_xsave())
 720         return kvm_put_fpu(env);
 721
 722     xsave = qemu_memalign(4096, sizeof(struct kvm_xsave));
 723     memset(xsave, 0, sizeof(struct kvm_xsave));
 724     cwd = swd = twd = fop = 0;
 725     swd = env->fpus & ~(7 << 11);
 726     swd |= (env->fpstt & 7) << 11;
 727     cwd = env->fpuc;
 728     for (i = 0; i < 8; ++i)
 729         twd |= (!env->fptags[i]) << i;
 730     xsave->region[0] = (uint32_t)(swd << 16) + cwd;
 731     xsave->region[1] = (uint32_t)(fop << 16) + twd;
 732     memcpy(&xsave->region[XSAVE_ST_SPACE], env->fpregs,
 733             sizeof env->fpregs);
 734     memcpy(&xsave->region[XSAVE_XMM_SPACE], env->xmm_regs,
 735             sizeof env->xmm_regs);
 736     xsave->region[XSAVE_MXCSR] = env->mxcsr;
 737     *(uint64_t *)&xsave->region[XSAVE_XSTATE_BV] = env->xstate_bv;
 738     memcpy(&xsave->region[XSAVE_YMMH_SPACE], env->ymmh_regs,
 739             sizeof env->ymmh_regs);
 740     r = kvm_vcpu_ioctl(env, KVM_SET_XSAVE, xsave);
 741     qemu_free(xsave);
 742     return r;
 743 #else
 744     return kvm_put_fpu(env);
 745 #endif
 746 }
 747
 748 static int kvm_put_xcrs(CPUState *env)
 749 {
 750 #ifdef KVM_CAP_XCRS
 751     struct kvm_xcrs xcrs;
 752
 753     if (!kvm_has_xcrs())
 754         return 0;
 755
 756     xcrs.nr_xcrs = 1;
 757     xcrs.flags = 0;
 758     xcrs.xcrs[0].xcr = 0;
 759     xcrs.xcrs[0].value = env->xcr0;
 760     return kvm_vcpu_ioctl(env, KVM_SET_XCRS, &xcrs);
 761 #else
 762     return 0;
 763 #endif
 764 }
 765
 766 static int kvm_put_sregs(CPUState *env)
 767 {
 768     struct kvm_sregs sregs;
 769
 770     memset(sregs.interrupt_bitmap, 0, sizeof(sregs.interrupt_bitmap));
 771     if (env->interrupt_injected >= 0) {
 772         sregs.interrupt_bitmap[env->interrupt_injected / 64] |=
 773                 (uint64_t)1 << (env->interrupt_injected % 64);
 774     }
 775
 776     if ((env->eflags & VM_MASK)) {
 777             set_v8086_seg(&sregs.cs, &env->segs[R_CS]);
 778             set_v8086_seg(&sregs.ds, &env->segs[R_DS]);
 779             set_v8086_seg(&sregs.es, &env->segs[R_ES]);
 780             set_v8086_seg(&sregs.fs, &env->segs[R_FS]);
 781             set_v8086_seg(&sregs.gs, &env->segs[R_GS]);
 782             set_v8086_seg(&sregs.ss, &env->segs[R_SS]);
 783     } else {
 784             set_seg(&sregs.cs, &env->segs[R_CS]);
 785             set_seg(&sregs.ds, &env->segs[R_DS]);
 786             set_seg(&sregs.es, &env->segs[R_ES]);
 787             set_seg(&sregs.fs, &env->segs[R_FS]);
 788             set_seg(&sregs.gs, &env->segs[R_GS]);
 789             set_seg(&sregs.ss, &env->segs[R_SS]);
 790
 791             if (env->cr[0] & CR0_PE_MASK) {
 792                 /* force ss cpl to cs cpl */
 793                 sregs.ss.selector = (sregs.ss.selector & ~3) |
 794                         (sregs.cs.selector & 3);
 795                 sregs.ss.dpl = sregs.ss.selector & 3;
 796             }
 797     }
 798
 799     set_seg(&sregs.tr, &env->tr);
 800     set_seg(&sregs.ldt, &env->ldt);
 801
 802     sregs.idt.limit = env->idt.limit;
 803     sregs.idt.base = env->idt.base;
 804     sregs.gdt.limit = env->gdt.limit;
 805     sregs.gdt.base = env->gdt.base;
 806
 807     sregs.cr0 = env->cr[0];
 808     sregs.cr2 = env->cr[2];
 809     sregs.cr3 = env->cr[3];
 810     sregs.cr4 = env->cr[4];
 811
 812     sregs.cr8 = cpu_get_apic_tpr(env->apic_state);
 813     sregs.apic_base = cpu_get_apic_base(env->apic_state);
 814
 815     sregs.efer = env->efer;
 816
 817     return kvm_vcpu_ioctl(env, KVM_SET_SREGS, &sregs);
 818 }
 819
 820 static void kvm_msr_entry_set(struct kvm_msr_entry *entry,
 821                               uint32_t index, uint64_t value)
 822 {
 823     entry->index = index;
 824     entry->data = value;
 825 }
 826
 827 #ifdef OBSOLETE_KVM_IMPL
 828 static int kvm_put_msrs(CPUState *env, int level)
 829 {
 830     struct {
 831         struct kvm_msrs info;
 832         struct kvm_msr_entry entries[100];
 833     } msr_data;
 834     struct kvm_msr_entry *msrs = msr_data.entries;
 835     int n = 0;
 836
 837     kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_CS, env->sysenter_cs);
 838     kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_ESP, env->sysenter_esp);
 839     kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_EIP, env->sysenter_eip);
 840     if (kvm_has_msr_star(env))
 841         kvm_msr_entry_set(&msrs[n++], MSR_STAR, env->star);
 842     if (kvm_has_msr_hsave_pa(env))
 843         kvm_msr_entry_set(&msrs[n++], MSR_VM_HSAVE_PA, env->vm_hsave);
 844 #ifdef TARGET_X86_64
 845     if (lm_capable_kernel) {
 846         kvm_msr_entry_set(&msrs[n++], MSR_CSTAR, env->cstar);
 847         kvm_msr_entry_set(&msrs[n++], MSR_KERNELGSBASE, env->kernelgsbase);
 848         kvm_msr_entry_set(&msrs[n++], MSR_FMASK, env->fmask);
 849         kvm_msr_entry_set(&msrs[n++], MSR_LSTAR, env->lstar);
 850     }
 851 #endif
 852     if (level == KVM_PUT_FULL_STATE) {
 853         /*
 854          * KVM is yet unable to synchronize TSC values of multiple VCPUs on
 855          * writeback. Until this is fixed, we only write the offset to SMP
 856          * guests after migration, desynchronizing the VCPUs, but avoiding
 857          * huge jump-backs that would occur without any writeback at all.
 858          */
 859         if (smp_cpus == 1 || env->tsc != 0) {
 860             kvm_msr_entry_set(&msrs[n++], MSR_IA32_TSC, env->tsc);
 861         }
 862         kvm_msr_entry_set(&msrs[n++], MSR_KVM_SYSTEM_TIME,
 863                           env->system_time_msr);
 864         kvm_msr_entry_set(&msrs[n++], MSR_KVM_WALL_CLOCK, env->wall_clock_msr);
 865     }
 866 #ifdef KVM_CAP_MCE
 867     if (env->mcg_cap) {
 868         int i;
 869         if (level == KVM_PUT_RESET_STATE)
 870             kvm_msr_entry_set(&msrs[n++], MSR_MCG_STATUS, env->mcg_status);
 871         else if (level == KVM_PUT_FULL_STATE) {
 872             kvm_msr_entry_set(&msrs[n++], MSR_MCG_STATUS, env->mcg_status);
 873             kvm_msr_entry_set(&msrs[n++], MSR_MCG_CTL, env->mcg_ctl);
 874             for (i = 0; i < (env->mcg_cap & 0xff) * 4; i++)
 875                 kvm_msr_entry_set(&msrs[n++], MSR_MC0_CTL + i, env->mce_banks[i]);
 876         }
 877     }
 878 #endif
 879
 880     msr_data.info.nmsrs = n;
 881
 882     return kvm_vcpu_ioctl(env, KVM_SET_MSRS, &msr_data);
 883
 884 }
 885
 886 #endif
 887
 888 static int kvm_get_fpu(CPUState *env)
 889 {
 890     struct kvm_fpu fpu;
 891     int i, ret;
 892
 893     ret = kvm_vcpu_ioctl(env, KVM_GET_FPU, &fpu);
 894     if (ret < 0)
 895         return ret;
 896
 897     env->fpstt = (fpu.fsw >> 11) & 7;
 898     env->fpus = fpu.fsw;
 899     env->fpuc = fpu.fcw;
 900     for (i = 0; i < 8; ++i)
 901         env->fptags[i] = !((fpu.ftwx >> i) & 1);
 902     memcpy(env->fpregs, fpu.fpr, sizeof env->fpregs);
 903     memcpy(env->xmm_regs, fpu.xmm, sizeof env->xmm_regs);
 904     env->mxcsr = fpu.mxcsr;
 905
 906     return 0;
 907 }
 908
 909 static int kvm_get_xsave(CPUState *env)
 910 {
 911 #ifdef KVM_CAP_XSAVE
 912     struct kvm_xsave* xsave;
 913     int ret, i;
 914     uint16_t cwd, swd, twd, fop;
 915
 916     if (!kvm_has_xsave())
 917         return kvm_get_fpu(env);
 918
 919     xsave = qemu_memalign(4096, sizeof(struct kvm_xsave));
 920     ret = kvm_vcpu_ioctl(env, KVM_GET_XSAVE, xsave);
 921     if (ret < 0) {
 922         qemu_free(xsave);
 923         return ret;
 924     }
 925
 926     cwd = (uint16_t)xsave->region[0];
 927     swd = (uint16_t)(xsave->region[0] >> 16);
 928     twd = (uint16_t)xsave->region[1];
 929     fop = (uint16_t)(xsave->region[1] >> 16);
 930     env->fpstt = (swd >> 11) & 7;
 931     env->fpus = swd;
 932     env->fpuc = cwd;
 933     for (i = 0; i < 8; ++i)
 934         env->fptags[i] = !((twd >> i) & 1);
 935     env->mxcsr = xsave->region[XSAVE_MXCSR];
 936     memcpy(env->fpregs, &xsave->region[XSAVE_ST_SPACE],
 937             sizeof env->fpregs);
 938     memcpy(env->xmm_regs, &xsave->region[XSAVE_XMM_SPACE],
 939             sizeof env->xmm_regs);
 940     env->xstate_bv = *(uint64_t *)&xsave->region[XSAVE_XSTATE_BV];
 941     memcpy(env->ymmh_regs, &xsave->region[XSAVE_YMMH_SPACE],
 942             sizeof env->ymmh_regs);
 943     qemu_free(xsave);
 944     return 0;
 945 #else
 946     return kvm_get_fpu(env);
 947 #endif
 948 }
 949
 950 static int kvm_get_xcrs(CPUState *env)
 951 {
 952 #ifdef KVM_CAP_XCRS
 953     int i, ret;
 954     struct kvm_xcrs xcrs;
 955
 956     if (!kvm_has_xcrs())
 957         return 0;
 958
 959     ret = kvm_vcpu_ioctl(env, KVM_GET_XCRS, &xcrs);
 960     if (ret < 0)
 961         return ret;
 962
 963     for (i = 0; i < xcrs.nr_xcrs; i++)
 964         /* Only support xcr0 now */
 965         if (xcrs.xcrs[0].xcr == 0) {
 966             env->xcr0 = xcrs.xcrs[0].value;
 967             break;
 968         }
 969     return 0;
 970 #else
 971     return 0;
 972 #endif
 973 }
 974
 975 static int kvm_get_sregs(CPUState *env)
 976 {
 977     struct kvm_sregs sregs;
 978     uint32_t hflags;
 979     int bit, i, ret;
 980
 981     ret = kvm_vcpu_ioctl(env, KVM_GET_SREGS, &sregs);
 982     if (ret < 0)
 983         return ret;
 984
 985     /* There can only be one pending IRQ set in the bitmap at a time, so try
 986        to find it and save its number instead (-1 for none). */
 987     env->interrupt_injected = -1;
 988     for (i = 0; i < ARRAY_SIZE(sregs.interrupt_bitmap); i++) {
 989         if (sregs.interrupt_bitmap[i]) {
 990             bit = ctz64(sregs.interrupt_bitmap[i]);
 991             env->interrupt_injected = i * 64 + bit;
 992             break;
 993         }
 994     }
 995
 996     get_seg(&env->segs[R_CS], &sregs.cs);
 997     get_seg(&env->segs[R_DS], &sregs.ds);
 998     get_seg(&env->segs[R_ES], &sregs.es);
 999     get_seg(&env->segs[R_FS], &sregs.fs);
1000     get_seg(&env->segs[R_GS], &sregs.gs);
1001     get_seg(&env->segs[R_SS], &sregs.ss);
1002
1003     get_seg(&env->tr, &sregs.tr);
1004     get_seg(&env->ldt, &sregs.ldt);
1005
1006     env->idt.limit = sregs.idt.limit;
1007     env->idt.base = sregs.idt.base;
1008     env->gdt.limit = sregs.gdt.limit;
1009     env->gdt.base = sregs.gdt.base;
1010
1011     env->cr[0] = sregs.cr0;
1012     env->cr[2] = sregs.cr2;
1013     env->cr[3] = sregs.cr3;
1014     env->cr[4] = sregs.cr4;
1015
1016     cpu_set_apic_base(env->apic_state, sregs.apic_base);
1017
1018     env->efer = sregs.efer;
1019     //cpu_set_apic_tpr(env->apic_state, sregs.cr8);
1020
1021 #define HFLAG_COPY_MASK ~( \
1022                         HF_CPL_MASK | HF_PE_MASK | HF_MP_MASK | HF_EM_MASK | \
1023                         HF_TS_MASK | HF_TF_MASK | HF_VM_MASK | HF_IOPL_MASK | \
1024                         HF_OSFXSR_MASK | HF_LMA_MASK | HF_CS32_MASK | \
1025                         HF_SS32_MASK | HF_CS64_MASK | HF_ADDSEG_MASK)
1026
1027
1028
1029     hflags = (env->segs[R_CS].flags >> DESC_DPL_SHIFT) & HF_CPL_MASK;
1030     hflags |= (env->cr[0] & CR0_PE_MASK) << (HF_PE_SHIFT - CR0_PE_SHIFT);
1031     hflags |= (env->cr[0] << (HF_MP_SHIFT - CR0_MP_SHIFT)) &
1032             (HF_MP_MASK | HF_EM_MASK | HF_TS_MASK);
1033     hflags |= (env->eflags & (HF_TF_MASK | HF_VM_MASK | HF_IOPL_MASK));
1034     hflags |= (env->cr[4] & CR4_OSFXSR_MASK) <<
1035             (HF_OSFXSR_SHIFT - CR4_OSFXSR_SHIFT);
1036
1037     if (env->efer & MSR_EFER_LMA) {
1038         hflags |= HF_LMA_MASK;
1039     }
1040
1041     if ((hflags & HF_LMA_MASK) && (env->segs[R_CS].flags & DESC_L_MASK)) {
1042         hflags |= HF_CS32_MASK | HF_SS32_MASK | HF_CS64_MASK;
1043     } else {
1044         hflags |= (env->segs[R_CS].flags & DESC_B_MASK) >>
1045                 (DESC_B_SHIFT - HF_CS32_SHIFT);
1046         hflags |= (env->segs[R_SS].flags & DESC_B_MASK) >>
1047                 (DESC_B_SHIFT - HF_SS32_SHIFT);
1048         if (!(env->cr[0] & CR0_PE_MASK) ||
1049                    (env->eflags & VM_MASK) ||
1050                    !(hflags & HF_CS32_MASK)) {
1051                 hflags |= HF_ADDSEG_MASK;
1052             } else {
1053                 hflags |= ((env->segs[R_DS].base |
1054                                 env->segs[R_ES].base |
1055                                 env->segs[R_SS].base) != 0) <<
1056                     HF_ADDSEG_SHIFT;
1057             }
1058     }
1059     env->hflags = (env->hflags & HFLAG_COPY_MASK) | hflags;
1060
1061     return 0;
1062 }
1063
1064 #ifdef OBSOLETE_KVM_IMPL
1065
1066 static int kvm_get_msrs(CPUState *env)
1067 {
1068     struct {
1069         struct kvm_msrs info;
1070         struct kvm_msr_entry entries[100];
1071     } msr_data;
1072     struct kvm_msr_entry *msrs = msr_data.entries;
1073     int ret, i, n;
1074
1075     n = 0;
1076     msrs[n++].index = MSR_IA32_SYSENTER_CS;
1077     msrs[n++].index = MSR_IA32_SYSENTER_ESP;
1078     msrs[n++].index = MSR_IA32_SYSENTER_EIP;
1079     if (kvm_has_msr_star(env))
1080         msrs[n++].index = MSR_STAR;
1081     if (kvm_has_msr_hsave_pa(env))
1082         msrs[n++].index = MSR_VM_HSAVE_PA;
1083     msrs[n++].index = MSR_IA32_TSC;
1084     msrs[n++].index = MSR_VM_HSAVE_PA;
1085 #ifdef TARGET_X86_64
1086     if (lm_capable_kernel) {
1087         msrs[n++].index = MSR_CSTAR;
1088         msrs[n++].index = MSR_KERNELGSBASE;
1089         msrs[n++].index = MSR_FMASK;
1090         msrs[n++].index = MSR_LSTAR;
1091     }
1092 #endif
1093     msrs[n++].index = MSR_KVM_SYSTEM_TIME;
1094     msrs[n++].index = MSR_KVM_WALL_CLOCK;
1095
1096 #ifdef KVM_CAP_MCE
1097     if (env->mcg_cap) {
1098         msrs[n++].index = MSR_MCG_STATUS;
1099         msrs[n++].index = MSR_MCG_CTL;
1100         for (i = 0; i < (env->mcg_cap & 0xff) * 4; i++)
1101             msrs[n++].index = MSR_MC0_CTL + i;
1102     }
1103 #endif
1104
1105     msr_data.info.nmsrs = n;
1106     ret = kvm_vcpu_ioctl(env, KVM_GET_MSRS, &msr_data);
1107     if (ret < 0)
1108         return ret;
1109
1110     for (i = 0; i < ret; i++) {
1111         switch (msrs[i].index) {
1112         case MSR_IA32_SYSENTER_CS:
1113             env->sysenter_cs = msrs[i].data;
1114             break;
1115         case MSR_IA32_SYSENTER_ESP:
1116             env->sysenter_esp = msrs[i].data;
1117             break;
1118         case MSR_IA32_SYSENTER_EIP:
1119             env->sysenter_eip = msrs[i].data;
1120             break;
1121         case MSR_STAR:
1122             env->star = msrs[i].data;
1123             break;
1124 #ifdef TARGET_X86_64
1125         case MSR_CSTAR:
1126             env->cstar = msrs[i].data;
1127             break;
1128         case MSR_KERNELGSBASE:
1129             env->kernelgsbase = msrs[i].data;
1130             break;
1131         case MSR_FMASK:
1132             env->fmask = msrs[i].data;
1133             break;
1134         case MSR_LSTAR:
1135             env->lstar = msrs[i].data;
1136             break;
1137 #endif
1138         case MSR_IA32_TSC:
1139             env->tsc = msrs[i].data;
1140             break;
1141         case MSR_VM_HSAVE_PA:
1142             env->vm_hsave = msrs[i].data;
1143             break;
1144         case MSR_KVM_SYSTEM_TIME:
1145             env->system_time_msr = msrs[i].data;
1146             break;
1147         case MSR_KVM_WALL_CLOCK:
1148             env->wall_clock_msr = msrs[i].data;
1149             break;
1150         case MSR_VM_HSAVE_PA:
1151             env->vm_hsave = msrs[i].data;
1152             break;
1153 #ifdef KVM_CAP_MCE
1154         case MSR_MCG_STATUS:
1155             env->mcg_status = msrs[i].data;
1156             break;
1157         case MSR_MCG_CTL:
1158             env->mcg_ctl = msrs[i].data;
1159             break;
1160 #endif
1161         default:
1162 #ifdef KVM_CAP_MCE
1163             if (msrs[i].index >= MSR_MC0_CTL &&
1164                 msrs[i].index < MSR_MC0_CTL + (env->mcg_cap & 0xff) * 4) {
1165                 env->mce_banks[msrs[i].index - MSR_MC0_CTL] = msrs[i].data;
1166             }
1167 #endif
1168             break;
1169         }
1170     }
1171
1172     return 0;
1173 }
1174
1175 static int kvm_put_mp_state(CPUState *env)
1176 {
1177     struct kvm_mp_state mp_state = { .mp_state = env->mp_state };
1178
1179     return kvm_vcpu_ioctl(env, KVM_SET_MP_STATE, &mp_state);
1180 }
1181
1182 static int kvm_get_mp_state(CPUState *env)
1183 {
1184     struct kvm_mp_state mp_state;
1185     int ret;
1186
1187     ret = kvm_vcpu_ioctl(env, KVM_GET_MP_STATE, &mp_state);
1188     if (ret < 0) {
1189         return ret;
1190     }
1191     env->mp_state = mp_state.mp_state;
1192     return 0;
1193 }
1194 #endif
1195
1196 static int kvm_put_vcpu_events(CPUState *env, int level)
1197 {
1198 #ifdef KVM_CAP_VCPU_EVENTS
1199     struct kvm_vcpu_events events;
1200
1201     if (!kvm_has_vcpu_events()) {
1202         return 0;
1203     }
1204
1205     events.exception.injected = (env->exception_injected >= 0);
1206     events.exception.nr = env->exception_injected;
1207     events.exception.has_error_code = env->has_error_code;
1208     events.exception.error_code = env->error_code;
1209
1210     events.interrupt.injected = (env->interrupt_injected >= 0);
1211     events.interrupt.nr = env->interrupt_injected;
1212     events.interrupt.soft = env->soft_interrupt;
1213
1214     events.nmi.injected = env->nmi_injected;
1215     events.nmi.pending = env->nmi_pending;
1216     events.nmi.masked = !!(env->hflags2 & HF2_NMI_MASK);
1217
1218     events.sipi_vector = env->sipi_vector;
1219
1220     events.flags = 0;
1221     if (level >= KVM_PUT_RESET_STATE) {
1222         events.flags |=
1223             KVM_VCPUEVENT_VALID_NMI_PENDING | KVM_VCPUEVENT_VALID_SIPI_VECTOR;
1224     }
1225
1226     return kvm_vcpu_ioctl(env, KVM_SET_VCPU_EVENTS, &events);
1227 #else
1228     return 0;
1229 #endif
1230 }
1231
1232 static int kvm_get_vcpu_events(CPUState *env)
1233 {
1234 #ifdef KVM_CAP_VCPU_EVENTS
1235     struct kvm_vcpu_events events;
1236     int ret;
1237
1238     if (!kvm_has_vcpu_events()) {
1239         return 0;
1240     }
1241
1242     ret = kvm_vcpu_ioctl(env, KVM_GET_VCPU_EVENTS, &events);
1243     if (ret < 0) {
1244        return ret;
1245     }
1246     env->exception_injected =
1247        events.exception.injected ? events.exception.nr : -1;
1248     env->has_error_code = events.exception.has_error_code;
1249     env->error_code = events.exception.error_code;
1250
1251     env->interrupt_injected =
1252         events.interrupt.injected ? events.interrupt.nr : -1;
1253     env->soft_interrupt = events.interrupt.soft;
1254
1255     env->nmi_injected = events.nmi.injected;
1256     env->nmi_pending = events.nmi.pending;
1257     if (events.nmi.masked) {
1258         env->hflags2 |= HF2_NMI_MASK;
1259     } else {
1260         env->hflags2 &= ~HF2_NMI_MASK;
1261     }
1262
1263     env->sipi_vector = events.sipi_vector;
1264 #endif
1265
1266     return 0;
1267 }
1268
1269 static int kvm_guest_debug_workarounds(CPUState *env)
1270 {
1271     int ret = 0;
1272 #ifdef KVM_CAP_SET_GUEST_DEBUG
1273     unsigned long reinject_trap = 0;
1274
1275     if (!kvm_has_vcpu_events()) {
1276         if (env->exception_injected == 1) {
1277             reinject_trap = KVM_GUESTDBG_INJECT_DB;
1278         } else if (env->exception_injected == 3) {
1279             reinject_trap = KVM_GUESTDBG_INJECT_BP;
1280         }
1281         env->exception_injected = -1;
1282     }
1283
1284     /*
1285      * Kernels before KVM_CAP_X86_ROBUST_SINGLESTEP overwrote flags.TF
1286      * injected via SET_GUEST_DEBUG while updating GP regs. Work around this
1287      * by updating the debug state once again if single-stepping is on.
1288      * Another reason to call kvm_update_guest_debug here is a pending debug
1289      * trap raise by the guest. On kernels without SET_VCPU_EVENTS we have to
1290      * reinject them via SET_GUEST_DEBUG.
1291      */
1292     if (reinject_trap ||
1293         (!kvm_has_robust_singlestep() && env->singlestep_enabled)) {
1294         ret = kvm_update_guest_debug(env, reinject_trap);
1295     }
1296 #endif /* KVM_CAP_SET_GUEST_DEBUG */
1297     return ret;
1298 }
1299
1300 static int kvm_put_debugregs(CPUState *env)
1301 {
1302 #ifdef KVM_CAP_DEBUGREGS
1303     struct kvm_debugregs dbgregs;
1304     int i;
1305
1306     if (!kvm_has_debugregs()) {
1307         return 0;
1308     }
1309
1310     for (i = 0; i < 4; i++) {
1311         dbgregs.db[i] = env->dr[i];
1312     }
1313     dbgregs.dr6 = env->dr[6];
1314     dbgregs.dr7 = env->dr[7];
1315     dbgregs.flags = 0;
1316
1317     return kvm_vcpu_ioctl(env, KVM_SET_DEBUGREGS, &dbgregs);
1318 #else
1319     return 0;
1320 #endif
1321 }
1322
1323 static int kvm_get_debugregs(CPUState *env)
1324 {
1325 #ifdef KVM_CAP_DEBUGREGS
1326     struct kvm_debugregs dbgregs;
1327     int i, ret;
1328
1329     if (!kvm_has_debugregs()) {
1330         return 0;
1331     }
1332
1333     ret = kvm_vcpu_ioctl(env, KVM_GET_DEBUGREGS, &dbgregs);
1334     if (ret < 0) {
1335        return ret;
1336     }
1337     for (i = 0; i < 4; i++) {
1338         env->dr[i] = dbgregs.db[i];
1339     }
1340     env->dr[4] = env->dr[6] = dbgregs.dr6;
1341     env->dr[5] = env->dr[7] = dbgregs.dr7;
1342 #endif
1343
1344     return 0;
1345 }
1346
1347 #ifdef OBSOLETE_KVM_IMPL
1348 int kvm_arch_put_registers(CPUState *env, int level)
1349 {
1350     int ret;
1351
1352     assert(cpu_is_stopped(env) || qemu_cpu_self(env));
1353
1354     ret = kvm_getput_regs(env, 1);
1355     if (ret < 0)
1356         return ret;
1357
1358     ret = kvm_put_xsave(env);
1359     if (ret < 0)
1360         return ret;
1361
1362     ret = kvm_put_xcrs(env);
1363     if (ret < 0)
1364         return ret;
1365
1366     ret = kvm_put_sregs(env);
1367     if (ret < 0)
1368         return ret;
1369
1370     ret = kvm_put_msrs(env, level);
1371     if (ret < 0)
1372         return ret;
1373
1374     if (level >= KVM_PUT_RESET_STATE) {
1375         ret = kvm_put_mp_state(env);
1376         if (ret < 0)
1377             return ret;
1378     }
1379
1380     ret = kvm_put_vcpu_events(env, level);
1381     if (ret < 0)
1382         return ret;
1383
1384     /* must be last */
1385     ret = kvm_guest_debug_workarounds(env);
1386     if (ret < 0)
1387         return ret;
1388
1389     ret = kvm_put_debugregs(env);
1390     if (ret < 0)
1391         return ret;
1392
1393     return 0;
1394 }
1395
1396 int kvm_arch_get_registers(CPUState *env)
1397 {
1398     int ret;
1399
1400     assert(cpu_is_stopped(env) || qemu_cpu_self(env));
1401
1402     ret = kvm_getput_regs(env, 0);
1403     if (ret < 0)
1404         return ret;
1405
1406     ret = kvm_get_xsave(env);
1407     if (ret < 0)
1408         return ret;
1409
1410     ret = kvm_get_xcrs(env);
1411     if (ret < 0)
1412         return ret;
1413
1414     ret = kvm_get_sregs(env);
1415     if (ret < 0)
1416         return ret;
1417
1418     ret = kvm_get_msrs(env);
1419     if (ret < 0)
1420         return ret;
1421
1422     ret = kvm_get_mp_state(env);
1423     if (ret < 0)
1424         return ret;
1425
1426     ret = kvm_get_vcpu_events(env);
1427     if (ret < 0)
1428         return ret;
1429
1430     ret = kvm_get_debugregs(env);
1431     if (ret < 0)
1432         return ret;
1433
1434     return 0;
1435 }
1436
1437 int kvm_arch_pre_run(CPUState *env, struct kvm_run *run)
1438 {
1439     /* Try to inject an interrupt if the guest can accept it */
1440     if (run->ready_for_interrupt_injection &&
1441         (env->interrupt_request & CPU_INTERRUPT_HARD) &&
1442         (env->eflags & IF_MASK)) {
1443         int irq;
1444
1445         env->interrupt_request &= ~CPU_INTERRUPT_HARD;
1446         irq = cpu_get_pic_interrupt(env);
1447         if (irq >= 0) {
1448             struct kvm_interrupt intr;
1449             intr.irq = irq;
1450             /* FIXME: errors */
1451             DPRINTF("injected interrupt %d\n", irq);
1452             kvm_vcpu_ioctl(env, KVM_INTERRUPT, &intr);
1453         }
1454     }
1455
1456     /* If we have an interrupt but the guest is not ready to receive an
1457      * interrupt, request an interrupt window exit.  This will
1458      * cause a return to userspace as soon as the guest is ready to
1459      * receive interrupts. */
1460     if ((env->interrupt_request & CPU_INTERRUPT_HARD))
1461         run->request_interrupt_window = 1;
1462     else
1463         run->request_interrupt_window = 0;
1464
1465     DPRINTF("setting tpr\n");
1466     run->cr8 = cpu_get_apic_tpr(env->apic_state);
1467
1468     return 0;
1469 }
1470 #endif
1471
1472 int kvm_arch_post_run(CPUState *env, struct kvm_run *run)
1473 {
1474     if (run->if_flag)
1475         env->eflags |= IF_MASK;
1476     else
1477         env->eflags &= ~IF_MASK;
1478
1479     cpu_set_apic_tpr(env->apic_state, run->cr8);
1480     cpu_set_apic_base(env->apic_state, run->apic_base);
1481
1482     return 0;
1483 }
1484
1485 #ifdef OBSOLETE_KVM_IMPL
1486
1487 int kvm_arch_process_irqchip_events(CPUState *env)
1488 {
1489     if (env->interrupt_request & CPU_INTERRUPT_INIT) {
1490         kvm_cpu_synchronize_state(env);
1491         do_cpu_init(env);
1492         env->exception_index = EXCP_HALTED;
1493     }
1494
1495     if (env->interrupt_request & CPU_INTERRUPT_SIPI) {
1496         kvm_cpu_synchronize_state(env);
1497         do_cpu_sipi(env);
1498     }
1499
1500     return env->halted;
1501 }
1502
1503 static int kvm_handle_halt(CPUState *env)
1504 {
1505     if (!((env->interrupt_request & CPU_INTERRUPT_HARD) &&
1506           (env->eflags & IF_MASK)) &&
1507         !(env->interrupt_request & CPU_INTERRUPT_NMI)) {
1508         env->halted = 1;
1509         env->exception_index = EXCP_HLT;
1510         return 0;
1511     }
1512
1513     return 1;
1514 }
1515
1516 int kvm_arch_handle_exit(CPUState *env, struct kvm_run *run)
1517 {
1518     int ret = 0;
1519
1520     switch (run->exit_reason) {
1521     case KVM_EXIT_HLT:
1522         DPRINTF("handle_hlt\n");
1523         ret = kvm_handle_halt(env);
1524         break;
1525     }
1526
1527     return ret;
1528 }
1529 #endif
1530
1531 #ifdef KVM_CAP_SET_GUEST_DEBUG
1532 int kvm_arch_insert_sw_breakpoint(CPUState *env, struct kvm_sw_breakpoint *bp)
1533 {
1534     static const uint8_t int3 = 0xcc;
1535
1536     if (cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&bp->saved_insn, 1, 0) ||
1537         cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&int3, 1, 1))
1538         return -EINVAL;
1539     return 0;
1540 }
1541
1542 int kvm_arch_remove_sw_breakpoint(CPUState *env, struct kvm_sw_breakpoint *bp)
1543 {
1544     uint8_t int3;
1545
1546     if (cpu_memory_rw_debug(env, bp->pc, &int3, 1, 0) || int3 != 0xcc ||
1547         cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&bp->saved_insn, 1, 1))
1548         return -EINVAL;
1549     return 0;
1550 }
1551
1552 static struct {
1553     target_ulong addr;
1554     int len;
1555     int type;
1556 } hw_breakpoint[4];
1557
1558 static int nb_hw_breakpoint;
1559
1560 static int find_hw_breakpoint(target_ulong addr, int len, int type)
1561 {
1562     int n;
1563
1564     for (n = 0; n < nb_hw_breakpoint; n++)
1565         if (hw_breakpoint[n].addr == addr && hw_breakpoint[n].type == type &&
1566             (hw_breakpoint[n].len == len || len == -1))
1567             return n;
1568     return -1;
1569 }
1570
1571 int kvm_arch_insert_hw_breakpoint(target_ulong addr,
1572                                   target_ulong len, int type)
1573 {
1574     switch (type) {
1575     case GDB_BREAKPOINT_HW:
1576         len = 1;
1577         break;
1578     case GDB_WATCHPOINT_WRITE:
1579     case GDB_WATCHPOINT_ACCESS:
1580         switch (len) {
1581         case 1:
1582             break;
1583         case 2:
1584         case 4:
1585         case 8:
1586             if (addr & (len - 1))
1587                 return -EINVAL;
1588             break;
1589         default:
1590             return -EINVAL;
1591         }
1592         break;
1593     default:
1594         return -ENOSYS;
1595     }
1596
1597     if (nb_hw_breakpoint == 4)
1598         return -ENOBUFS;
1599
1600     if (find_hw_breakpoint(addr, len, type) >= 0)
1601         return -EEXIST;
1602
1603     hw_breakpoint[nb_hw_breakpoint].addr = addr;
1604     hw_breakpoint[nb_hw_breakpoint].len = len;
1605     hw_breakpoint[nb_hw_breakpoint].type = type;
1606     nb_hw_breakpoint++;
1607
1608     return 0;
1609 }
1610
1611 int kvm_arch_remove_hw_breakpoint(target_ulong addr,
1612                                   target_ulong len, int type)
1613 {
1614     int n;
1615
1616     n = find_hw_breakpoint(addr, (type == GDB_BREAKPOINT_HW) ? 1 : len, type);
1617     if (n < 0)
1618         return -ENOENT;
1619
1620     nb_hw_breakpoint--;
1621     hw_breakpoint[n] = hw_breakpoint[nb_hw_breakpoint];
1622
1623     return 0;
1624 }
1625
1626 void kvm_arch_remove_all_hw_breakpoints(void)
1627 {
1628     nb_hw_breakpoint = 0;
1629 }
1630
1631 static CPUWatchpoint hw_watchpoint;
1632
1633 int kvm_arch_debug(struct kvm_debug_exit_arch *arch_info)
1634 {
1635     int handle = 0;
1636     int n;
1637
1638     if (arch_info->exception == 1) {
1639         if (arch_info->dr6 & (1 << 14)) {
1640             if (cpu_single_env->singlestep_enabled)
1641                 handle = 1;
1642         } else {
1643             for (n = 0; n < 4; n++)
1644                 if (arch_info->dr6 & (1 << n))
1645                     switch ((arch_info->dr7 >> (16 + n*4)) & 0x3) {
1646                     case 0x0:
1647                         handle = 1;
1648                         break;
1649                     case 0x1:
1650                         handle = 1;
1651                         cpu_single_env->watchpoint_hit = &hw_watchpoint;
1652                         hw_watchpoint.vaddr = hw_breakpoint[n].addr;
1653                         hw_watchpoint.flags = BP_MEM_WRITE;
1654                         break;
1655                     case 0x3:
1656                         handle = 1;
1657                         cpu_single_env->watchpoint_hit = &hw_watchpoint;
1658                         hw_watchpoint.vaddr = hw_breakpoint[n].addr;
1659                         hw_watchpoint.flags = BP_MEM_ACCESS;
1660                         break;
1661                     }
1662         }
1663     } else if (kvm_find_sw_breakpoint(cpu_single_env, arch_info->pc))
1664         handle = 1;
1665
1666     if (!handle) {
1667         cpu_synchronize_state(cpu_single_env);
1668         assert(cpu_single_env->exception_injected == -1);
1669
1670         cpu_single_env->exception_injected = arch_info->exception;
1671         cpu_single_env->has_error_code = 0;
1672     }
1673
1674     return handle;
1675 }
1676
1677 void kvm_arch_update_guest_debug(CPUState *env, struct kvm_guest_debug *dbg)
1678 {
1679     const uint8_t type_code[] = {
1680         [GDB_BREAKPOINT_HW] = 0x0,
1681         [GDB_WATCHPOINT_WRITE] = 0x1,
1682         [GDB_WATCHPOINT_ACCESS] = 0x3
1683     };
1684     const uint8_t len_code[] = {
1685         [1] = 0x0, [2] = 0x1, [4] = 0x3, [8] = 0x2
1686     };
1687     int n;
1688
1689     if (kvm_sw_breakpoints_active(env))
1690         dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP;
1691
1692     if (nb_hw_breakpoint > 0) {
1693         dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW_BP;
1694         dbg->arch.debugreg[7] = 0x0600;
1695         for (n = 0; n < nb_hw_breakpoint; n++) {
1696             dbg->arch.debugreg[n] = hw_breakpoint[n].addr;
1697             dbg->arch.debugreg[7] |= (2 << (n * 2)) |
1698                 (type_code[hw_breakpoint[n].type] << (16 + n*4)) |
1699                 (len_code[hw_breakpoint[n].len] << (18 + n*4));
1700         }
1701     }
1702     /* Legal xcr0 for loading */
1703     env->xcr0 = 1;
1704 }
1705 #endif /* KVM_CAP_SET_GUEST_DEBUG */
1706
1707 bool kvm_arch_stop_on_emulation_error(CPUState *env)
1708 {
1709       return !(env->cr[0] & CR0_PE_MASK) ||
1710               ((env->segs[R_CS].selector  & 3) != 3);
1711 }
1712
1713 static void hardware_memory_error(void)
1714 {
1715     fprintf(stderr, "Hardware memory error!\n");
1716     exit(1);
1717 }
1718
1719 #ifdef KVM_CAP_MCE
1720 static void kvm_mce_broadcast_rest(CPUState *env)
1721 {
1722     CPUState *cenv;
1723     int family, model, cpuver = env->cpuid_version;
1724
1725     family = (cpuver >> 8) & 0xf;
1726     model = ((cpuver >> 12) & 0xf0) + ((cpuver >> 4) & 0xf);
1727
1728     /* Broadcast MCA signal for processor version 06H_EH and above */
1729     if ((family == 6 && model >= 14) || family > 6) {
1730         for (cenv = first_cpu; cenv != NULL; cenv = cenv->next_cpu) {
1731             if (cenv == env) {
1732                 continue;
1733             }
1734             kvm_inject_x86_mce(cenv, 1, MCI_STATUS_VAL | MCI_STATUS_UC,
1735                                MCG_STATUS_MCIP | MCG_STATUS_RIPV, 0, 0, 1);
1736         }
1737     }
1738 }
1739 #endif
1740
1741 int kvm_on_sigbus_vcpu(CPUState *env, int code, void *addr)
1742 {
1743 #if defined(KVM_CAP_MCE)
1744     struct kvm_x86_mce mce = {
1745             .bank = 9,
1746     };
1747     void *vaddr;
1748     ram_addr_t ram_addr;
1749     target_phys_addr_t paddr;
1750     int r;
1751
1752     if ((env->mcg_cap & MCG_SER_P) && addr
1753         && (code == BUS_MCEERR_AR
1754             || code == BUS_MCEERR_AO)) {
1755         if (code == BUS_MCEERR_AR) {
1756             /* Fake an Intel architectural Data Load SRAR UCR */
1757             mce.status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN
1758                 | MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S
1759                 | MCI_STATUS_AR | 0x134;
1760             mce.misc = (MCM_ADDR_PHYS << 6) | 0xc;
1761             mce.mcg_status = MCG_STATUS_MCIP | MCG_STATUS_EIPV;
1762         } else {
1763             /*
1764              * If there is an MCE excpetion being processed, ignore
1765              * this SRAO MCE
1766              */
1767             r = kvm_mce_in_exception(env);
1768             if (r == -1) {
1769                 fprintf(stderr, "Failed to get MCE status\n");
1770             } else if (r) {
1771                 return 0;
1772             }
1773             /* Fake an Intel architectural Memory scrubbing UCR */
1774             mce.status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN
1775                 | MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S
1776                 | 0xc0;
1777             mce.misc = (MCM_ADDR_PHYS << 6) | 0xc;
1778             mce.mcg_status = MCG_STATUS_MCIP | MCG_STATUS_RIPV;
1779         }
1780         vaddr = (void *)addr;
1781         if (qemu_ram_addr_from_host(vaddr, &ram_addr) ||
1782             !kvm_physical_memory_addr_from_ram(env->kvm_state, ram_addr, &paddr)) {
1783             fprintf(stderr, "Hardware memory error for memory used by "
1784                     "QEMU itself instead of guest system!\n");
1785             /* Hope we are lucky for AO MCE */
1786             if (code == BUS_MCEERR_AO) {
1787                 return 0;
1788             } else {
1789                 hardware_memory_error();
1790             }
1791         }
1792         mce.addr = paddr;
1793         r = kvm_set_mce(env, &mce);
1794         if (r < 0) {
1795             fprintf(stderr, "kvm_set_mce: %s\n", strerror(errno));
1796             abort();
1797         }
1798         kvm_mce_broadcast_rest(env);
1799     } else
1800 #endif
1801     {
1802         if (code == BUS_MCEERR_AO) {
1803             return 0;
1804         } else if (code == BUS_MCEERR_AR) {
1805             hardware_memory_error();
1806         } else {
1807             return 1;
1808         }
1809     }
1810     return 0;
1811 }
1812
1813 int kvm_on_sigbus(int code, void *addr)
1814 {
1815 #if defined(KVM_CAP_MCE)
1816     if ((first_cpu->mcg_cap & MCG_SER_P) && addr && code == BUS_MCEERR_AO) {
1817         uint64_t status;
1818         void *vaddr;
1819         ram_addr_t ram_addr;
1820         target_phys_addr_t paddr;
1821
1822         /* Hope we are lucky for AO MCE */
1823         vaddr = addr;
1824         if (qemu_ram_addr_from_host(vaddr, &ram_addr) ||
1825             !kvm_physical_memory_addr_from_ram(first_cpu->kvm_state, ram_addr, &paddr)) {
1826             fprintf(stderr, "Hardware memory error for memory used by "
1827                     "QEMU itself instead of guest system!: %p\n", addr);
1828             return 0;
1829         }
1830         status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN
1831             | MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S
1832             | 0xc0;
1833         kvm_inject_x86_mce(first_cpu, 9, status,
1834                            MCG_STATUS_MCIP | MCG_STATUS_RIPV, paddr,
1835                            (MCM_ADDR_PHYS << 6) | 0xc, 1);
1836         kvm_mce_broadcast_rest(first_cpu);
1837     } else
1838 #endif
1839     {
1840         if (code == BUS_MCEERR_AO) {
1841             return 0;
1842         } else if (code == BUS_MCEERR_AR) {
1843             hardware_memory_error();
1844         } else {
1845             return 1;
1846         }
1847     }
1848     return 0;
1849 }
1850
1851 #include "qemu-kvm-x86.c"