target-i386/kvm.c

   1 /*
   2  * QEMU KVM support
   3  *
   4  * Copyright (C) 2006-2008 Qumranet Technologies
   5  * Copyright IBM, Corp. 2008
   6  *
   7  * Authors:
   8  *  Anthony Liguori   <aliguori@us.ibm.com>
   9  *
  10  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  11  * See the COPYING file in the top-level directory.
  12  *
  13  */
  14
  15 #include <sys/types.h>
  16 #include <sys/ioctl.h>
  17 #include <sys/mman.h>
  18 #include <sys/utsname.h>
  19
  20 #include <linux/kvm.h>
  21
  22 #include "qemu-common.h"
  23 #include "sysemu.h"
  24 #include "kvm.h"
  25 #include "cpu.h"
  26 #include "gdbstub.h"
  27 #include "host-utils.h"
  28 #include "hw/pc.h"
  29 #include "hw/apic.h"
  30 #include "ioport.h"
  31 #include "kvm_x86.h"
  32
  33 #ifdef CONFIG_KVM_PARA
  34 #include <linux/kvm_para.h>
  35 #endif
  36 //
  37 //#define DEBUG_KVM
  38
  39 #ifdef DEBUG_KVM
  40 #define DPRINTF(fmt, ...) \
  41     do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
  42 #else
  43 #define DPRINTF(fmt, ...) \
  44     do { } while (0)
  45 #endif
  46
  47 #define MSR_KVM_WALL_CLOCK  0x11
  48 #define MSR_KVM_SYSTEM_TIME 0x12
  49
  50 #ifndef BUS_MCEERR_AR
  51 #define BUS_MCEERR_AR 4
  52 #endif
  53 #ifndef BUS_MCEERR_AO
  54 #define BUS_MCEERR_AO 5
  55 #endif
  56
  57 const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
  58     KVM_CAP_INFO(SET_TSS_ADDR),
  59     KVM_CAP_INFO(EXT_CPUID),
  60     KVM_CAP_INFO(MP_STATE),
  61     KVM_CAP_LAST_INFO
  62 };
  63
  64 static bool has_msr_star;
  65 static bool has_msr_hsave_pa;
  66 #if defined(CONFIG_KVM_PARA) && defined(KVM_CAP_ASYNC_PF)
  67 static bool has_msr_async_pf_en;
  68 #endif
  69 static int lm_capable_kernel;
  70
  71 static struct kvm_cpuid2 *try_get_cpuid(KVMState *s, int max)
  72 {
  73     struct kvm_cpuid2 *cpuid;
  74     int r, size;
  75
  76     size = sizeof(*cpuid) + max * sizeof(*cpuid->entries);
  77     cpuid = (struct kvm_cpuid2 *)qemu_mallocz(size);
  78     cpuid->nent = max;
  79     r = kvm_ioctl(s, KVM_GET_SUPPORTED_CPUID, cpuid);
  80     if (r == 0 && cpuid->nent >= max) {
  81         r = -E2BIG;
  82     }
  83     if (r < 0) {
  84         if (r == -E2BIG) {
  85             qemu_free(cpuid);
  86             return NULL;
  87         } else {
  88             fprintf(stderr, "KVM_GET_SUPPORTED_CPUID failed: %s\n",
  89                     strerror(-r));
  90             exit(1);
  91         }
  92     }
  93     return cpuid;
  94 }
  95
  96 uint32_t kvm_arch_get_supported_cpuid(CPUState *env, uint32_t function,
  97                                       uint32_t index, int reg)
  98 {
  99     struct kvm_cpuid2 *cpuid;
 100     int i, max;
 101     uint32_t ret = 0;
 102     uint32_t cpuid_1_edx;
 103
 104     max = 1;
 105     while ((cpuid = try_get_cpuid(env->kvm_state, max)) == NULL) {
 106         max *= 2;
 107     }
 108
 109     for (i = 0; i < cpuid->nent; ++i) {
 110         if (cpuid->entries[i].function == function &&
 111             cpuid->entries[i].index == index) {
 112             switch (reg) {
 113             case R_EAX:
 114                 ret = cpuid->entries[i].eax;
 115                 break;
 116             case R_EBX:
 117                 ret = cpuid->entries[i].ebx;
 118                 break;
 119             case R_ECX:
 120                 ret = cpuid->entries[i].ecx;
 121                 break;
 122             case R_EDX:
 123                 ret = cpuid->entries[i].edx;
 124                 switch (function) {
 125                 case 1:
 126                     /* KVM before 2.6.30 misreports the following features */
 127                     ret |= CPUID_MTRR | CPUID_PAT | CPUID_MCE | CPUID_MCA;
 128                     break;
 129                 case 0x80000001:
 130                     /* On Intel, kvm returns cpuid according to the Intel spec,
 131                      * so add missing bits according to the AMD spec:
 132                      */
 133                     cpuid_1_edx = kvm_arch_get_supported_cpuid(env, 1, 0, R_EDX);
 134                     ret |= cpuid_1_edx & 0x183f7ff;
 135                     break;
 136                 }
 137                 break;
 138             }
 139         }
 140     }
 141
 142     qemu_free(cpuid);
 143
 144     return ret;
 145 }
 146
 147 #ifdef CONFIG_KVM_PARA
 148 struct kvm_para_features {
 149     int cap;
 150     int feature;
 151 } para_features[] = {
 152     { KVM_CAP_CLOCKSOURCE, KVM_FEATURE_CLOCKSOURCE },
 153     { KVM_CAP_NOP_IO_DELAY, KVM_FEATURE_NOP_IO_DELAY },
 154     { KVM_CAP_PV_MMU, KVM_FEATURE_MMU_OP },
 155 #ifdef KVM_CAP_ASYNC_PF
 156     { KVM_CAP_ASYNC_PF, KVM_FEATURE_ASYNC_PF },
 157 #endif
 158     { -1, -1 }
 159 };
 160
 161 static int get_para_features(CPUState *env)
 162 {
 163     int i, features = 0;
 164
 165     for (i = 0; i < ARRAY_SIZE(para_features) - 1; i++) {
 166         if (kvm_check_extension(env->kvm_state, para_features[i].cap)) {
 167             features |= (1 << para_features[i].feature);
 168         }
 169     }
 170 #ifdef KVM_CAP_ASYNC_PF
 171     has_msr_async_pf_en = features & (1 << KVM_FEATURE_ASYNC_PF);
 172 #endif
 173     return features;
 174 }
 175 #endif /* CONFIG_KVM_PARA */
 176
 177 #ifdef KVM_CAP_MCE
 178 static int kvm_get_mce_cap_supported(KVMState *s, uint64_t *mce_cap,
 179                                      int *max_banks)
 180 {
 181     int r;
 182
 183     r = kvm_check_extension(s, KVM_CAP_MCE);
 184     if (r > 0) {
 185         *max_banks = r;
 186         return kvm_ioctl(s, KVM_X86_GET_MCE_CAP_SUPPORTED, mce_cap);
 187     }
 188     return -ENOSYS;
 189 }
 190
 191 static int kvm_setup_mce(CPUState *env, uint64_t *mcg_cap)
 192 {
 193     return kvm_vcpu_ioctl(env, KVM_X86_SETUP_MCE, mcg_cap);
 194 }
 195
 196 static int kvm_set_mce(CPUState *env, struct kvm_x86_mce *m)
 197 {
 198     return kvm_vcpu_ioctl(env, KVM_X86_SET_MCE, m);
 199 }
 200
 201 static int kvm_get_msr(CPUState *env, struct kvm_msr_entry *msrs, int n)
 202 {
 203     struct kvm_msrs *kmsrs = qemu_malloc(sizeof *kmsrs + n * sizeof *msrs);
 204     int r;
 205
 206     kmsrs->nmsrs = n;
 207     memcpy(kmsrs->entries, msrs, n * sizeof *msrs);
 208     r = kvm_vcpu_ioctl(env, KVM_GET_MSRS, kmsrs);
 209     memcpy(msrs, kmsrs->entries, n * sizeof *msrs);
 210     free(kmsrs);
 211     return r;
 212 }
 213
 214 /* FIXME: kill this and kvm_get_msr, use env->mcg_status instead */
 215 static int kvm_mce_in_progress(CPUState *env)
 216 {
 217     struct kvm_msr_entry msr_mcg_status = {
 218         .index = MSR_MCG_STATUS,
 219     };
 220     int r;
 221
 222     r = kvm_get_msr(env, &msr_mcg_status, 1);
 223     if (r == -1 || r == 0) {
 224         fprintf(stderr, "Failed to get MCE status\n");
 225         return 0;
 226     }
 227     return !!(msr_mcg_status.data & MCG_STATUS_MCIP);
 228 }
 229
 230 struct kvm_x86_mce_data
 231 {
 232     CPUState *env;
 233     struct kvm_x86_mce *mce;
 234     int abort_on_error;
 235 };
 236
 237 static void kvm_do_inject_x86_mce(void *_data)
 238 {
 239     struct kvm_x86_mce_data *data = _data;
 240     int r;
 241
 242     /* If there is an MCE exception being processed, ignore this SRAO MCE */
 243     if ((data->env->mcg_cap & MCG_SER_P) &&
 244         !(data->mce->status & MCI_STATUS_AR)) {
 245         if (kvm_mce_in_progress(data->env)) {
 246             return;
 247         }
 248     }
 249
 250     r = kvm_set_mce(data->env, data->mce);
 251     if (r < 0) {
 252         perror("kvm_set_mce FAILED");
 253         if (data->abort_on_error) {
 254             abort();
 255         }
 256     }
 257 }
 258
 259 static void kvm_inject_x86_mce_on(CPUState *env, struct kvm_x86_mce *mce,
 260                                   int flag)
 261 {
 262     struct kvm_x86_mce_data data = {
 263         .env = env,
 264         .mce = mce,
 265         .abort_on_error = (flag & ABORT_ON_ERROR),
 266     };
 267
 268     if (!env->mcg_cap) {
 269         fprintf(stderr, "MCE support is not enabled!\n");
 270         return;
 271     }
 272
 273     run_on_cpu(env, kvm_do_inject_x86_mce, &data);
 274 }
 275
 276 static void kvm_mce_broadcast_rest(CPUState *env)
 277 {
 278     struct kvm_x86_mce mce = {
 279         .bank = 1,
 280         .status = MCI_STATUS_VAL | MCI_STATUS_UC,
 281         .mcg_status = MCG_STATUS_MCIP | MCG_STATUS_RIPV,
 282         .addr = 0,
 283         .misc = 0,
 284     };
 285     CPUState *cenv;
 286
 287     /* Broadcast MCA signal for processor version 06H_EH and above */
 288     if (cpu_x86_support_mca_broadcast(env)) {
 289         for (cenv = first_cpu; cenv != NULL; cenv = cenv->next_cpu) {
 290             if (cenv == env) {
 291                 continue;
 292             }
 293             kvm_inject_x86_mce_on(cenv, &mce, ABORT_ON_ERROR);
 294         }
 295     }
 296 }
 297
 298 static void kvm_mce_inj_srar_dataload(CPUState *env, target_phys_addr_t paddr)
 299 {
 300     struct kvm_x86_mce mce = {
 301         .bank = 9,
 302         .status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN
 303                   | MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S
 304                   | MCI_STATUS_AR | 0x134,
 305         .mcg_status = MCG_STATUS_MCIP | MCG_STATUS_EIPV,
 306         .addr = paddr,
 307         .misc = (MCM_ADDR_PHYS << 6) | 0xc,
 308     };
 309     int r;
 310
 311     r = kvm_set_mce(env, &mce);
 312     if (r < 0) {
 313         fprintf(stderr, "kvm_set_mce: %s\n", strerror(errno));
 314         abort();
 315     }
 316     kvm_mce_broadcast_rest(env);
 317 }
 318
 319 static void kvm_mce_inj_srao_memscrub(CPUState *env, target_phys_addr_t paddr)
 320 {
 321     struct kvm_x86_mce mce = {
 322         .bank = 9,
 323         .status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN
 324                   | MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S
 325                   | 0xc0,
 326         .mcg_status = MCG_STATUS_MCIP | MCG_STATUS_RIPV,
 327         .addr = paddr,
 328         .misc = (MCM_ADDR_PHYS << 6) | 0xc,
 329     };
 330     int r;
 331
 332     r = kvm_set_mce(env, &mce);
 333     if (r < 0) {
 334         fprintf(stderr, "kvm_set_mce: %s\n", strerror(errno));
 335         abort();
 336     }
 337     kvm_mce_broadcast_rest(env);
 338 }
 339
 340 static void kvm_mce_inj_srao_memscrub2(CPUState *env, target_phys_addr_t paddr)
 341 {
 342     struct kvm_x86_mce mce = {
 343         .bank = 9,
 344         .status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN
 345                   | MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S
 346                   | 0xc0,
 347         .mcg_status = MCG_STATUS_MCIP | MCG_STATUS_RIPV,
 348         .addr = paddr,
 349         .misc = (MCM_ADDR_PHYS << 6) | 0xc,
 350     };
 351
 352     kvm_inject_x86_mce_on(env, &mce, ABORT_ON_ERROR);
 353     kvm_mce_broadcast_rest(env);
 354 }
 355 #endif /* KVM_CAP_MCE */
 356
 357 static void hardware_memory_error(void)
 358 {
 359     fprintf(stderr, "Hardware memory error!\n");
 360     exit(1);
 361 }
 362
 363 int kvm_arch_on_sigbus_vcpu(CPUState *env, int code, void *addr)
 364 {
 365 #ifdef KVM_CAP_MCE
 366     void *vaddr;
 367     ram_addr_t ram_addr;
 368     target_phys_addr_t paddr;
 369
 370     if ((env->mcg_cap & MCG_SER_P) && addr
 371         && (code == BUS_MCEERR_AR
 372             || code == BUS_MCEERR_AO)) {
 373         vaddr = (void *)addr;
 374         if (qemu_ram_addr_from_host(vaddr, &ram_addr) ||
 375             !kvm_physical_memory_addr_from_ram(env->kvm_state, ram_addr, &paddr)) {
 376             fprintf(stderr, "Hardware memory error for memory used by "
 377                     "QEMU itself instead of guest system!\n");
 378             /* Hope we are lucky for AO MCE */
 379             if (code == BUS_MCEERR_AO) {
 380                 return 0;
 381             } else {
 382                 hardware_memory_error();
 383             }
 384         }
 385
 386         if (code == BUS_MCEERR_AR) {
 387             /* Fake an Intel architectural Data Load SRAR UCR */
 388             kvm_mce_inj_srar_dataload(env, paddr);
 389         } else {
 390             /*
 391              * If there is an MCE excpetion being processed, ignore
 392              * this SRAO MCE
 393              */
 394             if (!kvm_mce_in_progress(env)) {
 395                 /* Fake an Intel architectural Memory scrubbing UCR */
 396                 kvm_mce_inj_srao_memscrub(env, paddr);
 397             }
 398         }
 399     } else
 400 #endif /* KVM_CAP_MCE */
 401     {
 402         if (code == BUS_MCEERR_AO) {
 403             return 0;
 404         } else if (code == BUS_MCEERR_AR) {
 405             hardware_memory_error();
 406         } else {
 407             return 1;
 408         }
 409     }
 410     return 0;
 411 }
 412
 413 int kvm_arch_on_sigbus(int code, void *addr)
 414 {
 415 #ifdef KVM_CAP_MCE
 416     if ((first_cpu->mcg_cap & MCG_SER_P) && addr && code == BUS_MCEERR_AO) {
 417         void *vaddr;
 418         ram_addr_t ram_addr;
 419         target_phys_addr_t paddr;
 420
 421         /* Hope we are lucky for AO MCE */
 422         vaddr = addr;
 423         if (qemu_ram_addr_from_host(vaddr, &ram_addr) ||
 424             !kvm_physical_memory_addr_from_ram(first_cpu->kvm_state, ram_addr,
 425                                                &paddr)) {
 426             fprintf(stderr, "Hardware memory error for memory used by "
 427                     "QEMU itself instead of guest system!: %p\n", addr);
 428             return 0;
 429         }
 430         kvm_mce_inj_srao_memscrub2(first_cpu, paddr);
 431     } else
 432 #endif /* KVM_CAP_MCE */
 433     {
 434         if (code == BUS_MCEERR_AO) {
 435             return 0;
 436         } else if (code == BUS_MCEERR_AR) {
 437             hardware_memory_error();
 438         } else {
 439             return 1;
 440         }
 441     }
 442     return 0;
 443 }
 444
 445 void kvm_inject_x86_mce(CPUState *cenv, int bank, uint64_t status,
 446                         uint64_t mcg_status, uint64_t addr, uint64_t misc,
 447                         int flag)
 448 {
 449 #ifdef KVM_CAP_MCE
 450     struct kvm_x86_mce mce = {
 451         .bank = bank,
 452         .status = status,
 453         .mcg_status = mcg_status,
 454         .addr = addr,
 455         .misc = misc,
 456     };
 457
 458     if (flag & MCE_BROADCAST) {
 459         kvm_mce_broadcast_rest(cenv);
 460     }
 461
 462     kvm_inject_x86_mce_on(cenv, &mce, flag);
 463 #else /* !KVM_CAP_MCE*/
 464     if (flag & ABORT_ON_ERROR) {
 465         abort();
 466     }
 467 #endif /* !KVM_CAP_MCE*/
 468 }
 469
 470 static void cpu_update_state(void *opaque, int running, int reason)
 471 {
 472     CPUState *env = opaque;
 473
 474     if (running) {
 475         env->tsc_valid = false;
 476     }
 477 }
 478
 479 int kvm_arch_init_vcpu(CPUState *env)
 480 {
 481     struct {
 482         struct kvm_cpuid2 cpuid;
 483         struct kvm_cpuid_entry2 entries[100];
 484     } __attribute__((packed)) cpuid_data;
 485     uint32_t limit, i, j, cpuid_i;
 486     uint32_t unused;
 487     struct kvm_cpuid_entry2 *c;
 488 #ifdef CONFIG_KVM_PARA
 489     uint32_t signature[3];
 490 #endif
 491
 492     env->cpuid_features &= kvm_arch_get_supported_cpuid(env, 1, 0, R_EDX);
 493
 494     i = env->cpuid_ext_features & CPUID_EXT_HYPERVISOR;
 495     env->cpuid_ext_features &= kvm_arch_get_supported_cpuid(env, 1, 0, R_ECX);
 496     env->cpuid_ext_features |= i;
 497
 498     env->cpuid_ext2_features &= kvm_arch_get_supported_cpuid(env, 0x80000001,
 499                                                              0, R_EDX);
 500     env->cpuid_ext3_features &= kvm_arch_get_supported_cpuid(env, 0x80000001,
 501                                                              0, R_ECX);
 502     env->cpuid_svm_features  &= kvm_arch_get_supported_cpuid(env, 0x8000000A,
 503                                                              0, R_EDX);
 504
 505
 506     cpuid_i = 0;
 507
 508 #ifdef CONFIG_KVM_PARA
 509     /* Paravirtualization CPUIDs */
 510     memcpy(signature, "KVMKVMKVM\0\0\0", 12);
 511     c = &cpuid_data.entries[cpuid_i++];
 512     memset(c, 0, sizeof(*c));
 513     c->function = KVM_CPUID_SIGNATURE;
 514     c->eax = 0;
 515     c->ebx = signature[0];
 516     c->ecx = signature[1];
 517     c->edx = signature[2];
 518
 519     c = &cpuid_data.entries[cpuid_i++];
 520     memset(c, 0, sizeof(*c));
 521     c->function = KVM_CPUID_FEATURES;
 522     c->eax = env->cpuid_kvm_features & get_para_features(env);
 523 #endif
 524
 525     cpu_x86_cpuid(env, 0, 0, &limit, &unused, &unused, &unused);
 526
 527     for (i = 0; i <= limit; i++) {
 528         c = &cpuid_data.entries[cpuid_i++];
 529
 530         switch (i) {
 531         case 2: {
 532             /* Keep reading function 2 till all the input is received */
 533             int times;
 534
 535             c->function = i;
 536             c->flags = KVM_CPUID_FLAG_STATEFUL_FUNC |
 537                        KVM_CPUID_FLAG_STATE_READ_NEXT;
 538             cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 539             times = c->eax & 0xff;
 540
 541             for (j = 1; j < times; ++j) {
 542                 c = &cpuid_data.entries[cpuid_i++];
 543                 c->function = i;
 544                 c->flags = KVM_CPUID_FLAG_STATEFUL_FUNC;
 545                 cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 546             }
 547             break;
 548         }
 549         case 4:
 550         case 0xb:
 551         case 0xd:
 552             for (j = 0; ; j++) {
 553                 c->function = i;
 554                 c->flags = KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
 555                 c->index = j;
 556                 cpu_x86_cpuid(env, i, j, &c->eax, &c->ebx, &c->ecx, &c->edx);
 557
 558                 if (i == 4 && c->eax == 0) {
 559                     break;
 560                 }
 561                 if (i == 0xb && !(c->ecx & 0xff00)) {
 562                     break;
 563                 }
 564                 if (i == 0xd && c->eax == 0) {
 565                     break;
 566                 }
 567                 c = &cpuid_data.entries[cpuid_i++];
 568             }
 569             break;
 570         default:
 571             c->function = i;
 572             c->flags = 0;
 573             cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 574             break;
 575         }
 576     }
 577     cpu_x86_cpuid(env, 0x80000000, 0, &limit, &unused, &unused, &unused);
 578
 579     for (i = 0x80000000; i <= limit; i++) {
 580         c = &cpuid_data.entries[cpuid_i++];
 581
 582         c->function = i;
 583         c->flags = 0;
 584         cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
 585     }
 586
 587     cpuid_data.cpuid.nent = cpuid_i;
 588
 589 #ifdef KVM_CAP_MCE
 590     if (((env->cpuid_version >> 8)&0xF) >= 6
 591         && (env->cpuid_features&(CPUID_MCE|CPUID_MCA)) == (CPUID_MCE|CPUID_MCA)
 592         && kvm_check_extension(env->kvm_state, KVM_CAP_MCE) > 0) {
 593         uint64_t mcg_cap;
 594         int banks;
 595
 596         if (kvm_get_mce_cap_supported(env->kvm_state, &mcg_cap, &banks)) {
 597             perror("kvm_get_mce_cap_supported FAILED");
 598         } else {
 599             if (banks > MCE_BANKS_DEF)
 600                 banks = MCE_BANKS_DEF;
 601             mcg_cap &= MCE_CAP_DEF;
 602             mcg_cap |= banks;
 603             if (kvm_setup_mce(env, &mcg_cap)) {
 604                 perror("kvm_setup_mce FAILED");
 605             } else {
 606                 env->mcg_cap = mcg_cap;
 607             }
 608         }
 609     }
 610 #endif
 611
 612     qemu_add_vm_change_state_handler(cpu_update_state, env);
 613
 614     return kvm_vcpu_ioctl(env, KVM_SET_CPUID2, &cpuid_data);
 615 }
 616
 617 void kvm_arch_reset_vcpu(CPUState *env)
 618 {
 619     env->exception_injected = -1;
 620     env->interrupt_injected = -1;
 621     env->xcr0 = 1;
 622     if (kvm_irqchip_in_kernel()) {
 623         env->mp_state = cpu_is_bsp(env) ? KVM_MP_STATE_RUNNABLE :
 624                                           KVM_MP_STATE_UNINITIALIZED;
 625     } else {
 626         env->mp_state = KVM_MP_STATE_RUNNABLE;
 627     }
 628 }
 629
 630 static int kvm_get_supported_msrs(KVMState *s)
 631 {
 632     static int kvm_supported_msrs;
 633     int ret = 0;
 634
 635     /* first time */
 636     if (kvm_supported_msrs == 0) {
 637         struct kvm_msr_list msr_list, *kvm_msr_list;
 638
 639         kvm_supported_msrs = -1;
 640
 641         /* Obtain MSR list from KVM.  These are the MSRs that we must
 642          * save/restore */
 643         msr_list.nmsrs = 0;
 644         ret = kvm_ioctl(s, KVM_GET_MSR_INDEX_LIST, &msr_list);
 645         if (ret < 0 && ret != -E2BIG) {
 646             return ret;
 647         }
 648         /* Old kernel modules had a bug and could write beyond the provided
 649            memory. Allocate at least a safe amount of 1K. */
 650         kvm_msr_list = qemu_mallocz(MAX(1024, sizeof(msr_list) +
 651                                               msr_list.nmsrs *
 652                                               sizeof(msr_list.indices[0])));
 653
 654         kvm_msr_list->nmsrs = msr_list.nmsrs;
 655         ret = kvm_ioctl(s, KVM_GET_MSR_INDEX_LIST, kvm_msr_list);
 656         if (ret >= 0) {
 657             int i;
 658
 659             for (i = 0; i < kvm_msr_list->nmsrs; i++) {
 660                 if (kvm_msr_list->indices[i] == MSR_STAR) {
 661                     has_msr_star = true;
 662                     continue;
 663                 }
 664                 if (kvm_msr_list->indices[i] == MSR_VM_HSAVE_PA) {
 665                     has_msr_hsave_pa = true;
 666                     continue;
 667                 }
 668             }
 669         }
 670
 671         free(kvm_msr_list);
 672     }
 673
 674     return ret;
 675 }
 676
 677 int kvm_arch_init(KVMState *s)
 678 {
 679     uint64_t identity_base = 0xfffbc000;
 680     int ret;
 681     struct utsname utsname;
 682
 683     ret = kvm_get_supported_msrs(s);
 684     if (ret < 0) {
 685         return ret;
 686     }
 687
 688     uname(&utsname);
 689     lm_capable_kernel = strcmp(utsname.machine, "x86_64") == 0;
 690
 691     /*
 692      * On older Intel CPUs, KVM uses vm86 mode to emulate 16-bit code directly.
 693      * In order to use vm86 mode, an EPT identity map and a TSS  are needed.
 694      * Since these must be part of guest physical memory, we need to allocate
 695      * them, both by setting their start addresses in the kernel and by
 696      * creating a corresponding e820 entry. We need 4 pages before the BIOS.
 697      *
 698      * Older KVM versions may not support setting the identity map base. In
 699      * that case we need to stick with the default, i.e. a 256K maximum BIOS
 700      * size.
 701      */
 702 #ifdef KVM_CAP_SET_IDENTITY_MAP_ADDR
 703     if (kvm_check_extension(s, KVM_CAP_SET_IDENTITY_MAP_ADDR)) {
 704         /* Allows up to 16M BIOSes. */
 705         identity_base = 0xfeffc000;
 706
 707         ret = kvm_vm_ioctl(s, KVM_SET_IDENTITY_MAP_ADDR, &identity_base);
 708         if (ret < 0) {
 709             return ret;
 710         }
 711     }
 712 #endif
 713     /* Set TSS base one page after EPT identity map. */
 714     ret = kvm_vm_ioctl(s, KVM_SET_TSS_ADDR, identity_base + 0x1000);
 715     if (ret < 0) {
 716         return ret;
 717     }
 718
 719     /* Tell fw_cfg to notify the BIOS to reserve the range. */
 720     ret = e820_add_entry(identity_base, 0x4000, E820_RESERVED);
 721     if (ret < 0) {
 722         fprintf(stderr, "e820_add_entry() table is full\n");
 723         return ret;
 724     }
 725
 726     return 0;
 727 }
 728
 729 static void set_v8086_seg(struct kvm_segment *lhs, const SegmentCache *rhs)
 730 {
 731     lhs->selector = rhs->selector;
 732     lhs->base = rhs->base;
 733     lhs->limit = rhs->limit;
 734     lhs->type = 3;
 735     lhs->present = 1;
 736     lhs->dpl = 3;
 737     lhs->db = 0;
 738     lhs->s = 1;
 739     lhs->l = 0;
 740     lhs->g = 0;
 741     lhs->avl = 0;
 742     lhs->unusable = 0;
 743 }
 744
 745 static void set_seg(struct kvm_segment *lhs, const SegmentCache *rhs)
 746 {
 747     unsigned flags = rhs->flags;
 748     lhs->selector = rhs->selector;
 749     lhs->base = rhs->base;
 750     lhs->limit = rhs->limit;
 751     lhs->type = (flags >> DESC_TYPE_SHIFT) & 15;
 752     lhs->present = (flags & DESC_P_MASK) != 0;
 753     lhs->dpl = (flags >> DESC_DPL_SHIFT) & 3;
 754     lhs->db = (flags >> DESC_B_SHIFT) & 1;
 755     lhs->s = (flags & DESC_S_MASK) != 0;
 756     lhs->l = (flags >> DESC_L_SHIFT) & 1;
 757     lhs->g = (flags & DESC_G_MASK) != 0;
 758     lhs->avl = (flags & DESC_AVL_MASK) != 0;
 759     lhs->unusable = 0;
 760 }
 761
 762 static void get_seg(SegmentCache *lhs, const struct kvm_segment *rhs)
 763 {
 764     lhs->selector = rhs->selector;
 765     lhs->base = rhs->base;
 766     lhs->limit = rhs->limit;
 767     lhs->flags = (rhs->type << DESC_TYPE_SHIFT) |
 768                  (rhs->present * DESC_P_MASK) |
 769                  (rhs->dpl << DESC_DPL_SHIFT) |
 770                  (rhs->db << DESC_B_SHIFT) |
 771                  (rhs->s * DESC_S_MASK) |
 772                  (rhs->l << DESC_L_SHIFT) |
 773                  (rhs->g * DESC_G_MASK) |
 774                  (rhs->avl * DESC_AVL_MASK);
 775 }
 776
 777 static void kvm_getput_reg(__u64 *kvm_reg, target_ulong *qemu_reg, int set)
 778 {
 779     if (set) {
 780         *kvm_reg = *qemu_reg;
 781     } else {
 782         *qemu_reg = *kvm_reg;
 783     }
 784 }
 785
 786 static int kvm_getput_regs(CPUState *env, int set)
 787 {
 788     struct kvm_regs regs;
 789     int ret = 0;
 790
 791     if (!set) {
 792         ret = kvm_vcpu_ioctl(env, KVM_GET_REGS, &regs);
 793         if (ret < 0) {
 794             return ret;
 795         }
 796     }
 797
 798     kvm_getput_reg(&regs.rax, &env->regs[R_EAX], set);
 799     kvm_getput_reg(&regs.rbx, &env->regs[R_EBX], set);
 800     kvm_getput_reg(&regs.rcx, &env->regs[R_ECX], set);
 801     kvm_getput_reg(&regs.rdx, &env->regs[R_EDX], set);
 802     kvm_getput_reg(&regs.rsi, &env->regs[R_ESI], set);
 803     kvm_getput_reg(&regs.rdi, &env->regs[R_EDI], set);
 804     kvm_getput_reg(&regs.rsp, &env->regs[R_ESP], set);
 805     kvm_getput_reg(&regs.rbp, &env->regs[R_EBP], set);
 806 #ifdef TARGET_X86_64
 807     kvm_getput_reg(&regs.r8, &env->regs[8], set);
 808     kvm_getput_reg(&regs.r9, &env->regs[9], set);
 809     kvm_getput_reg(&regs.r10, &env->regs[10], set);
 810     kvm_getput_reg(&regs.r11, &env->regs[11], set);
 811     kvm_getput_reg(&regs.r12, &env->regs[12], set);
 812     kvm_getput_reg(&regs.r13, &env->regs[13], set);
 813     kvm_getput_reg(&regs.r14, &env->regs[14], set);
 814     kvm_getput_reg(&regs.r15, &env->regs[15], set);
 815 #endif
 816
 817     kvm_getput_reg(&regs.rflags, &env->eflags, set);
 818     kvm_getput_reg(&regs.rip, &env->eip, set);
 819
 820     if (set) {
 821         ret = kvm_vcpu_ioctl(env, KVM_SET_REGS, &regs);
 822     }
 823
 824     return ret;
 825 }
 826
 827 static int kvm_put_fpu(CPUState *env)
 828 {
 829     struct kvm_fpu fpu;
 830     int i;
 831
 832     memset(&fpu, 0, sizeof fpu);
 833     fpu.fsw = env->fpus & ~(7 << 11);
 834     fpu.fsw |= (env->fpstt & 7) << 11;
 835     fpu.fcw = env->fpuc;
 836     for (i = 0; i < 8; ++i) {
 837         fpu.ftwx |= (!env->fptags[i]) << i;
 838     }
 839     memcpy(fpu.fpr, env->fpregs, sizeof env->fpregs);
 840     memcpy(fpu.xmm, env->xmm_regs, sizeof env->xmm_regs);
 841     fpu.mxcsr = env->mxcsr;
 842
 843     return kvm_vcpu_ioctl(env, KVM_SET_FPU, &fpu);
 844 }
 845
 846 #ifdef KVM_CAP_XSAVE
 847 #define XSAVE_CWD_RIP     2
 848 #define XSAVE_CWD_RDP     4
 849 #define XSAVE_MXCSR       6
 850 #define XSAVE_ST_SPACE    8
 851 #define XSAVE_XMM_SPACE   40
 852 #define XSAVE_XSTATE_BV   128
 853 #define XSAVE_YMMH_SPACE  144
 854 #endif
 855
 856 static int kvm_put_xsave(CPUState *env)
 857 {
 858 #ifdef KVM_CAP_XSAVE
 859     int i, r;
 860     struct kvm_xsave* xsave;
 861     uint16_t cwd, swd, twd, fop;
 862
 863     if (!kvm_has_xsave()) {
 864         return kvm_put_fpu(env);
 865     }
 866
 867     xsave = qemu_memalign(4096, sizeof(struct kvm_xsave));
 868     memset(xsave, 0, sizeof(struct kvm_xsave));
 869     cwd = swd = twd = fop = 0;
 870     swd = env->fpus & ~(7 << 11);
 871     swd |= (env->fpstt & 7) << 11;
 872     cwd = env->fpuc;
 873     for (i = 0; i < 8; ++i) {
 874         twd |= (!env->fptags[i]) << i;
 875     }
 876     xsave->region[0] = (uint32_t)(swd << 16) + cwd;
 877     xsave->region[1] = (uint32_t)(fop << 16) + twd;
 878     memcpy(&xsave->region[XSAVE_ST_SPACE], env->fpregs,
 879             sizeof env->fpregs);
 880     memcpy(&xsave->region[XSAVE_XMM_SPACE], env->xmm_regs,
 881             sizeof env->xmm_regs);
 882     xsave->region[XSAVE_MXCSR] = env->mxcsr;
 883     *(uint64_t *)&xsave->region[XSAVE_XSTATE_BV] = env->xstate_bv;
 884     memcpy(&xsave->region[XSAVE_YMMH_SPACE], env->ymmh_regs,
 885             sizeof env->ymmh_regs);
 886     r = kvm_vcpu_ioctl(env, KVM_SET_XSAVE, xsave);
 887     qemu_free(xsave);
 888     return r;
 889 #else
 890     return kvm_put_fpu(env);
 891 #endif
 892 }
 893
 894 static int kvm_put_xcrs(CPUState *env)
 895 {
 896 #ifdef KVM_CAP_XCRS
 897     struct kvm_xcrs xcrs;
 898
 899     if (!kvm_has_xcrs()) {
 900         return 0;
 901     }
 902
 903     xcrs.nr_xcrs = 1;
 904     xcrs.flags = 0;
 905     xcrs.xcrs[0].xcr = 0;
 906     xcrs.xcrs[0].value = env->xcr0;
 907     return kvm_vcpu_ioctl(env, KVM_SET_XCRS, &xcrs);
 908 #else
 909     return 0;
 910 #endif
 911 }
 912
 913 static int kvm_put_sregs(CPUState *env)
 914 {
 915     struct kvm_sregs sregs;
 916
 917     memset(sregs.interrupt_bitmap, 0, sizeof(sregs.interrupt_bitmap));
 918     if (env->interrupt_injected >= 0) {
 919         sregs.interrupt_bitmap[env->interrupt_injected / 64] |=
 920                 (uint64_t)1 << (env->interrupt_injected % 64);
 921     }
 922
 923     if ((env->eflags & VM_MASK)) {
 924         set_v8086_seg(&sregs.cs, &env->segs[R_CS]);
 925         set_v8086_seg(&sregs.ds, &env->segs[R_DS]);
 926         set_v8086_seg(&sregs.es, &env->segs[R_ES]);
 927         set_v8086_seg(&sregs.fs, &env->segs[R_FS]);
 928         set_v8086_seg(&sregs.gs, &env->segs[R_GS]);
 929         set_v8086_seg(&sregs.ss, &env->segs[R_SS]);
 930     } else {
 931         set_seg(&sregs.cs, &env->segs[R_CS]);
 932         set_seg(&sregs.ds, &env->segs[R_DS]);
 933         set_seg(&sregs.es, &env->segs[R_ES]);
 934         set_seg(&sregs.fs, &env->segs[R_FS]);
 935         set_seg(&sregs.gs, &env->segs[R_GS]);
 936         set_seg(&sregs.ss, &env->segs[R_SS]);
 937     }
 938
 939     set_seg(&sregs.tr, &env->tr);
 940     set_seg(&sregs.ldt, &env->ldt);
 941
 942     sregs.idt.limit = env->idt.limit;
 943     sregs.idt.base = env->idt.base;
 944     sregs.gdt.limit = env->gdt.limit;
 945     sregs.gdt.base = env->gdt.base;
 946
 947     sregs.cr0 = env->cr[0];
 948     sregs.cr2 = env->cr[2];
 949     sregs.cr3 = env->cr[3];
 950     sregs.cr4 = env->cr[4];
 951
 952     sregs.cr8 = cpu_get_apic_tpr(env->apic_state);
 953     sregs.apic_base = cpu_get_apic_base(env->apic_state);
 954
 955     sregs.efer = env->efer;
 956
 957     return kvm_vcpu_ioctl(env, KVM_SET_SREGS, &sregs);
 958 }
 959
 960 static void kvm_msr_entry_set(struct kvm_msr_entry *entry,
 961                               uint32_t index, uint64_t value)
 962 {
 963     entry->index = index;
 964     entry->data = value;
 965 }
 966
 967 static int kvm_put_msrs(CPUState *env, int level)
 968 {
 969     struct {
 970         struct kvm_msrs info;
 971         struct kvm_msr_entry entries[100];
 972     } msr_data;
 973     struct kvm_msr_entry *msrs = msr_data.entries;
 974     int n = 0;
 975
 976     kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_CS, env->sysenter_cs);
 977     kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_ESP, env->sysenter_esp);
 978     kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_EIP, env->sysenter_eip);
 979     if (has_msr_star) {
 980         kvm_msr_entry_set(&msrs[n++], MSR_STAR, env->star);
 981     }
 982     if (has_msr_hsave_pa) {
 983         kvm_msr_entry_set(&msrs[n++], MSR_VM_HSAVE_PA, env->vm_hsave);
 984     }
 985 #ifdef TARGET_X86_64
 986     if (lm_capable_kernel) {
 987         kvm_msr_entry_set(&msrs[n++], MSR_CSTAR, env->cstar);
 988         kvm_msr_entry_set(&msrs[n++], MSR_KERNELGSBASE, env->kernelgsbase);
 989         kvm_msr_entry_set(&msrs[n++], MSR_FMASK, env->fmask);
 990         kvm_msr_entry_set(&msrs[n++], MSR_LSTAR, env->lstar);
 991     }
 992 #endif
 993     if (level == KVM_PUT_FULL_STATE) {
 994         /*
 995          * KVM is yet unable to synchronize TSC values of multiple VCPUs on
 996          * writeback. Until this is fixed, we only write the offset to SMP
 997          * guests after migration, desynchronizing the VCPUs, but avoiding
 998          * huge jump-backs that would occur without any writeback at all.
 999          */
1000         if (smp_cpus == 1 || env->tsc != 0) {
1001             kvm_msr_entry_set(&msrs[n++], MSR_IA32_TSC, env->tsc);
1002         }
1003     }
1004     /*
1005      * The following paravirtual MSRs have side effects on the guest or are
1006      * too heavy for normal writeback. Limit them to reset or full state
1007      * updates.
1008      */
1009     if (level >= KVM_PUT_RESET_STATE) {
1010         kvm_msr_entry_set(&msrs[n++], MSR_KVM_SYSTEM_TIME,
1011                           env->system_time_msr);
1012         kvm_msr_entry_set(&msrs[n++], MSR_KVM_WALL_CLOCK, env->wall_clock_msr);
1013 #if defined(CONFIG_KVM_PARA) && defined(KVM_CAP_ASYNC_PF)
1014         if (has_msr_async_pf_en) {
1015             kvm_msr_entry_set(&msrs[n++], MSR_KVM_ASYNC_PF_EN,
1016                               env->async_pf_en_msr);
1017         }
1018 #endif
1019     }
1020 #ifdef KVM_CAP_MCE
1021     if (env->mcg_cap) {
1022         int i;
1023
1024         if (level == KVM_PUT_RESET_STATE) {
1025             kvm_msr_entry_set(&msrs[n++], MSR_MCG_STATUS, env->mcg_status);
1026         } else if (level == KVM_PUT_FULL_STATE) {
1027             kvm_msr_entry_set(&msrs[n++], MSR_MCG_STATUS, env->mcg_status);
1028             kvm_msr_entry_set(&msrs[n++], MSR_MCG_CTL, env->mcg_ctl);
1029             for (i = 0; i < (env->mcg_cap & 0xff) * 4; i++) {
1030                 kvm_msr_entry_set(&msrs[n++], MSR_MC0_CTL + i, env->mce_banks[i]);
1031             }
1032         }
1033     }
1034 #endif
1035
1036     msr_data.info.nmsrs = n;
1037
1038     return kvm_vcpu_ioctl(env, KVM_SET_MSRS, &msr_data);
1039
1040 }
1041
1042
1043 static int kvm_get_fpu(CPUState *env)
1044 {
1045     struct kvm_fpu fpu;
1046     int i, ret;
1047
1048     ret = kvm_vcpu_ioctl(env, KVM_GET_FPU, &fpu);
1049     if (ret < 0) {
1050         return ret;
1051     }
1052
1053     env->fpstt = (fpu.fsw >> 11) & 7;
1054     env->fpus = fpu.fsw;
1055     env->fpuc = fpu.fcw;
1056     for (i = 0; i < 8; ++i) {
1057         env->fptags[i] = !((fpu.ftwx >> i) & 1);
1058     }
1059     memcpy(env->fpregs, fpu.fpr, sizeof env->fpregs);
1060     memcpy(env->xmm_regs, fpu.xmm, sizeof env->xmm_regs);
1061     env->mxcsr = fpu.mxcsr;
1062
1063     return 0;
1064 }
1065
1066 static int kvm_get_xsave(CPUState *env)
1067 {
1068 #ifdef KVM_CAP_XSAVE
1069     struct kvm_xsave* xsave;
1070     int ret, i;
1071     uint16_t cwd, swd, twd, fop;
1072
1073     if (!kvm_has_xsave()) {
1074         return kvm_get_fpu(env);
1075     }
1076
1077     xsave = qemu_memalign(4096, sizeof(struct kvm_xsave));
1078     ret = kvm_vcpu_ioctl(env, KVM_GET_XSAVE, xsave);
1079     if (ret < 0) {
1080         qemu_free(xsave);
1081         return ret;
1082     }
1083
1084     cwd = (uint16_t)xsave->region[0];
1085     swd = (uint16_t)(xsave->region[0] >> 16);
1086     twd = (uint16_t)xsave->region[1];
1087     fop = (uint16_t)(xsave->region[1] >> 16);
1088     env->fpstt = (swd >> 11) & 7;
1089     env->fpus = swd;
1090     env->fpuc = cwd;
1091     for (i = 0; i < 8; ++i) {
1092         env->fptags[i] = !((twd >> i) & 1);
1093     }
1094     env->mxcsr = xsave->region[XSAVE_MXCSR];
1095     memcpy(env->fpregs, &xsave->region[XSAVE_ST_SPACE],
1096             sizeof env->fpregs);
1097     memcpy(env->xmm_regs, &xsave->region[XSAVE_XMM_SPACE],
1098             sizeof env->xmm_regs);
1099     env->xstate_bv = *(uint64_t *)&xsave->region[XSAVE_XSTATE_BV];
1100     memcpy(env->ymmh_regs, &xsave->region[XSAVE_YMMH_SPACE],
1101             sizeof env->ymmh_regs);
1102     qemu_free(xsave);
1103     return 0;
1104 #else
1105     return kvm_get_fpu(env);
1106 #endif
1107 }
1108
1109 static int kvm_get_xcrs(CPUState *env)
1110 {
1111 #ifdef KVM_CAP_XCRS
1112     int i, ret;
1113     struct kvm_xcrs xcrs;
1114
1115     if (!kvm_has_xcrs()) {
1116         return 0;
1117     }
1118
1119     ret = kvm_vcpu_ioctl(env, KVM_GET_XCRS, &xcrs);
1120     if (ret < 0) {
1121         return ret;
1122     }
1123
1124     for (i = 0; i < xcrs.nr_xcrs; i++) {
1125         /* Only support xcr0 now */
1126         if (xcrs.xcrs[0].xcr == 0) {
1127             env->xcr0 = xcrs.xcrs[0].value;
1128             break;
1129         }
1130     }
1131     return 0;
1132 #else
1133     return 0;
1134 #endif
1135 }
1136
1137 static int kvm_get_sregs(CPUState *env)
1138 {
1139     struct kvm_sregs sregs;
1140     uint32_t hflags;
1141     int bit, i, ret;
1142
1143     ret = kvm_vcpu_ioctl(env, KVM_GET_SREGS, &sregs);
1144     if (ret < 0) {
1145         return ret;
1146     }
1147
1148     /* There can only be one pending IRQ set in the bitmap at a time, so try
1149        to find it and save its number instead (-1 for none). */
1150     env->interrupt_injected = -1;
1151     for (i = 0; i < ARRAY_SIZE(sregs.interrupt_bitmap); i++) {
1152         if (sregs.interrupt_bitmap[i]) {
1153             bit = ctz64(sregs.interrupt_bitmap[i]);
1154             env->interrupt_injected = i * 64 + bit;
1155             break;
1156         }
1157     }
1158
1159     get_seg(&env->segs[R_CS], &sregs.cs);
1160     get_seg(&env->segs[R_DS], &sregs.ds);
1161     get_seg(&env->segs[R_ES], &sregs.es);
1162     get_seg(&env->segs[R_FS], &sregs.fs);
1163     get_seg(&env->segs[R_GS], &sregs.gs);
1164     get_seg(&env->segs[R_SS], &sregs.ss);
1165
1166     get_seg(&env->tr, &sregs.tr);
1167     get_seg(&env->ldt, &sregs.ldt);
1168
1169     env->idt.limit = sregs.idt.limit;
1170     env->idt.base = sregs.idt.base;
1171     env->gdt.limit = sregs.gdt.limit;
1172     env->gdt.base = sregs.gdt.base;
1173
1174     env->cr[0] = sregs.cr0;
1175     env->cr[2] = sregs.cr2;
1176     env->cr[3] = sregs.cr3;
1177     env->cr[4] = sregs.cr4;
1178
1179     cpu_set_apic_base(env->apic_state, sregs.apic_base);
1180
1181     env->efer = sregs.efer;
1182     //cpu_set_apic_tpr(env->apic_state, sregs.cr8);
1183
1184 #define HFLAG_COPY_MASK \
1185     ~( HF_CPL_MASK | HF_PE_MASK | HF_MP_MASK | HF_EM_MASK | \
1186        HF_TS_MASK | HF_TF_MASK | HF_VM_MASK | HF_IOPL_MASK | \
1187        HF_OSFXSR_MASK | HF_LMA_MASK | HF_CS32_MASK | \
1188        HF_SS32_MASK | HF_CS64_MASK | HF_ADDSEG_MASK)
1189
1190     hflags = (env->segs[R_CS].flags >> DESC_DPL_SHIFT) & HF_CPL_MASK;
1191     hflags |= (env->cr[0] & CR0_PE_MASK) << (HF_PE_SHIFT - CR0_PE_SHIFT);
1192     hflags |= (env->cr[0] << (HF_MP_SHIFT - CR0_MP_SHIFT)) &
1193                 (HF_MP_MASK | HF_EM_MASK | HF_TS_MASK);
1194     hflags |= (env->eflags & (HF_TF_MASK | HF_VM_MASK | HF_IOPL_MASK));
1195     hflags |= (env->cr[4] & CR4_OSFXSR_MASK) <<
1196                 (HF_OSFXSR_SHIFT - CR4_OSFXSR_SHIFT);
1197
1198     if (env->efer & MSR_EFER_LMA) {
1199         hflags |= HF_LMA_MASK;
1200     }
1201
1202     if ((hflags & HF_LMA_MASK) && (env->segs[R_CS].flags & DESC_L_MASK)) {
1203         hflags |= HF_CS32_MASK | HF_SS32_MASK | HF_CS64_MASK;
1204     } else {
1205         hflags |= (env->segs[R_CS].flags & DESC_B_MASK) >>
1206                     (DESC_B_SHIFT - HF_CS32_SHIFT);
1207         hflags |= (env->segs[R_SS].flags & DESC_B_MASK) >>
1208                     (DESC_B_SHIFT - HF_SS32_SHIFT);
1209         if (!(env->cr[0] & CR0_PE_MASK) || (env->eflags & VM_MASK) ||
1210             !(hflags & HF_CS32_MASK)) {
1211             hflags |= HF_ADDSEG_MASK;
1212         } else {
1213             hflags |= ((env->segs[R_DS].base | env->segs[R_ES].base |
1214                         env->segs[R_SS].base) != 0) << HF_ADDSEG_SHIFT;
1215         }
1216     }
1217     env->hflags = (env->hflags & HFLAG_COPY_MASK) | hflags;
1218
1219     return 0;
1220 }
1221
1222 static int kvm_get_msrs(CPUState *env)
1223 {
1224     struct {
1225         struct kvm_msrs info;
1226         struct kvm_msr_entry entries[100];
1227     } msr_data;
1228     struct kvm_msr_entry *msrs = msr_data.entries;
1229     int ret, i, n;
1230
1231     n = 0;
1232     msrs[n++].index = MSR_IA32_SYSENTER_CS;
1233     msrs[n++].index = MSR_IA32_SYSENTER_ESP;
1234     msrs[n++].index = MSR_IA32_SYSENTER_EIP;
1235     if (has_msr_star) {
1236         msrs[n++].index = MSR_STAR;
1237     }
1238     if (has_msr_hsave_pa) {
1239         msrs[n++].index = MSR_VM_HSAVE_PA;
1240     }
1241
1242     if (!env->tsc_valid) {
1243         msrs[n++].index = MSR_IA32_TSC;
1244         env->tsc_valid = !vm_running;
1245     }
1246
1247 #ifdef TARGET_X86_64
1248     if (lm_capable_kernel) {
1249         msrs[n++].index = MSR_CSTAR;
1250         msrs[n++].index = MSR_KERNELGSBASE;
1251         msrs[n++].index = MSR_FMASK;
1252         msrs[n++].index = MSR_LSTAR;
1253     }
1254 #endif
1255     msrs[n++].index = MSR_KVM_SYSTEM_TIME;
1256     msrs[n++].index = MSR_KVM_WALL_CLOCK;
1257 #if defined(CONFIG_KVM_PARA) && defined(KVM_CAP_ASYNC_PF)
1258     if (has_msr_async_pf_en) {
1259         msrs[n++].index = MSR_KVM_ASYNC_PF_EN;
1260     }
1261 #endif
1262
1263 #ifdef KVM_CAP_MCE
1264     if (env->mcg_cap) {
1265         msrs[n++].index = MSR_MCG_STATUS;
1266         msrs[n++].index = MSR_MCG_CTL;
1267         for (i = 0; i < (env->mcg_cap & 0xff) * 4; i++) {
1268             msrs[n++].index = MSR_MC0_CTL + i;
1269         }
1270     }
1271 #endif
1272
1273     msr_data.info.nmsrs = n;
1274     ret = kvm_vcpu_ioctl(env, KVM_GET_MSRS, &msr_data);
1275     if (ret < 0) {
1276         return ret;
1277     }
1278
1279     for (i = 0; i < ret; i++) {
1280         switch (msrs[i].index) {
1281         case MSR_IA32_SYSENTER_CS:
1282             env->sysenter_cs = msrs[i].data;
1283             break;
1284         case MSR_IA32_SYSENTER_ESP:
1285             env->sysenter_esp = msrs[i].data;
1286             break;
1287         case MSR_IA32_SYSENTER_EIP:
1288             env->sysenter_eip = msrs[i].data;
1289             break;
1290         case MSR_STAR:
1291             env->star = msrs[i].data;
1292             break;
1293 #ifdef TARGET_X86_64
1294         case MSR_CSTAR:
1295             env->cstar = msrs[i].data;
1296             break;
1297         case MSR_KERNELGSBASE:
1298             env->kernelgsbase = msrs[i].data;
1299             break;
1300         case MSR_FMASK:
1301             env->fmask = msrs[i].data;
1302             break;
1303         case MSR_LSTAR:
1304             env->lstar = msrs[i].data;
1305             break;
1306 #endif
1307         case MSR_IA32_TSC:
1308             env->tsc = msrs[i].data;
1309             break;
1310         case MSR_VM_HSAVE_PA:
1311             env->vm_hsave = msrs[i].data;
1312             break;
1313         case MSR_KVM_SYSTEM_TIME:
1314             env->system_time_msr = msrs[i].data;
1315             break;
1316         case MSR_KVM_WALL_CLOCK:
1317             env->wall_clock_msr = msrs[i].data;
1318             break;
1319 #ifdef KVM_CAP_MCE
1320         case MSR_MCG_STATUS:
1321             env->mcg_status = msrs[i].data;
1322             break;
1323         case MSR_MCG_CTL:
1324             env->mcg_ctl = msrs[i].data;
1325             break;
1326 #endif
1327         default:
1328 #ifdef KVM_CAP_MCE
1329             if (msrs[i].index >= MSR_MC0_CTL &&
1330                 msrs[i].index < MSR_MC0_CTL + (env->mcg_cap & 0xff) * 4) {
1331                 env->mce_banks[msrs[i].index - MSR_MC0_CTL] = msrs[i].data;
1332             }
1333 #endif
1334             break;
1335 #if defined(CONFIG_KVM_PARA) && defined(KVM_CAP_ASYNC_PF)
1336         case MSR_KVM_ASYNC_PF_EN:
1337             env->async_pf_en_msr = msrs[i].data;
1338             break;
1339 #endif
1340         }
1341     }
1342
1343     return 0;
1344 }
1345
1346 static int kvm_put_mp_state(CPUState *env)
1347 {
1348     struct kvm_mp_state mp_state = { .mp_state = env->mp_state };
1349
1350     return kvm_vcpu_ioctl(env, KVM_SET_MP_STATE, &mp_state);
1351 }
1352
1353 static int kvm_get_mp_state(CPUState *env)
1354 {
1355     struct kvm_mp_state mp_state;
1356     int ret;
1357
1358     ret = kvm_vcpu_ioctl(env, KVM_GET_MP_STATE, &mp_state);
1359     if (ret < 0) {
1360         return ret;
1361     }
1362     env->mp_state = mp_state.mp_state;
1363     if (kvm_irqchip_in_kernel()) {
1364         env->halted = (mp_state.mp_state == KVM_MP_STATE_HALTED);
1365     }
1366     return 0;
1367 }
1368
1369 static int kvm_put_vcpu_events(CPUState *env, int level)
1370 {
1371 #ifdef KVM_CAP_VCPU_EVENTS
1372     struct kvm_vcpu_events events;
1373
1374     if (!kvm_has_vcpu_events()) {
1375         return 0;
1376     }
1377
1378     events.exception.injected = (env->exception_injected >= 0);
1379     events.exception.nr = env->exception_injected;
1380     events.exception.has_error_code = env->has_error_code;
1381     events.exception.error_code = env->error_code;
1382
1383     events.interrupt.injected = (env->interrupt_injected >= 0);
1384     events.interrupt.nr = env->interrupt_injected;
1385     events.interrupt.soft = env->soft_interrupt;
1386
1387     events.nmi.injected = env->nmi_injected;
1388     events.nmi.pending = env->nmi_pending;
1389     events.nmi.masked = !!(env->hflags2 & HF2_NMI_MASK);
1390
1391     events.sipi_vector = env->sipi_vector;
1392
1393     events.flags = 0;
1394     if (level >= KVM_PUT_RESET_STATE) {
1395         events.flags |=
1396             KVM_VCPUEVENT_VALID_NMI_PENDING | KVM_VCPUEVENT_VALID_SIPI_VECTOR;
1397     }
1398
1399     return kvm_vcpu_ioctl(env, KVM_SET_VCPU_EVENTS, &events);
1400 #else
1401     return 0;
1402 #endif
1403 }
1404
1405 static int kvm_get_vcpu_events(CPUState *env)
1406 {
1407 #ifdef KVM_CAP_VCPU_EVENTS
1408     struct kvm_vcpu_events events;
1409     int ret;
1410
1411     if (!kvm_has_vcpu_events()) {
1412         return 0;
1413     }
1414
1415     ret = kvm_vcpu_ioctl(env, KVM_GET_VCPU_EVENTS, &events);
1416     if (ret < 0) {
1417        return ret;
1418     }
1419     env->exception_injected =
1420        events.exception.injected ? events.exception.nr : -1;
1421     env->has_error_code = events.exception.has_error_code;
1422     env->error_code = events.exception.error_code;
1423
1424     env->interrupt_injected =
1425         events.interrupt.injected ? events.interrupt.nr : -1;
1426     env->soft_interrupt = events.interrupt.soft;
1427
1428     env->nmi_injected = events.nmi.injected;
1429     env->nmi_pending = events.nmi.pending;
1430     if (events.nmi.masked) {
1431         env->hflags2 |= HF2_NMI_MASK;
1432     } else {
1433         env->hflags2 &= ~HF2_NMI_MASK;
1434     }
1435
1436     env->sipi_vector = events.sipi_vector;
1437 #endif
1438
1439     return 0;
1440 }
1441
1442 static int kvm_guest_debug_workarounds(CPUState *env)
1443 {
1444     int ret = 0;
1445 #ifdef KVM_CAP_SET_GUEST_DEBUG
1446     unsigned long reinject_trap = 0;
1447
1448     if (!kvm_has_vcpu_events()) {
1449         if (env->exception_injected == 1) {
1450             reinject_trap = KVM_GUESTDBG_INJECT_DB;
1451         } else if (env->exception_injected == 3) {
1452             reinject_trap = KVM_GUESTDBG_INJECT_BP;
1453         }
1454         env->exception_injected = -1;
1455     }
1456
1457     /*
1458      * Kernels before KVM_CAP_X86_ROBUST_SINGLESTEP overwrote flags.TF
1459      * injected via SET_GUEST_DEBUG while updating GP regs. Work around this
1460      * by updating the debug state once again if single-stepping is on.
1461      * Another reason to call kvm_update_guest_debug here is a pending debug
1462      * trap raise by the guest. On kernels without SET_VCPU_EVENTS we have to
1463      * reinject them via SET_GUEST_DEBUG.
1464      */
1465     if (reinject_trap ||
1466         (!kvm_has_robust_singlestep() && env->singlestep_enabled)) {
1467         ret = kvm_update_guest_debug(env, reinject_trap);
1468     }
1469 #endif /* KVM_CAP_SET_GUEST_DEBUG */
1470     return ret;
1471 }
1472
1473 static int kvm_put_debugregs(CPUState *env)
1474 {
1475 #ifdef KVM_CAP_DEBUGREGS
1476     struct kvm_debugregs dbgregs;
1477     int i;
1478
1479     if (!kvm_has_debugregs()) {
1480         return 0;
1481     }
1482
1483     for (i = 0; i < 4; i++) {
1484         dbgregs.db[i] = env->dr[i];
1485     }
1486     dbgregs.dr6 = env->dr[6];
1487     dbgregs.dr7 = env->dr[7];
1488     dbgregs.flags = 0;
1489
1490     return kvm_vcpu_ioctl(env, KVM_SET_DEBUGREGS, &dbgregs);
1491 #else
1492     return 0;
1493 #endif
1494 }
1495
1496 static int kvm_get_debugregs(CPUState *env)
1497 {
1498 #ifdef KVM_CAP_DEBUGREGS
1499     struct kvm_debugregs dbgregs;
1500     int i, ret;
1501
1502     if (!kvm_has_debugregs()) {
1503         return 0;
1504     }
1505
1506     ret = kvm_vcpu_ioctl(env, KVM_GET_DEBUGREGS, &dbgregs);
1507     if (ret < 0) {
1508         return ret;
1509     }
1510     for (i = 0; i < 4; i++) {
1511         env->dr[i] = dbgregs.db[i];
1512     }
1513     env->dr[4] = env->dr[6] = dbgregs.dr6;
1514     env->dr[5] = env->dr[7] = dbgregs.dr7;
1515 #endif
1516
1517     return 0;
1518 }
1519
1520 int kvm_arch_put_registers(CPUState *env, int level)
1521 {
1522     int ret;
1523
1524     assert(cpu_is_stopped(env) || qemu_cpu_is_self(env));
1525
1526     ret = kvm_getput_regs(env, 1);
1527     if (ret < 0) {
1528         return ret;
1529     }
1530     ret = kvm_put_xsave(env);
1531     if (ret < 0) {
1532         return ret;
1533     }
1534     ret = kvm_put_xcrs(env);
1535     if (ret < 0) {
1536         return ret;
1537     }
1538     ret = kvm_put_sregs(env);
1539     if (ret < 0) {
1540         return ret;
1541     }
1542     ret = kvm_put_msrs(env, level);
1543     if (ret < 0) {
1544         return ret;
1545     }
1546     if (level >= KVM_PUT_RESET_STATE) {
1547         ret = kvm_put_mp_state(env);
1548         if (ret < 0) {
1549             return ret;
1550         }
1551     }
1552     ret = kvm_put_vcpu_events(env, level);
1553     if (ret < 0) {
1554         return ret;
1555     }
1556     ret = kvm_put_debugregs(env);
1557     if (ret < 0) {
1558         return ret;
1559     }
1560     /* must be last */
1561     ret = kvm_guest_debug_workarounds(env);
1562     if (ret < 0) {
1563         return ret;
1564     }
1565     return 0;
1566 }
1567
1568 int kvm_arch_get_registers(CPUState *env)
1569 {
1570     int ret;
1571
1572     assert(cpu_is_stopped(env) || qemu_cpu_is_self(env));
1573
1574     ret = kvm_getput_regs(env, 0);
1575     if (ret < 0) {
1576         return ret;
1577     }
1578     ret = kvm_get_xsave(env);
1579     if (ret < 0) {
1580         return ret;
1581     }
1582     ret = kvm_get_xcrs(env);
1583     if (ret < 0) {
1584         return ret;
1585     }
1586     ret = kvm_get_sregs(env);
1587     if (ret < 0) {
1588         return ret;
1589     }
1590     ret = kvm_get_msrs(env);
1591     if (ret < 0) {
1592         return ret;
1593     }
1594     ret = kvm_get_mp_state(env);
1595     if (ret < 0) {
1596         return ret;
1597     }
1598     ret = kvm_get_vcpu_events(env);
1599     if (ret < 0) {
1600         return ret;
1601     }
1602     ret = kvm_get_debugregs(env);
1603     if (ret < 0) {
1604         return ret;
1605     }
1606     return 0;
1607 }
1608
1609 void kvm_arch_pre_run(CPUState *env, struct kvm_run *run)
1610 {
1611     int ret;
1612
1613     /* Inject NMI */
1614     if (env->interrupt_request & CPU_INTERRUPT_NMI) {
1615         env->interrupt_request &= ~CPU_INTERRUPT_NMI;
1616         DPRINTF("injected NMI\n");
1617         ret = kvm_vcpu_ioctl(env, KVM_NMI);
1618         if (ret < 0) {
1619             fprintf(stderr, "KVM: injection failed, NMI lost (%s)\n",
1620                     strerror(-ret));
1621         }
1622     }
1623
1624     if (!kvm_irqchip_in_kernel()) {
1625         /* Force the VCPU out of its inner loop to process the INIT request */
1626         if (env->interrupt_request & CPU_INTERRUPT_INIT) {
1627             env->exit_request = 1;
1628         }
1629
1630         /* Try to inject an interrupt if the guest can accept it */
1631         if (run->ready_for_interrupt_injection &&
1632             (env->interrupt_request & CPU_INTERRUPT_HARD) &&
1633             (env->eflags & IF_MASK)) {
1634             int irq;
1635
1636             env->interrupt_request &= ~CPU_INTERRUPT_HARD;
1637             irq = cpu_get_pic_interrupt(env);
1638             if (irq >= 0) {
1639                 struct kvm_interrupt intr;
1640
1641                 intr.irq = irq;
1642                 DPRINTF("injected interrupt %d\n", irq);
1643                 ret = kvm_vcpu_ioctl(env, KVM_INTERRUPT, &intr);
1644                 if (ret < 0) {
1645                     fprintf(stderr,
1646                             "KVM: injection failed, interrupt lost (%s)\n",
1647                             strerror(-ret));
1648                 }
1649             }
1650         }
1651
1652         /* If we have an interrupt but the guest is not ready to receive an
1653          * interrupt, request an interrupt window exit.  This will
1654          * cause a return to userspace as soon as the guest is ready to
1655          * receive interrupts. */
1656         if ((env->interrupt_request & CPU_INTERRUPT_HARD)) {
1657             run->request_interrupt_window = 1;
1658         } else {
1659             run->request_interrupt_window = 0;
1660         }
1661
1662         DPRINTF("setting tpr\n");
1663         run->cr8 = cpu_get_apic_tpr(env->apic_state);
1664     }
1665 }
1666
1667 void kvm_arch_post_run(CPUState *env, struct kvm_run *run)
1668 {
1669     if (run->if_flag) {
1670         env->eflags |= IF_MASK;
1671     } else {
1672         env->eflags &= ~IF_MASK;
1673     }
1674     cpu_set_apic_tpr(env->apic_state, run->cr8);
1675     cpu_set_apic_base(env->apic_state, run->apic_base);
1676 }
1677
1678 int kvm_arch_process_async_events(CPUState *env)
1679 {
1680     if (kvm_irqchip_in_kernel()) {
1681         return 0;
1682     }
1683
1684     if (env->interrupt_request & (CPU_INTERRUPT_HARD | CPU_INTERRUPT_NMI)) {
1685         env->halted = 0;
1686     }
1687     if (env->interrupt_request & CPU_INTERRUPT_INIT) {
1688         kvm_cpu_synchronize_state(env);
1689         do_cpu_init(env);
1690     }
1691     if (env->interrupt_request & CPU_INTERRUPT_SIPI) {
1692         kvm_cpu_synchronize_state(env);
1693         do_cpu_sipi(env);
1694     }
1695
1696     return env->halted;
1697 }
1698
1699 static int kvm_handle_halt(CPUState *env)
1700 {
1701     if (!((env->interrupt_request & CPU_INTERRUPT_HARD) &&
1702           (env->eflags & IF_MASK)) &&
1703         !(env->interrupt_request & CPU_INTERRUPT_NMI)) {
1704         env->halted = 1;
1705         return 0;
1706     }
1707
1708     return 1;
1709 }
1710
1711 static bool host_supports_vmx(void)
1712 {
1713     uint32_t ecx, unused;
1714
1715     host_cpuid(1, 0, &unused, &unused, &ecx, &unused);
1716     return ecx & CPUID_EXT_VMX;
1717 }
1718
1719 #define VMX_INVALID_GUEST_STATE 0x80000021
1720
1721 int kvm_arch_handle_exit(CPUState *env, struct kvm_run *run)
1722 {
1723     uint64_t code;
1724     int ret = 0;
1725
1726     switch (run->exit_reason) {
1727     case KVM_EXIT_HLT:
1728         DPRINTF("handle_hlt\n");
1729         ret = kvm_handle_halt(env);
1730         break;
1731     case KVM_EXIT_SET_TPR:
1732         ret = 1;
1733         break;
1734     case KVM_EXIT_FAIL_ENTRY:
1735         code = run->fail_entry.hardware_entry_failure_reason;
1736         fprintf(stderr, "KVM: entry failed, hardware error 0x%" PRIx64 "\n",
1737                 code);
1738         if (host_supports_vmx() && code == VMX_INVALID_GUEST_STATE) {
1739             fprintf(stderr,
1740                     "\nIf you're runnning a guest on an Intel machine without "
1741                         "unrestricted mode\n"
1742                     "support, the failure can be most likely due to the guest "
1743                         "entering an invalid\n"
1744                     "state for Intel VT. For example, the guest maybe running "
1745                         "in big real mode\n"
1746                     "which is not supported on less recent Intel processors."
1747                         "\n\n");
1748         }
1749         ret = -1;
1750         break;
1751     case KVM_EXIT_EXCEPTION:
1752         fprintf(stderr, "KVM: exception %d exit (error code 0x%x)\n",
1753                 run->ex.exception, run->ex.error_code);
1754         ret = -1;
1755         break;
1756     default:
1757         fprintf(stderr, "KVM: unknown exit reason %d\n", run->exit_reason);
1758         ret = -1;
1759         break;
1760     }
1761
1762     return ret;
1763 }
1764
1765 #ifdef KVM_CAP_SET_GUEST_DEBUG
1766 int kvm_arch_insert_sw_breakpoint(CPUState *env, struct kvm_sw_breakpoint *bp)
1767 {
1768     static const uint8_t int3 = 0xcc;
1769
1770     if (cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&bp->saved_insn, 1, 0) ||
1771         cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&int3, 1, 1)) {
1772         return -EINVAL;
1773     }
1774     return 0;
1775 }
1776
1777 int kvm_arch_remove_sw_breakpoint(CPUState *env, struct kvm_sw_breakpoint *bp)
1778 {
1779     uint8_t int3;
1780
1781     if (cpu_memory_rw_debug(env, bp->pc, &int3, 1, 0) || int3 != 0xcc ||
1782         cpu_memory_rw_debug(env, bp->pc, (uint8_t *)&bp->saved_insn, 1, 1)) {
1783         return -EINVAL;
1784     }
1785     return 0;
1786 }
1787
1788 static struct {
1789     target_ulong addr;
1790     int len;
1791     int type;
1792 } hw_breakpoint[4];
1793
1794 static int nb_hw_breakpoint;
1795
1796 static int find_hw_breakpoint(target_ulong addr, int len, int type)
1797 {
1798     int n;
1799
1800     for (n = 0; n < nb_hw_breakpoint; n++) {
1801         if (hw_breakpoint[n].addr == addr && hw_breakpoint[n].type == type &&
1802             (hw_breakpoint[n].len == len || len == -1)) {
1803             return n;
1804         }
1805     }
1806     return -1;
1807 }
1808
1809 int kvm_arch_insert_hw_breakpoint(target_ulong addr,
1810                                   target_ulong len, int type)
1811 {
1812     switch (type) {
1813     case GDB_BREAKPOINT_HW:
1814         len = 1;
1815         break;
1816     case GDB_WATCHPOINT_WRITE:
1817     case GDB_WATCHPOINT_ACCESS:
1818         switch (len) {
1819         case 1:
1820             break;
1821         case 2:
1822         case 4:
1823         case 8:
1824             if (addr & (len - 1)) {
1825                 return -EINVAL;
1826             }
1827             break;
1828         default:
1829             return -EINVAL;
1830         }
1831         break;
1832     default:
1833         return -ENOSYS;
1834     }
1835
1836     if (nb_hw_breakpoint == 4) {
1837         return -ENOBUFS;
1838     }
1839     if (find_hw_breakpoint(addr, len, type) >= 0) {
1840         return -EEXIST;
1841     }
1842     hw_breakpoint[nb_hw_breakpoint].addr = addr;
1843     hw_breakpoint[nb_hw_breakpoint].len = len;
1844     hw_breakpoint[nb_hw_breakpoint].type = type;
1845     nb_hw_breakpoint++;
1846
1847     return 0;
1848 }
1849
1850 int kvm_arch_remove_hw_breakpoint(target_ulong addr,
1851                                   target_ulong len, int type)
1852 {
1853     int n;
1854
1855     n = find_hw_breakpoint(addr, (type == GDB_BREAKPOINT_HW) ? 1 : len, type);
1856     if (n < 0) {
1857         return -ENOENT;
1858     }
1859     nb_hw_breakpoint--;
1860     hw_breakpoint[n] = hw_breakpoint[nb_hw_breakpoint];
1861
1862     return 0;
1863 }
1864
1865 void kvm_arch_remove_all_hw_breakpoints(void)
1866 {
1867     nb_hw_breakpoint = 0;
1868 }
1869
1870 static CPUWatchpoint hw_watchpoint;
1871
1872 int kvm_arch_debug(struct kvm_debug_exit_arch *arch_info)
1873 {
1874     int handle = 0;
1875     int n;
1876
1877     if (arch_info->exception == 1) {
1878         if (arch_info->dr6 & (1 << 14)) {
1879             if (cpu_single_env->singlestep_enabled) {
1880                 handle = 1;
1881             }
1882         } else {
1883             for (n = 0; n < 4; n++) {
1884                 if (arch_info->dr6 & (1 << n)) {
1885                     switch ((arch_info->dr7 >> (16 + n*4)) & 0x3) {
1886                     case 0x0:
1887                         handle = 1;
1888                         break;
1889                     case 0x1:
1890                         handle = 1;
1891                         cpu_single_env->watchpoint_hit = &hw_watchpoint;
1892                         hw_watchpoint.vaddr = hw_breakpoint[n].addr;
1893                         hw_watchpoint.flags = BP_MEM_WRITE;
1894                         break;
1895                     case 0x3:
1896                         handle = 1;
1897                         cpu_single_env->watchpoint_hit = &hw_watchpoint;
1898                         hw_watchpoint.vaddr = hw_breakpoint[n].addr;
1899                         hw_watchpoint.flags = BP_MEM_ACCESS;
1900                         break;
1901                     }
1902                 }
1903             }
1904         }
1905     } else if (kvm_find_sw_breakpoint(cpu_single_env, arch_info->pc)) {
1906         handle = 1;
1907     }
1908     if (!handle) {
1909         cpu_synchronize_state(cpu_single_env);
1910         assert(cpu_single_env->exception_injected == -1);
1911
1912         cpu_single_env->exception_injected = arch_info->exception;
1913         cpu_single_env->has_error_code = 0;
1914     }
1915
1916     return handle;
1917 }
1918
1919 void kvm_arch_update_guest_debug(CPUState *env, struct kvm_guest_debug *dbg)
1920 {
1921     const uint8_t type_code[] = {
1922         [GDB_BREAKPOINT_HW] = 0x0,
1923         [GDB_WATCHPOINT_WRITE] = 0x1,
1924         [GDB_WATCHPOINT_ACCESS] = 0x3
1925     };
1926     const uint8_t len_code[] = {
1927         [1] = 0x0, [2] = 0x1, [4] = 0x3, [8] = 0x2
1928     };
1929     int n;
1930
1931     if (kvm_sw_breakpoints_active(env)) {
1932         dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP;
1933     }
1934     if (nb_hw_breakpoint > 0) {
1935         dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW_BP;
1936         dbg->arch.debugreg[7] = 0x0600;
1937         for (n = 0; n < nb_hw_breakpoint; n++) {
1938             dbg->arch.debugreg[n] = hw_breakpoint[n].addr;
1939             dbg->arch.debugreg[7] |= (2 << (n * 2)) |
1940                 (type_code[hw_breakpoint[n].type] << (16 + n*4)) |
1941                 ((uint32_t)len_code[hw_breakpoint[n].len] << (18 + n*4));
1942         }
1943     }
1944 }
1945 #endif /* KVM_CAP_SET_GUEST_DEBUG */
1946
1947 bool kvm_arch_stop_on_emulation_error(CPUState *env)
1948 {
1949     return !(env->cr[0] & CR0_PE_MASK) ||
1950            ((env->segs[R_CS].selector  & 3) != 3);
1951 }