target/i386/whpx-all.c

   1 /*
   2  * QEMU Windows Hypervisor Platform accelerator (WHPX)
   3  *
   4  * Copyright Microsoft Corp. 2017
   5  *
   6  * This work is licensed under the terms of the GNU GPL, version 2 or later.
   7  * See the COPYING file in the top-level directory.
   8  *
   9  */
  10
  11 #include "qemu/osdep.h"
  12 #include "cpu.h"
  13 #include "exec/address-spaces.h"
  14 #include "exec/ioport.h"
  15 #include "qemu-common.h"
  16 #include "sysemu/accel.h"
  17 #include "sysemu/whpx.h"
  18 #include "sysemu/cpus.h"
  19 #include "sysemu/runstate.h"
  20 #include "qemu/main-loop.h"
  21 #include "hw/boards.h"
  22 #include "qemu/error-report.h"
  23 #include "qapi/error.h"
  24 #include "migration/blocker.h"
  25 #include "whp-dispatch.h"
  26
  27 #include <WinHvPlatform.h>
  28 #include <WinHvEmulation.h>
  29
  30 struct whpx_state {
  31     uint64_t mem_quota;
  32     WHV_PARTITION_HANDLE partition;
  33 };
  34
  35 static const WHV_REGISTER_NAME whpx_register_names[] = {
  36
  37     /* X64 General purpose registers */
  38     WHvX64RegisterRax,
  39     WHvX64RegisterRcx,
  40     WHvX64RegisterRdx,
  41     WHvX64RegisterRbx,
  42     WHvX64RegisterRsp,
  43     WHvX64RegisterRbp,
  44     WHvX64RegisterRsi,
  45     WHvX64RegisterRdi,
  46     WHvX64RegisterR8,
  47     WHvX64RegisterR9,
  48     WHvX64RegisterR10,
  49     WHvX64RegisterR11,
  50     WHvX64RegisterR12,
  51     WHvX64RegisterR13,
  52     WHvX64RegisterR14,
  53     WHvX64RegisterR15,
  54     WHvX64RegisterRip,
  55     WHvX64RegisterRflags,
  56
  57     /* X64 Segment registers */
  58     WHvX64RegisterEs,
  59     WHvX64RegisterCs,
  60     WHvX64RegisterSs,
  61     WHvX64RegisterDs,
  62     WHvX64RegisterFs,
  63     WHvX64RegisterGs,
  64     WHvX64RegisterLdtr,
  65     WHvX64RegisterTr,
  66
  67     /* X64 Table registers */
  68     WHvX64RegisterIdtr,
  69     WHvX64RegisterGdtr,
  70
  71     /* X64 Control Registers */
  72     WHvX64RegisterCr0,
  73     WHvX64RegisterCr2,
  74     WHvX64RegisterCr3,
  75     WHvX64RegisterCr4,
  76     WHvX64RegisterCr8,
  77
  78     /* X64 Debug Registers */
  79     /*
  80      * WHvX64RegisterDr0,
  81      * WHvX64RegisterDr1,
  82      * WHvX64RegisterDr2,
  83      * WHvX64RegisterDr3,
  84      * WHvX64RegisterDr6,
  85      * WHvX64RegisterDr7,
  86      */
  87
  88     /* X64 Floating Point and Vector Registers */
  89     WHvX64RegisterXmm0,
  90     WHvX64RegisterXmm1,
  91     WHvX64RegisterXmm2,
  92     WHvX64RegisterXmm3,
  93     WHvX64RegisterXmm4,
  94     WHvX64RegisterXmm5,
  95     WHvX64RegisterXmm6,
  96     WHvX64RegisterXmm7,
  97     WHvX64RegisterXmm8,
  98     WHvX64RegisterXmm9,
  99     WHvX64RegisterXmm10,
 100     WHvX64RegisterXmm11,
 101     WHvX64RegisterXmm12,
 102     WHvX64RegisterXmm13,
 103     WHvX64RegisterXmm14,
 104     WHvX64RegisterXmm15,
 105     WHvX64RegisterFpMmx0,
 106     WHvX64RegisterFpMmx1,
 107     WHvX64RegisterFpMmx2,
 108     WHvX64RegisterFpMmx3,
 109     WHvX64RegisterFpMmx4,
 110     WHvX64RegisterFpMmx5,
 111     WHvX64RegisterFpMmx6,
 112     WHvX64RegisterFpMmx7,
 113     WHvX64RegisterFpControlStatus,
 114     WHvX64RegisterXmmControlStatus,
 115
 116     /* X64 MSRs */
 117     WHvX64RegisterTsc,
 118     WHvX64RegisterEfer,
 119 #ifdef TARGET_X86_64
 120     WHvX64RegisterKernelGsBase,
 121 #endif
 122     WHvX64RegisterApicBase,
 123     /* WHvX64RegisterPat, */
 124     WHvX64RegisterSysenterCs,
 125     WHvX64RegisterSysenterEip,
 126     WHvX64RegisterSysenterEsp,
 127     WHvX64RegisterStar,
 128 #ifdef TARGET_X86_64
 129     WHvX64RegisterLstar,
 130     WHvX64RegisterCstar,
 131     WHvX64RegisterSfmask,
 132 #endif
 133
 134     /* Interrupt / Event Registers */
 135     /*
 136      * WHvRegisterPendingInterruption,
 137      * WHvRegisterInterruptState,
 138      * WHvRegisterPendingEvent0,
 139      * WHvRegisterPendingEvent1
 140      * WHvX64RegisterDeliverabilityNotifications,
 141      */
 142 };
 143
 144 struct whpx_register_set {
 145     WHV_REGISTER_VALUE values[RTL_NUMBER_OF(whpx_register_names)];
 146 };
 147
 148 struct whpx_vcpu {
 149     WHV_EMULATOR_HANDLE emulator;
 150     bool window_registered;
 151     bool interruptable;
 152     uint64_t tpr;
 153     uint64_t apic_base;
 154     bool interruption_pending;
 155
 156     /* Must be the last field as it may have a tail */
 157     WHV_RUN_VP_EXIT_CONTEXT exit_ctx;
 158 };
 159
 160 static bool whpx_allowed;
 161 static bool whp_dispatch_initialized;
 162 static HMODULE hWinHvPlatform, hWinHvEmulation;
 163
 164 struct whpx_state whpx_global;
 165 struct WHPDispatch whp_dispatch;
 166
 167
 168 /*
 169  * VP support
 170  */
 171
 172 static struct whpx_vcpu *get_whpx_vcpu(CPUState *cpu)
 173 {
 174     return (struct whpx_vcpu *)cpu->hax_vcpu;
 175 }
 176
 177 static WHV_X64_SEGMENT_REGISTER whpx_seg_q2h(const SegmentCache *qs, int v86,
 178                                              int r86)
 179 {
 180     WHV_X64_SEGMENT_REGISTER hs;
 181     unsigned flags = qs->flags;
 182
 183     hs.Base = qs->base;
 184     hs.Limit = qs->limit;
 185     hs.Selector = qs->selector;
 186
 187     if (v86) {
 188         hs.Attributes = 0;
 189         hs.SegmentType = 3;
 190         hs.Present = 1;
 191         hs.DescriptorPrivilegeLevel = 3;
 192         hs.NonSystemSegment = 1;
 193
 194     } else {
 195         hs.Attributes = (flags >> DESC_TYPE_SHIFT);
 196
 197         if (r86) {
 198             /* hs.Base &= 0xfffff; */
 199         }
 200     }
 201
 202     return hs;
 203 }
 204
 205 static SegmentCache whpx_seg_h2q(const WHV_X64_SEGMENT_REGISTER *hs)
 206 {
 207     SegmentCache qs;
 208
 209     qs.base = hs->Base;
 210     qs.limit = hs->Limit;
 211     qs.selector = hs->Selector;
 212
 213     qs.flags = ((uint32_t)hs->Attributes) << DESC_TYPE_SHIFT;
 214
 215     return qs;
 216 }
 217
 218 static void whpx_set_registers(CPUState *cpu)
 219 {
 220     struct whpx_state *whpx = &whpx_global;
 221     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
 222     struct CPUX86State *env = (CPUArchState *)(cpu->env_ptr);
 223     X86CPU *x86_cpu = X86_CPU(cpu);
 224     struct whpx_register_set vcxt;
 225     HRESULT hr;
 226     int idx;
 227     int idx_next;
 228     int i;
 229     int v86, r86;
 230
 231     assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
 232
 233     memset(&vcxt, 0, sizeof(struct whpx_register_set));
 234
 235     v86 = (env->eflags & VM_MASK);
 236     r86 = !(env->cr[0] & CR0_PE_MASK);
 237
 238     vcpu->tpr = cpu_get_apic_tpr(x86_cpu->apic_state);
 239     vcpu->apic_base = cpu_get_apic_base(x86_cpu->apic_state);
 240
 241     idx = 0;
 242
 243     /* Indexes for first 16 registers match between HV and QEMU definitions */
 244     idx_next = 16;
 245     for (idx = 0; idx < CPU_NB_REGS; idx += 1) {
 246         vcxt.values[idx].Reg64 = (uint64_t)env->regs[idx];
 247     }
 248     idx = idx_next;
 249
 250     /* Same goes for RIP and RFLAGS */
 251     assert(whpx_register_names[idx] == WHvX64RegisterRip);
 252     vcxt.values[idx++].Reg64 = env->eip;
 253
 254     assert(whpx_register_names[idx] == WHvX64RegisterRflags);
 255     vcxt.values[idx++].Reg64 = env->eflags;
 256
 257     /* Translate 6+4 segment registers. HV and QEMU order matches  */
 258     assert(idx == WHvX64RegisterEs);
 259     for (i = 0; i < 6; i += 1, idx += 1) {
 260         vcxt.values[idx].Segment = whpx_seg_q2h(&env->segs[i], v86, r86);
 261     }
 262
 263     assert(idx == WHvX64RegisterLdtr);
 264     vcxt.values[idx++].Segment = whpx_seg_q2h(&env->ldt, 0, 0);
 265
 266     assert(idx == WHvX64RegisterTr);
 267     vcxt.values[idx++].Segment = whpx_seg_q2h(&env->tr, 0, 0);
 268
 269     assert(idx == WHvX64RegisterIdtr);
 270     vcxt.values[idx].Table.Base = env->idt.base;
 271     vcxt.values[idx].Table.Limit = env->idt.limit;
 272     idx += 1;
 273
 274     assert(idx == WHvX64RegisterGdtr);
 275     vcxt.values[idx].Table.Base = env->gdt.base;
 276     vcxt.values[idx].Table.Limit = env->gdt.limit;
 277     idx += 1;
 278
 279     /* CR0, 2, 3, 4, 8 */
 280     assert(whpx_register_names[idx] == WHvX64RegisterCr0);
 281     vcxt.values[idx++].Reg64 = env->cr[0];
 282     assert(whpx_register_names[idx] == WHvX64RegisterCr2);
 283     vcxt.values[idx++].Reg64 = env->cr[2];
 284     assert(whpx_register_names[idx] == WHvX64RegisterCr3);
 285     vcxt.values[idx++].Reg64 = env->cr[3];
 286     assert(whpx_register_names[idx] == WHvX64RegisterCr4);
 287     vcxt.values[idx++].Reg64 = env->cr[4];
 288     assert(whpx_register_names[idx] == WHvX64RegisterCr8);
 289     vcxt.values[idx++].Reg64 = vcpu->tpr;
 290
 291     /* 8 Debug Registers - Skipped */
 292
 293     /* 16 XMM registers */
 294     assert(whpx_register_names[idx] == WHvX64RegisterXmm0);
 295     idx_next = idx + 16;
 296     for (i = 0; i < sizeof(env->xmm_regs) / sizeof(ZMMReg); i += 1, idx += 1) {
 297         vcxt.values[idx].Reg128.Low64 = env->xmm_regs[i].ZMM_Q(0);
 298         vcxt.values[idx].Reg128.High64 = env->xmm_regs[i].ZMM_Q(1);
 299     }
 300     idx = idx_next;
 301
 302     /* 8 FP registers */
 303     assert(whpx_register_names[idx] == WHvX64RegisterFpMmx0);
 304     for (i = 0; i < 8; i += 1, idx += 1) {
 305         vcxt.values[idx].Fp.AsUINT128.Low64 = env->fpregs[i].mmx.MMX_Q(0);
 306         /* vcxt.values[idx].Fp.AsUINT128.High64 =
 307                env->fpregs[i].mmx.MMX_Q(1);
 308         */
 309     }
 310
 311     /* FP control status register */
 312     assert(whpx_register_names[idx] == WHvX64RegisterFpControlStatus);
 313     vcxt.values[idx].FpControlStatus.FpControl = env->fpuc;
 314     vcxt.values[idx].FpControlStatus.FpStatus =
 315         (env->fpus & ~0x3800) | (env->fpstt & 0x7) << 11;
 316     vcxt.values[idx].FpControlStatus.FpTag = 0;
 317     for (i = 0; i < 8; ++i) {
 318         vcxt.values[idx].FpControlStatus.FpTag |= (!env->fptags[i]) << i;
 319     }
 320     vcxt.values[idx].FpControlStatus.Reserved = 0;
 321     vcxt.values[idx].FpControlStatus.LastFpOp = env->fpop;
 322     vcxt.values[idx].FpControlStatus.LastFpRip = env->fpip;
 323     idx += 1;
 324
 325     /* XMM control status register */
 326     assert(whpx_register_names[idx] == WHvX64RegisterXmmControlStatus);
 327     vcxt.values[idx].XmmControlStatus.LastFpRdp = 0;
 328     vcxt.values[idx].XmmControlStatus.XmmStatusControl = env->mxcsr;
 329     vcxt.values[idx].XmmControlStatus.XmmStatusControlMask = 0x0000ffff;
 330     idx += 1;
 331
 332     /* MSRs */
 333     assert(whpx_register_names[idx] == WHvX64RegisterTsc);
 334     vcxt.values[idx++].Reg64 = env->tsc;
 335     assert(whpx_register_names[idx] == WHvX64RegisterEfer);
 336     vcxt.values[idx++].Reg64 = env->efer;
 337 #ifdef TARGET_X86_64
 338     assert(whpx_register_names[idx] == WHvX64RegisterKernelGsBase);
 339     vcxt.values[idx++].Reg64 = env->kernelgsbase;
 340 #endif
 341
 342     assert(whpx_register_names[idx] == WHvX64RegisterApicBase);
 343     vcxt.values[idx++].Reg64 = vcpu->apic_base;
 344
 345     /* WHvX64RegisterPat - Skipped */
 346
 347     assert(whpx_register_names[idx] == WHvX64RegisterSysenterCs);
 348     vcxt.values[idx++].Reg64 = env->sysenter_cs;
 349     assert(whpx_register_names[idx] == WHvX64RegisterSysenterEip);
 350     vcxt.values[idx++].Reg64 = env->sysenter_eip;
 351     assert(whpx_register_names[idx] == WHvX64RegisterSysenterEsp);
 352     vcxt.values[idx++].Reg64 = env->sysenter_esp;
 353     assert(whpx_register_names[idx] == WHvX64RegisterStar);
 354     vcxt.values[idx++].Reg64 = env->star;
 355 #ifdef TARGET_X86_64
 356     assert(whpx_register_names[idx] == WHvX64RegisterLstar);
 357     vcxt.values[idx++].Reg64 = env->lstar;
 358     assert(whpx_register_names[idx] == WHvX64RegisterCstar);
 359     vcxt.values[idx++].Reg64 = env->cstar;
 360     assert(whpx_register_names[idx] == WHvX64RegisterSfmask);
 361     vcxt.values[idx++].Reg64 = env->fmask;
 362 #endif
 363
 364     /* Interrupt / Event Registers - Skipped */
 365
 366     assert(idx == RTL_NUMBER_OF(whpx_register_names));
 367
 368     hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
 369         whpx->partition, cpu->cpu_index,
 370         whpx_register_names,
 371         RTL_NUMBER_OF(whpx_register_names),
 372         &vcxt.values[0]);
 373
 374     if (FAILED(hr)) {
 375         error_report("WHPX: Failed to set virtual processor context, hr=%08lx",
 376                      hr);
 377     }
 378
 379     return;
 380 }
 381
 382 static void whpx_get_registers(CPUState *cpu)
 383 {
 384     struct whpx_state *whpx = &whpx_global;
 385     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
 386     struct CPUX86State *env = (CPUArchState *)(cpu->env_ptr);
 387     X86CPU *x86_cpu = X86_CPU(cpu);
 388     struct whpx_register_set vcxt;
 389     uint64_t tpr, apic_base;
 390     HRESULT hr;
 391     int idx;
 392     int idx_next;
 393     int i;
 394
 395     assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
 396
 397     hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
 398         whpx->partition, cpu->cpu_index,
 399         whpx_register_names,
 400         RTL_NUMBER_OF(whpx_register_names),
 401         &vcxt.values[0]);
 402     if (FAILED(hr)) {
 403         error_report("WHPX: Failed to get virtual processor context, hr=%08lx",
 404                      hr);
 405     }
 406
 407     idx = 0;
 408
 409     /* Indexes for first 16 registers match between HV and QEMU definitions */
 410     idx_next = 16;
 411     for (idx = 0; idx < CPU_NB_REGS; idx += 1) {
 412         env->regs[idx] = vcxt.values[idx].Reg64;
 413     }
 414     idx = idx_next;
 415
 416     /* Same goes for RIP and RFLAGS */
 417     assert(whpx_register_names[idx] == WHvX64RegisterRip);
 418     env->eip = vcxt.values[idx++].Reg64;
 419     assert(whpx_register_names[idx] == WHvX64RegisterRflags);
 420     env->eflags = vcxt.values[idx++].Reg64;
 421
 422     /* Translate 6+4 segment registers. HV and QEMU order matches  */
 423     assert(idx == WHvX64RegisterEs);
 424     for (i = 0; i < 6; i += 1, idx += 1) {
 425         env->segs[i] = whpx_seg_h2q(&vcxt.values[idx].Segment);
 426     }
 427
 428     assert(idx == WHvX64RegisterLdtr);
 429     env->ldt = whpx_seg_h2q(&vcxt.values[idx++].Segment);
 430     assert(idx == WHvX64RegisterTr);
 431     env->tr = whpx_seg_h2q(&vcxt.values[idx++].Segment);
 432     assert(idx == WHvX64RegisterIdtr);
 433     env->idt.base = vcxt.values[idx].Table.Base;
 434     env->idt.limit = vcxt.values[idx].Table.Limit;
 435     idx += 1;
 436     assert(idx == WHvX64RegisterGdtr);
 437     env->gdt.base = vcxt.values[idx].Table.Base;
 438     env->gdt.limit = vcxt.values[idx].Table.Limit;
 439     idx += 1;
 440
 441     /* CR0, 2, 3, 4, 8 */
 442     assert(whpx_register_names[idx] == WHvX64RegisterCr0);
 443     env->cr[0] = vcxt.values[idx++].Reg64;
 444     assert(whpx_register_names[idx] == WHvX64RegisterCr2);
 445     env->cr[2] = vcxt.values[idx++].Reg64;
 446     assert(whpx_register_names[idx] == WHvX64RegisterCr3);
 447     env->cr[3] = vcxt.values[idx++].Reg64;
 448     assert(whpx_register_names[idx] == WHvX64RegisterCr4);
 449     env->cr[4] = vcxt.values[idx++].Reg64;
 450     assert(whpx_register_names[idx] == WHvX64RegisterCr8);
 451     tpr = vcxt.values[idx++].Reg64;
 452     if (tpr != vcpu->tpr) {
 453         vcpu->tpr = tpr;
 454         cpu_set_apic_tpr(x86_cpu->apic_state, tpr);
 455     }
 456
 457     /* 8 Debug Registers - Skipped */
 458
 459     /* 16 XMM registers */
 460     assert(whpx_register_names[idx] == WHvX64RegisterXmm0);
 461     idx_next = idx + 16;
 462     for (i = 0; i < sizeof(env->xmm_regs) / sizeof(ZMMReg); i += 1, idx += 1) {
 463         env->xmm_regs[i].ZMM_Q(0) = vcxt.values[idx].Reg128.Low64;
 464         env->xmm_regs[i].ZMM_Q(1) = vcxt.values[idx].Reg128.High64;
 465     }
 466     idx = idx_next;
 467
 468     /* 8 FP registers */
 469     assert(whpx_register_names[idx] == WHvX64RegisterFpMmx0);
 470     for (i = 0; i < 8; i += 1, idx += 1) {
 471         env->fpregs[i].mmx.MMX_Q(0) = vcxt.values[idx].Fp.AsUINT128.Low64;
 472         /* env->fpregs[i].mmx.MMX_Q(1) =
 473                vcxt.values[idx].Fp.AsUINT128.High64;
 474         */
 475     }
 476
 477     /* FP control status register */
 478     assert(whpx_register_names[idx] == WHvX64RegisterFpControlStatus);
 479     env->fpuc = vcxt.values[idx].FpControlStatus.FpControl;
 480     env->fpstt = (vcxt.values[idx].FpControlStatus.FpStatus >> 11) & 0x7;
 481     env->fpus = vcxt.values[idx].FpControlStatus.FpStatus & ~0x3800;
 482     for (i = 0; i < 8; ++i) {
 483         env->fptags[i] = !((vcxt.values[idx].FpControlStatus.FpTag >> i) & 1);
 484     }
 485     env->fpop = vcxt.values[idx].FpControlStatus.LastFpOp;
 486     env->fpip = vcxt.values[idx].FpControlStatus.LastFpRip;
 487     idx += 1;
 488
 489     /* XMM control status register */
 490     assert(whpx_register_names[idx] == WHvX64RegisterXmmControlStatus);
 491     env->mxcsr = vcxt.values[idx].XmmControlStatus.XmmStatusControl;
 492     idx += 1;
 493
 494     /* MSRs */
 495     assert(whpx_register_names[idx] == WHvX64RegisterTsc);
 496     env->tsc = vcxt.values[idx++].Reg64;
 497     assert(whpx_register_names[idx] == WHvX64RegisterEfer);
 498     env->efer = vcxt.values[idx++].Reg64;
 499 #ifdef TARGET_X86_64
 500     assert(whpx_register_names[idx] == WHvX64RegisterKernelGsBase);
 501     env->kernelgsbase = vcxt.values[idx++].Reg64;
 502 #endif
 503
 504     assert(whpx_register_names[idx] == WHvX64RegisterApicBase);
 505     apic_base = vcxt.values[idx++].Reg64;
 506     if (apic_base != vcpu->apic_base) {
 507         vcpu->apic_base = apic_base;
 508         cpu_set_apic_base(x86_cpu->apic_state, vcpu->apic_base);
 509     }
 510
 511     /* WHvX64RegisterPat - Skipped */
 512
 513     assert(whpx_register_names[idx] == WHvX64RegisterSysenterCs);
 514     env->sysenter_cs = vcxt.values[idx++].Reg64;
 515     assert(whpx_register_names[idx] == WHvX64RegisterSysenterEip);
 516     env->sysenter_eip = vcxt.values[idx++].Reg64;
 517     assert(whpx_register_names[idx] == WHvX64RegisterSysenterEsp);
 518     env->sysenter_esp = vcxt.values[idx++].Reg64;
 519     assert(whpx_register_names[idx] == WHvX64RegisterStar);
 520     env->star = vcxt.values[idx++].Reg64;
 521 #ifdef TARGET_X86_64
 522     assert(whpx_register_names[idx] == WHvX64RegisterLstar);
 523     env->lstar = vcxt.values[idx++].Reg64;
 524     assert(whpx_register_names[idx] == WHvX64RegisterCstar);
 525     env->cstar = vcxt.values[idx++].Reg64;
 526     assert(whpx_register_names[idx] == WHvX64RegisterSfmask);
 527     env->fmask = vcxt.values[idx++].Reg64;
 528 #endif
 529
 530     /* Interrupt / Event Registers - Skipped */
 531
 532     assert(idx == RTL_NUMBER_OF(whpx_register_names));
 533
 534     return;
 535 }
 536
 537 static HRESULT CALLBACK whpx_emu_ioport_callback(
 538     void *ctx,
 539     WHV_EMULATOR_IO_ACCESS_INFO *IoAccess)
 540 {
 541     MemTxAttrs attrs = { 0 };
 542     address_space_rw(&address_space_io, IoAccess->Port, attrs,
 543                      &IoAccess->Data, IoAccess->AccessSize,
 544                      IoAccess->Direction);
 545     return S_OK;
 546 }
 547
 548 static HRESULT CALLBACK whpx_emu_mmio_callback(
 549     void *ctx,
 550     WHV_EMULATOR_MEMORY_ACCESS_INFO *ma)
 551 {
 552     cpu_physical_memory_rw(ma->GpaAddress, ma->Data, ma->AccessSize,
 553                            ma->Direction);
 554     return S_OK;
 555 }
 556
 557 static HRESULT CALLBACK whpx_emu_getreg_callback(
 558     void *ctx,
 559     const WHV_REGISTER_NAME *RegisterNames,
 560     UINT32 RegisterCount,
 561     WHV_REGISTER_VALUE *RegisterValues)
 562 {
 563     HRESULT hr;
 564     struct whpx_state *whpx = &whpx_global;
 565     CPUState *cpu = (CPUState *)ctx;
 566
 567     hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
 568         whpx->partition, cpu->cpu_index,
 569         RegisterNames, RegisterCount,
 570         RegisterValues);
 571     if (FAILED(hr)) {
 572         error_report("WHPX: Failed to get virtual processor registers,"
 573                      " hr=%08lx", hr);
 574     }
 575
 576     return hr;
 577 }
 578
 579 static HRESULT CALLBACK whpx_emu_setreg_callback(
 580     void *ctx,
 581     const WHV_REGISTER_NAME *RegisterNames,
 582     UINT32 RegisterCount,
 583     const WHV_REGISTER_VALUE *RegisterValues)
 584 {
 585     HRESULT hr;
 586     struct whpx_state *whpx = &whpx_global;
 587     CPUState *cpu = (CPUState *)ctx;
 588
 589     hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
 590         whpx->partition, cpu->cpu_index,
 591         RegisterNames, RegisterCount,
 592         RegisterValues);
 593     if (FAILED(hr)) {
 594         error_report("WHPX: Failed to set virtual processor registers,"
 595                      " hr=%08lx", hr);
 596     }
 597
 598     /*
 599      * The emulator just successfully wrote the register state. We clear the
 600      * dirty state so we avoid the double write on resume of the VP.
 601      */
 602     cpu->vcpu_dirty = false;
 603
 604     return hr;
 605 }
 606
 607 static HRESULT CALLBACK whpx_emu_translate_callback(
 608     void *ctx,
 609     WHV_GUEST_VIRTUAL_ADDRESS Gva,
 610     WHV_TRANSLATE_GVA_FLAGS TranslateFlags,
 611     WHV_TRANSLATE_GVA_RESULT_CODE *TranslationResult,
 612     WHV_GUEST_PHYSICAL_ADDRESS *Gpa)
 613 {
 614     HRESULT hr;
 615     struct whpx_state *whpx = &whpx_global;
 616     CPUState *cpu = (CPUState *)ctx;
 617     WHV_TRANSLATE_GVA_RESULT res;
 618
 619     hr = whp_dispatch.WHvTranslateGva(whpx->partition, cpu->cpu_index,
 620                                       Gva, TranslateFlags, &res, Gpa);
 621     if (FAILED(hr)) {
 622         error_report("WHPX: Failed to translate GVA, hr=%08lx", hr);
 623     } else {
 624         *TranslationResult = res.ResultCode;
 625     }
 626
 627     return hr;
 628 }
 629
 630 static const WHV_EMULATOR_CALLBACKS whpx_emu_callbacks = {
 631     .Size = sizeof(WHV_EMULATOR_CALLBACKS),
 632     .WHvEmulatorIoPortCallback = whpx_emu_ioport_callback,
 633     .WHvEmulatorMemoryCallback = whpx_emu_mmio_callback,
 634     .WHvEmulatorGetVirtualProcessorRegisters = whpx_emu_getreg_callback,
 635     .WHvEmulatorSetVirtualProcessorRegisters = whpx_emu_setreg_callback,
 636     .WHvEmulatorTranslateGvaPage = whpx_emu_translate_callback,
 637 };
 638
 639 static int whpx_handle_mmio(CPUState *cpu, WHV_MEMORY_ACCESS_CONTEXT *ctx)
 640 {
 641     HRESULT hr;
 642     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
 643     WHV_EMULATOR_STATUS emu_status;
 644
 645     hr = whp_dispatch.WHvEmulatorTryMmioEmulation(
 646         vcpu->emulator, cpu,
 647         &vcpu->exit_ctx.VpContext, ctx,
 648         &emu_status);
 649     if (FAILED(hr)) {
 650         error_report("WHPX: Failed to parse MMIO access, hr=%08lx", hr);
 651         return -1;
 652     }
 653
 654     if (!emu_status.EmulationSuccessful) {
 655         error_report("WHPX: Failed to emulate MMIO access with"
 656                      " EmulatorReturnStatus: %u", emu_status.AsUINT32);
 657         return -1;
 658     }
 659
 660     return 0;
 661 }
 662
 663 static int whpx_handle_portio(CPUState *cpu,
 664                               WHV_X64_IO_PORT_ACCESS_CONTEXT *ctx)
 665 {
 666     HRESULT hr;
 667     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
 668     WHV_EMULATOR_STATUS emu_status;
 669
 670     hr = whp_dispatch.WHvEmulatorTryIoEmulation(
 671         vcpu->emulator, cpu,
 672         &vcpu->exit_ctx.VpContext, ctx,
 673         &emu_status);
 674     if (FAILED(hr)) {
 675         error_report("WHPX: Failed to parse PortIO access, hr=%08lx", hr);
 676         return -1;
 677     }
 678
 679     if (!emu_status.EmulationSuccessful) {
 680         error_report("WHPX: Failed to emulate PortIO access with"
 681                      " EmulatorReturnStatus: %u", emu_status.AsUINT32);
 682         return -1;
 683     }
 684
 685     return 0;
 686 }
 687
 688 static int whpx_handle_halt(CPUState *cpu)
 689 {
 690     struct CPUX86State *env = (CPUArchState *)(cpu->env_ptr);
 691     int ret = 0;
 692
 693     qemu_mutex_lock_iothread();
 694     if (!((cpu->interrupt_request & CPU_INTERRUPT_HARD) &&
 695           (env->eflags & IF_MASK)) &&
 696         !(cpu->interrupt_request & CPU_INTERRUPT_NMI)) {
 697         cpu->exception_index = EXCP_HLT;
 698         cpu->halted = true;
 699         ret = 1;
 700     }
 701     qemu_mutex_unlock_iothread();
 702
 703     return ret;
 704 }
 705
 706 static void whpx_vcpu_pre_run(CPUState *cpu)
 707 {
 708     HRESULT hr;
 709     struct whpx_state *whpx = &whpx_global;
 710     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
 711     struct CPUX86State *env = (CPUArchState *)(cpu->env_ptr);
 712     X86CPU *x86_cpu = X86_CPU(cpu);
 713     int irq;
 714     uint8_t tpr;
 715     WHV_X64_PENDING_INTERRUPTION_REGISTER new_int;
 716     UINT32 reg_count = 0;
 717     WHV_REGISTER_VALUE reg_values[3];
 718     WHV_REGISTER_NAME reg_names[3];
 719
 720     memset(&new_int, 0, sizeof(new_int));
 721     memset(reg_values, 0, sizeof(reg_values));
 722
 723     qemu_mutex_lock_iothread();
 724
 725     /* Inject NMI */
 726     if (!vcpu->interruption_pending &&
 727         cpu->interrupt_request & (CPU_INTERRUPT_NMI | CPU_INTERRUPT_SMI)) {
 728         if (cpu->interrupt_request & CPU_INTERRUPT_NMI) {
 729             cpu->interrupt_request &= ~CPU_INTERRUPT_NMI;
 730             vcpu->interruptable = false;
 731             new_int.InterruptionType = WHvX64PendingNmi;
 732             new_int.InterruptionPending = 1;
 733             new_int.InterruptionVector = 2;
 734         }
 735         if (cpu->interrupt_request & CPU_INTERRUPT_SMI) {
 736             cpu->interrupt_request &= ~CPU_INTERRUPT_SMI;
 737         }
 738     }
 739
 740     /*
 741      * Force the VCPU out of its inner loop to process any INIT requests or
 742      * commit pending TPR access.
 743      */
 744     if (cpu->interrupt_request & (CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR)) {
 745         if ((cpu->interrupt_request & CPU_INTERRUPT_INIT) &&
 746             !(env->hflags & HF_SMM_MASK)) {
 747             cpu->exit_request = 1;
 748         }
 749         if (cpu->interrupt_request & CPU_INTERRUPT_TPR) {
 750             cpu->exit_request = 1;
 751         }
 752     }
 753
 754     /* Get pending hard interruption or replay one that was overwritten */
 755     if (!vcpu->interruption_pending &&
 756         vcpu->interruptable && (env->eflags & IF_MASK)) {
 757         assert(!new_int.InterruptionPending);
 758         if (cpu->interrupt_request & CPU_INTERRUPT_HARD) {
 759             cpu->interrupt_request &= ~CPU_INTERRUPT_HARD;
 760             irq = cpu_get_pic_interrupt(env);
 761             if (irq >= 0) {
 762                 new_int.InterruptionType = WHvX64PendingInterrupt;
 763                 new_int.InterruptionPending = 1;
 764                 new_int.InterruptionVector = irq;
 765             }
 766         }
 767     }
 768
 769     /* Setup interrupt state if new one was prepared */
 770     if (new_int.InterruptionPending) {
 771         reg_values[reg_count].PendingInterruption = new_int;
 772         reg_names[reg_count] = WHvRegisterPendingInterruption;
 773         reg_count += 1;
 774     }
 775
 776     /* Sync the TPR to the CR8 if was modified during the intercept */
 777     tpr = cpu_get_apic_tpr(x86_cpu->apic_state);
 778     if (tpr != vcpu->tpr) {
 779         vcpu->tpr = tpr;
 780         reg_values[reg_count].Reg64 = tpr;
 781         cpu->exit_request = 1;
 782         reg_names[reg_count] = WHvX64RegisterCr8;
 783         reg_count += 1;
 784     }
 785
 786     /* Update the state of the interrupt delivery notification */
 787     if (!vcpu->window_registered &&
 788         cpu->interrupt_request & CPU_INTERRUPT_HARD) {
 789         reg_values[reg_count].DeliverabilityNotifications.InterruptNotification
 790             = 1;
 791         vcpu->window_registered = 1;
 792         reg_names[reg_count] = WHvX64RegisterDeliverabilityNotifications;
 793         reg_count += 1;
 794     }
 795
 796     qemu_mutex_unlock_iothread();
 797
 798     if (reg_count) {
 799         hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
 800             whpx->partition, cpu->cpu_index,
 801             reg_names, reg_count, reg_values);
 802         if (FAILED(hr)) {
 803             error_report("WHPX: Failed to set interrupt state registers,"
 804                          " hr=%08lx", hr);
 805         }
 806     }
 807
 808     return;
 809 }
 810
 811 static void whpx_vcpu_post_run(CPUState *cpu)
 812 {
 813     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
 814     struct CPUX86State *env = (CPUArchState *)(cpu->env_ptr);
 815     X86CPU *x86_cpu = X86_CPU(cpu);
 816
 817     env->eflags = vcpu->exit_ctx.VpContext.Rflags;
 818
 819     uint64_t tpr = vcpu->exit_ctx.VpContext.Cr8;
 820     if (vcpu->tpr != tpr) {
 821         vcpu->tpr = tpr;
 822         qemu_mutex_lock_iothread();
 823         cpu_set_apic_tpr(x86_cpu->apic_state, vcpu->tpr);
 824         qemu_mutex_unlock_iothread();
 825     }
 826
 827     vcpu->interruption_pending =
 828         vcpu->exit_ctx.VpContext.ExecutionState.InterruptionPending;
 829
 830     vcpu->interruptable =
 831         !vcpu->exit_ctx.VpContext.ExecutionState.InterruptShadow;
 832
 833     return;
 834 }
 835
 836 static void whpx_vcpu_process_async_events(CPUState *cpu)
 837 {
 838     struct CPUX86State *env = (CPUArchState *)(cpu->env_ptr);
 839     X86CPU *x86_cpu = X86_CPU(cpu);
 840     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
 841
 842     if ((cpu->interrupt_request & CPU_INTERRUPT_INIT) &&
 843         !(env->hflags & HF_SMM_MASK)) {
 844
 845         do_cpu_init(x86_cpu);
 846         cpu->vcpu_dirty = true;
 847         vcpu->interruptable = true;
 848     }
 849
 850     if (cpu->interrupt_request & CPU_INTERRUPT_POLL) {
 851         cpu->interrupt_request &= ~CPU_INTERRUPT_POLL;
 852         apic_poll_irq(x86_cpu->apic_state);
 853     }
 854
 855     if (((cpu->interrupt_request & CPU_INTERRUPT_HARD) &&
 856          (env->eflags & IF_MASK)) ||
 857         (cpu->interrupt_request & CPU_INTERRUPT_NMI)) {
 858         cpu->halted = false;
 859     }
 860
 861     if (cpu->interrupt_request & CPU_INTERRUPT_SIPI) {
 862         if (!cpu->vcpu_dirty) {
 863             whpx_get_registers(cpu);
 864         }
 865         do_cpu_sipi(x86_cpu);
 866     }
 867
 868     if (cpu->interrupt_request & CPU_INTERRUPT_TPR) {
 869         cpu->interrupt_request &= ~CPU_INTERRUPT_TPR;
 870         if (!cpu->vcpu_dirty) {
 871             whpx_get_registers(cpu);
 872         }
 873         apic_handle_tpr_access_report(x86_cpu->apic_state, env->eip,
 874                                       env->tpr_access_type);
 875     }
 876
 877     return;
 878 }
 879
 880 static int whpx_vcpu_run(CPUState *cpu)
 881 {
 882     HRESULT hr;
 883     struct whpx_state *whpx = &whpx_global;
 884     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
 885     int ret;
 886
 887     whpx_vcpu_process_async_events(cpu);
 888     if (cpu->halted) {
 889         cpu->exception_index = EXCP_HLT;
 890         atomic_set(&cpu->exit_request, false);
 891         return 0;
 892     }
 893
 894     qemu_mutex_unlock_iothread();
 895     cpu_exec_start(cpu);
 896
 897     do {
 898         if (cpu->vcpu_dirty) {
 899             whpx_set_registers(cpu);
 900             cpu->vcpu_dirty = false;
 901         }
 902
 903         whpx_vcpu_pre_run(cpu);
 904
 905         if (atomic_read(&cpu->exit_request)) {
 906             whpx_vcpu_kick(cpu);
 907         }
 908
 909         hr = whp_dispatch.WHvRunVirtualProcessor(
 910             whpx->partition, cpu->cpu_index,
 911             &vcpu->exit_ctx, sizeof(vcpu->exit_ctx));
 912
 913         if (FAILED(hr)) {
 914             error_report("WHPX: Failed to exec a virtual processor,"
 915                          " hr=%08lx", hr);
 916             ret = -1;
 917             break;
 918         }
 919
 920         whpx_vcpu_post_run(cpu);
 921
 922         switch (vcpu->exit_ctx.ExitReason) {
 923         case WHvRunVpExitReasonMemoryAccess:
 924             ret = whpx_handle_mmio(cpu, &vcpu->exit_ctx.MemoryAccess);
 925             break;
 926
 927         case WHvRunVpExitReasonX64IoPortAccess:
 928             ret = whpx_handle_portio(cpu, &vcpu->exit_ctx.IoPortAccess);
 929             break;
 930
 931         case WHvRunVpExitReasonX64InterruptWindow:
 932             vcpu->window_registered = 0;
 933             ret = 0;
 934             break;
 935
 936         case WHvRunVpExitReasonX64Halt:
 937             ret = whpx_handle_halt(cpu);
 938             break;
 939
 940         case WHvRunVpExitReasonCanceled:
 941             cpu->exception_index = EXCP_INTERRUPT;
 942             ret = 1;
 943             break;
 944
 945         case WHvRunVpExitReasonX64MsrAccess: {
 946             WHV_REGISTER_VALUE reg_values[3] = {0};
 947             WHV_REGISTER_NAME reg_names[3];
 948             UINT32 reg_count;
 949
 950             reg_names[0] = WHvX64RegisterRip;
 951             reg_names[1] = WHvX64RegisterRax;
 952             reg_names[2] = WHvX64RegisterRdx;
 953
 954             reg_values[0].Reg64 =
 955                 vcpu->exit_ctx.VpContext.Rip +
 956                 vcpu->exit_ctx.VpContext.InstructionLength;
 957
 958             /*
 959              * For all unsupported MSR access we:
 960              *     ignore writes
 961              *     return 0 on read.
 962              */
 963             reg_count = vcpu->exit_ctx.MsrAccess.AccessInfo.IsWrite ?
 964                         1 : 3;
 965
 966             hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
 967                 whpx->partition,
 968                 cpu->cpu_index,
 969                 reg_names, reg_count,
 970                 reg_values);
 971
 972             if (FAILED(hr)) {
 973                 error_report("WHPX: Failed to set MsrAccess state "
 974                              " registers, hr=%08lx", hr);
 975             }
 976             ret = 0;
 977             break;
 978         }
 979         case WHvRunVpExitReasonX64Cpuid: {
 980             WHV_REGISTER_VALUE reg_values[5];
 981             WHV_REGISTER_NAME reg_names[5];
 982             UINT32 reg_count = 5;
 983             UINT64 rip, rax, rcx, rdx, rbx;
 984
 985             memset(reg_values, 0, sizeof(reg_values));
 986
 987             rip = vcpu->exit_ctx.VpContext.Rip +
 988                   vcpu->exit_ctx.VpContext.InstructionLength;
 989             switch (vcpu->exit_ctx.CpuidAccess.Rax) {
 990             case 1:
 991                 rax = vcpu->exit_ctx.CpuidAccess.DefaultResultRax;
 992                 /* Advertise that we are running on a hypervisor */
 993                 rcx =
 994                     vcpu->exit_ctx.CpuidAccess.DefaultResultRcx |
 995                     CPUID_EXT_HYPERVISOR;
 996
 997                 rdx = vcpu->exit_ctx.CpuidAccess.DefaultResultRdx;
 998                 rbx = vcpu->exit_ctx.CpuidAccess.DefaultResultRbx;
 999                 break;
1000             case 0x80000001:
1001                 rax = vcpu->exit_ctx.CpuidAccess.DefaultResultRax;
1002                 /* Remove any support of OSVW */
1003                 rcx =
1004                     vcpu->exit_ctx.CpuidAccess.DefaultResultRcx &
1005                     ~CPUID_EXT3_OSVW;
1006
1007                 rdx = vcpu->exit_ctx.CpuidAccess.DefaultResultRdx;
1008                 rbx = vcpu->exit_ctx.CpuidAccess.DefaultResultRbx;
1009                 break;
1010             default:
1011                 rax = vcpu->exit_ctx.CpuidAccess.DefaultResultRax;
1012                 rcx = vcpu->exit_ctx.CpuidAccess.DefaultResultRcx;
1013                 rdx = vcpu->exit_ctx.CpuidAccess.DefaultResultRdx;
1014                 rbx = vcpu->exit_ctx.CpuidAccess.DefaultResultRbx;
1015             }
1016
1017             reg_names[0] = WHvX64RegisterRip;
1018             reg_names[1] = WHvX64RegisterRax;
1019             reg_names[2] = WHvX64RegisterRcx;
1020             reg_names[3] = WHvX64RegisterRdx;
1021             reg_names[4] = WHvX64RegisterRbx;
1022
1023             reg_values[0].Reg64 = rip;
1024             reg_values[1].Reg64 = rax;
1025             reg_values[2].Reg64 = rcx;
1026             reg_values[3].Reg64 = rdx;
1027             reg_values[4].Reg64 = rbx;
1028
1029             hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
1030                 whpx->partition, cpu->cpu_index,
1031                 reg_names,
1032                 reg_count,
1033                 reg_values);
1034
1035             if (FAILED(hr)) {
1036                 error_report("WHPX: Failed to set CpuidAccess state registers,"
1037                              " hr=%08lx", hr);
1038             }
1039             ret = 0;
1040             break;
1041         }
1042         case WHvRunVpExitReasonNone:
1043         case WHvRunVpExitReasonUnrecoverableException:
1044         case WHvRunVpExitReasonInvalidVpRegisterValue:
1045         case WHvRunVpExitReasonUnsupportedFeature:
1046         case WHvRunVpExitReasonException:
1047         default:
1048             error_report("WHPX: Unexpected VP exit code %d",
1049                          vcpu->exit_ctx.ExitReason);
1050             whpx_get_registers(cpu);
1051             qemu_mutex_lock_iothread();
1052             qemu_system_guest_panicked(cpu_get_crash_info(cpu));
1053             qemu_mutex_unlock_iothread();
1054             break;
1055         }
1056
1057     } while (!ret);
1058
1059     cpu_exec_end(cpu);
1060     qemu_mutex_lock_iothread();
1061     current_cpu = cpu;
1062
1063     atomic_set(&cpu->exit_request, false);
1064
1065     return ret < 0;
1066 }
1067
1068 static void do_whpx_cpu_synchronize_state(CPUState *cpu, run_on_cpu_data arg)
1069 {
1070     whpx_get_registers(cpu);
1071     cpu->vcpu_dirty = true;
1072 }
1073
1074 static void do_whpx_cpu_synchronize_post_reset(CPUState *cpu,
1075                                                run_on_cpu_data arg)
1076 {
1077     whpx_set_registers(cpu);
1078     cpu->vcpu_dirty = false;
1079 }
1080
1081 static void do_whpx_cpu_synchronize_post_init(CPUState *cpu,
1082                                               run_on_cpu_data arg)
1083 {
1084     whpx_set_registers(cpu);
1085     cpu->vcpu_dirty = false;
1086 }
1087
1088 static void do_whpx_cpu_synchronize_pre_loadvm(CPUState *cpu,
1089                                                run_on_cpu_data arg)
1090 {
1091     cpu->vcpu_dirty = true;
1092 }
1093
1094 /*
1095  * CPU support.
1096  */
1097
1098 void whpx_cpu_synchronize_state(CPUState *cpu)
1099 {
1100     if (!cpu->vcpu_dirty) {
1101         run_on_cpu(cpu, do_whpx_cpu_synchronize_state, RUN_ON_CPU_NULL);
1102     }
1103 }
1104
1105 void whpx_cpu_synchronize_post_reset(CPUState *cpu)
1106 {
1107     run_on_cpu(cpu, do_whpx_cpu_synchronize_post_reset, RUN_ON_CPU_NULL);
1108 }
1109
1110 void whpx_cpu_synchronize_post_init(CPUState *cpu)
1111 {
1112     run_on_cpu(cpu, do_whpx_cpu_synchronize_post_init, RUN_ON_CPU_NULL);
1113 }
1114
1115 void whpx_cpu_synchronize_pre_loadvm(CPUState *cpu)
1116 {
1117     run_on_cpu(cpu, do_whpx_cpu_synchronize_pre_loadvm, RUN_ON_CPU_NULL);
1118 }
1119
1120 /*
1121  * Vcpu support.
1122  */
1123
1124 static Error *whpx_migration_blocker;
1125
1126 int whpx_init_vcpu(CPUState *cpu)
1127 {
1128     HRESULT hr;
1129     struct whpx_state *whpx = &whpx_global;
1130     struct whpx_vcpu *vcpu;
1131     Error *local_error = NULL;
1132
1133     /* Add migration blockers for all unsupported features of the
1134      * Windows Hypervisor Platform
1135      */
1136     if (whpx_migration_blocker == NULL) {
1137         error_setg(&whpx_migration_blocker,
1138                "State blocked due to non-migratable CPUID feature support,"
1139                "dirty memory tracking support, and XSAVE/XRSTOR support");
1140
1141         (void)migrate_add_blocker(whpx_migration_blocker, &local_error);
1142         if (local_error) {
1143             error_report_err(local_error);
1144             migrate_del_blocker(whpx_migration_blocker);
1145             error_free(whpx_migration_blocker);
1146             return -EINVAL;
1147         }
1148     }
1149
1150     vcpu = g_malloc0(sizeof(struct whpx_vcpu));
1151
1152     if (!vcpu) {
1153         error_report("WHPX: Failed to allocte VCPU context.");
1154         return -ENOMEM;
1155     }
1156
1157     hr = whp_dispatch.WHvEmulatorCreateEmulator(
1158         &whpx_emu_callbacks,
1159         &vcpu->emulator);
1160     if (FAILED(hr)) {
1161         error_report("WHPX: Failed to setup instruction completion support,"
1162                      " hr=%08lx", hr);
1163         g_free(vcpu);
1164         return -EINVAL;
1165     }
1166
1167     hr = whp_dispatch.WHvCreateVirtualProcessor(
1168         whpx->partition, cpu->cpu_index, 0);
1169     if (FAILED(hr)) {
1170         error_report("WHPX: Failed to create a virtual processor,"
1171                      " hr=%08lx", hr);
1172         whp_dispatch.WHvEmulatorDestroyEmulator(vcpu->emulator);
1173         g_free(vcpu);
1174         return -EINVAL;
1175     }
1176
1177     vcpu->interruptable = true;
1178
1179     cpu->vcpu_dirty = true;
1180     cpu->hax_vcpu = (struct hax_vcpu_state *)vcpu;
1181
1182     return 0;
1183 }
1184
1185 int whpx_vcpu_exec(CPUState *cpu)
1186 {
1187     int ret;
1188     int fatal;
1189
1190     for (;;) {
1191         if (cpu->exception_index >= EXCP_INTERRUPT) {
1192             ret = cpu->exception_index;
1193             cpu->exception_index = -1;
1194             break;
1195         }
1196
1197         fatal = whpx_vcpu_run(cpu);
1198
1199         if (fatal) {
1200             error_report("WHPX: Failed to exec a virtual processor");
1201             abort();
1202         }
1203     }
1204
1205     return ret;
1206 }
1207
1208 void whpx_destroy_vcpu(CPUState *cpu)
1209 {
1210     struct whpx_state *whpx = &whpx_global;
1211     struct whpx_vcpu *vcpu = get_whpx_vcpu(cpu);
1212
1213     whp_dispatch.WHvDeleteVirtualProcessor(whpx->partition, cpu->cpu_index);
1214     whp_dispatch.WHvEmulatorDestroyEmulator(vcpu->emulator);
1215     g_free(cpu->hax_vcpu);
1216     return;
1217 }
1218
1219 void whpx_vcpu_kick(CPUState *cpu)
1220 {
1221     struct whpx_state *whpx = &whpx_global;
1222     whp_dispatch.WHvCancelRunVirtualProcessor(
1223         whpx->partition, cpu->cpu_index, 0);
1224 }
1225
1226 /*
1227  * Memory support.
1228  */
1229
1230 static void whpx_update_mapping(hwaddr start_pa, ram_addr_t size,
1231                                 void *host_va, int add, int rom,
1232                                 const char *name)
1233 {
1234     struct whpx_state *whpx = &whpx_global;
1235     HRESULT hr;
1236
1237     /*
1238     if (add) {
1239         printf("WHPX: ADD PA:%p Size:%p, Host:%p, %s, '%s'\n",
1240                (void*)start_pa, (void*)size, host_va,
1241                (rom ? "ROM" : "RAM"), name);
1242     } else {
1243         printf("WHPX: DEL PA:%p Size:%p, Host:%p,      '%s'\n",
1244                (void*)start_pa, (void*)size, host_va, name);
1245     }
1246     */
1247
1248     if (add) {
1249         hr = whp_dispatch.WHvMapGpaRange(whpx->partition,
1250                                          host_va,
1251                                          start_pa,
1252                                          size,
1253                                          (WHvMapGpaRangeFlagRead |
1254                                           WHvMapGpaRangeFlagExecute |
1255                                           (rom ? 0 : WHvMapGpaRangeFlagWrite)));
1256     } else {
1257         hr = whp_dispatch.WHvUnmapGpaRange(whpx->partition,
1258                                            start_pa,
1259                                            size);
1260     }
1261
1262     if (FAILED(hr)) {
1263         error_report("WHPX: Failed to %s GPA range '%s' PA:%p, Size:%p bytes,"
1264                      " Host:%p, hr=%08lx",
1265                      (add ? "MAP" : "UNMAP"), name,
1266                      (void *)(uintptr_t)start_pa, (void *)size, host_va, hr);
1267     }
1268 }
1269
1270 static void whpx_process_section(MemoryRegionSection *section, int add)
1271 {
1272     MemoryRegion *mr = section->mr;
1273     hwaddr start_pa = section->offset_within_address_space;
1274     ram_addr_t size = int128_get64(section->size);
1275     unsigned int delta;
1276     uint64_t host_va;
1277
1278     if (!memory_region_is_ram(mr)) {
1279         return;
1280     }
1281
1282     delta = qemu_real_host_page_size - (start_pa & ~qemu_real_host_page_mask);
1283     delta &= ~qemu_real_host_page_mask;
1284     if (delta > size) {
1285         return;
1286     }
1287     start_pa += delta;
1288     size -= delta;
1289     size &= qemu_real_host_page_mask;
1290     if (!size || (start_pa & ~qemu_real_host_page_mask)) {
1291         return;
1292     }
1293
1294     host_va = (uintptr_t)memory_region_get_ram_ptr(mr)
1295             + section->offset_within_region + delta;
1296
1297     whpx_update_mapping(start_pa, size, (void *)(uintptr_t)host_va, add,
1298                         memory_region_is_rom(mr), mr->name);
1299 }
1300
1301 static void whpx_region_add(MemoryListener *listener,
1302                            MemoryRegionSection *section)
1303 {
1304     memory_region_ref(section->mr);
1305     whpx_process_section(section, 1);
1306 }
1307
1308 static void whpx_region_del(MemoryListener *listener,
1309                            MemoryRegionSection *section)
1310 {
1311     whpx_process_section(section, 0);
1312     memory_region_unref(section->mr);
1313 }
1314
1315 static void whpx_transaction_begin(MemoryListener *listener)
1316 {
1317 }
1318
1319 static void whpx_transaction_commit(MemoryListener *listener)
1320 {
1321 }
1322
1323 static void whpx_log_sync(MemoryListener *listener,
1324                          MemoryRegionSection *section)
1325 {
1326     MemoryRegion *mr = section->mr;
1327
1328     if (!memory_region_is_ram(mr)) {
1329         return;
1330     }
1331
1332     memory_region_set_dirty(mr, 0, int128_get64(section->size));
1333 }
1334
1335 static MemoryListener whpx_memory_listener = {
1336     .begin = whpx_transaction_begin,
1337     .commit = whpx_transaction_commit,
1338     .region_add = whpx_region_add,
1339     .region_del = whpx_region_del,
1340     .log_sync = whpx_log_sync,
1341     .priority = 10,
1342 };
1343
1344 static void whpx_memory_init(void)
1345 {
1346     memory_listener_register(&whpx_memory_listener, &address_space_memory);
1347 }
1348
1349 static void whpx_handle_interrupt(CPUState *cpu, int mask)
1350 {
1351     cpu->interrupt_request |= mask;
1352
1353     if (!qemu_cpu_is_self(cpu)) {
1354         qemu_cpu_kick(cpu);
1355     }
1356 }
1357
1358 /*
1359  * Load the functions from the given library, using the given handle. If a
1360  * handle is provided, it is used, otherwise the library is opened. The
1361  * handle will be updated on return with the opened one.
1362  */
1363 static bool load_whp_dispatch_fns(HMODULE *handle,
1364     WHPFunctionList function_list)
1365 {
1366     HMODULE hLib = *handle;
1367
1368     #define WINHV_PLATFORM_DLL "WinHvPlatform.dll"
1369     #define WINHV_EMULATION_DLL "WinHvEmulation.dll"
1370     #define WHP_LOAD_FIELD(return_type, function_name, signature) \
1371         whp_dispatch.function_name = \
1372             (function_name ## _t)GetProcAddress(hLib, #function_name); \
1373         if (!whp_dispatch.function_name) { \
1374             error_report("Could not load function %s", #function_name); \
1375             goto error; \
1376         } \
1377
1378     #define WHP_LOAD_LIB(lib_name, handle_lib) \
1379     if (!handle_lib) { \
1380         handle_lib = LoadLibrary(lib_name); \
1381         if (!handle_lib) { \
1382             error_report("Could not load library %s.", lib_name); \
1383             goto error; \
1384         } \
1385     } \
1386
1387     switch (function_list) {
1388     case WINHV_PLATFORM_FNS_DEFAULT:
1389         WHP_LOAD_LIB(WINHV_PLATFORM_DLL, hLib)
1390         LIST_WINHVPLATFORM_FUNCTIONS(WHP_LOAD_FIELD)
1391         break;
1392
1393     case WINHV_EMULATION_FNS_DEFAULT:
1394         WHP_LOAD_LIB(WINHV_EMULATION_DLL, hLib)
1395         LIST_WINHVEMULATION_FUNCTIONS(WHP_LOAD_FIELD)
1396         break;
1397     }
1398
1399     *handle = hLib;
1400     return true;
1401
1402 error:
1403     if (hLib) {
1404         FreeLibrary(hLib);
1405     }
1406
1407     return false;
1408 }
1409
1410 /*
1411  * Partition support
1412  */
1413
1414 static int whpx_accel_init(MachineState *ms)
1415 {
1416     struct whpx_state *whpx;
1417     int ret;
1418     HRESULT hr;
1419     WHV_CAPABILITY whpx_cap;
1420     UINT32 whpx_cap_size;
1421     WHV_PARTITION_PROPERTY prop;
1422
1423     whpx = &whpx_global;
1424
1425     if (!init_whp_dispatch()) {
1426         ret = -ENOSYS;
1427         goto error;
1428     }
1429
1430     memset(whpx, 0, sizeof(struct whpx_state));
1431     whpx->mem_quota = ms->ram_size;
1432
1433     hr = whp_dispatch.WHvGetCapability(
1434         WHvCapabilityCodeHypervisorPresent, &whpx_cap,
1435         sizeof(whpx_cap), &whpx_cap_size);
1436     if (FAILED(hr) || !whpx_cap.HypervisorPresent) {
1437         error_report("WHPX: No accelerator found, hr=%08lx", hr);
1438         ret = -ENOSPC;
1439         goto error;
1440     }
1441
1442     hr = whp_dispatch.WHvCreatePartition(&whpx->partition);
1443     if (FAILED(hr)) {
1444         error_report("WHPX: Failed to create partition, hr=%08lx", hr);
1445         ret = -EINVAL;
1446         goto error;
1447     }
1448
1449     memset(&prop, 0, sizeof(WHV_PARTITION_PROPERTY));
1450     prop.ProcessorCount = ms->smp.cpus;
1451     hr = whp_dispatch.WHvSetPartitionProperty(
1452         whpx->partition,
1453         WHvPartitionPropertyCodeProcessorCount,
1454         &prop,
1455         sizeof(WHV_PARTITION_PROPERTY));
1456
1457     if (FAILED(hr)) {
1458         error_report("WHPX: Failed to set partition core count to %d,"
1459                      " hr=%08lx", ms->smp.cores, hr);
1460         ret = -EINVAL;
1461         goto error;
1462     }
1463
1464     memset(&prop, 0, sizeof(WHV_PARTITION_PROPERTY));
1465     prop.ExtendedVmExits.X64MsrExit = 1;
1466     prop.ExtendedVmExits.X64CpuidExit = 1;
1467     hr = whp_dispatch.WHvSetPartitionProperty(
1468         whpx->partition,
1469         WHvPartitionPropertyCodeExtendedVmExits,
1470         &prop,
1471         sizeof(WHV_PARTITION_PROPERTY));
1472
1473     if (FAILED(hr)) {
1474         error_report("WHPX: Failed to enable partition extended X64MsrExit and"
1475                      " X64CpuidExit hr=%08lx", hr);
1476         ret = -EINVAL;
1477         goto error;
1478     }
1479
1480     UINT32 cpuidExitList[] = {1, 0x80000001};
1481     hr = whp_dispatch.WHvSetPartitionProperty(
1482         whpx->partition,
1483         WHvPartitionPropertyCodeCpuidExitList,
1484         cpuidExitList,
1485         RTL_NUMBER_OF(cpuidExitList) * sizeof(UINT32));
1486
1487     if (FAILED(hr)) {
1488         error_report("WHPX: Failed to set partition CpuidExitList hr=%08lx",
1489                      hr);
1490         ret = -EINVAL;
1491         goto error;
1492     }
1493
1494     hr = whp_dispatch.WHvSetupPartition(whpx->partition);
1495     if (FAILED(hr)) {
1496         error_report("WHPX: Failed to setup partition, hr=%08lx", hr);
1497         ret = -EINVAL;
1498         goto error;
1499     }
1500
1501     whpx_memory_init();
1502
1503     cpu_interrupt_handler = whpx_handle_interrupt;
1504
1505     printf("Windows Hypervisor Platform accelerator is operational\n");
1506     return 0;
1507
1508   error:
1509
1510     if (NULL != whpx->partition) {
1511         whp_dispatch.WHvDeletePartition(whpx->partition);
1512         whpx->partition = NULL;
1513     }
1514
1515
1516     return ret;
1517 }
1518
1519 int whpx_enabled(void)
1520 {
1521     return whpx_allowed;
1522 }
1523
1524 static void whpx_accel_class_init(ObjectClass *oc, void *data)
1525 {
1526     AccelClass *ac = ACCEL_CLASS(oc);
1527     ac->name = "WHPX";
1528     ac->init_machine = whpx_accel_init;
1529     ac->allowed = &whpx_allowed;
1530 }
1531
1532 static const TypeInfo whpx_accel_type = {
1533     .name = ACCEL_CLASS_NAME("whpx"),
1534     .parent = TYPE_ACCEL,
1535     .class_init = whpx_accel_class_init,
1536 };
1537
1538 static void whpx_type_init(void)
1539 {
1540     type_register_static(&whpx_accel_type);
1541 }
1542
1543 bool init_whp_dispatch(void)
1544 {
1545     if (whp_dispatch_initialized) {
1546         return true;
1547     }
1548
1549     if (!load_whp_dispatch_fns(&hWinHvPlatform, WINHV_PLATFORM_FNS_DEFAULT)) {
1550         goto error;
1551     }
1552
1553     if (!load_whp_dispatch_fns(&hWinHvEmulation, WINHV_EMULATION_FNS_DEFAULT)) {
1554         goto error;
1555     }
1556
1557     whp_dispatch_initialized = true;
1558
1559     return true;
1560 error:
1561     if (hWinHvPlatform) {
1562         FreeLibrary(hWinHvPlatform);
1563     }
1564
1565     if (hWinHvEmulation) {
1566         FreeLibrary(hWinHvEmulation);
1567     }
1568
1569     return false;
1570 }
1571
1572 type_init(whpx_type_init);