target/i386/whpx/whpx-all.c

   1 /*
   2  * QEMU Windows Hypervisor Platform accelerator (WHPX)
   3  *
   4  * Copyright Microsoft Corp. 2017
   5  *
   6  * This work is licensed under the terms of the GNU GPL, version 2 or later.
   7  * See the COPYING file in the top-level directory.
   8  *
   9  */
  10
  11 #include "qemu/osdep.h"
  12 #include "cpu.h"
  13 #include "exec/address-spaces.h"
  14 #include "exec/ioport.h"
  15 #include "gdbstub/helpers.h"
  16 #include "qemu/accel.h"
  17 #include "sysemu/whpx.h"
  18 #include "sysemu/cpus.h"
  19 #include "sysemu/runstate.h"
  20 #include "qemu/main-loop.h"
  21 #include "hw/boards.h"
  22 #include "hw/intc/ioapic.h"
  23 #include "hw/i386/apic_internal.h"
  24 #include "qemu/error-report.h"
  25 #include "qapi/error.h"
  26 #include "qapi/qapi-types-common.h"
  27 #include "qapi/qapi-visit-common.h"
  28 #include "migration/blocker.h"
  29 #include <winerror.h>
  30
  31 #include "whpx-internal.h"
  32 #include "whpx-accel-ops.h"
  33
  34 #include <winhvplatform.h>
  35 #include <winhvemulation.h>
  36
  37 #define HYPERV_APIC_BUS_FREQUENCY      (200000000ULL)
  38
  39 #pragma GCC diagnostic ignored "-Wcast-function-type"
  40
  41 static const WHV_REGISTER_NAME whpx_register_names[] = {
  42
  43     /* X64 General purpose registers */
  44     WHvX64RegisterRax,
  45     WHvX64RegisterRcx,
  46     WHvX64RegisterRdx,
  47     WHvX64RegisterRbx,
  48     WHvX64RegisterRsp,
  49     WHvX64RegisterRbp,
  50     WHvX64RegisterRsi,
  51     WHvX64RegisterRdi,
  52     WHvX64RegisterR8,
  53     WHvX64RegisterR9,
  54     WHvX64RegisterR10,
  55     WHvX64RegisterR11,
  56     WHvX64RegisterR12,
  57     WHvX64RegisterR13,
  58     WHvX64RegisterR14,
  59     WHvX64RegisterR15,
  60     WHvX64RegisterRip,
  61     WHvX64RegisterRflags,
  62
  63     /* X64 Segment registers */
  64     WHvX64RegisterEs,
  65     WHvX64RegisterCs,
  66     WHvX64RegisterSs,
  67     WHvX64RegisterDs,
  68     WHvX64RegisterFs,
  69     WHvX64RegisterGs,
  70     WHvX64RegisterLdtr,
  71     WHvX64RegisterTr,
  72
  73     /* X64 Table registers */
  74     WHvX64RegisterIdtr,
  75     WHvX64RegisterGdtr,
  76
  77     /* X64 Control Registers */
  78     WHvX64RegisterCr0,
  79     WHvX64RegisterCr2,
  80     WHvX64RegisterCr3,
  81     WHvX64RegisterCr4,
  82     WHvX64RegisterCr8,
  83
  84     /* X64 Debug Registers */
  85     /*
  86      * WHvX64RegisterDr0,
  87      * WHvX64RegisterDr1,
  88      * WHvX64RegisterDr2,
  89      * WHvX64RegisterDr3,
  90      * WHvX64RegisterDr6,
  91      * WHvX64RegisterDr7,
  92      */
  93
  94     /* X64 Floating Point and Vector Registers */
  95     WHvX64RegisterXmm0,
  96     WHvX64RegisterXmm1,
  97     WHvX64RegisterXmm2,
  98     WHvX64RegisterXmm3,
  99     WHvX64RegisterXmm4,
 100     WHvX64RegisterXmm5,
 101     WHvX64RegisterXmm6,
 102     WHvX64RegisterXmm7,
 103     WHvX64RegisterXmm8,
 104     WHvX64RegisterXmm9,
 105     WHvX64RegisterXmm10,
 106     WHvX64RegisterXmm11,
 107     WHvX64RegisterXmm12,
 108     WHvX64RegisterXmm13,
 109     WHvX64RegisterXmm14,
 110     WHvX64RegisterXmm15,
 111     WHvX64RegisterFpMmx0,
 112     WHvX64RegisterFpMmx1,
 113     WHvX64RegisterFpMmx2,
 114     WHvX64RegisterFpMmx3,
 115     WHvX64RegisterFpMmx4,
 116     WHvX64RegisterFpMmx5,
 117     WHvX64RegisterFpMmx6,
 118     WHvX64RegisterFpMmx7,
 119     WHvX64RegisterFpControlStatus,
 120     WHvX64RegisterXmmControlStatus,
 121
 122     /* X64 MSRs */
 123     WHvX64RegisterEfer,
 124 #ifdef TARGET_X86_64
 125     WHvX64RegisterKernelGsBase,
 126 #endif
 127     WHvX64RegisterApicBase,
 128     /* WHvX64RegisterPat, */
 129     WHvX64RegisterSysenterCs,
 130     WHvX64RegisterSysenterEip,
 131     WHvX64RegisterSysenterEsp,
 132     WHvX64RegisterStar,
 133 #ifdef TARGET_X86_64
 134     WHvX64RegisterLstar,
 135     WHvX64RegisterCstar,
 136     WHvX64RegisterSfmask,
 137 #endif
 138
 139     /* Interrupt / Event Registers */
 140     /*
 141      * WHvRegisterPendingInterruption,
 142      * WHvRegisterInterruptState,
 143      * WHvRegisterPendingEvent0,
 144      * WHvRegisterPendingEvent1
 145      * WHvX64RegisterDeliverabilityNotifications,
 146      */
 147 };
 148
 149 struct whpx_register_set {
 150     WHV_REGISTER_VALUE values[RTL_NUMBER_OF(whpx_register_names)];
 151 };
 152
 153 /*
 154  * The current implementation of instruction stepping sets the TF flag
 155  * in RFLAGS, causing the CPU to raise an INT1 after each instruction.
 156  * This corresponds to the WHvX64ExceptionTypeDebugTrapOrFault exception.
 157  *
 158  * This approach has a few limitations:
 159  *     1. Stepping over a PUSHF/SAHF instruction will save the TF flag
 160  *        along with the other flags, possibly restoring it later. It would
 161  *        result in another INT1 when the flags are restored, triggering
 162  *        a stop in gdb that could be cleared by doing another step.
 163  *
 164  *        Stepping over a POPF/LAHF instruction will let it overwrite the
 165  *        TF flags, ending the stepping mode.
 166  *
 167  *     2. Stepping over an instruction raising an exception (e.g. INT, DIV,
 168  *        or anything that could result in a page fault) will save the flags
 169  *        to the stack, clear the TF flag, and let the guest execute the
 170  *        handler. Normally, the guest will restore the original flags,
 171  *        that will continue single-stepping.
 172  *
 173  *     3. Debuggers running on the guest may wish to set TF to do instruction
 174  *        stepping. INT1 events generated by it would be intercepted by us,
 175  *        as long as the gdb is connected to QEMU.
 176  *
 177  * In practice this means that:
 178  *     1. Stepping through flags-modifying instructions may cause gdb to
 179  *        continue or stop in unexpected places. This will be fully recoverable
 180  *        and will not crash the target.
 181  *
 182  *     2. Stepping over an instruction that triggers an exception will step
 183  *        over the exception handler, not into it.
 184  *
 185  *     3. Debugging the guest via gdb, while running debugger on the guest
 186  *        at the same time may lead to unexpected effects. Removing all
 187  *        breakpoints set via QEMU will prevent any further interference
 188  *        with the guest-level debuggers.
 189  *
 190  * The limitations can be addressed as shown below:
 191  *     1. PUSHF/SAHF/POPF/LAHF/IRET instructions can be emulated instead of
 192  *        stepping through them. The exact semantics of the instructions is
 193  *        defined in the "Combined Volume Set of Intel 64 and IA-32
 194  *        Architectures Software Developer's Manuals", however it involves a
 195  *        fair amount of corner cases due to compatibility with real mode,
 196  *        virtual 8086 mode, and differences between 64-bit and 32-bit modes.
 197  *
 198  *     2. We could step into the guest's exception handlers using the following
 199  *        sequence:
 200  *          a. Temporarily enable catching of all exception types via
 201  *             whpx_set_exception_exit_bitmap().
 202  *          b. Once an exception is intercepted, read the IDT/GDT and locate
 203  *             the original handler.
 204  *          c. Patch the original handler, injecting an INT3 at the beginning.
 205  *          d. Update the exception exit bitmap to only catch the
 206  *             WHvX64ExceptionTypeBreakpointTrap exception.
 207  *          e. Let the affected CPU run in the exclusive mode.
 208  *          f. Restore the original handler and the exception exit bitmap.
 209  *        Note that handling all corner cases related to IDT/GDT is harder
 210  *        than it may seem. See x86_cpu_get_phys_page_attrs_debug() for a
 211  *        rough idea.
 212  *
 213  *     3. In order to properly support guest-level debugging in parallel with
 214  *        the QEMU-level debugging, we would need to be able to pass some INT1
 215  *        events to the guest. This could be done via the following methods:
 216  *          a. Using the WHvRegisterPendingEvent register. As of Windows 21H1,
 217  *             it seems to only work for interrupts and not software
 218  *             exceptions.
 219  *          b. Locating and patching the original handler by parsing IDT/GDT.
 220  *             This involves relatively complex logic outlined in the previous
 221  *             paragraph.
 222  *          c. Emulating the exception invocation (i.e. manually updating RIP,
 223  *             RFLAGS, and pushing the old values to stack). This is even more
 224  *             complicated than the previous option, since it involves checking
 225  *             CPL, gate attributes, and doing various adjustments depending
 226  *             on the current CPU mode, whether the CPL is changing, etc.
 227  */
 228 typedef enum WhpxStepMode {
 229     WHPX_STEP_NONE = 0,
 230     /* Halt other VCPUs */
 231     WHPX_STEP_EXCLUSIVE,
 232 } WhpxStepMode;
 233
 234 struct AccelCPUState {
 235     WHV_EMULATOR_HANDLE emulator;
 236     bool window_registered;
 237     bool interruptable;
 238     bool ready_for_pic_interrupt;
 239     uint64_t tpr;
 240     uint64_t apic_base;
 241     bool interruption_pending;
 242     bool dirty;
 243
 244     /* Must be the last field as it may have a tail */
 245     WHV_RUN_VP_EXIT_CONTEXT exit_ctx;
 246 };
 247
 248 static bool whpx_allowed;
 249 static bool whp_dispatch_initialized;
 250 static HMODULE hWinHvPlatform, hWinHvEmulation;
 251 static uint32_t max_vcpu_index;
 252 static WHV_PROCESSOR_XSAVE_FEATURES whpx_xsave_cap;
 253
 254 struct whpx_state whpx_global;
 255 struct WHPDispatch whp_dispatch;
 256
 257 static bool whpx_has_xsave(void)
 258 {
 259     return whpx_xsave_cap.XsaveSupport;
 260 }
 261
 262 static WHV_X64_SEGMENT_REGISTER whpx_seg_q2h(const SegmentCache *qs, int v86,
 263                                              int r86)
 264 {
 265     WHV_X64_SEGMENT_REGISTER hs;
 266     unsigned flags = qs->flags;
 267
 268     hs.Base = qs->base;
 269     hs.Limit = qs->limit;
 270     hs.Selector = qs->selector;
 271
 272     if (v86) {
 273         hs.Attributes = 0;
 274         hs.SegmentType = 3;
 275         hs.Present = 1;
 276         hs.DescriptorPrivilegeLevel = 3;
 277         hs.NonSystemSegment = 1;
 278
 279     } else {
 280         hs.Attributes = (flags >> DESC_TYPE_SHIFT);
 281
 282         if (r86) {
 283             /* hs.Base &= 0xfffff; */
 284         }
 285     }
 286
 287     return hs;
 288 }
 289
 290 static SegmentCache whpx_seg_h2q(const WHV_X64_SEGMENT_REGISTER *hs)
 291 {
 292     SegmentCache qs;
 293
 294     qs.base = hs->Base;
 295     qs.limit = hs->Limit;
 296     qs.selector = hs->Selector;
 297
 298     qs.flags = ((uint32_t)hs->Attributes) << DESC_TYPE_SHIFT;
 299
 300     return qs;
 301 }
 302
 303 /* X64 Extended Control Registers */
 304 static void whpx_set_xcrs(CPUState *cpu)
 305 {
 306     HRESULT hr;
 307     struct whpx_state *whpx = &whpx_global;
 308     WHV_REGISTER_VALUE xcr0;
 309     WHV_REGISTER_NAME xcr0_name = WHvX64RegisterXCr0;
 310
 311     if (!whpx_has_xsave()) {
 312         return;
 313     }
 314
 315     /* Only xcr0 is supported by the hypervisor currently */
 316     xcr0.Reg64 = cpu_env(cpu)->xcr0;
 317     hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
 318         whpx->partition, cpu->cpu_index, &xcr0_name, 1, &xcr0);
 319     if (FAILED(hr)) {
 320         error_report("WHPX: Failed to set register xcr0, hr=%08lx", hr);
 321     }
 322 }
 323
 324 static int whpx_set_tsc(CPUState *cpu)
 325 {
 326     WHV_REGISTER_NAME tsc_reg = WHvX64RegisterTsc;
 327     WHV_REGISTER_VALUE tsc_val;
 328     HRESULT hr;
 329     struct whpx_state *whpx = &whpx_global;
 330
 331     /*
 332      * Suspend the partition prior to setting the TSC to reduce the variance
 333      * in TSC across vCPUs. When the first vCPU runs post suspend, the
 334      * partition is automatically resumed.
 335      */
 336     if (whp_dispatch.WHvSuspendPartitionTime) {
 337
 338         /*
 339          * Unable to suspend partition while setting TSC is not a fatal
 340          * error. It just increases the likelihood of TSC variance between
 341          * vCPUs and some guest OS are able to handle that just fine.
 342          */
 343         hr = whp_dispatch.WHvSuspendPartitionTime(whpx->partition);
 344         if (FAILED(hr)) {
 345             warn_report("WHPX: Failed to suspend partition, hr=%08lx", hr);
 346         }
 347     }
 348
 349     tsc_val.Reg64 = cpu_env(cpu)->tsc;
 350     hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
 351         whpx->partition, cpu->cpu_index, &tsc_reg, 1, &tsc_val);
 352     if (FAILED(hr)) {
 353         error_report("WHPX: Failed to set TSC, hr=%08lx", hr);
 354         return -1;
 355     }
 356
 357     return 0;
 358 }
 359
 360 /*
 361  * The CR8 register in the CPU is mapped to the TPR register of the APIC,
 362  * however, they use a slightly different encoding. Specifically:
 363  *
 364  *     APIC.TPR[bits 7:4] = CR8[bits 3:0]
 365  *
 366  * This mechanism is described in section 10.8.6.1 of Volume 3 of Intel 64
 367  * and IA-32 Architectures Software Developer's Manual.
 368  *
 369  * The functions below translate the value of CR8 to TPR and vice versa.
 370  */
 371
 372 static uint64_t whpx_apic_tpr_to_cr8(uint64_t tpr)
 373 {
 374     return tpr >> 4;
 375 }
 376
 377 static uint64_t whpx_cr8_to_apic_tpr(uint64_t cr8)
 378 {
 379     return cr8 << 4;
 380 }
 381
 382 static void whpx_set_registers(CPUState *cpu, int level)
 383 {
 384     struct whpx_state *whpx = &whpx_global;
 385     AccelCPUState *vcpu = cpu->accel;
 386     X86CPU *x86_cpu = X86_CPU(cpu);
 387     CPUX86State *env = &x86_cpu->env;
 388     struct whpx_register_set vcxt;
 389     HRESULT hr;
 390     int idx;
 391     int idx_next;
 392     int i;
 393     int v86, r86;
 394
 395     assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
 396
 397     /*
 398      * Following MSRs have side effects on the guest or are too heavy for
 399      * runtime. Limit them to full state update.
 400      */
 401     if (level >= WHPX_SET_RESET_STATE) {
 402         whpx_set_tsc(cpu);
 403     }
 404
 405     memset(&vcxt, 0, sizeof(struct whpx_register_set));
 406
 407     v86 = (env->eflags & VM_MASK);
 408     r86 = !(env->cr[0] & CR0_PE_MASK);
 409
 410     vcpu->tpr = whpx_apic_tpr_to_cr8(cpu_get_apic_tpr(x86_cpu->apic_state));
 411     vcpu->apic_base = cpu_get_apic_base(x86_cpu->apic_state);
 412
 413     idx = 0;
 414
 415     /* Indexes for first 16 registers match between HV and QEMU definitions */
 416     idx_next = 16;
 417     for (idx = 0; idx < CPU_NB_REGS; idx += 1) {
 418         vcxt.values[idx].Reg64 = (uint64_t)env->regs[idx];
 419     }
 420     idx = idx_next;
 421
 422     /* Same goes for RIP and RFLAGS */
 423     assert(whpx_register_names[idx] == WHvX64RegisterRip);
 424     vcxt.values[idx++].Reg64 = env->eip;
 425
 426     assert(whpx_register_names[idx] == WHvX64RegisterRflags);
 427     vcxt.values[idx++].Reg64 = env->eflags;
 428
 429     /* Translate 6+4 segment registers. HV and QEMU order matches  */
 430     assert(idx == WHvX64RegisterEs);
 431     for (i = 0; i < 6; i += 1, idx += 1) {
 432         vcxt.values[idx].Segment = whpx_seg_q2h(&env->segs[i], v86, r86);
 433     }
 434
 435     assert(idx == WHvX64RegisterLdtr);
 436     vcxt.values[idx++].Segment = whpx_seg_q2h(&env->ldt, 0, 0);
 437
 438     assert(idx == WHvX64RegisterTr);
 439     vcxt.values[idx++].Segment = whpx_seg_q2h(&env->tr, 0, 0);
 440
 441     assert(idx == WHvX64RegisterIdtr);
 442     vcxt.values[idx].Table.Base = env->idt.base;
 443     vcxt.values[idx].Table.Limit = env->idt.limit;
 444     idx += 1;
 445
 446     assert(idx == WHvX64RegisterGdtr);
 447     vcxt.values[idx].Table.Base = env->gdt.base;
 448     vcxt.values[idx].Table.Limit = env->gdt.limit;
 449     idx += 1;
 450
 451     /* CR0, 2, 3, 4, 8 */
 452     assert(whpx_register_names[idx] == WHvX64RegisterCr0);
 453     vcxt.values[idx++].Reg64 = env->cr[0];
 454     assert(whpx_register_names[idx] == WHvX64RegisterCr2);
 455     vcxt.values[idx++].Reg64 = env->cr[2];
 456     assert(whpx_register_names[idx] == WHvX64RegisterCr3);
 457     vcxt.values[idx++].Reg64 = env->cr[3];
 458     assert(whpx_register_names[idx] == WHvX64RegisterCr4);
 459     vcxt.values[idx++].Reg64 = env->cr[4];
 460     assert(whpx_register_names[idx] == WHvX64RegisterCr8);
 461     vcxt.values[idx++].Reg64 = vcpu->tpr;
 462
 463     /* 8 Debug Registers - Skipped */
 464
 465     /*
 466      * Extended control registers needs to be handled separately depending
 467      * on whether xsave is supported/enabled or not.
 468      */
 469     whpx_set_xcrs(cpu);
 470
 471     /* 16 XMM registers */
 472     assert(whpx_register_names[idx] == WHvX64RegisterXmm0);
 473     idx_next = idx + 16;
 474     for (i = 0; i < sizeof(env->xmm_regs) / sizeof(ZMMReg); i += 1, idx += 1) {
 475         vcxt.values[idx].Reg128.Low64 = env->xmm_regs[i].ZMM_Q(0);
 476         vcxt.values[idx].Reg128.High64 = env->xmm_regs[i].ZMM_Q(1);
 477     }
 478     idx = idx_next;
 479
 480     /* 8 FP registers */
 481     assert(whpx_register_names[idx] == WHvX64RegisterFpMmx0);
 482     for (i = 0; i < 8; i += 1, idx += 1) {
 483         vcxt.values[idx].Fp.AsUINT128.Low64 = env->fpregs[i].mmx.MMX_Q(0);
 484         /* vcxt.values[idx].Fp.AsUINT128.High64 =
 485                env->fpregs[i].mmx.MMX_Q(1);
 486         */
 487     }
 488
 489     /* FP control status register */
 490     assert(whpx_register_names[idx] == WHvX64RegisterFpControlStatus);
 491     vcxt.values[idx].FpControlStatus.FpControl = env->fpuc;
 492     vcxt.values[idx].FpControlStatus.FpStatus =
 493         (env->fpus & ~0x3800) | (env->fpstt & 0x7) << 11;
 494     vcxt.values[idx].FpControlStatus.FpTag = 0;
 495     for (i = 0; i < 8; ++i) {
 496         vcxt.values[idx].FpControlStatus.FpTag |= (!env->fptags[i]) << i;
 497     }
 498     vcxt.values[idx].FpControlStatus.Reserved = 0;
 499     vcxt.values[idx].FpControlStatus.LastFpOp = env->fpop;
 500     vcxt.values[idx].FpControlStatus.LastFpRip = env->fpip;
 501     idx += 1;
 502
 503     /* XMM control status register */
 504     assert(whpx_register_names[idx] == WHvX64RegisterXmmControlStatus);
 505     vcxt.values[idx].XmmControlStatus.LastFpRdp = 0;
 506     vcxt.values[idx].XmmControlStatus.XmmStatusControl = env->mxcsr;
 507     vcxt.values[idx].XmmControlStatus.XmmStatusControlMask = 0x0000ffff;
 508     idx += 1;
 509
 510     /* MSRs */
 511     assert(whpx_register_names[idx] == WHvX64RegisterEfer);
 512     vcxt.values[idx++].Reg64 = env->efer;
 513 #ifdef TARGET_X86_64
 514     assert(whpx_register_names[idx] == WHvX64RegisterKernelGsBase);
 515     vcxt.values[idx++].Reg64 = env->kernelgsbase;
 516 #endif
 517
 518     assert(whpx_register_names[idx] == WHvX64RegisterApicBase);
 519     vcxt.values[idx++].Reg64 = vcpu->apic_base;
 520
 521     /* WHvX64RegisterPat - Skipped */
 522
 523     assert(whpx_register_names[idx] == WHvX64RegisterSysenterCs);
 524     vcxt.values[idx++].Reg64 = env->sysenter_cs;
 525     assert(whpx_register_names[idx] == WHvX64RegisterSysenterEip);
 526     vcxt.values[idx++].Reg64 = env->sysenter_eip;
 527     assert(whpx_register_names[idx] == WHvX64RegisterSysenterEsp);
 528     vcxt.values[idx++].Reg64 = env->sysenter_esp;
 529     assert(whpx_register_names[idx] == WHvX64RegisterStar);
 530     vcxt.values[idx++].Reg64 = env->star;
 531 #ifdef TARGET_X86_64
 532     assert(whpx_register_names[idx] == WHvX64RegisterLstar);
 533     vcxt.values[idx++].Reg64 = env->lstar;
 534     assert(whpx_register_names[idx] == WHvX64RegisterCstar);
 535     vcxt.values[idx++].Reg64 = env->cstar;
 536     assert(whpx_register_names[idx] == WHvX64RegisterSfmask);
 537     vcxt.values[idx++].Reg64 = env->fmask;
 538 #endif
 539
 540     /* Interrupt / Event Registers - Skipped */
 541
 542     assert(idx == RTL_NUMBER_OF(whpx_register_names));
 543
 544     hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
 545         whpx->partition, cpu->cpu_index,
 546         whpx_register_names,
 547         RTL_NUMBER_OF(whpx_register_names),
 548         &vcxt.values[0]);
 549
 550     if (FAILED(hr)) {
 551         error_report("WHPX: Failed to set virtual processor context, hr=%08lx",
 552                      hr);
 553     }
 554
 555     return;
 556 }
 557
 558 static int whpx_get_tsc(CPUState *cpu)
 559 {
 560     WHV_REGISTER_NAME tsc_reg = WHvX64RegisterTsc;
 561     WHV_REGISTER_VALUE tsc_val;
 562     HRESULT hr;
 563     struct whpx_state *whpx = &whpx_global;
 564
 565     hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
 566         whpx->partition, cpu->cpu_index, &tsc_reg, 1, &tsc_val);
 567     if (FAILED(hr)) {
 568         error_report("WHPX: Failed to get TSC, hr=%08lx", hr);
 569         return -1;
 570     }
 571
 572     cpu_env(cpu)->tsc = tsc_val.Reg64;
 573     return 0;
 574 }
 575
 576 /* X64 Extended Control Registers */
 577 static void whpx_get_xcrs(CPUState *cpu)
 578 {
 579     HRESULT hr;
 580     struct whpx_state *whpx = &whpx_global;
 581     WHV_REGISTER_VALUE xcr0;
 582     WHV_REGISTER_NAME xcr0_name = WHvX64RegisterXCr0;
 583
 584     if (!whpx_has_xsave()) {
 585         return;
 586     }
 587
 588     /* Only xcr0 is supported by the hypervisor currently */
 589     hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
 590         whpx->partition, cpu->cpu_index, &xcr0_name, 1, &xcr0);
 591     if (FAILED(hr)) {
 592         error_report("WHPX: Failed to get register xcr0, hr=%08lx", hr);
 593         return;
 594     }
 595
 596     cpu_env(cpu)->xcr0 = xcr0.Reg64;
 597 }
 598
 599 static void whpx_get_registers(CPUState *cpu)
 600 {
 601     struct whpx_state *whpx = &whpx_global;
 602     AccelCPUState *vcpu = cpu->accel;
 603     X86CPU *x86_cpu = X86_CPU(cpu);
 604     CPUX86State *env = &x86_cpu->env;
 605     struct whpx_register_set vcxt;
 606     uint64_t tpr, apic_base;
 607     HRESULT hr;
 608     int idx;
 609     int idx_next;
 610     int i;
 611
 612     assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
 613
 614     if (!env->tsc_valid) {
 615         whpx_get_tsc(cpu);
 616         env->tsc_valid = !runstate_is_running();
 617     }
 618
 619     hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
 620         whpx->partition, cpu->cpu_index,
 621         whpx_register_names,
 622         RTL_NUMBER_OF(whpx_register_names),
 623         &vcxt.values[0]);
 624     if (FAILED(hr)) {
 625         error_report("WHPX: Failed to get virtual processor context, hr=%08lx",
 626                      hr);
 627     }
 628
 629     if (whpx_apic_in_platform()) {
 630         /*
 631          * Fetch the TPR value from the emulated APIC. It may get overwritten
 632          * below with the value from CR8 returned by
 633          * WHvGetVirtualProcessorRegisters().
 634          */
 635         whpx_apic_get(x86_cpu->apic_state);
 636         vcpu->tpr = whpx_apic_tpr_to_cr8(
 637             cpu_get_apic_tpr(x86_cpu->apic_state));
 638     }
 639
 640     idx = 0;
 641
 642     /* Indexes for first 16 registers match between HV and QEMU definitions */
 643     idx_next = 16;
 644     for (idx = 0; idx < CPU_NB_REGS; idx += 1) {
 645         env->regs[idx] = vcxt.values[idx].Reg64;
 646     }
 647     idx = idx_next;
 648
 649     /* Same goes for RIP and RFLAGS */
 650     assert(whpx_register_names[idx] == WHvX64RegisterRip);
 651     env->eip = vcxt.values[idx++].Reg64;
 652     assert(whpx_register_names[idx] == WHvX64RegisterRflags);
 653     env->eflags = vcxt.values[idx++].Reg64;
 654
 655     /* Translate 6+4 segment registers. HV and QEMU order matches  */
 656     assert(idx == WHvX64RegisterEs);
 657     for (i = 0; i < 6; i += 1, idx += 1) {
 658         env->segs[i] = whpx_seg_h2q(&vcxt.values[idx].Segment);
 659     }
 660
 661     assert(idx == WHvX64RegisterLdtr);
 662     env->ldt = whpx_seg_h2q(&vcxt.values[idx++].Segment);
 663     assert(idx == WHvX64RegisterTr);
 664     env->tr = whpx_seg_h2q(&vcxt.values[idx++].Segment);
 665     assert(idx == WHvX64RegisterIdtr);
 666     env->idt.base = vcxt.values[idx].Table.Base;
 667     env->idt.limit = vcxt.values[idx].Table.Limit;
 668     idx += 1;
 669     assert(idx == WHvX64RegisterGdtr);
 670     env->gdt.base = vcxt.values[idx].Table.Base;
 671     env->gdt.limit = vcxt.values[idx].Table.Limit;
 672     idx += 1;
 673
 674     /* CR0, 2, 3, 4, 8 */
 675     assert(whpx_register_names[idx] == WHvX64RegisterCr0);
 676     env->cr[0] = vcxt.values[idx++].Reg64;
 677     assert(whpx_register_names[idx] == WHvX64RegisterCr2);
 678     env->cr[2] = vcxt.values[idx++].Reg64;
 679     assert(whpx_register_names[idx] == WHvX64RegisterCr3);
 680     env->cr[3] = vcxt.values[idx++].Reg64;
 681     assert(whpx_register_names[idx] == WHvX64RegisterCr4);
 682     env->cr[4] = vcxt.values[idx++].Reg64;
 683     assert(whpx_register_names[idx] == WHvX64RegisterCr8);
 684     tpr = vcxt.values[idx++].Reg64;
 685     if (tpr != vcpu->tpr) {
 686         vcpu->tpr = tpr;
 687         cpu_set_apic_tpr(x86_cpu->apic_state, whpx_cr8_to_apic_tpr(tpr));
 688     }
 689
 690     /* 8 Debug Registers - Skipped */
 691
 692     /*
 693      * Extended control registers needs to be handled separately depending
 694      * on whether xsave is supported/enabled or not.
 695      */
 696     whpx_get_xcrs(cpu);
 697
 698     /* 16 XMM registers */
 699     assert(whpx_register_names[idx] == WHvX64RegisterXmm0);
 700     idx_next = idx + 16;
 701     for (i = 0; i < sizeof(env->xmm_regs) / sizeof(ZMMReg); i += 1, idx += 1) {
 702         env->xmm_regs[i].ZMM_Q(0) = vcxt.values[idx].Reg128.Low64;
 703         env->xmm_regs[i].ZMM_Q(1) = vcxt.values[idx].Reg128.High64;
 704     }
 705     idx = idx_next;
 706
 707     /* 8 FP registers */
 708     assert(whpx_register_names[idx] == WHvX64RegisterFpMmx0);
 709     for (i = 0; i < 8; i += 1, idx += 1) {
 710         env->fpregs[i].mmx.MMX_Q(0) = vcxt.values[idx].Fp.AsUINT128.Low64;
 711         /* env->fpregs[i].mmx.MMX_Q(1) =
 712                vcxt.values[idx].Fp.AsUINT128.High64;
 713         */
 714     }
 715
 716     /* FP control status register */
 717     assert(whpx_register_names[idx] == WHvX64RegisterFpControlStatus);
 718     env->fpuc = vcxt.values[idx].FpControlStatus.FpControl;
 719     env->fpstt = (vcxt.values[idx].FpControlStatus.FpStatus >> 11) & 0x7;
 720     env->fpus = vcxt.values[idx].FpControlStatus.FpStatus & ~0x3800;
 721     for (i = 0; i < 8; ++i) {
 722         env->fptags[i] = !((vcxt.values[idx].FpControlStatus.FpTag >> i) & 1);
 723     }
 724     env->fpop = vcxt.values[idx].FpControlStatus.LastFpOp;
 725     env->fpip = vcxt.values[idx].FpControlStatus.LastFpRip;
 726     idx += 1;
 727
 728     /* XMM control status register */
 729     assert(whpx_register_names[idx] == WHvX64RegisterXmmControlStatus);
 730     env->mxcsr = vcxt.values[idx].XmmControlStatus.XmmStatusControl;
 731     idx += 1;
 732
 733     /* MSRs */
 734     assert(whpx_register_names[idx] == WHvX64RegisterEfer);
 735     env->efer = vcxt.values[idx++].Reg64;
 736 #ifdef TARGET_X86_64
 737     assert(whpx_register_names[idx] == WHvX64RegisterKernelGsBase);
 738     env->kernelgsbase = vcxt.values[idx++].Reg64;
 739 #endif
 740
 741     assert(whpx_register_names[idx] == WHvX64RegisterApicBase);
 742     apic_base = vcxt.values[idx++].Reg64;
 743     if (apic_base != vcpu->apic_base) {
 744         vcpu->apic_base = apic_base;
 745         cpu_set_apic_base(x86_cpu->apic_state, vcpu->apic_base);
 746     }
 747
 748     /* WHvX64RegisterPat - Skipped */
 749
 750     assert(whpx_register_names[idx] == WHvX64RegisterSysenterCs);
 751     env->sysenter_cs = vcxt.values[idx++].Reg64;
 752     assert(whpx_register_names[idx] == WHvX64RegisterSysenterEip);
 753     env->sysenter_eip = vcxt.values[idx++].Reg64;
 754     assert(whpx_register_names[idx] == WHvX64RegisterSysenterEsp);
 755     env->sysenter_esp = vcxt.values[idx++].Reg64;
 756     assert(whpx_register_names[idx] == WHvX64RegisterStar);
 757     env->star = vcxt.values[idx++].Reg64;
 758 #ifdef TARGET_X86_64
 759     assert(whpx_register_names[idx] == WHvX64RegisterLstar);
 760     env->lstar = vcxt.values[idx++].Reg64;
 761     assert(whpx_register_names[idx] == WHvX64RegisterCstar);
 762     env->cstar = vcxt.values[idx++].Reg64;
 763     assert(whpx_register_names[idx] == WHvX64RegisterSfmask);
 764     env->fmask = vcxt.values[idx++].Reg64;
 765 #endif
 766
 767     /* Interrupt / Event Registers - Skipped */
 768
 769     assert(idx == RTL_NUMBER_OF(whpx_register_names));
 770
 771     if (whpx_apic_in_platform()) {
 772         whpx_apic_get(x86_cpu->apic_state);
 773     }
 774
 775     x86_update_hflags(env);
 776
 777     return;
 778 }
 779
 780 static HRESULT CALLBACK whpx_emu_ioport_callback(
 781     void *ctx,
 782     WHV_EMULATOR_IO_ACCESS_INFO *IoAccess)
 783 {
 784     MemTxAttrs attrs = { 0 };
 785     address_space_rw(&address_space_io, IoAccess->Port, attrs,
 786                      &IoAccess->Data, IoAccess->AccessSize,
 787                      IoAccess->Direction);
 788     return S_OK;
 789 }
 790
 791 static HRESULT CALLBACK whpx_emu_mmio_callback(
 792     void *ctx,
 793     WHV_EMULATOR_MEMORY_ACCESS_INFO *ma)
 794 {
 795     cpu_physical_memory_rw(ma->GpaAddress, ma->Data, ma->AccessSize,
 796                            ma->Direction);
 797     return S_OK;
 798 }
 799
 800 static HRESULT CALLBACK whpx_emu_getreg_callback(
 801     void *ctx,
 802     const WHV_REGISTER_NAME *RegisterNames,
 803     UINT32 RegisterCount,
 804     WHV_REGISTER_VALUE *RegisterValues)
 805 {
 806     HRESULT hr;
 807     struct whpx_state *whpx = &whpx_global;
 808     CPUState *cpu = (CPUState *)ctx;
 809
 810     hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
 811         whpx->partition, cpu->cpu_index,
 812         RegisterNames, RegisterCount,
 813         RegisterValues);
 814     if (FAILED(hr)) {
 815         error_report("WHPX: Failed to get virtual processor registers,"
 816                      " hr=%08lx", hr);
 817     }
 818
 819     return hr;
 820 }
 821
 822 static HRESULT CALLBACK whpx_emu_setreg_callback(
 823     void *ctx,
 824     const WHV_REGISTER_NAME *RegisterNames,
 825     UINT32 RegisterCount,
 826     const WHV_REGISTER_VALUE *RegisterValues)
 827 {
 828     HRESULT hr;
 829     struct whpx_state *whpx = &whpx_global;
 830     CPUState *cpu = (CPUState *)ctx;
 831
 832     hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
 833         whpx->partition, cpu->cpu_index,
 834         RegisterNames, RegisterCount,
 835         RegisterValues);
 836     if (FAILED(hr)) {
 837         error_report("WHPX: Failed to set virtual processor registers,"
 838                      " hr=%08lx", hr);
 839     }
 840
 841     /*
 842      * The emulator just successfully wrote the register state. We clear the
 843      * dirty state so we avoid the double write on resume of the VP.
 844      */
 845     cpu->accel->dirty = false;
 846
 847     return hr;
 848 }
 849
 850 static HRESULT CALLBACK whpx_emu_translate_callback(
 851     void *ctx,
 852     WHV_GUEST_VIRTUAL_ADDRESS Gva,
 853     WHV_TRANSLATE_GVA_FLAGS TranslateFlags,
 854     WHV_TRANSLATE_GVA_RESULT_CODE *TranslationResult,
 855     WHV_GUEST_PHYSICAL_ADDRESS *Gpa)
 856 {
 857     HRESULT hr;
 858     struct whpx_state *whpx = &whpx_global;
 859     CPUState *cpu = (CPUState *)ctx;
 860     WHV_TRANSLATE_GVA_RESULT res;
 861
 862     hr = whp_dispatch.WHvTranslateGva(whpx->partition, cpu->cpu_index,
 863                                       Gva, TranslateFlags, &res, Gpa);
 864     if (FAILED(hr)) {
 865         error_report("WHPX: Failed to translate GVA, hr=%08lx", hr);
 866     } else {
 867         *TranslationResult = res.ResultCode;
 868     }
 869
 870     return hr;
 871 }
 872
 873 static const WHV_EMULATOR_CALLBACKS whpx_emu_callbacks = {
 874     .Size = sizeof(WHV_EMULATOR_CALLBACKS),
 875     .WHvEmulatorIoPortCallback = whpx_emu_ioport_callback,
 876     .WHvEmulatorMemoryCallback = whpx_emu_mmio_callback,
 877     .WHvEmulatorGetVirtualProcessorRegisters = whpx_emu_getreg_callback,
 878     .WHvEmulatorSetVirtualProcessorRegisters = whpx_emu_setreg_callback,
 879     .WHvEmulatorTranslateGvaPage = whpx_emu_translate_callback,
 880 };
 881
 882 static int whpx_handle_mmio(CPUState *cpu, WHV_MEMORY_ACCESS_CONTEXT *ctx)
 883 {
 884     HRESULT hr;
 885     AccelCPUState *vcpu = cpu->accel;
 886     WHV_EMULATOR_STATUS emu_status;
 887
 888     hr = whp_dispatch.WHvEmulatorTryMmioEmulation(
 889         vcpu->emulator, cpu,
 890         &vcpu->exit_ctx.VpContext, ctx,
 891         &emu_status);
 892     if (FAILED(hr)) {
 893         error_report("WHPX: Failed to parse MMIO access, hr=%08lx", hr);
 894         return -1;
 895     }
 896
 897     if (!emu_status.EmulationSuccessful) {
 898         error_report("WHPX: Failed to emulate MMIO access with"
 899                      " EmulatorReturnStatus: %u", emu_status.AsUINT32);
 900         return -1;
 901     }
 902
 903     return 0;
 904 }
 905
 906 static int whpx_handle_portio(CPUState *cpu,
 907                               WHV_X64_IO_PORT_ACCESS_CONTEXT *ctx)
 908 {
 909     HRESULT hr;
 910     AccelCPUState *vcpu = cpu->accel;
 911     WHV_EMULATOR_STATUS emu_status;
 912
 913     hr = whp_dispatch.WHvEmulatorTryIoEmulation(
 914         vcpu->emulator, cpu,
 915         &vcpu->exit_ctx.VpContext, ctx,
 916         &emu_status);
 917     if (FAILED(hr)) {
 918         error_report("WHPX: Failed to parse PortIO access, hr=%08lx", hr);
 919         return -1;
 920     }
 921
 922     if (!emu_status.EmulationSuccessful) {
 923         error_report("WHPX: Failed to emulate PortIO access with"
 924                      " EmulatorReturnStatus: %u", emu_status.AsUINT32);
 925         return -1;
 926     }
 927
 928     return 0;
 929 }
 930
 931 /*
 932  * Controls whether we should intercept various exceptions on the guest,
 933  * namely breakpoint/single-step events.
 934  *
 935  * The 'exceptions' argument accepts a bitmask, e.g:
 936  * (1 << WHvX64ExceptionTypeDebugTrapOrFault) | (...)
 937  */
 938 static HRESULT whpx_set_exception_exit_bitmap(UINT64 exceptions)
 939 {
 940     struct whpx_state *whpx = &whpx_global;
 941     WHV_PARTITION_PROPERTY prop = { 0, };
 942     HRESULT hr;
 943
 944     if (exceptions == whpx->exception_exit_bitmap) {
 945         return S_OK;
 946     }
 947
 948     prop.ExceptionExitBitmap = exceptions;
 949
 950     hr = whp_dispatch.WHvSetPartitionProperty(
 951         whpx->partition,
 952         WHvPartitionPropertyCodeExceptionExitBitmap,
 953         &prop,
 954         sizeof(WHV_PARTITION_PROPERTY));
 955
 956     if (SUCCEEDED(hr)) {
 957         whpx->exception_exit_bitmap = exceptions;
 958     }
 959
 960     return hr;
 961 }
 962
 963
 964 /*
 965  * This function is called before/after stepping over a single instruction.
 966  * It will update the CPU registers to arm/disarm the instruction stepping
 967  * accordingly.
 968  */
 969 static HRESULT whpx_vcpu_configure_single_stepping(CPUState *cpu,
 970     bool set,
 971     uint64_t *exit_context_rflags)
 972 {
 973     WHV_REGISTER_NAME reg_name;
 974     WHV_REGISTER_VALUE reg_value;
 975     HRESULT hr;
 976     struct whpx_state *whpx = &whpx_global;
 977
 978     /*
 979      * If we are trying to step over a single instruction, we need to set the
 980      * TF bit in rflags. Otherwise, clear it.
 981      */
 982     reg_name = WHvX64RegisterRflags;
 983     hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
 984         whpx->partition,
 985         cpu->cpu_index,
 986         &reg_name,
 987         1,
 988         &reg_value);
 989
 990     if (FAILED(hr)) {
 991         error_report("WHPX: Failed to get rflags, hr=%08lx", hr);
 992         return hr;
 993     }
 994
 995     if (exit_context_rflags) {
 996         assert(*exit_context_rflags == reg_value.Reg64);
 997     }
 998
 999     if (set) {
1000         /* Raise WHvX64ExceptionTypeDebugTrapOrFault after each instruction */
1001         reg_value.Reg64 |= TF_MASK;
1002     } else {
1003         reg_value.Reg64 &= ~TF_MASK;
1004     }
1005
1006     if (exit_context_rflags) {
1007         *exit_context_rflags = reg_value.Reg64;
1008     }
1009
1010     hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
1011         whpx->partition,
1012         cpu->cpu_index,
1013         &reg_name,
1014         1,
1015         &reg_value);
1016
1017     if (FAILED(hr)) {
1018         error_report("WHPX: Failed to set rflags,"
1019             " hr=%08lx",
1020             hr);
1021         return hr;
1022     }
1023
1024     reg_name = WHvRegisterInterruptState;
1025     reg_value.Reg64 = 0;
1026
1027     /* Suspend delivery of hardware interrupts during single-stepping. */
1028     reg_value.InterruptState.InterruptShadow = set != 0;
1029
1030     hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
1031     whpx->partition,
1032         cpu->cpu_index,
1033         &reg_name,
1034         1,
1035         &reg_value);
1036
1037     if (FAILED(hr)) {
1038         error_report("WHPX: Failed to set InterruptState,"
1039             " hr=%08lx",
1040             hr);
1041         return hr;
1042     }
1043
1044     if (!set) {
1045         /*
1046          * We have just finished stepping over a single instruction,
1047          * and intercepted the INT1 generated by it.
1048          * We need to now hide the INT1 from the guest,
1049          * as it would not be expecting it.
1050          */
1051
1052         reg_name = WHvX64RegisterPendingDebugException;
1053         hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
1054         whpx->partition,
1055             cpu->cpu_index,
1056             &reg_name,
1057             1,
1058             &reg_value);
1059
1060         if (FAILED(hr)) {
1061             error_report("WHPX: Failed to get pending debug exceptions,"
1062                          "hr=%08lx", hr);
1063             return hr;
1064         }
1065
1066         if (reg_value.PendingDebugException.SingleStep) {
1067             reg_value.PendingDebugException.SingleStep = 0;
1068
1069             hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
1070                 whpx->partition,
1071                 cpu->cpu_index,
1072                 &reg_name,
1073                 1,
1074                 &reg_value);
1075
1076             if (FAILED(hr)) {
1077                 error_report("WHPX: Failed to clear pending debug exceptions,"
1078                              "hr=%08lx", hr);
1079              return hr;
1080             }
1081         }
1082
1083     }
1084
1085     return S_OK;
1086 }
1087
1088 /* Tries to find a breakpoint at the specified address. */
1089 static struct whpx_breakpoint *whpx_lookup_breakpoint_by_addr(uint64_t address)
1090 {
1091     struct whpx_state *whpx = &whpx_global;
1092     int i;
1093
1094     if (whpx->breakpoints.breakpoints) {
1095         for (i = 0; i < whpx->breakpoints.breakpoints->used; i++) {
1096             if (address == whpx->breakpoints.breakpoints->data[i].address) {
1097                 return &whpx->breakpoints.breakpoints->data[i];
1098             }
1099         }
1100     }
1101
1102     return NULL;
1103 }
1104
1105 /*
1106  * Linux uses int3 (0xCC) during startup (see int3_selftest()) and for
1107  * debugging user-mode applications. Since the WHPX API does not offer
1108  * an easy way to pass the intercepted exception back to the guest, we
1109  * resort to using INT1 instead, and let the guest always handle INT3.
1110  */
1111 static const uint8_t whpx_breakpoint_instruction = 0xF1;
1112
1113 /*
1114  * The WHPX QEMU backend implements breakpoints by writing the INT1
1115  * instruction into memory (ignoring the DRx registers). This raises a few
1116  * issues that need to be carefully handled:
1117  *
1118  * 1. Although unlikely, other parts of QEMU may set multiple breakpoints
1119  *    at the same location, and later remove them in arbitrary order.
1120  *    This should not cause memory corruption, and should only remove the
1121  *    physical breakpoint instruction when the last QEMU breakpoint is gone.
1122  *
1123  * 2. Writing arbitrary virtual memory may fail if it's not mapped to a valid
1124  *    physical location. Hence, physically adding/removing a breakpoint can
1125  *    theoretically fail at any time. We need to keep track of it.
1126  *
1127  * The function below rebuilds a list of low-level breakpoints (one per
1128  * address, tracking the original instruction and any errors) from the list of
1129  * high-level breakpoints (set via cpu_breakpoint_insert()).
1130  *
1131  * In order to optimize performance, this function stores the list of
1132  * high-level breakpoints (a.k.a. CPU breakpoints) used to compute the
1133  * low-level ones, so that it won't be re-invoked until these breakpoints
1134  * change.
1135  *
1136  * Note that this function decides which breakpoints should be inserted into,
1137  * memory, but doesn't actually do it. The memory accessing is done in
1138  * whpx_apply_breakpoints().
1139  */
1140 static void whpx_translate_cpu_breakpoints(
1141     struct whpx_breakpoints *breakpoints,
1142     CPUState *cpu,
1143     int cpu_breakpoint_count)
1144 {
1145     CPUBreakpoint *bp;
1146     int cpu_bp_index = 0;
1147
1148     breakpoints->original_addresses =
1149         g_renew(vaddr, breakpoints->original_addresses, cpu_breakpoint_count);
1150
1151     breakpoints->original_address_count = cpu_breakpoint_count;
1152
1153     int max_breakpoints = cpu_breakpoint_count +
1154         (breakpoints->breakpoints ? breakpoints->breakpoints->used : 0);
1155
1156     struct whpx_breakpoint_collection *new_breakpoints =
1157         g_malloc0(sizeof(struct whpx_breakpoint_collection)
1158                   + max_breakpoints * sizeof(struct whpx_breakpoint));
1159
1160     new_breakpoints->allocated = max_breakpoints;
1161     new_breakpoints->used = 0;
1162
1163     /*
1164      * 1. Preserve all old breakpoints that could not be automatically
1165      * cleared when the CPU got stopped.
1166      */
1167     if (breakpoints->breakpoints) {
1168         int i;
1169         for (i = 0; i < breakpoints->breakpoints->used; i++) {
1170             if (breakpoints->breakpoints->data[i].state != WHPX_BP_CLEARED) {
1171                 new_breakpoints->data[new_breakpoints->used++] =
1172                     breakpoints->breakpoints->data[i];
1173             }
1174         }
1175     }
1176
1177     /* 2. Map all CPU breakpoints to WHPX breakpoints */
1178     QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
1179         int i;
1180         bool found = false;
1181
1182         /* This will be used to detect changed CPU breakpoints later. */
1183         breakpoints->original_addresses[cpu_bp_index++] = bp->pc;
1184
1185         for (i = 0; i < new_breakpoints->used; i++) {
1186             /*
1187              * WARNING: This loop has O(N^2) complexity, where N is the
1188              * number of breakpoints. It should not be a bottleneck in
1189              * real-world scenarios, since it only needs to run once after
1190              * the breakpoints have been modified.
1191              * If this ever becomes a concern, it can be optimized by storing
1192              * high-level breakpoint objects in a tree or hash map.
1193              */
1194
1195             if (new_breakpoints->data[i].address == bp->pc) {
1196                 /* There was already a breakpoint at this address. */
1197                 if (new_breakpoints->data[i].state == WHPX_BP_CLEAR_PENDING) {
1198                     new_breakpoints->data[i].state = WHPX_BP_SET;
1199                 } else if (new_breakpoints->data[i].state == WHPX_BP_SET) {
1200                     new_breakpoints->data[i].state = WHPX_BP_SET_PENDING;
1201                 }
1202
1203                 found = true;
1204                 break;
1205             }
1206         }
1207
1208         if (!found && new_breakpoints->used < new_breakpoints->allocated) {
1209             /* No WHPX breakpoint at this address. Create one. */
1210             new_breakpoints->data[new_breakpoints->used].address = bp->pc;
1211             new_breakpoints->data[new_breakpoints->used].state =
1212                 WHPX_BP_SET_PENDING;
1213             new_breakpoints->used++;
1214         }
1215     }
1216
1217     /*
1218      * Free the previous breakpoint list. This can be optimized by keeping
1219      * it as shadow buffer for the next computation instead of freeing
1220      * it immediately.
1221      */
1222     g_free(breakpoints->breakpoints);
1223
1224     breakpoints->breakpoints = new_breakpoints;
1225 }
1226
1227 /*
1228  * Physically inserts/removes the breakpoints by reading and writing the
1229  * physical memory, keeping a track of the failed attempts.
1230  *
1231  * Passing resuming=true  will try to set all previously unset breakpoints.
1232  * Passing resuming=false will remove all inserted ones.
1233  */
1234 static void whpx_apply_breakpoints(
1235     struct whpx_breakpoint_collection *breakpoints,
1236     CPUState *cpu,
1237     bool resuming)
1238 {
1239     int i, rc;
1240     if (!breakpoints) {
1241         return;
1242     }
1243
1244     for (i = 0; i < breakpoints->used; i++) {
1245         /* Decide what to do right now based on the last known state. */
1246         WhpxBreakpointState state = breakpoints->data[i].state;
1247         switch (state) {
1248         case WHPX_BP_CLEARED:
1249             if (resuming) {
1250                 state = WHPX_BP_SET_PENDING;
1251             }
1252             break;
1253         case WHPX_BP_SET_PENDING:
1254             if (!resuming) {
1255                 state = WHPX_BP_CLEARED;
1256             }
1257             break;
1258         case WHPX_BP_SET:
1259             if (!resuming) {
1260                 state = WHPX_BP_CLEAR_PENDING;
1261             }
1262             break;
1263         case WHPX_BP_CLEAR_PENDING:
1264             if (resuming) {
1265                 state = WHPX_BP_SET;
1266             }
1267             break;
1268         }
1269
1270         if (state == WHPX_BP_SET_PENDING) {
1271             /* Remember the original instruction. */
1272             rc = cpu_memory_rw_debug(cpu,
1273                 breakpoints->data[i].address,
1274                 &breakpoints->data[i].original_instruction,
1275                 1,
1276                 false);
1277
1278             if (!rc) {
1279                 /* Write the breakpoint instruction. */
1280                 rc = cpu_memory_rw_debug(cpu,
1281                     breakpoints->data[i].address,
1282                     (void *)&whpx_breakpoint_instruction,
1283                     1,
1284                     true);
1285             }
1286
1287             if (!rc) {
1288                 state = WHPX_BP_SET;
1289             }
1290
1291         }
1292
1293         if (state == WHPX_BP_CLEAR_PENDING) {
1294             /* Restore the original instruction. */
1295             rc = cpu_memory_rw_debug(cpu,
1296                 breakpoints->data[i].address,
1297                 &breakpoints->data[i].original_instruction,
1298                 1,
1299                 true);
1300
1301             if (!rc) {
1302                 state = WHPX_BP_CLEARED;
1303             }
1304         }
1305
1306         breakpoints->data[i].state = state;
1307     }
1308 }
1309
1310 /*
1311  * This function is called when the a VCPU is about to start and no other
1312  * VCPUs have been started so far. Since the VCPU start order could be
1313  * arbitrary, it doesn't have to be VCPU#0.
1314  *
1315  * It is used to commit the breakpoints into memory, and configure WHPX
1316  * to intercept debug exceptions.
1317  *
1318  * Note that whpx_set_exception_exit_bitmap() cannot be called if one or
1319  * more VCPUs are already running, so this is the best place to do it.
1320  */
1321 static int whpx_first_vcpu_starting(CPUState *cpu)
1322 {
1323     struct whpx_state *whpx = &whpx_global;
1324     HRESULT hr;
1325
1326     g_assert(bql_locked());
1327
1328     if (!QTAILQ_EMPTY(&cpu->breakpoints) ||
1329             (whpx->breakpoints.breakpoints &&
1330              whpx->breakpoints.breakpoints->used)) {
1331         CPUBreakpoint *bp;
1332         int i = 0;
1333         bool update_pending = false;
1334
1335         QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
1336             if (i >= whpx->breakpoints.original_address_count ||
1337                 bp->pc != whpx->breakpoints.original_addresses[i]) {
1338                 update_pending = true;
1339             }
1340
1341             i++;
1342         }
1343
1344         if (i != whpx->breakpoints.original_address_count) {
1345             update_pending = true;
1346         }
1347
1348         if (update_pending) {
1349             /*
1350              * The CPU breakpoints have changed since the last call to
1351              * whpx_translate_cpu_breakpoints(). WHPX breakpoints must
1352              * now be recomputed.
1353              */
1354             whpx_translate_cpu_breakpoints(&whpx->breakpoints, cpu, i);
1355         }
1356
1357         /* Actually insert the breakpoints into the memory. */
1358         whpx_apply_breakpoints(whpx->breakpoints.breakpoints, cpu, true);
1359     }
1360
1361     uint64_t exception_mask;
1362     if (whpx->step_pending ||
1363         (whpx->breakpoints.breakpoints &&
1364          whpx->breakpoints.breakpoints->used)) {
1365         /*
1366          * We are either attempting to single-step one or more CPUs, or
1367          * have one or more breakpoints enabled. Both require intercepting
1368          * the WHvX64ExceptionTypeBreakpointTrap exception.
1369          */
1370
1371         exception_mask = 1UL << WHvX64ExceptionTypeDebugTrapOrFault;
1372     } else {
1373         /* Let the guest handle all exceptions. */
1374         exception_mask = 0;
1375     }
1376
1377     hr = whpx_set_exception_exit_bitmap(exception_mask);
1378     if (!SUCCEEDED(hr)) {
1379         error_report("WHPX: Failed to update exception exit mask,"
1380                      "hr=%08lx.", hr);
1381         return 1;
1382     }
1383
1384     return 0;
1385 }
1386
1387 /*
1388  * This function is called when the last VCPU has finished running.
1389  * It is used to remove any previously set breakpoints from memory.
1390  */
1391 static int whpx_last_vcpu_stopping(CPUState *cpu)
1392 {
1393     whpx_apply_breakpoints(whpx_global.breakpoints.breakpoints, cpu, false);
1394     return 0;
1395 }
1396
1397 /* Returns the address of the next instruction that is about to be executed. */
1398 static vaddr whpx_vcpu_get_pc(CPUState *cpu, bool exit_context_valid)
1399 {
1400     if (cpu->accel->dirty) {
1401         /* The CPU registers have been modified by other parts of QEMU. */
1402         return cpu_env(cpu)->eip;
1403     } else if (exit_context_valid) {
1404         /*
1405          * The CPU registers have not been modified by neither other parts
1406          * of QEMU, nor this port by calling WHvSetVirtualProcessorRegisters().
1407          * This is the most common case.
1408          */
1409         AccelCPUState *vcpu = cpu->accel;
1410         return vcpu->exit_ctx.VpContext.Rip;
1411     } else {
1412         /*
1413          * The CPU registers have been modified by a call to
1414          * WHvSetVirtualProcessorRegisters() and must be re-queried from
1415          * the target.
1416          */
1417         WHV_REGISTER_VALUE reg_value;
1418         WHV_REGISTER_NAME reg_name = WHvX64RegisterRip;
1419         HRESULT hr;
1420         struct whpx_state *whpx = &whpx_global;
1421
1422         hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
1423             whpx->partition,
1424             cpu->cpu_index,
1425             &reg_name,
1426             1,
1427             &reg_value);
1428
1429         if (FAILED(hr)) {
1430             error_report("WHPX: Failed to get PC, hr=%08lx", hr);
1431             return 0;
1432         }
1433
1434         return reg_value.Reg64;
1435     }
1436 }
1437
1438 static int whpx_handle_halt(CPUState *cpu)
1439 {
1440     int ret = 0;
1441
1442     bql_lock();
1443     if (!((cpu->interrupt_request & CPU_INTERRUPT_HARD) &&
1444           (cpu_env(cpu)->eflags & IF_MASK)) &&
1445         !(cpu->interrupt_request & CPU_INTERRUPT_NMI)) {
1446         cpu->exception_index = EXCP_HLT;
1447         cpu->halted = true;
1448         ret = 1;
1449     }
1450     bql_unlock();
1451
1452     return ret;
1453 }
1454
1455 static void whpx_vcpu_pre_run(CPUState *cpu)
1456 {
1457     HRESULT hr;
1458     struct whpx_state *whpx = &whpx_global;
1459     AccelCPUState *vcpu = cpu->accel;
1460     X86CPU *x86_cpu = X86_CPU(cpu);
1461     CPUX86State *env = &x86_cpu->env;
1462     int irq;
1463     uint8_t tpr;
1464     WHV_X64_PENDING_INTERRUPTION_REGISTER new_int;
1465     UINT32 reg_count = 0;
1466     WHV_REGISTER_VALUE reg_values[3];
1467     WHV_REGISTER_NAME reg_names[3];
1468
1469     memset(&new_int, 0, sizeof(new_int));
1470     memset(reg_values, 0, sizeof(reg_values));
1471
1472     bql_lock();
1473
1474     /* Inject NMI */
1475     if (!vcpu->interruption_pending &&
1476         cpu->interrupt_request & (CPU_INTERRUPT_NMI | CPU_INTERRUPT_SMI)) {
1477         if (cpu->interrupt_request & CPU_INTERRUPT_NMI) {
1478             cpu->interrupt_request &= ~CPU_INTERRUPT_NMI;
1479             vcpu->interruptable = false;
1480             new_int.InterruptionType = WHvX64PendingNmi;
1481             new_int.InterruptionPending = 1;
1482             new_int.InterruptionVector = 2;
1483         }
1484         if (cpu->interrupt_request & CPU_INTERRUPT_SMI) {
1485             cpu->interrupt_request &= ~CPU_INTERRUPT_SMI;
1486         }
1487     }
1488
1489     /*
1490      * Force the VCPU out of its inner loop to process any INIT requests or
1491      * commit pending TPR access.
1492      */
1493     if (cpu->interrupt_request & (CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR)) {
1494         if ((cpu->interrupt_request & CPU_INTERRUPT_INIT) &&
1495             !(env->hflags & HF_SMM_MASK)) {
1496             cpu->exit_request = 1;
1497         }
1498         if (cpu->interrupt_request & CPU_INTERRUPT_TPR) {
1499             cpu->exit_request = 1;
1500         }
1501     }
1502
1503     /* Get pending hard interruption or replay one that was overwritten */
1504     if (!whpx_apic_in_platform()) {
1505         if (!vcpu->interruption_pending &&
1506             vcpu->interruptable && (env->eflags & IF_MASK)) {
1507             assert(!new_int.InterruptionPending);
1508             if (cpu->interrupt_request & CPU_INTERRUPT_HARD) {
1509                 cpu->interrupt_request &= ~CPU_INTERRUPT_HARD;
1510                 irq = cpu_get_pic_interrupt(env);
1511                 if (irq >= 0) {
1512                     new_int.InterruptionType = WHvX64PendingInterrupt;
1513                     new_int.InterruptionPending = 1;
1514                     new_int.InterruptionVector = irq;
1515                 }
1516             }
1517         }
1518
1519         /* Setup interrupt state if new one was prepared */
1520         if (new_int.InterruptionPending) {
1521             reg_values[reg_count].PendingInterruption = new_int;
1522             reg_names[reg_count] = WHvRegisterPendingInterruption;
1523             reg_count += 1;
1524         }
1525     } else if (vcpu->ready_for_pic_interrupt &&
1526                (cpu->interrupt_request & CPU_INTERRUPT_HARD)) {
1527         cpu->interrupt_request &= ~CPU_INTERRUPT_HARD;
1528         irq = cpu_get_pic_interrupt(env);
1529         if (irq >= 0) {
1530             reg_names[reg_count] = WHvRegisterPendingEvent;
1531             reg_values[reg_count].ExtIntEvent = (WHV_X64_PENDING_EXT_INT_EVENT)
1532             {
1533                 .EventPending = 1,
1534                 .EventType = WHvX64PendingEventExtInt,
1535                 .Vector = irq,
1536             };
1537             reg_count += 1;
1538         }
1539      }
1540
1541     /* Sync the TPR to the CR8 if was modified during the intercept */
1542     tpr = whpx_apic_tpr_to_cr8(cpu_get_apic_tpr(x86_cpu->apic_state));
1543     if (tpr != vcpu->tpr) {
1544         vcpu->tpr = tpr;
1545         reg_values[reg_count].Reg64 = tpr;
1546         cpu->exit_request = 1;
1547         reg_names[reg_count] = WHvX64RegisterCr8;
1548         reg_count += 1;
1549     }
1550
1551     /* Update the state of the interrupt delivery notification */
1552     if (!vcpu->window_registered &&
1553         cpu->interrupt_request & CPU_INTERRUPT_HARD) {
1554         reg_values[reg_count].DeliverabilityNotifications =
1555             (WHV_X64_DELIVERABILITY_NOTIFICATIONS_REGISTER) {
1556                 .InterruptNotification = 1
1557             };
1558         vcpu->window_registered = 1;
1559         reg_names[reg_count] = WHvX64RegisterDeliverabilityNotifications;
1560         reg_count += 1;
1561     }
1562
1563     bql_unlock();
1564     vcpu->ready_for_pic_interrupt = false;
1565
1566     if (reg_count) {
1567         hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
1568             whpx->partition, cpu->cpu_index,
1569             reg_names, reg_count, reg_values);
1570         if (FAILED(hr)) {
1571             error_report("WHPX: Failed to set interrupt state registers,"
1572                          " hr=%08lx", hr);
1573         }
1574     }
1575
1576     return;
1577 }
1578
1579 static void whpx_vcpu_post_run(CPUState *cpu)
1580 {
1581     AccelCPUState *vcpu = cpu->accel;
1582     X86CPU *x86_cpu = X86_CPU(cpu);
1583     CPUX86State *env = &x86_cpu->env;
1584
1585     env->eflags = vcpu->exit_ctx.VpContext.Rflags;
1586
1587     uint64_t tpr = vcpu->exit_ctx.VpContext.Cr8;
1588     if (vcpu->tpr != tpr) {
1589         vcpu->tpr = tpr;
1590         bql_lock();
1591         cpu_set_apic_tpr(x86_cpu->apic_state, whpx_cr8_to_apic_tpr(vcpu->tpr));
1592         bql_unlock();
1593     }
1594
1595     vcpu->interruption_pending =
1596         vcpu->exit_ctx.VpContext.ExecutionState.InterruptionPending;
1597
1598     vcpu->interruptable =
1599         !vcpu->exit_ctx.VpContext.ExecutionState.InterruptShadow;
1600
1601     return;
1602 }
1603
1604 static void whpx_vcpu_process_async_events(CPUState *cpu)
1605 {
1606     X86CPU *x86_cpu = X86_CPU(cpu);
1607     CPUX86State *env = &x86_cpu->env;
1608     AccelCPUState *vcpu = cpu->accel;
1609
1610     if ((cpu->interrupt_request & CPU_INTERRUPT_INIT) &&
1611         !(env->hflags & HF_SMM_MASK)) {
1612         whpx_cpu_synchronize_state(cpu);
1613         do_cpu_init(x86_cpu);
1614         vcpu->interruptable = true;
1615     }
1616
1617     if (cpu->interrupt_request & CPU_INTERRUPT_POLL) {
1618         cpu->interrupt_request &= ~CPU_INTERRUPT_POLL;
1619         apic_poll_irq(x86_cpu->apic_state);
1620     }
1621
1622     if (((cpu->interrupt_request & CPU_INTERRUPT_HARD) &&
1623          (env->eflags & IF_MASK)) ||
1624         (cpu->interrupt_request & CPU_INTERRUPT_NMI)) {
1625         cpu->halted = false;
1626     }
1627
1628     if (cpu->interrupt_request & CPU_INTERRUPT_SIPI) {
1629         whpx_cpu_synchronize_state(cpu);
1630         do_cpu_sipi(x86_cpu);
1631     }
1632
1633     if (cpu->interrupt_request & CPU_INTERRUPT_TPR) {
1634         cpu->interrupt_request &= ~CPU_INTERRUPT_TPR;
1635         whpx_cpu_synchronize_state(cpu);
1636         apic_handle_tpr_access_report(x86_cpu->apic_state, env->eip,
1637                                       env->tpr_access_type);
1638     }
1639
1640     return;
1641 }
1642
1643 static int whpx_vcpu_run(CPUState *cpu)
1644 {
1645     HRESULT hr;
1646     struct whpx_state *whpx = &whpx_global;
1647     AccelCPUState *vcpu = cpu->accel;
1648     struct whpx_breakpoint *stepped_over_bp = NULL;
1649     WhpxStepMode exclusive_step_mode = WHPX_STEP_NONE;
1650     int ret;
1651
1652     g_assert(bql_locked());
1653
1654     if (whpx->running_cpus++ == 0) {
1655         /* Insert breakpoints into memory, update exception exit bitmap. */
1656         ret = whpx_first_vcpu_starting(cpu);
1657         if (ret != 0) {
1658             return ret;
1659         }
1660     }
1661
1662     if (whpx->breakpoints.breakpoints &&
1663         whpx->breakpoints.breakpoints->used > 0)
1664     {
1665         uint64_t pc = whpx_vcpu_get_pc(cpu, true);
1666         stepped_over_bp = whpx_lookup_breakpoint_by_addr(pc);
1667         if (stepped_over_bp && stepped_over_bp->state != WHPX_BP_SET) {
1668             stepped_over_bp = NULL;
1669         }
1670
1671         if (stepped_over_bp) {
1672             /*
1673              * We are trying to run the instruction overwritten by an active
1674              * breakpoint. We will temporarily disable the breakpoint, suspend
1675              * other CPUs, and step over the instruction.
1676              */
1677             exclusive_step_mode = WHPX_STEP_EXCLUSIVE;
1678         }
1679     }
1680
1681     if (exclusive_step_mode == WHPX_STEP_NONE) {
1682         whpx_vcpu_process_async_events(cpu);
1683         if (cpu->halted && !whpx_apic_in_platform()) {
1684             cpu->exception_index = EXCP_HLT;
1685             qatomic_set(&cpu->exit_request, false);
1686             return 0;
1687         }
1688     }
1689
1690     bql_unlock();
1691
1692     if (exclusive_step_mode != WHPX_STEP_NONE) {
1693         start_exclusive();
1694         g_assert(cpu == current_cpu);
1695         g_assert(!cpu->running);
1696         cpu->running = true;
1697
1698         hr = whpx_set_exception_exit_bitmap(
1699             1UL << WHvX64ExceptionTypeDebugTrapOrFault);
1700         if (!SUCCEEDED(hr)) {
1701             error_report("WHPX: Failed to update exception exit mask, "
1702                          "hr=%08lx.", hr);
1703             return 1;
1704         }
1705
1706         if (stepped_over_bp) {
1707             /* Temporarily disable the triggered breakpoint. */
1708             cpu_memory_rw_debug(cpu,
1709                 stepped_over_bp->address,
1710                 &stepped_over_bp->original_instruction,
1711                 1,
1712                 true);
1713         }
1714     } else {
1715         cpu_exec_start(cpu);
1716     }
1717
1718     do {
1719         if (cpu->accel->dirty) {
1720             whpx_set_registers(cpu, WHPX_SET_RUNTIME_STATE);
1721             cpu->accel->dirty = false;
1722         }
1723
1724         if (exclusive_step_mode == WHPX_STEP_NONE) {
1725             whpx_vcpu_pre_run(cpu);
1726
1727             if (qatomic_read(&cpu->exit_request)) {
1728                 whpx_vcpu_kick(cpu);
1729             }
1730         }
1731
1732         if (exclusive_step_mode != WHPX_STEP_NONE || cpu->singlestep_enabled) {
1733             whpx_vcpu_configure_single_stepping(cpu, true, NULL);
1734         }
1735
1736         hr = whp_dispatch.WHvRunVirtualProcessor(
1737             whpx->partition, cpu->cpu_index,
1738             &vcpu->exit_ctx, sizeof(vcpu->exit_ctx));
1739
1740         if (FAILED(hr)) {
1741             error_report("WHPX: Failed to exec a virtual processor,"
1742                          " hr=%08lx", hr);
1743             ret = -1;
1744             break;
1745         }
1746
1747         if (exclusive_step_mode != WHPX_STEP_NONE || cpu->singlestep_enabled) {
1748             whpx_vcpu_configure_single_stepping(cpu,
1749                 false,
1750                 &vcpu->exit_ctx.VpContext.Rflags);
1751         }
1752
1753         whpx_vcpu_post_run(cpu);
1754
1755         switch (vcpu->exit_ctx.ExitReason) {
1756         case WHvRunVpExitReasonMemoryAccess:
1757             ret = whpx_handle_mmio(cpu, &vcpu->exit_ctx.MemoryAccess);
1758             break;
1759
1760         case WHvRunVpExitReasonX64IoPortAccess:
1761             ret = whpx_handle_portio(cpu, &vcpu->exit_ctx.IoPortAccess);
1762             break;
1763
1764         case WHvRunVpExitReasonX64InterruptWindow:
1765             vcpu->ready_for_pic_interrupt = 1;
1766             vcpu->window_registered = 0;
1767             ret = 0;
1768             break;
1769
1770         case WHvRunVpExitReasonX64ApicEoi:
1771             assert(whpx_apic_in_platform());
1772             ioapic_eoi_broadcast(vcpu->exit_ctx.ApicEoi.InterruptVector);
1773             break;
1774
1775         case WHvRunVpExitReasonX64Halt:
1776             /*
1777              * WARNING: as of build 19043.1526 (21H1), this exit reason is no
1778              * longer used.
1779              */
1780             ret = whpx_handle_halt(cpu);
1781             break;
1782
1783         case WHvRunVpExitReasonX64ApicInitSipiTrap: {
1784             WHV_INTERRUPT_CONTROL ipi = {0};
1785             uint64_t icr = vcpu->exit_ctx.ApicInitSipi.ApicIcr;
1786             uint32_t delivery_mode =
1787                 (icr & APIC_ICR_DELIV_MOD) >> APIC_ICR_DELIV_MOD_SHIFT;
1788             int dest_shorthand =
1789                 (icr & APIC_ICR_DEST_SHORT) >> APIC_ICR_DEST_SHORT_SHIFT;
1790             bool broadcast = false;
1791             bool include_self = false;
1792             uint32_t i;
1793
1794             /* We only registered for INIT and SIPI exits. */
1795             if ((delivery_mode != APIC_DM_INIT) &&
1796                 (delivery_mode != APIC_DM_SIPI)) {
1797                 error_report(
1798                     "WHPX: Unexpected APIC exit that is not a INIT or SIPI");
1799                 break;
1800             }
1801
1802             if (delivery_mode == APIC_DM_INIT) {
1803                 ipi.Type = WHvX64InterruptTypeInit;
1804             } else {
1805                 ipi.Type = WHvX64InterruptTypeSipi;
1806             }
1807
1808             ipi.DestinationMode =
1809                 ((icr & APIC_ICR_DEST_MOD) >> APIC_ICR_DEST_MOD_SHIFT) ?
1810                     WHvX64InterruptDestinationModeLogical :
1811                     WHvX64InterruptDestinationModePhysical;
1812
1813             ipi.TriggerMode =
1814                 ((icr & APIC_ICR_TRIGGER_MOD) >> APIC_ICR_TRIGGER_MOD_SHIFT) ?
1815                     WHvX64InterruptTriggerModeLevel :
1816                     WHvX64InterruptTriggerModeEdge;
1817
1818             ipi.Vector = icr & APIC_VECTOR_MASK;
1819             switch (dest_shorthand) {
1820             /* no shorthand. Bits 56-63 contain the destination. */
1821             case 0:
1822                 ipi.Destination = (icr >> 56) & APIC_VECTOR_MASK;
1823                 hr = whp_dispatch.WHvRequestInterrupt(whpx->partition,
1824                         &ipi, sizeof(ipi));
1825                 if (FAILED(hr)) {
1826                     error_report("WHPX: Failed to request interrupt  hr=%08lx",
1827                         hr);
1828                 }
1829
1830                 break;
1831
1832             /* self */
1833             case 1:
1834                 include_self = true;
1835                 break;
1836
1837             /* broadcast, including self */
1838             case 2:
1839                 broadcast = true;
1840                 include_self = true;
1841                 break;
1842
1843             /* broadcast, excluding self */
1844             case 3:
1845                 broadcast = true;
1846                 break;
1847             }
1848
1849             if (!broadcast && !include_self) {
1850                 break;
1851             }
1852
1853             for (i = 0; i <= max_vcpu_index; i++) {
1854                 if (i == cpu->cpu_index && !include_self) {
1855                     continue;
1856                 }
1857
1858                 /*
1859                  * Assuming that APIC Ids are identity mapped since
1860                  * WHvX64RegisterApicId & WHvX64RegisterInitialApicId registers
1861                  * are not handled yet and the hypervisor doesn't allow the
1862                  * guest to modify the APIC ID.
1863                  */
1864                 ipi.Destination = i;
1865                 hr = whp_dispatch.WHvRequestInterrupt(whpx->partition,
1866                         &ipi, sizeof(ipi));
1867                 if (FAILED(hr)) {
1868                     error_report(
1869                         "WHPX: Failed to request SIPI for %d,  hr=%08lx",
1870                         i, hr);
1871                 }
1872             }
1873
1874             break;
1875         }
1876
1877         case WHvRunVpExitReasonCanceled:
1878             if (exclusive_step_mode != WHPX_STEP_NONE) {
1879                 /*
1880                  * We are trying to step over a single instruction, and
1881                  * likely got a request to stop from another thread.
1882                  * Delay it until we are done stepping
1883                  * over.
1884                  */
1885                 ret = 0;
1886             } else {
1887                 cpu->exception_index = EXCP_INTERRUPT;
1888                 ret = 1;
1889             }
1890             break;
1891         case WHvRunVpExitReasonX64MsrAccess: {
1892             WHV_REGISTER_VALUE reg_values[3] = {0};
1893             WHV_REGISTER_NAME reg_names[3];
1894             UINT32 reg_count;
1895
1896             reg_names[0] = WHvX64RegisterRip;
1897             reg_names[1] = WHvX64RegisterRax;
1898             reg_names[2] = WHvX64RegisterRdx;
1899
1900             reg_values[0].Reg64 =
1901                 vcpu->exit_ctx.VpContext.Rip +
1902                 vcpu->exit_ctx.VpContext.InstructionLength;
1903
1904             /*
1905              * For all unsupported MSR access we:
1906              *     ignore writes
1907              *     return 0 on read.
1908              */
1909             reg_count = vcpu->exit_ctx.MsrAccess.AccessInfo.IsWrite ?
1910                         1 : 3;
1911
1912             hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
1913                 whpx->partition,
1914                 cpu->cpu_index,
1915                 reg_names, reg_count,
1916                 reg_values);
1917
1918             if (FAILED(hr)) {
1919                 error_report("WHPX: Failed to set MsrAccess state "
1920                              " registers, hr=%08lx", hr);
1921             }
1922             ret = 0;
1923             break;
1924         }
1925         case WHvRunVpExitReasonX64Cpuid: {
1926             WHV_REGISTER_VALUE reg_values[5];
1927             WHV_REGISTER_NAME reg_names[5];
1928             UINT32 reg_count = 5;
1929             UINT64 cpuid_fn, rip = 0, rax = 0, rcx = 0, rdx = 0, rbx = 0;
1930             X86CPU *x86_cpu = X86_CPU(cpu);
1931             CPUX86State *env = &x86_cpu->env;
1932
1933             memset(reg_values, 0, sizeof(reg_values));
1934
1935             rip = vcpu->exit_ctx.VpContext.Rip +
1936                   vcpu->exit_ctx.VpContext.InstructionLength;
1937             cpuid_fn = vcpu->exit_ctx.CpuidAccess.Rax;
1938
1939             /*
1940              * Ideally, these should be supplied to the hypervisor during VCPU
1941              * initialization and it should be able to satisfy this request.
1942              * But, currently, WHPX doesn't support setting CPUID values in the
1943              * hypervisor once the partition has been setup, which is too late
1944              * since VCPUs are realized later. For now, use the values from
1945              * QEMU to satisfy these requests, until WHPX adds support for
1946              * being able to set these values in the hypervisor at runtime.
1947              */
1948             cpu_x86_cpuid(env, cpuid_fn, 0, (UINT32 *)&rax, (UINT32 *)&rbx,
1949                 (UINT32 *)&rcx, (UINT32 *)&rdx);
1950             switch (cpuid_fn) {
1951             case 0x40000000:
1952                 /* Expose the vmware cpu frequency cpuid leaf */
1953                 rax = 0x40000010;
1954                 rbx = rcx = rdx = 0;
1955                 break;
1956
1957             case 0x40000010:
1958                 rax = env->tsc_khz;
1959                 rbx = env->apic_bus_freq / 1000; /* Hz to KHz */
1960                 rcx = rdx = 0;
1961                 break;
1962
1963             case 0x80000001:
1964                 /* Remove any support of OSVW */
1965                 rcx &= ~CPUID_EXT3_OSVW;
1966                 break;
1967             }
1968
1969             reg_names[0] = WHvX64RegisterRip;
1970             reg_names[1] = WHvX64RegisterRax;
1971             reg_names[2] = WHvX64RegisterRcx;
1972             reg_names[3] = WHvX64RegisterRdx;
1973             reg_names[4] = WHvX64RegisterRbx;
1974
1975             reg_values[0].Reg64 = rip;
1976             reg_values[1].Reg64 = rax;
1977             reg_values[2].Reg64 = rcx;
1978             reg_values[3].Reg64 = rdx;
1979             reg_values[4].Reg64 = rbx;
1980
1981             hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
1982                 whpx->partition, cpu->cpu_index,
1983                 reg_names,
1984                 reg_count,
1985                 reg_values);
1986
1987             if (FAILED(hr)) {
1988                 error_report("WHPX: Failed to set CpuidAccess state registers,"
1989                              " hr=%08lx", hr);
1990             }
1991             ret = 0;
1992             break;
1993         }
1994         case WHvRunVpExitReasonException:
1995             whpx_get_registers(cpu);
1996
1997             if ((vcpu->exit_ctx.VpException.ExceptionType ==
1998                  WHvX64ExceptionTypeDebugTrapOrFault) &&
1999                 (vcpu->exit_ctx.VpException.InstructionByteCount >= 1) &&
2000                 (vcpu->exit_ctx.VpException.InstructionBytes[0] ==
2001                  whpx_breakpoint_instruction)) {
2002                 /* Stopped at a software breakpoint. */
2003                 cpu->exception_index = EXCP_DEBUG;
2004             } else if ((vcpu->exit_ctx.VpException.ExceptionType ==
2005                         WHvX64ExceptionTypeDebugTrapOrFault) &&
2006                        !cpu->singlestep_enabled) {
2007                 /*
2008                  * Just finished stepping over a breakpoint, but the
2009                  * gdb does not expect us to do single-stepping.
2010                  * Don't do anything special.
2011                  */
2012                 cpu->exception_index = EXCP_INTERRUPT;
2013             } else {
2014                 /* Another exception or debug event. Report it to GDB. */
2015                 cpu->exception_index = EXCP_DEBUG;
2016             }
2017
2018             ret = 1;
2019             break;
2020         case WHvRunVpExitReasonNone:
2021         case WHvRunVpExitReasonUnrecoverableException:
2022         case WHvRunVpExitReasonInvalidVpRegisterValue:
2023         case WHvRunVpExitReasonUnsupportedFeature:
2024         default:
2025             error_report("WHPX: Unexpected VP exit code %d",
2026                          vcpu->exit_ctx.ExitReason);
2027             whpx_get_registers(cpu);
2028             bql_lock();
2029             qemu_system_guest_panicked(cpu_get_crash_info(cpu));
2030             bql_unlock();
2031             break;
2032         }
2033
2034     } while (!ret);
2035
2036     if (stepped_over_bp) {
2037         /* Restore the breakpoint we stepped over */
2038         cpu_memory_rw_debug(cpu,
2039             stepped_over_bp->address,
2040             (void *)&whpx_breakpoint_instruction,
2041             1,
2042             true);
2043     }
2044
2045     if (exclusive_step_mode != WHPX_STEP_NONE) {
2046         g_assert(cpu_in_exclusive_context(cpu));
2047         cpu->running = false;
2048         end_exclusive();
2049
2050         exclusive_step_mode = WHPX_STEP_NONE;
2051     } else {
2052         cpu_exec_end(cpu);
2053     }
2054
2055     bql_lock();
2056     current_cpu = cpu;
2057
2058     if (--whpx->running_cpus == 0) {
2059         whpx_last_vcpu_stopping(cpu);
2060     }
2061
2062     qatomic_set(&cpu->exit_request, false);
2063
2064     return ret < 0;
2065 }
2066
2067 static void do_whpx_cpu_synchronize_state(CPUState *cpu, run_on_cpu_data arg)
2068 {
2069     if (!cpu->accel->dirty) {
2070         whpx_get_registers(cpu);
2071         cpu->accel->dirty = true;
2072     }
2073 }
2074
2075 static void do_whpx_cpu_synchronize_post_reset(CPUState *cpu,
2076                                                run_on_cpu_data arg)
2077 {
2078     whpx_set_registers(cpu, WHPX_SET_RESET_STATE);
2079     cpu->accel->dirty = false;
2080 }
2081
2082 static void do_whpx_cpu_synchronize_post_init(CPUState *cpu,
2083                                               run_on_cpu_data arg)
2084 {
2085     whpx_set_registers(cpu, WHPX_SET_FULL_STATE);
2086     cpu->accel->dirty = false;
2087 }
2088
2089 static void do_whpx_cpu_synchronize_pre_loadvm(CPUState *cpu,
2090                                                run_on_cpu_data arg)
2091 {
2092     cpu->accel->dirty = true;
2093 }
2094
2095 /*
2096  * CPU support.
2097  */
2098
2099 void whpx_cpu_synchronize_state(CPUState *cpu)
2100 {
2101     if (!cpu->accel->dirty) {
2102         run_on_cpu(cpu, do_whpx_cpu_synchronize_state, RUN_ON_CPU_NULL);
2103     }
2104 }
2105
2106 void whpx_cpu_synchronize_post_reset(CPUState *cpu)
2107 {
2108     run_on_cpu(cpu, do_whpx_cpu_synchronize_post_reset, RUN_ON_CPU_NULL);
2109 }
2110
2111 void whpx_cpu_synchronize_post_init(CPUState *cpu)
2112 {
2113     run_on_cpu(cpu, do_whpx_cpu_synchronize_post_init, RUN_ON_CPU_NULL);
2114 }
2115
2116 void whpx_cpu_synchronize_pre_loadvm(CPUState *cpu)
2117 {
2118     run_on_cpu(cpu, do_whpx_cpu_synchronize_pre_loadvm, RUN_ON_CPU_NULL);
2119 }
2120
2121 void whpx_cpu_synchronize_pre_resume(bool step_pending)
2122 {
2123     whpx_global.step_pending = step_pending;
2124 }
2125
2126 /*
2127  * Vcpu support.
2128  */
2129
2130 static Error *whpx_migration_blocker;
2131
2132 static void whpx_cpu_update_state(void *opaque, bool running, RunState state)
2133 {
2134     CPUX86State *env = opaque;
2135
2136     if (running) {
2137         env->tsc_valid = false;
2138     }
2139 }
2140
2141 int whpx_init_vcpu(CPUState *cpu)
2142 {
2143     HRESULT hr;
2144     struct whpx_state *whpx = &whpx_global;
2145     AccelCPUState *vcpu = NULL;
2146     Error *local_error = NULL;
2147     X86CPU *x86_cpu = X86_CPU(cpu);
2148     CPUX86State *env = &x86_cpu->env;
2149     UINT64 freq = 0;
2150     int ret;
2151
2152     /* Add migration blockers for all unsupported features of the
2153      * Windows Hypervisor Platform
2154      */
2155     if (whpx_migration_blocker == NULL) {
2156         error_setg(&whpx_migration_blocker,
2157                "State blocked due to non-migratable CPUID feature support,"
2158                "dirty memory tracking support, and XSAVE/XRSTOR support");
2159
2160         if (migrate_add_blocker(&whpx_migration_blocker, &local_error) < 0) {
2161             error_report_err(local_error);
2162             ret = -EINVAL;
2163             goto error;
2164         }
2165     }
2166
2167     vcpu = g_new0(AccelCPUState, 1);
2168
2169     hr = whp_dispatch.WHvEmulatorCreateEmulator(
2170         &whpx_emu_callbacks,
2171         &vcpu->emulator);
2172     if (FAILED(hr)) {
2173         error_report("WHPX: Failed to setup instruction completion support,"
2174                      " hr=%08lx", hr);
2175         ret = -EINVAL;
2176         goto error;
2177     }
2178
2179     hr = whp_dispatch.WHvCreateVirtualProcessor(
2180         whpx->partition, cpu->cpu_index, 0);
2181     if (FAILED(hr)) {
2182         error_report("WHPX: Failed to create a virtual processor,"
2183                      " hr=%08lx", hr);
2184         whp_dispatch.WHvEmulatorDestroyEmulator(vcpu->emulator);
2185         ret = -EINVAL;
2186         goto error;
2187     }
2188
2189     /*
2190      * vcpu's TSC frequency is either specified by user, or use the value
2191      * provided by Hyper-V if the former is not present. In the latter case, we
2192      * query it from Hyper-V and record in env->tsc_khz, so that vcpu's TSC
2193      * frequency can be migrated later via this field.
2194      */
2195     if (!env->tsc_khz) {
2196         hr = whp_dispatch.WHvGetCapability(
2197             WHvCapabilityCodeProcessorClockFrequency, &freq, sizeof(freq),
2198                 NULL);
2199         if (hr != WHV_E_UNKNOWN_CAPABILITY) {
2200             if (FAILED(hr)) {
2201                 printf("WHPX: Failed to query tsc frequency, hr=0x%08lx\n", hr);
2202             } else {
2203                 env->tsc_khz = freq / 1000; /* Hz to KHz */
2204             }
2205         }
2206     }
2207
2208     env->apic_bus_freq = HYPERV_APIC_BUS_FREQUENCY;
2209     hr = whp_dispatch.WHvGetCapability(
2210         WHvCapabilityCodeInterruptClockFrequency, &freq, sizeof(freq), NULL);
2211     if (hr != WHV_E_UNKNOWN_CAPABILITY) {
2212         if (FAILED(hr)) {
2213             printf("WHPX: Failed to query apic bus frequency hr=0x%08lx\n", hr);
2214         } else {
2215             env->apic_bus_freq = freq;
2216         }
2217     }
2218
2219     /*
2220      * If the vmware cpuid frequency leaf option is set, and we have a valid
2221      * tsc value, trap the corresponding cpuid's.
2222      */
2223     if (x86_cpu->vmware_cpuid_freq && env->tsc_khz) {
2224         UINT32 cpuidExitList[] = {1, 0x80000001, 0x40000000, 0x40000010};
2225
2226         hr = whp_dispatch.WHvSetPartitionProperty(
2227                 whpx->partition,
2228                 WHvPartitionPropertyCodeCpuidExitList,
2229                 cpuidExitList,
2230                 RTL_NUMBER_OF(cpuidExitList) * sizeof(UINT32));
2231
2232         if (FAILED(hr)) {
2233             error_report("WHPX: Failed to set partition CpuidExitList hr=%08lx",
2234                         hr);
2235             ret = -EINVAL;
2236             goto error;
2237         }
2238     }
2239
2240     vcpu->interruptable = true;
2241     vcpu->dirty = true;
2242     cpu->accel = vcpu;
2243     max_vcpu_index = max(max_vcpu_index, cpu->cpu_index);
2244     qemu_add_vm_change_state_handler(whpx_cpu_update_state, env);
2245
2246     return 0;
2247
2248 error:
2249     g_free(vcpu);
2250
2251     return ret;
2252 }
2253
2254 int whpx_vcpu_exec(CPUState *cpu)
2255 {
2256     int ret;
2257     int fatal;
2258
2259     for (;;) {
2260         if (cpu->exception_index >= EXCP_INTERRUPT) {
2261             ret = cpu->exception_index;
2262             cpu->exception_index = -1;
2263             break;
2264         }
2265
2266         fatal = whpx_vcpu_run(cpu);
2267
2268         if (fatal) {
2269             error_report("WHPX: Failed to exec a virtual processor");
2270             abort();
2271         }
2272     }
2273
2274     return ret;
2275 }
2276
2277 void whpx_destroy_vcpu(CPUState *cpu)
2278 {
2279     struct whpx_state *whpx = &whpx_global;
2280     AccelCPUState *vcpu = cpu->accel;
2281
2282     whp_dispatch.WHvDeleteVirtualProcessor(whpx->partition, cpu->cpu_index);
2283     whp_dispatch.WHvEmulatorDestroyEmulator(vcpu->emulator);
2284     g_free(cpu->accel);
2285     return;
2286 }
2287
2288 void whpx_vcpu_kick(CPUState *cpu)
2289 {
2290     struct whpx_state *whpx = &whpx_global;
2291     whp_dispatch.WHvCancelRunVirtualProcessor(
2292         whpx->partition, cpu->cpu_index, 0);
2293 }
2294
2295 /*
2296  * Memory support.
2297  */
2298
2299 static void whpx_update_mapping(hwaddr start_pa, ram_addr_t size,
2300                                 void *host_va, int add, int rom,
2301                                 const char *name)
2302 {
2303     struct whpx_state *whpx = &whpx_global;
2304     HRESULT hr;
2305
2306     /*
2307     if (add) {
2308         printf("WHPX: ADD PA:%p Size:%p, Host:%p, %s, '%s'\n",
2309                (void*)start_pa, (void*)size, host_va,
2310                (rom ? "ROM" : "RAM"), name);
2311     } else {
2312         printf("WHPX: DEL PA:%p Size:%p, Host:%p,      '%s'\n",
2313                (void*)start_pa, (void*)size, host_va, name);
2314     }
2315     */
2316
2317     if (add) {
2318         hr = whp_dispatch.WHvMapGpaRange(whpx->partition,
2319                                          host_va,
2320                                          start_pa,
2321                                          size,
2322                                          (WHvMapGpaRangeFlagRead |
2323                                           WHvMapGpaRangeFlagExecute |
2324                                           (rom ? 0 : WHvMapGpaRangeFlagWrite)));
2325     } else {
2326         hr = whp_dispatch.WHvUnmapGpaRange(whpx->partition,
2327                                            start_pa,
2328                                            size);
2329     }
2330
2331     if (FAILED(hr)) {
2332         error_report("WHPX: Failed to %s GPA range '%s' PA:%p, Size:%p bytes,"
2333                      " Host:%p, hr=%08lx",
2334                      (add ? "MAP" : "UNMAP"), name,
2335                      (void *)(uintptr_t)start_pa, (void *)size, host_va, hr);
2336     }
2337 }
2338
2339 static void whpx_process_section(MemoryRegionSection *section, int add)
2340 {
2341     MemoryRegion *mr = section->mr;
2342     hwaddr start_pa = section->offset_within_address_space;
2343     ram_addr_t size = int128_get64(section->size);
2344     unsigned int delta;
2345     uint64_t host_va;
2346
2347     if (!memory_region_is_ram(mr)) {
2348         return;
2349     }
2350
2351     delta = qemu_real_host_page_size() - (start_pa & ~qemu_real_host_page_mask());
2352     delta &= ~qemu_real_host_page_mask();
2353     if (delta > size) {
2354         return;
2355     }
2356     start_pa += delta;
2357     size -= delta;
2358     size &= qemu_real_host_page_mask();
2359     if (!size || (start_pa & ~qemu_real_host_page_mask())) {
2360         return;
2361     }
2362
2363     host_va = (uintptr_t)memory_region_get_ram_ptr(mr)
2364             + section->offset_within_region + delta;
2365
2366     whpx_update_mapping(start_pa, size, (void *)(uintptr_t)host_va, add,
2367                         memory_region_is_rom(mr), mr->name);
2368 }
2369
2370 static void whpx_region_add(MemoryListener *listener,
2371                            MemoryRegionSection *section)
2372 {
2373     memory_region_ref(section->mr);
2374     whpx_process_section(section, 1);
2375 }
2376
2377 static void whpx_region_del(MemoryListener *listener,
2378                            MemoryRegionSection *section)
2379 {
2380     whpx_process_section(section, 0);
2381     memory_region_unref(section->mr);
2382 }
2383
2384 static void whpx_transaction_begin(MemoryListener *listener)
2385 {
2386 }
2387
2388 static void whpx_transaction_commit(MemoryListener *listener)
2389 {
2390 }
2391
2392 static void whpx_log_sync(MemoryListener *listener,
2393                          MemoryRegionSection *section)
2394 {
2395     MemoryRegion *mr = section->mr;
2396
2397     if (!memory_region_is_ram(mr)) {
2398         return;
2399     }
2400
2401     memory_region_set_dirty(mr, 0, int128_get64(section->size));
2402 }
2403
2404 static MemoryListener whpx_memory_listener = {
2405     .name = "whpx",
2406     .begin = whpx_transaction_begin,
2407     .commit = whpx_transaction_commit,
2408     .region_add = whpx_region_add,
2409     .region_del = whpx_region_del,
2410     .log_sync = whpx_log_sync,
2411     .priority = MEMORY_LISTENER_PRIORITY_ACCEL,
2412 };
2413
2414 static void whpx_memory_init(void)
2415 {
2416     memory_listener_register(&whpx_memory_listener, &address_space_memory);
2417 }
2418
2419 /*
2420  * Load the functions from the given library, using the given handle. If a
2421  * handle is provided, it is used, otherwise the library is opened. The
2422  * handle will be updated on return with the opened one.
2423  */
2424 static bool load_whp_dispatch_fns(HMODULE *handle,
2425     WHPFunctionList function_list)
2426 {
2427     HMODULE hLib = *handle;
2428
2429     #define WINHV_PLATFORM_DLL "WinHvPlatform.dll"
2430     #define WINHV_EMULATION_DLL "WinHvEmulation.dll"
2431     #define WHP_LOAD_FIELD_OPTIONAL(return_type, function_name, signature) \
2432         whp_dispatch.function_name = \
2433             (function_name ## _t)GetProcAddress(hLib, #function_name); \
2434
2435     #define WHP_LOAD_FIELD(return_type, function_name, signature) \
2436         whp_dispatch.function_name = \
2437             (function_name ## _t)GetProcAddress(hLib, #function_name); \
2438         if (!whp_dispatch.function_name) { \
2439             error_report("Could not load function %s", #function_name); \
2440             goto error; \
2441         } \
2442
2443     #define WHP_LOAD_LIB(lib_name, handle_lib) \
2444     if (!handle_lib) { \
2445         handle_lib = LoadLibrary(lib_name); \
2446         if (!handle_lib) { \
2447             error_report("Could not load library %s.", lib_name); \
2448             goto error; \
2449         } \
2450     } \
2451
2452     switch (function_list) {
2453     case WINHV_PLATFORM_FNS_DEFAULT:
2454         WHP_LOAD_LIB(WINHV_PLATFORM_DLL, hLib)
2455         LIST_WINHVPLATFORM_FUNCTIONS(WHP_LOAD_FIELD)
2456         break;
2457
2458     case WINHV_EMULATION_FNS_DEFAULT:
2459         WHP_LOAD_LIB(WINHV_EMULATION_DLL, hLib)
2460         LIST_WINHVEMULATION_FUNCTIONS(WHP_LOAD_FIELD)
2461         break;
2462
2463     case WINHV_PLATFORM_FNS_SUPPLEMENTAL:
2464         WHP_LOAD_LIB(WINHV_PLATFORM_DLL, hLib)
2465         LIST_WINHVPLATFORM_FUNCTIONS_SUPPLEMENTAL(WHP_LOAD_FIELD_OPTIONAL)
2466         break;
2467     }
2468
2469     *handle = hLib;
2470     return true;
2471
2472 error:
2473     if (hLib) {
2474         FreeLibrary(hLib);
2475     }
2476
2477     return false;
2478 }
2479
2480 static void whpx_set_kernel_irqchip(Object *obj, Visitor *v,
2481                                    const char *name, void *opaque,
2482                                    Error **errp)
2483 {
2484     struct whpx_state *whpx = &whpx_global;
2485     OnOffSplit mode;
2486
2487     if (!visit_type_OnOffSplit(v, name, &mode, errp)) {
2488         return;
2489     }
2490
2491     switch (mode) {
2492     case ON_OFF_SPLIT_ON:
2493         whpx->kernel_irqchip_allowed = true;
2494         whpx->kernel_irqchip_required = true;
2495         break;
2496
2497     case ON_OFF_SPLIT_OFF:
2498         whpx->kernel_irqchip_allowed = false;
2499         whpx->kernel_irqchip_required = false;
2500         break;
2501
2502     case ON_OFF_SPLIT_SPLIT:
2503         error_setg(errp, "WHPX: split irqchip currently not supported");
2504         error_append_hint(errp,
2505             "Try without kernel-irqchip or with kernel-irqchip=on|off");
2506         break;
2507
2508     default:
2509         /*
2510          * The value was checked in visit_type_OnOffSplit() above. If
2511          * we get here, then something is wrong in QEMU.
2512          */
2513         abort();
2514     }
2515 }
2516
2517 /*
2518  * Partition support
2519  */
2520
2521 static int whpx_accel_init(MachineState *ms)
2522 {
2523     struct whpx_state *whpx;
2524     int ret;
2525     HRESULT hr;
2526     WHV_CAPABILITY whpx_cap;
2527     UINT32 whpx_cap_size;
2528     WHV_PARTITION_PROPERTY prop;
2529     UINT32 cpuidExitList[] = {1, 0x80000001};
2530     WHV_CAPABILITY_FEATURES features = {0};
2531
2532     whpx = &whpx_global;
2533
2534     if (!init_whp_dispatch()) {
2535         ret = -ENOSYS;
2536         goto error;
2537     }
2538
2539     whpx->mem_quota = ms->ram_size;
2540
2541     hr = whp_dispatch.WHvGetCapability(
2542         WHvCapabilityCodeHypervisorPresent, &whpx_cap,
2543         sizeof(whpx_cap), &whpx_cap_size);
2544     if (FAILED(hr) || !whpx_cap.HypervisorPresent) {
2545         error_report("WHPX: No accelerator found, hr=%08lx", hr);
2546         ret = -ENOSPC;
2547         goto error;
2548     }
2549
2550     hr = whp_dispatch.WHvGetCapability(
2551         WHvCapabilityCodeFeatures, &features, sizeof(features), NULL);
2552     if (FAILED(hr)) {
2553         error_report("WHPX: Failed to query capabilities, hr=%08lx", hr);
2554         ret = -EINVAL;
2555         goto error;
2556     }
2557
2558     hr = whp_dispatch.WHvCreatePartition(&whpx->partition);
2559     if (FAILED(hr)) {
2560         error_report("WHPX: Failed to create partition, hr=%08lx", hr);
2561         ret = -EINVAL;
2562         goto error;
2563     }
2564
2565     /*
2566      * Query the XSAVE capability of the partition. Any error here is not
2567      * considered fatal.
2568      */
2569     hr = whp_dispatch.WHvGetPartitionProperty(
2570         whpx->partition,
2571         WHvPartitionPropertyCodeProcessorXsaveFeatures,
2572         &whpx_xsave_cap,
2573         sizeof(whpx_xsave_cap),
2574         &whpx_cap_size);
2575
2576     /*
2577      * Windows version which don't support this property will return with the
2578      * specific error code.
2579      */
2580     if (FAILED(hr) && hr != WHV_E_UNKNOWN_PROPERTY) {
2581         error_report("WHPX: Failed to query XSAVE capability, hr=%08lx", hr);
2582     }
2583
2584     if (!whpx_has_xsave()) {
2585         printf("WHPX: Partition is not XSAVE capable\n");
2586     }
2587
2588     memset(&prop, 0, sizeof(WHV_PARTITION_PROPERTY));
2589     prop.ProcessorCount = ms->smp.cpus;
2590     hr = whp_dispatch.WHvSetPartitionProperty(
2591         whpx->partition,
2592         WHvPartitionPropertyCodeProcessorCount,
2593         &prop,
2594         sizeof(WHV_PARTITION_PROPERTY));
2595
2596     if (FAILED(hr)) {
2597         error_report("WHPX: Failed to set partition processor count to %u,"
2598                      " hr=%08lx", prop.ProcessorCount, hr);
2599         ret = -EINVAL;
2600         goto error;
2601     }
2602
2603     /*
2604      * Error out if WHP doesn't support apic emulation and user is requiring
2605      * it.
2606      */
2607     if (whpx->kernel_irqchip_required && (!features.LocalApicEmulation ||
2608             !whp_dispatch.WHvSetVirtualProcessorInterruptControllerState2)) {
2609         error_report("WHPX: kernel irqchip requested, but unavailable. "
2610             "Try without kernel-irqchip or with kernel-irqchip=off");
2611         ret = -EINVAL;
2612         goto error;
2613     }
2614
2615     if (whpx->kernel_irqchip_allowed && features.LocalApicEmulation &&
2616         whp_dispatch.WHvSetVirtualProcessorInterruptControllerState2) {
2617         WHV_X64_LOCAL_APIC_EMULATION_MODE mode =
2618             WHvX64LocalApicEmulationModeXApic;
2619         printf("WHPX: setting APIC emulation mode in the hypervisor\n");
2620         hr = whp_dispatch.WHvSetPartitionProperty(
2621             whpx->partition,
2622             WHvPartitionPropertyCodeLocalApicEmulationMode,
2623             &mode,
2624             sizeof(mode));
2625         if (FAILED(hr)) {
2626             error_report("WHPX: Failed to enable kernel irqchip hr=%08lx", hr);
2627             if (whpx->kernel_irqchip_required) {
2628                 error_report("WHPX: kernel irqchip requested, but unavailable");
2629                 ret = -EINVAL;
2630                 goto error;
2631             }
2632         } else {
2633             whpx->apic_in_platform = true;
2634         }
2635     }
2636
2637     /* Register for MSR and CPUID exits */
2638     memset(&prop, 0, sizeof(WHV_PARTITION_PROPERTY));
2639     prop.ExtendedVmExits.X64MsrExit = 1;
2640     prop.ExtendedVmExits.X64CpuidExit = 1;
2641     prop.ExtendedVmExits.ExceptionExit = 1;
2642     if (whpx_apic_in_platform()) {
2643         prop.ExtendedVmExits.X64ApicInitSipiExitTrap = 1;
2644     }
2645
2646     hr = whp_dispatch.WHvSetPartitionProperty(
2647             whpx->partition,
2648             WHvPartitionPropertyCodeExtendedVmExits,
2649             &prop,
2650             sizeof(WHV_PARTITION_PROPERTY));
2651     if (FAILED(hr)) {
2652         error_report("WHPX: Failed to enable MSR & CPUIDexit, hr=%08lx", hr);
2653         ret = -EINVAL;
2654         goto error;
2655     }
2656
2657     hr = whp_dispatch.WHvSetPartitionProperty(
2658         whpx->partition,
2659         WHvPartitionPropertyCodeCpuidExitList,
2660         cpuidExitList,
2661         RTL_NUMBER_OF(cpuidExitList) * sizeof(UINT32));
2662
2663     if (FAILED(hr)) {
2664         error_report("WHPX: Failed to set partition CpuidExitList hr=%08lx",
2665                      hr);
2666         ret = -EINVAL;
2667         goto error;
2668     }
2669
2670     /*
2671      * We do not want to intercept any exceptions from the guest,
2672      * until we actually start debugging with gdb.
2673      */
2674     whpx->exception_exit_bitmap = -1;
2675     hr = whpx_set_exception_exit_bitmap(0);
2676
2677     if (FAILED(hr)) {
2678         error_report("WHPX: Failed to set exception exit bitmap, hr=%08lx", hr);
2679         ret = -EINVAL;
2680         goto error;
2681     }
2682
2683     hr = whp_dispatch.WHvSetupPartition(whpx->partition);
2684     if (FAILED(hr)) {
2685         error_report("WHPX: Failed to setup partition, hr=%08lx", hr);
2686         ret = -EINVAL;
2687         goto error;
2688     }
2689
2690     whpx_memory_init();
2691
2692     printf("Windows Hypervisor Platform accelerator is operational\n");
2693     return 0;
2694
2695 error:
2696
2697     if (NULL != whpx->partition) {
2698         whp_dispatch.WHvDeletePartition(whpx->partition);
2699         whpx->partition = NULL;
2700     }
2701
2702     return ret;
2703 }
2704
2705 int whpx_enabled(void)
2706 {
2707     return whpx_allowed;
2708 }
2709
2710 bool whpx_apic_in_platform(void) {
2711     return whpx_global.apic_in_platform;
2712 }
2713
2714 static void whpx_accel_class_init(ObjectClass *oc, void *data)
2715 {
2716     AccelClass *ac = ACCEL_CLASS(oc);
2717     ac->name = "WHPX";
2718     ac->init_machine = whpx_accel_init;
2719     ac->allowed = &whpx_allowed;
2720
2721     object_class_property_add(oc, "kernel-irqchip", "on|off|split",
2722         NULL, whpx_set_kernel_irqchip,
2723         NULL, NULL);
2724     object_class_property_set_description(oc, "kernel-irqchip",
2725         "Configure WHPX in-kernel irqchip");
2726 }
2727
2728 static void whpx_accel_instance_init(Object *obj)
2729 {
2730     struct whpx_state *whpx = &whpx_global;
2731
2732     memset(whpx, 0, sizeof(struct whpx_state));
2733     /* Turn on kernel-irqchip, by default */
2734     whpx->kernel_irqchip_allowed = true;
2735 }
2736
2737 static const TypeInfo whpx_accel_type = {
2738     .name = ACCEL_CLASS_NAME("whpx"),
2739     .parent = TYPE_ACCEL,
2740     .instance_init = whpx_accel_instance_init,
2741     .class_init = whpx_accel_class_init,
2742 };
2743
2744 static void whpx_type_init(void)
2745 {
2746     type_register_static(&whpx_accel_type);
2747 }
2748
2749 bool init_whp_dispatch(void)
2750 {
2751     if (whp_dispatch_initialized) {
2752         return true;
2753     }
2754
2755     if (!load_whp_dispatch_fns(&hWinHvPlatform, WINHV_PLATFORM_FNS_DEFAULT)) {
2756         goto error;
2757     }
2758
2759     if (!load_whp_dispatch_fns(&hWinHvEmulation, WINHV_EMULATION_FNS_DEFAULT)) {
2760         goto error;
2761     }
2762
2763     assert(load_whp_dispatch_fns(&hWinHvPlatform,
2764         WINHV_PLATFORM_FNS_SUPPLEMENTAL));
2765     whp_dispatch_initialized = true;
2766
2767     return true;
2768 error:
2769     if (hWinHvPlatform) {
2770         FreeLibrary(hWinHvPlatform);
2771     }
2772
2773     if (hWinHvEmulation) {
2774         FreeLibrary(hWinHvEmulation);
2775     }
2776
2777     return false;
2778 }
2779
2780 type_init(whpx_type_init);