?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
docker: remove libcap development packages
[qemu/ar7.git] / target / i386 / kvm.c
blobbf1655645bce46a352d897e92aad1aa1b6be4085
1 /*
2 * QEMU KVM support
3 *
4 * Copyright (C) 2006-2008 Qumranet Technologies
5 * Copyright IBM, Corp. 2008
6 *
7 * Authors:
8 * Anthony Liguori <aliguori@us.ibm.com>
9 *
10 * This work is licensed under the terms of the GNU GPL, version 2 or later.
11 * See the COPYING file in the top-level directory.
12 *
13 */
14
15 #include "qemu/osdep.h"
16 #include "qapi/error.h"
17 #include <sys/ioctl.h>
18 #include <sys/utsname.h>
19
20 #include <linux/kvm.h>
21 #include "standard-headers/asm-x86/kvm_para.h"
22
23 #include "cpu.h"
24 #include "sysemu/sysemu.h"
25 #include "sysemu/hw_accel.h"
26 #include "sysemu/kvm_int.h"
27 #include "sysemu/reset.h"
28 #include "sysemu/runstate.h"
29 #include "kvm_i386.h"
30 #include "hyperv.h"
31 #include "hyperv-proto.h"
32
33 #include "exec/gdbstub.h"
34 #include "qemu/host-utils.h"
35 #include "qemu/main-loop.h"
36 #include "qemu/config-file.h"
37 #include "qemu/error-report.h"
38 #include "hw/i386/pc.h"
39 #include "hw/i386/apic.h"
40 #include "hw/i386/apic_internal.h"
41 #include "hw/i386/apic-msidef.h"
42 #include "hw/i386/intel_iommu.h"
43 #include "hw/i386/x86-iommu.h"
44 #include "hw/i386/e820_memory_layout.h"
45
46 #include "hw/pci/pci.h"
47 #include "hw/pci/msi.h"
48 #include "hw/pci/msix.h"
49 #include "migration/blocker.h"
50 #include "exec/memattrs.h"
51 #include "trace.h"
52
53 //#define DEBUG_KVM
54
55 #ifdef DEBUG_KVM
56 #define DPRINTF(fmt, ...) \
57 do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
58 #else
59 #define DPRINTF(fmt, ...) \
60 do { } while (0)
61 #endif
62
63 #define MSR_KVM_WALL_CLOCK 0x11
64 #define MSR_KVM_SYSTEM_TIME 0x12
65
66 /* A 4096-byte buffer can hold the 8-byte kvm_msrs header, plus
67 * 255 kvm_msr_entry structs */
68 #define MSR_BUF_SIZE 4096
69
70 const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
71 KVM_CAP_INFO(SET_TSS_ADDR),
72 KVM_CAP_INFO(EXT_CPUID),
73 KVM_CAP_INFO(MP_STATE),
74 KVM_CAP_LAST_INFO
75 };
76
77 static bool has_msr_star;
78 static bool has_msr_hsave_pa;
79 static bool has_msr_tsc_aux;
80 static bool has_msr_tsc_adjust;
81 static bool has_msr_tsc_deadline;
82 static bool has_msr_feature_control;
83 static bool has_msr_misc_enable;
84 static bool has_msr_smbase;
85 static bool has_msr_bndcfgs;
86 static int lm_capable_kernel;
87 static bool has_msr_hv_hypercall;
88 static bool has_msr_hv_crash;
89 static bool has_msr_hv_reset;
90 static bool has_msr_hv_vpindex;
91 static bool hv_vpindex_settable;
92 static bool has_msr_hv_runtime;
93 static bool has_msr_hv_synic;
94 static bool has_msr_hv_stimer;
95 static bool has_msr_hv_frequencies;
96 static bool has_msr_hv_reenlightenment;
97 static bool has_msr_xss;
98 static bool has_msr_umwait;
99 static bool has_msr_spec_ctrl;
100 static bool has_msr_tsx_ctrl;
101 static bool has_msr_virt_ssbd;
102 static bool has_msr_smi_count;
103 static bool has_msr_arch_capabs;
104 static bool has_msr_core_capabs;
105 static bool has_msr_vmx_vmfunc;
106
107 static uint32_t has_architectural_pmu_version;
108 static uint32_t num_architectural_pmu_gp_counters;
109 static uint32_t num_architectural_pmu_fixed_counters;
110
111 static int has_xsave;
112 static int has_xcrs;
113 static int has_pit_state2;
114 static int has_exception_payload;
115
116 static bool has_msr_mcg_ext_ctl;
117
118 static struct kvm_cpuid2 *cpuid_cache;
119 static struct kvm_msr_list *kvm_feature_msrs;
120
121 int kvm_has_pit_state2(void)
122 {
123 return has_pit_state2;
124 }
125
126 bool kvm_has_smm(void)
127 {
128 return kvm_check_extension(kvm_state, KVM_CAP_X86_SMM);
129 }
130
131 bool kvm_has_adjust_clock_stable(void)
132 {
133 int ret = kvm_check_extension(kvm_state, KVM_CAP_ADJUST_CLOCK);
134
135 return (ret == KVM_CLOCK_TSC_STABLE);
136 }
137
138 bool kvm_has_exception_payload(void)
139 {
140 return has_exception_payload;
141 }
142
143 bool kvm_allows_irq0_override(void)
144 {
145 return !kvm_irqchip_in_kernel() || kvm_has_gsi_routing();
146 }
147
148 static bool kvm_x2apic_api_set_flags(uint64_t flags)
149 {
150 KVMState *s = KVM_STATE(current_machine->accelerator);
151
152 return !kvm_vm_enable_cap(s, KVM_CAP_X2APIC_API, 0, flags);
153 }
154
155 #define MEMORIZE(fn, _result) \
156 ({ \
157 static bool _memorized; \
158 \
159 if (_memorized) { \
160 return _result; \
161 } \
162 _memorized = true; \
163 _result = fn; \
164 })
165
166 static bool has_x2apic_api;
167
168 bool kvm_has_x2apic_api(void)
169 {
170 return has_x2apic_api;
171 }
172
173 bool kvm_enable_x2apic(void)
174 {
175 return MEMORIZE(
176 kvm_x2apic_api_set_flags(KVM_X2APIC_API_USE_32BIT_IDS |
177 KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK),
178 has_x2apic_api);
179 }
180
181 bool kvm_hv_vpindex_settable(void)
182 {
183 return hv_vpindex_settable;
184 }
185
186 static int kvm_get_tsc(CPUState *cs)
187 {
188 X86CPU *cpu = X86_CPU(cs);
189 CPUX86State *env = &cpu->env;
190 struct {
191 struct kvm_msrs info;
192 struct kvm_msr_entry entries[1];
193 } msr_data = {};
194 int ret;
195
196 if (env->tsc_valid) {
197 return 0;
198 }
199
200 memset(&msr_data, 0, sizeof(msr_data));
201 msr_data.info.nmsrs = 1;
202 msr_data.entries[0].index = MSR_IA32_TSC;
203 env->tsc_valid = !runstate_is_running();
204
205 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MSRS, &msr_data);
206 if (ret < 0) {
207 return ret;
208 }
209
210 assert(ret == 1);
211 env->tsc = msr_data.entries[0].data;
212 return 0;
213 }
214
215 static inline void do_kvm_synchronize_tsc(CPUState *cpu, run_on_cpu_data arg)
216 {
217 kvm_get_tsc(cpu);
218 }
219
220 void kvm_synchronize_all_tsc(void)
221 {
222 CPUState *cpu;
223
224 if (kvm_enabled()) {
225 CPU_FOREACH(cpu) {
226 run_on_cpu(cpu, do_kvm_synchronize_tsc, RUN_ON_CPU_NULL);
227 }
228 }
229 }
230
231 static struct kvm_cpuid2 *try_get_cpuid(KVMState *s, int max)
232 {
233 struct kvm_cpuid2 *cpuid;
234 int r, size;
235
236 size = sizeof(*cpuid) + max * sizeof(*cpuid->entries);
237 cpuid = g_malloc0(size);
238 cpuid->nent = max;
239 r = kvm_ioctl(s, KVM_GET_SUPPORTED_CPUID, cpuid);
240 if (r == 0 && cpuid->nent >= max) {
241 r = -E2BIG;
242 }
243 if (r < 0) {
244 if (r == -E2BIG) {
245 g_free(cpuid);
246 return NULL;
247 } else {
248 fprintf(stderr, "KVM_GET_SUPPORTED_CPUID failed: %s\n",
249 strerror(-r));
250 exit(1);
251 }
252 }
253 return cpuid;
254 }
255
256 /* Run KVM_GET_SUPPORTED_CPUID ioctl(), allocating a buffer large enough
257 * for all entries.
258 */
259 static struct kvm_cpuid2 *get_supported_cpuid(KVMState *s)
260 {
261 struct kvm_cpuid2 *cpuid;
262 int max = 1;
263
264 if (cpuid_cache != NULL) {
265 return cpuid_cache;
266 }
267 while ((cpuid = try_get_cpuid(s, max)) == NULL) {
268 max *= 2;
269 }
270 cpuid_cache = cpuid;
271 return cpuid;
272 }
273
274 static const struct kvm_para_features {
275 int cap;
276 int feature;
277 } para_features[] = {
278 { KVM_CAP_CLOCKSOURCE, KVM_FEATURE_CLOCKSOURCE },
279 { KVM_CAP_NOP_IO_DELAY, KVM_FEATURE_NOP_IO_DELAY },
280 { KVM_CAP_PV_MMU, KVM_FEATURE_MMU_OP },
281 { KVM_CAP_ASYNC_PF, KVM_FEATURE_ASYNC_PF },
282 };
283
284 static int get_para_features(KVMState *s)
285 {
286 int i, features = 0;
287
288 for (i = 0; i < ARRAY_SIZE(para_features); i++) {
289 if (kvm_check_extension(s, para_features[i].cap)) {
290 features |= (1 << para_features[i].feature);
291 }
292 }
293
294 return features;
295 }
296
297 static bool host_tsx_blacklisted(void)
298 {
299 int family, model, stepping;\
300 char vendor[CPUID_VENDOR_SZ + 1];
301
302 host_vendor_fms(vendor, &family, &model, &stepping);
303
304 /* Check if we are running on a Haswell host known to have broken TSX */
305 return !strcmp(vendor, CPUID_VENDOR_INTEL) &&
306 (family == 6) &&
307 ((model == 63 && stepping < 4) ||
308 model == 60 || model == 69 || model == 70);
309 }
310
311 /* Returns the value for a specific register on the cpuid entry
312 */
313 static uint32_t cpuid_entry_get_reg(struct kvm_cpuid_entry2 *entry, int reg)
314 {
315 uint32_t ret = 0;
316 switch (reg) {
317 case R_EAX:
318 ret = entry->eax;
319 break;
320 case R_EBX:
321 ret = entry->ebx;
322 break;
323 case R_ECX:
324 ret = entry->ecx;
325 break;
326 case R_EDX:
327 ret = entry->edx;
328 break;
329 }
330 return ret;
331 }
332
333 /* Find matching entry for function/index on kvm_cpuid2 struct
334 */
335 static struct kvm_cpuid_entry2 *cpuid_find_entry(struct kvm_cpuid2 *cpuid,
336 uint32_t function,
337 uint32_t index)
338 {
339 int i;
340 for (i = 0; i < cpuid->nent; ++i) {
341 if (cpuid->entries[i].function == function &&
342 cpuid->entries[i].index == index) {
343 return &cpuid->entries[i];
344 }
345 }
346 /* not found: */
347 return NULL;
348 }
349
350 uint32_t kvm_arch_get_supported_cpuid(KVMState *s, uint32_t function,
351 uint32_t index, int reg)
352 {
353 struct kvm_cpuid2 *cpuid;
354 uint32_t ret = 0;
355 uint32_t cpuid_1_edx;
356 bool found = false;
357
358 cpuid = get_supported_cpuid(s);
359
360 struct kvm_cpuid_entry2 *entry = cpuid_find_entry(cpuid, function, index);
361 if (entry) {
362 found = true;
363 ret = cpuid_entry_get_reg(entry, reg);
364 }
365
366 /* Fixups for the data returned by KVM, below */
367
368 if (function == 1 && reg == R_EDX) {
369 /* KVM before 2.6.30 misreports the following features */
370 ret |= CPUID_MTRR | CPUID_PAT | CPUID_MCE | CPUID_MCA;
371 } else if (function == 1 && reg == R_ECX) {
372 /* We can set the hypervisor flag, even if KVM does not return it on
373 * GET_SUPPORTED_CPUID
374 */
375 ret |= CPUID_EXT_HYPERVISOR;
376 /* tsc-deadline flag is not returned by GET_SUPPORTED_CPUID, but it
377 * can be enabled if the kernel has KVM_CAP_TSC_DEADLINE_TIMER,
378 * and the irqchip is in the kernel.
379 */
380 if (kvm_irqchip_in_kernel() &&
381 kvm_check_extension(s, KVM_CAP_TSC_DEADLINE_TIMER)) {
382 ret |= CPUID_EXT_TSC_DEADLINE_TIMER;
383 }
384
385 /* x2apic is reported by GET_SUPPORTED_CPUID, but it can't be enabled
386 * without the in-kernel irqchip
387 */
388 if (!kvm_irqchip_in_kernel()) {
389 ret &= ~CPUID_EXT_X2APIC;
390 }
391
392 if (enable_cpu_pm) {
393 int disable_exits = kvm_check_extension(s,
394 KVM_CAP_X86_DISABLE_EXITS);
395
396 if (disable_exits & KVM_X86_DISABLE_EXITS_MWAIT) {
397 ret |= CPUID_EXT_MONITOR;
398 }
399 }
400 } else if (function == 6 && reg == R_EAX) {
401 ret |= CPUID_6_EAX_ARAT; /* safe to allow because of emulated APIC */
402 } else if (function == 7 && index == 0 && reg == R_EBX) {
403 if (host_tsx_blacklisted()) {
404 ret &= ~(CPUID_7_0_EBX_RTM | CPUID_7_0_EBX_HLE);
405 }
406 } else if (function == 7 && index == 0 && reg == R_ECX) {
407 if (enable_cpu_pm) {
408 ret |= CPUID_7_0_ECX_WAITPKG;
409 } else {
410 ret &= ~CPUID_7_0_ECX_WAITPKG;
411 }
412 } else if (function == 7 && index == 0 && reg == R_EDX) {
413 /*
414 * Linux v4.17-v4.20 incorrectly return ARCH_CAPABILITIES on SVM hosts.
415 * We can detect the bug by checking if MSR_IA32_ARCH_CAPABILITIES is
416 * returned by KVM_GET_MSR_INDEX_LIST.
417 */
418 if (!has_msr_arch_capabs) {
419 ret &= ~CPUID_7_0_EDX_ARCH_CAPABILITIES;
420 }
421 } else if (function == 0x80000001 && reg == R_ECX) {
422 /*
423 * It's safe to enable TOPOEXT even if it's not returned by
424 * GET_SUPPORTED_CPUID. Unconditionally enabling TOPOEXT here allows
425 * us to keep CPU models including TOPOEXT runnable on older kernels.
426 */
427 ret |= CPUID_EXT3_TOPOEXT;
428 } else if (function == 0x80000001 && reg == R_EDX) {
429 /* On Intel, kvm returns cpuid according to the Intel spec,
430 * so add missing bits according to the AMD spec:
431 */
432 cpuid_1_edx = kvm_arch_get_supported_cpuid(s, 1, 0, R_EDX);
433 ret |= cpuid_1_edx & CPUID_EXT2_AMD_ALIASES;
434 } else if (function == KVM_CPUID_FEATURES && reg == R_EAX) {
435 /* kvm_pv_unhalt is reported by GET_SUPPORTED_CPUID, but it can't
436 * be enabled without the in-kernel irqchip
437 */
438 if (!kvm_irqchip_in_kernel()) {
439 ret &= ~(1U << KVM_FEATURE_PV_UNHALT);
440 }
441 } else if (function == KVM_CPUID_FEATURES && reg == R_EDX) {
442 ret |= 1U << KVM_HINTS_REALTIME;
443 found = 1;
444 }
445
446 /* fallback for older kernels */
447 if ((function == KVM_CPUID_FEATURES) && !found) {
448 ret = get_para_features(s);
449 }
450
451 return ret;
452 }
453
454 uint64_t kvm_arch_get_supported_msr_feature(KVMState *s, uint32_t index)
455 {
456 struct {
457 struct kvm_msrs info;
458 struct kvm_msr_entry entries[1];
459 } msr_data = {};
460 uint64_t value;
461 uint32_t ret, can_be_one, must_be_one;
462
463 if (kvm_feature_msrs == NULL) { /* Host doesn't support feature MSRs */
464 return 0;
465 }
466
467 /* Check if requested MSR is supported feature MSR */
468 int i;
469 for (i = 0; i < kvm_feature_msrs->nmsrs; i++)
470 if (kvm_feature_msrs->indices[i] == index) {
471 break;
472 }
473 if (i == kvm_feature_msrs->nmsrs) {
474 return 0; /* if the feature MSR is not supported, simply return 0 */
475 }
476
477 msr_data.info.nmsrs = 1;
478 msr_data.entries[0].index = index;
479
480 ret = kvm_ioctl(s, KVM_GET_MSRS, &msr_data);
481 if (ret != 1) {
482 error_report("KVM get MSR (index=0x%x) feature failed, %s",
483 index, strerror(-ret));
484 exit(1);
485 }
486
487 value = msr_data.entries[0].data;
488 switch (index) {
489 case MSR_IA32_VMX_PROCBASED_CTLS2:
490 /* KVM forgot to add these bits for some time, do this ourselves. */
491 if (kvm_arch_get_supported_cpuid(s, 0xD, 1, R_ECX) & CPUID_XSAVE_XSAVES) {
492 value |= (uint64_t)VMX_SECONDARY_EXEC_XSAVES << 32;
493 }
494 if (kvm_arch_get_supported_cpuid(s, 1, 0, R_ECX) & CPUID_EXT_RDRAND) {
495 value |= (uint64_t)VMX_SECONDARY_EXEC_RDRAND_EXITING << 32;
496 }
497 if (kvm_arch_get_supported_cpuid(s, 7, 0, R_EBX) & CPUID_7_0_EBX_INVPCID) {
498 value |= (uint64_t)VMX_SECONDARY_EXEC_ENABLE_INVPCID << 32;
499 }
500 if (kvm_arch_get_supported_cpuid(s, 7, 0, R_EBX) & CPUID_7_0_EBX_RDSEED) {
501 value |= (uint64_t)VMX_SECONDARY_EXEC_RDSEED_EXITING << 32;
502 }
503 if (kvm_arch_get_supported_cpuid(s, 0x80000001, 0, R_EDX) & CPUID_EXT2_RDTSCP) {
504 value |= (uint64_t)VMX_SECONDARY_EXEC_RDTSCP << 32;
505 }
506 /* fall through */
507 case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
508 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
509 case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
510 case MSR_IA32_VMX_TRUE_EXIT_CTLS:
511 /*
512 * Return true for bits that can be one, but do not have to be one.
513 * The SDM tells us which bits could have a "must be one" setting,
514 * so we can do the opposite transformation in make_vmx_msr_value.
515 */
516 must_be_one = (uint32_t)value;
517 can_be_one = (uint32_t)(value >> 32);
518 return can_be_one & ~must_be_one;
519
520 default:
521 return value;
522 }
523 }
524
525
526 typedef struct HWPoisonPage {
527 ram_addr_t ram_addr;
528 QLIST_ENTRY(HWPoisonPage) list;
529 } HWPoisonPage;
530
531 static QLIST_HEAD(, HWPoisonPage) hwpoison_page_list =
532 QLIST_HEAD_INITIALIZER(hwpoison_page_list);
533
534 static void kvm_unpoison_all(void *param)
535 {
536 HWPoisonPage *page, *next_page;
537
538 QLIST_FOREACH_SAFE(page, &hwpoison_page_list, list, next_page) {
539 QLIST_REMOVE(page, list);
540 qemu_ram_remap(page->ram_addr, TARGET_PAGE_SIZE);
541 g_free(page);
542 }
543 }
544
545 static void kvm_hwpoison_page_add(ram_addr_t ram_addr)
546 {
547 HWPoisonPage *page;
548
549 QLIST_FOREACH(page, &hwpoison_page_list, list) {
550 if (page->ram_addr == ram_addr) {
551 return;
552 }
553 }
554 page = g_new(HWPoisonPage, 1);
555 page->ram_addr = ram_addr;
556 QLIST_INSERT_HEAD(&hwpoison_page_list, page, list);
557 }
558
559 static int kvm_get_mce_cap_supported(KVMState *s, uint64_t *mce_cap,
560 int *max_banks)
561 {
562 int r;
563
564 r = kvm_check_extension(s, KVM_CAP_MCE);
565 if (r > 0) {
566 *max_banks = r;
567 return kvm_ioctl(s, KVM_X86_GET_MCE_CAP_SUPPORTED, mce_cap);
568 }
569 return -ENOSYS;
570 }
571
572 static void kvm_mce_inject(X86CPU *cpu, hwaddr paddr, int code)
573 {
574 CPUState *cs = CPU(cpu);
575 CPUX86State *env = &cpu->env;
576 uint64_t status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN |
577 MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S;
578 uint64_t mcg_status = MCG_STATUS_MCIP;
579 int flags = 0;
580
581 if (code == BUS_MCEERR_AR) {
582 status |= MCI_STATUS_AR | 0x134;
583 mcg_status |= MCG_STATUS_EIPV;
584 } else {
585 status |= 0xc0;
586 mcg_status |= MCG_STATUS_RIPV;
587 }
588
589 flags = cpu_x86_support_mca_broadcast(env) ? MCE_INJECT_BROADCAST : 0;
590 /* We need to read back the value of MSR_EXT_MCG_CTL that was set by the
591 * guest kernel back into env->mcg_ext_ctl.
592 */
593 cpu_synchronize_state(cs);
594 if (env->mcg_ext_ctl & MCG_EXT_CTL_LMCE_EN) {
595 mcg_status |= MCG_STATUS_LMCE;
596 flags = 0;
597 }
598
599 cpu_x86_inject_mce(NULL, cpu, 9, status, mcg_status, paddr,
600 (MCM_ADDR_PHYS << 6) | 0xc, flags);
601 }
602
603 static void hardware_memory_error(void *host_addr)
604 {
605 error_report("QEMU got Hardware memory error at addr %p", host_addr);
606 exit(1);
607 }
608
609 void kvm_arch_on_sigbus_vcpu(CPUState *c, int code, void *addr)
610 {
611 X86CPU *cpu = X86_CPU(c);
612 CPUX86State *env = &cpu->env;
613 ram_addr_t ram_addr;
614 hwaddr paddr;
615
616 /* If we get an action required MCE, it has been injected by KVM
617 * while the VM was running. An action optional MCE instead should
618 * be coming from the main thread, which qemu_init_sigbus identifies
619 * as the "early kill" thread.
620 */
621 assert(code == BUS_MCEERR_AR || code == BUS_MCEERR_AO);
622
623 if ((env->mcg_cap & MCG_SER_P) && addr) {
624 ram_addr = qemu_ram_addr_from_host(addr);
625 if (ram_addr != RAM_ADDR_INVALID &&
626 kvm_physical_memory_addr_from_host(c->kvm_state, addr, &paddr)) {
627 kvm_hwpoison_page_add(ram_addr);
628 kvm_mce_inject(cpu, paddr, code);
629
630 /*
631 * Use different logging severity based on error type.
632 * If there is additional MCE reporting on the hypervisor, QEMU VA
633 * could be another source to identify the PA and MCE details.
634 */
635 if (code == BUS_MCEERR_AR) {
636 error_report("Guest MCE Memory Error at QEMU addr %p and "
637 "GUEST addr 0x%" HWADDR_PRIx " of type %s injected",
638 addr, paddr, "BUS_MCEERR_AR");
639 } else {
640 warn_report("Guest MCE Memory Error at QEMU addr %p and "
641 "GUEST addr 0x%" HWADDR_PRIx " of type %s injected",
642 addr, paddr, "BUS_MCEERR_AO");
643 }
644
645 return;
646 }
647
648 if (code == BUS_MCEERR_AO) {
649 warn_report("Hardware memory error at addr %p of type %s "
650 "for memory used by QEMU itself instead of guest system!",
651 addr, "BUS_MCEERR_AO");
652 }
653 }
654
655 if (code == BUS_MCEERR_AR) {
656 hardware_memory_error(addr);
657 }
658
659 /* Hope we are lucky for AO MCE */
660 }
661
662 static void kvm_reset_exception(CPUX86State *env)
663 {
664 env->exception_nr = -1;
665 env->exception_pending = 0;
666 env->exception_injected = 0;
667 env->exception_has_payload = false;
668 env->exception_payload = 0;
669 }
670
671 static void kvm_queue_exception(CPUX86State *env,
672 int32_t exception_nr,
673 uint8_t exception_has_payload,
674 uint64_t exception_payload)
675 {
676 assert(env->exception_nr == -1);
677 assert(!env->exception_pending);
678 assert(!env->exception_injected);
679 assert(!env->exception_has_payload);
680
681 env->exception_nr = exception_nr;
682
683 if (has_exception_payload) {
684 env->exception_pending = 1;
685
686 env->exception_has_payload = exception_has_payload;
687 env->exception_payload = exception_payload;
688 } else {
689 env->exception_injected = 1;
690
691 if (exception_nr == EXCP01_DB) {
692 assert(exception_has_payload);
693 env->dr[6] = exception_payload;
694 } else if (exception_nr == EXCP0E_PAGE) {
695 assert(exception_has_payload);
696 env->cr[2] = exception_payload;
697 } else {
698 assert(!exception_has_payload);
699 }
700 }
701 }
702
703 static int kvm_inject_mce_oldstyle(X86CPU *cpu)
704 {
705 CPUX86State *env = &cpu->env;
706
707 if (!kvm_has_vcpu_events() && env->exception_nr == EXCP12_MCHK) {
708 unsigned int bank, bank_num = env->mcg_cap & 0xff;
709 struct kvm_x86_mce mce;
710
711 kvm_reset_exception(env);
712
713 /*
714 * There must be at least one bank in use if an MCE is pending.
715 * Find it and use its values for the event injection.
716 */
717 for (bank = 0; bank < bank_num; bank++) {
718 if (env->mce_banks[bank * 4 + 1] & MCI_STATUS_VAL) {
719 break;
720 }
721 }
722 assert(bank < bank_num);
723
724 mce.bank = bank;
725 mce.status = env->mce_banks[bank * 4 + 1];
726 mce.mcg_status = env->mcg_status;
727 mce.addr = env->mce_banks[bank * 4 + 2];
728 mce.misc = env->mce_banks[bank * 4 + 3];
729
730 return kvm_vcpu_ioctl(CPU(cpu), KVM_X86_SET_MCE, &mce);
731 }
732 return 0;
733 }
734
735 static void cpu_update_state(void *opaque, int running, RunState state)
736 {
737 CPUX86State *env = opaque;
738
739 if (running) {
740 env->tsc_valid = false;
741 }
742 }
743
744 unsigned long kvm_arch_vcpu_id(CPUState *cs)
745 {
746 X86CPU *cpu = X86_CPU(cs);
747 return cpu->apic_id;
748 }
749
750 #ifndef KVM_CPUID_SIGNATURE_NEXT
751 #define KVM_CPUID_SIGNATURE_NEXT 0x40000100
752 #endif
753
754 static bool hyperv_enabled(X86CPU *cpu)
755 {
756 CPUState *cs = CPU(cpu);
757 return kvm_check_extension(cs->kvm_state, KVM_CAP_HYPERV) > 0 &&
758 ((cpu->hyperv_spinlock_attempts != HYPERV_SPINLOCK_NEVER_RETRY) ||
759 cpu->hyperv_features || cpu->hyperv_passthrough);
760 }
761
762 static int kvm_arch_set_tsc_khz(CPUState *cs)
763 {
764 X86CPU *cpu = X86_CPU(cs);
765 CPUX86State *env = &cpu->env;
766 int r;
767
768 if (!env->tsc_khz) {
769 return 0;
770 }
771
772 r = kvm_check_extension(cs->kvm_state, KVM_CAP_TSC_CONTROL) ?
773 kvm_vcpu_ioctl(cs, KVM_SET_TSC_KHZ, env->tsc_khz) :
774 -ENOTSUP;
775 if (r < 0) {
776 /* When KVM_SET_TSC_KHZ fails, it's an error only if the current
777 * TSC frequency doesn't match the one we want.
778 */
779 int cur_freq = kvm_check_extension(cs->kvm_state, KVM_CAP_GET_TSC_KHZ) ?
780 kvm_vcpu_ioctl(cs, KVM_GET_TSC_KHZ) :
781 -ENOTSUP;
782 if (cur_freq <= 0 || cur_freq != env->tsc_khz) {
783 warn_report("TSC frequency mismatch between "
784 "VM (%" PRId64 " kHz) and host (%d kHz), "
785 "and TSC scaling unavailable",
786 env->tsc_khz, cur_freq);
787 return r;
788 }
789 }
790
791 return 0;
792 }
793
794 static bool tsc_is_stable_and_known(CPUX86State *env)
795 {
796 if (!env->tsc_khz) {
797 return false;
798 }
799 return (env->features[FEAT_8000_0007_EDX] & CPUID_APM_INVTSC)
800 || env->user_tsc_khz;
801 }
802
803 static struct {
804 const char *desc;
805 struct {
806 uint32_t fw;
807 uint32_t bits;
808 } flags[2];
809 uint64_t dependencies;
810 } kvm_hyperv_properties[] = {
811 [HYPERV_FEAT_RELAXED] = {
812 .desc = "relaxed timing (hv-relaxed)",
813 .flags = {
814 {.fw = FEAT_HYPERV_EAX,
815 .bits = HV_HYPERCALL_AVAILABLE},
816 {.fw = FEAT_HV_RECOMM_EAX,
817 .bits = HV_RELAXED_TIMING_RECOMMENDED}
818 }
819 },
820 [HYPERV_FEAT_VAPIC] = {
821 .desc = "virtual APIC (hv-vapic)",
822 .flags = {
823 {.fw = FEAT_HYPERV_EAX,
824 .bits = HV_HYPERCALL_AVAILABLE | HV_APIC_ACCESS_AVAILABLE},
825 {.fw = FEAT_HV_RECOMM_EAX,
826 .bits = HV_APIC_ACCESS_RECOMMENDED}
827 }
828 },
829 [HYPERV_FEAT_TIME] = {
830 .desc = "clocksources (hv-time)",
831 .flags = {
832 {.fw = FEAT_HYPERV_EAX,
833 .bits = HV_HYPERCALL_AVAILABLE | HV_TIME_REF_COUNT_AVAILABLE |
834 HV_REFERENCE_TSC_AVAILABLE}
835 }
836 },
837 [HYPERV_FEAT_CRASH] = {
838 .desc = "crash MSRs (hv-crash)",
839 .flags = {
840 {.fw = FEAT_HYPERV_EDX,
841 .bits = HV_GUEST_CRASH_MSR_AVAILABLE}
842 }
843 },
844 [HYPERV_FEAT_RESET] = {
845 .desc = "reset MSR (hv-reset)",
846 .flags = {
847 {.fw = FEAT_HYPERV_EAX,
848 .bits = HV_RESET_AVAILABLE}
849 }
850 },
851 [HYPERV_FEAT_VPINDEX] = {
852 .desc = "VP_INDEX MSR (hv-vpindex)",
853 .flags = {
854 {.fw = FEAT_HYPERV_EAX,
855 .bits = HV_VP_INDEX_AVAILABLE}
856 }
857 },
858 [HYPERV_FEAT_RUNTIME] = {
859 .desc = "VP_RUNTIME MSR (hv-runtime)",
860 .flags = {
861 {.fw = FEAT_HYPERV_EAX,
862 .bits = HV_VP_RUNTIME_AVAILABLE}
863 }
864 },
865 [HYPERV_FEAT_SYNIC] = {
866 .desc = "synthetic interrupt controller (hv-synic)",
867 .flags = {
868 {.fw = FEAT_HYPERV_EAX,
869 .bits = HV_SYNIC_AVAILABLE}
870 }
871 },
872 [HYPERV_FEAT_STIMER] = {
873 .desc = "synthetic timers (hv-stimer)",
874 .flags = {
875 {.fw = FEAT_HYPERV_EAX,
876 .bits = HV_SYNTIMERS_AVAILABLE}
877 },
878 .dependencies = BIT(HYPERV_FEAT_SYNIC) | BIT(HYPERV_FEAT_TIME)
879 },
880 [HYPERV_FEAT_FREQUENCIES] = {
881 .desc = "frequency MSRs (hv-frequencies)",
882 .flags = {
883 {.fw = FEAT_HYPERV_EAX,
884 .bits = HV_ACCESS_FREQUENCY_MSRS},
885 {.fw = FEAT_HYPERV_EDX,
886 .bits = HV_FREQUENCY_MSRS_AVAILABLE}
887 }
888 },
889 [HYPERV_FEAT_REENLIGHTENMENT] = {
890 .desc = "reenlightenment MSRs (hv-reenlightenment)",
891 .flags = {
892 {.fw = FEAT_HYPERV_EAX,
893 .bits = HV_ACCESS_REENLIGHTENMENTS_CONTROL}
894 }
895 },
896 [HYPERV_FEAT_TLBFLUSH] = {
897 .desc = "paravirtualized TLB flush (hv-tlbflush)",
898 .flags = {
899 {.fw = FEAT_HV_RECOMM_EAX,
900 .bits = HV_REMOTE_TLB_FLUSH_RECOMMENDED |
901 HV_EX_PROCESSOR_MASKS_RECOMMENDED}
902 },
903 .dependencies = BIT(HYPERV_FEAT_VPINDEX)
904 },
905 [HYPERV_FEAT_EVMCS] = {
906 .desc = "enlightened VMCS (hv-evmcs)",
907 .flags = {
908 {.fw = FEAT_HV_RECOMM_EAX,
909 .bits = HV_ENLIGHTENED_VMCS_RECOMMENDED}
910 },
911 .dependencies = BIT(HYPERV_FEAT_VAPIC)
912 },
913 [HYPERV_FEAT_IPI] = {
914 .desc = "paravirtualized IPI (hv-ipi)",
915 .flags = {
916 {.fw = FEAT_HV_RECOMM_EAX,
917 .bits = HV_CLUSTER_IPI_RECOMMENDED |
918 HV_EX_PROCESSOR_MASKS_RECOMMENDED}
919 },
920 .dependencies = BIT(HYPERV_FEAT_VPINDEX)
921 },
922 [HYPERV_FEAT_STIMER_DIRECT] = {
923 .desc = "direct mode synthetic timers (hv-stimer-direct)",
924 .flags = {
925 {.fw = FEAT_HYPERV_EDX,
926 .bits = HV_STIMER_DIRECT_MODE_AVAILABLE}
927 },
928 .dependencies = BIT(HYPERV_FEAT_STIMER)
929 },
930 };
931
932 static struct kvm_cpuid2 *try_get_hv_cpuid(CPUState *cs, int max)
933 {
934 struct kvm_cpuid2 *cpuid;
935 int r, size;
936
937 size = sizeof(*cpuid) + max * sizeof(*cpuid->entries);
938 cpuid = g_malloc0(size);
939 cpuid->nent = max;
940
941 r = kvm_vcpu_ioctl(cs, KVM_GET_SUPPORTED_HV_CPUID, cpuid);
942 if (r == 0 && cpuid->nent >= max) {
943 r = -E2BIG;
944 }
945 if (r < 0) {
946 if (r == -E2BIG) {
947 g_free(cpuid);
948 return NULL;
949 } else {
950 fprintf(stderr, "KVM_GET_SUPPORTED_HV_CPUID failed: %s\n",
951 strerror(-r));
952 exit(1);
953 }
954 }
955 return cpuid;
956 }
957
958 /*
959 * Run KVM_GET_SUPPORTED_HV_CPUID ioctl(), allocating a buffer large enough
960 * for all entries.
961 */
962 static struct kvm_cpuid2 *get_supported_hv_cpuid(CPUState *cs)
963 {
964 struct kvm_cpuid2 *cpuid;
965 int max = 7; /* 0x40000000..0x40000005, 0x4000000A */
966
967 /*
968 * When the buffer is too small, KVM_GET_SUPPORTED_HV_CPUID fails with
969 * -E2BIG, however, it doesn't report back the right size. Keep increasing
970 * it and re-trying until we succeed.
971 */
972 while ((cpuid = try_get_hv_cpuid(cs, max)) == NULL) {
973 max++;
974 }
975 return cpuid;
976 }
977
978 /*
979 * When KVM_GET_SUPPORTED_HV_CPUID is not supported we fill CPUID feature
980 * leaves from KVM_CAP_HYPERV* and present MSRs data.
981 */
982 static struct kvm_cpuid2 *get_supported_hv_cpuid_legacy(CPUState *cs)
983 {
984 X86CPU *cpu = X86_CPU(cs);
985 struct kvm_cpuid2 *cpuid;
986 struct kvm_cpuid_entry2 *entry_feat, *entry_recomm;
987
988 /* HV_CPUID_FEATURES, HV_CPUID_ENLIGHTMENT_INFO */
989 cpuid = g_malloc0(sizeof(*cpuid) + 2 * sizeof(*cpuid->entries));
990 cpuid->nent = 2;
991
992 /* HV_CPUID_VENDOR_AND_MAX_FUNCTIONS */
993 entry_feat = &cpuid->entries[0];
994 entry_feat->function = HV_CPUID_FEATURES;
995
996 entry_recomm = &cpuid->entries[1];
997 entry_recomm->function = HV_CPUID_ENLIGHTMENT_INFO;
998 entry_recomm->ebx = cpu->hyperv_spinlock_attempts;
999
1000 if (kvm_check_extension(cs->kvm_state, KVM_CAP_HYPERV) > 0) {
1001 entry_feat->eax |= HV_HYPERCALL_AVAILABLE;
1002 entry_feat->eax |= HV_APIC_ACCESS_AVAILABLE;
1003 entry_feat->edx |= HV_CPU_DYNAMIC_PARTITIONING_AVAILABLE;
1004 entry_recomm->eax |= HV_RELAXED_TIMING_RECOMMENDED;
1005 entry_recomm->eax |= HV_APIC_ACCESS_RECOMMENDED;
1006 }
1007
1008 if (kvm_check_extension(cs->kvm_state, KVM_CAP_HYPERV_TIME) > 0) {
1009 entry_feat->eax |= HV_TIME_REF_COUNT_AVAILABLE;
1010 entry_feat->eax |= HV_REFERENCE_TSC_AVAILABLE;
1011 }
1012
1013 if (has_msr_hv_frequencies) {
1014 entry_feat->eax |= HV_ACCESS_FREQUENCY_MSRS;
1015 entry_feat->edx |= HV_FREQUENCY_MSRS_AVAILABLE;
1016 }
1017
1018 if (has_msr_hv_crash) {
1019 entry_feat->edx |= HV_GUEST_CRASH_MSR_AVAILABLE;
1020 }
1021
1022 if (has_msr_hv_reenlightenment) {
1023 entry_feat->eax |= HV_ACCESS_REENLIGHTENMENTS_CONTROL;
1024 }
1025
1026 if (has_msr_hv_reset) {
1027 entry_feat->eax |= HV_RESET_AVAILABLE;
1028 }
1029
1030 if (has_msr_hv_vpindex) {
1031 entry_feat->eax |= HV_VP_INDEX_AVAILABLE;
1032 }
1033
1034 if (has_msr_hv_runtime) {
1035 entry_feat->eax |= HV_VP_RUNTIME_AVAILABLE;
1036 }
1037
1038 if (has_msr_hv_synic) {
1039 unsigned int cap = cpu->hyperv_synic_kvm_only ?
1040 KVM_CAP_HYPERV_SYNIC : KVM_CAP_HYPERV_SYNIC2;
1041
1042 if (kvm_check_extension(cs->kvm_state, cap) > 0) {
1043 entry_feat->eax |= HV_SYNIC_AVAILABLE;
1044 }
1045 }
1046
1047 if (has_msr_hv_stimer) {
1048 entry_feat->eax |= HV_SYNTIMERS_AVAILABLE;
1049 }
1050
1051 if (kvm_check_extension(cs->kvm_state,
1052 KVM_CAP_HYPERV_TLBFLUSH) > 0) {
1053 entry_recomm->eax |= HV_REMOTE_TLB_FLUSH_RECOMMENDED;
1054 entry_recomm->eax |= HV_EX_PROCESSOR_MASKS_RECOMMENDED;
1055 }
1056
1057 if (kvm_check_extension(cs->kvm_state,
1058 KVM_CAP_HYPERV_ENLIGHTENED_VMCS) > 0) {
1059 entry_recomm->eax |= HV_ENLIGHTENED_VMCS_RECOMMENDED;
1060 }
1061
1062 if (kvm_check_extension(cs->kvm_state,
1063 KVM_CAP_HYPERV_SEND_IPI) > 0) {
1064 entry_recomm->eax |= HV_CLUSTER_IPI_RECOMMENDED;
1065 entry_recomm->eax |= HV_EX_PROCESSOR_MASKS_RECOMMENDED;
1066 }
1067
1068 return cpuid;
1069 }
1070
1071 static int hv_cpuid_get_fw(struct kvm_cpuid2 *cpuid, int fw, uint32_t *r)
1072 {
1073 struct kvm_cpuid_entry2 *entry;
1074 uint32_t func;
1075 int reg;
1076
1077 switch (fw) {
1078 case FEAT_HYPERV_EAX:
1079 reg = R_EAX;
1080 func = HV_CPUID_FEATURES;
1081 break;
1082 case FEAT_HYPERV_EDX:
1083 reg = R_EDX;
1084 func = HV_CPUID_FEATURES;
1085 break;
1086 case FEAT_HV_RECOMM_EAX:
1087 reg = R_EAX;
1088 func = HV_CPUID_ENLIGHTMENT_INFO;
1089 break;
1090 default:
1091 return -EINVAL;
1092 }
1093
1094 entry = cpuid_find_entry(cpuid, func, 0);
1095 if (!entry) {
1096 return -ENOENT;
1097 }
1098
1099 switch (reg) {
1100 case R_EAX:
1101 *r = entry->eax;
1102 break;
1103 case R_EDX:
1104 *r = entry->edx;
1105 break;
1106 default:
1107 return -EINVAL;
1108 }
1109
1110 return 0;
1111 }
1112
1113 static int hv_cpuid_check_and_set(CPUState *cs, struct kvm_cpuid2 *cpuid,
1114 int feature)
1115 {
1116 X86CPU *cpu = X86_CPU(cs);
1117 CPUX86State *env = &cpu->env;
1118 uint32_t r, fw, bits;
1119 uint64_t deps;
1120 int i, dep_feat;
1121
1122 if (!hyperv_feat_enabled(cpu, feature) && !cpu->hyperv_passthrough) {
1123 return 0;
1124 }
1125
1126 deps = kvm_hyperv_properties[feature].dependencies;
1127 while (deps) {
1128 dep_feat = ctz64(deps);
1129 if (!(hyperv_feat_enabled(cpu, dep_feat))) {
1130 fprintf(stderr,
1131 "Hyper-V %s requires Hyper-V %s\n",
1132 kvm_hyperv_properties[feature].desc,
1133 kvm_hyperv_properties[dep_feat].desc);
1134 return 1;
1135 }
1136 deps &= ~(1ull << dep_feat);
1137 }
1138
1139 for (i = 0; i < ARRAY_SIZE(kvm_hyperv_properties[feature].flags); i++) {
1140 fw = kvm_hyperv_properties[feature].flags[i].fw;
1141 bits = kvm_hyperv_properties[feature].flags[i].bits;
1142
1143 if (!fw) {
1144 continue;
1145 }
1146
1147 if (hv_cpuid_get_fw(cpuid, fw, &r) || (r & bits) != bits) {
1148 if (hyperv_feat_enabled(cpu, feature)) {
1149 fprintf(stderr,
1150 "Hyper-V %s is not supported by kernel\n",
1151 kvm_hyperv_properties[feature].desc);
1152 return 1;
1153 } else {
1154 return 0;
1155 }
1156 }
1157
1158 env->features[fw] |= bits;
1159 }
1160
1161 if (cpu->hyperv_passthrough) {
1162 cpu->hyperv_features |= BIT(feature);
1163 }
1164
1165 return 0;
1166 }
1167
1168 /*
1169 * Fill in Hyper-V CPUIDs. Returns the number of entries filled in cpuid_ent in
1170 * case of success, errno < 0 in case of failure and 0 when no Hyper-V
1171 * extentions are enabled.
1172 */
1173 static int hyperv_handle_properties(CPUState *cs,
1174 struct kvm_cpuid_entry2 *cpuid_ent)
1175 {
1176 X86CPU *cpu = X86_CPU(cs);
1177 CPUX86State *env = &cpu->env;
1178 struct kvm_cpuid2 *cpuid;
1179 struct kvm_cpuid_entry2 *c;
1180 uint32_t signature[3];
1181 uint32_t cpuid_i = 0;
1182 int r;
1183
1184 if (!hyperv_enabled(cpu))
1185 return 0;
1186
1187 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_EVMCS) ||
1188 cpu->hyperv_passthrough) {
1189 uint16_t evmcs_version;
1190
1191 r = kvm_vcpu_enable_cap(cs, KVM_CAP_HYPERV_ENLIGHTENED_VMCS, 0,
1192 (uintptr_t)&evmcs_version);
1193
1194 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_EVMCS) && r) {
1195 fprintf(stderr, "Hyper-V %s is not supported by kernel\n",
1196 kvm_hyperv_properties[HYPERV_FEAT_EVMCS].desc);
1197 return -ENOSYS;
1198 }
1199
1200 if (!r) {
1201 env->features[FEAT_HV_RECOMM_EAX] |=
1202 HV_ENLIGHTENED_VMCS_RECOMMENDED;
1203 env->features[FEAT_HV_NESTED_EAX] = evmcs_version;
1204 }
1205 }
1206
1207 if (kvm_check_extension(cs->kvm_state, KVM_CAP_HYPERV_CPUID) > 0) {
1208 cpuid = get_supported_hv_cpuid(cs);
1209 } else {
1210 cpuid = get_supported_hv_cpuid_legacy(cs);
1211 }
1212
1213 if (cpu->hyperv_passthrough) {
1214 memcpy(cpuid_ent, &cpuid->entries[0],
1215 cpuid->nent * sizeof(cpuid->entries[0]));
1216
1217 c = cpuid_find_entry(cpuid, HV_CPUID_FEATURES, 0);
1218 if (c) {
1219 env->features[FEAT_HYPERV_EAX] = c->eax;
1220 env->features[FEAT_HYPERV_EBX] = c->ebx;
1221 env->features[FEAT_HYPERV_EDX] = c->eax;
1222 }
1223 c = cpuid_find_entry(cpuid, HV_CPUID_ENLIGHTMENT_INFO, 0);
1224 if (c) {
1225 env->features[FEAT_HV_RECOMM_EAX] = c->eax;
1226
1227 /* hv-spinlocks may have been overriden */
1228 if (cpu->hyperv_spinlock_attempts != HYPERV_SPINLOCK_NEVER_RETRY) {
1229 c->ebx = cpu->hyperv_spinlock_attempts;
1230 }
1231 }
1232 c = cpuid_find_entry(cpuid, HV_CPUID_NESTED_FEATURES, 0);
1233 if (c) {
1234 env->features[FEAT_HV_NESTED_EAX] = c->eax;
1235 }
1236 }
1237
1238 if (cpu->hyperv_no_nonarch_cs == ON_OFF_AUTO_ON) {
1239 env->features[FEAT_HV_RECOMM_EAX] |= HV_NO_NONARCH_CORESHARING;
1240 } else if (cpu->hyperv_no_nonarch_cs == ON_OFF_AUTO_AUTO) {
1241 c = cpuid_find_entry(cpuid, HV_CPUID_ENLIGHTMENT_INFO, 0);
1242 if (c) {
1243 env->features[FEAT_HV_RECOMM_EAX] |=
1244 c->eax & HV_NO_NONARCH_CORESHARING;
1245 }
1246 }
1247
1248 /* Features */
1249 r = hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_RELAXED);
1250 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_VAPIC);
1251 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_TIME);
1252 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_CRASH);
1253 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_RESET);
1254 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_VPINDEX);
1255 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_RUNTIME);
1256 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_SYNIC);
1257 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_STIMER);
1258 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_FREQUENCIES);
1259 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_REENLIGHTENMENT);
1260 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_TLBFLUSH);
1261 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_EVMCS);
1262 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_IPI);
1263 r |= hv_cpuid_check_and_set(cs, cpuid, HYPERV_FEAT_STIMER_DIRECT);
1264
1265 /* Additional dependencies not covered by kvm_hyperv_properties[] */
1266 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_SYNIC) &&
1267 !cpu->hyperv_synic_kvm_only &&
1268 !hyperv_feat_enabled(cpu, HYPERV_FEAT_VPINDEX)) {
1269 fprintf(stderr, "Hyper-V %s requires Hyper-V %s\n",
1270 kvm_hyperv_properties[HYPERV_FEAT_SYNIC].desc,
1271 kvm_hyperv_properties[HYPERV_FEAT_VPINDEX].desc);
1272 r |= 1;
1273 }
1274
1275 /* Not exposed by KVM but needed to make CPU hotplug in Windows work */
1276 env->features[FEAT_HYPERV_EDX] |= HV_CPU_DYNAMIC_PARTITIONING_AVAILABLE;
1277
1278 if (r) {
1279 r = -ENOSYS;
1280 goto free;
1281 }
1282
1283 if (cpu->hyperv_passthrough) {
1284 /* We already copied all feature words from KVM as is */
1285 r = cpuid->nent;
1286 goto free;
1287 }
1288
1289 c = &cpuid_ent[cpuid_i++];
1290 c->function = HV_CPUID_VENDOR_AND_MAX_FUNCTIONS;
1291 if (!cpu->hyperv_vendor_id) {
1292 memcpy(signature, "Microsoft Hv", 12);
1293 } else {
1294 size_t len = strlen(cpu->hyperv_vendor_id);
1295
1296 if (len > 12) {
1297 error_report("hv-vendor-id truncated to 12 characters");
1298 len = 12;
1299 }
1300 memset(signature, 0, 12);
1301 memcpy(signature, cpu->hyperv_vendor_id, len);
1302 }
1303 c->eax = hyperv_feat_enabled(cpu, HYPERV_FEAT_EVMCS) ?
1304 HV_CPUID_NESTED_FEATURES : HV_CPUID_IMPLEMENT_LIMITS;
1305 c->ebx = signature[0];
1306 c->ecx = signature[1];
1307 c->edx = signature[2];
1308
1309 c = &cpuid_ent[cpuid_i++];
1310 c->function = HV_CPUID_INTERFACE;
1311 memcpy(signature, "Hv#1\0\0\0\0\0\0\0\0", 12);
1312 c->eax = signature[0];
1313 c->ebx = 0;
1314 c->ecx = 0;
1315 c->edx = 0;
1316
1317 c = &cpuid_ent[cpuid_i++];
1318 c->function = HV_CPUID_VERSION;
1319 c->eax = 0x00001bbc;
1320 c->ebx = 0x00060001;
1321
1322 c = &cpuid_ent[cpuid_i++];
1323 c->function = HV_CPUID_FEATURES;
1324 c->eax = env->features[FEAT_HYPERV_EAX];
1325 c->ebx = env->features[FEAT_HYPERV_EBX];
1326 c->edx = env->features[FEAT_HYPERV_EDX];
1327
1328 c = &cpuid_ent[cpuid_i++];
1329 c->function = HV_CPUID_ENLIGHTMENT_INFO;
1330 c->eax = env->features[FEAT_HV_RECOMM_EAX];
1331 c->ebx = cpu->hyperv_spinlock_attempts;
1332
1333 c = &cpuid_ent[cpuid_i++];
1334 c->function = HV_CPUID_IMPLEMENT_LIMITS;
1335 c->eax = cpu->hv_max_vps;
1336 c->ebx = 0x40;
1337
1338 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_EVMCS)) {
1339 __u32 function;
1340
1341 /* Create zeroed 0x40000006..0x40000009 leaves */
1342 for (function = HV_CPUID_IMPLEMENT_LIMITS + 1;
1343 function < HV_CPUID_NESTED_FEATURES; function++) {
1344 c = &cpuid_ent[cpuid_i++];
1345 c->function = function;
1346 }
1347
1348 c = &cpuid_ent[cpuid_i++];
1349 c->function = HV_CPUID_NESTED_FEATURES;
1350 c->eax = env->features[FEAT_HV_NESTED_EAX];
1351 }
1352 r = cpuid_i;
1353
1354 free:
1355 g_free(cpuid);
1356
1357 return r;
1358 }
1359
1360 static Error *hv_passthrough_mig_blocker;
1361 static Error *hv_no_nonarch_cs_mig_blocker;
1362
1363 static int hyperv_init_vcpu(X86CPU *cpu)
1364 {
1365 CPUState *cs = CPU(cpu);
1366 Error *local_err = NULL;
1367 int ret;
1368
1369 if (cpu->hyperv_passthrough && hv_passthrough_mig_blocker == NULL) {
1370 error_setg(&hv_passthrough_mig_blocker,
1371 "'hv-passthrough' CPU flag prevents migration, use explicit"
1372 " set of hv-* flags instead");
1373 ret = migrate_add_blocker(hv_passthrough_mig_blocker, &local_err);
1374 if (local_err) {
1375 error_report_err(local_err);
1376 error_free(hv_passthrough_mig_blocker);
1377 return ret;
1378 }
1379 }
1380
1381 if (cpu->hyperv_no_nonarch_cs == ON_OFF_AUTO_AUTO &&
1382 hv_no_nonarch_cs_mig_blocker == NULL) {
1383 error_setg(&hv_no_nonarch_cs_mig_blocker,
1384 "'hv-no-nonarch-coresharing=auto' CPU flag prevents migration"
1385 " use explicit 'hv-no-nonarch-coresharing=on' instead (but"
1386 " make sure SMT is disabled and/or that vCPUs are properly"
1387 " pinned)");
1388 ret = migrate_add_blocker(hv_no_nonarch_cs_mig_blocker, &local_err);
1389 if (local_err) {
1390 error_report_err(local_err);
1391 error_free(hv_no_nonarch_cs_mig_blocker);
1392 return ret;
1393 }
1394 }
1395
1396 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_VPINDEX) && !hv_vpindex_settable) {
1397 /*
1398 * the kernel doesn't support setting vp_index; assert that its value
1399 * is in sync
1400 */
1401 struct {
1402 struct kvm_msrs info;
1403 struct kvm_msr_entry entries[1];
1404 } msr_data = {
1405 .info.nmsrs = 1,
1406 .entries[0].index = HV_X64_MSR_VP_INDEX,
1407 };
1408
1409 ret = kvm_vcpu_ioctl(cs, KVM_GET_MSRS, &msr_data);
1410 if (ret < 0) {
1411 return ret;
1412 }
1413 assert(ret == 1);
1414
1415 if (msr_data.entries[0].data != hyperv_vp_index(CPU(cpu))) {
1416 error_report("kernel's vp_index != QEMU's vp_index");
1417 return -ENXIO;
1418 }
1419 }
1420
1421 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_SYNIC)) {
1422 uint32_t synic_cap = cpu->hyperv_synic_kvm_only ?
1423 KVM_CAP_HYPERV_SYNIC : KVM_CAP_HYPERV_SYNIC2;
1424 ret = kvm_vcpu_enable_cap(cs, synic_cap, 0);
1425 if (ret < 0) {
1426 error_report("failed to turn on HyperV SynIC in KVM: %s",
1427 strerror(-ret));
1428 return ret;
1429 }
1430
1431 if (!cpu->hyperv_synic_kvm_only) {
1432 ret = hyperv_x86_synic_add(cpu);
1433 if (ret < 0) {
1434 error_report("failed to create HyperV SynIC: %s",
1435 strerror(-ret));
1436 return ret;
1437 }
1438 }
1439 }
1440
1441 return 0;
1442 }
1443
1444 static Error *invtsc_mig_blocker;
1445
1446 #define KVM_MAX_CPUID_ENTRIES 100
1447
1448 int kvm_arch_init_vcpu(CPUState *cs)
1449 {
1450 struct {
1451 struct kvm_cpuid2 cpuid;
1452 struct kvm_cpuid_entry2 entries[KVM_MAX_CPUID_ENTRIES];
1453 } cpuid_data;
1454 /*
1455 * The kernel defines these structs with padding fields so there
1456 * should be no extra padding in our cpuid_data struct.
1457 */
1458 QEMU_BUILD_BUG_ON(sizeof(cpuid_data) !=
1459 sizeof(struct kvm_cpuid2) +
1460 sizeof(struct kvm_cpuid_entry2) * KVM_MAX_CPUID_ENTRIES);
1461
1462 X86CPU *cpu = X86_CPU(cs);
1463 CPUX86State *env = &cpu->env;
1464 uint32_t limit, i, j, cpuid_i;
1465 uint32_t unused;
1466 struct kvm_cpuid_entry2 *c;
1467 uint32_t signature[3];
1468 int kvm_base = KVM_CPUID_SIGNATURE;
1469 int max_nested_state_len;
1470 int r;
1471 Error *local_err = NULL;
1472
1473 memset(&cpuid_data, 0, sizeof(cpuid_data));
1474
1475 cpuid_i = 0;
1476
1477 r = kvm_arch_set_tsc_khz(cs);
1478 if (r < 0) {
1479 return r;
1480 }
1481
1482 /* vcpu's TSC frequency is either specified by user, or following
1483 * the value used by KVM if the former is not present. In the
1484 * latter case, we query it from KVM and record in env->tsc_khz,
1485 * so that vcpu's TSC frequency can be migrated later via this field.
1486 */
1487 if (!env->tsc_khz) {
1488 r = kvm_check_extension(cs->kvm_state, KVM_CAP_GET_TSC_KHZ) ?
1489 kvm_vcpu_ioctl(cs, KVM_GET_TSC_KHZ) :
1490 -ENOTSUP;
1491 if (r > 0) {
1492 env->tsc_khz = r;
1493 }
1494 }
1495
1496 /* Paravirtualization CPUIDs */
1497 r = hyperv_handle_properties(cs, cpuid_data.entries);
1498 if (r < 0) {
1499 return r;
1500 } else if (r > 0) {
1501 cpuid_i = r;
1502 kvm_base = KVM_CPUID_SIGNATURE_NEXT;
1503 has_msr_hv_hypercall = true;
1504 }
1505
1506 if (cpu->expose_kvm) {
1507 memcpy(signature, "KVMKVMKVM\0\0\0", 12);
1508 c = &cpuid_data.entries[cpuid_i++];
1509 c->function = KVM_CPUID_SIGNATURE | kvm_base;
1510 c->eax = KVM_CPUID_FEATURES | kvm_base;
1511 c->ebx = signature[0];
1512 c->ecx = signature[1];
1513 c->edx = signature[2];
1514
1515 c = &cpuid_data.entries[cpuid_i++];
1516 c->function = KVM_CPUID_FEATURES | kvm_base;
1517 c->eax = env->features[FEAT_KVM];
1518 c->edx = env->features[FEAT_KVM_HINTS];
1519 }
1520
1521 cpu_x86_cpuid(env, 0, 0, &limit, &unused, &unused, &unused);
1522
1523 for (i = 0; i <= limit; i++) {
1524 if (cpuid_i == KVM_MAX_CPUID_ENTRIES) {
1525 fprintf(stderr, "unsupported level value: 0x%x\n", limit);
1526 abort();
1527 }
1528 c = &cpuid_data.entries[cpuid_i++];
1529
1530 switch (i) {
1531 case 2: {
1532 /* Keep reading function 2 till all the input is received */
1533 int times;
1534
1535 c->function = i;
1536 c->flags = KVM_CPUID_FLAG_STATEFUL_FUNC |
1537 KVM_CPUID_FLAG_STATE_READ_NEXT;
1538 cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
1539 times = c->eax & 0xff;
1540
1541 for (j = 1; j < times; ++j) {
1542 if (cpuid_i == KVM_MAX_CPUID_ENTRIES) {
1543 fprintf(stderr, "cpuid_data is full, no space for "
1544 "cpuid(eax:2):eax & 0xf = 0x%x\n", times);
1545 abort();
1546 }
1547 c = &cpuid_data.entries[cpuid_i++];
1548 c->function = i;
1549 c->flags = KVM_CPUID_FLAG_STATEFUL_FUNC;
1550 cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
1551 }
1552 break;
1553 }
1554 case 0x1f:
1555 if (env->nr_dies < 2) {
1556 break;
1557 }
1558 case 4:
1559 case 0xb:
1560 case 0xd:
1561 for (j = 0; ; j++) {
1562 if (i == 0xd && j == 64) {
1563 break;
1564 }
1565
1566 if (i == 0x1f && j == 64) {
1567 break;
1568 }
1569
1570 c->function = i;
1571 c->flags = KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
1572 c->index = j;
1573 cpu_x86_cpuid(env, i, j, &c->eax, &c->ebx, &c->ecx, &c->edx);
1574
1575 if (i == 4 && c->eax == 0) {
1576 break;
1577 }
1578 if (i == 0xb && !(c->ecx & 0xff00)) {
1579 break;
1580 }
1581 if (i == 0x1f && !(c->ecx & 0xff00)) {
1582 break;
1583 }
1584 if (i == 0xd && c->eax == 0) {
1585 continue;
1586 }
1587 if (cpuid_i == KVM_MAX_CPUID_ENTRIES) {
1588 fprintf(stderr, "cpuid_data is full, no space for "
1589 "cpuid(eax:0x%x,ecx:0x%x)\n", i, j);
1590 abort();
1591 }
1592 c = &cpuid_data.entries[cpuid_i++];
1593 }
1594 break;
1595 case 0x7:
1596 case 0x14: {
1597 uint32_t times;
1598
1599 c->function = i;
1600 c->index = 0;
1601 c->flags = KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
1602 cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
1603 times = c->eax;
1604
1605 for (j = 1; j <= times; ++j) {
1606 if (cpuid_i == KVM_MAX_CPUID_ENTRIES) {
1607 fprintf(stderr, "cpuid_data is full, no space for "
1608 "cpuid(eax:0x%x,ecx:0x%x)\n", i, j);
1609 abort();
1610 }
1611 c = &cpuid_data.entries[cpuid_i++];
1612 c->function = i;
1613 c->index = j;
1614 c->flags = KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
1615 cpu_x86_cpuid(env, i, j, &c->eax, &c->ebx, &c->ecx, &c->edx);
1616 }
1617 break;
1618 }
1619 default:
1620 c->function = i;
1621 c->flags = 0;
1622 cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
1623 if (!c->eax && !c->ebx && !c->ecx && !c->edx) {
1624 /*
1625 * KVM already returns all zeroes if a CPUID entry is missing,
1626 * so we can omit it and avoid hitting KVM's 80-entry limit.
1627 */
1628 cpuid_i--;
1629 }
1630 break;
1631 }
1632 }
1633
1634 if (limit >= 0x0a) {
1635 uint32_t eax, edx;
1636
1637 cpu_x86_cpuid(env, 0x0a, 0, &eax, &unused, &unused, &edx);
1638
1639 has_architectural_pmu_version = eax & 0xff;
1640 if (has_architectural_pmu_version > 0) {
1641 num_architectural_pmu_gp_counters = (eax & 0xff00) >> 8;
1642
1643 /* Shouldn't be more than 32, since that's the number of bits
1644 * available in EBX to tell us _which_ counters are available.
1645 * Play it safe.
1646 */
1647 if (num_architectural_pmu_gp_counters > MAX_GP_COUNTERS) {
1648 num_architectural_pmu_gp_counters = MAX_GP_COUNTERS;
1649 }
1650
1651 if (has_architectural_pmu_version > 1) {
1652 num_architectural_pmu_fixed_counters = edx & 0x1f;
1653
1654 if (num_architectural_pmu_fixed_counters > MAX_FIXED_COUNTERS) {
1655 num_architectural_pmu_fixed_counters = MAX_FIXED_COUNTERS;
1656 }
1657 }
1658 }
1659 }
1660
1661 cpu_x86_cpuid(env, 0x80000000, 0, &limit, &unused, &unused, &unused);
1662
1663 for (i = 0x80000000; i <= limit; i++) {
1664 if (cpuid_i == KVM_MAX_CPUID_ENTRIES) {
1665 fprintf(stderr, "unsupported xlevel value: 0x%x\n", limit);
1666 abort();
1667 }
1668 c = &cpuid_data.entries[cpuid_i++];
1669
1670 switch (i) {
1671 case 0x8000001d:
1672 /* Query for all AMD cache information leaves */
1673 for (j = 0; ; j++) {
1674 c->function = i;
1675 c->flags = KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
1676 c->index = j;
1677 cpu_x86_cpuid(env, i, j, &c->eax, &c->ebx, &c->ecx, &c->edx);
1678
1679 if (c->eax == 0) {
1680 break;
1681 }
1682 if (cpuid_i == KVM_MAX_CPUID_ENTRIES) {
1683 fprintf(stderr, "cpuid_data is full, no space for "
1684 "cpuid(eax:0x%x,ecx:0x%x)\n", i, j);
1685 abort();
1686 }
1687 c = &cpuid_data.entries[cpuid_i++];
1688 }
1689 break;
1690 default:
1691 c->function = i;
1692 c->flags = 0;
1693 cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
1694 if (!c->eax && !c->ebx && !c->ecx && !c->edx) {
1695 /*
1696 * KVM already returns all zeroes if a CPUID entry is missing,
1697 * so we can omit it and avoid hitting KVM's 80-entry limit.
1698 */
1699 cpuid_i--;
1700 }
1701 break;
1702 }
1703 }
1704
1705 /* Call Centaur's CPUID instructions they are supported. */
1706 if (env->cpuid_xlevel2 > 0) {
1707 cpu_x86_cpuid(env, 0xC0000000, 0, &limit, &unused, &unused, &unused);
1708
1709 for (i = 0xC0000000; i <= limit; i++) {
1710 if (cpuid_i == KVM_MAX_CPUID_ENTRIES) {
1711 fprintf(stderr, "unsupported xlevel2 value: 0x%x\n", limit);
1712 abort();
1713 }
1714 c = &cpuid_data.entries[cpuid_i++];
1715
1716 c->function = i;
1717 c->flags = 0;
1718 cpu_x86_cpuid(env, i, 0, &c->eax, &c->ebx, &c->ecx, &c->edx);
1719 }
1720 }
1721
1722 cpuid_data.cpuid.nent = cpuid_i;
1723
1724 if (((env->cpuid_version >> 8)&0xF) >= 6
1725 && (env->features[FEAT_1_EDX] & (CPUID_MCE | CPUID_MCA)) ==
1726 (CPUID_MCE | CPUID_MCA)
1727 && kvm_check_extension(cs->kvm_state, KVM_CAP_MCE) > 0) {
1728 uint64_t mcg_cap, unsupported_caps;
1729 int banks;
1730 int ret;
1731
1732 ret = kvm_get_mce_cap_supported(cs->kvm_state, &mcg_cap, &banks);
1733 if (ret < 0) {
1734 fprintf(stderr, "kvm_get_mce_cap_supported: %s", strerror(-ret));
1735 return ret;
1736 }
1737
1738 if (banks < (env->mcg_cap & MCG_CAP_BANKS_MASK)) {
1739 error_report("kvm: Unsupported MCE bank count (QEMU = %d, KVM = %d)",
1740 (int)(env->mcg_cap & MCG_CAP_BANKS_MASK), banks);
1741 return -ENOTSUP;
1742 }
1743
1744 unsupported_caps = env->mcg_cap & ~(mcg_cap | MCG_CAP_BANKS_MASK);
1745 if (unsupported_caps) {
1746 if (unsupported_caps & MCG_LMCE_P) {
1747 error_report("kvm: LMCE not supported");
1748 return -ENOTSUP;
1749 }
1750 warn_report("Unsupported MCG_CAP bits: 0x%" PRIx64,
1751 unsupported_caps);
1752 }
1753
1754 env->mcg_cap &= mcg_cap | MCG_CAP_BANKS_MASK;
1755 ret = kvm_vcpu_ioctl(cs, KVM_X86_SETUP_MCE, &env->mcg_cap);
1756 if (ret < 0) {
1757 fprintf(stderr, "KVM_X86_SETUP_MCE: %s", strerror(-ret));
1758 return ret;
1759 }
1760 }
1761
1762 qemu_add_vm_change_state_handler(cpu_update_state, env);
1763
1764 c = cpuid_find_entry(&cpuid_data.cpuid, 1, 0);
1765 if (c) {
1766 has_msr_feature_control = !!(c->ecx & CPUID_EXT_VMX) ||
1767 !!(c->ecx & CPUID_EXT_SMX);
1768 }
1769
1770 if (env->mcg_cap & MCG_LMCE_P) {
1771 has_msr_mcg_ext_ctl = has_msr_feature_control = true;
1772 }
1773
1774 if (!env->user_tsc_khz) {
1775 if ((env->features[FEAT_8000_0007_EDX] & CPUID_APM_INVTSC) &&
1776 invtsc_mig_blocker == NULL) {
1777 error_setg(&invtsc_mig_blocker,
1778 "State blocked by non-migratable CPU device"
1779 " (invtsc flag)");
1780 r = migrate_add_blocker(invtsc_mig_blocker, &local_err);
1781 if (local_err) {
1782 error_report_err(local_err);
1783 error_free(invtsc_mig_blocker);
1784 return r;
1785 }
1786 }
1787 }
1788
1789 if (cpu->vmware_cpuid_freq
1790 /* Guests depend on 0x40000000 to detect this feature, so only expose
1791 * it if KVM exposes leaf 0x40000000. (Conflicts with Hyper-V) */
1792 && cpu->expose_kvm
1793 && kvm_base == KVM_CPUID_SIGNATURE
1794 /* TSC clock must be stable and known for this feature. */
1795 && tsc_is_stable_and_known(env)) {
1796
1797 c = &cpuid_data.entries[cpuid_i++];
1798 c->function = KVM_CPUID_SIGNATURE | 0x10;
1799 c->eax = env->tsc_khz;
1800 /* LAPIC resolution of 1ns (freq: 1GHz) is hardcoded in KVM's
1801 * APIC_BUS_CYCLE_NS */
1802 c->ebx = 1000000;
1803 c->ecx = c->edx = 0;
1804
1805 c = cpuid_find_entry(&cpuid_data.cpuid, kvm_base, 0);
1806 c->eax = MAX(c->eax, KVM_CPUID_SIGNATURE | 0x10);
1807 }
1808
1809 cpuid_data.cpuid.nent = cpuid_i;
1810
1811 cpuid_data.cpuid.padding = 0;
1812 r = kvm_vcpu_ioctl(cs, KVM_SET_CPUID2, &cpuid_data);
1813 if (r) {
1814 goto fail;
1815 }
1816
1817 if (has_xsave) {
1818 env->xsave_buf = qemu_memalign(4096, sizeof(struct kvm_xsave));
1819 memset(env->xsave_buf, 0, sizeof(struct kvm_xsave));
1820 }
1821
1822 max_nested_state_len = kvm_max_nested_state_length();
1823 if (max_nested_state_len > 0) {
1824 assert(max_nested_state_len >= offsetof(struct kvm_nested_state, data));
1825
1826 if (cpu_has_vmx(env)) {
1827 struct kvm_vmx_nested_state_hdr *vmx_hdr;
1828
1829 env->nested_state = g_malloc0(max_nested_state_len);
1830 env->nested_state->size = max_nested_state_len;
1831 env->nested_state->format = KVM_STATE_NESTED_FORMAT_VMX;
1832
1833 vmx_hdr = &env->nested_state->hdr.vmx;
1834 vmx_hdr->vmxon_pa = -1ull;
1835 vmx_hdr->vmcs12_pa = -1ull;
1836 }
1837 }
1838
1839 cpu->kvm_msr_buf = g_malloc0(MSR_BUF_SIZE);
1840
1841 if (!(env->features[FEAT_8000_0001_EDX] & CPUID_EXT2_RDTSCP)) {
1842 has_msr_tsc_aux = false;
1843 }
1844
1845 r = hyperv_init_vcpu(cpu);
1846 if (r) {
1847 goto fail;
1848 }
1849
1850 return 0;
1851
1852 fail:
1853 migrate_del_blocker(invtsc_mig_blocker);
1854
1855 return r;
1856 }
1857
1858 int kvm_arch_destroy_vcpu(CPUState *cs)
1859 {
1860 X86CPU *cpu = X86_CPU(cs);
1861 CPUX86State *env = &cpu->env;
1862
1863 if (cpu->kvm_msr_buf) {
1864 g_free(cpu->kvm_msr_buf);
1865 cpu->kvm_msr_buf = NULL;
1866 }
1867
1868 if (env->nested_state) {
1869 g_free(env->nested_state);
1870 env->nested_state = NULL;
1871 }
1872
1873 return 0;
1874 }
1875
1876 void kvm_arch_reset_vcpu(X86CPU *cpu)
1877 {
1878 CPUX86State *env = &cpu->env;
1879
1880 env->xcr0 = 1;
1881 if (kvm_irqchip_in_kernel()) {
1882 env->mp_state = cpu_is_bsp(cpu) ? KVM_MP_STATE_RUNNABLE :
1883 KVM_MP_STATE_UNINITIALIZED;
1884 } else {
1885 env->mp_state = KVM_MP_STATE_RUNNABLE;
1886 }
1887
1888 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_SYNIC)) {
1889 int i;
1890 for (i = 0; i < ARRAY_SIZE(env->msr_hv_synic_sint); i++) {
1891 env->msr_hv_synic_sint[i] = HV_SINT_MASKED;
1892 }
1893
1894 hyperv_x86_synic_reset(cpu);
1895 }
1896 /* enabled by default */
1897 env->poll_control_msr = 1;
1898 }
1899
1900 void kvm_arch_do_init_vcpu(X86CPU *cpu)
1901 {
1902 CPUX86State *env = &cpu->env;
1903
1904 /* APs get directly into wait-for-SIPI state. */
1905 if (env->mp_state == KVM_MP_STATE_UNINITIALIZED) {
1906 env->mp_state = KVM_MP_STATE_INIT_RECEIVED;
1907 }
1908 }
1909
1910 static int kvm_get_supported_feature_msrs(KVMState *s)
1911 {
1912 int ret = 0;
1913
1914 if (kvm_feature_msrs != NULL) {
1915 return 0;
1916 }
1917
1918 if (!kvm_check_extension(s, KVM_CAP_GET_MSR_FEATURES)) {
1919 return 0;
1920 }
1921
1922 struct kvm_msr_list msr_list;
1923
1924 msr_list.nmsrs = 0;
1925 ret = kvm_ioctl(s, KVM_GET_MSR_FEATURE_INDEX_LIST, &msr_list);
1926 if (ret < 0 && ret != -E2BIG) {
1927 error_report("Fetch KVM feature MSR list failed: %s",
1928 strerror(-ret));
1929 return ret;
1930 }
1931
1932 assert(msr_list.nmsrs > 0);
1933 kvm_feature_msrs = (struct kvm_msr_list *) \
1934 g_malloc0(sizeof(msr_list) +
1935 msr_list.nmsrs * sizeof(msr_list.indices[0]));
1936
1937 kvm_feature_msrs->nmsrs = msr_list.nmsrs;
1938 ret = kvm_ioctl(s, KVM_GET_MSR_FEATURE_INDEX_LIST, kvm_feature_msrs);
1939
1940 if (ret < 0) {
1941 error_report("Fetch KVM feature MSR list failed: %s",
1942 strerror(-ret));
1943 g_free(kvm_feature_msrs);
1944 kvm_feature_msrs = NULL;
1945 return ret;
1946 }
1947
1948 return 0;
1949 }
1950
1951 static int kvm_get_supported_msrs(KVMState *s)
1952 {
1953 int ret = 0;
1954 struct kvm_msr_list msr_list, *kvm_msr_list;
1955
1956 /*
1957 * Obtain MSR list from KVM. These are the MSRs that we must
1958 * save/restore.
1959 */
1960 msr_list.nmsrs = 0;
1961 ret = kvm_ioctl(s, KVM_GET_MSR_INDEX_LIST, &msr_list);
1962 if (ret < 0 && ret != -E2BIG) {
1963 return ret;
1964 }
1965 /*
1966 * Old kernel modules had a bug and could write beyond the provided
1967 * memory. Allocate at least a safe amount of 1K.
1968 */
1969 kvm_msr_list = g_malloc0(MAX(1024, sizeof(msr_list) +
1970 msr_list.nmsrs *
1971 sizeof(msr_list.indices[0])));
1972
1973 kvm_msr_list->nmsrs = msr_list.nmsrs;
1974 ret = kvm_ioctl(s, KVM_GET_MSR_INDEX_LIST, kvm_msr_list);
1975 if (ret >= 0) {
1976 int i;
1977
1978 for (i = 0; i < kvm_msr_list->nmsrs; i++) {
1979 switch (kvm_msr_list->indices[i]) {
1980 case MSR_STAR:
1981 has_msr_star = true;
1982 break;
1983 case MSR_VM_HSAVE_PA:
1984 has_msr_hsave_pa = true;
1985 break;
1986 case MSR_TSC_AUX:
1987 has_msr_tsc_aux = true;
1988 break;
1989 case MSR_TSC_ADJUST:
1990 has_msr_tsc_adjust = true;
1991 break;
1992 case MSR_IA32_TSCDEADLINE:
1993 has_msr_tsc_deadline = true;
1994 break;
1995 case MSR_IA32_SMBASE:
1996 has_msr_smbase = true;
1997 break;
1998 case MSR_SMI_COUNT:
1999 has_msr_smi_count = true;
2000 break;
2001 case MSR_IA32_MISC_ENABLE:
2002 has_msr_misc_enable = true;
2003 break;
2004 case MSR_IA32_BNDCFGS:
2005 has_msr_bndcfgs = true;
2006 break;
2007 case MSR_IA32_XSS:
2008 has_msr_xss = true;
2009 break;
2010 case MSR_IA32_UMWAIT_CONTROL:
2011 has_msr_umwait = true;
2012 break;
2013 case HV_X64_MSR_CRASH_CTL:
2014 has_msr_hv_crash = true;
2015 break;
2016 case HV_X64_MSR_RESET:
2017 has_msr_hv_reset = true;
2018 break;
2019 case HV_X64_MSR_VP_INDEX:
2020 has_msr_hv_vpindex = true;
2021 break;
2022 case HV_X64_MSR_VP_RUNTIME:
2023 has_msr_hv_runtime = true;
2024 break;
2025 case HV_X64_MSR_SCONTROL:
2026 has_msr_hv_synic = true;
2027 break;
2028 case HV_X64_MSR_STIMER0_CONFIG:
2029 has_msr_hv_stimer = true;
2030 break;
2031 case HV_X64_MSR_TSC_FREQUENCY:
2032 has_msr_hv_frequencies = true;
2033 break;
2034 case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
2035 has_msr_hv_reenlightenment = true;
2036 break;
2037 case MSR_IA32_SPEC_CTRL:
2038 has_msr_spec_ctrl = true;
2039 break;
2040 case MSR_IA32_TSX_CTRL:
2041 has_msr_tsx_ctrl = true;
2042 break;
2043 case MSR_VIRT_SSBD:
2044 has_msr_virt_ssbd = true;
2045 break;
2046 case MSR_IA32_ARCH_CAPABILITIES:
2047 has_msr_arch_capabs = true;
2048 break;
2049 case MSR_IA32_CORE_CAPABILITY:
2050 has_msr_core_capabs = true;
2051 break;
2052 case MSR_IA32_VMX_VMFUNC:
2053 has_msr_vmx_vmfunc = true;
2054 break;
2055 }
2056 }
2057 }
2058
2059 g_free(kvm_msr_list);
2060
2061 return ret;
2062 }
2063
2064 static Notifier smram_machine_done;
2065 static KVMMemoryListener smram_listener;
2066 static AddressSpace smram_address_space;
2067 static MemoryRegion smram_as_root;
2068 static MemoryRegion smram_as_mem;
2069
2070 static void register_smram_listener(Notifier *n, void *unused)
2071 {
2072 MemoryRegion *smram =
2073 (MemoryRegion *) object_resolve_path("/machine/smram", NULL);
2074
2075 /* Outer container... */
2076 memory_region_init(&smram_as_root, OBJECT(kvm_state), "mem-container-smram", ~0ull);
2077 memory_region_set_enabled(&smram_as_root, true);
2078
2079 /* ... with two regions inside: normal system memory with low
2080 * priority, and...
2081 */
2082 memory_region_init_alias(&smram_as_mem, OBJECT(kvm_state), "mem-smram",
2083 get_system_memory(), 0, ~0ull);
2084 memory_region_add_subregion_overlap(&smram_as_root, 0, &smram_as_mem, 0);
2085 memory_region_set_enabled(&smram_as_mem, true);
2086
2087 if (smram) {
2088 /* ... SMRAM with higher priority */
2089 memory_region_add_subregion_overlap(&smram_as_root, 0, smram, 10);
2090 memory_region_set_enabled(smram, true);
2091 }
2092
2093 address_space_init(&smram_address_space, &smram_as_root, "KVM-SMRAM");
2094 kvm_memory_listener_register(kvm_state, &smram_listener,
2095 &smram_address_space, 1);
2096 }
2097
2098 int kvm_arch_init(MachineState *ms, KVMState *s)
2099 {
2100 uint64_t identity_base = 0xfffbc000;
2101 uint64_t shadow_mem;
2102 int ret;
2103 struct utsname utsname;
2104
2105 has_xsave = kvm_check_extension(s, KVM_CAP_XSAVE);
2106 has_xcrs = kvm_check_extension(s, KVM_CAP_XCRS);
2107 has_pit_state2 = kvm_check_extension(s, KVM_CAP_PIT_STATE2);
2108
2109 hv_vpindex_settable = kvm_check_extension(s, KVM_CAP_HYPERV_VP_INDEX);
2110
2111 has_exception_payload = kvm_check_extension(s, KVM_CAP_EXCEPTION_PAYLOAD);
2112 if (has_exception_payload) {
2113 ret = kvm_vm_enable_cap(s, KVM_CAP_EXCEPTION_PAYLOAD, 0, true);
2114 if (ret < 0) {
2115 error_report("kvm: Failed to enable exception payload cap: %s",
2116 strerror(-ret));
2117 return ret;
2118 }
2119 }
2120
2121 ret = kvm_get_supported_msrs(s);
2122 if (ret < 0) {
2123 return ret;
2124 }
2125
2126 kvm_get_supported_feature_msrs(s);
2127
2128 uname(&utsname);
2129 lm_capable_kernel = strcmp(utsname.machine, "x86_64") == 0;
2130
2131 /*
2132 * On older Intel CPUs, KVM uses vm86 mode to emulate 16-bit code directly.
2133 * In order to use vm86 mode, an EPT identity map and a TSS are needed.
2134 * Since these must be part of guest physical memory, we need to allocate
2135 * them, both by setting their start addresses in the kernel and by
2136 * creating a corresponding e820 entry. We need 4 pages before the BIOS.
2137 *
2138 * Older KVM versions may not support setting the identity map base. In
2139 * that case we need to stick with the default, i.e. a 256K maximum BIOS
2140 * size.
2141 */
2142 if (kvm_check_extension(s, KVM_CAP_SET_IDENTITY_MAP_ADDR)) {
2143 /* Allows up to 16M BIOSes. */
2144 identity_base = 0xfeffc000;
2145
2146 ret = kvm_vm_ioctl(s, KVM_SET_IDENTITY_MAP_ADDR, &identity_base);
2147 if (ret < 0) {
2148 return ret;
2149 }
2150 }
2151
2152 /* Set TSS base one page after EPT identity map. */
2153 ret = kvm_vm_ioctl(s, KVM_SET_TSS_ADDR, identity_base + 0x1000);
2154 if (ret < 0) {
2155 return ret;
2156 }
2157
2158 /* Tell fw_cfg to notify the BIOS to reserve the range. */
2159 ret = e820_add_entry(identity_base, 0x4000, E820_RESERVED);
2160 if (ret < 0) {
2161 fprintf(stderr, "e820_add_entry() table is full\n");
2162 return ret;
2163 }
2164 qemu_register_reset(kvm_unpoison_all, NULL);
2165
2166 shadow_mem = machine_kvm_shadow_mem(ms);
2167 if (shadow_mem != -1) {
2168 shadow_mem /= 4096;
2169 ret = kvm_vm_ioctl(s, KVM_SET_NR_MMU_PAGES, shadow_mem);
2170 if (ret < 0) {
2171 return ret;
2172 }
2173 }
2174
2175 if (kvm_check_extension(s, KVM_CAP_X86_SMM) &&
2176 object_dynamic_cast(OBJECT(ms), TYPE_PC_MACHINE) &&
2177 pc_machine_is_smm_enabled(PC_MACHINE(ms))) {
2178 smram_machine_done.notify = register_smram_listener;
2179 qemu_add_machine_init_done_notifier(&smram_machine_done);
2180 }
2181
2182 if (enable_cpu_pm) {
2183 int disable_exits = kvm_check_extension(s, KVM_CAP_X86_DISABLE_EXITS);
2184 int ret;
2185
2186 /* Work around for kernel header with a typo. TODO: fix header and drop. */
2187 #if defined(KVM_X86_DISABLE_EXITS_HTL) && !defined(KVM_X86_DISABLE_EXITS_HLT)
2188 #define KVM_X86_DISABLE_EXITS_HLT KVM_X86_DISABLE_EXITS_HTL
2189 #endif
2190 if (disable_exits) {
2191 disable_exits &= (KVM_X86_DISABLE_EXITS_MWAIT |
2192 KVM_X86_DISABLE_EXITS_HLT |
2193 KVM_X86_DISABLE_EXITS_PAUSE |
2194 KVM_X86_DISABLE_EXITS_CSTATE);
2195 }
2196
2197 ret = kvm_vm_enable_cap(s, KVM_CAP_X86_DISABLE_EXITS, 0,
2198 disable_exits);
2199 if (ret < 0) {
2200 error_report("kvm: guest stopping CPU not supported: %s",
2201 strerror(-ret));
2202 }
2203 }
2204
2205 return 0;
2206 }
2207
2208 static void set_v8086_seg(struct kvm_segment *lhs, const SegmentCache *rhs)
2209 {
2210 lhs->selector = rhs->selector;
2211 lhs->base = rhs->base;
2212 lhs->limit = rhs->limit;
2213 lhs->type = 3;
2214 lhs->present = 1;
2215 lhs->dpl = 3;
2216 lhs->db = 0;
2217 lhs->s = 1;
2218 lhs->l = 0;
2219 lhs->g = 0;
2220 lhs->avl = 0;
2221 lhs->unusable = 0;
2222 }
2223
2224 static void set_seg(struct kvm_segment *lhs, const SegmentCache *rhs)
2225 {
2226 unsigned flags = rhs->flags;
2227 lhs->selector = rhs->selector;
2228 lhs->base = rhs->base;
2229 lhs->limit = rhs->limit;
2230 lhs->type = (flags >> DESC_TYPE_SHIFT) & 15;
2231 lhs->present = (flags & DESC_P_MASK) != 0;
2232 lhs->dpl = (flags >> DESC_DPL_SHIFT) & 3;
2233 lhs->db = (flags >> DESC_B_SHIFT) & 1;
2234 lhs->s = (flags & DESC_S_MASK) != 0;
2235 lhs->l = (flags >> DESC_L_SHIFT) & 1;
2236 lhs->g = (flags & DESC_G_MASK) != 0;
2237 lhs->avl = (flags & DESC_AVL_MASK) != 0;
2238 lhs->unusable = !lhs->present;
2239 lhs->padding = 0;
2240 }
2241
2242 static void get_seg(SegmentCache *lhs, const struct kvm_segment *rhs)
2243 {
2244 lhs->selector = rhs->selector;
2245 lhs->base = rhs->base;
2246 lhs->limit = rhs->limit;
2247 lhs->flags = (rhs->type << DESC_TYPE_SHIFT) |
2248 ((rhs->present && !rhs->unusable) * DESC_P_MASK) |
2249 (rhs->dpl << DESC_DPL_SHIFT) |
2250 (rhs->db << DESC_B_SHIFT) |
2251 (rhs->s * DESC_S_MASK) |
2252 (rhs->l << DESC_L_SHIFT) |
2253 (rhs->g * DESC_G_MASK) |
2254 (rhs->avl * DESC_AVL_MASK);
2255 }
2256
2257 static void kvm_getput_reg(__u64 *kvm_reg, target_ulong *qemu_reg, int set)
2258 {
2259 if (set) {
2260 *kvm_reg = *qemu_reg;
2261 } else {
2262 *qemu_reg = *kvm_reg;
2263 }
2264 }
2265
2266 static int kvm_getput_regs(X86CPU *cpu, int set)
2267 {
2268 CPUX86State *env = &cpu->env;
2269 struct kvm_regs regs;
2270 int ret = 0;
2271
2272 if (!set) {
2273 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_REGS, &regs);
2274 if (ret < 0) {
2275 return ret;
2276 }
2277 }
2278
2279 kvm_getput_reg(&regs.rax, &env->regs[R_EAX], set);
2280 kvm_getput_reg(&regs.rbx, &env->regs[R_EBX], set);
2281 kvm_getput_reg(&regs.rcx, &env->regs[R_ECX], set);
2282 kvm_getput_reg(&regs.rdx, &env->regs[R_EDX], set);
2283 kvm_getput_reg(&regs.rsi, &env->regs[R_ESI], set);
2284 kvm_getput_reg(&regs.rdi, &env->regs[R_EDI], set);
2285 kvm_getput_reg(&regs.rsp, &env->regs[R_ESP], set);
2286 kvm_getput_reg(&regs.rbp, &env->regs[R_EBP], set);
2287 #ifdef TARGET_X86_64
2288 kvm_getput_reg(&regs.r8, &env->regs[8], set);
2289 kvm_getput_reg(&regs.r9, &env->regs[9], set);
2290 kvm_getput_reg(&regs.r10, &env->regs[10], set);
2291 kvm_getput_reg(&regs.r11, &env->regs[11], set);
2292 kvm_getput_reg(&regs.r12, &env->regs[12], set);
2293 kvm_getput_reg(&regs.r13, &env->regs[13], set);
2294 kvm_getput_reg(&regs.r14, &env->regs[14], set);
2295 kvm_getput_reg(&regs.r15, &env->regs[15], set);
2296 #endif
2297
2298 kvm_getput_reg(&regs.rflags, &env->eflags, set);
2299 kvm_getput_reg(&regs.rip, &env->eip, set);
2300
2301 if (set) {
2302 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_REGS, &regs);
2303 }
2304
2305 return ret;
2306 }
2307
2308 static int kvm_put_fpu(X86CPU *cpu)
2309 {
2310 CPUX86State *env = &cpu->env;
2311 struct kvm_fpu fpu;
2312 int i;
2313
2314 memset(&fpu, 0, sizeof fpu);
2315 fpu.fsw = env->fpus & ~(7 << 11);
2316 fpu.fsw |= (env->fpstt & 7) << 11;
2317 fpu.fcw = env->fpuc;
2318 fpu.last_opcode = env->fpop;
2319 fpu.last_ip = env->fpip;
2320 fpu.last_dp = env->fpdp;
2321 for (i = 0; i < 8; ++i) {
2322 fpu.ftwx |= (!env->fptags[i]) << i;
2323 }
2324 memcpy(fpu.fpr, env->fpregs, sizeof env->fpregs);
2325 for (i = 0; i < CPU_NB_REGS; i++) {
2326 stq_p(&fpu.xmm[i][0], env->xmm_regs[i].ZMM_Q(0));
2327 stq_p(&fpu.xmm[i][8], env->xmm_regs[i].ZMM_Q(1));
2328 }
2329 fpu.mxcsr = env->mxcsr;
2330
2331 return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_FPU, &fpu);
2332 }
2333
2334 #define XSAVE_FCW_FSW 0
2335 #define XSAVE_FTW_FOP 1
2336 #define XSAVE_CWD_RIP 2
2337 #define XSAVE_CWD_RDP 4
2338 #define XSAVE_MXCSR 6
2339 #define XSAVE_ST_SPACE 8
2340 #define XSAVE_XMM_SPACE 40
2341 #define XSAVE_XSTATE_BV 128
2342 #define XSAVE_YMMH_SPACE 144
2343 #define XSAVE_BNDREGS 240
2344 #define XSAVE_BNDCSR 256
2345 #define XSAVE_OPMASK 272
2346 #define XSAVE_ZMM_Hi256 288
2347 #define XSAVE_Hi16_ZMM 416
2348 #define XSAVE_PKRU 672
2349
2350 #define XSAVE_BYTE_OFFSET(word_offset) \
2351 ((word_offset) * sizeof_field(struct kvm_xsave, region[0]))
2352
2353 #define ASSERT_OFFSET(word_offset, field) \
2354 QEMU_BUILD_BUG_ON(XSAVE_BYTE_OFFSET(word_offset) != \
2355 offsetof(X86XSaveArea, field))
2356
2357 ASSERT_OFFSET(XSAVE_FCW_FSW, legacy.fcw);
2358 ASSERT_OFFSET(XSAVE_FTW_FOP, legacy.ftw);
2359 ASSERT_OFFSET(XSAVE_CWD_RIP, legacy.fpip);
2360 ASSERT_OFFSET(XSAVE_CWD_RDP, legacy.fpdp);
2361 ASSERT_OFFSET(XSAVE_MXCSR, legacy.mxcsr);
2362 ASSERT_OFFSET(XSAVE_ST_SPACE, legacy.fpregs);
2363 ASSERT_OFFSET(XSAVE_XMM_SPACE, legacy.xmm_regs);
2364 ASSERT_OFFSET(XSAVE_XSTATE_BV, header.xstate_bv);
2365 ASSERT_OFFSET(XSAVE_YMMH_SPACE, avx_state);
2366 ASSERT_OFFSET(XSAVE_BNDREGS, bndreg_state);
2367 ASSERT_OFFSET(XSAVE_BNDCSR, bndcsr_state);
2368 ASSERT_OFFSET(XSAVE_OPMASK, opmask_state);
2369 ASSERT_OFFSET(XSAVE_ZMM_Hi256, zmm_hi256_state);
2370 ASSERT_OFFSET(XSAVE_Hi16_ZMM, hi16_zmm_state);
2371 ASSERT_OFFSET(XSAVE_PKRU, pkru_state);
2372
2373 static int kvm_put_xsave(X86CPU *cpu)
2374 {
2375 CPUX86State *env = &cpu->env;
2376 X86XSaveArea *xsave = env->xsave_buf;
2377
2378 if (!has_xsave) {
2379 return kvm_put_fpu(cpu);
2380 }
2381 x86_cpu_xsave_all_areas(cpu, xsave);
2382
2383 return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_XSAVE, xsave);
2384 }
2385
2386 static int kvm_put_xcrs(X86CPU *cpu)
2387 {
2388 CPUX86State *env = &cpu->env;
2389 struct kvm_xcrs xcrs = {};
2390
2391 if (!has_xcrs) {
2392 return 0;
2393 }
2394
2395 xcrs.nr_xcrs = 1;
2396 xcrs.flags = 0;
2397 xcrs.xcrs[0].xcr = 0;
2398 xcrs.xcrs[0].value = env->xcr0;
2399 return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_XCRS, &xcrs);
2400 }
2401
2402 static int kvm_put_sregs(X86CPU *cpu)
2403 {
2404 CPUX86State *env = &cpu->env;
2405 struct kvm_sregs sregs;
2406
2407 memset(sregs.interrupt_bitmap, 0, sizeof(sregs.interrupt_bitmap));
2408 if (env->interrupt_injected >= 0) {
2409 sregs.interrupt_bitmap[env->interrupt_injected / 64] |=
2410 (uint64_t)1 << (env->interrupt_injected % 64);
2411 }
2412
2413 if ((env->eflags & VM_MASK)) {
2414 set_v8086_seg(&sregs.cs, &env->segs[R_CS]);
2415 set_v8086_seg(&sregs.ds, &env->segs[R_DS]);
2416 set_v8086_seg(&sregs.es, &env->segs[R_ES]);
2417 set_v8086_seg(&sregs.fs, &env->segs[R_FS]);
2418 set_v8086_seg(&sregs.gs, &env->segs[R_GS]);
2419 set_v8086_seg(&sregs.ss, &env->segs[R_SS]);
2420 } else {
2421 set_seg(&sregs.cs, &env->segs[R_CS]);
2422 set_seg(&sregs.ds, &env->segs[R_DS]);
2423 set_seg(&sregs.es, &env->segs[R_ES]);
2424 set_seg(&sregs.fs, &env->segs[R_FS]);
2425 set_seg(&sregs.gs, &env->segs[R_GS]);
2426 set_seg(&sregs.ss, &env->segs[R_SS]);
2427 }
2428
2429 set_seg(&sregs.tr, &env->tr);
2430 set_seg(&sregs.ldt, &env->ldt);
2431
2432 sregs.idt.limit = env->idt.limit;
2433 sregs.idt.base = env->idt.base;
2434 memset(sregs.idt.padding, 0, sizeof sregs.idt.padding);
2435 sregs.gdt.limit = env->gdt.limit;
2436 sregs.gdt.base = env->gdt.base;
2437 memset(sregs.gdt.padding, 0, sizeof sregs.gdt.padding);
2438
2439 sregs.cr0 = env->cr[0];
2440 sregs.cr2 = env->cr[2];
2441 sregs.cr3 = env->cr[3];
2442 sregs.cr4 = env->cr[4];
2443
2444 sregs.cr8 = cpu_get_apic_tpr(cpu->apic_state);
2445 sregs.apic_base = cpu_get_apic_base(cpu->apic_state);
2446
2447 sregs.efer = env->efer;
2448
2449 return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_SREGS, &sregs);
2450 }
2451
2452 static void kvm_msr_buf_reset(X86CPU *cpu)
2453 {
2454 memset(cpu->kvm_msr_buf, 0, MSR_BUF_SIZE);
2455 }
2456
2457 static void kvm_msr_entry_add(X86CPU *cpu, uint32_t index, uint64_t value)
2458 {
2459 struct kvm_msrs *msrs = cpu->kvm_msr_buf;
2460 void *limit = ((void *)msrs) + MSR_BUF_SIZE;
2461 struct kvm_msr_entry *entry = &msrs->entries[msrs->nmsrs];
2462
2463 assert((void *)(entry + 1) <= limit);
2464
2465 entry->index = index;
2466 entry->reserved = 0;
2467 entry->data = value;
2468 msrs->nmsrs++;
2469 }
2470
2471 static int kvm_put_one_msr(X86CPU *cpu, int index, uint64_t value)
2472 {
2473 kvm_msr_buf_reset(cpu);
2474 kvm_msr_entry_add(cpu, index, value);
2475
2476 return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MSRS, cpu->kvm_msr_buf);
2477 }
2478
2479 void kvm_put_apicbase(X86CPU *cpu, uint64_t value)
2480 {
2481 int ret;
2482
2483 ret = kvm_put_one_msr(cpu, MSR_IA32_APICBASE, value);
2484 assert(ret == 1);
2485 }
2486
2487 static int kvm_put_tscdeadline_msr(X86CPU *cpu)
2488 {
2489 CPUX86State *env = &cpu->env;
2490 int ret;
2491
2492 if (!has_msr_tsc_deadline) {
2493 return 0;
2494 }
2495
2496 ret = kvm_put_one_msr(cpu, MSR_IA32_TSCDEADLINE, env->tsc_deadline);
2497 if (ret < 0) {
2498 return ret;
2499 }
2500
2501 assert(ret == 1);
2502 return 0;
2503 }
2504
2505 /*
2506 * Provide a separate write service for the feature control MSR in order to
2507 * kick the VCPU out of VMXON or even guest mode on reset. This has to be done
2508 * before writing any other state because forcibly leaving nested mode
2509 * invalidates the VCPU state.
2510 */
2511 static int kvm_put_msr_feature_control(X86CPU *cpu)
2512 {
2513 int ret;
2514
2515 if (!has_msr_feature_control) {
2516 return 0;
2517 }
2518
2519 ret = kvm_put_one_msr(cpu, MSR_IA32_FEATURE_CONTROL,
2520 cpu->env.msr_ia32_feature_control);
2521 if (ret < 0) {
2522 return ret;
2523 }
2524
2525 assert(ret == 1);
2526 return 0;
2527 }
2528
2529 static uint64_t make_vmx_msr_value(uint32_t index, uint32_t features)
2530 {
2531 uint32_t default1, can_be_one, can_be_zero;
2532 uint32_t must_be_one;
2533
2534 switch (index) {
2535 case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
2536 default1 = 0x00000016;
2537 break;
2538 case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
2539 default1 = 0x0401e172;
2540 break;
2541 case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
2542 default1 = 0x000011ff;
2543 break;
2544 case MSR_IA32_VMX_TRUE_EXIT_CTLS:
2545 default1 = 0x00036dff;
2546 break;
2547 case MSR_IA32_VMX_PROCBASED_CTLS2:
2548 default1 = 0;
2549 break;
2550 default:
2551 abort();
2552 }
2553
2554 /* If a feature bit is set, the control can be either set or clear.
2555 * Otherwise the value is limited to either 0 or 1 by default1.
2556 */
2557 can_be_one = features | default1;
2558 can_be_zero = features | ~default1;
2559 must_be_one = ~can_be_zero;
2560
2561 /*
2562 * Bit 0:31 -> 0 if the control bit can be zero (i.e. 1 if it must be one).
2563 * Bit 32:63 -> 1 if the control bit can be one.
2564 */
2565 return must_be_one | (((uint64_t)can_be_one) << 32);
2566 }
2567
2568 #define VMCS12_MAX_FIELD_INDEX (0x17)
2569
2570 static void kvm_msr_entry_add_vmx(X86CPU *cpu, FeatureWordArray f)
2571 {
2572 uint64_t kvm_vmx_basic =
2573 kvm_arch_get_supported_msr_feature(kvm_state,
2574 MSR_IA32_VMX_BASIC);
2575 uint64_t kvm_vmx_misc =
2576 kvm_arch_get_supported_msr_feature(kvm_state,
2577 MSR_IA32_VMX_MISC);
2578 uint64_t kvm_vmx_ept_vpid =
2579 kvm_arch_get_supported_msr_feature(kvm_state,
2580 MSR_IA32_VMX_EPT_VPID_CAP);
2581
2582 /*
2583 * If the guest is 64-bit, a value of 1 is allowed for the host address
2584 * space size vmexit control.
2585 */
2586 uint64_t fixed_vmx_exit = f[FEAT_8000_0001_EDX] & CPUID_EXT2_LM
2587 ? (uint64_t)VMX_VM_EXIT_HOST_ADDR_SPACE_SIZE << 32 : 0;
2588
2589 /*
2590 * Bits 0-30, 32-44 and 50-53 come from the host. KVM should
2591 * not change them for backwards compatibility.
2592 */
2593 uint64_t fixed_vmx_basic = kvm_vmx_basic &
2594 (MSR_VMX_BASIC_VMCS_REVISION_MASK |
2595 MSR_VMX_BASIC_VMXON_REGION_SIZE_MASK |
2596 MSR_VMX_BASIC_VMCS_MEM_TYPE_MASK);
2597
2598 /*
2599 * Same for bits 0-4 and 25-27. Bits 16-24 (CR3 target count) can
2600 * change in the future but are always zero for now, clear them to be
2601 * future proof. Bits 32-63 in theory could change, though KVM does
2602 * not support dual-monitor treatment and probably never will; mask
2603 * them out as well.
2604 */
2605 uint64_t fixed_vmx_misc = kvm_vmx_misc &
2606 (MSR_VMX_MISC_PREEMPTION_TIMER_SHIFT_MASK |
2607 MSR_VMX_MISC_MAX_MSR_LIST_SIZE_MASK);
2608
2609 /*
2610 * EPT memory types should not change either, so we do not bother
2611 * adding features for them.
2612 */
2613 uint64_t fixed_vmx_ept_mask =
2614 (f[FEAT_VMX_SECONDARY_CTLS] & VMX_SECONDARY_EXEC_ENABLE_EPT ?
2615 MSR_VMX_EPT_UC | MSR_VMX_EPT_WB : 0);
2616 uint64_t fixed_vmx_ept_vpid = kvm_vmx_ept_vpid & fixed_vmx_ept_mask;
2617
2618 kvm_msr_entry_add(cpu, MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
2619 make_vmx_msr_value(MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
2620 f[FEAT_VMX_PROCBASED_CTLS]));
2621 kvm_msr_entry_add(cpu, MSR_IA32_VMX_TRUE_PINBASED_CTLS,
2622 make_vmx_msr_value(MSR_IA32_VMX_TRUE_PINBASED_CTLS,
2623 f[FEAT_VMX_PINBASED_CTLS]));
2624 kvm_msr_entry_add(cpu, MSR_IA32_VMX_TRUE_EXIT_CTLS,
2625 make_vmx_msr_value(MSR_IA32_VMX_TRUE_EXIT_CTLS,
2626 f[FEAT_VMX_EXIT_CTLS]) | fixed_vmx_exit);
2627 kvm_msr_entry_add(cpu, MSR_IA32_VMX_TRUE_ENTRY_CTLS,
2628 make_vmx_msr_value(MSR_IA32_VMX_TRUE_ENTRY_CTLS,
2629 f[FEAT_VMX_ENTRY_CTLS]));
2630 kvm_msr_entry_add(cpu, MSR_IA32_VMX_PROCBASED_CTLS2,
2631 make_vmx_msr_value(MSR_IA32_VMX_PROCBASED_CTLS2,
2632 f[FEAT_VMX_SECONDARY_CTLS]));
2633 kvm_msr_entry_add(cpu, MSR_IA32_VMX_EPT_VPID_CAP,
2634 f[FEAT_VMX_EPT_VPID_CAPS] | fixed_vmx_ept_vpid);
2635 kvm_msr_entry_add(cpu, MSR_IA32_VMX_BASIC,
2636 f[FEAT_VMX_BASIC] | fixed_vmx_basic);
2637 kvm_msr_entry_add(cpu, MSR_IA32_VMX_MISC,
2638 f[FEAT_VMX_MISC] | fixed_vmx_misc);
2639 if (has_msr_vmx_vmfunc) {
2640 kvm_msr_entry_add(cpu, MSR_IA32_VMX_VMFUNC, f[FEAT_VMX_VMFUNC]);
2641 }
2642
2643 /*
2644 * Just to be safe, write these with constant values. The CRn_FIXED1
2645 * MSRs are generated by KVM based on the vCPU's CPUID.
2646 */
2647 kvm_msr_entry_add(cpu, MSR_IA32_VMX_CR0_FIXED0,
2648 CR0_PE_MASK | CR0_PG_MASK | CR0_NE_MASK);
2649 kvm_msr_entry_add(cpu, MSR_IA32_VMX_CR4_FIXED0,
2650 CR4_VMXE_MASK);
2651 kvm_msr_entry_add(cpu, MSR_IA32_VMX_VMCS_ENUM,
2652 VMCS12_MAX_FIELD_INDEX << 1);
2653 }
2654
2655 static int kvm_put_msrs(X86CPU *cpu, int level)
2656 {
2657 CPUX86State *env = &cpu->env;
2658 int i;
2659 int ret;
2660
2661 kvm_msr_buf_reset(cpu);
2662
2663 kvm_msr_entry_add(cpu, MSR_IA32_SYSENTER_CS, env->sysenter_cs);
2664 kvm_msr_entry_add(cpu, MSR_IA32_SYSENTER_ESP, env->sysenter_esp);
2665 kvm_msr_entry_add(cpu, MSR_IA32_SYSENTER_EIP, env->sysenter_eip);
2666 kvm_msr_entry_add(cpu, MSR_PAT, env->pat);
2667 if (has_msr_star) {
2668 kvm_msr_entry_add(cpu, MSR_STAR, env->star);
2669 }
2670 if (has_msr_hsave_pa) {
2671 kvm_msr_entry_add(cpu, MSR_VM_HSAVE_PA, env->vm_hsave);
2672 }
2673 if (has_msr_tsc_aux) {
2674 kvm_msr_entry_add(cpu, MSR_TSC_AUX, env->tsc_aux);
2675 }
2676 if (has_msr_tsc_adjust) {
2677 kvm_msr_entry_add(cpu, MSR_TSC_ADJUST, env->tsc_adjust);
2678 }
2679 if (has_msr_misc_enable) {
2680 kvm_msr_entry_add(cpu, MSR_IA32_MISC_ENABLE,
2681 env->msr_ia32_misc_enable);
2682 }
2683 if (has_msr_smbase) {
2684 kvm_msr_entry_add(cpu, MSR_IA32_SMBASE, env->smbase);
2685 }
2686 if (has_msr_smi_count) {
2687 kvm_msr_entry_add(cpu, MSR_SMI_COUNT, env->msr_smi_count);
2688 }
2689 if (has_msr_bndcfgs) {
2690 kvm_msr_entry_add(cpu, MSR_IA32_BNDCFGS, env->msr_bndcfgs);
2691 }
2692 if (has_msr_xss) {
2693 kvm_msr_entry_add(cpu, MSR_IA32_XSS, env->xss);
2694 }
2695 if (has_msr_umwait) {
2696 kvm_msr_entry_add(cpu, MSR_IA32_UMWAIT_CONTROL, env->umwait);
2697 }
2698 if (has_msr_spec_ctrl) {
2699 kvm_msr_entry_add(cpu, MSR_IA32_SPEC_CTRL, env->spec_ctrl);
2700 }
2701 if (has_msr_tsx_ctrl) {
2702 kvm_msr_entry_add(cpu, MSR_IA32_TSX_CTRL, env->tsx_ctrl);
2703 }
2704 if (has_msr_virt_ssbd) {
2705 kvm_msr_entry_add(cpu, MSR_VIRT_SSBD, env->virt_ssbd);
2706 }
2707
2708 #ifdef TARGET_X86_64
2709 if (lm_capable_kernel) {
2710 kvm_msr_entry_add(cpu, MSR_CSTAR, env->cstar);
2711 kvm_msr_entry_add(cpu, MSR_KERNELGSBASE, env->kernelgsbase);
2712 kvm_msr_entry_add(cpu, MSR_FMASK, env->fmask);
2713 kvm_msr_entry_add(cpu, MSR_LSTAR, env->lstar);
2714 }
2715 #endif
2716
2717 /* If host supports feature MSR, write down. */
2718 if (has_msr_arch_capabs) {
2719 kvm_msr_entry_add(cpu, MSR_IA32_ARCH_CAPABILITIES,
2720 env->features[FEAT_ARCH_CAPABILITIES]);
2721 }
2722
2723 if (has_msr_core_capabs) {
2724 kvm_msr_entry_add(cpu, MSR_IA32_CORE_CAPABILITY,
2725 env->features[FEAT_CORE_CAPABILITY]);
2726 }
2727
2728 /*
2729 * The following MSRs have side effects on the guest or are too heavy
2730 * for normal writeback. Limit them to reset or full state updates.
2731 */
2732 if (level >= KVM_PUT_RESET_STATE) {
2733 kvm_msr_entry_add(cpu, MSR_IA32_TSC, env->tsc);
2734 kvm_msr_entry_add(cpu, MSR_KVM_SYSTEM_TIME, env->system_time_msr);
2735 kvm_msr_entry_add(cpu, MSR_KVM_WALL_CLOCK, env->wall_clock_msr);
2736 if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_ASYNC_PF)) {
2737 kvm_msr_entry_add(cpu, MSR_KVM_ASYNC_PF_EN, env->async_pf_en_msr);
2738 }
2739 if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_PV_EOI)) {
2740 kvm_msr_entry_add(cpu, MSR_KVM_PV_EOI_EN, env->pv_eoi_en_msr);
2741 }
2742 if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_STEAL_TIME)) {
2743 kvm_msr_entry_add(cpu, MSR_KVM_STEAL_TIME, env->steal_time_msr);
2744 }
2745
2746 if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_POLL_CONTROL)) {
2747 kvm_msr_entry_add(cpu, MSR_KVM_POLL_CONTROL, env->poll_control_msr);
2748 }
2749
2750 if (has_architectural_pmu_version > 0) {
2751 if (has_architectural_pmu_version > 1) {
2752 /* Stop the counter. */
2753 kvm_msr_entry_add(cpu, MSR_CORE_PERF_FIXED_CTR_CTRL, 0);
2754 kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_CTRL, 0);
2755 }
2756
2757 /* Set the counter values. */
2758 for (i = 0; i < num_architectural_pmu_fixed_counters; i++) {
2759 kvm_msr_entry_add(cpu, MSR_CORE_PERF_FIXED_CTR0 + i,
2760 env->msr_fixed_counters[i]);
2761 }
2762 for (i = 0; i < num_architectural_pmu_gp_counters; i++) {
2763 kvm_msr_entry_add(cpu, MSR_P6_PERFCTR0 + i,
2764 env->msr_gp_counters[i]);
2765 kvm_msr_entry_add(cpu, MSR_P6_EVNTSEL0 + i,
2766 env->msr_gp_evtsel[i]);
2767 }
2768 if (has_architectural_pmu_version > 1) {
2769 kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_STATUS,
2770 env->msr_global_status);
2771 kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_OVF_CTRL,
2772 env->msr_global_ovf_ctrl);
2773
2774 /* Now start the PMU. */
2775 kvm_msr_entry_add(cpu, MSR_CORE_PERF_FIXED_CTR_CTRL,
2776 env->msr_fixed_ctr_ctrl);
2777 kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_CTRL,
2778 env->msr_global_ctrl);
2779 }
2780 }
2781 /*
2782 * Hyper-V partition-wide MSRs: to avoid clearing them on cpu hot-add,
2783 * only sync them to KVM on the first cpu
2784 */
2785 if (current_cpu == first_cpu) {
2786 if (has_msr_hv_hypercall) {
2787 kvm_msr_entry_add(cpu, HV_X64_MSR_GUEST_OS_ID,
2788 env->msr_hv_guest_os_id);
2789 kvm_msr_entry_add(cpu, HV_X64_MSR_HYPERCALL,
2790 env->msr_hv_hypercall);
2791 }
2792 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_TIME)) {
2793 kvm_msr_entry_add(cpu, HV_X64_MSR_REFERENCE_TSC,
2794 env->msr_hv_tsc);
2795 }
2796 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_REENLIGHTENMENT)) {
2797 kvm_msr_entry_add(cpu, HV_X64_MSR_REENLIGHTENMENT_CONTROL,
2798 env->msr_hv_reenlightenment_control);
2799 kvm_msr_entry_add(cpu, HV_X64_MSR_TSC_EMULATION_CONTROL,
2800 env->msr_hv_tsc_emulation_control);
2801 kvm_msr_entry_add(cpu, HV_X64_MSR_TSC_EMULATION_STATUS,
2802 env->msr_hv_tsc_emulation_status);
2803 }
2804 }
2805 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_VAPIC)) {
2806 kvm_msr_entry_add(cpu, HV_X64_MSR_APIC_ASSIST_PAGE,
2807 env->msr_hv_vapic);
2808 }
2809 if (has_msr_hv_crash) {
2810 int j;
2811
2812 for (j = 0; j < HV_CRASH_PARAMS; j++)
2813 kvm_msr_entry_add(cpu, HV_X64_MSR_CRASH_P0 + j,
2814 env->msr_hv_crash_params[j]);
2815
2816 kvm_msr_entry_add(cpu, HV_X64_MSR_CRASH_CTL, HV_CRASH_CTL_NOTIFY);
2817 }
2818 if (has_msr_hv_runtime) {
2819 kvm_msr_entry_add(cpu, HV_X64_MSR_VP_RUNTIME, env->msr_hv_runtime);
2820 }
2821 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_VPINDEX)
2822 && hv_vpindex_settable) {
2823 kvm_msr_entry_add(cpu, HV_X64_MSR_VP_INDEX,
2824 hyperv_vp_index(CPU(cpu)));
2825 }
2826 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_SYNIC)) {
2827 int j;
2828
2829 kvm_msr_entry_add(cpu, HV_X64_MSR_SVERSION, HV_SYNIC_VERSION);
2830
2831 kvm_msr_entry_add(cpu, HV_X64_MSR_SCONTROL,
2832 env->msr_hv_synic_control);
2833 kvm_msr_entry_add(cpu, HV_X64_MSR_SIEFP,
2834 env->msr_hv_synic_evt_page);
2835 kvm_msr_entry_add(cpu, HV_X64_MSR_SIMP,
2836 env->msr_hv_synic_msg_page);
2837
2838 for (j = 0; j < ARRAY_SIZE(env->msr_hv_synic_sint); j++) {
2839 kvm_msr_entry_add(cpu, HV_X64_MSR_SINT0 + j,
2840 env->msr_hv_synic_sint[j]);
2841 }
2842 }
2843 if (has_msr_hv_stimer) {
2844 int j;
2845
2846 for (j = 0; j < ARRAY_SIZE(env->msr_hv_stimer_config); j++) {
2847 kvm_msr_entry_add(cpu, HV_X64_MSR_STIMER0_CONFIG + j * 2,
2848 env->msr_hv_stimer_config[j]);
2849 }
2850
2851 for (j = 0; j < ARRAY_SIZE(env->msr_hv_stimer_count); j++) {
2852 kvm_msr_entry_add(cpu, HV_X64_MSR_STIMER0_COUNT + j * 2,
2853 env->msr_hv_stimer_count[j]);
2854 }
2855 }
2856 if (env->features[FEAT_1_EDX] & CPUID_MTRR) {
2857 uint64_t phys_mask = MAKE_64BIT_MASK(0, cpu->phys_bits);
2858
2859 kvm_msr_entry_add(cpu, MSR_MTRRdefType, env->mtrr_deftype);
2860 kvm_msr_entry_add(cpu, MSR_MTRRfix64K_00000, env->mtrr_fixed[0]);
2861 kvm_msr_entry_add(cpu, MSR_MTRRfix16K_80000, env->mtrr_fixed[1]);
2862 kvm_msr_entry_add(cpu, MSR_MTRRfix16K_A0000, env->mtrr_fixed[2]);
2863 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_C0000, env->mtrr_fixed[3]);
2864 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_C8000, env->mtrr_fixed[4]);
2865 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_D0000, env->mtrr_fixed[5]);
2866 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_D8000, env->mtrr_fixed[6]);
2867 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_E0000, env->mtrr_fixed[7]);
2868 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_E8000, env->mtrr_fixed[8]);
2869 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_F0000, env->mtrr_fixed[9]);
2870 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_F8000, env->mtrr_fixed[10]);
2871 for (i = 0; i < MSR_MTRRcap_VCNT; i++) {
2872 /* The CPU GPs if we write to a bit above the physical limit of
2873 * the host CPU (and KVM emulates that)
2874 */
2875 uint64_t mask = env->mtrr_var[i].mask;
2876 mask &= phys_mask;
2877
2878 kvm_msr_entry_add(cpu, MSR_MTRRphysBase(i),
2879 env->mtrr_var[i].base);
2880 kvm_msr_entry_add(cpu, MSR_MTRRphysMask(i), mask);
2881 }
2882 }
2883 if (env->features[FEAT_7_0_EBX] & CPUID_7_0_EBX_INTEL_PT) {
2884 int addr_num = kvm_arch_get_supported_cpuid(kvm_state,
2885 0x14, 1, R_EAX) & 0x7;
2886
2887 kvm_msr_entry_add(cpu, MSR_IA32_RTIT_CTL,
2888 env->msr_rtit_ctrl);
2889 kvm_msr_entry_add(cpu, MSR_IA32_RTIT_STATUS,
2890 env->msr_rtit_status);
2891 kvm_msr_entry_add(cpu, MSR_IA32_RTIT_OUTPUT_BASE,
2892 env->msr_rtit_output_base);
2893 kvm_msr_entry_add(cpu, MSR_IA32_RTIT_OUTPUT_MASK,
2894 env->msr_rtit_output_mask);
2895 kvm_msr_entry_add(cpu, MSR_IA32_RTIT_CR3_MATCH,
2896 env->msr_rtit_cr3_match);
2897 for (i = 0; i < addr_num; i++) {
2898 kvm_msr_entry_add(cpu, MSR_IA32_RTIT_ADDR0_A + i,
2899 env->msr_rtit_addrs[i]);
2900 }
2901 }
2902
2903 /* Note: MSR_IA32_FEATURE_CONTROL is written separately, see
2904 * kvm_put_msr_feature_control. */
2905
2906 /*
2907 * Older kernels do not include VMX MSRs in KVM_GET_MSR_INDEX_LIST, but
2908 * all kernels with MSR features should have them.
2909 */
2910 if (kvm_feature_msrs && cpu_has_vmx(env)) {
2911 kvm_msr_entry_add_vmx(cpu, env->features);
2912 }
2913 }
2914
2915 if (env->mcg_cap) {
2916 int i;
2917
2918 kvm_msr_entry_add(cpu, MSR_MCG_STATUS, env->mcg_status);
2919 kvm_msr_entry_add(cpu, MSR_MCG_CTL, env->mcg_ctl);
2920 if (has_msr_mcg_ext_ctl) {
2921 kvm_msr_entry_add(cpu, MSR_MCG_EXT_CTL, env->mcg_ext_ctl);
2922 }
2923 for (i = 0; i < (env->mcg_cap & 0xff) * 4; i++) {
2924 kvm_msr_entry_add(cpu, MSR_MC0_CTL + i, env->mce_banks[i]);
2925 }
2926 }
2927
2928 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MSRS, cpu->kvm_msr_buf);
2929 if (ret < 0) {
2930 return ret;
2931 }
2932
2933 if (ret < cpu->kvm_msr_buf->nmsrs) {
2934 struct kvm_msr_entry *e = &cpu->kvm_msr_buf->entries[ret];
2935 error_report("error: failed to set MSR 0x%" PRIx32 " to 0x%" PRIx64,
2936 (uint32_t)e->index, (uint64_t)e->data);
2937 }
2938
2939 assert(ret == cpu->kvm_msr_buf->nmsrs);
2940 return 0;
2941 }
2942
2943
2944 static int kvm_get_fpu(X86CPU *cpu)
2945 {
2946 CPUX86State *env = &cpu->env;
2947 struct kvm_fpu fpu;
2948 int i, ret;
2949
2950 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_FPU, &fpu);
2951 if (ret < 0) {
2952 return ret;
2953 }
2954
2955 env->fpstt = (fpu.fsw >> 11) & 7;
2956 env->fpus = fpu.fsw;
2957 env->fpuc = fpu.fcw;
2958 env->fpop = fpu.last_opcode;
2959 env->fpip = fpu.last_ip;
2960 env->fpdp = fpu.last_dp;
2961 for (i = 0; i < 8; ++i) {
2962 env->fptags[i] = !((fpu.ftwx >> i) & 1);
2963 }
2964 memcpy(env->fpregs, fpu.fpr, sizeof env->fpregs);
2965 for (i = 0; i < CPU_NB_REGS; i++) {
2966 env->xmm_regs[i].ZMM_Q(0) = ldq_p(&fpu.xmm[i][0]);
2967 env->xmm_regs[i].ZMM_Q(1) = ldq_p(&fpu.xmm[i][8]);
2968 }
2969 env->mxcsr = fpu.mxcsr;
2970
2971 return 0;
2972 }
2973
2974 static int kvm_get_xsave(X86CPU *cpu)
2975 {
2976 CPUX86State *env = &cpu->env;
2977 X86XSaveArea *xsave = env->xsave_buf;
2978 int ret;
2979
2980 if (!has_xsave) {
2981 return kvm_get_fpu(cpu);
2982 }
2983
2984 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_XSAVE, xsave);
2985 if (ret < 0) {
2986 return ret;
2987 }
2988 x86_cpu_xrstor_all_areas(cpu, xsave);
2989
2990 return 0;
2991 }
2992
2993 static int kvm_get_xcrs(X86CPU *cpu)
2994 {
2995 CPUX86State *env = &cpu->env;
2996 int i, ret;
2997 struct kvm_xcrs xcrs;
2998
2999 if (!has_xcrs) {
3000 return 0;
3001 }
3002
3003 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_XCRS, &xcrs);
3004 if (ret < 0) {
3005 return ret;
3006 }
3007
3008 for (i = 0; i < xcrs.nr_xcrs; i++) {
3009 /* Only support xcr0 now */
3010 if (xcrs.xcrs[i].xcr == 0) {
3011 env->xcr0 = xcrs.xcrs[i].value;
3012 break;
3013 }
3014 }
3015 return 0;
3016 }
3017
3018 static int kvm_get_sregs(X86CPU *cpu)
3019 {
3020 CPUX86State *env = &cpu->env;
3021 struct kvm_sregs sregs;
3022 int bit, i, ret;
3023
3024 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_SREGS, &sregs);
3025 if (ret < 0) {
3026 return ret;
3027 }
3028
3029 /* There can only be one pending IRQ set in the bitmap at a time, so try
3030 to find it and save its number instead (-1 for none). */
3031 env->interrupt_injected = -1;
3032 for (i = 0; i < ARRAY_SIZE(sregs.interrupt_bitmap); i++) {
3033 if (sregs.interrupt_bitmap[i]) {
3034 bit = ctz64(sregs.interrupt_bitmap[i]);
3035 env->interrupt_injected = i * 64 + bit;
3036 break;
3037 }
3038 }
3039
3040 get_seg(&env->segs[R_CS], &sregs.cs);
3041 get_seg(&env->segs[R_DS], &sregs.ds);
3042 get_seg(&env->segs[R_ES], &sregs.es);
3043 get_seg(&env->segs[R_FS], &sregs.fs);
3044 get_seg(&env->segs[R_GS], &sregs.gs);
3045 get_seg(&env->segs[R_SS], &sregs.ss);
3046
3047 get_seg(&env->tr, &sregs.tr);
3048 get_seg(&env->ldt, &sregs.ldt);
3049
3050 env->idt.limit = sregs.idt.limit;
3051 env->idt.base = sregs.idt.base;
3052 env->gdt.limit = sregs.gdt.limit;
3053 env->gdt.base = sregs.gdt.base;
3054
3055 env->cr[0] = sregs.cr0;
3056 env->cr[2] = sregs.cr2;
3057 env->cr[3] = sregs.cr3;
3058 env->cr[4] = sregs.cr4;
3059
3060 env->efer = sregs.efer;
3061
3062 /* changes to apic base and cr8/tpr are read back via kvm_arch_post_run */
3063 x86_update_hflags(env);
3064
3065 return 0;
3066 }
3067
3068 static int kvm_get_msrs(X86CPU *cpu)
3069 {
3070 CPUX86State *env = &cpu->env;
3071 struct kvm_msr_entry *msrs = cpu->kvm_msr_buf->entries;
3072 int ret, i;
3073 uint64_t mtrr_top_bits;
3074
3075 kvm_msr_buf_reset(cpu);
3076
3077 kvm_msr_entry_add(cpu, MSR_IA32_SYSENTER_CS, 0);
3078 kvm_msr_entry_add(cpu, MSR_IA32_SYSENTER_ESP, 0);
3079 kvm_msr_entry_add(cpu, MSR_IA32_SYSENTER_EIP, 0);
3080 kvm_msr_entry_add(cpu, MSR_PAT, 0);
3081 if (has_msr_star) {
3082 kvm_msr_entry_add(cpu, MSR_STAR, 0);
3083 }
3084 if (has_msr_hsave_pa) {
3085 kvm_msr_entry_add(cpu, MSR_VM_HSAVE_PA, 0);
3086 }
3087 if (has_msr_tsc_aux) {
3088 kvm_msr_entry_add(cpu, MSR_TSC_AUX, 0);
3089 }
3090 if (has_msr_tsc_adjust) {
3091 kvm_msr_entry_add(cpu, MSR_TSC_ADJUST, 0);
3092 }
3093 if (has_msr_tsc_deadline) {
3094 kvm_msr_entry_add(cpu, MSR_IA32_TSCDEADLINE, 0);
3095 }
3096 if (has_msr_misc_enable) {
3097 kvm_msr_entry_add(cpu, MSR_IA32_MISC_ENABLE, 0);
3098 }
3099 if (has_msr_smbase) {
3100 kvm_msr_entry_add(cpu, MSR_IA32_SMBASE, 0);
3101 }
3102 if (has_msr_smi_count) {
3103 kvm_msr_entry_add(cpu, MSR_SMI_COUNT, 0);
3104 }
3105 if (has_msr_feature_control) {
3106 kvm_msr_entry_add(cpu, MSR_IA32_FEATURE_CONTROL, 0);
3107 }
3108 if (has_msr_bndcfgs) {
3109 kvm_msr_entry_add(cpu, MSR_IA32_BNDCFGS, 0);
3110 }
3111 if (has_msr_xss) {
3112 kvm_msr_entry_add(cpu, MSR_IA32_XSS, 0);
3113 }
3114 if (has_msr_umwait) {
3115 kvm_msr_entry_add(cpu, MSR_IA32_UMWAIT_CONTROL, 0);
3116 }
3117 if (has_msr_spec_ctrl) {
3118 kvm_msr_entry_add(cpu, MSR_IA32_SPEC_CTRL, 0);
3119 }
3120 if (has_msr_tsx_ctrl) {
3121 kvm_msr_entry_add(cpu, MSR_IA32_TSX_CTRL, 0);
3122 }
3123 if (has_msr_virt_ssbd) {
3124 kvm_msr_entry_add(cpu, MSR_VIRT_SSBD, 0);
3125 }
3126 if (!env->tsc_valid) {
3127 kvm_msr_entry_add(cpu, MSR_IA32_TSC, 0);
3128 env->tsc_valid = !runstate_is_running();
3129 }
3130
3131 #ifdef TARGET_X86_64
3132 if (lm_capable_kernel) {
3133 kvm_msr_entry_add(cpu, MSR_CSTAR, 0);
3134 kvm_msr_entry_add(cpu, MSR_KERNELGSBASE, 0);
3135 kvm_msr_entry_add(cpu, MSR_FMASK, 0);
3136 kvm_msr_entry_add(cpu, MSR_LSTAR, 0);
3137 }
3138 #endif
3139 kvm_msr_entry_add(cpu, MSR_KVM_SYSTEM_TIME, 0);
3140 kvm_msr_entry_add(cpu, MSR_KVM_WALL_CLOCK, 0);
3141 if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_ASYNC_PF)) {
3142 kvm_msr_entry_add(cpu, MSR_KVM_ASYNC_PF_EN, 0);
3143 }
3144 if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_PV_EOI)) {
3145 kvm_msr_entry_add(cpu, MSR_KVM_PV_EOI_EN, 0);
3146 }
3147 if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_STEAL_TIME)) {
3148 kvm_msr_entry_add(cpu, MSR_KVM_STEAL_TIME, 0);
3149 }
3150 if (env->features[FEAT_KVM] & (1 << KVM_FEATURE_POLL_CONTROL)) {
3151 kvm_msr_entry_add(cpu, MSR_KVM_POLL_CONTROL, 1);
3152 }
3153 if (has_architectural_pmu_version > 0) {
3154 if (has_architectural_pmu_version > 1) {
3155 kvm_msr_entry_add(cpu, MSR_CORE_PERF_FIXED_CTR_CTRL, 0);
3156 kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_CTRL, 0);
3157 kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_STATUS, 0);
3158 kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_OVF_CTRL, 0);
3159 }
3160 for (i = 0; i < num_architectural_pmu_fixed_counters; i++) {
3161 kvm_msr_entry_add(cpu, MSR_CORE_PERF_FIXED_CTR0 + i, 0);
3162 }
3163 for (i = 0; i < num_architectural_pmu_gp_counters; i++) {
3164 kvm_msr_entry_add(cpu, MSR_P6_PERFCTR0 + i, 0);
3165 kvm_msr_entry_add(cpu, MSR_P6_EVNTSEL0 + i, 0);
3166 }
3167 }
3168
3169 if (env->mcg_cap) {
3170 kvm_msr_entry_add(cpu, MSR_MCG_STATUS, 0);
3171 kvm_msr_entry_add(cpu, MSR_MCG_CTL, 0);
3172 if (has_msr_mcg_ext_ctl) {
3173 kvm_msr_entry_add(cpu, MSR_MCG_EXT_CTL, 0);
3174 }
3175 for (i = 0; i < (env->mcg_cap & 0xff) * 4; i++) {
3176 kvm_msr_entry_add(cpu, MSR_MC0_CTL + i, 0);
3177 }
3178 }
3179
3180 if (has_msr_hv_hypercall) {
3181 kvm_msr_entry_add(cpu, HV_X64_MSR_HYPERCALL, 0);
3182 kvm_msr_entry_add(cpu, HV_X64_MSR_GUEST_OS_ID, 0);
3183 }
3184 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_VAPIC)) {
3185 kvm_msr_entry_add(cpu, HV_X64_MSR_APIC_ASSIST_PAGE, 0);
3186 }
3187 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_TIME)) {
3188 kvm_msr_entry_add(cpu, HV_X64_MSR_REFERENCE_TSC, 0);
3189 }
3190 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_REENLIGHTENMENT)) {
3191 kvm_msr_entry_add(cpu, HV_X64_MSR_REENLIGHTENMENT_CONTROL, 0);
3192 kvm_msr_entry_add(cpu, HV_X64_MSR_TSC_EMULATION_CONTROL, 0);
3193 kvm_msr_entry_add(cpu, HV_X64_MSR_TSC_EMULATION_STATUS, 0);
3194 }
3195 if (has_msr_hv_crash) {
3196 int j;
3197
3198 for (j = 0; j < HV_CRASH_PARAMS; j++) {
3199 kvm_msr_entry_add(cpu, HV_X64_MSR_CRASH_P0 + j, 0);
3200 }
3201 }
3202 if (has_msr_hv_runtime) {
3203 kvm_msr_entry_add(cpu, HV_X64_MSR_VP_RUNTIME, 0);
3204 }
3205 if (hyperv_feat_enabled(cpu, HYPERV_FEAT_SYNIC)) {
3206 uint32_t msr;
3207
3208 kvm_msr_entry_add(cpu, HV_X64_MSR_SCONTROL, 0);
3209 kvm_msr_entry_add(cpu, HV_X64_MSR_SIEFP, 0);
3210 kvm_msr_entry_add(cpu, HV_X64_MSR_SIMP, 0);
3211 for (msr = HV_X64_MSR_SINT0; msr <= HV_X64_MSR_SINT15; msr++) {
3212 kvm_msr_entry_add(cpu, msr, 0);
3213 }
3214 }
3215 if (has_msr_hv_stimer) {
3216 uint32_t msr;
3217
3218 for (msr = HV_X64_MSR_STIMER0_CONFIG; msr <= HV_X64_MSR_STIMER3_COUNT;
3219 msr++) {
3220 kvm_msr_entry_add(cpu, msr, 0);
3221 }
3222 }
3223 if (env->features[FEAT_1_EDX] & CPUID_MTRR) {
3224 kvm_msr_entry_add(cpu, MSR_MTRRdefType, 0);
3225 kvm_msr_entry_add(cpu, MSR_MTRRfix64K_00000, 0);
3226 kvm_msr_entry_add(cpu, MSR_MTRRfix16K_80000, 0);
3227 kvm_msr_entry_add(cpu, MSR_MTRRfix16K_A0000, 0);
3228 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_C0000, 0);
3229 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_C8000, 0);
3230 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_D0000, 0);
3231 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_D8000, 0);
3232 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_E0000, 0);
3233 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_E8000, 0);
3234 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_F0000, 0);
3235 kvm_msr_entry_add(cpu, MSR_MTRRfix4K_F8000, 0);
3236 for (i = 0; i < MSR_MTRRcap_VCNT; i++) {
3237 kvm_msr_entry_add(cpu, MSR_MTRRphysBase(i), 0);
3238 kvm_msr_entry_add(cpu, MSR_MTRRphysMask(i), 0);
3239 }
3240 }
3241
3242 if (env->features[FEAT_7_0_EBX] & CPUID_7_0_EBX_INTEL_PT) {
3243 int addr_num =
3244 kvm_arch_get_supported_cpuid(kvm_state, 0x14, 1, R_EAX) & 0x7;
3245
3246 kvm_msr_entry_add(cpu, MSR_IA32_RTIT_CTL, 0);
3247 kvm_msr_entry_add(cpu, MSR_IA32_RTIT_STATUS, 0);
3248 kvm_msr_entry_add(cpu, MSR_IA32_RTIT_OUTPUT_BASE, 0);
3249 kvm_msr_entry_add(cpu, MSR_IA32_RTIT_OUTPUT_MASK, 0);
3250 kvm_msr_entry_add(cpu, MSR_IA32_RTIT_CR3_MATCH, 0);
3251 for (i = 0; i < addr_num; i++) {
3252 kvm_msr_entry_add(cpu, MSR_IA32_RTIT_ADDR0_A + i, 0);
3253 }
3254 }
3255
3256 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MSRS, cpu->kvm_msr_buf);
3257 if (ret < 0) {
3258 return ret;
3259 }
3260
3261 if (ret < cpu->kvm_msr_buf->nmsrs) {
3262 struct kvm_msr_entry *e = &cpu->kvm_msr_buf->entries[ret];
3263 error_report("error: failed to get MSR 0x%" PRIx32,
3264 (uint32_t)e->index);
3265 }
3266
3267 assert(ret == cpu->kvm_msr_buf->nmsrs);
3268 /*
3269 * MTRR masks: Each mask consists of 5 parts
3270 * a 10..0: must be zero
3271 * b 11 : valid bit
3272 * c n-1.12: actual mask bits
3273 * d 51..n: reserved must be zero
3274 * e 63.52: reserved must be zero
3275 *
3276 * 'n' is the number of physical bits supported by the CPU and is
3277 * apparently always <= 52. We know our 'n' but don't know what
3278 * the destinations 'n' is; it might be smaller, in which case
3279 * it masks (c) on loading. It might be larger, in which case
3280 * we fill 'd' so that d..c is consistent irrespetive of the 'n'
3281 * we're migrating to.
3282 */
3283
3284 if (cpu->fill_mtrr_mask) {
3285 QEMU_BUILD_BUG_ON(TARGET_PHYS_ADDR_SPACE_BITS > 52);
3286 assert(cpu->phys_bits <= TARGET_PHYS_ADDR_SPACE_BITS);
3287 mtrr_top_bits = MAKE_64BIT_MASK(cpu->phys_bits, 52 - cpu->phys_bits);
3288 } else {
3289 mtrr_top_bits = 0;
3290 }
3291
3292 for (i = 0; i < ret; i++) {
3293 uint32_t index = msrs[i].index;
3294 switch (index) {
3295 case MSR_IA32_SYSENTER_CS:
3296 env->sysenter_cs = msrs[i].data;
3297 break;
3298 case MSR_IA32_SYSENTER_ESP:
3299 env->sysenter_esp = msrs[i].data;
3300 break;
3301 case MSR_IA32_SYSENTER_EIP:
3302 env->sysenter_eip = msrs[i].data;
3303 break;
3304 case MSR_PAT:
3305 env->pat = msrs[i].data;
3306 break;
3307 case MSR_STAR:
3308 env->star = msrs[i].data;
3309 break;
3310 #ifdef TARGET_X86_64
3311 case MSR_CSTAR:
3312 env->cstar = msrs[i].data;
3313 break;
3314 case MSR_KERNELGSBASE:
3315 env->kernelgsbase = msrs[i].data;
3316 break;
3317 case MSR_FMASK:
3318 env->fmask = msrs[i].data;
3319 break;
3320 case MSR_LSTAR:
3321 env->lstar = msrs[i].data;
3322 break;
3323 #endif
3324 case MSR_IA32_TSC:
3325 env->tsc = msrs[i].data;
3326 break;
3327 case MSR_TSC_AUX:
3328 env->tsc_aux = msrs[i].data;
3329 break;
3330 case MSR_TSC_ADJUST:
3331 env->tsc_adjust = msrs[i].data;
3332 break;
3333 case MSR_IA32_TSCDEADLINE:
3334 env->tsc_deadline = msrs[i].data;
3335 break;
3336 case MSR_VM_HSAVE_PA:
3337 env->vm_hsave = msrs[i].data;
3338 break;
3339 case MSR_KVM_SYSTEM_TIME:
3340 env->system_time_msr = msrs[i].data;
3341 break;
3342 case MSR_KVM_WALL_CLOCK:
3343 env->wall_clock_msr = msrs[i].data;
3344 break;
3345 case MSR_MCG_STATUS:
3346 env->mcg_status = msrs[i].data;
3347 break;
3348 case MSR_MCG_CTL:
3349 env->mcg_ctl = msrs[i].data;
3350 break;
3351 case MSR_MCG_EXT_CTL:
3352 env->mcg_ext_ctl = msrs[i].data;
3353 break;
3354 case MSR_IA32_MISC_ENABLE:
3355 env->msr_ia32_misc_enable = msrs[i].data;
3356 break;
3357 case MSR_IA32_SMBASE:
3358 env->smbase = msrs[i].data;
3359 break;
3360 case MSR_SMI_COUNT:
3361 env->msr_smi_count = msrs[i].data;
3362 break;
3363 case MSR_IA32_FEATURE_CONTROL:
3364 env->msr_ia32_feature_control = msrs[i].data;
3365 break;
3366 case MSR_IA32_BNDCFGS:
3367 env->msr_bndcfgs = msrs[i].data;
3368 break;
3369 case MSR_IA32_XSS:
3370 env->xss = msrs[i].data;
3371 break;
3372 case MSR_IA32_UMWAIT_CONTROL:
3373 env->umwait = msrs[i].data;
3374 break;
3375 default:
3376 if (msrs[i].index >= MSR_MC0_CTL &&
3377 msrs[i].index < MSR_MC0_CTL + (env->mcg_cap & 0xff) * 4) {
3378 env->mce_banks[msrs[i].index - MSR_MC0_CTL] = msrs[i].data;
3379 }
3380 break;
3381 case MSR_KVM_ASYNC_PF_EN:
3382 env->async_pf_en_msr = msrs[i].data;
3383 break;
3384 case MSR_KVM_PV_EOI_EN:
3385 env->pv_eoi_en_msr = msrs[i].data;
3386 break;
3387 case MSR_KVM_STEAL_TIME:
3388 env->steal_time_msr = msrs[i].data;
3389 break;
3390 case MSR_KVM_POLL_CONTROL: {
3391 env->poll_control_msr = msrs[i].data;
3392 break;
3393 }
3394 case MSR_CORE_PERF_FIXED_CTR_CTRL:
3395 env->msr_fixed_ctr_ctrl = msrs[i].data;
3396 break;
3397 case MSR_CORE_PERF_GLOBAL_CTRL:
3398 env->msr_global_ctrl = msrs[i].data;
3399 break;
3400 case MSR_CORE_PERF_GLOBAL_STATUS:
3401 env->msr_global_status = msrs[i].data;
3402 break;
3403 case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
3404 env->msr_global_ovf_ctrl = msrs[i].data;
3405 break;
3406 case MSR_CORE_PERF_FIXED_CTR0 ... MSR_CORE_PERF_FIXED_CTR0 + MAX_FIXED_COUNTERS - 1:
3407 env->msr_fixed_counters[index - MSR_CORE_PERF_FIXED_CTR0] = msrs[i].data;
3408 break;
3409 case MSR_P6_PERFCTR0 ... MSR_P6_PERFCTR0 + MAX_GP_COUNTERS - 1:
3410 env->msr_gp_counters[index - MSR_P6_PERFCTR0] = msrs[i].data;
3411 break;
3412 case MSR_P6_EVNTSEL0 ... MSR_P6_EVNTSEL0 + MAX_GP_COUNTERS - 1:
3413 env->msr_gp_evtsel[index - MSR_P6_EVNTSEL0] = msrs[i].data;
3414 break;
3415 case HV_X64_MSR_HYPERCALL:
3416 env->msr_hv_hypercall = msrs[i].data;
3417 break;
3418 case HV_X64_MSR_GUEST_OS_ID:
3419 env->msr_hv_guest_os_id = msrs[i].data;
3420 break;
3421 case HV_X64_MSR_APIC_ASSIST_PAGE:
3422 env->msr_hv_vapic = msrs[i].data;
3423 break;
3424 case HV_X64_MSR_REFERENCE_TSC:
3425 env->msr_hv_tsc = msrs[i].data;
3426 break;
3427 case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
3428 env->msr_hv_crash_params[index - HV_X64_MSR_CRASH_P0] = msrs[i].data;
3429 break;
3430 case HV_X64_MSR_VP_RUNTIME:
3431 env->msr_hv_runtime = msrs[i].data;
3432 break;
3433 case HV_X64_MSR_SCONTROL:
3434 env->msr_hv_synic_control = msrs[i].data;
3435 break;
3436 case HV_X64_MSR_SIEFP:
3437 env->msr_hv_synic_evt_page = msrs[i].data;
3438 break;
3439 case HV_X64_MSR_SIMP:
3440 env->msr_hv_synic_msg_page = msrs[i].data;
3441 break;
3442 case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15:
3443 env->msr_hv_synic_sint[index - HV_X64_MSR_SINT0] = msrs[i].data;
3444 break;
3445 case HV_X64_MSR_STIMER0_CONFIG:
3446 case HV_X64_MSR_STIMER1_CONFIG:
3447 case HV_X64_MSR_STIMER2_CONFIG:
3448 case HV_X64_MSR_STIMER3_CONFIG:
3449 env->msr_hv_stimer_config[(index - HV_X64_MSR_STIMER0_CONFIG)/2] =
3450 msrs[i].data;
3451 break;
3452 case HV_X64_MSR_STIMER0_COUNT:
3453 case HV_X64_MSR_STIMER1_COUNT:
3454 case HV_X64_MSR_STIMER2_COUNT:
3455 case HV_X64_MSR_STIMER3_COUNT:
3456 env->msr_hv_stimer_count[(index - HV_X64_MSR_STIMER0_COUNT)/2] =
3457 msrs[i].data;
3458 break;
3459 case HV_X64_MSR_REENLIGHTENMENT_CONTROL:
3460 env->msr_hv_reenlightenment_control = msrs[i].data;
3461 break;
3462 case HV_X64_MSR_TSC_EMULATION_CONTROL:
3463 env->msr_hv_tsc_emulation_control = msrs[i].data;
3464 break;
3465 case HV_X64_MSR_TSC_EMULATION_STATUS:
3466 env->msr_hv_tsc_emulation_status = msrs[i].data;
3467 break;
3468 case MSR_MTRRdefType:
3469 env->mtrr_deftype = msrs[i].data;
3470 break;
3471 case MSR_MTRRfix64K_00000:
3472 env->mtrr_fixed[0] = msrs[i].data;
3473 break;
3474 case MSR_MTRRfix16K_80000:
3475 env->mtrr_fixed[1] = msrs[i].data;
3476 break;
3477 case MSR_MTRRfix16K_A0000:
3478 env->mtrr_fixed[2] = msrs[i].data;
3479 break;
3480 case MSR_MTRRfix4K_C0000:
3481 env->mtrr_fixed[3] = msrs[i].data;
3482 break;
3483 case MSR_MTRRfix4K_C8000:
3484 env->mtrr_fixed[4] = msrs[i].data;
3485 break;
3486 case MSR_MTRRfix4K_D0000:
3487 env->mtrr_fixed[5] = msrs[i].data;
3488 break;
3489 case MSR_MTRRfix4K_D8000:
3490 env->mtrr_fixed[6] = msrs[i].data;
3491 break;
3492 case MSR_MTRRfix4K_E0000:
3493 env->mtrr_fixed[7] = msrs[i].data;
3494 break;
3495 case MSR_MTRRfix4K_E8000:
3496 env->mtrr_fixed[8] = msrs[i].data;
3497 break;
3498 case MSR_MTRRfix4K_F0000:
3499 env->mtrr_fixed[9] = msrs[i].data;
3500 break;
3501 case MSR_MTRRfix4K_F8000:
3502 env->mtrr_fixed[10] = msrs[i].data;
3503 break;
3504 case MSR_MTRRphysBase(0) ... MSR_MTRRphysMask(MSR_MTRRcap_VCNT - 1):
3505 if (index & 1) {
3506 env->mtrr_var[MSR_MTRRphysIndex(index)].mask = msrs[i].data |
3507 mtrr_top_bits;
3508 } else {
3509 env->mtrr_var[MSR_MTRRphysIndex(index)].base = msrs[i].data;
3510 }
3511 break;
3512 case MSR_IA32_SPEC_CTRL:
3513 env->spec_ctrl = msrs[i].data;
3514 break;
3515 case MSR_IA32_TSX_CTRL:
3516 env->tsx_ctrl = msrs[i].data;
3517 break;
3518 case MSR_VIRT_SSBD:
3519 env->virt_ssbd = msrs[i].data;
3520 break;
3521 case MSR_IA32_RTIT_CTL:
3522 env->msr_rtit_ctrl = msrs[i].data;
3523 break;
3524 case MSR_IA32_RTIT_STATUS:
3525 env->msr_rtit_status = msrs[i].data;
3526 break;
3527 case MSR_IA32_RTIT_OUTPUT_BASE:
3528 env->msr_rtit_output_base = msrs[i].data;
3529 break;
3530 case MSR_IA32_RTIT_OUTPUT_MASK:
3531 env->msr_rtit_output_mask = msrs[i].data;
3532 break;
3533 case MSR_IA32_RTIT_CR3_MATCH:
3534 env->msr_rtit_cr3_match = msrs[i].data;
3535 break;
3536 case MSR_IA32_RTIT_ADDR0_A ... MSR_IA32_RTIT_ADDR3_B:
3537 env->msr_rtit_addrs[index - MSR_IA32_RTIT_ADDR0_A] = msrs[i].data;
3538 break;
3539 }
3540 }
3541
3542 return 0;
3543 }
3544
3545 static int kvm_put_mp_state(X86CPU *cpu)
3546 {
3547 struct kvm_mp_state mp_state = { .mp_state = cpu->env.mp_state };
3548
3549 return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MP_STATE, &mp_state);
3550 }
3551
3552 static int kvm_get_mp_state(X86CPU *cpu)
3553 {
3554 CPUState *cs = CPU(cpu);
3555 CPUX86State *env = &cpu->env;
3556 struct kvm_mp_state mp_state;
3557 int ret;
3558
3559 ret = kvm_vcpu_ioctl(cs, KVM_GET_MP_STATE, &mp_state);
3560 if (ret < 0) {
3561 return ret;
3562 }
3563 env->mp_state = mp_state.mp_state;
3564 if (kvm_irqchip_in_kernel()) {
3565 cs->halted = (mp_state.mp_state == KVM_MP_STATE_HALTED);
3566 }
3567 return 0;
3568 }
3569
3570 static int kvm_get_apic(X86CPU *cpu)
3571 {
3572 DeviceState *apic = cpu->apic_state;
3573 struct kvm_lapic_state kapic;
3574 int ret;
3575
3576 if (apic && kvm_irqchip_in_kernel()) {
3577 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_LAPIC, &kapic);
3578 if (ret < 0) {
3579 return ret;
3580 }
3581
3582 kvm_get_apic_state(apic, &kapic);
3583 }
3584 return 0;
3585 }
3586
3587 static int kvm_put_vcpu_events(X86CPU *cpu, int level)
3588 {
3589 CPUState *cs = CPU(cpu);
3590 CPUX86State *env = &cpu->env;
3591 struct kvm_vcpu_events events = {};
3592
3593 if (!kvm_has_vcpu_events()) {
3594 return 0;
3595 }
3596
3597 events.flags = 0;
3598
3599 if (has_exception_payload) {
3600 events.flags |= KVM_VCPUEVENT_VALID_PAYLOAD;
3601 events.exception.pending = env->exception_pending;
3602 events.exception_has_payload = env->exception_has_payload;
3603 events.exception_payload = env->exception_payload;
3604 }
3605 events.exception.nr = env->exception_nr;
3606 events.exception.injected = env->exception_injected;
3607 events.exception.has_error_code = env->has_error_code;
3608 events.exception.error_code = env->error_code;
3609
3610 events.interrupt.injected = (env->interrupt_injected >= 0);
3611 events.interrupt.nr = env->interrupt_injected;
3612 events.interrupt.soft = env->soft_interrupt;
3613
3614 events.nmi.injected = env->nmi_injected;
3615 events.nmi.pending = env->nmi_pending;
3616 events.nmi.masked = !!(env->hflags2 & HF2_NMI_MASK);
3617
3618 events.sipi_vector = env->sipi_vector;
3619
3620 if (has_msr_smbase) {
3621 events.smi.smm = !!(env->hflags & HF_SMM_MASK);
3622 events.smi.smm_inside_nmi = !!(env->hflags2 & HF2_SMM_INSIDE_NMI_MASK);
3623 if (kvm_irqchip_in_kernel()) {
3624 /* As soon as these are moved to the kernel, remove them
3625 * from cs->interrupt_request.
3626 */
3627 events.smi.pending = cs->interrupt_request & CPU_INTERRUPT_SMI;
3628 events.smi.latched_init = cs->interrupt_request & CPU_INTERRUPT_INIT;
3629 cs->interrupt_request &= ~(CPU_INTERRUPT_INIT | CPU_INTERRUPT_SMI);
3630 } else {
3631 /* Keep these in cs->interrupt_request. */
3632 events.smi.pending = 0;
3633 events.smi.latched_init = 0;
3634 }
3635 /* Stop SMI delivery on old machine types to avoid a reboot
3636 * on an inward migration of an old VM.
3637 */
3638 if (!cpu->kvm_no_smi_migration) {
3639 events.flags |= KVM_VCPUEVENT_VALID_SMM;
3640 }
3641 }
3642
3643 if (level >= KVM_PUT_RESET_STATE) {
3644 events.flags |= KVM_VCPUEVENT_VALID_NMI_PENDING;
3645 if (env->mp_state == KVM_MP_STATE_SIPI_RECEIVED) {
3646 events.flags |= KVM_VCPUEVENT_VALID_SIPI_VECTOR;
3647 }
3648 }
3649
3650 return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_VCPU_EVENTS, &events);
3651 }
3652
3653 static int kvm_get_vcpu_events(X86CPU *cpu)
3654 {
3655 CPUX86State *env = &cpu->env;
3656 struct kvm_vcpu_events events;
3657 int ret;
3658
3659 if (!kvm_has_vcpu_events()) {
3660 return 0;
3661 }
3662
3663 memset(&events, 0, sizeof(events));
3664 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_VCPU_EVENTS, &events);
3665 if (ret < 0) {
3666 return ret;
3667 }
3668
3669 if (events.flags & KVM_VCPUEVENT_VALID_PAYLOAD) {
3670 env->exception_pending = events.exception.pending;
3671 env->exception_has_payload = events.exception_has_payload;
3672 env->exception_payload = events.exception_payload;
3673 } else {
3674 env->exception_pending = 0;
3675 env->exception_has_payload = false;
3676 }
3677 env->exception_injected = events.exception.injected;
3678 env->exception_nr =
3679 (env->exception_pending || env->exception_injected) ?
3680 events.exception.nr : -1;
3681 env->has_error_code = events.exception.has_error_code;
3682 env->error_code = events.exception.error_code;
3683
3684 env->interrupt_injected =
3685 events.interrupt.injected ? events.interrupt.nr : -1;
3686 env->soft_interrupt = events.interrupt.soft;
3687
3688 env->nmi_injected = events.nmi.injected;
3689 env->nmi_pending = events.nmi.pending;
3690 if (events.nmi.masked) {
3691 env->hflags2 |= HF2_NMI_MASK;
3692 } else {
3693 env->hflags2 &= ~HF2_NMI_MASK;
3694 }
3695
3696 if (events.flags & KVM_VCPUEVENT_VALID_SMM) {
3697 if (events.smi.smm) {
3698 env->hflags |= HF_SMM_MASK;
3699 } else {
3700 env->hflags &= ~HF_SMM_MASK;
3701 }
3702 if (events.smi.pending) {
3703 cpu_interrupt(CPU(cpu), CPU_INTERRUPT_SMI);
3704 } else {
3705 cpu_reset_interrupt(CPU(cpu), CPU_INTERRUPT_SMI);
3706 }
3707 if (events.smi.smm_inside_nmi) {
3708 env->hflags2 |= HF2_SMM_INSIDE_NMI_MASK;
3709 } else {
3710 env->hflags2 &= ~HF2_SMM_INSIDE_NMI_MASK;
3711 }
3712 if (events.smi.latched_init) {
3713 cpu_interrupt(CPU(cpu), CPU_INTERRUPT_INIT);
3714 } else {
3715 cpu_reset_interrupt(CPU(cpu), CPU_INTERRUPT_INIT);
3716 }
3717 }
3718
3719 env->sipi_vector = events.sipi_vector;
3720
3721 return 0;
3722 }
3723
3724 static int kvm_guest_debug_workarounds(X86CPU *cpu)
3725 {
3726 CPUState *cs = CPU(cpu);
3727 CPUX86State *env = &cpu->env;
3728 int ret = 0;
3729 unsigned long reinject_trap = 0;
3730
3731 if (!kvm_has_vcpu_events()) {
3732 if (env->exception_nr == EXCP01_DB) {
3733 reinject_trap = KVM_GUESTDBG_INJECT_DB;
3734 } else if (env->exception_injected == EXCP03_INT3) {
3735 reinject_trap = KVM_GUESTDBG_INJECT_BP;
3736 }
3737 kvm_reset_exception(env);
3738 }
3739
3740 /*
3741 * Kernels before KVM_CAP_X86_ROBUST_SINGLESTEP overwrote flags.TF
3742 * injected via SET_GUEST_DEBUG while updating GP regs. Work around this
3743 * by updating the debug state once again if single-stepping is on.
3744 * Another reason to call kvm_update_guest_debug here is a pending debug
3745 * trap raise by the guest. On kernels without SET_VCPU_EVENTS we have to
3746 * reinject them via SET_GUEST_DEBUG.
3747 */
3748 if (reinject_trap ||
3749 (!kvm_has_robust_singlestep() && cs->singlestep_enabled)) {
3750 ret = kvm_update_guest_debug(cs, reinject_trap);
3751 }
3752 return ret;
3753 }
3754
3755 static int kvm_put_debugregs(X86CPU *cpu)
3756 {
3757 CPUX86State *env = &cpu->env;
3758 struct kvm_debugregs dbgregs;
3759 int i;
3760
3761 if (!kvm_has_debugregs()) {
3762 return 0;
3763 }
3764
3765 memset(&dbgregs, 0, sizeof(dbgregs));
3766 for (i = 0; i < 4; i++) {
3767 dbgregs.db[i] = env->dr[i];
3768 }
3769 dbgregs.dr6 = env->dr[6];
3770 dbgregs.dr7 = env->dr[7];
3771 dbgregs.flags = 0;
3772
3773 return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_DEBUGREGS, &dbgregs);
3774 }
3775
3776 static int kvm_get_debugregs(X86CPU *cpu)
3777 {
3778 CPUX86State *env = &cpu->env;
3779 struct kvm_debugregs dbgregs;
3780 int i, ret;
3781
3782 if (!kvm_has_debugregs()) {
3783 return 0;
3784 }
3785
3786 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_DEBUGREGS, &dbgregs);
3787 if (ret < 0) {
3788 return ret;
3789 }
3790 for (i = 0; i < 4; i++) {
3791 env->dr[i] = dbgregs.db[i];
3792 }
3793 env->dr[4] = env->dr[6] = dbgregs.dr6;
3794 env->dr[5] = env->dr[7] = dbgregs.dr7;
3795
3796 return 0;
3797 }
3798
3799 static int kvm_put_nested_state(X86CPU *cpu)
3800 {
3801 CPUX86State *env = &cpu->env;
3802 int max_nested_state_len = kvm_max_nested_state_length();
3803
3804 if (!env->nested_state) {
3805 return 0;
3806 }
3807
3808 assert(env->nested_state->size <= max_nested_state_len);
3809 return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_NESTED_STATE, env->nested_state);
3810 }
3811
3812 static int kvm_get_nested_state(X86CPU *cpu)
3813 {
3814 CPUX86State *env = &cpu->env;
3815 int max_nested_state_len = kvm_max_nested_state_length();
3816 int ret;
3817
3818 if (!env->nested_state) {
3819 return 0;
3820 }
3821
3822 /*
3823 * It is possible that migration restored a smaller size into
3824 * nested_state->hdr.size than what our kernel support.
3825 * We preserve migration origin nested_state->hdr.size for
3826 * call to KVM_SET_NESTED_STATE but wish that our next call
3827 * to KVM_GET_NESTED_STATE will use max size our kernel support.
3828 */
3829 env->nested_state->size = max_nested_state_len;
3830
3831 ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_NESTED_STATE, env->nested_state);
3832 if (ret < 0) {
3833 return ret;
3834 }
3835
3836 if (env->nested_state->flags & KVM_STATE_NESTED_GUEST_MODE) {
3837 env->hflags |= HF_GUEST_MASK;
3838 } else {
3839 env->hflags &= ~HF_GUEST_MASK;
3840 }
3841
3842 return ret;
3843 }
3844
3845 int kvm_arch_put_registers(CPUState *cpu, int level)
3846 {
3847 X86CPU *x86_cpu = X86_CPU(cpu);
3848 int ret;
3849
3850 assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
3851
3852 if (level >= KVM_PUT_RESET_STATE) {
3853 ret = kvm_put_nested_state(x86_cpu);
3854 if (ret < 0) {
3855 return ret;
3856 }
3857
3858 ret = kvm_put_msr_feature_control(x86_cpu);
3859 if (ret < 0) {
3860 return ret;
3861 }
3862 }
3863
3864 if (level == KVM_PUT_FULL_STATE) {
3865 /* We don't check for kvm_arch_set_tsc_khz() errors here,
3866 * because TSC frequency mismatch shouldn't abort migration,
3867 * unless the user explicitly asked for a more strict TSC
3868 * setting (e.g. using an explicit "tsc-freq" option).
3869 */
3870 kvm_arch_set_tsc_khz(cpu);
3871 }
3872
3873 ret = kvm_getput_regs(x86_cpu, 1);
3874 if (ret < 0) {
3875 return ret;
3876 }
3877 ret = kvm_put_xsave(x86_cpu);
3878 if (ret < 0) {
3879 return ret;
3880 }
3881 ret = kvm_put_xcrs(x86_cpu);
3882 if (ret < 0) {
3883 return ret;
3884 }
3885 ret = kvm_put_sregs(x86_cpu);
3886 if (ret < 0) {
3887 return ret;
3888 }
3889 /* must be before kvm_put_msrs */
3890 ret = kvm_inject_mce_oldstyle(x86_cpu);
3891 if (ret < 0) {
3892 return ret;
3893 }
3894 ret = kvm_put_msrs(x86_cpu, level);
3895 if (ret < 0) {
3896 return ret;
3897 }
3898 ret = kvm_put_vcpu_events(x86_cpu, level);
3899 if (ret < 0) {
3900 return ret;
3901 }
3902 if (level >= KVM_PUT_RESET_STATE) {
3903 ret = kvm_put_mp_state(x86_cpu);
3904 if (ret < 0) {
3905 return ret;
3906 }
3907 }
3908
3909 ret = kvm_put_tscdeadline_msr(x86_cpu);
3910 if (ret < 0) {
3911 return ret;
3912 }
3913 ret = kvm_put_debugregs(x86_cpu);
3914 if (ret < 0) {
3915 return ret;
3916 }
3917 /* must be last */
3918 ret = kvm_guest_debug_workarounds(x86_cpu);
3919 if (ret < 0) {
3920 return ret;
3921 }
3922 return 0;
3923 }
3924
3925 int kvm_arch_get_registers(CPUState *cs)
3926 {
3927 X86CPU *cpu = X86_CPU(cs);
3928 int ret;
3929
3930 assert(cpu_is_stopped(cs) || qemu_cpu_is_self(cs));
3931
3932 ret = kvm_get_vcpu_events(cpu);
3933 if (ret < 0) {
3934 goto out;
3935 }
3936 /*
3937 * KVM_GET_MPSTATE can modify CS and RIP, call it before
3938 * KVM_GET_REGS and KVM_GET_SREGS.
3939 */
3940 ret = kvm_get_mp_state(cpu);
3941 if (ret < 0) {
3942 goto out;
3943 }
3944 ret = kvm_getput_regs(cpu, 0);
3945 if (ret < 0) {
3946 goto out;
3947 }
3948 ret = kvm_get_xsave(cpu);
3949 if (ret < 0) {
3950 goto out;
3951 }
3952 ret = kvm_get_xcrs(cpu);
3953 if (ret < 0) {
3954 goto out;
3955 }
3956 ret = kvm_get_sregs(cpu);
3957 if (ret < 0) {
3958 goto out;
3959 }
3960 ret = kvm_get_msrs(cpu);
3961 if (ret < 0) {
3962 goto out;
3963 }
3964 ret = kvm_get_apic(cpu);
3965 if (ret < 0) {
3966 goto out;
3967 }
3968 ret = kvm_get_debugregs(cpu);
3969 if (ret < 0) {
3970 goto out;
3971 }
3972 ret = kvm_get_nested_state(cpu);
3973 if (ret < 0) {
3974 goto out;
3975 }
3976 ret = 0;
3977 out:
3978 cpu_sync_bndcs_hflags(&cpu->env);
3979 return ret;
3980 }
3981
3982 void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run)
3983 {
3984 X86CPU *x86_cpu = X86_CPU(cpu);
3985 CPUX86State *env = &x86_cpu->env;
3986 int ret;
3987
3988 /* Inject NMI */
3989 if (cpu->interrupt_request & (CPU_INTERRUPT_NMI | CPU_INTERRUPT_SMI)) {
3990 if (cpu->interrupt_request & CPU_INTERRUPT_NMI) {
3991 qemu_mutex_lock_iothread();
3992 cpu->interrupt_request &= ~CPU_INTERRUPT_NMI;
3993 qemu_mutex_unlock_iothread();
3994 DPRINTF("injected NMI\n");
3995 ret = kvm_vcpu_ioctl(cpu, KVM_NMI);
3996 if (ret < 0) {
3997 fprintf(stderr, "KVM: injection failed, NMI lost (%s)\n",
3998 strerror(-ret));
3999 }
4000 }
4001 if (cpu->interrupt_request & CPU_INTERRUPT_SMI) {
4002 qemu_mutex_lock_iothread();
4003 cpu->interrupt_request &= ~CPU_INTERRUPT_SMI;
4004 qemu_mutex_unlock_iothread();
4005 DPRINTF("injected SMI\n");
4006 ret = kvm_vcpu_ioctl(cpu, KVM_SMI);
4007 if (ret < 0) {
4008 fprintf(stderr, "KVM: injection failed, SMI lost (%s)\n",
4009 strerror(-ret));
4010 }
4011 }
4012 }
4013
4014 if (!kvm_pic_in_kernel()) {
4015 qemu_mutex_lock_iothread();
4016 }
4017
4018 /* Force the VCPU out of its inner loop to process any INIT requests
4019 * or (for userspace APIC, but it is cheap to combine the checks here)
4020 * pending TPR access reports.
4021 */
4022 if (cpu->interrupt_request & (CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR)) {
4023 if ((cpu->interrupt_request & CPU_INTERRUPT_INIT) &&
4024 !(env->hflags & HF_SMM_MASK)) {
4025 cpu->exit_request = 1;
4026 }
4027 if (cpu->interrupt_request & CPU_INTERRUPT_TPR) {
4028 cpu->exit_request = 1;
4029 }
4030 }
4031
4032 if (!kvm_pic_in_kernel()) {
4033 /* Try to inject an interrupt if the guest can accept it */
4034 if (run->ready_for_interrupt_injection &&
4035 (cpu->interrupt_request & CPU_INTERRUPT_HARD) &&
4036 (env->eflags & IF_MASK)) {
4037 int irq;
4038
4039 cpu->interrupt_request &= ~CPU_INTERRUPT_HARD;
4040 irq = cpu_get_pic_interrupt(env);
4041 if (irq >= 0) {
4042 struct kvm_interrupt intr;
4043
4044 intr.irq = irq;
4045 DPRINTF("injected interrupt %d\n", irq);
4046 ret = kvm_vcpu_ioctl(cpu, KVM_INTERRUPT, &intr);
4047 if (ret < 0) {
4048 fprintf(stderr,
4049 "KVM: injection failed, interrupt lost (%s)\n",
4050 strerror(-ret));
4051 }
4052 }
4053 }
4054
4055 /* If we have an interrupt but the guest is not ready to receive an
4056 * interrupt, request an interrupt window exit. This will
4057 * cause a return to userspace as soon as the guest is ready to
4058 * receive interrupts. */
4059 if ((cpu->interrupt_request & CPU_INTERRUPT_HARD)) {
4060 run->request_interrupt_window = 1;
4061 } else {
4062 run->request_interrupt_window = 0;
4063 }
4064
4065 DPRINTF("setting tpr\n");
4066 run->cr8 = cpu_get_apic_tpr(x86_cpu->apic_state);
4067
4068 qemu_mutex_unlock_iothread();
4069 }
4070 }
4071
4072 MemTxAttrs kvm_arch_post_run(CPUState *cpu, struct kvm_run *run)
4073 {
4074 X86CPU *x86_cpu = X86_CPU(cpu);
4075 CPUX86State *env = &x86_cpu->env;
4076
4077 if (run->flags & KVM_RUN_X86_SMM) {
4078 env->hflags |= HF_SMM_MASK;
4079 } else {
4080 env->hflags &= ~HF_SMM_MASK;
4081 }
4082 if (run->if_flag) {
4083 env->eflags |= IF_MASK;
4084 } else {
4085 env->eflags &= ~IF_MASK;
4086 }
4087
4088 /* We need to protect the apic state against concurrent accesses from
4089 * different threads in case the userspace irqchip is used. */
4090 if (!kvm_irqchip_in_kernel()) {
4091 qemu_mutex_lock_iothread();
4092 }
4093 cpu_set_apic_tpr(x86_cpu->apic_state, run->cr8);
4094 cpu_set_apic_base(x86_cpu->apic_state, run->apic_base);
4095 if (!kvm_irqchip_in_kernel()) {
4096 qemu_mutex_unlock_iothread();
4097 }
4098 return cpu_get_mem_attrs(env);
4099 }
4100
4101 int kvm_arch_process_async_events(CPUState *cs)
4102 {
4103 X86CPU *cpu = X86_CPU(cs);
4104 CPUX86State *env = &cpu->env;
4105
4106 if (cs->interrupt_request & CPU_INTERRUPT_MCE) {
4107 /* We must not raise CPU_INTERRUPT_MCE if it's not supported. */
4108 assert(env->mcg_cap);
4109
4110 cs->interrupt_request &= ~CPU_INTERRUPT_MCE;
4111
4112 kvm_cpu_synchronize_state(cs);
4113
4114 if (env->exception_nr == EXCP08_DBLE) {
4115 /* this means triple fault */
4116 qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET);
4117 cs->exit_request = 1;
4118 return 0;
4119 }
4120 kvm_queue_exception(env, EXCP12_MCHK, 0, 0);
4121 env->has_error_code = 0;
4122
4123 cs->halted = 0;
4124 if (kvm_irqchip_in_kernel() && env->mp_state == KVM_MP_STATE_HALTED) {
4125 env->mp_state = KVM_MP_STATE_RUNNABLE;
4126 }
4127 }
4128
4129 if ((cs->interrupt_request & CPU_INTERRUPT_INIT) &&
4130 !(env->hflags & HF_SMM_MASK)) {
4131 kvm_cpu_synchronize_state(cs);
4132 do_cpu_init(cpu);
4133 }
4134
4135 if (kvm_irqchip_in_kernel()) {
4136 return 0;
4137 }
4138
4139 if (cs->interrupt_request & CPU_INTERRUPT_POLL) {
4140 cs->interrupt_request &= ~CPU_INTERRUPT_POLL;
4141 apic_poll_irq(cpu->apic_state);
4142 }
4143 if (((cs->interrupt_request & CPU_INTERRUPT_HARD) &&
4144 (env->eflags & IF_MASK)) ||
4145 (cs->interrupt_request & CPU_INTERRUPT_NMI)) {
4146 cs->halted = 0;
4147 }
4148 if (cs->interrupt_request & CPU_INTERRUPT_SIPI) {
4149 kvm_cpu_synchronize_state(cs);
4150 do_cpu_sipi(cpu);
4151 }
4152 if (cs->interrupt_request & CPU_INTERRUPT_TPR) {
4153 cs->interrupt_request &= ~CPU_INTERRUPT_TPR;
4154 kvm_cpu_synchronize_state(cs);
4155 apic_handle_tpr_access_report(cpu->apic_state, env->eip,
4156 env->tpr_access_type);
4157 }
4158
4159 return cs->halted;
4160 }
4161
4162 static int kvm_handle_halt(X86CPU *cpu)
4163 {
4164 CPUState *cs = CPU(cpu);
4165 CPUX86State *env = &cpu->env;
4166
4167 if (!((cs->interrupt_request & CPU_INTERRUPT_HARD) &&
4168 (env->eflags & IF_MASK)) &&
4169 !(cs->interrupt_request & CPU_INTERRUPT_NMI)) {
4170 cs->halted = 1;
4171 return EXCP_HLT;
4172 }
4173
4174 return 0;
4175 }
4176
4177 static int kvm_handle_tpr_access(X86CPU *cpu)
4178 {
4179 CPUState *cs = CPU(cpu);
4180 struct kvm_run *run = cs->kvm_run;
4181
4182 apic_handle_tpr_access_report(cpu->apic_state, run->tpr_access.rip,
4183 run->tpr_access.is_write ? TPR_ACCESS_WRITE
4184 : TPR_ACCESS_READ);
4185 return 1;
4186 }
4187
4188 int kvm_arch_insert_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
4189 {
4190 static const uint8_t int3 = 0xcc;
4191
4192 if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 1, 0) ||
4193 cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&int3, 1, 1)) {
4194 return -EINVAL;
4195 }
4196 return 0;
4197 }
4198
4199 int kvm_arch_remove_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
4200 {
4201 uint8_t int3;
4202
4203 if (cpu_memory_rw_debug(cs, bp->pc, &int3, 1, 0) || int3 != 0xcc ||
4204 cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 1, 1)) {
4205 return -EINVAL;
4206 }
4207 return 0;
4208 }
4209
4210 static struct {
4211 target_ulong addr;
4212 int len;
4213 int type;
4214 } hw_breakpoint[4];
4215
4216 static int nb_hw_breakpoint;
4217
4218 static int find_hw_breakpoint(target_ulong addr, int len, int type)
4219 {
4220 int n;
4221
4222 for (n = 0; n < nb_hw_breakpoint; n++) {
4223 if (hw_breakpoint[n].addr == addr && hw_breakpoint[n].type == type &&
4224 (hw_breakpoint[n].len == len || len == -1)) {
4225 return n;
4226 }
4227 }
4228 return -1;
4229 }
4230
4231 int kvm_arch_insert_hw_breakpoint(target_ulong addr,
4232 target_ulong len, int type)
4233 {
4234 switch (type) {
4235 case GDB_BREAKPOINT_HW:
4236 len = 1;
4237 break;
4238 case GDB_WATCHPOINT_WRITE:
4239 case GDB_WATCHPOINT_ACCESS:
4240 switch (len) {
4241 case 1:
4242 break;
4243 case 2:
4244 case 4:
4245 case 8:
4246 if (addr & (len - 1)) {
4247 return -EINVAL;
4248 }
4249 break;
4250 default:
4251 return -EINVAL;
4252 }
4253 break;
4254 default:
4255 return -ENOSYS;
4256 }
4257
4258 if (nb_hw_breakpoint == 4) {
4259 return -ENOBUFS;
4260 }
4261 if (find_hw_breakpoint(addr, len, type) >= 0) {
4262 return -EEXIST;
4263 }
4264 hw_breakpoint[nb_hw_breakpoint].addr = addr;
4265 hw_breakpoint[nb_hw_breakpoint].len = len;
4266 hw_breakpoint[nb_hw_breakpoint].type = type;
4267 nb_hw_breakpoint++;
4268
4269 return 0;
4270 }
4271
4272 int kvm_arch_remove_hw_breakpoint(target_ulong addr,
4273 target_ulong len, int type)
4274 {
4275 int n;
4276
4277 n = find_hw_breakpoint(addr, (type == GDB_BREAKPOINT_HW) ? 1 : len, type);
4278 if (n < 0) {
4279 return -ENOENT;
4280 }
4281 nb_hw_breakpoint--;
4282 hw_breakpoint[n] = hw_breakpoint[nb_hw_breakpoint];
4283
4284 return 0;
4285 }
4286
4287 void kvm_arch_remove_all_hw_breakpoints(void)
4288 {
4289 nb_hw_breakpoint = 0;
4290 }
4291
4292 static CPUWatchpoint hw_watchpoint;
4293
4294 static int kvm_handle_debug(X86CPU *cpu,
4295 struct kvm_debug_exit_arch *arch_info)
4296 {
4297 CPUState *cs = CPU(cpu);
4298 CPUX86State *env = &cpu->env;
4299 int ret = 0;
4300 int n;
4301
4302 if (arch_info->exception == EXCP01_DB) {
4303 if (arch_info->dr6 & DR6_BS) {
4304 if (cs->singlestep_enabled) {
4305 ret = EXCP_DEBUG;
4306 }
4307 } else {
4308 for (n = 0; n < 4; n++) {
4309 if (arch_info->dr6 & (1 << n)) {
4310 switch ((arch_info->dr7 >> (16 + n*4)) & 0x3) {
4311 case 0x0:
4312 ret = EXCP_DEBUG;
4313 break;
4314 case 0x1:
4315 ret = EXCP_DEBUG;
4316 cs->watchpoint_hit = &hw_watchpoint;
4317 hw_watchpoint.vaddr = hw_breakpoint[n].addr;
4318 hw_watchpoint.flags = BP_MEM_WRITE;
4319 break;
4320 case 0x3:
4321 ret = EXCP_DEBUG;
4322 cs->watchpoint_hit = &hw_watchpoint;
4323 hw_watchpoint.vaddr = hw_breakpoint[n].addr;
4324 hw_watchpoint.flags = BP_MEM_ACCESS;
4325 break;
4326 }
4327 }
4328 }
4329 }
4330 } else if (kvm_find_sw_breakpoint(cs, arch_info->pc)) {
4331 ret = EXCP_DEBUG;
4332 }
4333 if (ret == 0) {
4334 cpu_synchronize_state(cs);
4335 assert(env->exception_nr == -1);
4336
4337 /* pass to guest */
4338 kvm_queue_exception(env, arch_info->exception,
4339 arch_info->exception == EXCP01_DB,
4340 arch_info->dr6);
4341 env->has_error_code = 0;
4342 }
4343
4344 return ret;
4345 }
4346
4347 void kvm_arch_update_guest_debug(CPUState *cpu, struct kvm_guest_debug *dbg)
4348 {
4349 const uint8_t type_code[] = {
4350 [GDB_BREAKPOINT_HW] = 0x0,
4351 [GDB_WATCHPOINT_WRITE] = 0x1,
4352 [GDB_WATCHPOINT_ACCESS] = 0x3
4353 };
4354 const uint8_t len_code[] = {
4355 [1] = 0x0, [2] = 0x1, [4] = 0x3, [8] = 0x2
4356 };
4357 int n;
4358
4359 if (kvm_sw_breakpoints_active(cpu)) {
4360 dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP;
4361 }
4362 if (nb_hw_breakpoint > 0) {
4363 dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW_BP;
4364 dbg->arch.debugreg[7] = 0x0600;
4365 for (n = 0; n < nb_hw_breakpoint; n++) {
4366 dbg->arch.debugreg[n] = hw_breakpoint[n].addr;
4367 dbg->arch.debugreg[7] |= (2 << (n * 2)) |
4368 (type_code[hw_breakpoint[n].type] << (16 + n*4)) |
4369 ((uint32_t)len_code[hw_breakpoint[n].len] << (18 + n*4));
4370 }
4371 }
4372 }
4373
4374 static bool host_supports_vmx(void)
4375 {
4376 uint32_t ecx, unused;
4377
4378 host_cpuid(1, 0, &unused, &unused, &ecx, &unused);
4379 return ecx & CPUID_EXT_VMX;
4380 }
4381
4382 #define VMX_INVALID_GUEST_STATE 0x80000021
4383
4384 int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
4385 {
4386 X86CPU *cpu = X86_CPU(cs);
4387 uint64_t code;
4388 int ret;
4389
4390 switch (run->exit_reason) {
4391 case KVM_EXIT_HLT:
4392 DPRINTF("handle_hlt\n");
4393 qemu_mutex_lock_iothread();
4394 ret = kvm_handle_halt(cpu);
4395 qemu_mutex_unlock_iothread();
4396 break;
4397 case KVM_EXIT_SET_TPR:
4398 ret = 0;
4399 break;
4400 case KVM_EXIT_TPR_ACCESS:
4401 qemu_mutex_lock_iothread();
4402 ret = kvm_handle_tpr_access(cpu);
4403 qemu_mutex_unlock_iothread();
4404 break;
4405 case KVM_EXIT_FAIL_ENTRY:
4406 code = run->fail_entry.hardware_entry_failure_reason;
4407 fprintf(stderr, "KVM: entry failed, hardware error 0x%" PRIx64 "\n",
4408 code);
4409 if (host_supports_vmx() && code == VMX_INVALID_GUEST_STATE) {
4410 fprintf(stderr,
4411 "\nIf you're running a guest on an Intel machine without "
4412 "unrestricted mode\n"
4413 "support, the failure can be most likely due to the guest "
4414 "entering an invalid\n"
4415 "state for Intel VT. For example, the guest maybe running "
4416 "in big real mode\n"
4417 "which is not supported on less recent Intel processors."
4418 "\n\n");
4419 }
4420 ret = -1;
4421 break;
4422 case KVM_EXIT_EXCEPTION:
4423 fprintf(stderr, "KVM: exception %d exit (error code 0x%x)\n",
4424 run->ex.exception, run->ex.error_code);
4425 ret = -1;
4426 break;
4427 case KVM_EXIT_DEBUG:
4428 DPRINTF("kvm_exit_debug\n");
4429 qemu_mutex_lock_iothread();
4430 ret = kvm_handle_debug(cpu, &run->debug.arch);
4431 qemu_mutex_unlock_iothread();
4432 break;
4433 case KVM_EXIT_HYPERV:
4434 ret = kvm_hv_handle_exit(cpu, &run->hyperv);
4435 break;
4436 case KVM_EXIT_IOAPIC_EOI:
4437 ioapic_eoi_broadcast(run->eoi.vector);
4438 ret = 0;
4439 break;
4440 default:
4441 fprintf(stderr, "KVM: unknown exit reason %d\n", run->exit_reason);
4442 ret = -1;
4443 break;
4444 }
4445
4446 return ret;
4447 }
4448
4449 bool kvm_arch_stop_on_emulation_error(CPUState *cs)
4450 {
4451 X86CPU *cpu = X86_CPU(cs);
4452 CPUX86State *env = &cpu->env;
4453
4454 kvm_cpu_synchronize_state(cs);
4455 return !(env->cr[0] & CR0_PE_MASK) ||
4456 ((env->segs[R_CS].selector & 3) != 3);
4457 }
4458
4459 void kvm_arch_init_irq_routing(KVMState *s)
4460 {
4461 if (!kvm_check_extension(s, KVM_CAP_IRQ_ROUTING)) {
4462 /* If kernel can't do irq routing, interrupt source
4463 * override 0->2 cannot be set up as required by HPET.
4464 * So we have to disable it.
4465 */
4466 no_hpet = 1;
4467 }
4468 /* We know at this point that we're using the in-kernel
4469 * irqchip, so we can use irqfds, and on x86 we know
4470 * we can use msi via irqfd and GSI routing.
4471 */
4472 kvm_msi_via_irqfd_allowed = true;
4473 kvm_gsi_routing_allowed = true;
4474
4475 if (kvm_irqchip_is_split()) {
4476 int i;
4477
4478 /* If the ioapic is in QEMU and the lapics are in KVM, reserve
4479 MSI routes for signaling interrupts to the local apics. */
4480 for (i = 0; i < IOAPIC_NUM_PINS; i++) {
4481 if (kvm_irqchip_add_msi_route(s, 0, NULL) < 0) {
4482 error_report("Could not enable split IRQ mode.");
4483 exit(1);
4484 }
4485 }
4486 }
4487 }
4488
4489 int kvm_arch_irqchip_create(MachineState *ms, KVMState *s)
4490 {
4491 int ret;
4492 if (machine_kernel_irqchip_split(ms)) {
4493 ret = kvm_vm_enable_cap(s, KVM_CAP_SPLIT_IRQCHIP, 0, 24);
4494 if (ret) {
4495 error_report("Could not enable split irqchip mode: %s",
4496 strerror(-ret));
4497 exit(1);
4498 } else {
4499 DPRINTF("Enabled KVM_CAP_SPLIT_IRQCHIP\n");
4500 kvm_split_irqchip = true;
4501 return 1;
4502 }
4503 } else {
4504 return 0;
4505 }
4506 }
4507
4508 /* Classic KVM device assignment interface. Will remain x86 only. */
4509 int kvm_device_pci_assign(KVMState *s, PCIHostDeviceAddress *dev_addr,
4510 uint32_t flags, uint32_t *dev_id)
4511 {
4512 struct kvm_assigned_pci_dev dev_data = {
4513 .segnr = dev_addr->domain,
4514 .busnr = dev_addr->bus,
4515 .devfn = PCI_DEVFN(dev_addr->slot, dev_addr->function),
4516 .flags = flags,
4517 };
4518 int ret;
4519
4520 dev_data.assigned_dev_id =
4521 (dev_addr->domain << 16) | (dev_addr->bus << 8) | dev_data.devfn;
4522
4523 ret = kvm_vm_ioctl(s, KVM_ASSIGN_PCI_DEVICE, &dev_data);
4524 if (ret < 0) {
4525 return ret;
4526 }
4527
4528 *dev_id = dev_data.assigned_dev_id;
4529
4530 return 0;
4531 }
4532
4533 int kvm_device_pci_deassign(KVMState *s, uint32_t dev_id)
4534 {
4535 struct kvm_assigned_pci_dev dev_data = {
4536 .assigned_dev_id = dev_id,
4537 };
4538
4539 return kvm_vm_ioctl(s, KVM_DEASSIGN_PCI_DEVICE, &dev_data);
4540 }
4541
4542 static int kvm_assign_irq_internal(KVMState *s, uint32_t dev_id,
4543 uint32_t irq_type, uint32_t guest_irq)
4544 {
4545 struct kvm_assigned_irq assigned_irq = {
4546 .assigned_dev_id = dev_id,
4547 .guest_irq = guest_irq,
4548 .flags = irq_type,
4549 };
4550
4551 if (kvm_check_extension(s, KVM_CAP_ASSIGN_DEV_IRQ)) {
4552 return kvm_vm_ioctl(s, KVM_ASSIGN_DEV_IRQ, &assigned_irq);
4553 } else {
4554 return kvm_vm_ioctl(s, KVM_ASSIGN_IRQ, &assigned_irq);
4555 }
4556 }
4557
4558 int kvm_device_intx_assign(KVMState *s, uint32_t dev_id, bool use_host_msi,
4559 uint32_t guest_irq)
4560 {
4561 uint32_t irq_type = KVM_DEV_IRQ_GUEST_INTX |
4562 (use_host_msi ? KVM_DEV_IRQ_HOST_MSI : KVM_DEV_IRQ_HOST_INTX);
4563
4564 return kvm_assign_irq_internal(s, dev_id, irq_type, guest_irq);
4565 }
4566
4567 int kvm_device_intx_set_mask(KVMState *s, uint32_t dev_id, bool masked)
4568 {
4569 struct kvm_assigned_pci_dev dev_data = {
4570 .assigned_dev_id = dev_id,
4571 .flags = masked ? KVM_DEV_ASSIGN_MASK_INTX : 0,
4572 };
4573
4574 return kvm_vm_ioctl(s, KVM_ASSIGN_SET_INTX_MASK, &dev_data);
4575 }
4576
4577 static int kvm_deassign_irq_internal(KVMState *s, uint32_t dev_id,
4578 uint32_t type)
4579 {
4580 struct kvm_assigned_irq assigned_irq = {
4581 .assigned_dev_id = dev_id,
4582 .flags = type,
4583 };
4584
4585 return kvm_vm_ioctl(s, KVM_DEASSIGN_DEV_IRQ, &assigned_irq);
4586 }
4587
4588 int kvm_device_intx_deassign(KVMState *s, uint32_t dev_id, bool use_host_msi)
4589 {
4590 return kvm_deassign_irq_internal(s, dev_id, KVM_DEV_IRQ_GUEST_INTX |
4591 (use_host_msi ? KVM_DEV_IRQ_HOST_MSI : KVM_DEV_IRQ_HOST_INTX));
4592 }
4593
4594 int kvm_device_msi_assign(KVMState *s, uint32_t dev_id, int virq)
4595 {
4596 return kvm_assign_irq_internal(s, dev_id, KVM_DEV_IRQ_HOST_MSI |
4597 KVM_DEV_IRQ_GUEST_MSI, virq);
4598 }
4599
4600 int kvm_device_msi_deassign(KVMState *s, uint32_t dev_id)
4601 {
4602 return kvm_deassign_irq_internal(s, dev_id, KVM_DEV_IRQ_GUEST_MSI |
4603 KVM_DEV_IRQ_HOST_MSI);
4604 }
4605
4606 bool kvm_device_msix_supported(KVMState *s)
4607 {
4608 /* The kernel lacks a corresponding KVM_CAP, so we probe by calling
4609 * KVM_ASSIGN_SET_MSIX_NR with an invalid parameter. */
4610 return kvm_vm_ioctl(s, KVM_ASSIGN_SET_MSIX_NR, NULL) == -EFAULT;
4611 }
4612
4613 int kvm_device_msix_init_vectors(KVMState *s, uint32_t dev_id,
4614 uint32_t nr_vectors)
4615 {
4616 struct kvm_assigned_msix_nr msix_nr = {
4617 .assigned_dev_id = dev_id,
4618 .entry_nr = nr_vectors,
4619 };
4620
4621 return kvm_vm_ioctl(s, KVM_ASSIGN_SET_MSIX_NR, &msix_nr);
4622 }
4623
4624 int kvm_device_msix_set_vector(KVMState *s, uint32_t dev_id, uint32_t vector,
4625 int virq)
4626 {
4627 struct kvm_assigned_msix_entry msix_entry = {
4628 .assigned_dev_id = dev_id,
4629 .gsi = virq,
4630 .entry = vector,
4631 };
4632
4633 return kvm_vm_ioctl(s, KVM_ASSIGN_SET_MSIX_ENTRY, &msix_entry);
4634 }
4635
4636 int kvm_device_msix_assign(KVMState *s, uint32_t dev_id)
4637 {
4638 return kvm_assign_irq_internal(s, dev_id, KVM_DEV_IRQ_HOST_MSIX |
4639 KVM_DEV_IRQ_GUEST_MSIX, 0);
4640 }
4641
4642 int kvm_device_msix_deassign(KVMState *s, uint32_t dev_id)
4643 {
4644 return kvm_deassign_irq_internal(s, dev_id, KVM_DEV_IRQ_GUEST_MSIX |
4645 KVM_DEV_IRQ_HOST_MSIX);
4646 }
4647
4648 int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
4649 uint64_t address, uint32_t data, PCIDevice *dev)
4650 {
4651 X86IOMMUState *iommu = x86_iommu_get_default();
4652
4653 if (iommu) {
4654 int ret;
4655 MSIMessage src, dst;
4656 X86IOMMUClass *class = X86_IOMMU_GET_CLASS(iommu);
4657
4658 if (!class->int_remap) {
4659 return 0;
4660 }
4661
4662 src.address = route->u.msi.address_hi;
4663 src.address <<= VTD_MSI_ADDR_HI_SHIFT;
4664 src.address |= route->u.msi.address_lo;
4665 src.data = route->u.msi.data;
4666
4667 ret = class->int_remap(iommu, &src, &dst, dev ? \
4668 pci_requester_id(dev) : \
4669 X86_IOMMU_SID_INVALID);
4670 if (ret) {
4671 trace_kvm_x86_fixup_msi_error(route->gsi);
4672 return 1;
4673 }
4674
4675 route->u.msi.address_hi = dst.address >> VTD_MSI_ADDR_HI_SHIFT;
4676 route->u.msi.address_lo = dst.address & VTD_MSI_ADDR_LO_MASK;
4677 route->u.msi.data = dst.data;
4678 }
4679
4680 return 0;
4681 }
4682
4683 typedef struct MSIRouteEntry MSIRouteEntry;
4684
4685 struct MSIRouteEntry {
4686 PCIDevice *dev; /* Device pointer */
4687 int vector; /* MSI/MSIX vector index */
4688 int virq; /* Virtual IRQ index */
4689 QLIST_ENTRY(MSIRouteEntry) list;
4690 };
4691
4692 /* List of used GSI routes */
4693 static QLIST_HEAD(, MSIRouteEntry) msi_route_list = \
4694 QLIST_HEAD_INITIALIZER(msi_route_list);
4695
4696 static void kvm_update_msi_routes_all(void *private, bool global,
4697 uint32_t index, uint32_t mask)
4698 {
4699 int cnt = 0, vector;
4700 MSIRouteEntry *entry;
4701 MSIMessage msg;
4702 PCIDevice *dev;
4703
4704 /* TODO: explicit route update */
4705 QLIST_FOREACH(entry, &msi_route_list, list) {
4706 cnt++;
4707 vector = entry->vector;
4708 dev = entry->dev;
4709 if (msix_enabled(dev) && !msix_is_masked(dev, vector)) {
4710 msg = msix_get_message(dev, vector);
4711 } else if (msi_enabled(dev) && !msi_is_masked(dev, vector)) {
4712 msg = msi_get_message(dev, vector);
4713 } else {
4714 /*
4715 * Either MSI/MSIX is disabled for the device, or the
4716 * specific message was masked out. Skip this one.
4717 */
4718 continue;
4719 }
4720 kvm_irqchip_update_msi_route(kvm_state, entry->virq, msg, dev);
4721 }
4722 kvm_irqchip_commit_routes(kvm_state);
4723 trace_kvm_x86_update_msi_routes(cnt);
4724 }
4725
4726 int kvm_arch_add_msi_route_post(struct kvm_irq_routing_entry *route,
4727 int vector, PCIDevice *dev)
4728 {
4729 static bool notify_list_inited = false;
4730 MSIRouteEntry *entry;
4731
4732 if (!dev) {
4733 /* These are (possibly) IOAPIC routes only used for split
4734 * kernel irqchip mode, while what we are housekeeping are
4735 * PCI devices only. */
4736 return 0;
4737 }
4738
4739 entry = g_new0(MSIRouteEntry, 1);
4740 entry->dev = dev;
4741 entry->vector = vector;
4742 entry->virq = route->gsi;
4743 QLIST_INSERT_HEAD(&msi_route_list, entry, list);
4744
4745 trace_kvm_x86_add_msi_route(route->gsi);
4746
4747 if (!notify_list_inited) {
4748 /* For the first time we do add route, add ourselves into
4749 * IOMMU's IEC notify list if needed. */
4750 X86IOMMUState *iommu = x86_iommu_get_default();
4751 if (iommu) {
4752 x86_iommu_iec_register_notifier(iommu,
4753 kvm_update_msi_routes_all,
4754 NULL);
4755 }
4756 notify_list_inited = true;
4757 }
4758 return 0;
4759 }
4760
4761 int kvm_arch_release_virq_post(int virq)
4762 {
4763 MSIRouteEntry *entry, *next;
4764 QLIST_FOREACH_SAFE(entry, &msi_route_list, list, next) {
4765 if (entry->virq == virq) {
4766 trace_kvm_x86_remove_msi_route(virq);
4767 QLIST_REMOVE(entry, list);
4768 g_free(entry);
4769 break;
4770 }
4771 }
4772 return 0;
4773 }
4774
4775 int kvm_arch_msi_data_to_gsi(uint32_t data)
4776 {
4777 abort();
4778 }
AR7 emulation for QEMU