util/mmap-alloc: Factor out calculation of the pagesize for the guard page
[qemu/kevin.git] / target / arm / kvm64.c
blob37ceadd9a9dd92714fd4b7cbfe95678061bcb505
1 /*
2 * ARM implementation of KVM hooks, 64 bit specific code
4 * Copyright Mian-M. Hamayun 2013, Virtual Open Systems
5 * Copyright Alex Bennée 2014, Linaro
7 * This work is licensed under the terms of the GNU GPL, version 2 or later.
8 * See the COPYING file in the top-level directory.
12 #include "qemu/osdep.h"
13 #include <sys/ioctl.h>
14 #include <sys/ptrace.h>
16 #include <linux/elf.h>
17 #include <linux/kvm.h>
19 #include "qemu-common.h"
20 #include "qapi/error.h"
21 #include "cpu.h"
22 #include "qemu/timer.h"
23 #include "qemu/error-report.h"
24 #include "qemu/host-utils.h"
25 #include "qemu/main-loop.h"
26 #include "exec/gdbstub.h"
27 #include "sysemu/runstate.h"
28 #include "sysemu/kvm.h"
29 #include "sysemu/kvm_int.h"
30 #include "kvm_arm.h"
31 #include "internals.h"
32 #include "hw/acpi/acpi.h"
33 #include "hw/acpi/ghes.h"
34 #include "hw/arm/virt.h"
36 static bool have_guest_debug;
39 * Although the ARM implementation of hardware assisted debugging
40 * allows for different breakpoints per-core, the current GDB
41 * interface treats them as a global pool of registers (which seems to
42 * be the case for x86, ppc and s390). As a result we store one copy
43 * of registers which is used for all active cores.
45 * Write access is serialised by virtue of the GDB protocol which
46 * updates things. Read access (i.e. when the values are copied to the
47 * vCPU) is also gated by GDB's run control.
49 * This is not unreasonable as most of the time debugging kernels you
50 * never know which core will eventually execute your function.
53 typedef struct {
54 uint64_t bcr;
55 uint64_t bvr;
56 } HWBreakpoint;
58 /* The watchpoint registers can cover more area than the requested
59 * watchpoint so we need to store the additional information
60 * somewhere. We also need to supply a CPUWatchpoint to the GDB stub
61 * when the watchpoint is hit.
63 typedef struct {
64 uint64_t wcr;
65 uint64_t wvr;
66 CPUWatchpoint details;
67 } HWWatchpoint;
69 /* Maximum and current break/watch point counts */
70 int max_hw_bps, max_hw_wps;
71 GArray *hw_breakpoints, *hw_watchpoints;
73 #define cur_hw_wps (hw_watchpoints->len)
74 #define cur_hw_bps (hw_breakpoints->len)
75 #define get_hw_bp(i) (&g_array_index(hw_breakpoints, HWBreakpoint, i))
76 #define get_hw_wp(i) (&g_array_index(hw_watchpoints, HWWatchpoint, i))
78 /**
79 * kvm_arm_init_debug() - check for guest debug capabilities
80 * @cs: CPUState
82 * kvm_check_extension returns the number of debug registers we have
83 * or 0 if we have none.
86 static void kvm_arm_init_debug(CPUState *cs)
88 have_guest_debug = kvm_check_extension(cs->kvm_state,
89 KVM_CAP_SET_GUEST_DEBUG);
91 max_hw_wps = kvm_check_extension(cs->kvm_state, KVM_CAP_GUEST_DEBUG_HW_WPS);
92 hw_watchpoints = g_array_sized_new(true, true,
93 sizeof(HWWatchpoint), max_hw_wps);
95 max_hw_bps = kvm_check_extension(cs->kvm_state, KVM_CAP_GUEST_DEBUG_HW_BPS);
96 hw_breakpoints = g_array_sized_new(true, true,
97 sizeof(HWBreakpoint), max_hw_bps);
98 return;
102 * insert_hw_breakpoint()
103 * @addr: address of breakpoint
105 * See ARM ARM D2.9.1 for details but here we are only going to create
106 * simple un-linked breakpoints (i.e. we don't chain breakpoints
107 * together to match address and context or vmid). The hardware is
108 * capable of fancier matching but that will require exposing that
109 * fanciness to GDB's interface
111 * DBGBCR<n>_EL1, Debug Breakpoint Control Registers
113 * 31 24 23 20 19 16 15 14 13 12 9 8 5 4 3 2 1 0
114 * +------+------+-------+-----+----+------+-----+------+-----+---+
115 * | RES0 | BT | LBN | SSC | HMC| RES0 | BAS | RES0 | PMC | E |
116 * +------+------+-------+-----+----+------+-----+------+-----+---+
118 * BT: Breakpoint type (0 = unlinked address match)
119 * LBN: Linked BP number (0 = unused)
120 * SSC/HMC/PMC: Security, Higher and Priv access control (Table D-12)
121 * BAS: Byte Address Select (RES1 for AArch64)
122 * E: Enable bit
124 * DBGBVR<n>_EL1, Debug Breakpoint Value Registers
126 * 63 53 52 49 48 2 1 0
127 * +------+-----------+----------+-----+
128 * | RESS | VA[52:49] | VA[48:2] | 0 0 |
129 * +------+-----------+----------+-----+
131 * Depending on the addressing mode bits the top bits of the register
132 * are a sign extension of the highest applicable VA bit. Some
133 * versions of GDB don't do it correctly so we ensure they are correct
134 * here so future PC comparisons will work properly.
137 static int insert_hw_breakpoint(target_ulong addr)
139 HWBreakpoint brk = {
140 .bcr = 0x1, /* BCR E=1, enable */
141 .bvr = sextract64(addr, 0, 53)
144 if (cur_hw_bps >= max_hw_bps) {
145 return -ENOBUFS;
148 brk.bcr = deposit32(brk.bcr, 1, 2, 0x3); /* PMC = 11 */
149 brk.bcr = deposit32(brk.bcr, 5, 4, 0xf); /* BAS = RES1 */
151 g_array_append_val(hw_breakpoints, brk);
153 return 0;
157 * delete_hw_breakpoint()
158 * @pc: address of breakpoint
160 * Delete a breakpoint and shuffle any above down
163 static int delete_hw_breakpoint(target_ulong pc)
165 int i;
166 for (i = 0; i < hw_breakpoints->len; i++) {
167 HWBreakpoint *brk = get_hw_bp(i);
168 if (brk->bvr == pc) {
169 g_array_remove_index(hw_breakpoints, i);
170 return 0;
173 return -ENOENT;
177 * insert_hw_watchpoint()
178 * @addr: address of watch point
179 * @len: size of area
180 * @type: type of watch point
182 * See ARM ARM D2.10. As with the breakpoints we can do some advanced
183 * stuff if we want to. The watch points can be linked with the break
184 * points above to make them context aware. However for simplicity
185 * currently we only deal with simple read/write watch points.
187 * D7.3.11 DBGWCR<n>_EL1, Debug Watchpoint Control Registers
189 * 31 29 28 24 23 21 20 19 16 15 14 13 12 5 4 3 2 1 0
190 * +------+-------+------+----+-----+-----+-----+-----+-----+-----+---+
191 * | RES0 | MASK | RES0 | WT | LBN | SSC | HMC | BAS | LSC | PAC | E |
192 * +------+-------+------+----+-----+-----+-----+-----+-----+-----+---+
194 * MASK: num bits addr mask (0=none,01/10=res,11=3 bits (8 bytes))
195 * WT: 0 - unlinked, 1 - linked (not currently used)
196 * LBN: Linked BP number (not currently used)
197 * SSC/HMC/PAC: Security, Higher and Priv access control (Table D2-11)
198 * BAS: Byte Address Select
199 * LSC: Load/Store control (01: load, 10: store, 11: both)
200 * E: Enable
202 * The bottom 2 bits of the value register are masked. Therefore to
203 * break on any sizes smaller than an unaligned word you need to set
204 * MASK=0, BAS=bit per byte in question. For larger regions (^2) you
205 * need to ensure you mask the address as required and set BAS=0xff
208 static int insert_hw_watchpoint(target_ulong addr,
209 target_ulong len, int type)
211 HWWatchpoint wp = {
212 .wcr = 1, /* E=1, enable */
213 .wvr = addr & (~0x7ULL),
214 .details = { .vaddr = addr, .len = len }
217 if (cur_hw_wps >= max_hw_wps) {
218 return -ENOBUFS;
222 * HMC=0 SSC=0 PAC=3 will hit EL0 or EL1, any security state,
223 * valid whether EL3 is implemented or not
225 wp.wcr = deposit32(wp.wcr, 1, 2, 3);
227 switch (type) {
228 case GDB_WATCHPOINT_READ:
229 wp.wcr = deposit32(wp.wcr, 3, 2, 1);
230 wp.details.flags = BP_MEM_READ;
231 break;
232 case GDB_WATCHPOINT_WRITE:
233 wp.wcr = deposit32(wp.wcr, 3, 2, 2);
234 wp.details.flags = BP_MEM_WRITE;
235 break;
236 case GDB_WATCHPOINT_ACCESS:
237 wp.wcr = deposit32(wp.wcr, 3, 2, 3);
238 wp.details.flags = BP_MEM_ACCESS;
239 break;
240 default:
241 g_assert_not_reached();
242 break;
244 if (len <= 8) {
245 /* we align the address and set the bits in BAS */
246 int off = addr & 0x7;
247 int bas = (1 << len) - 1;
249 wp.wcr = deposit32(wp.wcr, 5 + off, 8 - off, bas);
250 } else {
251 /* For ranges above 8 bytes we need to be a power of 2 */
252 if (is_power_of_2(len)) {
253 int bits = ctz64(len);
255 wp.wvr &= ~((1 << bits) - 1);
256 wp.wcr = deposit32(wp.wcr, 24, 4, bits);
257 wp.wcr = deposit32(wp.wcr, 5, 8, 0xff);
258 } else {
259 return -ENOBUFS;
263 g_array_append_val(hw_watchpoints, wp);
264 return 0;
268 static bool check_watchpoint_in_range(int i, target_ulong addr)
270 HWWatchpoint *wp = get_hw_wp(i);
271 uint64_t addr_top, addr_bottom = wp->wvr;
272 int bas = extract32(wp->wcr, 5, 8);
273 int mask = extract32(wp->wcr, 24, 4);
275 if (mask) {
276 addr_top = addr_bottom + (1 << mask);
277 } else {
278 /* BAS must be contiguous but can offset against the base
279 * address in DBGWVR */
280 addr_bottom = addr_bottom + ctz32(bas);
281 addr_top = addr_bottom + clo32(bas);
284 if (addr >= addr_bottom && addr <= addr_top) {
285 return true;
288 return false;
292 * delete_hw_watchpoint()
293 * @addr: address of breakpoint
295 * Delete a breakpoint and shuffle any above down
298 static int delete_hw_watchpoint(target_ulong addr,
299 target_ulong len, int type)
301 int i;
302 for (i = 0; i < cur_hw_wps; i++) {
303 if (check_watchpoint_in_range(i, addr)) {
304 g_array_remove_index(hw_watchpoints, i);
305 return 0;
308 return -ENOENT;
312 int kvm_arch_insert_hw_breakpoint(target_ulong addr,
313 target_ulong len, int type)
315 switch (type) {
316 case GDB_BREAKPOINT_HW:
317 return insert_hw_breakpoint(addr);
318 break;
319 case GDB_WATCHPOINT_READ:
320 case GDB_WATCHPOINT_WRITE:
321 case GDB_WATCHPOINT_ACCESS:
322 return insert_hw_watchpoint(addr, len, type);
323 default:
324 return -ENOSYS;
328 int kvm_arch_remove_hw_breakpoint(target_ulong addr,
329 target_ulong len, int type)
331 switch (type) {
332 case GDB_BREAKPOINT_HW:
333 return delete_hw_breakpoint(addr);
334 case GDB_WATCHPOINT_READ:
335 case GDB_WATCHPOINT_WRITE:
336 case GDB_WATCHPOINT_ACCESS:
337 return delete_hw_watchpoint(addr, len, type);
338 default:
339 return -ENOSYS;
344 void kvm_arch_remove_all_hw_breakpoints(void)
346 if (cur_hw_wps > 0) {
347 g_array_remove_range(hw_watchpoints, 0, cur_hw_wps);
349 if (cur_hw_bps > 0) {
350 g_array_remove_range(hw_breakpoints, 0, cur_hw_bps);
354 void kvm_arm_copy_hw_debug_data(struct kvm_guest_debug_arch *ptr)
356 int i;
357 memset(ptr, 0, sizeof(struct kvm_guest_debug_arch));
359 for (i = 0; i < max_hw_wps; i++) {
360 HWWatchpoint *wp = get_hw_wp(i);
361 ptr->dbg_wcr[i] = wp->wcr;
362 ptr->dbg_wvr[i] = wp->wvr;
364 for (i = 0; i < max_hw_bps; i++) {
365 HWBreakpoint *bp = get_hw_bp(i);
366 ptr->dbg_bcr[i] = bp->bcr;
367 ptr->dbg_bvr[i] = bp->bvr;
371 bool kvm_arm_hw_debug_active(CPUState *cs)
373 return ((cur_hw_wps > 0) || (cur_hw_bps > 0));
376 static bool find_hw_breakpoint(CPUState *cpu, target_ulong pc)
378 int i;
380 for (i = 0; i < cur_hw_bps; i++) {
381 HWBreakpoint *bp = get_hw_bp(i);
382 if (bp->bvr == pc) {
383 return true;
386 return false;
389 static CPUWatchpoint *find_hw_watchpoint(CPUState *cpu, target_ulong addr)
391 int i;
393 for (i = 0; i < cur_hw_wps; i++) {
394 if (check_watchpoint_in_range(i, addr)) {
395 return &get_hw_wp(i)->details;
398 return NULL;
401 static bool kvm_arm_set_device_attr(CPUState *cs, struct kvm_device_attr *attr,
402 const char *name)
404 int err;
406 err = kvm_vcpu_ioctl(cs, KVM_HAS_DEVICE_ATTR, attr);
407 if (err != 0) {
408 error_report("%s: KVM_HAS_DEVICE_ATTR: %s", name, strerror(-err));
409 return false;
412 err = kvm_vcpu_ioctl(cs, KVM_SET_DEVICE_ATTR, attr);
413 if (err != 0) {
414 error_report("%s: KVM_SET_DEVICE_ATTR: %s", name, strerror(-err));
415 return false;
418 return true;
421 void kvm_arm_pmu_init(CPUState *cs)
423 struct kvm_device_attr attr = {
424 .group = KVM_ARM_VCPU_PMU_V3_CTRL,
425 .attr = KVM_ARM_VCPU_PMU_V3_INIT,
428 if (!ARM_CPU(cs)->has_pmu) {
429 return;
431 if (!kvm_arm_set_device_attr(cs, &attr, "PMU")) {
432 error_report("failed to init PMU");
433 abort();
437 void kvm_arm_pmu_set_irq(CPUState *cs, int irq)
439 struct kvm_device_attr attr = {
440 .group = KVM_ARM_VCPU_PMU_V3_CTRL,
441 .addr = (intptr_t)&irq,
442 .attr = KVM_ARM_VCPU_PMU_V3_IRQ,
445 if (!ARM_CPU(cs)->has_pmu) {
446 return;
448 if (!kvm_arm_set_device_attr(cs, &attr, "PMU")) {
449 error_report("failed to set irq for PMU");
450 abort();
454 void kvm_arm_pvtime_init(CPUState *cs, uint64_t ipa)
456 struct kvm_device_attr attr = {
457 .group = KVM_ARM_VCPU_PVTIME_CTRL,
458 .attr = KVM_ARM_VCPU_PVTIME_IPA,
459 .addr = (uint64_t)&ipa,
462 if (ARM_CPU(cs)->kvm_steal_time == ON_OFF_AUTO_OFF) {
463 return;
465 if (!kvm_arm_set_device_attr(cs, &attr, "PVTIME IPA")) {
466 error_report("failed to init PVTIME IPA");
467 abort();
471 static int read_sys_reg32(int fd, uint32_t *pret, uint64_t id)
473 uint64_t ret;
474 struct kvm_one_reg idreg = { .id = id, .addr = (uintptr_t)&ret };
475 int err;
477 assert((id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64);
478 err = ioctl(fd, KVM_GET_ONE_REG, &idreg);
479 if (err < 0) {
480 return -1;
482 *pret = ret;
483 return 0;
486 static int read_sys_reg64(int fd, uint64_t *pret, uint64_t id)
488 struct kvm_one_reg idreg = { .id = id, .addr = (uintptr_t)pret };
490 assert((id & KVM_REG_SIZE_MASK) == KVM_REG_SIZE_U64);
491 return ioctl(fd, KVM_GET_ONE_REG, &idreg);
494 bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
496 /* Identify the feature bits corresponding to the host CPU, and
497 * fill out the ARMHostCPUClass fields accordingly. To do this
498 * we have to create a scratch VM, create a single CPU inside it,
499 * and then query that CPU for the relevant ID registers.
501 int fdarray[3];
502 bool sve_supported;
503 uint64_t features = 0;
504 uint64_t t;
505 int err;
507 /* Old kernels may not know about the PREFERRED_TARGET ioctl: however
508 * we know these will only support creating one kind of guest CPU,
509 * which is its preferred CPU type. Fortunately these old kernels
510 * support only a very limited number of CPUs.
512 static const uint32_t cpus_to_try[] = {
513 KVM_ARM_TARGET_AEM_V8,
514 KVM_ARM_TARGET_FOUNDATION_V8,
515 KVM_ARM_TARGET_CORTEX_A57,
516 QEMU_KVM_ARM_TARGET_NONE
519 * target = -1 informs kvm_arm_create_scratch_host_vcpu()
520 * to use the preferred target
522 struct kvm_vcpu_init init = { .target = -1, };
524 if (!kvm_arm_create_scratch_host_vcpu(cpus_to_try, fdarray, &init)) {
525 return false;
528 ahcf->target = init.target;
529 ahcf->dtb_compatible = "arm,arm-v8";
531 err = read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64pfr0,
532 ARM64_SYS_REG(3, 0, 0, 4, 0));
533 if (unlikely(err < 0)) {
535 * Before v4.15, the kernel only exposed a limited number of system
536 * registers, not including any of the interesting AArch64 ID regs.
537 * For the most part we could leave these fields as zero with minimal
538 * effect, since this does not affect the values seen by the guest.
540 * However, it could cause problems down the line for QEMU,
541 * so provide a minimal v8.0 default.
543 * ??? Could read MIDR and use knowledge from cpu64.c.
544 * ??? Could map a page of memory into our temp guest and
545 * run the tiniest of hand-crafted kernels to extract
546 * the values seen by the guest.
547 * ??? Either of these sounds like too much effort just
548 * to work around running a modern host kernel.
550 ahcf->isar.id_aa64pfr0 = 0x00000011; /* EL1&0, AArch64 only */
551 err = 0;
552 } else {
553 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64pfr1,
554 ARM64_SYS_REG(3, 0, 0, 4, 1));
555 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64dfr0,
556 ARM64_SYS_REG(3, 0, 0, 5, 0));
557 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64dfr1,
558 ARM64_SYS_REG(3, 0, 0, 5, 1));
559 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64isar0,
560 ARM64_SYS_REG(3, 0, 0, 6, 0));
561 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64isar1,
562 ARM64_SYS_REG(3, 0, 0, 6, 1));
563 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64mmfr0,
564 ARM64_SYS_REG(3, 0, 0, 7, 0));
565 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64mmfr1,
566 ARM64_SYS_REG(3, 0, 0, 7, 1));
567 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64mmfr2,
568 ARM64_SYS_REG(3, 0, 0, 7, 2));
571 * Note that if AArch32 support is not present in the host,
572 * the AArch32 sysregs are present to be read, but will
573 * return UNKNOWN values. This is neither better nor worse
574 * than skipping the reads and leaving 0, as we must avoid
575 * considering the values in every case.
577 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_pfr0,
578 ARM64_SYS_REG(3, 0, 0, 1, 0));
579 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_pfr1,
580 ARM64_SYS_REG(3, 0, 0, 1, 1));
581 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_pfr2,
582 ARM64_SYS_REG(3, 0, 0, 3, 4));
583 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_dfr0,
584 ARM64_SYS_REG(3, 0, 0, 1, 2));
585 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr0,
586 ARM64_SYS_REG(3, 0, 0, 1, 4));
587 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr1,
588 ARM64_SYS_REG(3, 0, 0, 1, 5));
589 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr2,
590 ARM64_SYS_REG(3, 0, 0, 1, 6));
591 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr3,
592 ARM64_SYS_REG(3, 0, 0, 1, 7));
593 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar0,
594 ARM64_SYS_REG(3, 0, 0, 2, 0));
595 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar1,
596 ARM64_SYS_REG(3, 0, 0, 2, 1));
597 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar2,
598 ARM64_SYS_REG(3, 0, 0, 2, 2));
599 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar3,
600 ARM64_SYS_REG(3, 0, 0, 2, 3));
601 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar4,
602 ARM64_SYS_REG(3, 0, 0, 2, 4));
603 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar5,
604 ARM64_SYS_REG(3, 0, 0, 2, 5));
605 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_mmfr4,
606 ARM64_SYS_REG(3, 0, 0, 2, 6));
607 err |= read_sys_reg32(fdarray[2], &ahcf->isar.id_isar6,
608 ARM64_SYS_REG(3, 0, 0, 2, 7));
610 err |= read_sys_reg32(fdarray[2], &ahcf->isar.mvfr0,
611 ARM64_SYS_REG(3, 0, 0, 3, 0));
612 err |= read_sys_reg32(fdarray[2], &ahcf->isar.mvfr1,
613 ARM64_SYS_REG(3, 0, 0, 3, 1));
614 err |= read_sys_reg32(fdarray[2], &ahcf->isar.mvfr2,
615 ARM64_SYS_REG(3, 0, 0, 3, 2));
618 * DBGDIDR is a bit complicated because the kernel doesn't
619 * provide an accessor for it in 64-bit mode, which is what this
620 * scratch VM is in, and there's no architected "64-bit sysreg
621 * which reads the same as the 32-bit register" the way there is
622 * for other ID registers. Instead we synthesize a value from the
623 * AArch64 ID_AA64DFR0, the same way the kernel code in
624 * arch/arm64/kvm/sys_regs.c:trap_dbgidr() does.
625 * We only do this if the CPU supports AArch32 at EL1.
627 if (FIELD_EX32(ahcf->isar.id_aa64pfr0, ID_AA64PFR0, EL1) >= 2) {
628 int wrps = FIELD_EX64(ahcf->isar.id_aa64dfr0, ID_AA64DFR0, WRPS);
629 int brps = FIELD_EX64(ahcf->isar.id_aa64dfr0, ID_AA64DFR0, BRPS);
630 int ctx_cmps =
631 FIELD_EX64(ahcf->isar.id_aa64dfr0, ID_AA64DFR0, CTX_CMPS);
632 int version = 6; /* ARMv8 debug architecture */
633 bool has_el3 =
634 !!FIELD_EX32(ahcf->isar.id_aa64pfr0, ID_AA64PFR0, EL3);
635 uint32_t dbgdidr = 0;
637 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, WRPS, wrps);
638 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, BRPS, brps);
639 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, CTX_CMPS, ctx_cmps);
640 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, VERSION, version);
641 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, NSUHD_IMP, has_el3);
642 dbgdidr = FIELD_DP32(dbgdidr, DBGDIDR, SE_IMP, has_el3);
643 dbgdidr |= (1 << 15); /* RES1 bit */
644 ahcf->isar.dbgdidr = dbgdidr;
648 sve_supported = ioctl(fdarray[0], KVM_CHECK_EXTENSION, KVM_CAP_ARM_SVE) > 0;
650 /* Add feature bits that can't appear until after VCPU init. */
651 if (sve_supported) {
652 t = ahcf->isar.id_aa64pfr0;
653 t = FIELD_DP64(t, ID_AA64PFR0, SVE, 1);
654 ahcf->isar.id_aa64pfr0 = t;
657 * Before v5.1, KVM did not support SVE and did not expose
658 * ID_AA64ZFR0_EL1 even as RAZ. After v5.1, KVM still does
659 * not expose the register to "user" requests like this
660 * unless the host supports SVE.
662 err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64zfr0,
663 ARM64_SYS_REG(3, 0, 0, 4, 4));
666 kvm_arm_destroy_scratch_host_vcpu(fdarray);
668 if (err < 0) {
669 return false;
673 * We can assume any KVM supporting CPU is at least a v8
674 * with VFPv4+Neon; this in turn implies most of the other
675 * feature bits.
677 features |= 1ULL << ARM_FEATURE_V8;
678 features |= 1ULL << ARM_FEATURE_NEON;
679 features |= 1ULL << ARM_FEATURE_AARCH64;
680 features |= 1ULL << ARM_FEATURE_PMU;
681 features |= 1ULL << ARM_FEATURE_GENERIC_TIMER;
683 ahcf->features = features;
685 return true;
688 void kvm_arm_steal_time_finalize(ARMCPU *cpu, Error **errp)
690 bool has_steal_time = kvm_arm_steal_time_supported();
692 if (cpu->kvm_steal_time == ON_OFF_AUTO_AUTO) {
693 if (!has_steal_time || !arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) {
694 cpu->kvm_steal_time = ON_OFF_AUTO_OFF;
695 } else {
696 cpu->kvm_steal_time = ON_OFF_AUTO_ON;
698 } else if (cpu->kvm_steal_time == ON_OFF_AUTO_ON) {
699 if (!has_steal_time) {
700 error_setg(errp, "'kvm-steal-time' cannot be enabled "
701 "on this host");
702 return;
703 } else if (!arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) {
705 * DEN0057A chapter 2 says "This specification only covers
706 * systems in which the Execution state of the hypervisor
707 * as well as EL1 of virtual machines is AArch64.". And,
708 * to ensure that, the smc/hvc calls are only specified as
709 * smc64/hvc64.
711 error_setg(errp, "'kvm-steal-time' cannot be enabled "
712 "for AArch32 guests");
713 return;
718 bool kvm_arm_aarch32_supported(void)
720 return kvm_check_extension(kvm_state, KVM_CAP_ARM_EL1_32BIT);
723 bool kvm_arm_sve_supported(void)
725 return kvm_check_extension(kvm_state, KVM_CAP_ARM_SVE);
728 bool kvm_arm_steal_time_supported(void)
730 return kvm_check_extension(kvm_state, KVM_CAP_STEAL_TIME);
733 QEMU_BUILD_BUG_ON(KVM_ARM64_SVE_VQ_MIN != 1);
735 void kvm_arm_sve_get_vls(CPUState *cs, unsigned long *map)
737 /* Only call this function if kvm_arm_sve_supported() returns true. */
738 static uint64_t vls[KVM_ARM64_SVE_VLS_WORDS];
739 static bool probed;
740 uint32_t vq = 0;
741 int i, j;
743 bitmap_clear(map, 0, ARM_MAX_VQ);
746 * KVM ensures all host CPUs support the same set of vector lengths.
747 * So we only need to create the scratch VCPUs once and then cache
748 * the results.
750 if (!probed) {
751 struct kvm_vcpu_init init = {
752 .target = -1,
753 .features[0] = (1 << KVM_ARM_VCPU_SVE),
755 struct kvm_one_reg reg = {
756 .id = KVM_REG_ARM64_SVE_VLS,
757 .addr = (uint64_t)&vls[0],
759 int fdarray[3], ret;
761 probed = true;
763 if (!kvm_arm_create_scratch_host_vcpu(NULL, fdarray, &init)) {
764 error_report("failed to create scratch VCPU with SVE enabled");
765 abort();
767 ret = ioctl(fdarray[2], KVM_GET_ONE_REG, &reg);
768 kvm_arm_destroy_scratch_host_vcpu(fdarray);
769 if (ret) {
770 error_report("failed to get KVM_REG_ARM64_SVE_VLS: %s",
771 strerror(errno));
772 abort();
775 for (i = KVM_ARM64_SVE_VLS_WORDS - 1; i >= 0; --i) {
776 if (vls[i]) {
777 vq = 64 - clz64(vls[i]) + i * 64;
778 break;
781 if (vq > ARM_MAX_VQ) {
782 warn_report("KVM supports vector lengths larger than "
783 "QEMU can enable");
787 for (i = 0; i < KVM_ARM64_SVE_VLS_WORDS; ++i) {
788 if (!vls[i]) {
789 continue;
791 for (j = 1; j <= 64; ++j) {
792 vq = j + i * 64;
793 if (vq > ARM_MAX_VQ) {
794 return;
796 if (vls[i] & (1UL << (j - 1))) {
797 set_bit(vq - 1, map);
803 static int kvm_arm_sve_set_vls(CPUState *cs)
805 uint64_t vls[KVM_ARM64_SVE_VLS_WORDS] = {0};
806 struct kvm_one_reg reg = {
807 .id = KVM_REG_ARM64_SVE_VLS,
808 .addr = (uint64_t)&vls[0],
810 ARMCPU *cpu = ARM_CPU(cs);
811 uint32_t vq;
812 int i, j;
814 assert(cpu->sve_max_vq <= KVM_ARM64_SVE_VQ_MAX);
816 for (vq = 1; vq <= cpu->sve_max_vq; ++vq) {
817 if (test_bit(vq - 1, cpu->sve_vq_map)) {
818 i = (vq - 1) / 64;
819 j = (vq - 1) % 64;
820 vls[i] |= 1UL << j;
824 return kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
827 #define ARM_CPU_ID_MPIDR 3, 0, 0, 0, 5
829 int kvm_arch_init_vcpu(CPUState *cs)
831 int ret;
832 uint64_t mpidr;
833 ARMCPU *cpu = ARM_CPU(cs);
834 CPUARMState *env = &cpu->env;
836 if (cpu->kvm_target == QEMU_KVM_ARM_TARGET_NONE ||
837 !object_dynamic_cast(OBJECT(cpu), TYPE_AARCH64_CPU)) {
838 error_report("KVM is not supported for this guest CPU type");
839 return -EINVAL;
842 qemu_add_vm_change_state_handler(kvm_arm_vm_state_change, cs);
844 /* Determine init features for this CPU */
845 memset(cpu->kvm_init_features, 0, sizeof(cpu->kvm_init_features));
846 if (cs->start_powered_off) {
847 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_POWER_OFF;
849 if (kvm_check_extension(cs->kvm_state, KVM_CAP_ARM_PSCI_0_2)) {
850 cpu->psci_version = 2;
851 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_PSCI_0_2;
853 if (!arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) {
854 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_EL1_32BIT;
856 if (!kvm_check_extension(cs->kvm_state, KVM_CAP_ARM_PMU_V3)) {
857 cpu->has_pmu = false;
859 if (cpu->has_pmu) {
860 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_PMU_V3;
861 } else {
862 env->features &= ~(1ULL << ARM_FEATURE_PMU);
864 if (cpu_isar_feature(aa64_sve, cpu)) {
865 assert(kvm_arm_sve_supported());
866 cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_SVE;
869 /* Do KVM_ARM_VCPU_INIT ioctl */
870 ret = kvm_arm_vcpu_init(cs);
871 if (ret) {
872 return ret;
875 if (cpu_isar_feature(aa64_sve, cpu)) {
876 ret = kvm_arm_sve_set_vls(cs);
877 if (ret) {
878 return ret;
880 ret = kvm_arm_vcpu_finalize(cs, KVM_ARM_VCPU_SVE);
881 if (ret) {
882 return ret;
887 * When KVM is in use, PSCI is emulated in-kernel and not by qemu.
888 * Currently KVM has its own idea about MPIDR assignment, so we
889 * override our defaults with what we get from KVM.
891 ret = kvm_get_one_reg(cs, ARM64_SYS_REG(ARM_CPU_ID_MPIDR), &mpidr);
892 if (ret) {
893 return ret;
895 cpu->mp_affinity = mpidr & ARM64_AFFINITY_MASK;
897 kvm_arm_init_debug(cs);
899 /* Check whether user space can specify guest syndrome value */
900 kvm_arm_init_serror_injection(cs);
902 return kvm_arm_init_cpreg_list(cpu);
905 int kvm_arch_destroy_vcpu(CPUState *cs)
907 return 0;
910 bool kvm_arm_reg_syncs_via_cpreg_list(uint64_t regidx)
912 /* Return true if the regidx is a register we should synchronize
913 * via the cpreg_tuples array (ie is not a core or sve reg that
914 * we sync by hand in kvm_arch_get/put_registers())
916 switch (regidx & KVM_REG_ARM_COPROC_MASK) {
917 case KVM_REG_ARM_CORE:
918 case KVM_REG_ARM64_SVE:
919 return false;
920 default:
921 return true;
925 typedef struct CPRegStateLevel {
926 uint64_t regidx;
927 int level;
928 } CPRegStateLevel;
930 /* All system registers not listed in the following table are assumed to be
931 * of the level KVM_PUT_RUNTIME_STATE. If a register should be written less
932 * often, you must add it to this table with a state of either
933 * KVM_PUT_RESET_STATE or KVM_PUT_FULL_STATE.
935 static const CPRegStateLevel non_runtime_cpregs[] = {
936 { KVM_REG_ARM_TIMER_CNT, KVM_PUT_FULL_STATE },
939 int kvm_arm_cpreg_level(uint64_t regidx)
941 int i;
943 for (i = 0; i < ARRAY_SIZE(non_runtime_cpregs); i++) {
944 const CPRegStateLevel *l = &non_runtime_cpregs[i];
945 if (l->regidx == regidx) {
946 return l->level;
950 return KVM_PUT_RUNTIME_STATE;
953 /* Callers must hold the iothread mutex lock */
954 static void kvm_inject_arm_sea(CPUState *c)
956 ARMCPU *cpu = ARM_CPU(c);
957 CPUARMState *env = &cpu->env;
958 uint32_t esr;
959 bool same_el;
961 c->exception_index = EXCP_DATA_ABORT;
962 env->exception.target_el = 1;
965 * Set the DFSC to synchronous external abort and set FnV to not valid,
966 * this will tell guest the FAR_ELx is UNKNOWN for this abort.
968 same_el = arm_current_el(env) == env->exception.target_el;
969 esr = syn_data_abort_no_iss(same_el, 1, 0, 0, 0, 0, 0x10);
971 env->exception.syndrome = esr;
973 arm_cpu_do_interrupt(c);
976 #define AARCH64_CORE_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U64 | \
977 KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x))
979 #define AARCH64_SIMD_CORE_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U128 | \
980 KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x))
982 #define AARCH64_SIMD_CTRL_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U32 | \
983 KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x))
985 static int kvm_arch_put_fpsimd(CPUState *cs)
987 CPUARMState *env = &ARM_CPU(cs)->env;
988 struct kvm_one_reg reg;
989 int i, ret;
991 for (i = 0; i < 32; i++) {
992 uint64_t *q = aa64_vfp_qreg(env, i);
993 #ifdef HOST_WORDS_BIGENDIAN
994 uint64_t fp_val[2] = { q[1], q[0] };
995 reg.addr = (uintptr_t)fp_val;
996 #else
997 reg.addr = (uintptr_t)q;
998 #endif
999 reg.id = AARCH64_SIMD_CORE_REG(fp_regs.vregs[i]);
1000 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1001 if (ret) {
1002 return ret;
1006 return 0;
1010 * KVM SVE registers come in slices where ZREGs have a slice size of 2048 bits
1011 * and PREGS and the FFR have a slice size of 256 bits. However we simply hard
1012 * code the slice index to zero for now as it's unlikely we'll need more than
1013 * one slice for quite some time.
1015 static int kvm_arch_put_sve(CPUState *cs)
1017 ARMCPU *cpu = ARM_CPU(cs);
1018 CPUARMState *env = &cpu->env;
1019 uint64_t tmp[ARM_MAX_VQ * 2];
1020 uint64_t *r;
1021 struct kvm_one_reg reg;
1022 int n, ret;
1024 for (n = 0; n < KVM_ARM64_SVE_NUM_ZREGS; ++n) {
1025 r = sve_bswap64(tmp, &env->vfp.zregs[n].d[0], cpu->sve_max_vq * 2);
1026 reg.addr = (uintptr_t)r;
1027 reg.id = KVM_REG_ARM64_SVE_ZREG(n, 0);
1028 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1029 if (ret) {
1030 return ret;
1034 for (n = 0; n < KVM_ARM64_SVE_NUM_PREGS; ++n) {
1035 r = sve_bswap64(tmp, r = &env->vfp.pregs[n].p[0],
1036 DIV_ROUND_UP(cpu->sve_max_vq * 2, 8));
1037 reg.addr = (uintptr_t)r;
1038 reg.id = KVM_REG_ARM64_SVE_PREG(n, 0);
1039 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1040 if (ret) {
1041 return ret;
1045 r = sve_bswap64(tmp, &env->vfp.pregs[FFR_PRED_NUM].p[0],
1046 DIV_ROUND_UP(cpu->sve_max_vq * 2, 8));
1047 reg.addr = (uintptr_t)r;
1048 reg.id = KVM_REG_ARM64_SVE_FFR(0);
1049 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1050 if (ret) {
1051 return ret;
1054 return 0;
1057 int kvm_arch_put_registers(CPUState *cs, int level)
1059 struct kvm_one_reg reg;
1060 uint64_t val;
1061 uint32_t fpr;
1062 int i, ret;
1063 unsigned int el;
1065 ARMCPU *cpu = ARM_CPU(cs);
1066 CPUARMState *env = &cpu->env;
1068 /* If we are in AArch32 mode then we need to copy the AArch32 regs to the
1069 * AArch64 registers before pushing them out to 64-bit KVM.
1071 if (!is_a64(env)) {
1072 aarch64_sync_32_to_64(env);
1075 for (i = 0; i < 31; i++) {
1076 reg.id = AARCH64_CORE_REG(regs.regs[i]);
1077 reg.addr = (uintptr_t) &env->xregs[i];
1078 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1079 if (ret) {
1080 return ret;
1084 /* KVM puts SP_EL0 in regs.sp and SP_EL1 in regs.sp_el1. On the
1085 * QEMU side we keep the current SP in xregs[31] as well.
1087 aarch64_save_sp(env, 1);
1089 reg.id = AARCH64_CORE_REG(regs.sp);
1090 reg.addr = (uintptr_t) &env->sp_el[0];
1091 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1092 if (ret) {
1093 return ret;
1096 reg.id = AARCH64_CORE_REG(sp_el1);
1097 reg.addr = (uintptr_t) &env->sp_el[1];
1098 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1099 if (ret) {
1100 return ret;
1103 /* Note that KVM thinks pstate is 64 bit but we use a uint32_t */
1104 if (is_a64(env)) {
1105 val = pstate_read(env);
1106 } else {
1107 val = cpsr_read(env);
1109 reg.id = AARCH64_CORE_REG(regs.pstate);
1110 reg.addr = (uintptr_t) &val;
1111 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1112 if (ret) {
1113 return ret;
1116 reg.id = AARCH64_CORE_REG(regs.pc);
1117 reg.addr = (uintptr_t) &env->pc;
1118 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1119 if (ret) {
1120 return ret;
1123 reg.id = AARCH64_CORE_REG(elr_el1);
1124 reg.addr = (uintptr_t) &env->elr_el[1];
1125 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1126 if (ret) {
1127 return ret;
1130 /* Saved Program State Registers
1132 * Before we restore from the banked_spsr[] array we need to
1133 * ensure that any modifications to env->spsr are correctly
1134 * reflected in the banks.
1136 el = arm_current_el(env);
1137 if (el > 0 && !is_a64(env)) {
1138 i = bank_number(env->uncached_cpsr & CPSR_M);
1139 env->banked_spsr[i] = env->spsr;
1142 /* KVM 0-4 map to QEMU banks 1-5 */
1143 for (i = 0; i < KVM_NR_SPSR; i++) {
1144 reg.id = AARCH64_CORE_REG(spsr[i]);
1145 reg.addr = (uintptr_t) &env->banked_spsr[i + 1];
1146 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1147 if (ret) {
1148 return ret;
1152 if (cpu_isar_feature(aa64_sve, cpu)) {
1153 ret = kvm_arch_put_sve(cs);
1154 } else {
1155 ret = kvm_arch_put_fpsimd(cs);
1157 if (ret) {
1158 return ret;
1161 reg.addr = (uintptr_t)(&fpr);
1162 fpr = vfp_get_fpsr(env);
1163 reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr);
1164 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1165 if (ret) {
1166 return ret;
1169 reg.addr = (uintptr_t)(&fpr);
1170 fpr = vfp_get_fpcr(env);
1171 reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr);
1172 ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
1173 if (ret) {
1174 return ret;
1177 write_cpustate_to_list(cpu, true);
1179 if (!write_list_to_kvmstate(cpu, level)) {
1180 return -EINVAL;
1184 * Setting VCPU events should be triggered after syncing the registers
1185 * to avoid overwriting potential changes made by KVM upon calling
1186 * KVM_SET_VCPU_EVENTS ioctl
1188 ret = kvm_put_vcpu_events(cpu);
1189 if (ret) {
1190 return ret;
1193 kvm_arm_sync_mpstate_to_kvm(cpu);
1195 return ret;
1198 static int kvm_arch_get_fpsimd(CPUState *cs)
1200 CPUARMState *env = &ARM_CPU(cs)->env;
1201 struct kvm_one_reg reg;
1202 int i, ret;
1204 for (i = 0; i < 32; i++) {
1205 uint64_t *q = aa64_vfp_qreg(env, i);
1206 reg.id = AARCH64_SIMD_CORE_REG(fp_regs.vregs[i]);
1207 reg.addr = (uintptr_t)q;
1208 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1209 if (ret) {
1210 return ret;
1211 } else {
1212 #ifdef HOST_WORDS_BIGENDIAN
1213 uint64_t t;
1214 t = q[0], q[0] = q[1], q[1] = t;
1215 #endif
1219 return 0;
1223 * KVM SVE registers come in slices where ZREGs have a slice size of 2048 bits
1224 * and PREGS and the FFR have a slice size of 256 bits. However we simply hard
1225 * code the slice index to zero for now as it's unlikely we'll need more than
1226 * one slice for quite some time.
1228 static int kvm_arch_get_sve(CPUState *cs)
1230 ARMCPU *cpu = ARM_CPU(cs);
1231 CPUARMState *env = &cpu->env;
1232 struct kvm_one_reg reg;
1233 uint64_t *r;
1234 int n, ret;
1236 for (n = 0; n < KVM_ARM64_SVE_NUM_ZREGS; ++n) {
1237 r = &env->vfp.zregs[n].d[0];
1238 reg.addr = (uintptr_t)r;
1239 reg.id = KVM_REG_ARM64_SVE_ZREG(n, 0);
1240 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1241 if (ret) {
1242 return ret;
1244 sve_bswap64(r, r, cpu->sve_max_vq * 2);
1247 for (n = 0; n < KVM_ARM64_SVE_NUM_PREGS; ++n) {
1248 r = &env->vfp.pregs[n].p[0];
1249 reg.addr = (uintptr_t)r;
1250 reg.id = KVM_REG_ARM64_SVE_PREG(n, 0);
1251 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1252 if (ret) {
1253 return ret;
1255 sve_bswap64(r, r, DIV_ROUND_UP(cpu->sve_max_vq * 2, 8));
1258 r = &env->vfp.pregs[FFR_PRED_NUM].p[0];
1259 reg.addr = (uintptr_t)r;
1260 reg.id = KVM_REG_ARM64_SVE_FFR(0);
1261 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1262 if (ret) {
1263 return ret;
1265 sve_bswap64(r, r, DIV_ROUND_UP(cpu->sve_max_vq * 2, 8));
1267 return 0;
1270 int kvm_arch_get_registers(CPUState *cs)
1272 struct kvm_one_reg reg;
1273 uint64_t val;
1274 unsigned int el;
1275 uint32_t fpr;
1276 int i, ret;
1278 ARMCPU *cpu = ARM_CPU(cs);
1279 CPUARMState *env = &cpu->env;
1281 for (i = 0; i < 31; i++) {
1282 reg.id = AARCH64_CORE_REG(regs.regs[i]);
1283 reg.addr = (uintptr_t) &env->xregs[i];
1284 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1285 if (ret) {
1286 return ret;
1290 reg.id = AARCH64_CORE_REG(regs.sp);
1291 reg.addr = (uintptr_t) &env->sp_el[0];
1292 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1293 if (ret) {
1294 return ret;
1297 reg.id = AARCH64_CORE_REG(sp_el1);
1298 reg.addr = (uintptr_t) &env->sp_el[1];
1299 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1300 if (ret) {
1301 return ret;
1304 reg.id = AARCH64_CORE_REG(regs.pstate);
1305 reg.addr = (uintptr_t) &val;
1306 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1307 if (ret) {
1308 return ret;
1311 env->aarch64 = ((val & PSTATE_nRW) == 0);
1312 if (is_a64(env)) {
1313 pstate_write(env, val);
1314 } else {
1315 cpsr_write(env, val, 0xffffffff, CPSRWriteRaw);
1318 /* KVM puts SP_EL0 in regs.sp and SP_EL1 in regs.sp_el1. On the
1319 * QEMU side we keep the current SP in xregs[31] as well.
1321 aarch64_restore_sp(env, 1);
1323 reg.id = AARCH64_CORE_REG(regs.pc);
1324 reg.addr = (uintptr_t) &env->pc;
1325 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1326 if (ret) {
1327 return ret;
1330 /* If we are in AArch32 mode then we need to sync the AArch32 regs with the
1331 * incoming AArch64 regs received from 64-bit KVM.
1332 * We must perform this after all of the registers have been acquired from
1333 * the kernel.
1335 if (!is_a64(env)) {
1336 aarch64_sync_64_to_32(env);
1339 reg.id = AARCH64_CORE_REG(elr_el1);
1340 reg.addr = (uintptr_t) &env->elr_el[1];
1341 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1342 if (ret) {
1343 return ret;
1346 /* Fetch the SPSR registers
1348 * KVM SPSRs 0-4 map to QEMU banks 1-5
1350 for (i = 0; i < KVM_NR_SPSR; i++) {
1351 reg.id = AARCH64_CORE_REG(spsr[i]);
1352 reg.addr = (uintptr_t) &env->banked_spsr[i + 1];
1353 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1354 if (ret) {
1355 return ret;
1359 el = arm_current_el(env);
1360 if (el > 0 && !is_a64(env)) {
1361 i = bank_number(env->uncached_cpsr & CPSR_M);
1362 env->spsr = env->banked_spsr[i];
1365 if (cpu_isar_feature(aa64_sve, cpu)) {
1366 ret = kvm_arch_get_sve(cs);
1367 } else {
1368 ret = kvm_arch_get_fpsimd(cs);
1370 if (ret) {
1371 return ret;
1374 reg.addr = (uintptr_t)(&fpr);
1375 reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr);
1376 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1377 if (ret) {
1378 return ret;
1380 vfp_set_fpsr(env, fpr);
1382 reg.addr = (uintptr_t)(&fpr);
1383 reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr);
1384 ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
1385 if (ret) {
1386 return ret;
1388 vfp_set_fpcr(env, fpr);
1390 ret = kvm_get_vcpu_events(cpu);
1391 if (ret) {
1392 return ret;
1395 if (!write_kvmstate_to_list(cpu)) {
1396 return -EINVAL;
1398 /* Note that it's OK to have registers which aren't in CPUState,
1399 * so we can ignore a failure return here.
1401 write_list_to_cpustate(cpu);
1403 kvm_arm_sync_mpstate_to_qemu(cpu);
1405 /* TODO: other registers */
1406 return ret;
1409 void kvm_arch_on_sigbus_vcpu(CPUState *c, int code, void *addr)
1411 ram_addr_t ram_addr;
1412 hwaddr paddr;
1413 Object *obj = qdev_get_machine();
1414 VirtMachineState *vms = VIRT_MACHINE(obj);
1415 bool acpi_enabled = virt_is_acpi_enabled(vms);
1417 assert(code == BUS_MCEERR_AR || code == BUS_MCEERR_AO);
1419 if (acpi_enabled && addr &&
1420 object_property_get_bool(obj, "ras", NULL)) {
1421 ram_addr = qemu_ram_addr_from_host(addr);
1422 if (ram_addr != RAM_ADDR_INVALID &&
1423 kvm_physical_memory_addr_from_host(c->kvm_state, addr, &paddr)) {
1424 kvm_hwpoison_page_add(ram_addr);
1426 * If this is a BUS_MCEERR_AR, we know we have been called
1427 * synchronously from the vCPU thread, so we can easily
1428 * synchronize the state and inject an error.
1430 * TODO: we currently don't tell the guest at all about
1431 * BUS_MCEERR_AO. In that case we might either be being
1432 * called synchronously from the vCPU thread, or a bit
1433 * later from the main thread, so doing the injection of
1434 * the error would be more complicated.
1436 if (code == BUS_MCEERR_AR) {
1437 kvm_cpu_synchronize_state(c);
1438 if (!acpi_ghes_record_errors(ACPI_HEST_SRC_ID_SEA, paddr)) {
1439 kvm_inject_arm_sea(c);
1440 } else {
1441 error_report("failed to record the error");
1442 abort();
1445 return;
1447 if (code == BUS_MCEERR_AO) {
1448 error_report("Hardware memory error at addr %p for memory used by "
1449 "QEMU itself instead of guest system!", addr);
1453 if (code == BUS_MCEERR_AR) {
1454 error_report("Hardware memory error!");
1455 exit(1);
1459 /* C6.6.29 BRK instruction */
1460 static const uint32_t brk_insn = 0xd4200000;
1462 int kvm_arch_insert_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
1464 if (have_guest_debug) {
1465 if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 4, 0) ||
1466 cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&brk_insn, 4, 1)) {
1467 return -EINVAL;
1469 return 0;
1470 } else {
1471 error_report("guest debug not supported on this kernel");
1472 return -EINVAL;
1476 int kvm_arch_remove_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
1478 static uint32_t brk;
1480 if (have_guest_debug) {
1481 if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&brk, 4, 0) ||
1482 brk != brk_insn ||
1483 cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 4, 1)) {
1484 return -EINVAL;
1486 return 0;
1487 } else {
1488 error_report("guest debug not supported on this kernel");
1489 return -EINVAL;
1493 /* See v8 ARM ARM D7.2.27 ESR_ELx, Exception Syndrome Register
1495 * To minimise translating between kernel and user-space the kernel
1496 * ABI just provides user-space with the full exception syndrome
1497 * register value to be decoded in QEMU.
1500 bool kvm_arm_handle_debug(CPUState *cs, struct kvm_debug_exit_arch *debug_exit)
1502 int hsr_ec = syn_get_ec(debug_exit->hsr);
1503 ARMCPU *cpu = ARM_CPU(cs);
1504 CPUARMState *env = &cpu->env;
1506 /* Ensure PC is synchronised */
1507 kvm_cpu_synchronize_state(cs);
1509 switch (hsr_ec) {
1510 case EC_SOFTWARESTEP:
1511 if (cs->singlestep_enabled) {
1512 return true;
1513 } else {
1515 * The kernel should have suppressed the guest's ability to
1516 * single step at this point so something has gone wrong.
1518 error_report("%s: guest single-step while debugging unsupported"
1519 " (%"PRIx64", %"PRIx32")",
1520 __func__, env->pc, debug_exit->hsr);
1521 return false;
1523 break;
1524 case EC_AA64_BKPT:
1525 if (kvm_find_sw_breakpoint(cs, env->pc)) {
1526 return true;
1528 break;
1529 case EC_BREAKPOINT:
1530 if (find_hw_breakpoint(cs, env->pc)) {
1531 return true;
1533 break;
1534 case EC_WATCHPOINT:
1536 CPUWatchpoint *wp = find_hw_watchpoint(cs, debug_exit->far);
1537 if (wp) {
1538 cs->watchpoint_hit = wp;
1539 return true;
1541 break;
1543 default:
1544 error_report("%s: unhandled debug exit (%"PRIx32", %"PRIx64")",
1545 __func__, debug_exit->hsr, env->pc);
1548 /* If we are not handling the debug exception it must belong to
1549 * the guest. Let's re-use the existing TCG interrupt code to set
1550 * everything up properly.
1552 cs->exception_index = EXCP_BKPT;
1553 env->exception.syndrome = debug_exit->hsr;
1554 env->exception.vaddress = debug_exit->far;
1555 env->exception.target_el = 1;
1556 qemu_mutex_lock_iothread();
1557 arm_cpu_do_interrupt(cs);
1558 qemu_mutex_unlock_iothread();
1560 return false;
1563 #define ARM64_REG_ESR_EL1 ARM64_SYS_REG(3, 0, 5, 2, 0)
1564 #define ARM64_REG_TCR_EL1 ARM64_SYS_REG(3, 0, 2, 0, 2)
1567 * ESR_EL1
1568 * ISS encoding
1569 * AARCH64: DFSC, bits [5:0]
1570 * AARCH32:
1571 * TTBCR.EAE == 0
1572 * FS[4] - DFSR[10]
1573 * FS[3:0] - DFSR[3:0]
1574 * TTBCR.EAE == 1
1575 * FS, bits [5:0]
1577 #define ESR_DFSC(aarch64, lpae, v) \
1578 ((aarch64 || (lpae)) ? ((v) & 0x3F) \
1579 : (((v) >> 6) | ((v) & 0x1F)))
1581 #define ESR_DFSC_EXTABT(aarch64, lpae) \
1582 ((aarch64) ? 0x10 : (lpae) ? 0x10 : 0x8)
1584 bool kvm_arm_verify_ext_dabt_pending(CPUState *cs)
1586 uint64_t dfsr_val;
1588 if (!kvm_get_one_reg(cs, ARM64_REG_ESR_EL1, &dfsr_val)) {
1589 ARMCPU *cpu = ARM_CPU(cs);
1590 CPUARMState *env = &cpu->env;
1591 int aarch64_mode = arm_feature(env, ARM_FEATURE_AARCH64);
1592 int lpae = 0;
1594 if (!aarch64_mode) {
1595 uint64_t ttbcr;
1597 if (!kvm_get_one_reg(cs, ARM64_REG_TCR_EL1, &ttbcr)) {
1598 lpae = arm_feature(env, ARM_FEATURE_LPAE)
1599 && (ttbcr & TTBCR_EAE);
1603 * The verification here is based on the DFSC bits
1604 * of the ESR_EL1 reg only
1606 return (ESR_DFSC(aarch64_mode, lpae, dfsr_val) ==
1607 ESR_DFSC_EXTABT(aarch64_mode, lpae));
1609 return false;