2 * x86 exception helpers - sysemu code
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
22 #include "exec/exec-all.h"
23 #include "tcg/helper-tcg.h"
25 int get_pg_mode(CPUX86State
*env
)
28 if (!(env
->cr
[0] & CR0_PG_MASK
)) {
31 if (env
->cr
[0] & CR0_WP_MASK
) {
32 pg_mode
|= PG_MODE_WP
;
34 if (env
->cr
[4] & CR4_PAE_MASK
) {
35 pg_mode
|= PG_MODE_PAE
;
36 if (env
->efer
& MSR_EFER_NXE
) {
37 pg_mode
|= PG_MODE_NXE
;
40 if (env
->cr
[4] & CR4_PSE_MASK
) {
41 pg_mode
|= PG_MODE_PSE
;
43 if (env
->cr
[4] & CR4_SMEP_MASK
) {
44 pg_mode
|= PG_MODE_SMEP
;
46 if (env
->hflags
& HF_LMA_MASK
) {
47 pg_mode
|= PG_MODE_LMA
;
48 if (env
->cr
[4] & CR4_PKE_MASK
) {
49 pg_mode
|= PG_MODE_PKE
;
51 if (env
->cr
[4] & CR4_PKS_MASK
) {
52 pg_mode
|= PG_MODE_PKS
;
54 if (env
->cr
[4] & CR4_LA57_MASK
) {
55 pg_mode
|= PG_MODE_LA57
;
61 #define PG_ERROR_OK (-1)
63 typedef hwaddr (*MMUTranslateFunc
)(CPUState
*cs
, hwaddr gphys
, MMUAccessType access_type
,
66 #define GET_HPHYS(cs, gpa, access_type, prot) \
67 (get_hphys_func ? get_hphys_func(cs, gpa, access_type, prot) : gpa)
69 static int mmu_translate(CPUState
*cs
, hwaddr addr
, MMUTranslateFunc get_hphys_func
,
70 uint64_t cr3
, int is_write1
, int mmu_idx
, int pg_mode
,
71 hwaddr
*xlat
, int *page_size
, int *prot
)
73 X86CPU
*cpu
= X86_CPU(cs
);
74 CPUX86State
*env
= &cpu
->env
;
77 target_ulong pde_addr
, pte_addr
;
79 int is_dirty
, is_write
, is_user
;
80 uint64_t rsvd_mask
= PG_ADDRESS_MASK
& ~MAKE_64BIT_MASK(0, cpu
->phys_bits
);
84 is_user
= (mmu_idx
== MMU_USER_IDX
);
85 is_write
= is_write1
& 1;
86 a20_mask
= x86_get_a20_mask(env
);
88 if (!(pg_mode
& PG_MODE_NXE
)) {
89 rsvd_mask
|= PG_NX_MASK
;
92 if (pg_mode
& PG_MODE_PAE
) {
94 target_ulong pdpe_addr
;
97 if (pg_mode
& PG_MODE_LMA
) {
98 bool la57
= pg_mode
& PG_MODE_LA57
;
99 uint64_t pml5e_addr
, pml5e
;
100 uint64_t pml4e_addr
, pml4e
;
103 pml5e_addr
= ((cr3
& ~0xfff) +
104 (((addr
>> 48) & 0x1ff) << 3)) & a20_mask
;
105 pml5e_addr
= GET_HPHYS(cs
, pml5e_addr
, MMU_DATA_STORE
, NULL
);
106 pml5e
= x86_ldq_phys(cs
, pml5e_addr
);
107 if (!(pml5e
& PG_PRESENT_MASK
)) {
110 if (pml5e
& (rsvd_mask
| PG_PSE_MASK
)) {
113 if (!(pml5e
& PG_ACCESSED_MASK
)) {
114 pml5e
|= PG_ACCESSED_MASK
;
115 x86_stl_phys_notdirty(cs
, pml5e_addr
, pml5e
);
117 ptep
= pml5e
^ PG_NX_MASK
;
120 ptep
= PG_NX_MASK
| PG_USER_MASK
| PG_RW_MASK
;
123 pml4e_addr
= ((pml5e
& PG_ADDRESS_MASK
) +
124 (((addr
>> 39) & 0x1ff) << 3)) & a20_mask
;
125 pml4e_addr
= GET_HPHYS(cs
, pml4e_addr
, MMU_DATA_STORE
, NULL
);
126 pml4e
= x86_ldq_phys(cs
, pml4e_addr
);
127 if (!(pml4e
& PG_PRESENT_MASK
)) {
130 if (pml4e
& (rsvd_mask
| PG_PSE_MASK
)) {
133 if (!(pml4e
& PG_ACCESSED_MASK
)) {
134 pml4e
|= PG_ACCESSED_MASK
;
135 x86_stl_phys_notdirty(cs
, pml4e_addr
, pml4e
);
137 ptep
&= pml4e
^ PG_NX_MASK
;
138 pdpe_addr
= ((pml4e
& PG_ADDRESS_MASK
) + (((addr
>> 30) & 0x1ff) << 3)) &
140 pdpe_addr
= GET_HPHYS(cs
, pdpe_addr
, MMU_DATA_STORE
, NULL
);
141 pdpe
= x86_ldq_phys(cs
, pdpe_addr
);
142 if (!(pdpe
& PG_PRESENT_MASK
)) {
145 if (pdpe
& rsvd_mask
) {
148 ptep
&= pdpe
^ PG_NX_MASK
;
149 if (!(pdpe
& PG_ACCESSED_MASK
)) {
150 pdpe
|= PG_ACCESSED_MASK
;
151 x86_stl_phys_notdirty(cs
, pdpe_addr
, pdpe
);
153 if (pdpe
& PG_PSE_MASK
) {
155 *page_size
= 1024 * 1024 * 1024;
156 pte_addr
= pdpe_addr
;
158 goto do_check_protect
;
163 /* XXX: load them when cr3 is loaded ? */
164 pdpe_addr
= ((cr3
& ~0x1f) + ((addr
>> 27) & 0x18)) &
166 pdpe_addr
= GET_HPHYS(cs
, pdpe_addr
, MMU_DATA_STORE
, NULL
);
167 pdpe
= x86_ldq_phys(cs
, pdpe_addr
);
168 if (!(pdpe
& PG_PRESENT_MASK
)) {
171 rsvd_mask
|= PG_HI_USER_MASK
;
172 if (pdpe
& (rsvd_mask
| PG_NX_MASK
)) {
175 ptep
= PG_NX_MASK
| PG_USER_MASK
| PG_RW_MASK
;
178 pde_addr
= ((pdpe
& PG_ADDRESS_MASK
) + (((addr
>> 21) & 0x1ff) << 3)) &
180 pde_addr
= GET_HPHYS(cs
, pde_addr
, MMU_DATA_STORE
, NULL
);
181 pde
= x86_ldq_phys(cs
, pde_addr
);
182 if (!(pde
& PG_PRESENT_MASK
)) {
185 if (pde
& rsvd_mask
) {
188 ptep
&= pde
^ PG_NX_MASK
;
189 if (pde
& PG_PSE_MASK
) {
191 *page_size
= 2048 * 1024;
194 goto do_check_protect
;
197 if (!(pde
& PG_ACCESSED_MASK
)) {
198 pde
|= PG_ACCESSED_MASK
;
199 x86_stl_phys_notdirty(cs
, pde_addr
, pde
);
201 pte_addr
= ((pde
& PG_ADDRESS_MASK
) + (((addr
>> 12) & 0x1ff) << 3)) &
203 pte_addr
= GET_HPHYS(cs
, pte_addr
, MMU_DATA_STORE
, NULL
);
204 pte
= x86_ldq_phys(cs
, pte_addr
);
205 if (!(pte
& PG_PRESENT_MASK
)) {
208 if (pte
& rsvd_mask
) {
211 /* combine pde and pte nx, user and rw protections */
212 ptep
&= pte
^ PG_NX_MASK
;
217 /* page directory entry */
218 pde_addr
= ((cr3
& ~0xfff) + ((addr
>> 20) & 0xffc)) &
220 pde_addr
= GET_HPHYS(cs
, pde_addr
, MMU_DATA_STORE
, NULL
);
221 pde
= x86_ldl_phys(cs
, pde_addr
);
222 if (!(pde
& PG_PRESENT_MASK
)) {
225 ptep
= pde
| PG_NX_MASK
;
227 /* if PSE bit is set, then we use a 4MB page */
228 if ((pde
& PG_PSE_MASK
) && (pg_mode
& PG_MODE_PSE
)) {
229 *page_size
= 4096 * 1024;
232 /* Bits 20-13 provide bits 39-32 of the address, bit 21 is reserved.
233 * Leave bits 20-13 in place for setting accessed/dirty bits below.
235 pte
= pde
| ((pde
& 0x1fe000LL
) << (32 - 13));
236 rsvd_mask
= 0x200000;
237 goto do_check_protect_pse36
;
240 if (!(pde
& PG_ACCESSED_MASK
)) {
241 pde
|= PG_ACCESSED_MASK
;
242 x86_stl_phys_notdirty(cs
, pde_addr
, pde
);
245 /* page directory entry */
246 pte_addr
= ((pde
& ~0xfff) + ((addr
>> 10) & 0xffc)) &
248 pte_addr
= GET_HPHYS(cs
, pte_addr
, MMU_DATA_STORE
, NULL
);
249 pte
= x86_ldl_phys(cs
, pte_addr
);
250 if (!(pte
& PG_PRESENT_MASK
)) {
253 /* combine pde and pte user and rw protections */
254 ptep
&= pte
| PG_NX_MASK
;
260 rsvd_mask
|= (*page_size
- 1) & PG_ADDRESS_MASK
& ~PG_PSE_PAT_MASK
;
261 do_check_protect_pse36
:
262 if (pte
& rsvd_mask
) {
267 /* can the page can be put in the TLB? prot will tell us */
268 if (is_user
&& !(ptep
& PG_USER_MASK
)) {
269 goto do_fault_protect
;
273 if (mmu_idx
!= MMU_KSMAP_IDX
|| !(ptep
& PG_USER_MASK
)) {
275 if ((ptep
& PG_RW_MASK
) || !(is_user
|| (pg_mode
& PG_MODE_WP
))) {
279 if (!(ptep
& PG_NX_MASK
) &&
280 (mmu_idx
== MMU_USER_IDX
||
281 !((pg_mode
& PG_MODE_SMEP
) && (ptep
& PG_USER_MASK
)))) {
285 if (ptep
& PG_USER_MASK
) {
286 pkr
= pg_mode
& PG_MODE_PKE
? env
->pkru
: 0;
288 pkr
= pg_mode
& PG_MODE_PKS
? env
->pkrs
: 0;
291 uint32_t pk
= (pte
& PG_PKRU_MASK
) >> PG_PKRU_BIT
;
292 uint32_t pkr_ad
= (pkr
>> pk
* 2) & 1;
293 uint32_t pkr_wd
= (pkr
>> pk
* 2) & 2;
294 uint32_t pkr_prot
= PAGE_READ
| PAGE_WRITE
| PAGE_EXEC
;
297 pkr_prot
&= ~(PAGE_READ
| PAGE_WRITE
);
298 } else if (pkr_wd
&& (is_user
|| (pg_mode
& PG_MODE_WP
))) {
299 pkr_prot
&= ~PAGE_WRITE
;
303 if ((pkr_prot
& (1 << is_write1
)) == 0) {
304 assert(is_write1
!= 2);
305 error_code
|= PG_ERROR_PK_MASK
;
306 goto do_fault_protect
;
310 if ((*prot
& (1 << is_write1
)) == 0) {
311 goto do_fault_protect
;
315 is_dirty
= is_write
&& !(pte
& PG_DIRTY_MASK
);
316 if (!(pte
& PG_ACCESSED_MASK
) || is_dirty
) {
317 pte
|= PG_ACCESSED_MASK
;
319 pte
|= PG_DIRTY_MASK
;
321 x86_stl_phys_notdirty(cs
, pte_addr
, pte
);
324 if (!(pte
& PG_DIRTY_MASK
)) {
325 /* only set write access if already dirty... otherwise wait
328 *prot
&= ~PAGE_WRITE
;
331 pte
= pte
& a20_mask
;
333 /* align to page_size */
334 pte
&= PG_ADDRESS_MASK
& ~(*page_size
- 1);
335 page_offset
= addr
& (*page_size
- 1);
336 *xlat
= GET_HPHYS(cs
, pte
+ page_offset
, is_write1
, prot
);
340 error_code
|= PG_ERROR_RSVD_MASK
;
342 error_code
|= PG_ERROR_P_MASK
;
344 error_code
|= (is_write
<< PG_ERROR_W_BIT
);
346 error_code
|= PG_ERROR_U_MASK
;
347 if (is_write1
== 2 &&
348 ((pg_mode
& PG_MODE_NXE
) || (pg_mode
& PG_MODE_SMEP
)))
349 error_code
|= PG_ERROR_I_D_MASK
;
353 hwaddr
get_hphys(CPUState
*cs
, hwaddr gphys
, MMUAccessType access_type
,
356 CPUX86State
*env
= &X86_CPU(cs
)->env
;
357 uint64_t exit_info_1
;
362 if (likely(!(env
->hflags2
& HF2_NPT_MASK
))) {
366 exit_info_1
= mmu_translate(cs
, gphys
, NULL
, env
->nested_cr3
,
367 access_type
, MMU_USER_IDX
, env
->nested_pg_mode
,
368 &hphys
, &page_size
, &next_prot
);
369 if (exit_info_1
== PG_ERROR_OK
) {
376 x86_stq_phys(cs
, env
->vm_vmcb
+ offsetof(struct vmcb
, control
.exit_info_2
),
379 exit_info_1
|= SVM_NPTEXIT_GPA
;
380 } else { /* page table access */
381 exit_info_1
|= SVM_NPTEXIT_GPT
;
383 cpu_vmexit(env
, SVM_EXIT_NPF
, exit_info_1
, env
->retaddr
);
387 * -1 = cannot handle fault
388 * 0 = nothing more to do
389 * 1 = generate PF fault
391 static int handle_mmu_fault(CPUState
*cs
, vaddr addr
, int size
,
392 int is_write1
, int mmu_idx
)
394 X86CPU
*cpu
= X86_CPU(cs
);
395 CPUX86State
*env
= &cpu
->env
;
396 int error_code
= PG_ERROR_OK
;
397 int pg_mode
, prot
, page_size
;
401 #if defined(DEBUG_MMU)
402 printf("MMU fault: addr=%" VADDR_PRIx
" w=%d mmu=%d eip=" TARGET_FMT_lx
"\n",
403 addr
, is_write1
, mmu_idx
, env
->eip
);
406 if (!(env
->cr
[0] & CR0_PG_MASK
)) {
409 if (!(env
->hflags
& HF_LMA_MASK
)) {
410 /* Without long mode we can only address 32bits in real mode */
411 paddr
= (uint32_t)paddr
;
414 prot
= PAGE_READ
| PAGE_WRITE
| PAGE_EXEC
;
417 pg_mode
= get_pg_mode(env
);
418 if (pg_mode
& PG_MODE_LMA
) {
421 /* test virtual address sign extension */
422 sext
= (int64_t)addr
>> (pg_mode
& PG_MODE_LA57
? 56 : 47);
423 if (sext
!= 0 && sext
!= -1) {
425 cs
->exception_index
= EXCP0D_GPF
;
430 error_code
= mmu_translate(cs
, addr
, get_hphys
, env
->cr
[3], is_write1
,
432 &paddr
, &page_size
, &prot
);
435 if (error_code
== PG_ERROR_OK
) {
436 /* Even if 4MB pages, we map only one 4KB page in the cache to
437 avoid filling it too fast */
438 vaddr
= addr
& TARGET_PAGE_MASK
;
439 paddr
&= TARGET_PAGE_MASK
;
441 assert(prot
& (1 << is_write1
));
442 tlb_set_page_with_attrs(cs
, vaddr
, paddr
, cpu_get_mem_attrs(env
),
443 prot
, mmu_idx
, page_size
);
446 if (env
->intercept_exceptions
& (1 << EXCP0E_PAGE
)) {
447 /* cr2 is not modified in case of exceptions */
449 env
->vm_vmcb
+ offsetof(struct vmcb
, control
.exit_info_2
),
454 env
->error_code
= error_code
;
455 cs
->exception_index
= EXCP0E_PAGE
;
460 bool x86_cpu_tlb_fill(CPUState
*cs
, vaddr addr
, int size
,
461 MMUAccessType access_type
, int mmu_idx
,
462 bool probe
, uintptr_t retaddr
)
464 X86CPU
*cpu
= X86_CPU(cs
);
465 CPUX86State
*env
= &cpu
->env
;
467 env
->retaddr
= retaddr
;
468 if (handle_mmu_fault(cs
, addr
, size
, access_type
, mmu_idx
)) {
469 /* FIXME: On error in get_hphys we have already jumped out. */
471 raise_exception_err_ra(env
, cs
->exception_index
,
472 env
->error_code
, retaddr
);