2 * Kernel-based Virtual Machine driver for Linux
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
9 * Copyright (C) 2006 Qumranet, Inc.
12 * Yaniv Kamay <yaniv@qumranet.com>
13 * Avi Kivity <avi@qumranet.com>
15 * This work is licensed under the terms of the GNU GPL, version 2. See
16 * the COPYING file in the top-level directory.
21 * We need the mmu code to access both 32-bit and 64-bit guest ptes,
22 * so the code in this file is compiled twice, once per pte size.
26 #define pt_element_t u64
27 #define guest_walker guest_walker64
28 #define shadow_walker shadow_walker64
29 #define FNAME(name) paging##64_##name
30 #define PT_BASE_ADDR_MASK PT64_BASE_ADDR_MASK
31 #define PT_DIR_BASE_ADDR_MASK PT64_DIR_BASE_ADDR_MASK
32 #define PT_INDEX(addr, level) PT64_INDEX(addr, level)
33 #define PT_LEVEL_MASK(level) PT64_LEVEL_MASK(level)
34 #define PT_LEVEL_BITS PT64_LEVEL_BITS
36 #define PT_MAX_FULL_LEVELS 4
37 #define CMPXCHG cmpxchg
39 #define CMPXCHG cmpxchg64
40 #define PT_MAX_FULL_LEVELS 2
43 #define pt_element_t u32
44 #define guest_walker guest_walker32
45 #define shadow_walker shadow_walker32
46 #define FNAME(name) paging##32_##name
47 #define PT_BASE_ADDR_MASK PT32_BASE_ADDR_MASK
48 #define PT_DIR_BASE_ADDR_MASK PT32_DIR_BASE_ADDR_MASK
49 #define PT_INDEX(addr, level) PT32_INDEX(addr, level)
50 #define PT_LEVEL_MASK(level) PT32_LEVEL_MASK(level)
51 #define PT_LEVEL_BITS PT32_LEVEL_BITS
52 #define PT_MAX_FULL_LEVELS 2
53 #define CMPXCHG cmpxchg
55 #error Invalid PTTYPE value
58 #define gpte_to_gfn FNAME(gpte_to_gfn)
59 #define gpte_to_gfn_pde FNAME(gpte_to_gfn_pde)
62 * The guest_walker structure emulates the behavior of the hardware page
67 gfn_t table_gfn
[PT_MAX_FULL_LEVELS
];
68 pt_element_t ptes
[PT_MAX_FULL_LEVELS
];
69 gpa_t pte_gpa
[PT_MAX_FULL_LEVELS
];
76 struct shadow_walker
{
77 struct kvm_shadow_walk walker
;
78 struct guest_walker
*guest_walker
;
88 static gfn_t
gpte_to_gfn(pt_element_t gpte
)
90 return (gpte
& PT_BASE_ADDR_MASK
) >> PAGE_SHIFT
;
93 static gfn_t
gpte_to_gfn_pde(pt_element_t gpte
)
95 return (gpte
& PT_DIR_BASE_ADDR_MASK
) >> PAGE_SHIFT
;
98 static bool FNAME(cmpxchg_gpte
)(struct kvm
*kvm
,
99 gfn_t table_gfn
, unsigned index
,
100 pt_element_t orig_pte
, pt_element_t new_pte
)
106 page
= gfn_to_page(kvm
, table_gfn
);
108 table
= kmap_atomic(page
, KM_USER0
);
109 ret
= CMPXCHG(&table
[index
], orig_pte
, new_pte
);
110 kunmap_atomic(table
, KM_USER0
);
112 kvm_release_page_dirty(page
);
114 return (ret
!= orig_pte
);
117 static unsigned FNAME(gpte_access
)(struct kvm_vcpu
*vcpu
, pt_element_t gpte
)
121 access
= (gpte
& (PT_WRITABLE_MASK
| PT_USER_MASK
)) | ACC_EXEC_MASK
;
124 access
&= ~(gpte
>> PT64_NX_SHIFT
);
130 * Fetch a guest pte for a guest virtual address
132 static int FNAME(walk_addr
)(struct guest_walker
*walker
,
133 struct kvm_vcpu
*vcpu
, gva_t addr
,
134 int write_fault
, int user_fault
, int fetch_fault
)
138 unsigned index
, pt_access
, pte_access
;
141 pgprintk("%s: addr %lx\n", __func__
, addr
);
143 walker
->level
= vcpu
->arch
.mmu
.root_level
;
144 pte
= vcpu
->arch
.cr3
;
146 if (!is_long_mode(vcpu
)) {
147 pte
= vcpu
->arch
.pdptrs
[(addr
>> 30) & 3];
148 if (!is_present_pte(pte
))
153 ASSERT((!is_long_mode(vcpu
) && is_pae(vcpu
)) ||
154 (vcpu
->arch
.cr3
& CR3_NONPAE_RESERVED_BITS
) == 0);
159 index
= PT_INDEX(addr
, walker
->level
);
161 table_gfn
= gpte_to_gfn(pte
);
162 pte_gpa
= gfn_to_gpa(table_gfn
);
163 pte_gpa
+= index
* sizeof(pt_element_t
);
164 walker
->table_gfn
[walker
->level
- 1] = table_gfn
;
165 walker
->pte_gpa
[walker
->level
- 1] = pte_gpa
;
166 pgprintk("%s: table_gfn[%d] %lx\n", __func__
,
167 walker
->level
- 1, table_gfn
);
169 kvm_read_guest(vcpu
->kvm
, pte_gpa
, &pte
, sizeof(pte
));
171 if (!is_present_pte(pte
))
174 if (write_fault
&& !is_writeble_pte(pte
))
175 if (user_fault
|| is_write_protection(vcpu
))
178 if (user_fault
&& !(pte
& PT_USER_MASK
))
182 if (fetch_fault
&& is_nx(vcpu
) && (pte
& PT64_NX_MASK
))
186 if (!(pte
& PT_ACCESSED_MASK
)) {
187 mark_page_dirty(vcpu
->kvm
, table_gfn
);
188 if (FNAME(cmpxchg_gpte
)(vcpu
->kvm
, table_gfn
,
189 index
, pte
, pte
|PT_ACCESSED_MASK
))
191 pte
|= PT_ACCESSED_MASK
;
194 pte_access
= pt_access
& FNAME(gpte_access
)(vcpu
, pte
);
196 walker
->ptes
[walker
->level
- 1] = pte
;
198 if (walker
->level
== PT_PAGE_TABLE_LEVEL
) {
199 walker
->gfn
= gpte_to_gfn(pte
);
203 if (walker
->level
== PT_DIRECTORY_LEVEL
204 && (pte
& PT_PAGE_SIZE_MASK
)
205 && (PTTYPE
== 64 || is_pse(vcpu
))) {
206 walker
->gfn
= gpte_to_gfn_pde(pte
);
207 walker
->gfn
+= PT_INDEX(addr
, PT_PAGE_TABLE_LEVEL
);
208 if (PTTYPE
== 32 && is_cpuid_PSE36())
209 walker
->gfn
+= pse36_gfn_delta(pte
);
213 pt_access
= pte_access
;
217 if (write_fault
&& !is_dirty_pte(pte
)) {
220 mark_page_dirty(vcpu
->kvm
, table_gfn
);
221 ret
= FNAME(cmpxchg_gpte
)(vcpu
->kvm
, table_gfn
, index
, pte
,
225 pte
|= PT_DIRTY_MASK
;
226 kvm_mmu_pte_write(vcpu
, pte_gpa
, (u8
*)&pte
, sizeof(pte
), 0);
227 walker
->ptes
[walker
->level
- 1] = pte
;
230 walker
->pt_access
= pt_access
;
231 walker
->pte_access
= pte_access
;
232 pgprintk("%s: pte %llx pte_access %x pt_access %x\n",
233 __func__
, (u64
)pte
, pt_access
, pte_access
);
237 walker
->error_code
= 0;
241 walker
->error_code
= PFERR_PRESENT_MASK
;
245 walker
->error_code
|= PFERR_WRITE_MASK
;
247 walker
->error_code
|= PFERR_USER_MASK
;
249 walker
->error_code
|= PFERR_FETCH_MASK
;
253 static void FNAME(update_pte
)(struct kvm_vcpu
*vcpu
, struct kvm_mmu_page
*page
,
254 u64
*spte
, const void *pte
)
259 int largepage
= vcpu
->arch
.update_pte
.largepage
;
261 gpte
= *(const pt_element_t
*)pte
;
262 if (~gpte
& (PT_PRESENT_MASK
| PT_ACCESSED_MASK
)) {
263 if (!is_present_pte(gpte
))
264 set_shadow_pte(spte
, shadow_notrap_nonpresent_pte
);
267 pgprintk("%s: gpte %llx spte %p\n", __func__
, (u64
)gpte
, spte
);
268 pte_access
= page
->role
.access
& FNAME(gpte_access
)(vcpu
, gpte
);
269 if (gpte_to_gfn(gpte
) != vcpu
->arch
.update_pte
.gfn
)
271 pfn
= vcpu
->arch
.update_pte
.pfn
;
272 if (is_error_pfn(pfn
))
274 if (mmu_notifier_retry(vcpu
, vcpu
->arch
.update_pte
.mmu_seq
))
277 mmu_set_spte(vcpu
, spte
, page
->role
.access
, pte_access
, 0, 0,
278 gpte
& PT_DIRTY_MASK
, NULL
, largepage
,
279 gpte
& PT_GLOBAL_MASK
, gpte_to_gfn(gpte
),
284 * Fetch a shadow pte for a specific level in the paging hierarchy.
286 static u64
*FNAME(fetch
)(struct kvm_vcpu
*vcpu
, gva_t addr
,
287 struct guest_walker
*gw
,
288 int user_fault
, int write_fault
, int largepage
,
289 int *ptwrite
, pfn_t pfn
)
291 unsigned access
= gw
->pt_access
;
292 struct kvm_mmu_page
*shadow_page
;
298 pt_element_t curr_pte
;
299 struct kvm_shadow_walk_iterator iterator
;
301 if (!is_present_pte(gw
->ptes
[gw
->level
- 1]))
304 for_each_shadow_entry(vcpu
, addr
, iterator
) {
305 level
= iterator
.level
;
306 sptep
= iterator
.sptep
;
307 if (level
== PT_PAGE_TABLE_LEVEL
308 || (largepage
&& level
== PT_DIRECTORY_LEVEL
)) {
309 mmu_set_spte(vcpu
, sptep
, access
,
310 gw
->pte_access
& access
,
311 user_fault
, write_fault
,
312 gw
->ptes
[gw
->level
-1] & PT_DIRTY_MASK
,
314 gw
->ptes
[gw
->level
-1] & PT_GLOBAL_MASK
,
315 gw
->gfn
, pfn
, false);
319 if (is_shadow_present_pte(*sptep
) && !is_large_pte(*sptep
))
322 if (is_large_pte(*sptep
)) {
323 set_shadow_pte(sptep
, shadow_trap_nonpresent_pte
);
324 kvm_flush_remote_tlbs(vcpu
->kvm
);
325 rmap_remove(vcpu
->kvm
, sptep
);
328 if (level
== PT_DIRECTORY_LEVEL
329 && gw
->level
== PT_DIRECTORY_LEVEL
) {
331 if (!is_dirty_pte(gw
->ptes
[level
- 1]))
332 access
&= ~ACC_WRITE_MASK
;
333 table_gfn
= gpte_to_gfn(gw
->ptes
[level
- 1]);
336 table_gfn
= gw
->table_gfn
[level
- 2];
338 shadow_page
= kvm_mmu_get_page(vcpu
, table_gfn
, addr
, level
-1,
339 metaphysical
, access
, sptep
);
341 r
= kvm_read_guest_atomic(vcpu
->kvm
,
342 gw
->pte_gpa
[level
- 2],
343 &curr_pte
, sizeof(curr_pte
));
344 if (r
|| curr_pte
!= gw
->ptes
[level
- 2]) {
345 kvm_mmu_put_page(shadow_page
, sptep
);
346 kvm_release_pfn_clean(pfn
);
352 spte
= __pa(shadow_page
->spt
)
353 | PT_PRESENT_MASK
| PT_ACCESSED_MASK
354 | PT_WRITABLE_MASK
| PT_USER_MASK
;
362 * Page fault handler. There are several causes for a page fault:
363 * - there is no shadow pte for the guest pte
364 * - write access through a shadow pte marked read only so that we can set
366 * - write access to a shadow pte marked read only so we can update the page
367 * dirty bitmap, when userspace requests it
368 * - mmio access; in this case we will never install a present shadow pte
369 * - normal guest page fault due to the guest pte marked not present, not
370 * writable, or not executable
372 * Returns: 1 if we need to emulate the instruction, 0 otherwise, or
373 * a negative value on error.
375 static int FNAME(page_fault
)(struct kvm_vcpu
*vcpu
, gva_t addr
,
378 int write_fault
= error_code
& PFERR_WRITE_MASK
;
379 int user_fault
= error_code
& PFERR_USER_MASK
;
380 int fetch_fault
= error_code
& PFERR_FETCH_MASK
;
381 struct guest_walker walker
;
387 unsigned long mmu_seq
;
389 pgprintk("%s: addr %lx err %x\n", __func__
, addr
, error_code
);
390 kvm_mmu_audit(vcpu
, "pre page fault");
392 r
= mmu_topup_memory_caches(vcpu
);
397 * Look up the shadow pte for the faulting address.
399 r
= FNAME(walk_addr
)(&walker
, vcpu
, addr
, write_fault
, user_fault
,
403 * The page is not mapped by the guest. Let the guest handle it.
406 pgprintk("%s: guest page fault\n", __func__
);
407 inject_page_fault(vcpu
, addr
, walker
.error_code
);
408 vcpu
->arch
.last_pt_write_count
= 0; /* reset fork detector */
412 if (walker
.level
== PT_DIRECTORY_LEVEL
) {
414 large_gfn
= walker
.gfn
& ~(KVM_PAGES_PER_HPAGE
-1);
415 if (is_largepage_backed(vcpu
, large_gfn
)) {
416 walker
.gfn
= large_gfn
;
420 mmu_seq
= vcpu
->kvm
->mmu_notifier_seq
;
422 pfn
= gfn_to_pfn(vcpu
->kvm
, walker
.gfn
);
425 if (is_error_pfn(pfn
)) {
426 pgprintk("gfn %lx is mmio\n", walker
.gfn
);
427 kvm_release_pfn_clean(pfn
);
431 spin_lock(&vcpu
->kvm
->mmu_lock
);
432 if (mmu_notifier_retry(vcpu
, mmu_seq
))
434 kvm_mmu_free_some_pages(vcpu
);
435 shadow_pte
= FNAME(fetch
)(vcpu
, addr
, &walker
, user_fault
, write_fault
,
436 largepage
, &write_pt
, pfn
);
438 pgprintk("%s: shadow pte %p %llx ptwrite %d\n", __func__
,
439 shadow_pte
, *shadow_pte
, write_pt
);
442 vcpu
->arch
.last_pt_write_count
= 0; /* reset fork detector */
444 ++vcpu
->stat
.pf_fixed
;
445 kvm_mmu_audit(vcpu
, "post page fault (fixed)");
446 spin_unlock(&vcpu
->kvm
->mmu_lock
);
451 spin_unlock(&vcpu
->kvm
->mmu_lock
);
452 kvm_release_pfn_clean(pfn
);
456 static int FNAME(shadow_invlpg_entry
)(struct kvm_shadow_walk
*_sw
,
457 struct kvm_vcpu
*vcpu
, u64 addr
,
458 u64
*sptep
, int level
)
460 struct shadow_walker
*sw
=
461 container_of(_sw
, struct shadow_walker
, walker
);
463 /* FIXME: properly handle invlpg on large guest pages */
464 if (level
== PT_PAGE_TABLE_LEVEL
||
465 ((level
== PT_DIRECTORY_LEVEL
) && is_large_pte(*sptep
))) {
466 struct kvm_mmu_page
*sp
= page_header(__pa(sptep
));
468 sw
->pte_gpa
= (sp
->gfn
<< PAGE_SHIFT
);
469 sw
->pte_gpa
+= (sptep
- sp
->spt
) * sizeof(pt_element_t
);
471 if (is_shadow_present_pte(*sptep
)) {
472 rmap_remove(vcpu
->kvm
, sptep
);
473 if (is_large_pte(*sptep
))
474 --vcpu
->kvm
->stat
.lpages
;
476 set_shadow_pte(sptep
, shadow_trap_nonpresent_pte
);
479 if (!is_shadow_present_pte(*sptep
))
484 static void FNAME(invlpg
)(struct kvm_vcpu
*vcpu
, gva_t gva
)
487 struct shadow_walker walker
= {
488 .walker
= { .entry
= FNAME(shadow_invlpg_entry
), },
492 spin_lock(&vcpu
->kvm
->mmu_lock
);
493 walk_shadow(&walker
.walker
, vcpu
, gva
);
494 spin_unlock(&vcpu
->kvm
->mmu_lock
);
495 if (walker
.pte_gpa
== -1)
497 if (kvm_read_guest_atomic(vcpu
->kvm
, walker
.pte_gpa
, &gpte
,
498 sizeof(pt_element_t
)))
500 if (is_present_pte(gpte
) && (gpte
& PT_ACCESSED_MASK
)) {
501 if (mmu_topup_memory_caches(vcpu
))
503 kvm_mmu_pte_write(vcpu
, walker
.pte_gpa
, (const u8
*)&gpte
,
504 sizeof(pt_element_t
), 0);
508 static gpa_t
FNAME(gva_to_gpa
)(struct kvm_vcpu
*vcpu
, gva_t vaddr
)
510 struct guest_walker walker
;
511 gpa_t gpa
= UNMAPPED_GVA
;
514 r
= FNAME(walk_addr
)(&walker
, vcpu
, vaddr
, 0, 0, 0);
517 gpa
= gfn_to_gpa(walker
.gfn
);
518 gpa
|= vaddr
& ~PAGE_MASK
;
524 static void FNAME(prefetch_page
)(struct kvm_vcpu
*vcpu
,
525 struct kvm_mmu_page
*sp
)
528 pt_element_t pt
[256 / sizeof(pt_element_t
)];
531 if (sp
->role
.metaphysical
532 || (PTTYPE
== 32 && sp
->role
.level
> PT_PAGE_TABLE_LEVEL
)) {
533 nonpaging_prefetch_page(vcpu
, sp
);
537 pte_gpa
= gfn_to_gpa(sp
->gfn
);
539 offset
= sp
->role
.quadrant
<< PT64_LEVEL_BITS
;
540 pte_gpa
+= offset
* sizeof(pt_element_t
);
543 for (i
= 0; i
< PT64_ENT_PER_PAGE
; i
+= ARRAY_SIZE(pt
)) {
544 r
= kvm_read_guest_atomic(vcpu
->kvm
, pte_gpa
, pt
, sizeof pt
);
545 pte_gpa
+= ARRAY_SIZE(pt
) * sizeof(pt_element_t
);
546 for (j
= 0; j
< ARRAY_SIZE(pt
); ++j
)
547 if (r
|| is_present_pte(pt
[j
]))
548 sp
->spt
[i
+j
] = shadow_trap_nonpresent_pte
;
550 sp
->spt
[i
+j
] = shadow_notrap_nonpresent_pte
;
555 * Using the cached information from sp->gfns is safe because:
556 * - The spte has a reference to the struct page, so the pfn for a given gfn
557 * can't change unless all sptes pointing to it are nuked first.
558 * - Alias changes zap the entire shadow cache.
560 static int FNAME(sync_page
)(struct kvm_vcpu
*vcpu
, struct kvm_mmu_page
*sp
)
562 int i
, offset
, nr_present
;
564 offset
= nr_present
= 0;
567 offset
= sp
->role
.quadrant
<< PT64_LEVEL_BITS
;
569 for (i
= 0; i
< PT64_ENT_PER_PAGE
; i
++) {
573 gfn_t gfn
= sp
->gfns
[i
];
575 if (!is_shadow_present_pte(sp
->spt
[i
]))
578 pte_gpa
= gfn_to_gpa(sp
->gfn
);
579 pte_gpa
+= (i
+offset
) * sizeof(pt_element_t
);
581 if (kvm_read_guest_atomic(vcpu
->kvm
, pte_gpa
, &gpte
,
582 sizeof(pt_element_t
)))
585 if (gpte_to_gfn(gpte
) != gfn
|| !is_present_pte(gpte
) ||
586 !(gpte
& PT_ACCESSED_MASK
)) {
589 rmap_remove(vcpu
->kvm
, &sp
->spt
[i
]);
590 if (is_present_pte(gpte
))
591 nonpresent
= shadow_trap_nonpresent_pte
;
593 nonpresent
= shadow_notrap_nonpresent_pte
;
594 set_shadow_pte(&sp
->spt
[i
], nonpresent
);
599 pte_access
= sp
->role
.access
& FNAME(gpte_access
)(vcpu
, gpte
);
600 set_spte(vcpu
, &sp
->spt
[i
], pte_access
, 0, 0,
601 is_dirty_pte(gpte
), 0, gpte
& PT_GLOBAL_MASK
, gfn
,
602 spte_to_pfn(sp
->spt
[i
]), true, false);
612 #undef PT_BASE_ADDR_MASK
615 #undef PT_DIR_BASE_ADDR_MASK
617 #undef PT_MAX_FULL_LEVELS
619 #undef gpte_to_gfn_pde