?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
KVM: Enable MTRR for EPT
[linux-2.6/mini2440.git] / arch / x86 / kvm / mmu.c
blob09d05f57bf66e6c1664b6146e17f7a6a471de619
1 /*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * MMU support
8 *
9 * Copyright (C) 2006 Qumranet, Inc.
10 *
11 * Authors:
12 * Yaniv Kamay <yaniv@qumranet.com>
13 * Avi Kivity <avi@qumranet.com>
14 *
15 * This work is licensed under the terms of the GNU GPL, version 2. See
16 * the COPYING file in the top-level directory.
17 *
18 */
19
20 #include "vmx.h"
21 #include "mmu.h"
22
23 #include <linux/kvm_host.h>
24 #include <linux/types.h>
25 #include <linux/string.h>
26 #include <linux/mm.h>
27 #include <linux/highmem.h>
28 #include <linux/module.h>
29 #include <linux/swap.h>
30 #include <linux/hugetlb.h>
31 #include <linux/compiler.h>
32
33 #include <asm/page.h>
34 #include <asm/cmpxchg.h>
35 #include <asm/io.h>
36
37 /*
38 * When setting this variable to true it enables Two-Dimensional-Paging
39 * where the hardware walks 2 page tables:
40 * 1. the guest-virtual to guest-physical
41 * 2. while doing 1. it walks guest-physical to host-physical
42 * If the hardware supports that we don't need to do shadow paging.
43 */
44 bool tdp_enabled = false;
45
46 #undef MMU_DEBUG
47
48 #undef AUDIT
49
50 #ifdef AUDIT
51 static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg);
52 #else
53 static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg) {}
54 #endif
55
56 #ifdef MMU_DEBUG
57
58 #define pgprintk(x...) do { if (dbg) printk(x); } while (0)
59 #define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
60
61 #else
62
63 #define pgprintk(x...) do { } while (0)
64 #define rmap_printk(x...) do { } while (0)
65
66 #endif
67
68 #if defined(MMU_DEBUG) || defined(AUDIT)
69 static int dbg = 0;
70 module_param(dbg, bool, 0644);
71 #endif
72
73 static int oos_shadow = 1;
74 module_param(oos_shadow, bool, 0644);
75
76 #ifndef MMU_DEBUG
77 #define ASSERT(x) do { } while (0)
78 #else
79 #define ASSERT(x) \
80 if (!(x)) { \
81 printk(KERN_WARNING "assertion failed %s:%d: %s\n", \
82 __FILE__, __LINE__, #x); \
83 }
84 #endif
85
86 #define PT_FIRST_AVAIL_BITS_SHIFT 9
87 #define PT64_SECOND_AVAIL_BITS_SHIFT 52
88
89 #define VALID_PAGE(x) ((x) != INVALID_PAGE)
90
91 #define PT64_LEVEL_BITS 9
92
93 #define PT64_LEVEL_SHIFT(level) \
94 (PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS)
95
96 #define PT64_LEVEL_MASK(level) \
97 (((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level))
98
99 #define PT64_INDEX(address, level)\
100 (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
101
102
103 #define PT32_LEVEL_BITS 10
104
105 #define PT32_LEVEL_SHIFT(level) \
106 (PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
107
108 #define PT32_LEVEL_MASK(level) \
109 (((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level))
110
111 #define PT32_INDEX(address, level)\
112 (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
113
114
115 #define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
116 #define PT64_DIR_BASE_ADDR_MASK \
117 (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
118
119 #define PT32_BASE_ADDR_MASK PAGE_MASK
120 #define PT32_DIR_BASE_ADDR_MASK \
121 (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
122
123 #define PT64_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | PT_USER_MASK \
124 | PT64_NX_MASK)
125
126 #define PFERR_PRESENT_MASK (1U << 0)
127 #define PFERR_WRITE_MASK (1U << 1)
128 #define PFERR_USER_MASK (1U << 2)
129 #define PFERR_FETCH_MASK (1U << 4)
130
131 #define PT_DIRECTORY_LEVEL 2
132 #define PT_PAGE_TABLE_LEVEL 1
133
134 #define RMAP_EXT 4
135
136 #define ACC_EXEC_MASK 1
137 #define ACC_WRITE_MASK PT_WRITABLE_MASK
138 #define ACC_USER_MASK PT_USER_MASK
139 #define ACC_ALL (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
140
141 #define SHADOW_PT_INDEX(addr, level) PT64_INDEX(addr, level)
142
143 struct kvm_rmap_desc {
144 u64 *shadow_ptes[RMAP_EXT];
145 struct kvm_rmap_desc *more;
146 };
147
148 struct kvm_shadow_walk {
149 int (*entry)(struct kvm_shadow_walk *walk, struct kvm_vcpu *vcpu,
150 u64 addr, u64 *spte, int level);
151 };
152
153 struct kvm_unsync_walk {
154 int (*entry) (struct kvm_mmu_page *sp, struct kvm_unsync_walk *walk);
155 };
156
157 typedef int (*mmu_parent_walk_fn) (struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp);
158
159 static struct kmem_cache *pte_chain_cache;
160 static struct kmem_cache *rmap_desc_cache;
161 static struct kmem_cache *mmu_page_header_cache;
162
163 static u64 __read_mostly shadow_trap_nonpresent_pte;
164 static u64 __read_mostly shadow_notrap_nonpresent_pte;
165 static u64 __read_mostly shadow_base_present_pte;
166 static u64 __read_mostly shadow_nx_mask;
167 static u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */
168 static u64 __read_mostly shadow_user_mask;
169 static u64 __read_mostly shadow_accessed_mask;
170 static u64 __read_mostly shadow_dirty_mask;
171 static u64 __read_mostly shadow_mt_mask;
172
173 void kvm_mmu_set_nonpresent_ptes(u64 trap_pte, u64 notrap_pte)
174 {
175 shadow_trap_nonpresent_pte = trap_pte;
176 shadow_notrap_nonpresent_pte = notrap_pte;
177 }
178 EXPORT_SYMBOL_GPL(kvm_mmu_set_nonpresent_ptes);
179
180 void kvm_mmu_set_base_ptes(u64 base_pte)
181 {
182 shadow_base_present_pte = base_pte;
183 }
184 EXPORT_SYMBOL_GPL(kvm_mmu_set_base_ptes);
185
186 void kvm_mmu_set_mask_ptes(u64 user_mask, u64 accessed_mask,
187 u64 dirty_mask, u64 nx_mask, u64 x_mask, u64 mt_mask)
188 {
189 shadow_user_mask = user_mask;
190 shadow_accessed_mask = accessed_mask;
191 shadow_dirty_mask = dirty_mask;
192 shadow_nx_mask = nx_mask;
193 shadow_x_mask = x_mask;
194 shadow_mt_mask = mt_mask;
195 }
196 EXPORT_SYMBOL_GPL(kvm_mmu_set_mask_ptes);
197
198 static int is_write_protection(struct kvm_vcpu *vcpu)
199 {
200 return vcpu->arch.cr0 & X86_CR0_WP;
201 }
202
203 static int is_cpuid_PSE36(void)
204 {
205 return 1;
206 }
207
208 static int is_nx(struct kvm_vcpu *vcpu)
209 {
210 return vcpu->arch.shadow_efer & EFER_NX;
211 }
212
213 static int is_present_pte(unsigned long pte)
214 {
215 return pte & PT_PRESENT_MASK;
216 }
217
218 static int is_shadow_present_pte(u64 pte)
219 {
220 return pte != shadow_trap_nonpresent_pte
221 && pte != shadow_notrap_nonpresent_pte;
222 }
223
224 static int is_large_pte(u64 pte)
225 {
226 return pte & PT_PAGE_SIZE_MASK;
227 }
228
229 static int is_writeble_pte(unsigned long pte)
230 {
231 return pte & PT_WRITABLE_MASK;
232 }
233
234 static int is_dirty_pte(unsigned long pte)
235 {
236 return pte & shadow_dirty_mask;
237 }
238
239 static int is_rmap_pte(u64 pte)
240 {
241 return is_shadow_present_pte(pte);
242 }
243
244 static pfn_t spte_to_pfn(u64 pte)
245 {
246 return (pte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
247 }
248
249 static gfn_t pse36_gfn_delta(u32 gpte)
250 {
251 int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;
252
253 return (gpte & PT32_DIR_PSE36_MASK) << shift;
254 }
255
256 static void set_shadow_pte(u64 *sptep, u64 spte)
257 {
258 #ifdef CONFIG_X86_64
259 set_64bit((unsigned long *)sptep, spte);
260 #else
261 set_64bit((unsigned long long *)sptep, spte);
262 #endif
263 }
264
265 static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
266 struct kmem_cache *base_cache, int min)
267 {
268 void *obj;
269
270 if (cache->nobjs >= min)
271 return 0;
272 while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
273 obj = kmem_cache_zalloc(base_cache, GFP_KERNEL);
274 if (!obj)
275 return -ENOMEM;
276 cache->objects[cache->nobjs++] = obj;
277 }
278 return 0;
279 }
280
281 static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
282 {
283 while (mc->nobjs)
284 kfree(mc->objects[--mc->nobjs]);
285 }
286
287 static int mmu_topup_memory_cache_page(struct kvm_mmu_memory_cache *cache,
288 int min)
289 {
290 struct page *page;
291
292 if (cache->nobjs >= min)
293 return 0;
294 while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
295 page = alloc_page(GFP_KERNEL);
296 if (!page)
297 return -ENOMEM;
298 set_page_private(page, 0);
299 cache->objects[cache->nobjs++] = page_address(page);
300 }
301 return 0;
302 }
303
304 static void mmu_free_memory_cache_page(struct kvm_mmu_memory_cache *mc)
305 {
306 while (mc->nobjs)
307 free_page((unsigned long)mc->objects[--mc->nobjs]);
308 }
309
310 static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
311 {
312 int r;
313
314 r = mmu_topup_memory_cache(&vcpu->arch.mmu_pte_chain_cache,
315 pte_chain_cache, 4);
316 if (r)
317 goto out;
318 r = mmu_topup_memory_cache(&vcpu->arch.mmu_rmap_desc_cache,
319 rmap_desc_cache, 4);
320 if (r)
321 goto out;
322 r = mmu_topup_memory_cache_page(&vcpu->arch.mmu_page_cache, 8);
323 if (r)
324 goto out;
325 r = mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
326 mmu_page_header_cache, 4);
327 out:
328 return r;
329 }
330
331 static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
332 {
333 mmu_free_memory_cache(&vcpu->arch.mmu_pte_chain_cache);
334 mmu_free_memory_cache(&vcpu->arch.mmu_rmap_desc_cache);
335 mmu_free_memory_cache_page(&vcpu->arch.mmu_page_cache);
336 mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache);
337 }
338
339 static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc,
340 size_t size)
341 {
342 void *p;
343
344 BUG_ON(!mc->nobjs);
345 p = mc->objects[--mc->nobjs];
346 memset(p, 0, size);
347 return p;
348 }
349
350 static struct kvm_pte_chain *mmu_alloc_pte_chain(struct kvm_vcpu *vcpu)
351 {
352 return mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_chain_cache,
353 sizeof(struct kvm_pte_chain));
354 }
355
356 static void mmu_free_pte_chain(struct kvm_pte_chain *pc)
357 {
358 kfree(pc);
359 }
360
361 static struct kvm_rmap_desc *mmu_alloc_rmap_desc(struct kvm_vcpu *vcpu)
362 {
363 return mmu_memory_cache_alloc(&vcpu->arch.mmu_rmap_desc_cache,
364 sizeof(struct kvm_rmap_desc));
365 }
366
367 static void mmu_free_rmap_desc(struct kvm_rmap_desc *rd)
368 {
369 kfree(rd);
370 }
371
372 /*
373 * Return the pointer to the largepage write count for a given
374 * gfn, handling slots that are not large page aligned.
375 */
376 static int *slot_largepage_idx(gfn_t gfn, struct kvm_memory_slot *slot)
377 {
378 unsigned long idx;
379
380 idx = (gfn / KVM_PAGES_PER_HPAGE) -
381 (slot->base_gfn / KVM_PAGES_PER_HPAGE);
382 return &slot->lpage_info[idx].write_count;
383 }
384
385 static void account_shadowed(struct kvm *kvm, gfn_t gfn)
386 {
387 int *write_count;
388
389 write_count = slot_largepage_idx(gfn, gfn_to_memslot(kvm, gfn));
390 *write_count += 1;
391 }
392
393 static void unaccount_shadowed(struct kvm *kvm, gfn_t gfn)
394 {
395 int *write_count;
396
397 write_count = slot_largepage_idx(gfn, gfn_to_memslot(kvm, gfn));
398 *write_count -= 1;
399 WARN_ON(*write_count < 0);
400 }
401
402 static int has_wrprotected_page(struct kvm *kvm, gfn_t gfn)
403 {
404 struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
405 int *largepage_idx;
406
407 if (slot) {
408 largepage_idx = slot_largepage_idx(gfn, slot);
409 return *largepage_idx;
410 }
411
412 return 1;
413 }
414
415 static int host_largepage_backed(struct kvm *kvm, gfn_t gfn)
416 {
417 struct vm_area_struct *vma;
418 unsigned long addr;
419 int ret = 0;
420
421 addr = gfn_to_hva(kvm, gfn);
422 if (kvm_is_error_hva(addr))
423 return ret;
424
425 down_read(&current->mm->mmap_sem);
426 vma = find_vma(current->mm, addr);
427 if (vma && is_vm_hugetlb_page(vma))
428 ret = 1;
429 up_read(&current->mm->mmap_sem);
430
431 return ret;
432 }
433
434 static int is_largepage_backed(struct kvm_vcpu *vcpu, gfn_t large_gfn)
435 {
436 struct kvm_memory_slot *slot;
437
438 if (has_wrprotected_page(vcpu->kvm, large_gfn))
439 return 0;
440
441 if (!host_largepage_backed(vcpu->kvm, large_gfn))
442 return 0;
443
444 slot = gfn_to_memslot(vcpu->kvm, large_gfn);
445 if (slot && slot->dirty_bitmap)
446 return 0;
447
448 return 1;
449 }
450
451 /*
452 * Take gfn and return the reverse mapping to it.
453 * Note: gfn must be unaliased before this function get called
454 */
455
456 static unsigned long *gfn_to_rmap(struct kvm *kvm, gfn_t gfn, int lpage)
457 {
458 struct kvm_memory_slot *slot;
459 unsigned long idx;
460
461 slot = gfn_to_memslot(kvm, gfn);
462 if (!lpage)
463 return &slot->rmap[gfn - slot->base_gfn];
464
465 idx = (gfn / KVM_PAGES_PER_HPAGE) -
466 (slot->base_gfn / KVM_PAGES_PER_HPAGE);
467
468 return &slot->lpage_info[idx].rmap_pde;
469 }
470
471 /*
472 * Reverse mapping data structures:
473 *
474 * If rmapp bit zero is zero, then rmapp point to the shadw page table entry
475 * that points to page_address(page).
476 *
477 * If rmapp bit zero is one, (then rmap & ~1) points to a struct kvm_rmap_desc
478 * containing more mappings.
479 */
480 static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn, int lpage)
481 {
482 struct kvm_mmu_page *sp;
483 struct kvm_rmap_desc *desc;
484 unsigned long *rmapp;
485 int i;
486
487 if (!is_rmap_pte(*spte))
488 return;
489 gfn = unalias_gfn(vcpu->kvm, gfn);
490 sp = page_header(__pa(spte));
491 sp->gfns[spte - sp->spt] = gfn;
492 rmapp = gfn_to_rmap(vcpu->kvm, gfn, lpage);
493 if (!*rmapp) {
494 rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte);
495 *rmapp = (unsigned long)spte;
496 } else if (!(*rmapp & 1)) {
497 rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte);
498 desc = mmu_alloc_rmap_desc(vcpu);
499 desc->shadow_ptes[0] = (u64 *)*rmapp;
500 desc->shadow_ptes[1] = spte;
501 *rmapp = (unsigned long)desc | 1;
502 } else {
503 rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte);
504 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
505 while (desc->shadow_ptes[RMAP_EXT-1] && desc->more)
506 desc = desc->more;
507 if (desc->shadow_ptes[RMAP_EXT-1]) {
508 desc->more = mmu_alloc_rmap_desc(vcpu);
509 desc = desc->more;
510 }
511 for (i = 0; desc->shadow_ptes[i]; ++i)
512 ;
513 desc->shadow_ptes[i] = spte;
514 }
515 }
516
517 static void rmap_desc_remove_entry(unsigned long *rmapp,
518 struct kvm_rmap_desc *desc,
519 int i,
520 struct kvm_rmap_desc *prev_desc)
521 {
522 int j;
523
524 for (j = RMAP_EXT - 1; !desc->shadow_ptes[j] && j > i; --j)
525 ;
526 desc->shadow_ptes[i] = desc->shadow_ptes[j];
527 desc->shadow_ptes[j] = NULL;
528 if (j != 0)
529 return;
530 if (!prev_desc && !desc->more)
531 *rmapp = (unsigned long)desc->shadow_ptes[0];
532 else
533 if (prev_desc)
534 prev_desc->more = desc->more;
535 else
536 *rmapp = (unsigned long)desc->more | 1;
537 mmu_free_rmap_desc(desc);
538 }
539
540 static void rmap_remove(struct kvm *kvm, u64 *spte)
541 {
542 struct kvm_rmap_desc *desc;
543 struct kvm_rmap_desc *prev_desc;
544 struct kvm_mmu_page *sp;
545 pfn_t pfn;
546 unsigned long *rmapp;
547 int i;
548
549 if (!is_rmap_pte(*spte))
550 return;
551 sp = page_header(__pa(spte));
552 pfn = spte_to_pfn(*spte);
553 if (*spte & shadow_accessed_mask)
554 kvm_set_pfn_accessed(pfn);
555 if (is_writeble_pte(*spte))
556 kvm_release_pfn_dirty(pfn);
557 else
558 kvm_release_pfn_clean(pfn);
559 rmapp = gfn_to_rmap(kvm, sp->gfns[spte - sp->spt], is_large_pte(*spte));
560 if (!*rmapp) {
561 printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte);
562 BUG();
563 } else if (!(*rmapp & 1)) {
564 rmap_printk("rmap_remove: %p %llx 1->0\n", spte, *spte);
565 if ((u64 *)*rmapp != spte) {
566 printk(KERN_ERR "rmap_remove: %p %llx 1->BUG\n",
567 spte, *spte);
568 BUG();
569 }
570 *rmapp = 0;
571 } else {
572 rmap_printk("rmap_remove: %p %llx many->many\n", spte, *spte);
573 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
574 prev_desc = NULL;
575 while (desc) {
576 for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i)
577 if (desc->shadow_ptes[i] == spte) {
578 rmap_desc_remove_entry(rmapp,
579 desc, i,
580 prev_desc);
581 return;
582 }
583 prev_desc = desc;
584 desc = desc->more;
585 }
586 BUG();
587 }
588 }
589
590 static u64 *rmap_next(struct kvm *kvm, unsigned long *rmapp, u64 *spte)
591 {
592 struct kvm_rmap_desc *desc;
593 struct kvm_rmap_desc *prev_desc;
594 u64 *prev_spte;
595 int i;
596
597 if (!*rmapp)
598 return NULL;
599 else if (!(*rmapp & 1)) {
600 if (!spte)
601 return (u64 *)*rmapp;
602 return NULL;
603 }
604 desc = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
605 prev_desc = NULL;
606 prev_spte = NULL;
607 while (desc) {
608 for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i) {
609 if (prev_spte == spte)
610 return desc->shadow_ptes[i];
611 prev_spte = desc->shadow_ptes[i];
612 }
613 desc = desc->more;
614 }
615 return NULL;
616 }
617
618 static void rmap_write_protect(struct kvm *kvm, u64 gfn)
619 {
620 unsigned long *rmapp;
621 u64 *spte;
622 int write_protected = 0;
623
624 gfn = unalias_gfn(kvm, gfn);
625 rmapp = gfn_to_rmap(kvm, gfn, 0);
626
627 spte = rmap_next(kvm, rmapp, NULL);
628 while (spte) {
629 BUG_ON(!spte);
630 BUG_ON(!(*spte & PT_PRESENT_MASK));
631 rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
632 if (is_writeble_pte(*spte)) {
633 set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK);
634 write_protected = 1;
635 }
636 spte = rmap_next(kvm, rmapp, spte);
637 }
638 if (write_protected) {
639 pfn_t pfn;
640
641 spte = rmap_next(kvm, rmapp, NULL);
642 pfn = spte_to_pfn(*spte);
643 kvm_set_pfn_dirty(pfn);
644 }
645
646 /* check for huge page mappings */
647 rmapp = gfn_to_rmap(kvm, gfn, 1);
648 spte = rmap_next(kvm, rmapp, NULL);
649 while (spte) {
650 BUG_ON(!spte);
651 BUG_ON(!(*spte & PT_PRESENT_MASK));
652 BUG_ON((*spte & (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK)) != (PT_PAGE_SIZE_MASK|PT_PRESENT_MASK));
653 pgprintk("rmap_write_protect(large): spte %p %llx %lld\n", spte, *spte, gfn);
654 if (is_writeble_pte(*spte)) {
655 rmap_remove(kvm, spte);
656 --kvm->stat.lpages;
657 set_shadow_pte(spte, shadow_trap_nonpresent_pte);
658 spte = NULL;
659 write_protected = 1;
660 }
661 spte = rmap_next(kvm, rmapp, spte);
662 }
663
664 if (write_protected)
665 kvm_flush_remote_tlbs(kvm);
666 }
667
668 static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp)
669 {
670 u64 *spte;
671 int need_tlb_flush = 0;
672
673 while ((spte = rmap_next(kvm, rmapp, NULL))) {
674 BUG_ON(!(*spte & PT_PRESENT_MASK));
675 rmap_printk("kvm_rmap_unmap_hva: spte %p %llx\n", spte, *spte);
676 rmap_remove(kvm, spte);
677 set_shadow_pte(spte, shadow_trap_nonpresent_pte);
678 need_tlb_flush = 1;
679 }
680 return need_tlb_flush;
681 }
682
683 static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
684 int (*handler)(struct kvm *kvm, unsigned long *rmapp))
685 {
686 int i;
687 int retval = 0;
688
689 /*
690 * If mmap_sem isn't taken, we can look the memslots with only
691 * the mmu_lock by skipping over the slots with userspace_addr == 0.
692 */
693 for (i = 0; i < kvm->nmemslots; i++) {
694 struct kvm_memory_slot *memslot = &kvm->memslots[i];
695 unsigned long start = memslot->userspace_addr;
696 unsigned long end;
697
698 /* mmu_lock protects userspace_addr */
699 if (!start)
700 continue;
701
702 end = start + (memslot->npages << PAGE_SHIFT);
703 if (hva >= start && hva < end) {
704 gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT;
705 retval |= handler(kvm, &memslot->rmap[gfn_offset]);
706 retval |= handler(kvm,
707 &memslot->lpage_info[
708 gfn_offset /
709 KVM_PAGES_PER_HPAGE].rmap_pde);
710 }
711 }
712
713 return retval;
714 }
715
716 int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
717 {
718 return kvm_handle_hva(kvm, hva, kvm_unmap_rmapp);
719 }
720
721 static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp)
722 {
723 u64 *spte;
724 int young = 0;
725
726 /* always return old for EPT */
727 if (!shadow_accessed_mask)
728 return 0;
729
730 spte = rmap_next(kvm, rmapp, NULL);
731 while (spte) {
732 int _young;
733 u64 _spte = *spte;
734 BUG_ON(!(_spte & PT_PRESENT_MASK));
735 _young = _spte & PT_ACCESSED_MASK;
736 if (_young) {
737 young = 1;
738 clear_bit(PT_ACCESSED_SHIFT, (unsigned long *)spte);
739 }
740 spte = rmap_next(kvm, rmapp, spte);
741 }
742 return young;
743 }
744
745 int kvm_age_hva(struct kvm *kvm, unsigned long hva)
746 {
747 return kvm_handle_hva(kvm, hva, kvm_age_rmapp);
748 }
749
750 #ifdef MMU_DEBUG
751 static int is_empty_shadow_page(u64 *spt)
752 {
753 u64 *pos;
754 u64 *end;
755
756 for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
757 if (is_shadow_present_pte(*pos)) {
758 printk(KERN_ERR "%s: %p %llx\n", __func__,
759 pos, *pos);
760 return 0;
761 }
762 return 1;
763 }
764 #endif
765
766 static void kvm_mmu_free_page(struct kvm *kvm, struct kvm_mmu_page *sp)
767 {
768 ASSERT(is_empty_shadow_page(sp->spt));
769 list_del(&sp->link);
770 __free_page(virt_to_page(sp->spt));
771 __free_page(virt_to_page(sp->gfns));
772 kfree(sp);
773 ++kvm->arch.n_free_mmu_pages;
774 }
775
776 static unsigned kvm_page_table_hashfn(gfn_t gfn)
777 {
778 return gfn & ((1 << KVM_MMU_HASH_SHIFT) - 1);
779 }
780
781 static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
782 u64 *parent_pte)
783 {
784 struct kvm_mmu_page *sp;
785
786 sp = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache, sizeof *sp);
787 sp->spt = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE);
788 sp->gfns = mmu_memory_cache_alloc(&vcpu->arch.mmu_page_cache, PAGE_SIZE);
789 set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
790 list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
791 ASSERT(is_empty_shadow_page(sp->spt));
792 sp->slot_bitmap = 0;
793 sp->multimapped = 0;
794 sp->parent_pte = parent_pte;
795 --vcpu->kvm->arch.n_free_mmu_pages;
796 return sp;
797 }
798
799 static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
800 struct kvm_mmu_page *sp, u64 *parent_pte)
801 {
802 struct kvm_pte_chain *pte_chain;
803 struct hlist_node *node;
804 int i;
805
806 if (!parent_pte)
807 return;
808 if (!sp->multimapped) {
809 u64 *old = sp->parent_pte;
810
811 if (!old) {
812 sp->parent_pte = parent_pte;
813 return;
814 }
815 sp->multimapped = 1;
816 pte_chain = mmu_alloc_pte_chain(vcpu);
817 INIT_HLIST_HEAD(&sp->parent_ptes);
818 hlist_add_head(&pte_chain->link, &sp->parent_ptes);
819 pte_chain->parent_ptes[0] = old;
820 }
821 hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link) {
822 if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1])
823 continue;
824 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i)
825 if (!pte_chain->parent_ptes[i]) {
826 pte_chain->parent_ptes[i] = parent_pte;
827 return;
828 }
829 }
830 pte_chain = mmu_alloc_pte_chain(vcpu);
831 BUG_ON(!pte_chain);
832 hlist_add_head(&pte_chain->link, &sp->parent_ptes);
833 pte_chain->parent_ptes[0] = parent_pte;
834 }
835
836 static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
837 u64 *parent_pte)
838 {
839 struct kvm_pte_chain *pte_chain;
840 struct hlist_node *node;
841 int i;
842
843 if (!sp->multimapped) {
844 BUG_ON(sp->parent_pte != parent_pte);
845 sp->parent_pte = NULL;
846 return;
847 }
848 hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link)
849 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
850 if (!pte_chain->parent_ptes[i])
851 break;
852 if (pte_chain->parent_ptes[i] != parent_pte)
853 continue;
854 while (i + 1 < NR_PTE_CHAIN_ENTRIES
855 && pte_chain->parent_ptes[i + 1]) {
856 pte_chain->parent_ptes[i]
857 = pte_chain->parent_ptes[i + 1];
858 ++i;
859 }
860 pte_chain->parent_ptes[i] = NULL;
861 if (i == 0) {
862 hlist_del(&pte_chain->link);
863 mmu_free_pte_chain(pte_chain);
864 if (hlist_empty(&sp->parent_ptes)) {
865 sp->multimapped = 0;
866 sp->parent_pte = NULL;
867 }
868 }
869 return;
870 }
871 BUG();
872 }
873
874
875 static void mmu_parent_walk(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
876 mmu_parent_walk_fn fn)
877 {
878 struct kvm_pte_chain *pte_chain;
879 struct hlist_node *node;
880 struct kvm_mmu_page *parent_sp;
881 int i;
882
883 if (!sp->multimapped && sp->parent_pte) {
884 parent_sp = page_header(__pa(sp->parent_pte));
885 fn(vcpu, parent_sp);
886 mmu_parent_walk(vcpu, parent_sp, fn);
887 return;
888 }
889 hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link)
890 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
891 if (!pte_chain->parent_ptes[i])
892 break;
893 parent_sp = page_header(__pa(pte_chain->parent_ptes[i]));
894 fn(vcpu, parent_sp);
895 mmu_parent_walk(vcpu, parent_sp, fn);
896 }
897 }
898
899 static void kvm_mmu_update_unsync_bitmap(u64 *spte)
900 {
901 unsigned int index;
902 struct kvm_mmu_page *sp = page_header(__pa(spte));
903
904 index = spte - sp->spt;
905 __set_bit(index, sp->unsync_child_bitmap);
906 sp->unsync_children = 1;
907 }
908
909 static void kvm_mmu_update_parents_unsync(struct kvm_mmu_page *sp)
910 {
911 struct kvm_pte_chain *pte_chain;
912 struct hlist_node *node;
913 int i;
914
915 if (!sp->parent_pte)
916 return;
917
918 if (!sp->multimapped) {
919 kvm_mmu_update_unsync_bitmap(sp->parent_pte);
920 return;
921 }
922
923 hlist_for_each_entry(pte_chain, node, &sp->parent_ptes, link)
924 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
925 if (!pte_chain->parent_ptes[i])
926 break;
927 kvm_mmu_update_unsync_bitmap(pte_chain->parent_ptes[i]);
928 }
929 }
930
931 static int unsync_walk_fn(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
932 {
933 sp->unsync_children = 1;
934 kvm_mmu_update_parents_unsync(sp);
935 return 1;
936 }
937
938 static void kvm_mmu_mark_parents_unsync(struct kvm_vcpu *vcpu,
939 struct kvm_mmu_page *sp)
940 {
941 mmu_parent_walk(vcpu, sp, unsync_walk_fn);
942 kvm_mmu_update_parents_unsync(sp);
943 }
944
945 static void nonpaging_prefetch_page(struct kvm_vcpu *vcpu,
946 struct kvm_mmu_page *sp)
947 {
948 int i;
949
950 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
951 sp->spt[i] = shadow_trap_nonpresent_pte;
952 }
953
954 static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
955 struct kvm_mmu_page *sp)
956 {
957 return 1;
958 }
959
960 static void nonpaging_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
961 {
962 }
963
964 #define for_each_unsync_children(bitmap, idx) \
965 for (idx = find_first_bit(bitmap, 512); \
966 idx < 512; \
967 idx = find_next_bit(bitmap, 512, idx+1))
968
969 static int mmu_unsync_walk(struct kvm_mmu_page *sp,
970 struct kvm_unsync_walk *walker)
971 {
972 int i, ret;
973
974 if (!sp->unsync_children)
975 return 0;
976
977 for_each_unsync_children(sp->unsync_child_bitmap, i) {
978 u64 ent = sp->spt[i];
979
980 if (is_shadow_present_pte(ent)) {
981 struct kvm_mmu_page *child;
982 child = page_header(ent & PT64_BASE_ADDR_MASK);
983
984 if (child->unsync_children) {
985 ret = mmu_unsync_walk(child, walker);
986 if (ret)
987 return ret;
988 __clear_bit(i, sp->unsync_child_bitmap);
989 }
990
991 if (child->unsync) {
992 ret = walker->entry(child, walker);
993 __clear_bit(i, sp->unsync_child_bitmap);
994 if (ret)
995 return ret;
996 }
997 }
998 }
999
1000 if (find_first_bit(sp->unsync_child_bitmap, 512) == 512)
1001 sp->unsync_children = 0;
1002
1003 return 0;
1004 }
1005
1006 static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm *kvm, gfn_t gfn)
1007 {
1008 unsigned index;
1009 struct hlist_head *bucket;
1010 struct kvm_mmu_page *sp;
1011 struct hlist_node *node;
1012
1013 pgprintk("%s: looking for gfn %lx\n", __func__, gfn);
1014 index = kvm_page_table_hashfn(gfn);
1015 bucket = &kvm->arch.mmu_page_hash[index];
1016 hlist_for_each_entry(sp, node, bucket, hash_link)
1017 if (sp->gfn == gfn && !sp->role.metaphysical
1018 && !sp->role.invalid) {
1019 pgprintk("%s: found role %x\n",
1020 __func__, sp->role.word);
1021 return sp;
1022 }
1023 return NULL;
1024 }
1025
1026 static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
1027 {
1028 WARN_ON(!sp->unsync);
1029 sp->unsync = 0;
1030 --kvm->stat.mmu_unsync;
1031 }
1032
1033 static int kvm_mmu_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp);
1034
1035 static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
1036 {
1037 if (sp->role.glevels != vcpu->arch.mmu.root_level) {
1038 kvm_mmu_zap_page(vcpu->kvm, sp);
1039 return 1;
1040 }
1041
1042 rmap_write_protect(vcpu->kvm, sp->gfn);
1043 kvm_unlink_unsync_page(vcpu->kvm, sp);
1044 if (vcpu->arch.mmu.sync_page(vcpu, sp)) {
1045 kvm_mmu_zap_page(vcpu->kvm, sp);
1046 return 1;
1047 }
1048
1049 kvm_mmu_flush_tlb(vcpu);
1050 return 0;
1051 }
1052
1053 struct sync_walker {
1054 struct kvm_vcpu *vcpu;
1055 struct kvm_unsync_walk walker;
1056 };
1057
1058 static int mmu_sync_fn(struct kvm_mmu_page *sp, struct kvm_unsync_walk *walk)
1059 {
1060 struct sync_walker *sync_walk = container_of(walk, struct sync_walker,
1061 walker);
1062 struct kvm_vcpu *vcpu = sync_walk->vcpu;
1063
1064 kvm_sync_page(vcpu, sp);
1065 return (need_resched() || spin_needbreak(&vcpu->kvm->mmu_lock));
1066 }
1067
1068 static void mmu_sync_children(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
1069 {
1070 struct sync_walker walker = {
1071 .walker = { .entry = mmu_sync_fn, },
1072 .vcpu = vcpu,
1073 };
1074
1075 while (mmu_unsync_walk(sp, &walker.walker))
1076 cond_resched_lock(&vcpu->kvm->mmu_lock);
1077 }
1078
1079 static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
1080 gfn_t gfn,
1081 gva_t gaddr,
1082 unsigned level,
1083 int metaphysical,
1084 unsigned access,
1085 u64 *parent_pte)
1086 {
1087 union kvm_mmu_page_role role;
1088 unsigned index;
1089 unsigned quadrant;
1090 struct hlist_head *bucket;
1091 struct kvm_mmu_page *sp;
1092 struct hlist_node *node, *tmp;
1093
1094 role.word = 0;
1095 role.glevels = vcpu->arch.mmu.root_level;
1096 role.level = level;
1097 role.metaphysical = metaphysical;
1098 role.access = access;
1099 if (vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) {
1100 quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
1101 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
1102 role.quadrant = quadrant;
1103 }
1104 pgprintk("%s: looking gfn %lx role %x\n", __func__,
1105 gfn, role.word);
1106 index = kvm_page_table_hashfn(gfn);
1107 bucket = &vcpu->kvm->arch.mmu_page_hash[index];
1108 hlist_for_each_entry_safe(sp, node, tmp, bucket, hash_link)
1109 if (sp->gfn == gfn) {
1110 if (sp->unsync)
1111 if (kvm_sync_page(vcpu, sp))
1112 continue;
1113
1114 if (sp->role.word != role.word)
1115 continue;
1116
1117 mmu_page_add_parent_pte(vcpu, sp, parent_pte);
1118 if (sp->unsync_children) {
1119 set_bit(KVM_REQ_MMU_SYNC, &vcpu->requests);
1120 kvm_mmu_mark_parents_unsync(vcpu, sp);
1121 }
1122 pgprintk("%s: found\n", __func__);
1123 return sp;
1124 }
1125 ++vcpu->kvm->stat.mmu_cache_miss;
1126 sp = kvm_mmu_alloc_page(vcpu, parent_pte);
1127 if (!sp)
1128 return sp;
1129 pgprintk("%s: adding gfn %lx role %x\n", __func__, gfn, role.word);
1130 sp->gfn = gfn;
1131 sp->role = role;
1132 hlist_add_head(&sp->hash_link, bucket);
1133 if (!metaphysical) {
1134 rmap_write_protect(vcpu->kvm, gfn);
1135 account_shadowed(vcpu->kvm, gfn);
1136 }
1137 if (shadow_trap_nonpresent_pte != shadow_notrap_nonpresent_pte)
1138 vcpu->arch.mmu.prefetch_page(vcpu, sp);
1139 else
1140 nonpaging_prefetch_page(vcpu, sp);
1141 return sp;
1142 }
1143
1144 static int walk_shadow(struct kvm_shadow_walk *walker,
1145 struct kvm_vcpu *vcpu, u64 addr)
1146 {
1147 hpa_t shadow_addr;
1148 int level;
1149 int r;
1150 u64 *sptep;
1151 unsigned index;
1152
1153 shadow_addr = vcpu->arch.mmu.root_hpa;
1154 level = vcpu->arch.mmu.shadow_root_level;
1155 if (level == PT32E_ROOT_LEVEL) {
1156 shadow_addr = vcpu->arch.mmu.pae_root[(addr >> 30) & 3];
1157 shadow_addr &= PT64_BASE_ADDR_MASK;
1158 --level;
1159 }
1160
1161 while (level >= PT_PAGE_TABLE_LEVEL) {
1162 index = SHADOW_PT_INDEX(addr, level);
1163 sptep = ((u64 *)__va(shadow_addr)) + index;
1164 r = walker->entry(walker, vcpu, addr, sptep, level);
1165 if (r)
1166 return r;
1167 shadow_addr = *sptep & PT64_BASE_ADDR_MASK;
1168 --level;
1169 }
1170 return 0;
1171 }
1172
1173 static void kvm_mmu_page_unlink_children(struct kvm *kvm,
1174 struct kvm_mmu_page *sp)
1175 {
1176 unsigned i;
1177 u64 *pt;
1178 u64 ent;
1179
1180 pt = sp->spt;
1181
1182 if (sp->role.level == PT_PAGE_TABLE_LEVEL) {
1183 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
1184 if (is_shadow_present_pte(pt[i]))
1185 rmap_remove(kvm, &pt[i]);
1186 pt[i] = shadow_trap_nonpresent_pte;
1187 }
1188 return;
1189 }
1190
1191 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
1192 ent = pt[i];
1193
1194 if (is_shadow_present_pte(ent)) {
1195 if (!is_large_pte(ent)) {
1196 ent &= PT64_BASE_ADDR_MASK;
1197 mmu_page_remove_parent_pte(page_header(ent),
1198 &pt[i]);
1199 } else {
1200 --kvm->stat.lpages;
1201 rmap_remove(kvm, &pt[i]);
1202 }
1203 }
1204 pt[i] = shadow_trap_nonpresent_pte;
1205 }
1206 }
1207
1208 static void kvm_mmu_put_page(struct kvm_mmu_page *sp, u64 *parent_pte)
1209 {
1210 mmu_page_remove_parent_pte(sp, parent_pte);
1211 }
1212
1213 static void kvm_mmu_reset_last_pte_updated(struct kvm *kvm)
1214 {
1215 int i;
1216
1217 for (i = 0; i < KVM_MAX_VCPUS; ++i)
1218 if (kvm->vcpus[i])
1219 kvm->vcpus[i]->arch.last_pte_updated = NULL;
1220 }
1221
1222 static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
1223 {
1224 u64 *parent_pte;
1225
1226 while (sp->multimapped || sp->parent_pte) {
1227 if (!sp->multimapped)
1228 parent_pte = sp->parent_pte;
1229 else {
1230 struct kvm_pte_chain *chain;
1231
1232 chain = container_of(sp->parent_ptes.first,
1233 struct kvm_pte_chain, link);
1234 parent_pte = chain->parent_ptes[0];
1235 }
1236 BUG_ON(!parent_pte);
1237 kvm_mmu_put_page(sp, parent_pte);
1238 set_shadow_pte(parent_pte, shadow_trap_nonpresent_pte);
1239 }
1240 }
1241
1242 struct zap_walker {
1243 struct kvm_unsync_walk walker;
1244 struct kvm *kvm;
1245 int zapped;
1246 };
1247
1248 static int mmu_zap_fn(struct kvm_mmu_page *sp, struct kvm_unsync_walk *walk)
1249 {
1250 struct zap_walker *zap_walk = container_of(walk, struct zap_walker,
1251 walker);
1252 kvm_mmu_zap_page(zap_walk->kvm, sp);
1253 zap_walk->zapped = 1;
1254 return 0;
1255 }
1256
1257 static int mmu_zap_unsync_children(struct kvm *kvm, struct kvm_mmu_page *sp)
1258 {
1259 struct zap_walker walker = {
1260 .walker = { .entry = mmu_zap_fn, },
1261 .kvm = kvm,
1262 .zapped = 0,
1263 };
1264
1265 if (sp->role.level == PT_PAGE_TABLE_LEVEL)
1266 return 0;
1267 mmu_unsync_walk(sp, &walker.walker);
1268 return walker.zapped;
1269 }
1270
1271 static int kvm_mmu_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp)
1272 {
1273 int ret;
1274 ++kvm->stat.mmu_shadow_zapped;
1275 ret = mmu_zap_unsync_children(kvm, sp);
1276 kvm_mmu_page_unlink_children(kvm, sp);
1277 kvm_mmu_unlink_parents(kvm, sp);
1278 kvm_flush_remote_tlbs(kvm);
1279 if (!sp->role.invalid && !sp->role.metaphysical)
1280 unaccount_shadowed(kvm, sp->gfn);
1281 if (sp->unsync)
1282 kvm_unlink_unsync_page(kvm, sp);
1283 if (!sp->root_count) {
1284 hlist_del(&sp->hash_link);
1285 kvm_mmu_free_page(kvm, sp);
1286 } else {
1287 sp->role.invalid = 1;
1288 list_move(&sp->link, &kvm->arch.active_mmu_pages);
1289 kvm_reload_remote_mmus(kvm);
1290 }
1291 kvm_mmu_reset_last_pte_updated(kvm);
1292 return ret;
1293 }
1294
1295 /*
1296 * Changing the number of mmu pages allocated to the vm
1297 * Note: if kvm_nr_mmu_pages is too small, you will get dead lock
1298 */
1299 void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned int kvm_nr_mmu_pages)
1300 {
1301 /*
1302 * If we set the number of mmu pages to be smaller be than the
1303 * number of actived pages , we must to free some mmu pages before we
1304 * change the value
1305 */
1306
1307 if ((kvm->arch.n_alloc_mmu_pages - kvm->arch.n_free_mmu_pages) >
1308 kvm_nr_mmu_pages) {
1309 int n_used_mmu_pages = kvm->arch.n_alloc_mmu_pages
1310 - kvm->arch.n_free_mmu_pages;
1311
1312 while (n_used_mmu_pages > kvm_nr_mmu_pages) {
1313 struct kvm_mmu_page *page;
1314
1315 page = container_of(kvm->arch.active_mmu_pages.prev,
1316 struct kvm_mmu_page, link);
1317 kvm_mmu_zap_page(kvm, page);
1318 n_used_mmu_pages--;
1319 }
1320 kvm->arch.n_free_mmu_pages = 0;
1321 }
1322 else
1323 kvm->arch.n_free_mmu_pages += kvm_nr_mmu_pages
1324 - kvm->arch.n_alloc_mmu_pages;
1325
1326 kvm->arch.n_alloc_mmu_pages = kvm_nr_mmu_pages;
1327 }
1328
1329 static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
1330 {
1331 unsigned index;
1332 struct hlist_head *bucket;
1333 struct kvm_mmu_page *sp;
1334 struct hlist_node *node, *n;
1335 int r;
1336
1337 pgprintk("%s: looking for gfn %lx\n", __func__, gfn);
1338 r = 0;
1339 index = kvm_page_table_hashfn(gfn);
1340 bucket = &kvm->arch.mmu_page_hash[index];
1341 hlist_for_each_entry_safe(sp, node, n, bucket, hash_link)
1342 if (sp->gfn == gfn && !sp->role.metaphysical) {
1343 pgprintk("%s: gfn %lx role %x\n", __func__, gfn,
1344 sp->role.word);
1345 r = 1;
1346 if (kvm_mmu_zap_page(kvm, sp))
1347 n = bucket->first;
1348 }
1349 return r;
1350 }
1351
1352 static void mmu_unshadow(struct kvm *kvm, gfn_t gfn)
1353 {
1354 struct kvm_mmu_page *sp;
1355
1356 while ((sp = kvm_mmu_lookup_page(kvm, gfn)) != NULL) {
1357 pgprintk("%s: zap %lx %x\n", __func__, gfn, sp->role.word);
1358 kvm_mmu_zap_page(kvm, sp);
1359 }
1360 }
1361
1362 static void page_header_update_slot(struct kvm *kvm, void *pte, gfn_t gfn)
1363 {
1364 int slot = memslot_id(kvm, gfn_to_memslot(kvm, gfn));
1365 struct kvm_mmu_page *sp = page_header(__pa(pte));
1366
1367 __set_bit(slot, &sp->slot_bitmap);
1368 }
1369
1370 static void mmu_convert_notrap(struct kvm_mmu_page *sp)
1371 {
1372 int i;
1373 u64 *pt = sp->spt;
1374
1375 if (shadow_trap_nonpresent_pte == shadow_notrap_nonpresent_pte)
1376 return;
1377
1378 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
1379 if (pt[i] == shadow_notrap_nonpresent_pte)
1380 set_shadow_pte(&pt[i], shadow_trap_nonpresent_pte);
1381 }
1382 }
1383
1384 struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva)
1385 {
1386 struct page *page;
1387
1388 gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva);
1389
1390 if (gpa == UNMAPPED_GVA)
1391 return NULL;
1392
1393 page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
1394
1395 return page;
1396 }
1397
1398 /*
1399 * The function is based on mtrr_type_lookup() in
1400 * arch/x86/kernel/cpu/mtrr/generic.c
1401 */
1402 static int get_mtrr_type(struct mtrr_state_type *mtrr_state,
1403 u64 start, u64 end)
1404 {
1405 int i;
1406 u64 base, mask;
1407 u8 prev_match, curr_match;
1408 int num_var_ranges = KVM_NR_VAR_MTRR;
1409
1410 if (!mtrr_state->enabled)
1411 return 0xFF;
1412
1413 /* Make end inclusive end, instead of exclusive */
1414 end--;
1415
1416 /* Look in fixed ranges. Just return the type as per start */
1417 if (mtrr_state->have_fixed && (start < 0x100000)) {
1418 int idx;
1419
1420 if (start < 0x80000) {
1421 idx = 0;
1422 idx += (start >> 16);
1423 return mtrr_state->fixed_ranges[idx];
1424 } else if (start < 0xC0000) {
1425 idx = 1 * 8;
1426 idx += ((start - 0x80000) >> 14);
1427 return mtrr_state->fixed_ranges[idx];
1428 } else if (start < 0x1000000) {
1429 idx = 3 * 8;
1430 idx += ((start - 0xC0000) >> 12);
1431 return mtrr_state->fixed_ranges[idx];
1432 }
1433 }
1434
1435 /*
1436 * Look in variable ranges
1437 * Look of multiple ranges matching this address and pick type
1438 * as per MTRR precedence
1439 */
1440 if (!(mtrr_state->enabled & 2))
1441 return mtrr_state->def_type;
1442
1443 prev_match = 0xFF;
1444 for (i = 0; i < num_var_ranges; ++i) {
1445 unsigned short start_state, end_state;
1446
1447 if (!(mtrr_state->var_ranges[i].mask_lo & (1 << 11)))
1448 continue;
1449
1450 base = (((u64)mtrr_state->var_ranges[i].base_hi) << 32) +
1451 (mtrr_state->var_ranges[i].base_lo & PAGE_MASK);
1452 mask = (((u64)mtrr_state->var_ranges[i].mask_hi) << 32) +
1453 (mtrr_state->var_ranges[i].mask_lo & PAGE_MASK);
1454
1455 start_state = ((start & mask) == (base & mask));
1456 end_state = ((end & mask) == (base & mask));
1457 if (start_state != end_state)
1458 return 0xFE;
1459
1460 if ((start & mask) != (base & mask))
1461 continue;
1462
1463 curr_match = mtrr_state->var_ranges[i].base_lo & 0xff;
1464 if (prev_match == 0xFF) {
1465 prev_match = curr_match;
1466 continue;
1467 }
1468
1469 if (prev_match == MTRR_TYPE_UNCACHABLE ||
1470 curr_match == MTRR_TYPE_UNCACHABLE)
1471 return MTRR_TYPE_UNCACHABLE;
1472
1473 if ((prev_match == MTRR_TYPE_WRBACK &&
1474 curr_match == MTRR_TYPE_WRTHROUGH) ||
1475 (prev_match == MTRR_TYPE_WRTHROUGH &&
1476 curr_match == MTRR_TYPE_WRBACK)) {
1477 prev_match = MTRR_TYPE_WRTHROUGH;
1478 curr_match = MTRR_TYPE_WRTHROUGH;
1479 }
1480
1481 if (prev_match != curr_match)
1482 return MTRR_TYPE_UNCACHABLE;
1483 }
1484
1485 if (prev_match != 0xFF)
1486 return prev_match;
1487
1488 return mtrr_state->def_type;
1489 }
1490
1491 static u8 get_memory_type(struct kvm_vcpu *vcpu, gfn_t gfn)
1492 {
1493 u8 mtrr;
1494
1495 mtrr = get_mtrr_type(&vcpu->arch.mtrr_state, gfn << PAGE_SHIFT,
1496 (gfn << PAGE_SHIFT) + PAGE_SIZE);
1497 if (mtrr == 0xfe || mtrr == 0xff)
1498 mtrr = MTRR_TYPE_WRBACK;
1499 return mtrr;
1500 }
1501
1502 static int kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
1503 {
1504 unsigned index;
1505 struct hlist_head *bucket;
1506 struct kvm_mmu_page *s;
1507 struct hlist_node *node, *n;
1508
1509 index = kvm_page_table_hashfn(sp->gfn);
1510 bucket = &vcpu->kvm->arch.mmu_page_hash[index];
1511 /* don't unsync if pagetable is shadowed with multiple roles */
1512 hlist_for_each_entry_safe(s, node, n, bucket, hash_link) {
1513 if (s->gfn != sp->gfn || s->role.metaphysical)
1514 continue;
1515 if (s->role.word != sp->role.word)
1516 return 1;
1517 }
1518 kvm_mmu_mark_parents_unsync(vcpu, sp);
1519 ++vcpu->kvm->stat.mmu_unsync;
1520 sp->unsync = 1;
1521 mmu_convert_notrap(sp);
1522 return 0;
1523 }
1524
1525 static int mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn,
1526 bool can_unsync)
1527 {
1528 struct kvm_mmu_page *shadow;
1529
1530 shadow = kvm_mmu_lookup_page(vcpu->kvm, gfn);
1531 if (shadow) {
1532 if (shadow->role.level != PT_PAGE_TABLE_LEVEL)
1533 return 1;
1534 if (shadow->unsync)
1535 return 0;
1536 if (can_unsync && oos_shadow)
1537 return kvm_unsync_page(vcpu, shadow);
1538 return 1;
1539 }
1540 return 0;
1541 }
1542
1543 static int set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
1544 unsigned pte_access, int user_fault,
1545 int write_fault, int dirty, int largepage,
1546 gfn_t gfn, pfn_t pfn, bool speculative,
1547 bool can_unsync)
1548 {
1549 u64 spte;
1550 int ret = 0;
1551 u64 mt_mask = shadow_mt_mask;
1552
1553 /*
1554 * We don't set the accessed bit, since we sometimes want to see
1555 * whether the guest actually used the pte (in order to detect
1556 * demand paging).
1557 */
1558 spte = shadow_base_present_pte | shadow_dirty_mask;
1559 if (!speculative)
1560 spte |= shadow_accessed_mask;
1561 if (!dirty)
1562 pte_access &= ~ACC_WRITE_MASK;
1563 if (pte_access & ACC_EXEC_MASK)
1564 spte |= shadow_x_mask;
1565 else
1566 spte |= shadow_nx_mask;
1567 if (pte_access & ACC_USER_MASK)
1568 spte |= shadow_user_mask;
1569 if (largepage)
1570 spte |= PT_PAGE_SIZE_MASK;
1571 if (mt_mask) {
1572 mt_mask = get_memory_type(vcpu, gfn) <<
1573 kvm_x86_ops->get_mt_mask_shift();
1574 spte |= mt_mask;
1575 }
1576
1577 spte |= (u64)pfn << PAGE_SHIFT;
1578
1579 if ((pte_access & ACC_WRITE_MASK)
1580 || (write_fault && !is_write_protection(vcpu) && !user_fault)) {
1581
1582 if (largepage && has_wrprotected_page(vcpu->kvm, gfn)) {
1583 ret = 1;
1584 spte = shadow_trap_nonpresent_pte;
1585 goto set_pte;
1586 }
1587
1588 spte |= PT_WRITABLE_MASK;
1589
1590 if (mmu_need_write_protect(vcpu, gfn, can_unsync)) {
1591 pgprintk("%s: found shadow page for %lx, marking ro\n",
1592 __func__, gfn);
1593 ret = 1;
1594 pte_access &= ~ACC_WRITE_MASK;
1595 if (is_writeble_pte(spte))
1596 spte &= ~PT_WRITABLE_MASK;
1597 }
1598 }
1599
1600 if (pte_access & ACC_WRITE_MASK)
1601 mark_page_dirty(vcpu->kvm, gfn);
1602
1603 set_pte:
1604 set_shadow_pte(shadow_pte, spte);
1605 return ret;
1606 }
1607
1608 static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte,
1609 unsigned pt_access, unsigned pte_access,
1610 int user_fault, int write_fault, int dirty,
1611 int *ptwrite, int largepage, gfn_t gfn,
1612 pfn_t pfn, bool speculative)
1613 {
1614 int was_rmapped = 0;
1615 int was_writeble = is_writeble_pte(*shadow_pte);
1616
1617 pgprintk("%s: spte %llx access %x write_fault %d"
1618 " user_fault %d gfn %lx\n",
1619 __func__, *shadow_pte, pt_access,
1620 write_fault, user_fault, gfn);
1621
1622 if (is_rmap_pte(*shadow_pte)) {
1623 /*
1624 * If we overwrite a PTE page pointer with a 2MB PMD, unlink
1625 * the parent of the now unreachable PTE.
1626 */
1627 if (largepage && !is_large_pte(*shadow_pte)) {
1628 struct kvm_mmu_page *child;
1629 u64 pte = *shadow_pte;
1630
1631 child = page_header(pte & PT64_BASE_ADDR_MASK);
1632 mmu_page_remove_parent_pte(child, shadow_pte);
1633 } else if (pfn != spte_to_pfn(*shadow_pte)) {
1634 pgprintk("hfn old %lx new %lx\n",
1635 spte_to_pfn(*shadow_pte), pfn);
1636 rmap_remove(vcpu->kvm, shadow_pte);
1637 } else {
1638 if (largepage)
1639 was_rmapped = is_large_pte(*shadow_pte);
1640 else
1641 was_rmapped = 1;
1642 }
1643 }
1644 if (set_spte(vcpu, shadow_pte, pte_access, user_fault, write_fault,
1645 dirty, largepage, gfn, pfn, speculative, true)) {
1646 if (write_fault)
1647 *ptwrite = 1;
1648 kvm_x86_ops->tlb_flush(vcpu);
1649 }
1650
1651 pgprintk("%s: setting spte %llx\n", __func__, *shadow_pte);
1652 pgprintk("instantiating %s PTE (%s) at %ld (%llx) addr %p\n",
1653 is_large_pte(*shadow_pte)? "2MB" : "4kB",
1654 is_present_pte(*shadow_pte)?"RW":"R", gfn,
1655 *shadow_pte, shadow_pte);
1656 if (!was_rmapped && is_large_pte(*shadow_pte))
1657 ++vcpu->kvm->stat.lpages;
1658
1659 page_header_update_slot(vcpu->kvm, shadow_pte, gfn);
1660 if (!was_rmapped) {
1661 rmap_add(vcpu, shadow_pte, gfn, largepage);
1662 if (!is_rmap_pte(*shadow_pte))
1663 kvm_release_pfn_clean(pfn);
1664 } else {
1665 if (was_writeble)
1666 kvm_release_pfn_dirty(pfn);
1667 else
1668 kvm_release_pfn_clean(pfn);
1669 }
1670 if (speculative) {
1671 vcpu->arch.last_pte_updated = shadow_pte;
1672 vcpu->arch.last_pte_gfn = gfn;
1673 }
1674 }
1675
1676 static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
1677 {
1678 }
1679
1680 struct direct_shadow_walk {
1681 struct kvm_shadow_walk walker;
1682 pfn_t pfn;
1683 int write;
1684 int largepage;
1685 int pt_write;
1686 };
1687
1688 static int direct_map_entry(struct kvm_shadow_walk *_walk,
1689 struct kvm_vcpu *vcpu,
1690 u64 addr, u64 *sptep, int level)
1691 {
1692 struct direct_shadow_walk *walk =
1693 container_of(_walk, struct direct_shadow_walk, walker);
1694 struct kvm_mmu_page *sp;
1695 gfn_t pseudo_gfn;
1696 gfn_t gfn = addr >> PAGE_SHIFT;
1697
1698 if (level == PT_PAGE_TABLE_LEVEL
1699 || (walk->largepage && level == PT_DIRECTORY_LEVEL)) {
1700 mmu_set_spte(vcpu, sptep, ACC_ALL, ACC_ALL,
1701 0, walk->write, 1, &walk->pt_write,
1702 walk->largepage, gfn, walk->pfn, false);
1703 ++vcpu->stat.pf_fixed;
1704 return 1;
1705 }
1706
1707 if (*sptep == shadow_trap_nonpresent_pte) {
1708 pseudo_gfn = (addr & PT64_DIR_BASE_ADDR_MASK) >> PAGE_SHIFT;
1709 sp = kvm_mmu_get_page(vcpu, pseudo_gfn, (gva_t)addr, level - 1,
1710 1, ACC_ALL, sptep);
1711 if (!sp) {
1712 pgprintk("nonpaging_map: ENOMEM\n");
1713 kvm_release_pfn_clean(walk->pfn);
1714 return -ENOMEM;
1715 }
1716
1717 set_shadow_pte(sptep,
1718 __pa(sp->spt)
1719 | PT_PRESENT_MASK | PT_WRITABLE_MASK
1720 | shadow_user_mask | shadow_x_mask);
1721 }
1722 return 0;
1723 }
1724
1725 static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
1726 int largepage, gfn_t gfn, pfn_t pfn)
1727 {
1728 int r;
1729 struct direct_shadow_walk walker = {
1730 .walker = { .entry = direct_map_entry, },
1731 .pfn = pfn,
1732 .largepage = largepage,
1733 .write = write,
1734 .pt_write = 0,
1735 };
1736
1737 r = walk_shadow(&walker.walker, vcpu, gfn << PAGE_SHIFT);
1738 if (r < 0)
1739 return r;
1740 return walker.pt_write;
1741 }
1742
1743 static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn)
1744 {
1745 int r;
1746 int largepage = 0;
1747 pfn_t pfn;
1748 unsigned long mmu_seq;
1749
1750 if (is_largepage_backed(vcpu, gfn & ~(KVM_PAGES_PER_HPAGE-1))) {
1751 gfn &= ~(KVM_PAGES_PER_HPAGE-1);
1752 largepage = 1;
1753 }
1754
1755 mmu_seq = vcpu->kvm->mmu_notifier_seq;
1756 smp_rmb();
1757 pfn = gfn_to_pfn(vcpu->kvm, gfn);
1758
1759 /* mmio */
1760 if (is_error_pfn(pfn)) {
1761 kvm_release_pfn_clean(pfn);
1762 return 1;
1763 }
1764
1765 spin_lock(&vcpu->kvm->mmu_lock);
1766 if (mmu_notifier_retry(vcpu, mmu_seq))
1767 goto out_unlock;
1768 kvm_mmu_free_some_pages(vcpu);
1769 r = __direct_map(vcpu, v, write, largepage, gfn, pfn);
1770 spin_unlock(&vcpu->kvm->mmu_lock);
1771
1772
1773 return r;
1774
1775 out_unlock:
1776 spin_unlock(&vcpu->kvm->mmu_lock);
1777 kvm_release_pfn_clean(pfn);
1778 return 0;
1779 }
1780
1781
1782 static void mmu_free_roots(struct kvm_vcpu *vcpu)
1783 {
1784 int i;
1785 struct kvm_mmu_page *sp;
1786
1787 if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
1788 return;
1789 spin_lock(&vcpu->kvm->mmu_lock);
1790 if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
1791 hpa_t root = vcpu->arch.mmu.root_hpa;
1792
1793 sp = page_header(root);
1794 --sp->root_count;
1795 if (!sp->root_count && sp->role.invalid)
1796 kvm_mmu_zap_page(vcpu->kvm, sp);
1797 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
1798 spin_unlock(&vcpu->kvm->mmu_lock);
1799 return;
1800 }
1801 for (i = 0; i < 4; ++i) {
1802 hpa_t root = vcpu->arch.mmu.pae_root[i];
1803
1804 if (root) {
1805 root &= PT64_BASE_ADDR_MASK;
1806 sp = page_header(root);
1807 --sp->root_count;
1808 if (!sp->root_count && sp->role.invalid)
1809 kvm_mmu_zap_page(vcpu->kvm, sp);
1810 }
1811 vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
1812 }
1813 spin_unlock(&vcpu->kvm->mmu_lock);
1814 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
1815 }
1816
1817 static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
1818 {
1819 int i;
1820 gfn_t root_gfn;
1821 struct kvm_mmu_page *sp;
1822 int metaphysical = 0;
1823
1824 root_gfn = vcpu->arch.cr3 >> PAGE_SHIFT;
1825
1826 if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
1827 hpa_t root = vcpu->arch.mmu.root_hpa;
1828
1829 ASSERT(!VALID_PAGE(root));
1830 if (tdp_enabled)
1831 metaphysical = 1;
1832 sp = kvm_mmu_get_page(vcpu, root_gfn, 0,
1833 PT64_ROOT_LEVEL, metaphysical,
1834 ACC_ALL, NULL);
1835 root = __pa(sp->spt);
1836 ++sp->root_count;
1837 vcpu->arch.mmu.root_hpa = root;
1838 return;
1839 }
1840 metaphysical = !is_paging(vcpu);
1841 if (tdp_enabled)
1842 metaphysical = 1;
1843 for (i = 0; i < 4; ++i) {
1844 hpa_t root = vcpu->arch.mmu.pae_root[i];
1845
1846 ASSERT(!VALID_PAGE(root));
1847 if (vcpu->arch.mmu.root_level == PT32E_ROOT_LEVEL) {
1848 if (!is_present_pte(vcpu->arch.pdptrs[i])) {
1849 vcpu->arch.mmu.pae_root[i] = 0;
1850 continue;
1851 }
1852 root_gfn = vcpu->arch.pdptrs[i] >> PAGE_SHIFT;
1853 } else if (vcpu->arch.mmu.root_level == 0)
1854 root_gfn = 0;
1855 sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
1856 PT32_ROOT_LEVEL, metaphysical,
1857 ACC_ALL, NULL);
1858 root = __pa(sp->spt);
1859 ++sp->root_count;
1860 vcpu->arch.mmu.pae_root[i] = root | PT_PRESENT_MASK;
1861 }
1862 vcpu->arch.mmu.root_hpa = __pa(vcpu->arch.mmu.pae_root);
1863 }
1864
1865 static void mmu_sync_roots(struct kvm_vcpu *vcpu)
1866 {
1867 int i;
1868 struct kvm_mmu_page *sp;
1869
1870 if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
1871 return;
1872 if (vcpu->arch.mmu.shadow_root_level == PT64_ROOT_LEVEL) {
1873 hpa_t root = vcpu->arch.mmu.root_hpa;
1874 sp = page_header(root);
1875 mmu_sync_children(vcpu, sp);
1876 return;
1877 }
1878 for (i = 0; i < 4; ++i) {
1879 hpa_t root = vcpu->arch.mmu.pae_root[i];
1880
1881 if (root) {
1882 root &= PT64_BASE_ADDR_MASK;
1883 sp = page_header(root);
1884 mmu_sync_children(vcpu, sp);
1885 }
1886 }
1887 }
1888
1889 void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
1890 {
1891 spin_lock(&vcpu->kvm->mmu_lock);
1892 mmu_sync_roots(vcpu);
1893 spin_unlock(&vcpu->kvm->mmu_lock);
1894 }
1895
1896 static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr)
1897 {
1898 return vaddr;
1899 }
1900
1901 static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
1902 u32 error_code)
1903 {
1904 gfn_t gfn;
1905 int r;
1906
1907 pgprintk("%s: gva %lx error %x\n", __func__, gva, error_code);
1908 r = mmu_topup_memory_caches(vcpu);
1909 if (r)
1910 return r;
1911
1912 ASSERT(vcpu);
1913 ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
1914
1915 gfn = gva >> PAGE_SHIFT;
1916
1917 return nonpaging_map(vcpu, gva & PAGE_MASK,
1918 error_code & PFERR_WRITE_MASK, gfn);
1919 }
1920
1921 static int tdp_page_fault(struct kvm_vcpu *vcpu, gva_t gpa,
1922 u32 error_code)
1923 {
1924 pfn_t pfn;
1925 int r;
1926 int largepage = 0;
1927 gfn_t gfn = gpa >> PAGE_SHIFT;
1928 unsigned long mmu_seq;
1929
1930 ASSERT(vcpu);
1931 ASSERT(VALID_PAGE(vcpu->arch.mmu.root_hpa));
1932
1933 r = mmu_topup_memory_caches(vcpu);
1934 if (r)
1935 return r;
1936
1937 if (is_largepage_backed(vcpu, gfn & ~(KVM_PAGES_PER_HPAGE-1))) {
1938 gfn &= ~(KVM_PAGES_PER_HPAGE-1);
1939 largepage = 1;
1940 }
1941 mmu_seq = vcpu->kvm->mmu_notifier_seq;
1942 smp_rmb();
1943 pfn = gfn_to_pfn(vcpu->kvm, gfn);
1944 if (is_error_pfn(pfn)) {
1945 kvm_release_pfn_clean(pfn);
1946 return 1;
1947 }
1948 spin_lock(&vcpu->kvm->mmu_lock);
1949 if (mmu_notifier_retry(vcpu, mmu_seq))
1950 goto out_unlock;
1951 kvm_mmu_free_some_pages(vcpu);
1952 r = __direct_map(vcpu, gpa, error_code & PFERR_WRITE_MASK,
1953 largepage, gfn, pfn);
1954 spin_unlock(&vcpu->kvm->mmu_lock);
1955
1956 return r;
1957
1958 out_unlock:
1959 spin_unlock(&vcpu->kvm->mmu_lock);
1960 kvm_release_pfn_clean(pfn);
1961 return 0;
1962 }
1963
1964 static void nonpaging_free(struct kvm_vcpu *vcpu)
1965 {
1966 mmu_free_roots(vcpu);
1967 }
1968
1969 static int nonpaging_init_context(struct kvm_vcpu *vcpu)
1970 {
1971 struct kvm_mmu *context = &vcpu->arch.mmu;
1972
1973 context->new_cr3 = nonpaging_new_cr3;
1974 context->page_fault = nonpaging_page_fault;
1975 context->gva_to_gpa = nonpaging_gva_to_gpa;
1976 context->free = nonpaging_free;
1977 context->prefetch_page = nonpaging_prefetch_page;
1978 context->sync_page = nonpaging_sync_page;
1979 context->invlpg = nonpaging_invlpg;
1980 context->root_level = 0;
1981 context->shadow_root_level = PT32E_ROOT_LEVEL;
1982 context->root_hpa = INVALID_PAGE;
1983 return 0;
1984 }
1985
1986 void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
1987 {
1988 ++vcpu->stat.tlb_flush;
1989 kvm_x86_ops->tlb_flush(vcpu);
1990 }
1991
1992 static void paging_new_cr3(struct kvm_vcpu *vcpu)
1993 {
1994 pgprintk("%s: cr3 %lx\n", __func__, vcpu->arch.cr3);
1995 mmu_free_roots(vcpu);
1996 }
1997
1998 static void inject_page_fault(struct kvm_vcpu *vcpu,
1999 u64 addr,
2000 u32 err_code)
2001 {
2002 kvm_inject_page_fault(vcpu, addr, err_code);
2003 }
2004
2005 static void paging_free(struct kvm_vcpu *vcpu)
2006 {
2007 nonpaging_free(vcpu);
2008 }
2009
2010 #define PTTYPE 64
2011 #include "paging_tmpl.h"
2012 #undef PTTYPE
2013
2014 #define PTTYPE 32
2015 #include "paging_tmpl.h"
2016 #undef PTTYPE
2017
2018 static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level)
2019 {
2020 struct kvm_mmu *context = &vcpu->arch.mmu;
2021
2022 ASSERT(is_pae(vcpu));
2023 context->new_cr3 = paging_new_cr3;
2024 context->page_fault = paging64_page_fault;
2025 context->gva_to_gpa = paging64_gva_to_gpa;
2026 context->prefetch_page = paging64_prefetch_page;
2027 context->sync_page = paging64_sync_page;
2028 context->invlpg = paging64_invlpg;
2029 context->free = paging_free;
2030 context->root_level = level;
2031 context->shadow_root_level = level;
2032 context->root_hpa = INVALID_PAGE;
2033 return 0;
2034 }
2035
2036 static int paging64_init_context(struct kvm_vcpu *vcpu)
2037 {
2038 return paging64_init_context_common(vcpu, PT64_ROOT_LEVEL);
2039 }
2040
2041 static int paging32_init_context(struct kvm_vcpu *vcpu)
2042 {
2043 struct kvm_mmu *context = &vcpu->arch.mmu;
2044
2045 context->new_cr3 = paging_new_cr3;
2046 context->page_fault = paging32_page_fault;
2047 context->gva_to_gpa = paging32_gva_to_gpa;
2048 context->free = paging_free;
2049 context->prefetch_page = paging32_prefetch_page;
2050 context->sync_page = paging32_sync_page;
2051 context->invlpg = paging32_invlpg;
2052 context->root_level = PT32_ROOT_LEVEL;
2053 context->shadow_root_level = PT32E_ROOT_LEVEL;
2054 context->root_hpa = INVALID_PAGE;
2055 return 0;
2056 }
2057
2058 static int paging32E_init_context(struct kvm_vcpu *vcpu)
2059 {
2060 return paging64_init_context_common(vcpu, PT32E_ROOT_LEVEL);
2061 }
2062
2063 static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
2064 {
2065 struct kvm_mmu *context = &vcpu->arch.mmu;
2066
2067 context->new_cr3 = nonpaging_new_cr3;
2068 context->page_fault = tdp_page_fault;
2069 context->free = nonpaging_free;
2070 context->prefetch_page = nonpaging_prefetch_page;
2071 context->sync_page = nonpaging_sync_page;
2072 context->invlpg = nonpaging_invlpg;
2073 context->shadow_root_level = kvm_x86_ops->get_tdp_level();
2074 context->root_hpa = INVALID_PAGE;
2075
2076 if (!is_paging(vcpu)) {
2077 context->gva_to_gpa = nonpaging_gva_to_gpa;
2078 context->root_level = 0;
2079 } else if (is_long_mode(vcpu)) {
2080 context->gva_to_gpa = paging64_gva_to_gpa;
2081 context->root_level = PT64_ROOT_LEVEL;
2082 } else if (is_pae(vcpu)) {
2083 context->gva_to_gpa = paging64_gva_to_gpa;
2084 context->root_level = PT32E_ROOT_LEVEL;
2085 } else {
2086 context->gva_to_gpa = paging32_gva_to_gpa;
2087 context->root_level = PT32_ROOT_LEVEL;
2088 }
2089
2090 return 0;
2091 }
2092
2093 static int init_kvm_softmmu(struct kvm_vcpu *vcpu)
2094 {
2095 ASSERT(vcpu);
2096 ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
2097
2098 if (!is_paging(vcpu))
2099 return nonpaging_init_context(vcpu);
2100 else if (is_long_mode(vcpu))
2101 return paging64_init_context(vcpu);
2102 else if (is_pae(vcpu))
2103 return paging32E_init_context(vcpu);
2104 else
2105 return paging32_init_context(vcpu);
2106 }
2107
2108 static int init_kvm_mmu(struct kvm_vcpu *vcpu)
2109 {
2110 vcpu->arch.update_pte.pfn = bad_pfn;
2111
2112 if (tdp_enabled)
2113 return init_kvm_tdp_mmu(vcpu);
2114 else
2115 return init_kvm_softmmu(vcpu);
2116 }
2117
2118 static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
2119 {
2120 ASSERT(vcpu);
2121 if (VALID_PAGE(vcpu->arch.mmu.root_hpa)) {
2122 vcpu->arch.mmu.free(vcpu);
2123 vcpu->arch.mmu.root_hpa = INVALID_PAGE;
2124 }
2125 }
2126
2127 int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
2128 {
2129 destroy_kvm_mmu(vcpu);
2130 return init_kvm_mmu(vcpu);
2131 }
2132 EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
2133
2134 int kvm_mmu_load(struct kvm_vcpu *vcpu)
2135 {
2136 int r;
2137
2138 r = mmu_topup_memory_caches(vcpu);
2139 if (r)
2140 goto out;
2141 spin_lock(&vcpu->kvm->mmu_lock);
2142 kvm_mmu_free_some_pages(vcpu);
2143 mmu_alloc_roots(vcpu);
2144 mmu_sync_roots(vcpu);
2145 spin_unlock(&vcpu->kvm->mmu_lock);
2146 kvm_x86_ops->set_cr3(vcpu, vcpu->arch.mmu.root_hpa);
2147 kvm_mmu_flush_tlb(vcpu);
2148 out:
2149 return r;
2150 }
2151 EXPORT_SYMBOL_GPL(kvm_mmu_load);
2152
2153 void kvm_mmu_unload(struct kvm_vcpu *vcpu)
2154 {
2155 mmu_free_roots(vcpu);
2156 }
2157
2158 static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu,
2159 struct kvm_mmu_page *sp,
2160 u64 *spte)
2161 {
2162 u64 pte;
2163 struct kvm_mmu_page *child;
2164
2165 pte = *spte;
2166 if (is_shadow_present_pte(pte)) {
2167 if (sp->role.level == PT_PAGE_TABLE_LEVEL ||
2168 is_large_pte(pte))
2169 rmap_remove(vcpu->kvm, spte);
2170 else {
2171 child = page_header(pte & PT64_BASE_ADDR_MASK);
2172 mmu_page_remove_parent_pte(child, spte);
2173 }
2174 }
2175 set_shadow_pte(spte, shadow_trap_nonpresent_pte);
2176 if (is_large_pte(pte))
2177 --vcpu->kvm->stat.lpages;
2178 }
2179
2180 static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
2181 struct kvm_mmu_page *sp,
2182 u64 *spte,
2183 const void *new)
2184 {
2185 if (sp->role.level != PT_PAGE_TABLE_LEVEL) {
2186 if (!vcpu->arch.update_pte.largepage ||
2187 sp->role.glevels == PT32_ROOT_LEVEL) {
2188 ++vcpu->kvm->stat.mmu_pde_zapped;
2189 return;
2190 }
2191 }
2192
2193 ++vcpu->kvm->stat.mmu_pte_updated;
2194 if (sp->role.glevels == PT32_ROOT_LEVEL)
2195 paging32_update_pte(vcpu, sp, spte, new);
2196 else
2197 paging64_update_pte(vcpu, sp, spte, new);
2198 }
2199
2200 static bool need_remote_flush(u64 old, u64 new)
2201 {
2202 if (!is_shadow_present_pte(old))
2203 return false;
2204 if (!is_shadow_present_pte(new))
2205 return true;
2206 if ((old ^ new) & PT64_BASE_ADDR_MASK)
2207 return true;
2208 old ^= PT64_NX_MASK;
2209 new ^= PT64_NX_MASK;
2210 return (old & ~new & PT64_PERM_MASK) != 0;
2211 }
2212
2213 static void mmu_pte_write_flush_tlb(struct kvm_vcpu *vcpu, u64 old, u64 new)
2214 {
2215 if (need_remote_flush(old, new))
2216 kvm_flush_remote_tlbs(vcpu->kvm);
2217 else
2218 kvm_mmu_flush_tlb(vcpu);
2219 }
2220
2221 static bool last_updated_pte_accessed(struct kvm_vcpu *vcpu)
2222 {
2223 u64 *spte = vcpu->arch.last_pte_updated;
2224
2225 return !!(spte && (*spte & shadow_accessed_mask));
2226 }
2227
2228 static void mmu_guess_page_from_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
2229 const u8 *new, int bytes)
2230 {
2231 gfn_t gfn;
2232 int r;
2233 u64 gpte = 0;
2234 pfn_t pfn;
2235
2236 vcpu->arch.update_pte.largepage = 0;
2237
2238 if (bytes != 4 && bytes != 8)
2239 return;
2240
2241 /*
2242 * Assume that the pte write on a page table of the same type
2243 * as the current vcpu paging mode. This is nearly always true
2244 * (might be false while changing modes). Note it is verified later
2245 * by update_pte().
2246 */
2247 if (is_pae(vcpu)) {
2248 /* Handle a 32-bit guest writing two halves of a 64-bit gpte */
2249 if ((bytes == 4) && (gpa % 4 == 0)) {
2250 r = kvm_read_guest(vcpu->kvm, gpa & ~(u64)7, &gpte, 8);
2251 if (r)
2252 return;
2253 memcpy((void *)&gpte + (gpa % 8), new, 4);
2254 } else if ((bytes == 8) && (gpa % 8 == 0)) {
2255 memcpy((void *)&gpte, new, 8);
2256 }
2257 } else {
2258 if ((bytes == 4) && (gpa % 4 == 0))
2259 memcpy((void *)&gpte, new, 4);
2260 }
2261 if (!is_present_pte(gpte))
2262 return;
2263 gfn = (gpte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT;
2264
2265 if (is_large_pte(gpte) && is_largepage_backed(vcpu, gfn)) {
2266 gfn &= ~(KVM_PAGES_PER_HPAGE-1);
2267 vcpu->arch.update_pte.largepage = 1;
2268 }
2269 vcpu->arch.update_pte.mmu_seq = vcpu->kvm->mmu_notifier_seq;
2270 smp_rmb();
2271 pfn = gfn_to_pfn(vcpu->kvm, gfn);
2272
2273 if (is_error_pfn(pfn)) {
2274 kvm_release_pfn_clean(pfn);
2275 return;
2276 }
2277 vcpu->arch.update_pte.gfn = gfn;
2278 vcpu->arch.update_pte.pfn = pfn;
2279 }
2280
2281 static void kvm_mmu_access_page(struct kvm_vcpu *vcpu, gfn_t gfn)
2282 {
2283 u64 *spte = vcpu->arch.last_pte_updated;
2284
2285 if (spte
2286 && vcpu->arch.last_pte_gfn == gfn
2287 && shadow_accessed_mask
2288 && !(*spte & shadow_accessed_mask)
2289 && is_shadow_present_pte(*spte))
2290 set_bit(PT_ACCESSED_SHIFT, (unsigned long *)spte);
2291 }
2292
2293 void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
2294 const u8 *new, int bytes)
2295 {
2296 gfn_t gfn = gpa >> PAGE_SHIFT;
2297 struct kvm_mmu_page *sp;
2298 struct hlist_node *node, *n;
2299 struct hlist_head *bucket;
2300 unsigned index;
2301 u64 entry, gentry;
2302 u64 *spte;
2303 unsigned offset = offset_in_page(gpa);
2304 unsigned pte_size;
2305 unsigned page_offset;
2306 unsigned misaligned;
2307 unsigned quadrant;
2308 int level;
2309 int flooded = 0;
2310 int npte;
2311 int r;
2312
2313 pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);
2314 mmu_guess_page_from_pte_write(vcpu, gpa, new, bytes);
2315 spin_lock(&vcpu->kvm->mmu_lock);
2316 kvm_mmu_access_page(vcpu, gfn);
2317 kvm_mmu_free_some_pages(vcpu);
2318 ++vcpu->kvm->stat.mmu_pte_write;
2319 kvm_mmu_audit(vcpu, "pre pte write");
2320 if (gfn == vcpu->arch.last_pt_write_gfn
2321 && !last_updated_pte_accessed(vcpu)) {
2322 ++vcpu->arch.last_pt_write_count;
2323 if (vcpu->arch.last_pt_write_count >= 3)
2324 flooded = 1;
2325 } else {
2326 vcpu->arch.last_pt_write_gfn = gfn;
2327 vcpu->arch.last_pt_write_count = 1;
2328 vcpu->arch.last_pte_updated = NULL;
2329 }
2330 index = kvm_page_table_hashfn(gfn);
2331 bucket = &vcpu->kvm->arch.mmu_page_hash[index];
2332 hlist_for_each_entry_safe(sp, node, n, bucket, hash_link) {
2333 if (sp->gfn != gfn || sp->role.metaphysical || sp->role.invalid)
2334 continue;
2335 pte_size = sp->role.glevels == PT32_ROOT_LEVEL ? 4 : 8;
2336 misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
2337 misaligned |= bytes < 4;
2338 if (misaligned || flooded) {
2339 /*
2340 * Misaligned accesses are too much trouble to fix
2341 * up; also, they usually indicate a page is not used
2342 * as a page table.
2343 *
2344 * If we're seeing too many writes to a page,
2345 * it may no longer be a page table, or we may be
2346 * forking, in which case it is better to unmap the
2347 * page.
2348 */
2349 pgprintk("misaligned: gpa %llx bytes %d role %x\n",
2350 gpa, bytes, sp->role.word);
2351 if (kvm_mmu_zap_page(vcpu->kvm, sp))
2352 n = bucket->first;
2353 ++vcpu->kvm->stat.mmu_flooded;
2354 continue;
2355 }
2356 page_offset = offset;
2357 level = sp->role.level;
2358 npte = 1;
2359 if (sp->role.glevels == PT32_ROOT_LEVEL) {
2360 page_offset <<= 1; /* 32->64 */
2361 /*
2362 * A 32-bit pde maps 4MB while the shadow pdes map
2363 * only 2MB. So we need to double the offset again
2364 * and zap two pdes instead of one.
2365 */
2366 if (level == PT32_ROOT_LEVEL) {
2367 page_offset &= ~7; /* kill rounding error */
2368 page_offset <<= 1;
2369 npte = 2;
2370 }
2371 quadrant = page_offset >> PAGE_SHIFT;
2372 page_offset &= ~PAGE_MASK;
2373 if (quadrant != sp->role.quadrant)
2374 continue;
2375 }
2376 spte = &sp->spt[page_offset / sizeof(*spte)];
2377 if ((gpa & (pte_size - 1)) || (bytes < pte_size)) {
2378 gentry = 0;
2379 r = kvm_read_guest_atomic(vcpu->kvm,
2380 gpa & ~(u64)(pte_size - 1),
2381 &gentry, pte_size);
2382 new = (const void *)&gentry;
2383 if (r < 0)
2384 new = NULL;
2385 }
2386 while (npte--) {
2387 entry = *spte;
2388 mmu_pte_write_zap_pte(vcpu, sp, spte);
2389 if (new)
2390 mmu_pte_write_new_pte(vcpu, sp, spte, new);
2391 mmu_pte_write_flush_tlb(vcpu, entry, *spte);
2392 ++spte;
2393 }
2394 }
2395 kvm_mmu_audit(vcpu, "post pte write");
2396 spin_unlock(&vcpu->kvm->mmu_lock);
2397 if (!is_error_pfn(vcpu->arch.update_pte.pfn)) {
2398 kvm_release_pfn_clean(vcpu->arch.update_pte.pfn);
2399 vcpu->arch.update_pte.pfn = bad_pfn;
2400 }
2401 }
2402
2403 int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
2404 {
2405 gpa_t gpa;
2406 int r;
2407
2408 gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, gva);
2409
2410 spin_lock(&vcpu->kvm->mmu_lock);
2411 r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);
2412 spin_unlock(&vcpu->kvm->mmu_lock);
2413 return r;
2414 }
2415 EXPORT_SYMBOL_GPL(kvm_mmu_unprotect_page_virt);
2416
2417 void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
2418 {
2419 while (vcpu->kvm->arch.n_free_mmu_pages < KVM_REFILL_PAGES) {
2420 struct kvm_mmu_page *sp;
2421
2422 sp = container_of(vcpu->kvm->arch.active_mmu_pages.prev,
2423 struct kvm_mmu_page, link);
2424 kvm_mmu_zap_page(vcpu->kvm, sp);
2425 ++vcpu->kvm->stat.mmu_recycled;
2426 }
2427 }
2428
2429 int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t cr2, u32 error_code)
2430 {
2431 int r;
2432 enum emulation_result er;
2433
2434 r = vcpu->arch.mmu.page_fault(vcpu, cr2, error_code);
2435 if (r < 0)
2436 goto out;
2437
2438 if (!r) {
2439 r = 1;
2440 goto out;
2441 }
2442
2443 r = mmu_topup_memory_caches(vcpu);
2444 if (r)
2445 goto out;
2446
2447 er = emulate_instruction(vcpu, vcpu->run, cr2, error_code, 0);
2448
2449 switch (er) {
2450 case EMULATE_DONE:
2451 return 1;
2452 case EMULATE_DO_MMIO:
2453 ++vcpu->stat.mmio_exits;
2454 return 0;
2455 case EMULATE_FAIL:
2456 kvm_report_emulation_failure(vcpu, "pagetable");
2457 return 1;
2458 default:
2459 BUG();
2460 }
2461 out:
2462 return r;
2463 }
2464 EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);
2465
2466 void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
2467 {
2468 spin_lock(&vcpu->kvm->mmu_lock);
2469 vcpu->arch.mmu.invlpg(vcpu, gva);
2470 spin_unlock(&vcpu->kvm->mmu_lock);
2471 kvm_mmu_flush_tlb(vcpu);
2472 ++vcpu->stat.invlpg;
2473 }
2474 EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);
2475
2476 void kvm_enable_tdp(void)
2477 {
2478 tdp_enabled = true;
2479 }
2480 EXPORT_SYMBOL_GPL(kvm_enable_tdp);
2481
2482 void kvm_disable_tdp(void)
2483 {
2484 tdp_enabled = false;
2485 }
2486 EXPORT_SYMBOL_GPL(kvm_disable_tdp);
2487
2488 static void free_mmu_pages(struct kvm_vcpu *vcpu)
2489 {
2490 struct kvm_mmu_page *sp;
2491
2492 while (!list_empty(&vcpu->kvm->arch.active_mmu_pages)) {
2493 sp = container_of(vcpu->kvm->arch.active_mmu_pages.next,
2494 struct kvm_mmu_page, link);
2495 kvm_mmu_zap_page(vcpu->kvm, sp);
2496 cond_resched();
2497 }
2498 free_page((unsigned long)vcpu->arch.mmu.pae_root);
2499 }
2500
2501 static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
2502 {
2503 struct page *page;
2504 int i;
2505
2506 ASSERT(vcpu);
2507
2508 if (vcpu->kvm->arch.n_requested_mmu_pages)
2509 vcpu->kvm->arch.n_free_mmu_pages =
2510 vcpu->kvm->arch.n_requested_mmu_pages;
2511 else
2512 vcpu->kvm->arch.n_free_mmu_pages =
2513 vcpu->kvm->arch.n_alloc_mmu_pages;
2514 /*
2515 * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
2516 * Therefore we need to allocate shadow page tables in the first
2517 * 4GB of memory, which happens to fit the DMA32 zone.
2518 */
2519 page = alloc_page(GFP_KERNEL | __GFP_DMA32);
2520 if (!page)
2521 goto error_1;
2522 vcpu->arch.mmu.pae_root = page_address(page);
2523 for (i = 0; i < 4; ++i)
2524 vcpu->arch.mmu.pae_root[i] = INVALID_PAGE;
2525
2526 return 0;
2527
2528 error_1:
2529 free_mmu_pages(vcpu);
2530 return -ENOMEM;
2531 }
2532
2533 int kvm_mmu_create(struct kvm_vcpu *vcpu)
2534 {
2535 ASSERT(vcpu);
2536 ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
2537
2538 return alloc_mmu_pages(vcpu);
2539 }
2540
2541 int kvm_mmu_setup(struct kvm_vcpu *vcpu)
2542 {
2543 ASSERT(vcpu);
2544 ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa));
2545
2546 return init_kvm_mmu(vcpu);
2547 }
2548
2549 void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
2550 {
2551 ASSERT(vcpu);
2552
2553 destroy_kvm_mmu(vcpu);
2554 free_mmu_pages(vcpu);
2555 mmu_free_memory_caches(vcpu);
2556 }
2557
2558 void kvm_mmu_slot_remove_write_access(struct kvm *kvm, int slot)
2559 {
2560 struct kvm_mmu_page *sp;
2561
2562 spin_lock(&kvm->mmu_lock);
2563 list_for_each_entry(sp, &kvm->arch.active_mmu_pages, link) {
2564 int i;
2565 u64 *pt;
2566
2567 if (!test_bit(slot, &sp->slot_bitmap))
2568 continue;
2569
2570 pt = sp->spt;
2571 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
2572 /* avoid RMW */
2573 if (pt[i] & PT_WRITABLE_MASK)
2574 pt[i] &= ~PT_WRITABLE_MASK;
2575 }
2576 kvm_flush_remote_tlbs(kvm);
2577 spin_unlock(&kvm->mmu_lock);
2578 }
2579
2580 void kvm_mmu_zap_all(struct kvm *kvm)
2581 {
2582 struct kvm_mmu_page *sp, *node;
2583
2584 spin_lock(&kvm->mmu_lock);
2585 list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link)
2586 if (kvm_mmu_zap_page(kvm, sp))
2587 node = container_of(kvm->arch.active_mmu_pages.next,
2588 struct kvm_mmu_page, link);
2589 spin_unlock(&kvm->mmu_lock);
2590
2591 kvm_flush_remote_tlbs(kvm);
2592 }
2593
2594 static void kvm_mmu_remove_one_alloc_mmu_page(struct kvm *kvm)
2595 {
2596 struct kvm_mmu_page *page;
2597
2598 page = container_of(kvm->arch.active_mmu_pages.prev,
2599 struct kvm_mmu_page, link);
2600 kvm_mmu_zap_page(kvm, page);
2601 }
2602
2603 static int mmu_shrink(int nr_to_scan, gfp_t gfp_mask)
2604 {
2605 struct kvm *kvm;
2606 struct kvm *kvm_freed = NULL;
2607 int cache_count = 0;
2608
2609 spin_lock(&kvm_lock);
2610
2611 list_for_each_entry(kvm, &vm_list, vm_list) {
2612 int npages;
2613
2614 if (!down_read_trylock(&kvm->slots_lock))
2615 continue;
2616 spin_lock(&kvm->mmu_lock);
2617 npages = kvm->arch.n_alloc_mmu_pages -
2618 kvm->arch.n_free_mmu_pages;
2619 cache_count += npages;
2620 if (!kvm_freed && nr_to_scan > 0 && npages > 0) {
2621 kvm_mmu_remove_one_alloc_mmu_page(kvm);
2622 cache_count--;
2623 kvm_freed = kvm;
2624 }
2625 nr_to_scan--;
2626
2627 spin_unlock(&kvm->mmu_lock);
2628 up_read(&kvm->slots_lock);
2629 }
2630 if (kvm_freed)
2631 list_move_tail(&kvm_freed->vm_list, &vm_list);
2632
2633 spin_unlock(&kvm_lock);
2634
2635 return cache_count;
2636 }
2637
2638 static struct shrinker mmu_shrinker = {
2639 .shrink = mmu_shrink,
2640 .seeks = DEFAULT_SEEKS * 10,
2641 };
2642
2643 static void mmu_destroy_caches(void)
2644 {
2645 if (pte_chain_cache)
2646 kmem_cache_destroy(pte_chain_cache);
2647 if (rmap_desc_cache)
2648 kmem_cache_destroy(rmap_desc_cache);
2649 if (mmu_page_header_cache)
2650 kmem_cache_destroy(mmu_page_header_cache);
2651 }
2652
2653 void kvm_mmu_module_exit(void)
2654 {
2655 mmu_destroy_caches();
2656 unregister_shrinker(&mmu_shrinker);
2657 }
2658
2659 int kvm_mmu_module_init(void)
2660 {
2661 pte_chain_cache = kmem_cache_create("kvm_pte_chain",
2662 sizeof(struct kvm_pte_chain),
2663 0, 0, NULL);
2664 if (!pte_chain_cache)
2665 goto nomem;
2666 rmap_desc_cache = kmem_cache_create("kvm_rmap_desc",
2667 sizeof(struct kvm_rmap_desc),
2668 0, 0, NULL);
2669 if (!rmap_desc_cache)
2670 goto nomem;
2671
2672 mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
2673 sizeof(struct kvm_mmu_page),
2674 0, 0, NULL);
2675 if (!mmu_page_header_cache)
2676 goto nomem;
2677
2678 register_shrinker(&mmu_shrinker);
2679
2680 return 0;
2681
2682 nomem:
2683 mmu_destroy_caches();
2684 return -ENOMEM;
2685 }
2686
2687 /*
2688 * Caculate mmu pages needed for kvm.
2689 */
2690 unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm)
2691 {
2692 int i;
2693 unsigned int nr_mmu_pages;
2694 unsigned int nr_pages = 0;
2695
2696 for (i = 0; i < kvm->nmemslots; i++)
2697 nr_pages += kvm->memslots[i].npages;
2698
2699 nr_mmu_pages = nr_pages * KVM_PERMILLE_MMU_PAGES / 1000;
2700 nr_mmu_pages = max(nr_mmu_pages,
2701 (unsigned int) KVM_MIN_ALLOC_MMU_PAGES);
2702
2703 return nr_mmu_pages;
2704 }
2705
2706 static void *pv_mmu_peek_buffer(struct kvm_pv_mmu_op_buffer *buffer,
2707 unsigned len)
2708 {
2709 if (len > buffer->len)
2710 return NULL;
2711 return buffer->ptr;
2712 }
2713
2714 static void *pv_mmu_read_buffer(struct kvm_pv_mmu_op_buffer *buffer,
2715 unsigned len)
2716 {
2717 void *ret;
2718
2719 ret = pv_mmu_peek_buffer(buffer, len);
2720 if (!ret)
2721 return ret;
2722 buffer->ptr += len;
2723 buffer->len -= len;
2724 buffer->processed += len;
2725 return ret;
2726 }
2727
2728 static int kvm_pv_mmu_write(struct kvm_vcpu *vcpu,
2729 gpa_t addr, gpa_t value)
2730 {
2731 int bytes = 8;
2732 int r;
2733
2734 if (!is_long_mode(vcpu) && !is_pae(vcpu))
2735 bytes = 4;
2736
2737 r = mmu_topup_memory_caches(vcpu);
2738 if (r)
2739 return r;
2740
2741 if (!emulator_write_phys(vcpu, addr, &value, bytes))
2742 return -EFAULT;
2743
2744 return 1;
2745 }
2746
2747 static int kvm_pv_mmu_flush_tlb(struct kvm_vcpu *vcpu)
2748 {
2749 kvm_x86_ops->tlb_flush(vcpu);
2750 set_bit(KVM_REQ_MMU_SYNC, &vcpu->requests);
2751 return 1;
2752 }
2753
2754 static int kvm_pv_mmu_release_pt(struct kvm_vcpu *vcpu, gpa_t addr)
2755 {
2756 spin_lock(&vcpu->kvm->mmu_lock);
2757 mmu_unshadow(vcpu->kvm, addr >> PAGE_SHIFT);
2758 spin_unlock(&vcpu->kvm->mmu_lock);
2759 return 1;
2760 }
2761
2762 static int kvm_pv_mmu_op_one(struct kvm_vcpu *vcpu,
2763 struct kvm_pv_mmu_op_buffer *buffer)
2764 {
2765 struct kvm_mmu_op_header *header;
2766
2767 header = pv_mmu_peek_buffer(buffer, sizeof *header);
2768 if (!header)
2769 return 0;
2770 switch (header->op) {
2771 case KVM_MMU_OP_WRITE_PTE: {
2772 struct kvm_mmu_op_write_pte *wpte;
2773
2774 wpte = pv_mmu_read_buffer(buffer, sizeof *wpte);
2775 if (!wpte)
2776 return 0;
2777 return kvm_pv_mmu_write(vcpu, wpte->pte_phys,
2778 wpte->pte_val);
2779 }
2780 case KVM_MMU_OP_FLUSH_TLB: {
2781 struct kvm_mmu_op_flush_tlb *ftlb;
2782
2783 ftlb = pv_mmu_read_buffer(buffer, sizeof *ftlb);
2784 if (!ftlb)
2785 return 0;
2786 return kvm_pv_mmu_flush_tlb(vcpu);
2787 }
2788 case KVM_MMU_OP_RELEASE_PT: {
2789 struct kvm_mmu_op_release_pt *rpt;
2790
2791 rpt = pv_mmu_read_buffer(buffer, sizeof *rpt);
2792 if (!rpt)
2793 return 0;
2794 return kvm_pv_mmu_release_pt(vcpu, rpt->pt_phys);
2795 }
2796 default: return 0;
2797 }
2798 }
2799
2800 int kvm_pv_mmu_op(struct kvm_vcpu *vcpu, unsigned long bytes,
2801 gpa_t addr, unsigned long *ret)
2802 {
2803 int r;
2804 struct kvm_pv_mmu_op_buffer *buffer = &vcpu->arch.mmu_op_buffer;
2805
2806 buffer->ptr = buffer->buf;
2807 buffer->len = min_t(unsigned long, bytes, sizeof buffer->buf);
2808 buffer->processed = 0;
2809
2810 r = kvm_read_guest(vcpu->kvm, addr, buffer->buf, buffer->len);
2811 if (r)
2812 goto out;
2813
2814 while (buffer->len) {
2815 r = kvm_pv_mmu_op_one(vcpu, buffer);
2816 if (r < 0)
2817 goto out;
2818 if (r == 0)
2819 break;
2820 }
2821
2822 r = 1;
2823 out:
2824 *ret = buffer->processed;
2825 return r;
2826 }
2827
2828 #ifdef AUDIT
2829
2830 static const char *audit_msg;
2831
2832 static gva_t canonicalize(gva_t gva)
2833 {
2834 #ifdef CONFIG_X86_64
2835 gva = (long long)(gva << 16) >> 16;
2836 #endif
2837 return gva;
2838 }
2839
2840 static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte,
2841 gva_t va, int level)
2842 {
2843 u64 *pt = __va(page_pte & PT64_BASE_ADDR_MASK);
2844 int i;
2845 gva_t va_delta = 1ul << (PAGE_SHIFT + 9 * (level - 1));
2846
2847 for (i = 0; i < PT64_ENT_PER_PAGE; ++i, va += va_delta) {
2848 u64 ent = pt[i];
2849
2850 if (ent == shadow_trap_nonpresent_pte)
2851 continue;
2852
2853 va = canonicalize(va);
2854 if (level > 1) {
2855 if (ent == shadow_notrap_nonpresent_pte)
2856 printk(KERN_ERR "audit: (%s) nontrapping pte"
2857 " in nonleaf level: levels %d gva %lx"
2858 " level %d pte %llx\n", audit_msg,
2859 vcpu->arch.mmu.root_level, va, level, ent);
2860
2861 audit_mappings_page(vcpu, ent, va, level - 1);
2862 } else {
2863 gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, va);
2864 hpa_t hpa = (hpa_t)gpa_to_pfn(vcpu, gpa) << PAGE_SHIFT;
2865
2866 if (is_shadow_present_pte(ent)
2867 && (ent & PT64_BASE_ADDR_MASK) != hpa)
2868 printk(KERN_ERR "xx audit error: (%s) levels %d"
2869 " gva %lx gpa %llx hpa %llx ent %llx %d\n",
2870 audit_msg, vcpu->arch.mmu.root_level,
2871 va, gpa, hpa, ent,
2872 is_shadow_present_pte(ent));
2873 else if (ent == shadow_notrap_nonpresent_pte
2874 && !is_error_hpa(hpa))
2875 printk(KERN_ERR "audit: (%s) notrap shadow,"
2876 " valid guest gva %lx\n", audit_msg, va);
2877 kvm_release_pfn_clean(pfn);
2878
2879 }
2880 }
2881 }
2882
2883 static void audit_mappings(struct kvm_vcpu *vcpu)
2884 {
2885 unsigned i;
2886
2887 if (vcpu->arch.mmu.root_level == 4)
2888 audit_mappings_page(vcpu, vcpu->arch.mmu.root_hpa, 0, 4);
2889 else
2890 for (i = 0; i < 4; ++i)
2891 if (vcpu->arch.mmu.pae_root[i] & PT_PRESENT_MASK)
2892 audit_mappings_page(vcpu,
2893 vcpu->arch.mmu.pae_root[i],
2894 i << 30,
2895 2);
2896 }
2897
2898 static int count_rmaps(struct kvm_vcpu *vcpu)
2899 {
2900 int nmaps = 0;
2901 int i, j, k;
2902
2903 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
2904 struct kvm_memory_slot *m = &vcpu->kvm->memslots[i];
2905 struct kvm_rmap_desc *d;
2906
2907 for (j = 0; j < m->npages; ++j) {
2908 unsigned long *rmapp = &m->rmap[j];
2909
2910 if (!*rmapp)
2911 continue;
2912 if (!(*rmapp & 1)) {
2913 ++nmaps;
2914 continue;
2915 }
2916 d = (struct kvm_rmap_desc *)(*rmapp & ~1ul);
2917 while (d) {
2918 for (k = 0; k < RMAP_EXT; ++k)
2919 if (d->shadow_ptes[k])
2920 ++nmaps;
2921 else
2922 break;
2923 d = d->more;
2924 }
2925 }
2926 }
2927 return nmaps;
2928 }
2929
2930 static int count_writable_mappings(struct kvm_vcpu *vcpu)
2931 {
2932 int nmaps = 0;
2933 struct kvm_mmu_page *sp;
2934 int i;
2935
2936 list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) {
2937 u64 *pt = sp->spt;
2938
2939 if (sp->role.level != PT_PAGE_TABLE_LEVEL)
2940 continue;
2941
2942 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
2943 u64 ent = pt[i];
2944
2945 if (!(ent & PT_PRESENT_MASK))
2946 continue;
2947 if (!(ent & PT_WRITABLE_MASK))
2948 continue;
2949 ++nmaps;
2950 }
2951 }
2952 return nmaps;
2953 }
2954
2955 static void audit_rmap(struct kvm_vcpu *vcpu)
2956 {
2957 int n_rmap = count_rmaps(vcpu);
2958 int n_actual = count_writable_mappings(vcpu);
2959
2960 if (n_rmap != n_actual)
2961 printk(KERN_ERR "%s: (%s) rmap %d actual %d\n",
2962 __func__, audit_msg, n_rmap, n_actual);
2963 }
2964
2965 static void audit_write_protection(struct kvm_vcpu *vcpu)
2966 {
2967 struct kvm_mmu_page *sp;
2968 struct kvm_memory_slot *slot;
2969 unsigned long *rmapp;
2970 gfn_t gfn;
2971
2972 list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) {
2973 if (sp->role.metaphysical)
2974 continue;
2975
2976 slot = gfn_to_memslot(vcpu->kvm, sp->gfn);
2977 gfn = unalias_gfn(vcpu->kvm, sp->gfn);
2978 rmapp = &slot->rmap[gfn - slot->base_gfn];
2979 if (*rmapp)
2980 printk(KERN_ERR "%s: (%s) shadow page has writable"
2981 " mappings: gfn %lx role %x\n",
2982 __func__, audit_msg, sp->gfn,
2983 sp->role.word);
2984 }
2985 }
2986
2987 static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg)
2988 {
2989 int olddbg = dbg;
2990
2991 dbg = 0;
2992 audit_msg = msg;
2993 audit_rmap(vcpu);
2994 audit_write_protection(vcpu);
2995 audit_mappings(vcpu);
2996 dbg = olddbg;
2997 }
2998
2999 #endif
Support for the Chinese Samsung S3C2440 based development boards