?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
KVM: MMU: Fix Wrong tlb flush order
[linux-2.6/x86.git] / drivers / kvm / mmu.c
blob49ffbd3da749b3f281ab4d7ee816d5512d917dcd
1 /*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * MMU support
8 *
9 * Copyright (C) 2006 Qumranet, Inc.
10 *
11 * Authors:
12 * Yaniv Kamay <yaniv@qumranet.com>
13 * Avi Kivity <avi@qumranet.com>
14 *
15 * This work is licensed under the terms of the GNU GPL, version 2. See
16 * the COPYING file in the top-level directory.
17 *
18 */
19 #include <linux/types.h>
20 #include <linux/string.h>
21 #include <asm/page.h>
22 #include <linux/mm.h>
23 #include <linux/highmem.h>
24 #include <linux/module.h>
25 #include <asm/cmpxchg.h>
26
27 #include "vmx.h"
28 #include "kvm.h"
29
30 #undef MMU_DEBUG
31
32 #undef AUDIT
33
34 #ifdef AUDIT
35 static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg);
36 #else
37 static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg) {}
38 #endif
39
40 #ifdef MMU_DEBUG
41
42 #define pgprintk(x...) do { if (dbg) printk(x); } while (0)
43 #define rmap_printk(x...) do { if (dbg) printk(x); } while (0)
44
45 #else
46
47 #define pgprintk(x...) do { } while (0)
48 #define rmap_printk(x...) do { } while (0)
49
50 #endif
51
52 #if defined(MMU_DEBUG) || defined(AUDIT)
53 static int dbg = 1;
54 #endif
55
56 #ifndef MMU_DEBUG
57 #define ASSERT(x) do { } while (0)
58 #else
59 #define ASSERT(x) \
60 if (!(x)) { \
61 printk(KERN_WARNING "assertion failed %s:%d: %s\n", \
62 __FILE__, __LINE__, #x); \
63 }
64 #endif
65
66 #define PT64_PT_BITS 9
67 #define PT64_ENT_PER_PAGE (1 << PT64_PT_BITS)
68 #define PT32_PT_BITS 10
69 #define PT32_ENT_PER_PAGE (1 << PT32_PT_BITS)
70
71 #define PT_WRITABLE_SHIFT 1
72
73 #define PT_PRESENT_MASK (1ULL << 0)
74 #define PT_WRITABLE_MASK (1ULL << PT_WRITABLE_SHIFT)
75 #define PT_USER_MASK (1ULL << 2)
76 #define PT_PWT_MASK (1ULL << 3)
77 #define PT_PCD_MASK (1ULL << 4)
78 #define PT_ACCESSED_MASK (1ULL << 5)
79 #define PT_DIRTY_MASK (1ULL << 6)
80 #define PT_PAGE_SIZE_MASK (1ULL << 7)
81 #define PT_PAT_MASK (1ULL << 7)
82 #define PT_GLOBAL_MASK (1ULL << 8)
83 #define PT64_NX_MASK (1ULL << 63)
84
85 #define PT_PAT_SHIFT 7
86 #define PT_DIR_PAT_SHIFT 12
87 #define PT_DIR_PAT_MASK (1ULL << PT_DIR_PAT_SHIFT)
88
89 #define PT32_DIR_PSE36_SIZE 4
90 #define PT32_DIR_PSE36_SHIFT 13
91 #define PT32_DIR_PSE36_MASK (((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT)
92
93
94 #define PT_FIRST_AVAIL_BITS_SHIFT 9
95 #define PT64_SECOND_AVAIL_BITS_SHIFT 52
96
97 #define PT_SHADOW_IO_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
98
99 #define VALID_PAGE(x) ((x) != INVALID_PAGE)
100
101 #define PT64_LEVEL_BITS 9
102
103 #define PT64_LEVEL_SHIFT(level) \
104 ( PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS )
105
106 #define PT64_LEVEL_MASK(level) \
107 (((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level))
108
109 #define PT64_INDEX(address, level)\
110 (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
111
112
113 #define PT32_LEVEL_BITS 10
114
115 #define PT32_LEVEL_SHIFT(level) \
116 ( PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS )
117
118 #define PT32_LEVEL_MASK(level) \
119 (((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level))
120
121 #define PT32_INDEX(address, level)\
122 (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
123
124
125 #define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
126 #define PT64_DIR_BASE_ADDR_MASK \
127 (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
128
129 #define PT32_BASE_ADDR_MASK PAGE_MASK
130 #define PT32_DIR_BASE_ADDR_MASK \
131 (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
132
133
134 #define PFERR_PRESENT_MASK (1U << 0)
135 #define PFERR_WRITE_MASK (1U << 1)
136 #define PFERR_USER_MASK (1U << 2)
137 #define PFERR_FETCH_MASK (1U << 4)
138
139 #define PT64_ROOT_LEVEL 4
140 #define PT32_ROOT_LEVEL 2
141 #define PT32E_ROOT_LEVEL 3
142
143 #define PT_DIRECTORY_LEVEL 2
144 #define PT_PAGE_TABLE_LEVEL 1
145
146 #define RMAP_EXT 4
147
148 struct kvm_rmap_desc {
149 u64 *shadow_ptes[RMAP_EXT];
150 struct kvm_rmap_desc *more;
151 };
152
153 static struct kmem_cache *pte_chain_cache;
154 static struct kmem_cache *rmap_desc_cache;
155 static struct kmem_cache *mmu_page_cache;
156 static struct kmem_cache *mmu_page_header_cache;
157
158 static int is_write_protection(struct kvm_vcpu *vcpu)
159 {
160 return vcpu->cr0 & CR0_WP_MASK;
161 }
162
163 static int is_cpuid_PSE36(void)
164 {
165 return 1;
166 }
167
168 static int is_nx(struct kvm_vcpu *vcpu)
169 {
170 return vcpu->shadow_efer & EFER_NX;
171 }
172
173 static int is_present_pte(unsigned long pte)
174 {
175 return pte & PT_PRESENT_MASK;
176 }
177
178 static int is_writeble_pte(unsigned long pte)
179 {
180 return pte & PT_WRITABLE_MASK;
181 }
182
183 static int is_io_pte(unsigned long pte)
184 {
185 return pte & PT_SHADOW_IO_MARK;
186 }
187
188 static int is_rmap_pte(u64 pte)
189 {
190 return (pte & (PT_WRITABLE_MASK | PT_PRESENT_MASK))
191 == (PT_WRITABLE_MASK | PT_PRESENT_MASK);
192 }
193
194 static void set_shadow_pte(u64 *sptep, u64 spte)
195 {
196 #ifdef CONFIG_X86_64
197 set_64bit((unsigned long *)sptep, spte);
198 #else
199 set_64bit((unsigned long long *)sptep, spte);
200 #endif
201 }
202
203 static int mmu_topup_memory_cache(struct kvm_mmu_memory_cache *cache,
204 struct kmem_cache *base_cache, int min,
205 gfp_t gfp_flags)
206 {
207 void *obj;
208
209 if (cache->nobjs >= min)
210 return 0;
211 while (cache->nobjs < ARRAY_SIZE(cache->objects)) {
212 obj = kmem_cache_zalloc(base_cache, gfp_flags);
213 if (!obj)
214 return -ENOMEM;
215 cache->objects[cache->nobjs++] = obj;
216 }
217 return 0;
218 }
219
220 static void mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc)
221 {
222 while (mc->nobjs)
223 kfree(mc->objects[--mc->nobjs]);
224 }
225
226 static int __mmu_topup_memory_caches(struct kvm_vcpu *vcpu, gfp_t gfp_flags)
227 {
228 int r;
229
230 r = mmu_topup_memory_cache(&vcpu->mmu_pte_chain_cache,
231 pte_chain_cache, 4, gfp_flags);
232 if (r)
233 goto out;
234 r = mmu_topup_memory_cache(&vcpu->mmu_rmap_desc_cache,
235 rmap_desc_cache, 1, gfp_flags);
236 if (r)
237 goto out;
238 r = mmu_topup_memory_cache(&vcpu->mmu_page_cache,
239 mmu_page_cache, 4, gfp_flags);
240 if (r)
241 goto out;
242 r = mmu_topup_memory_cache(&vcpu->mmu_page_header_cache,
243 mmu_page_header_cache, 4, gfp_flags);
244 out:
245 return r;
246 }
247
248 static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu)
249 {
250 int r;
251
252 r = __mmu_topup_memory_caches(vcpu, GFP_NOWAIT);
253 if (r < 0) {
254 spin_unlock(&vcpu->kvm->lock);
255 kvm_arch_ops->vcpu_put(vcpu);
256 r = __mmu_topup_memory_caches(vcpu, GFP_KERNEL);
257 kvm_arch_ops->vcpu_load(vcpu);
258 spin_lock(&vcpu->kvm->lock);
259 }
260 return r;
261 }
262
263 static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
264 {
265 mmu_free_memory_cache(&vcpu->mmu_pte_chain_cache);
266 mmu_free_memory_cache(&vcpu->mmu_rmap_desc_cache);
267 mmu_free_memory_cache(&vcpu->mmu_page_cache);
268 mmu_free_memory_cache(&vcpu->mmu_page_header_cache);
269 }
270
271 static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc,
272 size_t size)
273 {
274 void *p;
275
276 BUG_ON(!mc->nobjs);
277 p = mc->objects[--mc->nobjs];
278 memset(p, 0, size);
279 return p;
280 }
281
282 static void mmu_memory_cache_free(struct kvm_mmu_memory_cache *mc, void *obj)
283 {
284 if (mc->nobjs < KVM_NR_MEM_OBJS)
285 mc->objects[mc->nobjs++] = obj;
286 else
287 kfree(obj);
288 }
289
290 static struct kvm_pte_chain *mmu_alloc_pte_chain(struct kvm_vcpu *vcpu)
291 {
292 return mmu_memory_cache_alloc(&vcpu->mmu_pte_chain_cache,
293 sizeof(struct kvm_pte_chain));
294 }
295
296 static void mmu_free_pte_chain(struct kvm_vcpu *vcpu,
297 struct kvm_pte_chain *pc)
298 {
299 mmu_memory_cache_free(&vcpu->mmu_pte_chain_cache, pc);
300 }
301
302 static struct kvm_rmap_desc *mmu_alloc_rmap_desc(struct kvm_vcpu *vcpu)
303 {
304 return mmu_memory_cache_alloc(&vcpu->mmu_rmap_desc_cache,
305 sizeof(struct kvm_rmap_desc));
306 }
307
308 static void mmu_free_rmap_desc(struct kvm_vcpu *vcpu,
309 struct kvm_rmap_desc *rd)
310 {
311 mmu_memory_cache_free(&vcpu->mmu_rmap_desc_cache, rd);
312 }
313
314 /*
315 * Reverse mapping data structures:
316 *
317 * If page->private bit zero is zero, then page->private points to the
318 * shadow page table entry that points to page_address(page).
319 *
320 * If page->private bit zero is one, (then page->private & ~1) points
321 * to a struct kvm_rmap_desc containing more mappings.
322 */
323 static void rmap_add(struct kvm_vcpu *vcpu, u64 *spte)
324 {
325 struct page *page;
326 struct kvm_rmap_desc *desc;
327 int i;
328
329 if (!is_rmap_pte(*spte))
330 return;
331 page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT);
332 if (!page_private(page)) {
333 rmap_printk("rmap_add: %p %llx 0->1\n", spte, *spte);
334 set_page_private(page,(unsigned long)spte);
335 } else if (!(page_private(page) & 1)) {
336 rmap_printk("rmap_add: %p %llx 1->many\n", spte, *spte);
337 desc = mmu_alloc_rmap_desc(vcpu);
338 desc->shadow_ptes[0] = (u64 *)page_private(page);
339 desc->shadow_ptes[1] = spte;
340 set_page_private(page,(unsigned long)desc | 1);
341 } else {
342 rmap_printk("rmap_add: %p %llx many->many\n", spte, *spte);
343 desc = (struct kvm_rmap_desc *)(page_private(page) & ~1ul);
344 while (desc->shadow_ptes[RMAP_EXT-1] && desc->more)
345 desc = desc->more;
346 if (desc->shadow_ptes[RMAP_EXT-1]) {
347 desc->more = mmu_alloc_rmap_desc(vcpu);
348 desc = desc->more;
349 }
350 for (i = 0; desc->shadow_ptes[i]; ++i)
351 ;
352 desc->shadow_ptes[i] = spte;
353 }
354 }
355
356 static void rmap_desc_remove_entry(struct kvm_vcpu *vcpu,
357 struct page *page,
358 struct kvm_rmap_desc *desc,
359 int i,
360 struct kvm_rmap_desc *prev_desc)
361 {
362 int j;
363
364 for (j = RMAP_EXT - 1; !desc->shadow_ptes[j] && j > i; --j)
365 ;
366 desc->shadow_ptes[i] = desc->shadow_ptes[j];
367 desc->shadow_ptes[j] = NULL;
368 if (j != 0)
369 return;
370 if (!prev_desc && !desc->more)
371 set_page_private(page,(unsigned long)desc->shadow_ptes[0]);
372 else
373 if (prev_desc)
374 prev_desc->more = desc->more;
375 else
376 set_page_private(page,(unsigned long)desc->more | 1);
377 mmu_free_rmap_desc(vcpu, desc);
378 }
379
380 static void rmap_remove(struct kvm_vcpu *vcpu, u64 *spte)
381 {
382 struct page *page;
383 struct kvm_rmap_desc *desc;
384 struct kvm_rmap_desc *prev_desc;
385 int i;
386
387 if (!is_rmap_pte(*spte))
388 return;
389 page = pfn_to_page((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT);
390 if (!page_private(page)) {
391 printk(KERN_ERR "rmap_remove: %p %llx 0->BUG\n", spte, *spte);
392 BUG();
393 } else if (!(page_private(page) & 1)) {
394 rmap_printk("rmap_remove: %p %llx 1->0\n", spte, *spte);
395 if ((u64 *)page_private(page) != spte) {
396 printk(KERN_ERR "rmap_remove: %p %llx 1->BUG\n",
397 spte, *spte);
398 BUG();
399 }
400 set_page_private(page,0);
401 } else {
402 rmap_printk("rmap_remove: %p %llx many->many\n", spte, *spte);
403 desc = (struct kvm_rmap_desc *)(page_private(page) & ~1ul);
404 prev_desc = NULL;
405 while (desc) {
406 for (i = 0; i < RMAP_EXT && desc->shadow_ptes[i]; ++i)
407 if (desc->shadow_ptes[i] == spte) {
408 rmap_desc_remove_entry(vcpu, page,
409 desc, i,
410 prev_desc);
411 return;
412 }
413 prev_desc = desc;
414 desc = desc->more;
415 }
416 BUG();
417 }
418 }
419
420 static void rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn)
421 {
422 struct kvm *kvm = vcpu->kvm;
423 struct page *page;
424 struct kvm_rmap_desc *desc;
425 u64 *spte;
426
427 page = gfn_to_page(kvm, gfn);
428 BUG_ON(!page);
429
430 while (page_private(page)) {
431 if (!(page_private(page) & 1))
432 spte = (u64 *)page_private(page);
433 else {
434 desc = (struct kvm_rmap_desc *)(page_private(page) & ~1ul);
435 spte = desc->shadow_ptes[0];
436 }
437 BUG_ON(!spte);
438 BUG_ON((*spte & PT64_BASE_ADDR_MASK) >> PAGE_SHIFT
439 != page_to_pfn(page));
440 BUG_ON(!(*spte & PT_PRESENT_MASK));
441 BUG_ON(!(*spte & PT_WRITABLE_MASK));
442 rmap_printk("rmap_write_protect: spte %p %llx\n", spte, *spte);
443 rmap_remove(vcpu, spte);
444 set_shadow_pte(spte, *spte & ~PT_WRITABLE_MASK);
445 kvm_flush_remote_tlbs(vcpu->kvm);
446 }
447 }
448
449 #ifdef MMU_DEBUG
450 static int is_empty_shadow_page(u64 *spt)
451 {
452 u64 *pos;
453 u64 *end;
454
455 for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
456 if (*pos != 0) {
457 printk(KERN_ERR "%s: %p %llx\n", __FUNCTION__,
458 pos, *pos);
459 return 0;
460 }
461 return 1;
462 }
463 #endif
464
465 static void kvm_mmu_free_page(struct kvm_vcpu *vcpu,
466 struct kvm_mmu_page *page_head)
467 {
468 ASSERT(is_empty_shadow_page(page_head->spt));
469 list_del(&page_head->link);
470 mmu_memory_cache_free(&vcpu->mmu_page_cache, page_head->spt);
471 mmu_memory_cache_free(&vcpu->mmu_page_header_cache, page_head);
472 ++vcpu->kvm->n_free_mmu_pages;
473 }
474
475 static unsigned kvm_page_table_hashfn(gfn_t gfn)
476 {
477 return gfn;
478 }
479
480 static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu,
481 u64 *parent_pte)
482 {
483 struct kvm_mmu_page *page;
484
485 if (!vcpu->kvm->n_free_mmu_pages)
486 return NULL;
487
488 page = mmu_memory_cache_alloc(&vcpu->mmu_page_header_cache,
489 sizeof *page);
490 page->spt = mmu_memory_cache_alloc(&vcpu->mmu_page_cache, PAGE_SIZE);
491 set_page_private(virt_to_page(page->spt), (unsigned long)page);
492 list_add(&page->link, &vcpu->kvm->active_mmu_pages);
493 ASSERT(is_empty_shadow_page(page->spt));
494 page->slot_bitmap = 0;
495 page->multimapped = 0;
496 page->parent_pte = parent_pte;
497 --vcpu->kvm->n_free_mmu_pages;
498 return page;
499 }
500
501 static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
502 struct kvm_mmu_page *page, u64 *parent_pte)
503 {
504 struct kvm_pte_chain *pte_chain;
505 struct hlist_node *node;
506 int i;
507
508 if (!parent_pte)
509 return;
510 if (!page->multimapped) {
511 u64 *old = page->parent_pte;
512
513 if (!old) {
514 page->parent_pte = parent_pte;
515 return;
516 }
517 page->multimapped = 1;
518 pte_chain = mmu_alloc_pte_chain(vcpu);
519 INIT_HLIST_HEAD(&page->parent_ptes);
520 hlist_add_head(&pte_chain->link, &page->parent_ptes);
521 pte_chain->parent_ptes[0] = old;
522 }
523 hlist_for_each_entry(pte_chain, node, &page->parent_ptes, link) {
524 if (pte_chain->parent_ptes[NR_PTE_CHAIN_ENTRIES-1])
525 continue;
526 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i)
527 if (!pte_chain->parent_ptes[i]) {
528 pte_chain->parent_ptes[i] = parent_pte;
529 return;
530 }
531 }
532 pte_chain = mmu_alloc_pte_chain(vcpu);
533 BUG_ON(!pte_chain);
534 hlist_add_head(&pte_chain->link, &page->parent_ptes);
535 pte_chain->parent_ptes[0] = parent_pte;
536 }
537
538 static void mmu_page_remove_parent_pte(struct kvm_vcpu *vcpu,
539 struct kvm_mmu_page *page,
540 u64 *parent_pte)
541 {
542 struct kvm_pte_chain *pte_chain;
543 struct hlist_node *node;
544 int i;
545
546 if (!page->multimapped) {
547 BUG_ON(page->parent_pte != parent_pte);
548 page->parent_pte = NULL;
549 return;
550 }
551 hlist_for_each_entry(pte_chain, node, &page->parent_ptes, link)
552 for (i = 0; i < NR_PTE_CHAIN_ENTRIES; ++i) {
553 if (!pte_chain->parent_ptes[i])
554 break;
555 if (pte_chain->parent_ptes[i] != parent_pte)
556 continue;
557 while (i + 1 < NR_PTE_CHAIN_ENTRIES
558 && pte_chain->parent_ptes[i + 1]) {
559 pte_chain->parent_ptes[i]
560 = pte_chain->parent_ptes[i + 1];
561 ++i;
562 }
563 pte_chain->parent_ptes[i] = NULL;
564 if (i == 0) {
565 hlist_del(&pte_chain->link);
566 mmu_free_pte_chain(vcpu, pte_chain);
567 if (hlist_empty(&page->parent_ptes)) {
568 page->multimapped = 0;
569 page->parent_pte = NULL;
570 }
571 }
572 return;
573 }
574 BUG();
575 }
576
577 static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm_vcpu *vcpu,
578 gfn_t gfn)
579 {
580 unsigned index;
581 struct hlist_head *bucket;
582 struct kvm_mmu_page *page;
583 struct hlist_node *node;
584
585 pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn);
586 index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
587 bucket = &vcpu->kvm->mmu_page_hash[index];
588 hlist_for_each_entry(page, node, bucket, hash_link)
589 if (page->gfn == gfn && !page->role.metaphysical) {
590 pgprintk("%s: found role %x\n",
591 __FUNCTION__, page->role.word);
592 return page;
593 }
594 return NULL;
595 }
596
597 static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
598 gfn_t gfn,
599 gva_t gaddr,
600 unsigned level,
601 int metaphysical,
602 unsigned hugepage_access,
603 u64 *parent_pte)
604 {
605 union kvm_mmu_page_role role;
606 unsigned index;
607 unsigned quadrant;
608 struct hlist_head *bucket;
609 struct kvm_mmu_page *page;
610 struct hlist_node *node;
611
612 role.word = 0;
613 role.glevels = vcpu->mmu.root_level;
614 role.level = level;
615 role.metaphysical = metaphysical;
616 role.hugepage_access = hugepage_access;
617 if (vcpu->mmu.root_level <= PT32_ROOT_LEVEL) {
618 quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
619 quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
620 role.quadrant = quadrant;
621 }
622 pgprintk("%s: looking gfn %lx role %x\n", __FUNCTION__,
623 gfn, role.word);
624 index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
625 bucket = &vcpu->kvm->mmu_page_hash[index];
626 hlist_for_each_entry(page, node, bucket, hash_link)
627 if (page->gfn == gfn && page->role.word == role.word) {
628 mmu_page_add_parent_pte(vcpu, page, parent_pte);
629 pgprintk("%s: found\n", __FUNCTION__);
630 return page;
631 }
632 page = kvm_mmu_alloc_page(vcpu, parent_pte);
633 if (!page)
634 return page;
635 pgprintk("%s: adding gfn %lx role %x\n", __FUNCTION__, gfn, role.word);
636 page->gfn = gfn;
637 page->role = role;
638 hlist_add_head(&page->hash_link, bucket);
639 if (!metaphysical)
640 rmap_write_protect(vcpu, gfn);
641 return page;
642 }
643
644 static void kvm_mmu_page_unlink_children(struct kvm_vcpu *vcpu,
645 struct kvm_mmu_page *page)
646 {
647 unsigned i;
648 u64 *pt;
649 u64 ent;
650
651 pt = page->spt;
652
653 if (page->role.level == PT_PAGE_TABLE_LEVEL) {
654 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
655 if (pt[i] & PT_PRESENT_MASK)
656 rmap_remove(vcpu, &pt[i]);
657 pt[i] = 0;
658 }
659 kvm_flush_remote_tlbs(vcpu->kvm);
660 return;
661 }
662
663 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
664 ent = pt[i];
665
666 pt[i] = 0;
667 if (!(ent & PT_PRESENT_MASK))
668 continue;
669 ent &= PT64_BASE_ADDR_MASK;
670 mmu_page_remove_parent_pte(vcpu, page_header(ent), &pt[i]);
671 }
672 kvm_flush_remote_tlbs(vcpu->kvm);
673 }
674
675 static void kvm_mmu_put_page(struct kvm_vcpu *vcpu,
676 struct kvm_mmu_page *page,
677 u64 *parent_pte)
678 {
679 mmu_page_remove_parent_pte(vcpu, page, parent_pte);
680 }
681
682 static void kvm_mmu_zap_page(struct kvm_vcpu *vcpu,
683 struct kvm_mmu_page *page)
684 {
685 u64 *parent_pte;
686
687 while (page->multimapped || page->parent_pte) {
688 if (!page->multimapped)
689 parent_pte = page->parent_pte;
690 else {
691 struct kvm_pte_chain *chain;
692
693 chain = container_of(page->parent_ptes.first,
694 struct kvm_pte_chain, link);
695 parent_pte = chain->parent_ptes[0];
696 }
697 BUG_ON(!parent_pte);
698 kvm_mmu_put_page(vcpu, page, parent_pte);
699 set_shadow_pte(parent_pte, 0);
700 }
701 kvm_mmu_page_unlink_children(vcpu, page);
702 if (!page->root_count) {
703 hlist_del(&page->hash_link);
704 kvm_mmu_free_page(vcpu, page);
705 } else
706 list_move(&page->link, &vcpu->kvm->active_mmu_pages);
707 }
708
709 static int kvm_mmu_unprotect_page(struct kvm_vcpu *vcpu, gfn_t gfn)
710 {
711 unsigned index;
712 struct hlist_head *bucket;
713 struct kvm_mmu_page *page;
714 struct hlist_node *node, *n;
715 int r;
716
717 pgprintk("%s: looking for gfn %lx\n", __FUNCTION__, gfn);
718 r = 0;
719 index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
720 bucket = &vcpu->kvm->mmu_page_hash[index];
721 hlist_for_each_entry_safe(page, node, n, bucket, hash_link)
722 if (page->gfn == gfn && !page->role.metaphysical) {
723 pgprintk("%s: gfn %lx role %x\n", __FUNCTION__, gfn,
724 page->role.word);
725 kvm_mmu_zap_page(vcpu, page);
726 r = 1;
727 }
728 return r;
729 }
730
731 static void mmu_unshadow(struct kvm_vcpu *vcpu, gfn_t gfn)
732 {
733 struct kvm_mmu_page *page;
734
735 while ((page = kvm_mmu_lookup_page(vcpu, gfn)) != NULL) {
736 pgprintk("%s: zap %lx %x\n",
737 __FUNCTION__, gfn, page->role.word);
738 kvm_mmu_zap_page(vcpu, page);
739 }
740 }
741
742 static void page_header_update_slot(struct kvm *kvm, void *pte, gpa_t gpa)
743 {
744 int slot = memslot_id(kvm, gfn_to_memslot(kvm, gpa >> PAGE_SHIFT));
745 struct kvm_mmu_page *page_head = page_header(__pa(pte));
746
747 __set_bit(slot, &page_head->slot_bitmap);
748 }
749
750 hpa_t safe_gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa)
751 {
752 hpa_t hpa = gpa_to_hpa(vcpu, gpa);
753
754 return is_error_hpa(hpa) ? bad_page_address | (gpa & ~PAGE_MASK): hpa;
755 }
756
757 hpa_t gpa_to_hpa(struct kvm_vcpu *vcpu, gpa_t gpa)
758 {
759 struct page *page;
760
761 ASSERT((gpa & HPA_ERR_MASK) == 0);
762 page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
763 if (!page)
764 return gpa | HPA_ERR_MASK;
765 return ((hpa_t)page_to_pfn(page) << PAGE_SHIFT)
766 | (gpa & (PAGE_SIZE-1));
767 }
768
769 hpa_t gva_to_hpa(struct kvm_vcpu *vcpu, gva_t gva)
770 {
771 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);
772
773 if (gpa == UNMAPPED_GVA)
774 return UNMAPPED_GVA;
775 return gpa_to_hpa(vcpu, gpa);
776 }
777
778 struct page *gva_to_page(struct kvm_vcpu *vcpu, gva_t gva)
779 {
780 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);
781
782 if (gpa == UNMAPPED_GVA)
783 return NULL;
784 return pfn_to_page(gpa_to_hpa(vcpu, gpa) >> PAGE_SHIFT);
785 }
786
787 static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
788 {
789 }
790
791 static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, hpa_t p)
792 {
793 int level = PT32E_ROOT_LEVEL;
794 hpa_t table_addr = vcpu->mmu.root_hpa;
795
796 for (; ; level--) {
797 u32 index = PT64_INDEX(v, level);
798 u64 *table;
799 u64 pte;
800
801 ASSERT(VALID_PAGE(table_addr));
802 table = __va(table_addr);
803
804 if (level == 1) {
805 pte = table[index];
806 if (is_present_pte(pte) && is_writeble_pte(pte))
807 return 0;
808 mark_page_dirty(vcpu->kvm, v >> PAGE_SHIFT);
809 page_header_update_slot(vcpu->kvm, table, v);
810 table[index] = p | PT_PRESENT_MASK | PT_WRITABLE_MASK |
811 PT_USER_MASK;
812 rmap_add(vcpu, &table[index]);
813 return 0;
814 }
815
816 if (table[index] == 0) {
817 struct kvm_mmu_page *new_table;
818 gfn_t pseudo_gfn;
819
820 pseudo_gfn = (v & PT64_DIR_BASE_ADDR_MASK)
821 >> PAGE_SHIFT;
822 new_table = kvm_mmu_get_page(vcpu, pseudo_gfn,
823 v, level - 1,
824 1, 0, &table[index]);
825 if (!new_table) {
826 pgprintk("nonpaging_map: ENOMEM\n");
827 return -ENOMEM;
828 }
829
830 table[index] = __pa(new_table->spt) | PT_PRESENT_MASK
831 | PT_WRITABLE_MASK | PT_USER_MASK;
832 }
833 table_addr = table[index] & PT64_BASE_ADDR_MASK;
834 }
835 }
836
837 static void mmu_free_roots(struct kvm_vcpu *vcpu)
838 {
839 int i;
840 struct kvm_mmu_page *page;
841
842 if (!VALID_PAGE(vcpu->mmu.root_hpa))
843 return;
844 #ifdef CONFIG_X86_64
845 if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) {
846 hpa_t root = vcpu->mmu.root_hpa;
847
848 page = page_header(root);
849 --page->root_count;
850 vcpu->mmu.root_hpa = INVALID_PAGE;
851 return;
852 }
853 #endif
854 for (i = 0; i < 4; ++i) {
855 hpa_t root = vcpu->mmu.pae_root[i];
856
857 if (root) {
858 root &= PT64_BASE_ADDR_MASK;
859 page = page_header(root);
860 --page->root_count;
861 }
862 vcpu->mmu.pae_root[i] = INVALID_PAGE;
863 }
864 vcpu->mmu.root_hpa = INVALID_PAGE;
865 }
866
867 static void mmu_alloc_roots(struct kvm_vcpu *vcpu)
868 {
869 int i;
870 gfn_t root_gfn;
871 struct kvm_mmu_page *page;
872
873 root_gfn = vcpu->cr3 >> PAGE_SHIFT;
874
875 #ifdef CONFIG_X86_64
876 if (vcpu->mmu.shadow_root_level == PT64_ROOT_LEVEL) {
877 hpa_t root = vcpu->mmu.root_hpa;
878
879 ASSERT(!VALID_PAGE(root));
880 page = kvm_mmu_get_page(vcpu, root_gfn, 0,
881 PT64_ROOT_LEVEL, 0, 0, NULL);
882 root = __pa(page->spt);
883 ++page->root_count;
884 vcpu->mmu.root_hpa = root;
885 return;
886 }
887 #endif
888 for (i = 0; i < 4; ++i) {
889 hpa_t root = vcpu->mmu.pae_root[i];
890
891 ASSERT(!VALID_PAGE(root));
892 if (vcpu->mmu.root_level == PT32E_ROOT_LEVEL) {
893 if (!is_present_pte(vcpu->pdptrs[i])) {
894 vcpu->mmu.pae_root[i] = 0;
895 continue;
896 }
897 root_gfn = vcpu->pdptrs[i] >> PAGE_SHIFT;
898 } else if (vcpu->mmu.root_level == 0)
899 root_gfn = 0;
900 page = kvm_mmu_get_page(vcpu, root_gfn, i << 30,
901 PT32_ROOT_LEVEL, !is_paging(vcpu),
902 0, NULL);
903 root = __pa(page->spt);
904 ++page->root_count;
905 vcpu->mmu.pae_root[i] = root | PT_PRESENT_MASK;
906 }
907 vcpu->mmu.root_hpa = __pa(vcpu->mmu.pae_root);
908 }
909
910 static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, gva_t vaddr)
911 {
912 return vaddr;
913 }
914
915 static int nonpaging_page_fault(struct kvm_vcpu *vcpu, gva_t gva,
916 u32 error_code)
917 {
918 gpa_t addr = gva;
919 hpa_t paddr;
920 int r;
921
922 r = mmu_topup_memory_caches(vcpu);
923 if (r)
924 return r;
925
926 ASSERT(vcpu);
927 ASSERT(VALID_PAGE(vcpu->mmu.root_hpa));
928
929
930 paddr = gpa_to_hpa(vcpu , addr & PT64_BASE_ADDR_MASK);
931
932 if (is_error_hpa(paddr))
933 return 1;
934
935 return nonpaging_map(vcpu, addr & PAGE_MASK, paddr);
936 }
937
938 static void nonpaging_free(struct kvm_vcpu *vcpu)
939 {
940 mmu_free_roots(vcpu);
941 }
942
943 static int nonpaging_init_context(struct kvm_vcpu *vcpu)
944 {
945 struct kvm_mmu *context = &vcpu->mmu;
946
947 context->new_cr3 = nonpaging_new_cr3;
948 context->page_fault = nonpaging_page_fault;
949 context->gva_to_gpa = nonpaging_gva_to_gpa;
950 context->free = nonpaging_free;
951 context->root_level = 0;
952 context->shadow_root_level = PT32E_ROOT_LEVEL;
953 context->root_hpa = INVALID_PAGE;
954 return 0;
955 }
956
957 static void kvm_mmu_flush_tlb(struct kvm_vcpu *vcpu)
958 {
959 ++vcpu->stat.tlb_flush;
960 kvm_arch_ops->tlb_flush(vcpu);
961 }
962
963 static void paging_new_cr3(struct kvm_vcpu *vcpu)
964 {
965 pgprintk("%s: cr3 %lx\n", __FUNCTION__, vcpu->cr3);
966 mmu_free_roots(vcpu);
967 }
968
969 static void inject_page_fault(struct kvm_vcpu *vcpu,
970 u64 addr,
971 u32 err_code)
972 {
973 kvm_arch_ops->inject_page_fault(vcpu, addr, err_code);
974 }
975
976 static void paging_free(struct kvm_vcpu *vcpu)
977 {
978 nonpaging_free(vcpu);
979 }
980
981 #define PTTYPE 64
982 #include "paging_tmpl.h"
983 #undef PTTYPE
984
985 #define PTTYPE 32
986 #include "paging_tmpl.h"
987 #undef PTTYPE
988
989 static int paging64_init_context_common(struct kvm_vcpu *vcpu, int level)
990 {
991 struct kvm_mmu *context = &vcpu->mmu;
992
993 ASSERT(is_pae(vcpu));
994 context->new_cr3 = paging_new_cr3;
995 context->page_fault = paging64_page_fault;
996 context->gva_to_gpa = paging64_gva_to_gpa;
997 context->free = paging_free;
998 context->root_level = level;
999 context->shadow_root_level = level;
1000 context->root_hpa = INVALID_PAGE;
1001 return 0;
1002 }
1003
1004 static int paging64_init_context(struct kvm_vcpu *vcpu)
1005 {
1006 return paging64_init_context_common(vcpu, PT64_ROOT_LEVEL);
1007 }
1008
1009 static int paging32_init_context(struct kvm_vcpu *vcpu)
1010 {
1011 struct kvm_mmu *context = &vcpu->mmu;
1012
1013 context->new_cr3 = paging_new_cr3;
1014 context->page_fault = paging32_page_fault;
1015 context->gva_to_gpa = paging32_gva_to_gpa;
1016 context->free = paging_free;
1017 context->root_level = PT32_ROOT_LEVEL;
1018 context->shadow_root_level = PT32E_ROOT_LEVEL;
1019 context->root_hpa = INVALID_PAGE;
1020 return 0;
1021 }
1022
1023 static int paging32E_init_context(struct kvm_vcpu *vcpu)
1024 {
1025 return paging64_init_context_common(vcpu, PT32E_ROOT_LEVEL);
1026 }
1027
1028 static int init_kvm_mmu(struct kvm_vcpu *vcpu)
1029 {
1030 ASSERT(vcpu);
1031 ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
1032
1033 if (!is_paging(vcpu))
1034 return nonpaging_init_context(vcpu);
1035 else if (is_long_mode(vcpu))
1036 return paging64_init_context(vcpu);
1037 else if (is_pae(vcpu))
1038 return paging32E_init_context(vcpu);
1039 else
1040 return paging32_init_context(vcpu);
1041 }
1042
1043 static void destroy_kvm_mmu(struct kvm_vcpu *vcpu)
1044 {
1045 ASSERT(vcpu);
1046 if (VALID_PAGE(vcpu->mmu.root_hpa)) {
1047 vcpu->mmu.free(vcpu);
1048 vcpu->mmu.root_hpa = INVALID_PAGE;
1049 }
1050 }
1051
1052 int kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
1053 {
1054 destroy_kvm_mmu(vcpu);
1055 return init_kvm_mmu(vcpu);
1056 }
1057
1058 int kvm_mmu_load(struct kvm_vcpu *vcpu)
1059 {
1060 int r;
1061
1062 spin_lock(&vcpu->kvm->lock);
1063 r = mmu_topup_memory_caches(vcpu);
1064 if (r)
1065 goto out;
1066 mmu_alloc_roots(vcpu);
1067 kvm_arch_ops->set_cr3(vcpu, vcpu->mmu.root_hpa);
1068 kvm_mmu_flush_tlb(vcpu);
1069 out:
1070 spin_unlock(&vcpu->kvm->lock);
1071 return r;
1072 }
1073 EXPORT_SYMBOL_GPL(kvm_mmu_load);
1074
1075 void kvm_mmu_unload(struct kvm_vcpu *vcpu)
1076 {
1077 mmu_free_roots(vcpu);
1078 }
1079
1080 static void mmu_pte_write_zap_pte(struct kvm_vcpu *vcpu,
1081 struct kvm_mmu_page *page,
1082 u64 *spte)
1083 {
1084 u64 pte;
1085 struct kvm_mmu_page *child;
1086
1087 pte = *spte;
1088 if (is_present_pte(pte)) {
1089 if (page->role.level == PT_PAGE_TABLE_LEVEL)
1090 rmap_remove(vcpu, spte);
1091 else {
1092 child = page_header(pte & PT64_BASE_ADDR_MASK);
1093 mmu_page_remove_parent_pte(vcpu, child, spte);
1094 }
1095 }
1096 *spte = 0;
1097 kvm_flush_remote_tlbs(vcpu->kvm);
1098 }
1099
1100 static void mmu_pte_write_new_pte(struct kvm_vcpu *vcpu,
1101 struct kvm_mmu_page *page,
1102 u64 *spte,
1103 const void *new, int bytes)
1104 {
1105 if (page->role.level != PT_PAGE_TABLE_LEVEL)
1106 return;
1107
1108 if (page->role.glevels == PT32_ROOT_LEVEL)
1109 paging32_update_pte(vcpu, page, spte, new, bytes);
1110 else
1111 paging64_update_pte(vcpu, page, spte, new, bytes);
1112 }
1113
1114 void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
1115 const u8 *old, const u8 *new, int bytes)
1116 {
1117 gfn_t gfn = gpa >> PAGE_SHIFT;
1118 struct kvm_mmu_page *page;
1119 struct hlist_node *node, *n;
1120 struct hlist_head *bucket;
1121 unsigned index;
1122 u64 *spte;
1123 unsigned offset = offset_in_page(gpa);
1124 unsigned pte_size;
1125 unsigned page_offset;
1126 unsigned misaligned;
1127 unsigned quadrant;
1128 int level;
1129 int flooded = 0;
1130 int npte;
1131
1132 pgprintk("%s: gpa %llx bytes %d\n", __FUNCTION__, gpa, bytes);
1133 if (gfn == vcpu->last_pt_write_gfn) {
1134 ++vcpu->last_pt_write_count;
1135 if (vcpu->last_pt_write_count >= 3)
1136 flooded = 1;
1137 } else {
1138 vcpu->last_pt_write_gfn = gfn;
1139 vcpu->last_pt_write_count = 1;
1140 }
1141 index = kvm_page_table_hashfn(gfn) % KVM_NUM_MMU_PAGES;
1142 bucket = &vcpu->kvm->mmu_page_hash[index];
1143 hlist_for_each_entry_safe(page, node, n, bucket, hash_link) {
1144 if (page->gfn != gfn || page->role.metaphysical)
1145 continue;
1146 pte_size = page->role.glevels == PT32_ROOT_LEVEL ? 4 : 8;
1147 misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
1148 misaligned |= bytes < 4;
1149 if (misaligned || flooded) {
1150 /*
1151 * Misaligned accesses are too much trouble to fix
1152 * up; also, they usually indicate a page is not used
1153 * as a page table.
1154 *
1155 * If we're seeing too many writes to a page,
1156 * it may no longer be a page table, or we may be
1157 * forking, in which case it is better to unmap the
1158 * page.
1159 */
1160 pgprintk("misaligned: gpa %llx bytes %d role %x\n",
1161 gpa, bytes, page->role.word);
1162 kvm_mmu_zap_page(vcpu, page);
1163 continue;
1164 }
1165 page_offset = offset;
1166 level = page->role.level;
1167 npte = 1;
1168 if (page->role.glevels == PT32_ROOT_LEVEL) {
1169 page_offset <<= 1; /* 32->64 */
1170 /*
1171 * A 32-bit pde maps 4MB while the shadow pdes map
1172 * only 2MB. So we need to double the offset again
1173 * and zap two pdes instead of one.
1174 */
1175 if (level == PT32_ROOT_LEVEL) {
1176 page_offset &= ~7; /* kill rounding error */
1177 page_offset <<= 1;
1178 npte = 2;
1179 }
1180 quadrant = page_offset >> PAGE_SHIFT;
1181 page_offset &= ~PAGE_MASK;
1182 if (quadrant != page->role.quadrant)
1183 continue;
1184 }
1185 spte = &page->spt[page_offset / sizeof(*spte)];
1186 while (npte--) {
1187 mmu_pte_write_zap_pte(vcpu, page, spte);
1188 mmu_pte_write_new_pte(vcpu, page, spte, new, bytes);
1189 ++spte;
1190 }
1191 }
1192 }
1193
1194 int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
1195 {
1196 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);
1197
1198 return kvm_mmu_unprotect_page(vcpu, gpa >> PAGE_SHIFT);
1199 }
1200
1201 void kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu)
1202 {
1203 while (vcpu->kvm->n_free_mmu_pages < KVM_REFILL_PAGES) {
1204 struct kvm_mmu_page *page;
1205
1206 page = container_of(vcpu->kvm->active_mmu_pages.prev,
1207 struct kvm_mmu_page, link);
1208 kvm_mmu_zap_page(vcpu, page);
1209 }
1210 }
1211 EXPORT_SYMBOL_GPL(kvm_mmu_free_some_pages);
1212
1213 static void free_mmu_pages(struct kvm_vcpu *vcpu)
1214 {
1215 struct kvm_mmu_page *page;
1216
1217 while (!list_empty(&vcpu->kvm->active_mmu_pages)) {
1218 page = container_of(vcpu->kvm->active_mmu_pages.next,
1219 struct kvm_mmu_page, link);
1220 kvm_mmu_zap_page(vcpu, page);
1221 }
1222 free_page((unsigned long)vcpu->mmu.pae_root);
1223 }
1224
1225 static int alloc_mmu_pages(struct kvm_vcpu *vcpu)
1226 {
1227 struct page *page;
1228 int i;
1229
1230 ASSERT(vcpu);
1231
1232 vcpu->kvm->n_free_mmu_pages = KVM_NUM_MMU_PAGES;
1233
1234 /*
1235 * When emulating 32-bit mode, cr3 is only 32 bits even on x86_64.
1236 * Therefore we need to allocate shadow page tables in the first
1237 * 4GB of memory, which happens to fit the DMA32 zone.
1238 */
1239 page = alloc_page(GFP_KERNEL | __GFP_DMA32);
1240 if (!page)
1241 goto error_1;
1242 vcpu->mmu.pae_root = page_address(page);
1243 for (i = 0; i < 4; ++i)
1244 vcpu->mmu.pae_root[i] = INVALID_PAGE;
1245
1246 return 0;
1247
1248 error_1:
1249 free_mmu_pages(vcpu);
1250 return -ENOMEM;
1251 }
1252
1253 int kvm_mmu_create(struct kvm_vcpu *vcpu)
1254 {
1255 ASSERT(vcpu);
1256 ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
1257
1258 return alloc_mmu_pages(vcpu);
1259 }
1260
1261 int kvm_mmu_setup(struct kvm_vcpu *vcpu)
1262 {
1263 ASSERT(vcpu);
1264 ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
1265
1266 return init_kvm_mmu(vcpu);
1267 }
1268
1269 void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
1270 {
1271 ASSERT(vcpu);
1272
1273 destroy_kvm_mmu(vcpu);
1274 free_mmu_pages(vcpu);
1275 mmu_free_memory_caches(vcpu);
1276 }
1277
1278 void kvm_mmu_slot_remove_write_access(struct kvm_vcpu *vcpu, int slot)
1279 {
1280 struct kvm *kvm = vcpu->kvm;
1281 struct kvm_mmu_page *page;
1282
1283 list_for_each_entry(page, &kvm->active_mmu_pages, link) {
1284 int i;
1285 u64 *pt;
1286
1287 if (!test_bit(slot, &page->slot_bitmap))
1288 continue;
1289
1290 pt = page->spt;
1291 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
1292 /* avoid RMW */
1293 if (pt[i] & PT_WRITABLE_MASK) {
1294 rmap_remove(vcpu, &pt[i]);
1295 pt[i] &= ~PT_WRITABLE_MASK;
1296 }
1297 }
1298 }
1299
1300 void kvm_mmu_zap_all(struct kvm_vcpu *vcpu)
1301 {
1302 destroy_kvm_mmu(vcpu);
1303
1304 while (!list_empty(&vcpu->kvm->active_mmu_pages)) {
1305 struct kvm_mmu_page *page;
1306
1307 page = container_of(vcpu->kvm->active_mmu_pages.next,
1308 struct kvm_mmu_page, link);
1309 kvm_mmu_zap_page(vcpu, page);
1310 }
1311
1312 mmu_free_memory_caches(vcpu);
1313 kvm_flush_remote_tlbs(vcpu->kvm);
1314 init_kvm_mmu(vcpu);
1315 }
1316
1317 void kvm_mmu_module_exit(void)
1318 {
1319 if (pte_chain_cache)
1320 kmem_cache_destroy(pte_chain_cache);
1321 if (rmap_desc_cache)
1322 kmem_cache_destroy(rmap_desc_cache);
1323 if (mmu_page_cache)
1324 kmem_cache_destroy(mmu_page_cache);
1325 if (mmu_page_header_cache)
1326 kmem_cache_destroy(mmu_page_header_cache);
1327 }
1328
1329 int kvm_mmu_module_init(void)
1330 {
1331 pte_chain_cache = kmem_cache_create("kvm_pte_chain",
1332 sizeof(struct kvm_pte_chain),
1333 0, 0, NULL, NULL);
1334 if (!pte_chain_cache)
1335 goto nomem;
1336 rmap_desc_cache = kmem_cache_create("kvm_rmap_desc",
1337 sizeof(struct kvm_rmap_desc),
1338 0, 0, NULL, NULL);
1339 if (!rmap_desc_cache)
1340 goto nomem;
1341
1342 mmu_page_cache = kmem_cache_create("kvm_mmu_page",
1343 PAGE_SIZE,
1344 PAGE_SIZE, 0, NULL, NULL);
1345 if (!mmu_page_cache)
1346 goto nomem;
1347
1348 mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
1349 sizeof(struct kvm_mmu_page),
1350 0, 0, NULL, NULL);
1351 if (!mmu_page_header_cache)
1352 goto nomem;
1353
1354 return 0;
1355
1356 nomem:
1357 kvm_mmu_module_exit();
1358 return -ENOMEM;
1359 }
1360
1361 #ifdef AUDIT
1362
1363 static const char *audit_msg;
1364
1365 static gva_t canonicalize(gva_t gva)
1366 {
1367 #ifdef CONFIG_X86_64
1368 gva = (long long)(gva << 16) >> 16;
1369 #endif
1370 return gva;
1371 }
1372
1373 static void audit_mappings_page(struct kvm_vcpu *vcpu, u64 page_pte,
1374 gva_t va, int level)
1375 {
1376 u64 *pt = __va(page_pte & PT64_BASE_ADDR_MASK);
1377 int i;
1378 gva_t va_delta = 1ul << (PAGE_SHIFT + 9 * (level - 1));
1379
1380 for (i = 0; i < PT64_ENT_PER_PAGE; ++i, va += va_delta) {
1381 u64 ent = pt[i];
1382
1383 if (!(ent & PT_PRESENT_MASK))
1384 continue;
1385
1386 va = canonicalize(va);
1387 if (level > 1)
1388 audit_mappings_page(vcpu, ent, va, level - 1);
1389 else {
1390 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, va);
1391 hpa_t hpa = gpa_to_hpa(vcpu, gpa);
1392
1393 if ((ent & PT_PRESENT_MASK)
1394 && (ent & PT64_BASE_ADDR_MASK) != hpa)
1395 printk(KERN_ERR "audit error: (%s) levels %d"
1396 " gva %lx gpa %llx hpa %llx ent %llx\n",
1397 audit_msg, vcpu->mmu.root_level,
1398 va, gpa, hpa, ent);
1399 }
1400 }
1401 }
1402
1403 static void audit_mappings(struct kvm_vcpu *vcpu)
1404 {
1405 unsigned i;
1406
1407 if (vcpu->mmu.root_level == 4)
1408 audit_mappings_page(vcpu, vcpu->mmu.root_hpa, 0, 4);
1409 else
1410 for (i = 0; i < 4; ++i)
1411 if (vcpu->mmu.pae_root[i] & PT_PRESENT_MASK)
1412 audit_mappings_page(vcpu,
1413 vcpu->mmu.pae_root[i],
1414 i << 30,
1415 2);
1416 }
1417
1418 static int count_rmaps(struct kvm_vcpu *vcpu)
1419 {
1420 int nmaps = 0;
1421 int i, j, k;
1422
1423 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
1424 struct kvm_memory_slot *m = &vcpu->kvm->memslots[i];
1425 struct kvm_rmap_desc *d;
1426
1427 for (j = 0; j < m->npages; ++j) {
1428 struct page *page = m->phys_mem[j];
1429
1430 if (!page->private)
1431 continue;
1432 if (!(page->private & 1)) {
1433 ++nmaps;
1434 continue;
1435 }
1436 d = (struct kvm_rmap_desc *)(page->private & ~1ul);
1437 while (d) {
1438 for (k = 0; k < RMAP_EXT; ++k)
1439 if (d->shadow_ptes[k])
1440 ++nmaps;
1441 else
1442 break;
1443 d = d->more;
1444 }
1445 }
1446 }
1447 return nmaps;
1448 }
1449
1450 static int count_writable_mappings(struct kvm_vcpu *vcpu)
1451 {
1452 int nmaps = 0;
1453 struct kvm_mmu_page *page;
1454 int i;
1455
1456 list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) {
1457 u64 *pt = page->spt;
1458
1459 if (page->role.level != PT_PAGE_TABLE_LEVEL)
1460 continue;
1461
1462 for (i = 0; i < PT64_ENT_PER_PAGE; ++i) {
1463 u64 ent = pt[i];
1464
1465 if (!(ent & PT_PRESENT_MASK))
1466 continue;
1467 if (!(ent & PT_WRITABLE_MASK))
1468 continue;
1469 ++nmaps;
1470 }
1471 }
1472 return nmaps;
1473 }
1474
1475 static void audit_rmap(struct kvm_vcpu *vcpu)
1476 {
1477 int n_rmap = count_rmaps(vcpu);
1478 int n_actual = count_writable_mappings(vcpu);
1479
1480 if (n_rmap != n_actual)
1481 printk(KERN_ERR "%s: (%s) rmap %d actual %d\n",
1482 __FUNCTION__, audit_msg, n_rmap, n_actual);
1483 }
1484
1485 static void audit_write_protection(struct kvm_vcpu *vcpu)
1486 {
1487 struct kvm_mmu_page *page;
1488
1489 list_for_each_entry(page, &vcpu->kvm->active_mmu_pages, link) {
1490 hfn_t hfn;
1491 struct page *pg;
1492
1493 if (page->role.metaphysical)
1494 continue;
1495
1496 hfn = gpa_to_hpa(vcpu, (gpa_t)page->gfn << PAGE_SHIFT)
1497 >> PAGE_SHIFT;
1498 pg = pfn_to_page(hfn);
1499 if (pg->private)
1500 printk(KERN_ERR "%s: (%s) shadow page has writable"
1501 " mappings: gfn %lx role %x\n",
1502 __FUNCTION__, audit_msg, page->gfn,
1503 page->role.word);
1504 }
1505 }
1506
1507 static void kvm_mmu_audit(struct kvm_vcpu *vcpu, const char *msg)
1508 {
1509 int olddbg = dbg;
1510
1511 dbg = 0;
1512 audit_msg = msg;
1513 audit_rmap(vcpu);
1514 audit_write_protection(vcpu);
1515 audit_mappings(vcpu);
1516 dbg = olddbg;
1517 }
1518
1519 #endif
x86 "subsystem" repository