?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[PATCH] KVM: Make loading cr3 more robust
[linux-2.6/openmoko-kernel/knife-kernel.git] / drivers / kvm / kvm_main.c
blob67c1154960f0a308f72e41c7a5e2f54f2afc2894
1 /*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * Copyright (C) 2006 Qumranet, Inc.
8 *
9 * Authors:
10 * Avi Kivity <avi@qumranet.com>
11 * Yaniv Kamay <yaniv@qumranet.com>
12 *
13 * This work is licensed under the terms of the GNU GPL, version 2. See
14 * the COPYING file in the top-level directory.
15 *
16 */
17
18 #include "kvm.h"
19
20 #include <linux/kvm.h>
21 #include <linux/module.h>
22 #include <linux/errno.h>
23 #include <asm/processor.h>
24 #include <linux/percpu.h>
25 #include <linux/gfp.h>
26 #include <asm/msr.h>
27 #include <linux/mm.h>
28 #include <linux/miscdevice.h>
29 #include <linux/vmalloc.h>
30 #include <asm/uaccess.h>
31 #include <linux/reboot.h>
32 #include <asm/io.h>
33 #include <linux/debugfs.h>
34 #include <linux/highmem.h>
35 #include <linux/file.h>
36 #include <asm/desc.h>
37
38 #include "x86_emulate.h"
39 #include "segment_descriptor.h"
40
41 MODULE_AUTHOR("Qumranet");
42 MODULE_LICENSE("GPL");
43
44 struct kvm_arch_ops *kvm_arch_ops;
45 struct kvm_stat kvm_stat;
46 EXPORT_SYMBOL_GPL(kvm_stat);
47
48 static struct kvm_stats_debugfs_item {
49 const char *name;
50 u32 *data;
51 struct dentry *dentry;
52 } debugfs_entries[] = {
53 { "pf_fixed", &kvm_stat.pf_fixed },
54 { "pf_guest", &kvm_stat.pf_guest },
55 { "tlb_flush", &kvm_stat.tlb_flush },
56 { "invlpg", &kvm_stat.invlpg },
57 { "exits", &kvm_stat.exits },
58 { "io_exits", &kvm_stat.io_exits },
59 { "mmio_exits", &kvm_stat.mmio_exits },
60 { "signal_exits", &kvm_stat.signal_exits },
61 { "irq_window", &kvm_stat.irq_window_exits },
62 { "halt_exits", &kvm_stat.halt_exits },
63 { "request_irq", &kvm_stat.request_irq_exits },
64 { "irq_exits", &kvm_stat.irq_exits },
65 { 0, 0 }
66 };
67
68 static struct dentry *debugfs_dir;
69
70 #define MAX_IO_MSRS 256
71
72 #define CR0_RESEVED_BITS 0xffffffff1ffaffc0ULL
73 #define LMSW_GUEST_MASK 0x0eULL
74 #define CR4_RESEVED_BITS (~((1ULL << 11) - 1))
75 #define CR8_RESEVED_BITS (~0x0fULL)
76 #define EFER_RESERVED_BITS 0xfffffffffffff2fe
77
78 #ifdef CONFIG_X86_64
79 // LDT or TSS descriptor in the GDT. 16 bytes.
80 struct segment_descriptor_64 {
81 struct segment_descriptor s;
82 u32 base_higher;
83 u32 pad_zero;
84 };
85
86 #endif
87
88 unsigned long segment_base(u16 selector)
89 {
90 struct descriptor_table gdt;
91 struct segment_descriptor *d;
92 unsigned long table_base;
93 typedef unsigned long ul;
94 unsigned long v;
95
96 if (selector == 0)
97 return 0;
98
99 asm ("sgdt %0" : "=m"(gdt));
100 table_base = gdt.base;
101
102 if (selector & 4) { /* from ldt */
103 u16 ldt_selector;
104
105 asm ("sldt %0" : "=g"(ldt_selector));
106 table_base = segment_base(ldt_selector);
107 }
108 d = (struct segment_descriptor *)(table_base + (selector & ~7));
109 v = d->base_low | ((ul)d->base_mid << 16) | ((ul)d->base_high << 24);
110 #ifdef CONFIG_X86_64
111 if (d->system == 0
112 && (d->type == 2 || d->type == 9 || d->type == 11))
113 v |= ((ul)((struct segment_descriptor_64 *)d)->base_higher) << 32;
114 #endif
115 return v;
116 }
117 EXPORT_SYMBOL_GPL(segment_base);
118
119 static inline int valid_vcpu(int n)
120 {
121 return likely(n >= 0 && n < KVM_MAX_VCPUS);
122 }
123
124 int kvm_read_guest(struct kvm_vcpu *vcpu,
125 gva_t addr,
126 unsigned long size,
127 void *dest)
128 {
129 unsigned char *host_buf = dest;
130 unsigned long req_size = size;
131
132 while (size) {
133 hpa_t paddr;
134 unsigned now;
135 unsigned offset;
136 hva_t guest_buf;
137
138 paddr = gva_to_hpa(vcpu, addr);
139
140 if (is_error_hpa(paddr))
141 break;
142
143 guest_buf = (hva_t)kmap_atomic(
144 pfn_to_page(paddr >> PAGE_SHIFT),
145 KM_USER0);
146 offset = addr & ~PAGE_MASK;
147 guest_buf |= offset;
148 now = min(size, PAGE_SIZE - offset);
149 memcpy(host_buf, (void*)guest_buf, now);
150 host_buf += now;
151 addr += now;
152 size -= now;
153 kunmap_atomic((void *)(guest_buf & PAGE_MASK), KM_USER0);
154 }
155 return req_size - size;
156 }
157 EXPORT_SYMBOL_GPL(kvm_read_guest);
158
159 int kvm_write_guest(struct kvm_vcpu *vcpu,
160 gva_t addr,
161 unsigned long size,
162 void *data)
163 {
164 unsigned char *host_buf = data;
165 unsigned long req_size = size;
166
167 while (size) {
168 hpa_t paddr;
169 unsigned now;
170 unsigned offset;
171 hva_t guest_buf;
172
173 paddr = gva_to_hpa(vcpu, addr);
174
175 if (is_error_hpa(paddr))
176 break;
177
178 guest_buf = (hva_t)kmap_atomic(
179 pfn_to_page(paddr >> PAGE_SHIFT), KM_USER0);
180 offset = addr & ~PAGE_MASK;
181 guest_buf |= offset;
182 now = min(size, PAGE_SIZE - offset);
183 memcpy((void*)guest_buf, host_buf, now);
184 host_buf += now;
185 addr += now;
186 size -= now;
187 kunmap_atomic((void *)(guest_buf & PAGE_MASK), KM_USER0);
188 }
189 return req_size - size;
190 }
191 EXPORT_SYMBOL_GPL(kvm_write_guest);
192
193 static int vcpu_slot(struct kvm_vcpu *vcpu)
194 {
195 return vcpu - vcpu->kvm->vcpus;
196 }
197
198 /*
199 * Switches to specified vcpu, until a matching vcpu_put()
200 */
201 static struct kvm_vcpu *vcpu_load(struct kvm *kvm, int vcpu_slot)
202 {
203 struct kvm_vcpu *vcpu = &kvm->vcpus[vcpu_slot];
204
205 mutex_lock(&vcpu->mutex);
206 if (unlikely(!vcpu->vmcs)) {
207 mutex_unlock(&vcpu->mutex);
208 return 0;
209 }
210 return kvm_arch_ops->vcpu_load(vcpu);
211 }
212
213 static void vcpu_put(struct kvm_vcpu *vcpu)
214 {
215 kvm_arch_ops->vcpu_put(vcpu);
216 mutex_unlock(&vcpu->mutex);
217 }
218
219 static int kvm_dev_open(struct inode *inode, struct file *filp)
220 {
221 struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
222 int i;
223
224 if (!kvm)
225 return -ENOMEM;
226
227 spin_lock_init(&kvm->lock);
228 INIT_LIST_HEAD(&kvm->active_mmu_pages);
229 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
230 struct kvm_vcpu *vcpu = &kvm->vcpus[i];
231
232 mutex_init(&vcpu->mutex);
233 vcpu->kvm = kvm;
234 vcpu->mmu.root_hpa = INVALID_PAGE;
235 INIT_LIST_HEAD(&vcpu->free_pages);
236 }
237 filp->private_data = kvm;
238 return 0;
239 }
240
241 /*
242 * Free any memory in @free but not in @dont.
243 */
244 static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
245 struct kvm_memory_slot *dont)
246 {
247 int i;
248
249 if (!dont || free->phys_mem != dont->phys_mem)
250 if (free->phys_mem) {
251 for (i = 0; i < free->npages; ++i)
252 if (free->phys_mem[i])
253 __free_page(free->phys_mem[i]);
254 vfree(free->phys_mem);
255 }
256
257 if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
258 vfree(free->dirty_bitmap);
259
260 free->phys_mem = 0;
261 free->npages = 0;
262 free->dirty_bitmap = 0;
263 }
264
265 static void kvm_free_physmem(struct kvm *kvm)
266 {
267 int i;
268
269 for (i = 0; i < kvm->nmemslots; ++i)
270 kvm_free_physmem_slot(&kvm->memslots[i], 0);
271 }
272
273 static void kvm_free_vcpu(struct kvm_vcpu *vcpu)
274 {
275 kvm_mmu_destroy(vcpu);
276 kvm_arch_ops->vcpu_free(vcpu);
277 }
278
279 static void kvm_free_vcpus(struct kvm *kvm)
280 {
281 unsigned int i;
282
283 for (i = 0; i < KVM_MAX_VCPUS; ++i)
284 kvm_free_vcpu(&kvm->vcpus[i]);
285 }
286
287 static int kvm_dev_release(struct inode *inode, struct file *filp)
288 {
289 struct kvm *kvm = filp->private_data;
290
291 kvm_free_vcpus(kvm);
292 kvm_free_physmem(kvm);
293 kfree(kvm);
294 return 0;
295 }
296
297 static void inject_gp(struct kvm_vcpu *vcpu)
298 {
299 kvm_arch_ops->inject_gp(vcpu, 0);
300 }
301
302 /*
303 * Load the pae pdptrs. Return true is they are all valid.
304 */
305 static int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
306 {
307 gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT;
308 unsigned offset = ((cr3 & (PAGE_SIZE-1)) >> 5) << 2;
309 int i;
310 u64 pdpte;
311 u64 *pdpt;
312 int ret;
313 struct kvm_memory_slot *memslot;
314
315 spin_lock(&vcpu->kvm->lock);
316 memslot = gfn_to_memslot(vcpu->kvm, pdpt_gfn);
317 /* FIXME: !memslot - emulate? 0xff? */
318 pdpt = kmap_atomic(gfn_to_page(memslot, pdpt_gfn), KM_USER0);
319
320 ret = 1;
321 for (i = 0; i < 4; ++i) {
322 pdpte = pdpt[offset + i];
323 if ((pdpte & 1) && (pdpte & 0xfffffff0000001e6ull)) {
324 ret = 0;
325 goto out;
326 }
327 }
328
329 for (i = 0; i < 4; ++i)
330 vcpu->pdptrs[i] = pdpt[offset + i];
331
332 out:
333 kunmap_atomic(pdpt, KM_USER0);
334 spin_unlock(&vcpu->kvm->lock);
335
336 return ret;
337 }
338
339 void set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
340 {
341 if (cr0 & CR0_RESEVED_BITS) {
342 printk(KERN_DEBUG "set_cr0: 0x%lx #GP, reserved bits 0x%lx\n",
343 cr0, vcpu->cr0);
344 inject_gp(vcpu);
345 return;
346 }
347
348 if ((cr0 & CR0_NW_MASK) && !(cr0 & CR0_CD_MASK)) {
349 printk(KERN_DEBUG "set_cr0: #GP, CD == 0 && NW == 1\n");
350 inject_gp(vcpu);
351 return;
352 }
353
354 if ((cr0 & CR0_PG_MASK) && !(cr0 & CR0_PE_MASK)) {
355 printk(KERN_DEBUG "set_cr0: #GP, set PG flag "
356 "and a clear PE flag\n");
357 inject_gp(vcpu);
358 return;
359 }
360
361 if (!is_paging(vcpu) && (cr0 & CR0_PG_MASK)) {
362 #ifdef CONFIG_X86_64
363 if ((vcpu->shadow_efer & EFER_LME)) {
364 int cs_db, cs_l;
365
366 if (!is_pae(vcpu)) {
367 printk(KERN_DEBUG "set_cr0: #GP, start paging "
368 "in long mode while PAE is disabled\n");
369 inject_gp(vcpu);
370 return;
371 }
372 kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
373 if (cs_l) {
374 printk(KERN_DEBUG "set_cr0: #GP, start paging "
375 "in long mode while CS.L == 1\n");
376 inject_gp(vcpu);
377 return;
378
379 }
380 } else
381 #endif
382 if (is_pae(vcpu) && !load_pdptrs(vcpu, vcpu->cr3)) {
383 printk(KERN_DEBUG "set_cr0: #GP, pdptrs "
384 "reserved bits\n");
385 inject_gp(vcpu);
386 return;
387 }
388
389 }
390
391 kvm_arch_ops->set_cr0(vcpu, cr0);
392 vcpu->cr0 = cr0;
393
394 spin_lock(&vcpu->kvm->lock);
395 kvm_mmu_reset_context(vcpu);
396 spin_unlock(&vcpu->kvm->lock);
397 return;
398 }
399 EXPORT_SYMBOL_GPL(set_cr0);
400
401 void lmsw(struct kvm_vcpu *vcpu, unsigned long msw)
402 {
403 kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
404 set_cr0(vcpu, (vcpu->cr0 & ~0x0ful) | (msw & 0x0f));
405 }
406 EXPORT_SYMBOL_GPL(lmsw);
407
408 void set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
409 {
410 if (cr4 & CR4_RESEVED_BITS) {
411 printk(KERN_DEBUG "set_cr4: #GP, reserved bits\n");
412 inject_gp(vcpu);
413 return;
414 }
415
416 if (is_long_mode(vcpu)) {
417 if (!(cr4 & CR4_PAE_MASK)) {
418 printk(KERN_DEBUG "set_cr4: #GP, clearing PAE while "
419 "in long mode\n");
420 inject_gp(vcpu);
421 return;
422 }
423 } else if (is_paging(vcpu) && !is_pae(vcpu) && (cr4 & CR4_PAE_MASK)
424 && !load_pdptrs(vcpu, vcpu->cr3)) {
425 printk(KERN_DEBUG "set_cr4: #GP, pdptrs reserved bits\n");
426 inject_gp(vcpu);
427 }
428
429 if (cr4 & CR4_VMXE_MASK) {
430 printk(KERN_DEBUG "set_cr4: #GP, setting VMXE\n");
431 inject_gp(vcpu);
432 return;
433 }
434 kvm_arch_ops->set_cr4(vcpu, cr4);
435 spin_lock(&vcpu->kvm->lock);
436 kvm_mmu_reset_context(vcpu);
437 spin_unlock(&vcpu->kvm->lock);
438 }
439 EXPORT_SYMBOL_GPL(set_cr4);
440
441 void set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
442 {
443 if (is_long_mode(vcpu)) {
444 if ( cr3 & CR3_L_MODE_RESEVED_BITS) {
445 printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
446 inject_gp(vcpu);
447 return;
448 }
449 } else {
450 if (cr3 & CR3_RESEVED_BITS) {
451 printk(KERN_DEBUG "set_cr3: #GP, reserved bits\n");
452 inject_gp(vcpu);
453 return;
454 }
455 if (is_paging(vcpu) && is_pae(vcpu) &&
456 !load_pdptrs(vcpu, cr3)) {
457 printk(KERN_DEBUG "set_cr3: #GP, pdptrs "
458 "reserved bits\n");
459 inject_gp(vcpu);
460 return;
461 }
462 }
463
464 vcpu->cr3 = cr3;
465 spin_lock(&vcpu->kvm->lock);
466 /*
467 * Does the new cr3 value map to physical memory? (Note, we
468 * catch an invalid cr3 even in real-mode, because it would
469 * cause trouble later on when we turn on paging anyway.)
470 *
471 * A real CPU would silently accept an invalid cr3 and would
472 * attempt to use it - with largely undefined (and often hard
473 * to debug) behavior on the guest side.
474 */
475 if (unlikely(!gfn_to_memslot(vcpu->kvm, cr3 >> PAGE_SHIFT)))
476 inject_gp(vcpu);
477 else
478 vcpu->mmu.new_cr3(vcpu);
479 spin_unlock(&vcpu->kvm->lock);
480 }
481 EXPORT_SYMBOL_GPL(set_cr3);
482
483 void set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8)
484 {
485 if ( cr8 & CR8_RESEVED_BITS) {
486 printk(KERN_DEBUG "set_cr8: #GP, reserved bits 0x%lx\n", cr8);
487 inject_gp(vcpu);
488 return;
489 }
490 vcpu->cr8 = cr8;
491 }
492 EXPORT_SYMBOL_GPL(set_cr8);
493
494 void fx_init(struct kvm_vcpu *vcpu)
495 {
496 struct __attribute__ ((__packed__)) fx_image_s {
497 u16 control; //fcw
498 u16 status; //fsw
499 u16 tag; // ftw
500 u16 opcode; //fop
501 u64 ip; // fpu ip
502 u64 operand;// fpu dp
503 u32 mxcsr;
504 u32 mxcsr_mask;
505
506 } *fx_image;
507
508 fx_save(vcpu->host_fx_image);
509 fpu_init();
510 fx_save(vcpu->guest_fx_image);
511 fx_restore(vcpu->host_fx_image);
512
513 fx_image = (struct fx_image_s *)vcpu->guest_fx_image;
514 fx_image->mxcsr = 0x1f80;
515 memset(vcpu->guest_fx_image + sizeof(struct fx_image_s),
516 0, FX_IMAGE_SIZE - sizeof(struct fx_image_s));
517 }
518 EXPORT_SYMBOL_GPL(fx_init);
519
520 /*
521 * Creates some virtual cpus. Good luck creating more than one.
522 */
523 static int kvm_dev_ioctl_create_vcpu(struct kvm *kvm, int n)
524 {
525 int r;
526 struct kvm_vcpu *vcpu;
527
528 r = -EINVAL;
529 if (!valid_vcpu(n))
530 goto out;
531
532 vcpu = &kvm->vcpus[n];
533
534 mutex_lock(&vcpu->mutex);
535
536 if (vcpu->vmcs) {
537 mutex_unlock(&vcpu->mutex);
538 return -EEXIST;
539 }
540
541 vcpu->host_fx_image = (char*)ALIGN((hva_t)vcpu->fx_buf,
542 FX_IMAGE_ALIGN);
543 vcpu->guest_fx_image = vcpu->host_fx_image + FX_IMAGE_SIZE;
544
545 vcpu->cpu = -1; /* First load will set up TR */
546 r = kvm_arch_ops->vcpu_create(vcpu);
547 if (r < 0)
548 goto out_free_vcpus;
549
550 r = kvm_mmu_create(vcpu);
551 if (r < 0)
552 goto out_free_vcpus;
553
554 kvm_arch_ops->vcpu_load(vcpu);
555 r = kvm_mmu_setup(vcpu);
556 if (r >= 0)
557 r = kvm_arch_ops->vcpu_setup(vcpu);
558 vcpu_put(vcpu);
559
560 if (r < 0)
561 goto out_free_vcpus;
562
563 return 0;
564
565 out_free_vcpus:
566 kvm_free_vcpu(vcpu);
567 mutex_unlock(&vcpu->mutex);
568 out:
569 return r;
570 }
571
572 /*
573 * Allocate some memory and give it an address in the guest physical address
574 * space.
575 *
576 * Discontiguous memory is allowed, mostly for framebuffers.
577 */
578 static int kvm_dev_ioctl_set_memory_region(struct kvm *kvm,
579 struct kvm_memory_region *mem)
580 {
581 int r;
582 gfn_t base_gfn;
583 unsigned long npages;
584 unsigned long i;
585 struct kvm_memory_slot *memslot;
586 struct kvm_memory_slot old, new;
587 int memory_config_version;
588
589 r = -EINVAL;
590 /* General sanity checks */
591 if (mem->memory_size & (PAGE_SIZE - 1))
592 goto out;
593 if (mem->guest_phys_addr & (PAGE_SIZE - 1))
594 goto out;
595 if (mem->slot >= KVM_MEMORY_SLOTS)
596 goto out;
597 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
598 goto out;
599
600 memslot = &kvm->memslots[mem->slot];
601 base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
602 npages = mem->memory_size >> PAGE_SHIFT;
603
604 if (!npages)
605 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
606
607 raced:
608 spin_lock(&kvm->lock);
609
610 memory_config_version = kvm->memory_config_version;
611 new = old = *memslot;
612
613 new.base_gfn = base_gfn;
614 new.npages = npages;
615 new.flags = mem->flags;
616
617 /* Disallow changing a memory slot's size. */
618 r = -EINVAL;
619 if (npages && old.npages && npages != old.npages)
620 goto out_unlock;
621
622 /* Check for overlaps */
623 r = -EEXIST;
624 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
625 struct kvm_memory_slot *s = &kvm->memslots[i];
626
627 if (s == memslot)
628 continue;
629 if (!((base_gfn + npages <= s->base_gfn) ||
630 (base_gfn >= s->base_gfn + s->npages)))
631 goto out_unlock;
632 }
633 /*
634 * Do memory allocations outside lock. memory_config_version will
635 * detect any races.
636 */
637 spin_unlock(&kvm->lock);
638
639 /* Deallocate if slot is being removed */
640 if (!npages)
641 new.phys_mem = 0;
642
643 /* Free page dirty bitmap if unneeded */
644 if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
645 new.dirty_bitmap = 0;
646
647 r = -ENOMEM;
648
649 /* Allocate if a slot is being created */
650 if (npages && !new.phys_mem) {
651 new.phys_mem = vmalloc(npages * sizeof(struct page *));
652
653 if (!new.phys_mem)
654 goto out_free;
655
656 memset(new.phys_mem, 0, npages * sizeof(struct page *));
657 for (i = 0; i < npages; ++i) {
658 new.phys_mem[i] = alloc_page(GFP_HIGHUSER
659 | __GFP_ZERO);
660 if (!new.phys_mem[i])
661 goto out_free;
662 new.phys_mem[i]->private = 0;
663 }
664 }
665
666 /* Allocate page dirty bitmap if needed */
667 if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
668 unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;
669
670 new.dirty_bitmap = vmalloc(dirty_bytes);
671 if (!new.dirty_bitmap)
672 goto out_free;
673 memset(new.dirty_bitmap, 0, dirty_bytes);
674 }
675
676 spin_lock(&kvm->lock);
677
678 if (memory_config_version != kvm->memory_config_version) {
679 spin_unlock(&kvm->lock);
680 kvm_free_physmem_slot(&new, &old);
681 goto raced;
682 }
683
684 r = -EAGAIN;
685 if (kvm->busy)
686 goto out_unlock;
687
688 if (mem->slot >= kvm->nmemslots)
689 kvm->nmemslots = mem->slot + 1;
690
691 *memslot = new;
692 ++kvm->memory_config_version;
693
694 spin_unlock(&kvm->lock);
695
696 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
697 struct kvm_vcpu *vcpu;
698
699 vcpu = vcpu_load(kvm, i);
700 if (!vcpu)
701 continue;
702 kvm_mmu_reset_context(vcpu);
703 vcpu_put(vcpu);
704 }
705
706 kvm_free_physmem_slot(&old, &new);
707 return 0;
708
709 out_unlock:
710 spin_unlock(&kvm->lock);
711 out_free:
712 kvm_free_physmem_slot(&new, &old);
713 out:
714 return r;
715 }
716
717 static void do_remove_write_access(struct kvm_vcpu *vcpu, int slot)
718 {
719 spin_lock(&vcpu->kvm->lock);
720 kvm_mmu_slot_remove_write_access(vcpu, slot);
721 spin_unlock(&vcpu->kvm->lock);
722 }
723
724 /*
725 * Get (and clear) the dirty memory log for a memory slot.
726 */
727 static int kvm_dev_ioctl_get_dirty_log(struct kvm *kvm,
728 struct kvm_dirty_log *log)
729 {
730 struct kvm_memory_slot *memslot;
731 int r, i;
732 int n;
733 int cleared;
734 unsigned long any = 0;
735
736 spin_lock(&kvm->lock);
737
738 /*
739 * Prevent changes to guest memory configuration even while the lock
740 * is not taken.
741 */
742 ++kvm->busy;
743 spin_unlock(&kvm->lock);
744 r = -EINVAL;
745 if (log->slot >= KVM_MEMORY_SLOTS)
746 goto out;
747
748 memslot = &kvm->memslots[log->slot];
749 r = -ENOENT;
750 if (!memslot->dirty_bitmap)
751 goto out;
752
753 n = ALIGN(memslot->npages, 8) / 8;
754
755 for (i = 0; !any && i < n; ++i)
756 any = memslot->dirty_bitmap[i];
757
758 r = -EFAULT;
759 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
760 goto out;
761
762
763 if (any) {
764 cleared = 0;
765 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
766 struct kvm_vcpu *vcpu = vcpu_load(kvm, i);
767
768 if (!vcpu)
769 continue;
770 if (!cleared) {
771 do_remove_write_access(vcpu, log->slot);
772 memset(memslot->dirty_bitmap, 0, n);
773 cleared = 1;
774 }
775 kvm_arch_ops->tlb_flush(vcpu);
776 vcpu_put(vcpu);
777 }
778 }
779
780 r = 0;
781
782 out:
783 spin_lock(&kvm->lock);
784 --kvm->busy;
785 spin_unlock(&kvm->lock);
786 return r;
787 }
788
789 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
790 {
791 int i;
792
793 for (i = 0; i < kvm->nmemslots; ++i) {
794 struct kvm_memory_slot *memslot = &kvm->memslots[i];
795
796 if (gfn >= memslot->base_gfn
797 && gfn < memslot->base_gfn + memslot->npages)
798 return memslot;
799 }
800 return 0;
801 }
802 EXPORT_SYMBOL_GPL(gfn_to_memslot);
803
804 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
805 {
806 int i;
807 struct kvm_memory_slot *memslot = 0;
808 unsigned long rel_gfn;
809
810 for (i = 0; i < kvm->nmemslots; ++i) {
811 memslot = &kvm->memslots[i];
812
813 if (gfn >= memslot->base_gfn
814 && gfn < memslot->base_gfn + memslot->npages) {
815
816 if (!memslot || !memslot->dirty_bitmap)
817 return;
818
819 rel_gfn = gfn - memslot->base_gfn;
820
821 /* avoid RMW */
822 if (!test_bit(rel_gfn, memslot->dirty_bitmap))
823 set_bit(rel_gfn, memslot->dirty_bitmap);
824 return;
825 }
826 }
827 }
828
829 static int emulator_read_std(unsigned long addr,
830 unsigned long *val,
831 unsigned int bytes,
832 struct x86_emulate_ctxt *ctxt)
833 {
834 struct kvm_vcpu *vcpu = ctxt->vcpu;
835 void *data = val;
836
837 while (bytes) {
838 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
839 unsigned offset = addr & (PAGE_SIZE-1);
840 unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset);
841 unsigned long pfn;
842 struct kvm_memory_slot *memslot;
843 void *page;
844
845 if (gpa == UNMAPPED_GVA)
846 return X86EMUL_PROPAGATE_FAULT;
847 pfn = gpa >> PAGE_SHIFT;
848 memslot = gfn_to_memslot(vcpu->kvm, pfn);
849 if (!memslot)
850 return X86EMUL_UNHANDLEABLE;
851 page = kmap_atomic(gfn_to_page(memslot, pfn), KM_USER0);
852
853 memcpy(data, page + offset, tocopy);
854
855 kunmap_atomic(page, KM_USER0);
856
857 bytes -= tocopy;
858 data += tocopy;
859 addr += tocopy;
860 }
861
862 return X86EMUL_CONTINUE;
863 }
864
865 static int emulator_write_std(unsigned long addr,
866 unsigned long val,
867 unsigned int bytes,
868 struct x86_emulate_ctxt *ctxt)
869 {
870 printk(KERN_ERR "emulator_write_std: addr %lx n %d\n",
871 addr, bytes);
872 return X86EMUL_UNHANDLEABLE;
873 }
874
875 static int emulator_read_emulated(unsigned long addr,
876 unsigned long *val,
877 unsigned int bytes,
878 struct x86_emulate_ctxt *ctxt)
879 {
880 struct kvm_vcpu *vcpu = ctxt->vcpu;
881
882 if (vcpu->mmio_read_completed) {
883 memcpy(val, vcpu->mmio_data, bytes);
884 vcpu->mmio_read_completed = 0;
885 return X86EMUL_CONTINUE;
886 } else if (emulator_read_std(addr, val, bytes, ctxt)
887 == X86EMUL_CONTINUE)
888 return X86EMUL_CONTINUE;
889 else {
890 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
891 if (gpa == UNMAPPED_GVA)
892 return vcpu_printf(vcpu, "not present\n"), X86EMUL_PROPAGATE_FAULT;
893 vcpu->mmio_needed = 1;
894 vcpu->mmio_phys_addr = gpa;
895 vcpu->mmio_size = bytes;
896 vcpu->mmio_is_write = 0;
897
898 return X86EMUL_UNHANDLEABLE;
899 }
900 }
901
902 static int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa,
903 unsigned long val, int bytes)
904 {
905 struct kvm_memory_slot *m;
906 struct page *page;
907 void *virt;
908
909 if (((gpa + bytes - 1) >> PAGE_SHIFT) != (gpa >> PAGE_SHIFT))
910 return 0;
911 m = gfn_to_memslot(vcpu->kvm, gpa >> PAGE_SHIFT);
912 if (!m)
913 return 0;
914 page = gfn_to_page(m, gpa >> PAGE_SHIFT);
915 kvm_mmu_pre_write(vcpu, gpa, bytes);
916 virt = kmap_atomic(page, KM_USER0);
917 memcpy(virt + offset_in_page(gpa), &val, bytes);
918 kunmap_atomic(virt, KM_USER0);
919 kvm_mmu_post_write(vcpu, gpa, bytes);
920 return 1;
921 }
922
923 static int emulator_write_emulated(unsigned long addr,
924 unsigned long val,
925 unsigned int bytes,
926 struct x86_emulate_ctxt *ctxt)
927 {
928 struct kvm_vcpu *vcpu = ctxt->vcpu;
929 gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, addr);
930
931 if (gpa == UNMAPPED_GVA)
932 return X86EMUL_PROPAGATE_FAULT;
933
934 if (emulator_write_phys(vcpu, gpa, val, bytes))
935 return X86EMUL_CONTINUE;
936
937 vcpu->mmio_needed = 1;
938 vcpu->mmio_phys_addr = gpa;
939 vcpu->mmio_size = bytes;
940 vcpu->mmio_is_write = 1;
941 memcpy(vcpu->mmio_data, &val, bytes);
942
943 return X86EMUL_CONTINUE;
944 }
945
946 static int emulator_cmpxchg_emulated(unsigned long addr,
947 unsigned long old,
948 unsigned long new,
949 unsigned int bytes,
950 struct x86_emulate_ctxt *ctxt)
951 {
952 static int reported;
953
954 if (!reported) {
955 reported = 1;
956 printk(KERN_WARNING "kvm: emulating exchange as write\n");
957 }
958 return emulator_write_emulated(addr, new, bytes, ctxt);
959 }
960
961 #ifdef CONFIG_X86_32
962
963 static int emulator_cmpxchg8b_emulated(unsigned long addr,
964 unsigned long old_lo,
965 unsigned long old_hi,
966 unsigned long new_lo,
967 unsigned long new_hi,
968 struct x86_emulate_ctxt *ctxt)
969 {
970 static int reported;
971 int r;
972
973 if (!reported) {
974 reported = 1;
975 printk(KERN_WARNING "kvm: emulating exchange8b as write\n");
976 }
977 r = emulator_write_emulated(addr, new_lo, 4, ctxt);
978 if (r != X86EMUL_CONTINUE)
979 return r;
980 return emulator_write_emulated(addr+4, new_hi, 4, ctxt);
981 }
982
983 #endif
984
985 static unsigned long get_segment_base(struct kvm_vcpu *vcpu, int seg)
986 {
987 return kvm_arch_ops->get_segment_base(vcpu, seg);
988 }
989
990 int emulate_invlpg(struct kvm_vcpu *vcpu, gva_t address)
991 {
992 return X86EMUL_CONTINUE;
993 }
994
995 int emulate_clts(struct kvm_vcpu *vcpu)
996 {
997 unsigned long cr0;
998
999 kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
1000 cr0 = vcpu->cr0 & ~CR0_TS_MASK;
1001 kvm_arch_ops->set_cr0(vcpu, cr0);
1002 return X86EMUL_CONTINUE;
1003 }
1004
1005 int emulator_get_dr(struct x86_emulate_ctxt* ctxt, int dr, unsigned long *dest)
1006 {
1007 struct kvm_vcpu *vcpu = ctxt->vcpu;
1008
1009 switch (dr) {
1010 case 0 ... 3:
1011 *dest = kvm_arch_ops->get_dr(vcpu, dr);
1012 return X86EMUL_CONTINUE;
1013 default:
1014 printk(KERN_DEBUG "%s: unexpected dr %u\n",
1015 __FUNCTION__, dr);
1016 return X86EMUL_UNHANDLEABLE;
1017 }
1018 }
1019
1020 int emulator_set_dr(struct x86_emulate_ctxt *ctxt, int dr, unsigned long value)
1021 {
1022 unsigned long mask = (ctxt->mode == X86EMUL_MODE_PROT64) ? ~0ULL : ~0U;
1023 int exception;
1024
1025 kvm_arch_ops->set_dr(ctxt->vcpu, dr, value & mask, &exception);
1026 if (exception) {
1027 /* FIXME: better handling */
1028 return X86EMUL_UNHANDLEABLE;
1029 }
1030 return X86EMUL_CONTINUE;
1031 }
1032
1033 static void report_emulation_failure(struct x86_emulate_ctxt *ctxt)
1034 {
1035 static int reported;
1036 u8 opcodes[4];
1037 unsigned long rip = ctxt->vcpu->rip;
1038 unsigned long rip_linear;
1039
1040 rip_linear = rip + get_segment_base(ctxt->vcpu, VCPU_SREG_CS);
1041
1042 if (reported)
1043 return;
1044
1045 emulator_read_std(rip_linear, (void *)opcodes, 4, ctxt);
1046
1047 printk(KERN_ERR "emulation failed but !mmio_needed?"
1048 " rip %lx %02x %02x %02x %02x\n",
1049 rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]);
1050 reported = 1;
1051 }
1052
1053 struct x86_emulate_ops emulate_ops = {
1054 .read_std = emulator_read_std,
1055 .write_std = emulator_write_std,
1056 .read_emulated = emulator_read_emulated,
1057 .write_emulated = emulator_write_emulated,
1058 .cmpxchg_emulated = emulator_cmpxchg_emulated,
1059 #ifdef CONFIG_X86_32
1060 .cmpxchg8b_emulated = emulator_cmpxchg8b_emulated,
1061 #endif
1062 };
1063
1064 int emulate_instruction(struct kvm_vcpu *vcpu,
1065 struct kvm_run *run,
1066 unsigned long cr2,
1067 u16 error_code)
1068 {
1069 struct x86_emulate_ctxt emulate_ctxt;
1070 int r;
1071 int cs_db, cs_l;
1072
1073 kvm_arch_ops->cache_regs(vcpu);
1074
1075 kvm_arch_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
1076
1077 emulate_ctxt.vcpu = vcpu;
1078 emulate_ctxt.eflags = kvm_arch_ops->get_rflags(vcpu);
1079 emulate_ctxt.cr2 = cr2;
1080 emulate_ctxt.mode = (emulate_ctxt.eflags & X86_EFLAGS_VM)
1081 ? X86EMUL_MODE_REAL : cs_l
1082 ? X86EMUL_MODE_PROT64 : cs_db
1083 ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16;
1084
1085 if (emulate_ctxt.mode == X86EMUL_MODE_PROT64) {
1086 emulate_ctxt.cs_base = 0;
1087 emulate_ctxt.ds_base = 0;
1088 emulate_ctxt.es_base = 0;
1089 emulate_ctxt.ss_base = 0;
1090 } else {
1091 emulate_ctxt.cs_base = get_segment_base(vcpu, VCPU_SREG_CS);
1092 emulate_ctxt.ds_base = get_segment_base(vcpu, VCPU_SREG_DS);
1093 emulate_ctxt.es_base = get_segment_base(vcpu, VCPU_SREG_ES);
1094 emulate_ctxt.ss_base = get_segment_base(vcpu, VCPU_SREG_SS);
1095 }
1096
1097 emulate_ctxt.gs_base = get_segment_base(vcpu, VCPU_SREG_GS);
1098 emulate_ctxt.fs_base = get_segment_base(vcpu, VCPU_SREG_FS);
1099
1100 vcpu->mmio_is_write = 0;
1101 r = x86_emulate_memop(&emulate_ctxt, &emulate_ops);
1102
1103 if ((r || vcpu->mmio_is_write) && run) {
1104 run->mmio.phys_addr = vcpu->mmio_phys_addr;
1105 memcpy(run->mmio.data, vcpu->mmio_data, 8);
1106 run->mmio.len = vcpu->mmio_size;
1107 run->mmio.is_write = vcpu->mmio_is_write;
1108 }
1109
1110 if (r) {
1111 if (kvm_mmu_unprotect_page_virt(vcpu, cr2))
1112 return EMULATE_DONE;
1113 if (!vcpu->mmio_needed) {
1114 report_emulation_failure(&emulate_ctxt);
1115 return EMULATE_FAIL;
1116 }
1117 return EMULATE_DO_MMIO;
1118 }
1119
1120 kvm_arch_ops->decache_regs(vcpu);
1121 kvm_arch_ops->set_rflags(vcpu, emulate_ctxt.eflags);
1122
1123 if (vcpu->mmio_is_write)
1124 return EMULATE_DO_MMIO;
1125
1126 return EMULATE_DONE;
1127 }
1128 EXPORT_SYMBOL_GPL(emulate_instruction);
1129
1130 static u64 mk_cr_64(u64 curr_cr, u32 new_val)
1131 {
1132 return (curr_cr & ~((1ULL << 32) - 1)) | new_val;
1133 }
1134
1135 void realmode_lgdt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
1136 {
1137 struct descriptor_table dt = { limit, base };
1138
1139 kvm_arch_ops->set_gdt(vcpu, &dt);
1140 }
1141
1142 void realmode_lidt(struct kvm_vcpu *vcpu, u16 limit, unsigned long base)
1143 {
1144 struct descriptor_table dt = { limit, base };
1145
1146 kvm_arch_ops->set_idt(vcpu, &dt);
1147 }
1148
1149 void realmode_lmsw(struct kvm_vcpu *vcpu, unsigned long msw,
1150 unsigned long *rflags)
1151 {
1152 lmsw(vcpu, msw);
1153 *rflags = kvm_arch_ops->get_rflags(vcpu);
1154 }
1155
1156 unsigned long realmode_get_cr(struct kvm_vcpu *vcpu, int cr)
1157 {
1158 kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
1159 switch (cr) {
1160 case 0:
1161 return vcpu->cr0;
1162 case 2:
1163 return vcpu->cr2;
1164 case 3:
1165 return vcpu->cr3;
1166 case 4:
1167 return vcpu->cr4;
1168 default:
1169 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
1170 return 0;
1171 }
1172 }
1173
1174 void realmode_set_cr(struct kvm_vcpu *vcpu, int cr, unsigned long val,
1175 unsigned long *rflags)
1176 {
1177 switch (cr) {
1178 case 0:
1179 set_cr0(vcpu, mk_cr_64(vcpu->cr0, val));
1180 *rflags = kvm_arch_ops->get_rflags(vcpu);
1181 break;
1182 case 2:
1183 vcpu->cr2 = val;
1184 break;
1185 case 3:
1186 set_cr3(vcpu, val);
1187 break;
1188 case 4:
1189 set_cr4(vcpu, mk_cr_64(vcpu->cr4, val));
1190 break;
1191 default:
1192 vcpu_printf(vcpu, "%s: unexpected cr %u\n", __FUNCTION__, cr);
1193 }
1194 }
1195
1196 int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata)
1197 {
1198 u64 data;
1199
1200 switch (msr) {
1201 case 0xc0010010: /* SYSCFG */
1202 case 0xc0010015: /* HWCR */
1203 case MSR_IA32_PLATFORM_ID:
1204 case MSR_IA32_P5_MC_ADDR:
1205 case MSR_IA32_P5_MC_TYPE:
1206 case MSR_IA32_MC0_CTL:
1207 case MSR_IA32_MCG_STATUS:
1208 case MSR_IA32_MCG_CAP:
1209 case MSR_IA32_MC0_MISC:
1210 case MSR_IA32_MC0_MISC+4:
1211 case MSR_IA32_MC0_MISC+8:
1212 case MSR_IA32_MC0_MISC+12:
1213 case MSR_IA32_MC0_MISC+16:
1214 case MSR_IA32_UCODE_REV:
1215 case MSR_IA32_PERF_STATUS:
1216 /* MTRR registers */
1217 case 0xfe:
1218 case 0x200 ... 0x2ff:
1219 data = 0;
1220 break;
1221 case 0xcd: /* fsb frequency */
1222 data = 3;
1223 break;
1224 case MSR_IA32_APICBASE:
1225 data = vcpu->apic_base;
1226 break;
1227 #ifdef CONFIG_X86_64
1228 case MSR_EFER:
1229 data = vcpu->shadow_efer;
1230 break;
1231 #endif
1232 default:
1233 printk(KERN_ERR "kvm: unhandled rdmsr: 0x%x\n", msr);
1234 return 1;
1235 }
1236 *pdata = data;
1237 return 0;
1238 }
1239 EXPORT_SYMBOL_GPL(kvm_get_msr_common);
1240
1241 /*
1242 * Reads an msr value (of 'msr_index') into 'pdata'.
1243 * Returns 0 on success, non-0 otherwise.
1244 * Assumes vcpu_load() was already called.
1245 */
1246 static int get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
1247 {
1248 return kvm_arch_ops->get_msr(vcpu, msr_index, pdata);
1249 }
1250
1251 #ifdef CONFIG_X86_64
1252
1253 static void set_efer(struct kvm_vcpu *vcpu, u64 efer)
1254 {
1255 if (efer & EFER_RESERVED_BITS) {
1256 printk(KERN_DEBUG "set_efer: 0x%llx #GP, reserved bits\n",
1257 efer);
1258 inject_gp(vcpu);
1259 return;
1260 }
1261
1262 if (is_paging(vcpu)
1263 && (vcpu->shadow_efer & EFER_LME) != (efer & EFER_LME)) {
1264 printk(KERN_DEBUG "set_efer: #GP, change LME while paging\n");
1265 inject_gp(vcpu);
1266 return;
1267 }
1268
1269 kvm_arch_ops->set_efer(vcpu, efer);
1270
1271 efer &= ~EFER_LMA;
1272 efer |= vcpu->shadow_efer & EFER_LMA;
1273
1274 vcpu->shadow_efer = efer;
1275 }
1276
1277 #endif
1278
1279 int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
1280 {
1281 switch (msr) {
1282 #ifdef CONFIG_X86_64
1283 case MSR_EFER:
1284 set_efer(vcpu, data);
1285 break;
1286 #endif
1287 case MSR_IA32_MC0_STATUS:
1288 printk(KERN_WARNING "%s: MSR_IA32_MC0_STATUS 0x%llx, nop\n",
1289 __FUNCTION__, data);
1290 break;
1291 case MSR_IA32_UCODE_REV:
1292 case MSR_IA32_UCODE_WRITE:
1293 case 0x200 ... 0x2ff: /* MTRRs */
1294 break;
1295 case MSR_IA32_APICBASE:
1296 vcpu->apic_base = data;
1297 break;
1298 default:
1299 printk(KERN_ERR "kvm: unhandled wrmsr: 0x%x\n", msr);
1300 return 1;
1301 }
1302 return 0;
1303 }
1304 EXPORT_SYMBOL_GPL(kvm_set_msr_common);
1305
1306 /*
1307 * Writes msr value into into the appropriate "register".
1308 * Returns 0 on success, non-0 otherwise.
1309 * Assumes vcpu_load() was already called.
1310 */
1311 static int set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
1312 {
1313 return kvm_arch_ops->set_msr(vcpu, msr_index, data);
1314 }
1315
1316 void kvm_resched(struct kvm_vcpu *vcpu)
1317 {
1318 vcpu_put(vcpu);
1319 cond_resched();
1320 /* Cannot fail - no vcpu unplug yet. */
1321 vcpu_load(vcpu->kvm, vcpu_slot(vcpu));
1322 }
1323 EXPORT_SYMBOL_GPL(kvm_resched);
1324
1325 void load_msrs(struct vmx_msr_entry *e, int n)
1326 {
1327 int i;
1328
1329 for (i = 0; i < n; ++i)
1330 wrmsrl(e[i].index, e[i].data);
1331 }
1332 EXPORT_SYMBOL_GPL(load_msrs);
1333
1334 void save_msrs(struct vmx_msr_entry *e, int n)
1335 {
1336 int i;
1337
1338 for (i = 0; i < n; ++i)
1339 rdmsrl(e[i].index, e[i].data);
1340 }
1341 EXPORT_SYMBOL_GPL(save_msrs);
1342
1343 static int kvm_dev_ioctl_run(struct kvm *kvm, struct kvm_run *kvm_run)
1344 {
1345 struct kvm_vcpu *vcpu;
1346 int r;
1347
1348 if (!valid_vcpu(kvm_run->vcpu))
1349 return -EINVAL;
1350
1351 vcpu = vcpu_load(kvm, kvm_run->vcpu);
1352 if (!vcpu)
1353 return -ENOENT;
1354
1355 if (kvm_run->emulated) {
1356 kvm_arch_ops->skip_emulated_instruction(vcpu);
1357 kvm_run->emulated = 0;
1358 }
1359
1360 if (kvm_run->mmio_completed) {
1361 memcpy(vcpu->mmio_data, kvm_run->mmio.data, 8);
1362 vcpu->mmio_read_completed = 1;
1363 }
1364
1365 vcpu->mmio_needed = 0;
1366
1367 r = kvm_arch_ops->run(vcpu, kvm_run);
1368
1369 vcpu_put(vcpu);
1370 return r;
1371 }
1372
1373 static int kvm_dev_ioctl_get_regs(struct kvm *kvm, struct kvm_regs *regs)
1374 {
1375 struct kvm_vcpu *vcpu;
1376
1377 if (!valid_vcpu(regs->vcpu))
1378 return -EINVAL;
1379
1380 vcpu = vcpu_load(kvm, regs->vcpu);
1381 if (!vcpu)
1382 return -ENOENT;
1383
1384 kvm_arch_ops->cache_regs(vcpu);
1385
1386 regs->rax = vcpu->regs[VCPU_REGS_RAX];
1387 regs->rbx = vcpu->regs[VCPU_REGS_RBX];
1388 regs->rcx = vcpu->regs[VCPU_REGS_RCX];
1389 regs->rdx = vcpu->regs[VCPU_REGS_RDX];
1390 regs->rsi = vcpu->regs[VCPU_REGS_RSI];
1391 regs->rdi = vcpu->regs[VCPU_REGS_RDI];
1392 regs->rsp = vcpu->regs[VCPU_REGS_RSP];
1393 regs->rbp = vcpu->regs[VCPU_REGS_RBP];
1394 #ifdef CONFIG_X86_64
1395 regs->r8 = vcpu->regs[VCPU_REGS_R8];
1396 regs->r9 = vcpu->regs[VCPU_REGS_R9];
1397 regs->r10 = vcpu->regs[VCPU_REGS_R10];
1398 regs->r11 = vcpu->regs[VCPU_REGS_R11];
1399 regs->r12 = vcpu->regs[VCPU_REGS_R12];
1400 regs->r13 = vcpu->regs[VCPU_REGS_R13];
1401 regs->r14 = vcpu->regs[VCPU_REGS_R14];
1402 regs->r15 = vcpu->regs[VCPU_REGS_R15];
1403 #endif
1404
1405 regs->rip = vcpu->rip;
1406 regs->rflags = kvm_arch_ops->get_rflags(vcpu);
1407
1408 /*
1409 * Don't leak debug flags in case they were set for guest debugging
1410 */
1411 if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep)
1412 regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
1413
1414 vcpu_put(vcpu);
1415
1416 return 0;
1417 }
1418
1419 static int kvm_dev_ioctl_set_regs(struct kvm *kvm, struct kvm_regs *regs)
1420 {
1421 struct kvm_vcpu *vcpu;
1422
1423 if (!valid_vcpu(regs->vcpu))
1424 return -EINVAL;
1425
1426 vcpu = vcpu_load(kvm, regs->vcpu);
1427 if (!vcpu)
1428 return -ENOENT;
1429
1430 vcpu->regs[VCPU_REGS_RAX] = regs->rax;
1431 vcpu->regs[VCPU_REGS_RBX] = regs->rbx;
1432 vcpu->regs[VCPU_REGS_RCX] = regs->rcx;
1433 vcpu->regs[VCPU_REGS_RDX] = regs->rdx;
1434 vcpu->regs[VCPU_REGS_RSI] = regs->rsi;
1435 vcpu->regs[VCPU_REGS_RDI] = regs->rdi;
1436 vcpu->regs[VCPU_REGS_RSP] = regs->rsp;
1437 vcpu->regs[VCPU_REGS_RBP] = regs->rbp;
1438 #ifdef CONFIG_X86_64
1439 vcpu->regs[VCPU_REGS_R8] = regs->r8;
1440 vcpu->regs[VCPU_REGS_R9] = regs->r9;
1441 vcpu->regs[VCPU_REGS_R10] = regs->r10;
1442 vcpu->regs[VCPU_REGS_R11] = regs->r11;
1443 vcpu->regs[VCPU_REGS_R12] = regs->r12;
1444 vcpu->regs[VCPU_REGS_R13] = regs->r13;
1445 vcpu->regs[VCPU_REGS_R14] = regs->r14;
1446 vcpu->regs[VCPU_REGS_R15] = regs->r15;
1447 #endif
1448
1449 vcpu->rip = regs->rip;
1450 kvm_arch_ops->set_rflags(vcpu, regs->rflags);
1451
1452 kvm_arch_ops->decache_regs(vcpu);
1453
1454 vcpu_put(vcpu);
1455
1456 return 0;
1457 }
1458
1459 static void get_segment(struct kvm_vcpu *vcpu,
1460 struct kvm_segment *var, int seg)
1461 {
1462 return kvm_arch_ops->get_segment(vcpu, var, seg);
1463 }
1464
1465 static int kvm_dev_ioctl_get_sregs(struct kvm *kvm, struct kvm_sregs *sregs)
1466 {
1467 struct kvm_vcpu *vcpu;
1468 struct descriptor_table dt;
1469
1470 if (!valid_vcpu(sregs->vcpu))
1471 return -EINVAL;
1472 vcpu = vcpu_load(kvm, sregs->vcpu);
1473 if (!vcpu)
1474 return -ENOENT;
1475
1476 get_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
1477 get_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
1478 get_segment(vcpu, &sregs->es, VCPU_SREG_ES);
1479 get_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
1480 get_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
1481 get_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
1482
1483 get_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
1484 get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
1485
1486 kvm_arch_ops->get_idt(vcpu, &dt);
1487 sregs->idt.limit = dt.limit;
1488 sregs->idt.base = dt.base;
1489 kvm_arch_ops->get_gdt(vcpu, &dt);
1490 sregs->gdt.limit = dt.limit;
1491 sregs->gdt.base = dt.base;
1492
1493 kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
1494 sregs->cr0 = vcpu->cr0;
1495 sregs->cr2 = vcpu->cr2;
1496 sregs->cr3 = vcpu->cr3;
1497 sregs->cr4 = vcpu->cr4;
1498 sregs->cr8 = vcpu->cr8;
1499 sregs->efer = vcpu->shadow_efer;
1500 sregs->apic_base = vcpu->apic_base;
1501
1502 memcpy(sregs->interrupt_bitmap, vcpu->irq_pending,
1503 sizeof sregs->interrupt_bitmap);
1504
1505 vcpu_put(vcpu);
1506
1507 return 0;
1508 }
1509
1510 static void set_segment(struct kvm_vcpu *vcpu,
1511 struct kvm_segment *var, int seg)
1512 {
1513 return kvm_arch_ops->set_segment(vcpu, var, seg);
1514 }
1515
1516 static int kvm_dev_ioctl_set_sregs(struct kvm *kvm, struct kvm_sregs *sregs)
1517 {
1518 struct kvm_vcpu *vcpu;
1519 int mmu_reset_needed = 0;
1520 int i;
1521 struct descriptor_table dt;
1522
1523 if (!valid_vcpu(sregs->vcpu))
1524 return -EINVAL;
1525 vcpu = vcpu_load(kvm, sregs->vcpu);
1526 if (!vcpu)
1527 return -ENOENT;
1528
1529 set_segment(vcpu, &sregs->cs, VCPU_SREG_CS);
1530 set_segment(vcpu, &sregs->ds, VCPU_SREG_DS);
1531 set_segment(vcpu, &sregs->es, VCPU_SREG_ES);
1532 set_segment(vcpu, &sregs->fs, VCPU_SREG_FS);
1533 set_segment(vcpu, &sregs->gs, VCPU_SREG_GS);
1534 set_segment(vcpu, &sregs->ss, VCPU_SREG_SS);
1535
1536 set_segment(vcpu, &sregs->tr, VCPU_SREG_TR);
1537 set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR);
1538
1539 dt.limit = sregs->idt.limit;
1540 dt.base = sregs->idt.base;
1541 kvm_arch_ops->set_idt(vcpu, &dt);
1542 dt.limit = sregs->gdt.limit;
1543 dt.base = sregs->gdt.base;
1544 kvm_arch_ops->set_gdt(vcpu, &dt);
1545
1546 vcpu->cr2 = sregs->cr2;
1547 mmu_reset_needed |= vcpu->cr3 != sregs->cr3;
1548 vcpu->cr3 = sregs->cr3;
1549
1550 vcpu->cr8 = sregs->cr8;
1551
1552 mmu_reset_needed |= vcpu->shadow_efer != sregs->efer;
1553 #ifdef CONFIG_X86_64
1554 kvm_arch_ops->set_efer(vcpu, sregs->efer);
1555 #endif
1556 vcpu->apic_base = sregs->apic_base;
1557
1558 kvm_arch_ops->decache_cr0_cr4_guest_bits(vcpu);
1559
1560 mmu_reset_needed |= vcpu->cr0 != sregs->cr0;
1561 kvm_arch_ops->set_cr0_no_modeswitch(vcpu, sregs->cr0);
1562
1563 mmu_reset_needed |= vcpu->cr4 != sregs->cr4;
1564 kvm_arch_ops->set_cr4(vcpu, sregs->cr4);
1565 if (!is_long_mode(vcpu) && is_pae(vcpu))
1566 load_pdptrs(vcpu, vcpu->cr3);
1567
1568 if (mmu_reset_needed)
1569 kvm_mmu_reset_context(vcpu);
1570
1571 memcpy(vcpu->irq_pending, sregs->interrupt_bitmap,
1572 sizeof vcpu->irq_pending);
1573 vcpu->irq_summary = 0;
1574 for (i = 0; i < NR_IRQ_WORDS; ++i)
1575 if (vcpu->irq_pending[i])
1576 __set_bit(i, &vcpu->irq_summary);
1577
1578 vcpu_put(vcpu);
1579
1580 return 0;
1581 }
1582
1583 /*
1584 * List of msr numbers which we expose to userspace through KVM_GET_MSRS
1585 * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST.
1586 *
1587 * This list is modified at module load time to reflect the
1588 * capabilities of the host cpu.
1589 */
1590 static u32 msrs_to_save[] = {
1591 MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
1592 MSR_K6_STAR,
1593 #ifdef CONFIG_X86_64
1594 MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR,
1595 #endif
1596 MSR_IA32_TIME_STAMP_COUNTER,
1597 };
1598
1599 static unsigned num_msrs_to_save;
1600
1601 static __init void kvm_init_msr_list(void)
1602 {
1603 u32 dummy[2];
1604 unsigned i, j;
1605
1606 for (i = j = 0; i < ARRAY_SIZE(msrs_to_save); i++) {
1607 if (rdmsr_safe(msrs_to_save[i], &dummy[0], &dummy[1]) < 0)
1608 continue;
1609 if (j < i)
1610 msrs_to_save[j] = msrs_to_save[i];
1611 j++;
1612 }
1613 num_msrs_to_save = j;
1614 }
1615
1616 /*
1617 * Adapt set_msr() to msr_io()'s calling convention
1618 */
1619 static int do_set_msr(struct kvm_vcpu *vcpu, unsigned index, u64 *data)
1620 {
1621 return set_msr(vcpu, index, *data);
1622 }
1623
1624 /*
1625 * Read or write a bunch of msrs. All parameters are kernel addresses.
1626 *
1627 * @return number of msrs set successfully.
1628 */
1629 static int __msr_io(struct kvm *kvm, struct kvm_msrs *msrs,
1630 struct kvm_msr_entry *entries,
1631 int (*do_msr)(struct kvm_vcpu *vcpu,
1632 unsigned index, u64 *data))
1633 {
1634 struct kvm_vcpu *vcpu;
1635 int i;
1636
1637 if (!valid_vcpu(msrs->vcpu))
1638 return -EINVAL;
1639
1640 vcpu = vcpu_load(kvm, msrs->vcpu);
1641 if (!vcpu)
1642 return -ENOENT;
1643
1644 for (i = 0; i < msrs->nmsrs; ++i)
1645 if (do_msr(vcpu, entries[i].index, &entries[i].data))
1646 break;
1647
1648 vcpu_put(vcpu);
1649
1650 return i;
1651 }
1652
1653 /*
1654 * Read or write a bunch of msrs. Parameters are user addresses.
1655 *
1656 * @return number of msrs set successfully.
1657 */
1658 static int msr_io(struct kvm *kvm, struct kvm_msrs __user *user_msrs,
1659 int (*do_msr)(struct kvm_vcpu *vcpu,
1660 unsigned index, u64 *data),
1661 int writeback)
1662 {
1663 struct kvm_msrs msrs;
1664 struct kvm_msr_entry *entries;
1665 int r, n;
1666 unsigned size;
1667
1668 r = -EFAULT;
1669 if (copy_from_user(&msrs, user_msrs, sizeof msrs))
1670 goto out;
1671
1672 r = -E2BIG;
1673 if (msrs.nmsrs >= MAX_IO_MSRS)
1674 goto out;
1675
1676 r = -ENOMEM;
1677 size = sizeof(struct kvm_msr_entry) * msrs.nmsrs;
1678 entries = vmalloc(size);
1679 if (!entries)
1680 goto out;
1681
1682 r = -EFAULT;
1683 if (copy_from_user(entries, user_msrs->entries, size))
1684 goto out_free;
1685
1686 r = n = __msr_io(kvm, &msrs, entries, do_msr);
1687 if (r < 0)
1688 goto out_free;
1689
1690 r = -EFAULT;
1691 if (writeback && copy_to_user(user_msrs->entries, entries, size))
1692 goto out_free;
1693
1694 r = n;
1695
1696 out_free:
1697 vfree(entries);
1698 out:
1699 return r;
1700 }
1701
1702 /*
1703 * Translate a guest virtual address to a guest physical address.
1704 */
1705 static int kvm_dev_ioctl_translate(struct kvm *kvm, struct kvm_translation *tr)
1706 {
1707 unsigned long vaddr = tr->linear_address;
1708 struct kvm_vcpu *vcpu;
1709 gpa_t gpa;
1710
1711 vcpu = vcpu_load(kvm, tr->vcpu);
1712 if (!vcpu)
1713 return -ENOENT;
1714 spin_lock(&kvm->lock);
1715 gpa = vcpu->mmu.gva_to_gpa(vcpu, vaddr);
1716 tr->physical_address = gpa;
1717 tr->valid = gpa != UNMAPPED_GVA;
1718 tr->writeable = 1;
1719 tr->usermode = 0;
1720 spin_unlock(&kvm->lock);
1721 vcpu_put(vcpu);
1722
1723 return 0;
1724 }
1725
1726 static int kvm_dev_ioctl_interrupt(struct kvm *kvm, struct kvm_interrupt *irq)
1727 {
1728 struct kvm_vcpu *vcpu;
1729
1730 if (!valid_vcpu(irq->vcpu))
1731 return -EINVAL;
1732 if (irq->irq < 0 || irq->irq >= 256)
1733 return -EINVAL;
1734 vcpu = vcpu_load(kvm, irq->vcpu);
1735 if (!vcpu)
1736 return -ENOENT;
1737
1738 set_bit(irq->irq, vcpu->irq_pending);
1739 set_bit(irq->irq / BITS_PER_LONG, &vcpu->irq_summary);
1740
1741 vcpu_put(vcpu);
1742
1743 return 0;
1744 }
1745
1746 static int kvm_dev_ioctl_debug_guest(struct kvm *kvm,
1747 struct kvm_debug_guest *dbg)
1748 {
1749 struct kvm_vcpu *vcpu;
1750 int r;
1751
1752 if (!valid_vcpu(dbg->vcpu))
1753 return -EINVAL;
1754 vcpu = vcpu_load(kvm, dbg->vcpu);
1755 if (!vcpu)
1756 return -ENOENT;
1757
1758 r = kvm_arch_ops->set_guest_debug(vcpu, dbg);
1759
1760 vcpu_put(vcpu);
1761
1762 return r;
1763 }
1764
1765 static long kvm_dev_ioctl(struct file *filp,
1766 unsigned int ioctl, unsigned long arg)
1767 {
1768 struct kvm *kvm = filp->private_data;
1769 int r = -EINVAL;
1770
1771 switch (ioctl) {
1772 case KVM_GET_API_VERSION:
1773 r = KVM_API_VERSION;
1774 break;
1775 case KVM_CREATE_VCPU: {
1776 r = kvm_dev_ioctl_create_vcpu(kvm, arg);
1777 if (r)
1778 goto out;
1779 break;
1780 }
1781 case KVM_RUN: {
1782 struct kvm_run kvm_run;
1783
1784 r = -EFAULT;
1785 if (copy_from_user(&kvm_run, (void *)arg, sizeof kvm_run))
1786 goto out;
1787 r = kvm_dev_ioctl_run(kvm, &kvm_run);
1788 if (r < 0 && r != -EINTR)
1789 goto out;
1790 if (copy_to_user((void *)arg, &kvm_run, sizeof kvm_run)) {
1791 r = -EFAULT;
1792 goto out;
1793 }
1794 break;
1795 }
1796 case KVM_GET_REGS: {
1797 struct kvm_regs kvm_regs;
1798
1799 r = -EFAULT;
1800 if (copy_from_user(&kvm_regs, (void *)arg, sizeof kvm_regs))
1801 goto out;
1802 r = kvm_dev_ioctl_get_regs(kvm, &kvm_regs);
1803 if (r)
1804 goto out;
1805 r = -EFAULT;
1806 if (copy_to_user((void *)arg, &kvm_regs, sizeof kvm_regs))
1807 goto out;
1808 r = 0;
1809 break;
1810 }
1811 case KVM_SET_REGS: {
1812 struct kvm_regs kvm_regs;
1813
1814 r = -EFAULT;
1815 if (copy_from_user(&kvm_regs, (void *)arg, sizeof kvm_regs))
1816 goto out;
1817 r = kvm_dev_ioctl_set_regs(kvm, &kvm_regs);
1818 if (r)
1819 goto out;
1820 r = 0;
1821 break;
1822 }
1823 case KVM_GET_SREGS: {
1824 struct kvm_sregs kvm_sregs;
1825
1826 r = -EFAULT;
1827 if (copy_from_user(&kvm_sregs, (void *)arg, sizeof kvm_sregs))
1828 goto out;
1829 r = kvm_dev_ioctl_get_sregs(kvm, &kvm_sregs);
1830 if (r)
1831 goto out;
1832 r = -EFAULT;
1833 if (copy_to_user((void *)arg, &kvm_sregs, sizeof kvm_sregs))
1834 goto out;
1835 r = 0;
1836 break;
1837 }
1838 case KVM_SET_SREGS: {
1839 struct kvm_sregs kvm_sregs;
1840
1841 r = -EFAULT;
1842 if (copy_from_user(&kvm_sregs, (void *)arg, sizeof kvm_sregs))
1843 goto out;
1844 r = kvm_dev_ioctl_set_sregs(kvm, &kvm_sregs);
1845 if (r)
1846 goto out;
1847 r = 0;
1848 break;
1849 }
1850 case KVM_TRANSLATE: {
1851 struct kvm_translation tr;
1852
1853 r = -EFAULT;
1854 if (copy_from_user(&tr, (void *)arg, sizeof tr))
1855 goto out;
1856 r = kvm_dev_ioctl_translate(kvm, &tr);
1857 if (r)
1858 goto out;
1859 r = -EFAULT;
1860 if (copy_to_user((void *)arg, &tr, sizeof tr))
1861 goto out;
1862 r = 0;
1863 break;
1864 }
1865 case KVM_INTERRUPT: {
1866 struct kvm_interrupt irq;
1867
1868 r = -EFAULT;
1869 if (copy_from_user(&irq, (void *)arg, sizeof irq))
1870 goto out;
1871 r = kvm_dev_ioctl_interrupt(kvm, &irq);
1872 if (r)
1873 goto out;
1874 r = 0;
1875 break;
1876 }
1877 case KVM_DEBUG_GUEST: {
1878 struct kvm_debug_guest dbg;
1879
1880 r = -EFAULT;
1881 if (copy_from_user(&dbg, (void *)arg, sizeof dbg))
1882 goto out;
1883 r = kvm_dev_ioctl_debug_guest(kvm, &dbg);
1884 if (r)
1885 goto out;
1886 r = 0;
1887 break;
1888 }
1889 case KVM_SET_MEMORY_REGION: {
1890 struct kvm_memory_region kvm_mem;
1891
1892 r = -EFAULT;
1893 if (copy_from_user(&kvm_mem, (void *)arg, sizeof kvm_mem))
1894 goto out;
1895 r = kvm_dev_ioctl_set_memory_region(kvm, &kvm_mem);
1896 if (r)
1897 goto out;
1898 break;
1899 }
1900 case KVM_GET_DIRTY_LOG: {
1901 struct kvm_dirty_log log;
1902
1903 r = -EFAULT;
1904 if (copy_from_user(&log, (void *)arg, sizeof log))
1905 goto out;
1906 r = kvm_dev_ioctl_get_dirty_log(kvm, &log);
1907 if (r)
1908 goto out;
1909 break;
1910 }
1911 case KVM_GET_MSRS:
1912 r = msr_io(kvm, (void __user *)arg, get_msr, 1);
1913 break;
1914 case KVM_SET_MSRS:
1915 r = msr_io(kvm, (void __user *)arg, do_set_msr, 0);
1916 break;
1917 case KVM_GET_MSR_INDEX_LIST: {
1918 struct kvm_msr_list __user *user_msr_list = (void __user *)arg;
1919 struct kvm_msr_list msr_list;
1920 unsigned n;
1921
1922 r = -EFAULT;
1923 if (copy_from_user(&msr_list, user_msr_list, sizeof msr_list))
1924 goto out;
1925 n = msr_list.nmsrs;
1926 msr_list.nmsrs = num_msrs_to_save;
1927 if (copy_to_user(user_msr_list, &msr_list, sizeof msr_list))
1928 goto out;
1929 r = -E2BIG;
1930 if (n < num_msrs_to_save)
1931 goto out;
1932 r = -EFAULT;
1933 if (copy_to_user(user_msr_list->indices, &msrs_to_save,
1934 num_msrs_to_save * sizeof(u32)))
1935 goto out;
1936 r = 0;
1937 break;
1938 }
1939 default:
1940 ;
1941 }
1942 out:
1943 return r;
1944 }
1945
1946 static struct page *kvm_dev_nopage(struct vm_area_struct *vma,
1947 unsigned long address,
1948 int *type)
1949 {
1950 struct kvm *kvm = vma->vm_file->private_data;
1951 unsigned long pgoff;
1952 struct kvm_memory_slot *slot;
1953 struct page *page;
1954
1955 *type = VM_FAULT_MINOR;
1956 pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
1957 slot = gfn_to_memslot(kvm, pgoff);
1958 if (!slot)
1959 return NOPAGE_SIGBUS;
1960 page = gfn_to_page(slot, pgoff);
1961 if (!page)
1962 return NOPAGE_SIGBUS;
1963 get_page(page);
1964 return page;
1965 }
1966
1967 static struct vm_operations_struct kvm_dev_vm_ops = {
1968 .nopage = kvm_dev_nopage,
1969 };
1970
1971 static int kvm_dev_mmap(struct file *file, struct vm_area_struct *vma)
1972 {
1973 vma->vm_ops = &kvm_dev_vm_ops;
1974 return 0;
1975 }
1976
1977 static struct file_operations kvm_chardev_ops = {
1978 .open = kvm_dev_open,
1979 .release = kvm_dev_release,
1980 .unlocked_ioctl = kvm_dev_ioctl,
1981 .compat_ioctl = kvm_dev_ioctl,
1982 .mmap = kvm_dev_mmap,
1983 };
1984
1985 static struct miscdevice kvm_dev = {
1986 MISC_DYNAMIC_MINOR,
1987 "kvm",
1988 &kvm_chardev_ops,
1989 };
1990
1991 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
1992 void *v)
1993 {
1994 if (val == SYS_RESTART) {
1995 /*
1996 * Some (well, at least mine) BIOSes hang on reboot if
1997 * in vmx root mode.
1998 */
1999 printk(KERN_INFO "kvm: exiting hardware virtualization\n");
2000 on_each_cpu(kvm_arch_ops->hardware_disable, 0, 0, 1);
2001 }
2002 return NOTIFY_OK;
2003 }
2004
2005 static struct notifier_block kvm_reboot_notifier = {
2006 .notifier_call = kvm_reboot,
2007 .priority = 0,
2008 };
2009
2010 static __init void kvm_init_debug(void)
2011 {
2012 struct kvm_stats_debugfs_item *p;
2013
2014 debugfs_dir = debugfs_create_dir("kvm", 0);
2015 for (p = debugfs_entries; p->name; ++p)
2016 p->dentry = debugfs_create_u32(p->name, 0444, debugfs_dir,
2017 p->data);
2018 }
2019
2020 static void kvm_exit_debug(void)
2021 {
2022 struct kvm_stats_debugfs_item *p;
2023
2024 for (p = debugfs_entries; p->name; ++p)
2025 debugfs_remove(p->dentry);
2026 debugfs_remove(debugfs_dir);
2027 }
2028
2029 hpa_t bad_page_address;
2030
2031 int kvm_init_arch(struct kvm_arch_ops *ops, struct module *module)
2032 {
2033 int r;
2034
2035 if (kvm_arch_ops) {
2036 printk(KERN_ERR "kvm: already loaded the other module\n");
2037 return -EEXIST;
2038 }
2039
2040 if (!ops->cpu_has_kvm_support()) {
2041 printk(KERN_ERR "kvm: no hardware support\n");
2042 return -EOPNOTSUPP;
2043 }
2044 if (ops->disabled_by_bios()) {
2045 printk(KERN_ERR "kvm: disabled by bios\n");
2046 return -EOPNOTSUPP;
2047 }
2048
2049 kvm_arch_ops = ops;
2050
2051 r = kvm_arch_ops->hardware_setup();
2052 if (r < 0)
2053 return r;
2054
2055 on_each_cpu(kvm_arch_ops->hardware_enable, 0, 0, 1);
2056 register_reboot_notifier(&kvm_reboot_notifier);
2057
2058 kvm_chardev_ops.owner = module;
2059
2060 r = misc_register(&kvm_dev);
2061 if (r) {
2062 printk (KERN_ERR "kvm: misc device register failed\n");
2063 goto out_free;
2064 }
2065
2066 return r;
2067
2068 out_free:
2069 unregister_reboot_notifier(&kvm_reboot_notifier);
2070 on_each_cpu(kvm_arch_ops->hardware_disable, 0, 0, 1);
2071 kvm_arch_ops->hardware_unsetup();
2072 return r;
2073 }
2074
2075 void kvm_exit_arch(void)
2076 {
2077 misc_deregister(&kvm_dev);
2078
2079 unregister_reboot_notifier(&kvm_reboot_notifier);
2080 on_each_cpu(kvm_arch_ops->hardware_disable, 0, 0, 1);
2081 kvm_arch_ops->hardware_unsetup();
2082 kvm_arch_ops = NULL;
2083 }
2084
2085 static __init int kvm_init(void)
2086 {
2087 static struct page *bad_page;
2088 int r = 0;
2089
2090 kvm_init_debug();
2091
2092 kvm_init_msr_list();
2093
2094 if ((bad_page = alloc_page(GFP_KERNEL)) == NULL) {
2095 r = -ENOMEM;
2096 goto out;
2097 }
2098
2099 bad_page_address = page_to_pfn(bad_page) << PAGE_SHIFT;
2100 memset(__va(bad_page_address), 0, PAGE_SIZE);
2101
2102 return r;
2103
2104 out:
2105 kvm_exit_debug();
2106 return r;
2107 }
2108
2109 static __exit void kvm_exit(void)
2110 {
2111 kvm_exit_debug();
2112 __free_page(pfn_to_page(bad_page_address >> PAGE_SHIFT));
2113 }
2114
2115 module_init(kvm_init)
2116 module_exit(kvm_exit)
2117
2118 EXPORT_SYMBOL_GPL(kvm_init_arch);
2119 EXPORT_SYMBOL_GPL(kvm_exit_arch);
Community contributions to the Openmoko Kernel