2 * Kernel-based Virtual Machine driver for Linux
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
7 * Copyright (C) 2006 Qumranet, Inc.
10 * Avi Kivity <avi@qumranet.com>
11 * Yaniv Kamay <yaniv@qumranet.com>
13 * This work is licensed under the terms of the GNU GPL, version 2. See
14 * the COPYING file in the top-level directory.
20 #include <linux/kvm_host.h>
21 #include <linux/kvm.h>
22 #include <linux/module.h>
23 #include <linux/errno.h>
24 #include <linux/percpu.h>
25 #include <linux/gfp.h>
27 #include <linux/miscdevice.h>
28 #include <linux/vmalloc.h>
29 #include <linux/reboot.h>
30 #include <linux/debugfs.h>
31 #include <linux/highmem.h>
32 #include <linux/file.h>
33 #include <linux/sysdev.h>
34 #include <linux/cpu.h>
35 #include <linux/sched.h>
36 #include <linux/cpumask.h>
37 #include <linux/smp.h>
38 #include <linux/anon_inodes.h>
39 #include <linux/profile.h>
40 #include <linux/kvm_para.h>
41 #include <linux/pagemap.h>
42 #include <linux/mman.h>
43 #include <linux/swap.h>
45 #include <asm/processor.h>
47 #include <asm/uaccess.h>
48 #include <asm/pgtable.h>
50 MODULE_AUTHOR("Qumranet");
51 MODULE_LICENSE("GPL");
53 DEFINE_SPINLOCK(kvm_lock
);
56 static cpumask_t cpus_hardware_enabled
;
58 struct kmem_cache
*kvm_vcpu_cache
;
59 EXPORT_SYMBOL_GPL(kvm_vcpu_cache
);
61 static __read_mostly
struct preempt_ops kvm_preempt_ops
;
63 struct dentry
*kvm_debugfs_dir
;
65 static long kvm_vcpu_ioctl(struct file
*file
, unsigned int ioctl
,
68 static inline int valid_vcpu(int n
)
70 return likely(n
>= 0 && n
< KVM_MAX_VCPUS
);
74 * Switches to specified vcpu, until a matching vcpu_put()
76 void vcpu_load(struct kvm_vcpu
*vcpu
)
80 mutex_lock(&vcpu
->mutex
);
82 preempt_notifier_register(&vcpu
->preempt_notifier
);
83 kvm_arch_vcpu_load(vcpu
, cpu
);
87 void vcpu_put(struct kvm_vcpu
*vcpu
)
90 kvm_arch_vcpu_put(vcpu
);
91 preempt_notifier_unregister(&vcpu
->preempt_notifier
);
93 mutex_unlock(&vcpu
->mutex
);
96 static void ack_flush(void *_completed
)
100 void kvm_flush_remote_tlbs(struct kvm
*kvm
)
104 struct kvm_vcpu
*vcpu
;
107 for (i
= 0; i
< KVM_MAX_VCPUS
; ++i
) {
108 vcpu
= kvm
->vcpus
[i
];
111 if (test_and_set_bit(KVM_REQ_TLB_FLUSH
, &vcpu
->requests
))
114 if (cpu
!= -1 && cpu
!= raw_smp_processor_id())
117 if (cpus_empty(cpus
))
119 ++kvm
->stat
.remote_tlb_flush
;
120 smp_call_function_mask(cpus
, ack_flush
, NULL
, 1);
123 void kvm_reload_remote_mmus(struct kvm
*kvm
)
127 struct kvm_vcpu
*vcpu
;
130 for (i
= 0; i
< KVM_MAX_VCPUS
; ++i
) {
131 vcpu
= kvm
->vcpus
[i
];
134 if (test_and_set_bit(KVM_REQ_MMU_RELOAD
, &vcpu
->requests
))
137 if (cpu
!= -1 && cpu
!= raw_smp_processor_id())
140 if (cpus_empty(cpus
))
142 smp_call_function_mask(cpus
, ack_flush
, NULL
, 1);
146 int kvm_vcpu_init(struct kvm_vcpu
*vcpu
, struct kvm
*kvm
, unsigned id
)
151 mutex_init(&vcpu
->mutex
);
155 init_waitqueue_head(&vcpu
->wq
);
157 page
= alloc_page(GFP_KERNEL
| __GFP_ZERO
);
162 vcpu
->run
= page_address(page
);
164 r
= kvm_arch_vcpu_init(vcpu
);
170 free_page((unsigned long)vcpu
->run
);
174 EXPORT_SYMBOL_GPL(kvm_vcpu_init
);
176 void kvm_vcpu_uninit(struct kvm_vcpu
*vcpu
)
178 kvm_arch_vcpu_uninit(vcpu
);
179 free_page((unsigned long)vcpu
->run
);
181 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit
);
183 static struct kvm
*kvm_create_vm(void)
185 struct kvm
*kvm
= kvm_arch_create_vm();
190 kvm
->mm
= current
->mm
;
191 atomic_inc(&kvm
->mm
->mm_count
);
192 spin_lock_init(&kvm
->mmu_lock
);
193 kvm_io_bus_init(&kvm
->pio_bus
);
194 mutex_init(&kvm
->lock
);
195 kvm_io_bus_init(&kvm
->mmio_bus
);
196 init_rwsem(&kvm
->slots_lock
);
197 atomic_set(&kvm
->users_count
, 1);
198 spin_lock(&kvm_lock
);
199 list_add(&kvm
->vm_list
, &vm_list
);
200 spin_unlock(&kvm_lock
);
206 * Free any memory in @free but not in @dont.
208 static void kvm_free_physmem_slot(struct kvm_memory_slot
*free
,
209 struct kvm_memory_slot
*dont
)
211 if (!dont
|| free
->rmap
!= dont
->rmap
)
214 if (!dont
|| free
->dirty_bitmap
!= dont
->dirty_bitmap
)
215 vfree(free
->dirty_bitmap
);
217 if (!dont
|| free
->lpage_info
!= dont
->lpage_info
)
218 vfree(free
->lpage_info
);
221 free
->dirty_bitmap
= NULL
;
223 free
->lpage_info
= NULL
;
226 void kvm_free_physmem(struct kvm
*kvm
)
230 for (i
= 0; i
< kvm
->nmemslots
; ++i
)
231 kvm_free_physmem_slot(&kvm
->memslots
[i
], NULL
);
234 static void kvm_destroy_vm(struct kvm
*kvm
)
236 struct mm_struct
*mm
= kvm
->mm
;
238 spin_lock(&kvm_lock
);
239 list_del(&kvm
->vm_list
);
240 spin_unlock(&kvm_lock
);
241 kvm_io_bus_destroy(&kvm
->pio_bus
);
242 kvm_io_bus_destroy(&kvm
->mmio_bus
);
243 kvm_arch_destroy_vm(kvm
);
247 void kvm_get_kvm(struct kvm
*kvm
)
249 atomic_inc(&kvm
->users_count
);
251 EXPORT_SYMBOL_GPL(kvm_get_kvm
);
253 void kvm_put_kvm(struct kvm
*kvm
)
255 if (atomic_dec_and_test(&kvm
->users_count
))
258 EXPORT_SYMBOL_GPL(kvm_put_kvm
);
261 static int kvm_vm_release(struct inode
*inode
, struct file
*filp
)
263 struct kvm
*kvm
= filp
->private_data
;
270 * Allocate some memory and give it an address in the guest physical address
273 * Discontiguous memory is allowed, mostly for framebuffers.
275 * Must be called holding mmap_sem for write.
277 int __kvm_set_memory_region(struct kvm
*kvm
,
278 struct kvm_userspace_memory_region
*mem
,
283 unsigned long npages
;
285 struct kvm_memory_slot
*memslot
;
286 struct kvm_memory_slot old
, new;
289 /* General sanity checks */
290 if (mem
->memory_size
& (PAGE_SIZE
- 1))
292 if (mem
->guest_phys_addr
& (PAGE_SIZE
- 1))
294 if (mem
->slot
>= KVM_MEMORY_SLOTS
+ KVM_PRIVATE_MEM_SLOTS
)
296 if (mem
->guest_phys_addr
+ mem
->memory_size
< mem
->guest_phys_addr
)
299 memslot
= &kvm
->memslots
[mem
->slot
];
300 base_gfn
= mem
->guest_phys_addr
>> PAGE_SHIFT
;
301 npages
= mem
->memory_size
>> PAGE_SHIFT
;
304 mem
->flags
&= ~KVM_MEM_LOG_DIRTY_PAGES
;
306 new = old
= *memslot
;
308 new.base_gfn
= base_gfn
;
310 new.flags
= mem
->flags
;
312 /* Disallow changing a memory slot's size. */
314 if (npages
&& old
.npages
&& npages
!= old
.npages
)
317 /* Check for overlaps */
319 for (i
= 0; i
< KVM_MEMORY_SLOTS
; ++i
) {
320 struct kvm_memory_slot
*s
= &kvm
->memslots
[i
];
324 if (!((base_gfn
+ npages
<= s
->base_gfn
) ||
325 (base_gfn
>= s
->base_gfn
+ s
->npages
)))
329 /* Free page dirty bitmap if unneeded */
330 if (!(new.flags
& KVM_MEM_LOG_DIRTY_PAGES
))
331 new.dirty_bitmap
= NULL
;
335 /* Allocate if a slot is being created */
336 if (npages
&& !new.rmap
) {
337 new.rmap
= vmalloc(npages
* sizeof(struct page
*));
342 memset(new.rmap
, 0, npages
* sizeof(*new.rmap
));
344 new.user_alloc
= user_alloc
;
345 new.userspace_addr
= mem
->userspace_addr
;
347 if (npages
&& !new.lpage_info
) {
348 int largepages
= npages
/ KVM_PAGES_PER_HPAGE
;
349 if (npages
% KVM_PAGES_PER_HPAGE
)
351 if (base_gfn
% KVM_PAGES_PER_HPAGE
)
354 new.lpage_info
= vmalloc(largepages
* sizeof(*new.lpage_info
));
359 memset(new.lpage_info
, 0, largepages
* sizeof(*new.lpage_info
));
361 if (base_gfn
% KVM_PAGES_PER_HPAGE
)
362 new.lpage_info
[0].write_count
= 1;
363 if ((base_gfn
+npages
) % KVM_PAGES_PER_HPAGE
)
364 new.lpage_info
[largepages
-1].write_count
= 1;
367 /* Allocate page dirty bitmap if needed */
368 if ((new.flags
& KVM_MEM_LOG_DIRTY_PAGES
) && !new.dirty_bitmap
) {
369 unsigned dirty_bytes
= ALIGN(npages
, BITS_PER_LONG
) / 8;
371 new.dirty_bitmap
= vmalloc(dirty_bytes
);
372 if (!new.dirty_bitmap
)
374 memset(new.dirty_bitmap
, 0, dirty_bytes
);
377 if (mem
->slot
>= kvm
->nmemslots
)
378 kvm
->nmemslots
= mem
->slot
+ 1;
381 kvm_arch_flush_shadow(kvm
);
385 r
= kvm_arch_set_memory_region(kvm
, mem
, old
, user_alloc
);
391 kvm_free_physmem_slot(&old
, &new);
395 kvm_free_physmem_slot(&new, &old
);
400 EXPORT_SYMBOL_GPL(__kvm_set_memory_region
);
402 int kvm_set_memory_region(struct kvm
*kvm
,
403 struct kvm_userspace_memory_region
*mem
,
408 down_write(&kvm
->slots_lock
);
409 r
= __kvm_set_memory_region(kvm
, mem
, user_alloc
);
410 up_write(&kvm
->slots_lock
);
413 EXPORT_SYMBOL_GPL(kvm_set_memory_region
);
415 int kvm_vm_ioctl_set_memory_region(struct kvm
*kvm
,
417 kvm_userspace_memory_region
*mem
,
420 if (mem
->slot
>= KVM_MEMORY_SLOTS
)
422 return kvm_set_memory_region(kvm
, mem
, user_alloc
);
425 int kvm_get_dirty_log(struct kvm
*kvm
,
426 struct kvm_dirty_log
*log
, int *is_dirty
)
428 struct kvm_memory_slot
*memslot
;
431 unsigned long any
= 0;
434 if (log
->slot
>= KVM_MEMORY_SLOTS
)
437 memslot
= &kvm
->memslots
[log
->slot
];
439 if (!memslot
->dirty_bitmap
)
442 n
= ALIGN(memslot
->npages
, BITS_PER_LONG
) / 8;
444 for (i
= 0; !any
&& i
< n
/sizeof(long); ++i
)
445 any
= memslot
->dirty_bitmap
[i
];
448 if (copy_to_user(log
->dirty_bitmap
, memslot
->dirty_bitmap
, n
))
459 int is_error_page(struct page
*page
)
461 return page
== bad_page
;
463 EXPORT_SYMBOL_GPL(is_error_page
);
465 int is_error_pfn(pfn_t pfn
)
467 return pfn
== bad_pfn
;
469 EXPORT_SYMBOL_GPL(is_error_pfn
);
471 static inline unsigned long bad_hva(void)
476 int kvm_is_error_hva(unsigned long addr
)
478 return addr
== bad_hva();
480 EXPORT_SYMBOL_GPL(kvm_is_error_hva
);
482 static struct kvm_memory_slot
*__gfn_to_memslot(struct kvm
*kvm
, gfn_t gfn
)
486 for (i
= 0; i
< kvm
->nmemslots
; ++i
) {
487 struct kvm_memory_slot
*memslot
= &kvm
->memslots
[i
];
489 if (gfn
>= memslot
->base_gfn
490 && gfn
< memslot
->base_gfn
+ memslot
->npages
)
496 struct kvm_memory_slot
*gfn_to_memslot(struct kvm
*kvm
, gfn_t gfn
)
498 gfn
= unalias_gfn(kvm
, gfn
);
499 return __gfn_to_memslot(kvm
, gfn
);
502 int kvm_is_visible_gfn(struct kvm
*kvm
, gfn_t gfn
)
506 gfn
= unalias_gfn(kvm
, gfn
);
507 for (i
= 0; i
< KVM_MEMORY_SLOTS
; ++i
) {
508 struct kvm_memory_slot
*memslot
= &kvm
->memslots
[i
];
510 if (gfn
>= memslot
->base_gfn
511 && gfn
< memslot
->base_gfn
+ memslot
->npages
)
516 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn
);
518 unsigned long gfn_to_hva(struct kvm
*kvm
, gfn_t gfn
)
520 struct kvm_memory_slot
*slot
;
522 gfn
= unalias_gfn(kvm
, gfn
);
523 slot
= __gfn_to_memslot(kvm
, gfn
);
526 return (slot
->userspace_addr
+ (gfn
- slot
->base_gfn
) * PAGE_SIZE
);
528 EXPORT_SYMBOL_GPL(gfn_to_hva
);
531 * Requires current->mm->mmap_sem to be held
533 pfn_t
gfn_to_pfn(struct kvm
*kvm
, gfn_t gfn
)
535 struct page
*page
[1];
541 addr
= gfn_to_hva(kvm
, gfn
);
542 if (kvm_is_error_hva(addr
)) {
544 return page_to_pfn(bad_page
);
547 npages
= get_user_pages(current
, current
->mm
, addr
, 1, 1, 1, page
,
552 return page_to_pfn(bad_page
);
555 return page_to_pfn(page
[0]);
558 EXPORT_SYMBOL_GPL(gfn_to_pfn
);
560 struct page
*gfn_to_page(struct kvm
*kvm
, gfn_t gfn
)
562 return pfn_to_page(gfn_to_pfn(kvm
, gfn
));
565 EXPORT_SYMBOL_GPL(gfn_to_page
);
567 void kvm_release_page_clean(struct page
*page
)
569 kvm_release_pfn_clean(page_to_pfn(page
));
571 EXPORT_SYMBOL_GPL(kvm_release_page_clean
);
573 void kvm_release_pfn_clean(pfn_t pfn
)
575 put_page(pfn_to_page(pfn
));
577 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean
);
579 void kvm_release_page_dirty(struct page
*page
)
581 kvm_release_pfn_dirty(page_to_pfn(page
));
583 EXPORT_SYMBOL_GPL(kvm_release_page_dirty
);
585 void kvm_release_pfn_dirty(pfn_t pfn
)
587 kvm_set_pfn_dirty(pfn
);
588 kvm_release_pfn_clean(pfn
);
590 EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty
);
592 void kvm_set_page_dirty(struct page
*page
)
594 kvm_set_pfn_dirty(page_to_pfn(page
));
596 EXPORT_SYMBOL_GPL(kvm_set_page_dirty
);
598 void kvm_set_pfn_dirty(pfn_t pfn
)
600 struct page
*page
= pfn_to_page(pfn
);
601 if (!PageReserved(page
))
604 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty
);
606 void kvm_set_pfn_accessed(pfn_t pfn
)
608 mark_page_accessed(pfn_to_page(pfn
));
610 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed
);
612 void kvm_get_pfn(pfn_t pfn
)
614 get_page(pfn_to_page(pfn
));
616 EXPORT_SYMBOL_GPL(kvm_get_pfn
);
618 static int next_segment(unsigned long len
, int offset
)
620 if (len
> PAGE_SIZE
- offset
)
621 return PAGE_SIZE
- offset
;
626 int kvm_read_guest_page(struct kvm
*kvm
, gfn_t gfn
, void *data
, int offset
,
632 addr
= gfn_to_hva(kvm
, gfn
);
633 if (kvm_is_error_hva(addr
))
635 r
= copy_from_user(data
, (void __user
*)addr
+ offset
, len
);
640 EXPORT_SYMBOL_GPL(kvm_read_guest_page
);
642 int kvm_read_guest(struct kvm
*kvm
, gpa_t gpa
, void *data
, unsigned long len
)
644 gfn_t gfn
= gpa
>> PAGE_SHIFT
;
646 int offset
= offset_in_page(gpa
);
649 while ((seg
= next_segment(len
, offset
)) != 0) {
650 ret
= kvm_read_guest_page(kvm
, gfn
, data
, offset
, seg
);
660 EXPORT_SYMBOL_GPL(kvm_read_guest
);
662 int kvm_read_guest_atomic(struct kvm
*kvm
, gpa_t gpa
, void *data
,
667 gfn_t gfn
= gpa
>> PAGE_SHIFT
;
668 int offset
= offset_in_page(gpa
);
670 addr
= gfn_to_hva(kvm
, gfn
);
671 if (kvm_is_error_hva(addr
))
674 r
= __copy_from_user_inatomic(data
, (void __user
*)addr
+ offset
, len
);
680 EXPORT_SYMBOL(kvm_read_guest_atomic
);
682 int kvm_write_guest_page(struct kvm
*kvm
, gfn_t gfn
, const void *data
,
688 addr
= gfn_to_hva(kvm
, gfn
);
689 if (kvm_is_error_hva(addr
))
691 r
= copy_to_user((void __user
*)addr
+ offset
, data
, len
);
694 mark_page_dirty(kvm
, gfn
);
697 EXPORT_SYMBOL_GPL(kvm_write_guest_page
);
699 int kvm_write_guest(struct kvm
*kvm
, gpa_t gpa
, const void *data
,
702 gfn_t gfn
= gpa
>> PAGE_SHIFT
;
704 int offset
= offset_in_page(gpa
);
707 while ((seg
= next_segment(len
, offset
)) != 0) {
708 ret
= kvm_write_guest_page(kvm
, gfn
, data
, offset
, seg
);
719 int kvm_clear_guest_page(struct kvm
*kvm
, gfn_t gfn
, int offset
, int len
)
721 return kvm_write_guest_page(kvm
, gfn
, empty_zero_page
, offset
, len
);
723 EXPORT_SYMBOL_GPL(kvm_clear_guest_page
);
725 int kvm_clear_guest(struct kvm
*kvm
, gpa_t gpa
, unsigned long len
)
727 gfn_t gfn
= gpa
>> PAGE_SHIFT
;
729 int offset
= offset_in_page(gpa
);
732 while ((seg
= next_segment(len
, offset
)) != 0) {
733 ret
= kvm_clear_guest_page(kvm
, gfn
, offset
, seg
);
742 EXPORT_SYMBOL_GPL(kvm_clear_guest
);
744 void mark_page_dirty(struct kvm
*kvm
, gfn_t gfn
)
746 struct kvm_memory_slot
*memslot
;
748 gfn
= unalias_gfn(kvm
, gfn
);
749 memslot
= __gfn_to_memslot(kvm
, gfn
);
750 if (memslot
&& memslot
->dirty_bitmap
) {
751 unsigned long rel_gfn
= gfn
- memslot
->base_gfn
;
754 if (!test_bit(rel_gfn
, memslot
->dirty_bitmap
))
755 set_bit(rel_gfn
, memslot
->dirty_bitmap
);
760 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
762 void kvm_vcpu_block(struct kvm_vcpu
*vcpu
)
767 prepare_to_wait(&vcpu
->wq
, &wait
, TASK_INTERRUPTIBLE
);
769 if (kvm_cpu_has_interrupt(vcpu
))
771 if (kvm_cpu_has_pending_timer(vcpu
))
773 if (kvm_arch_vcpu_runnable(vcpu
))
775 if (signal_pending(current
))
783 finish_wait(&vcpu
->wq
, &wait
);
786 void kvm_resched(struct kvm_vcpu
*vcpu
)
792 EXPORT_SYMBOL_GPL(kvm_resched
);
794 static int kvm_vcpu_fault(struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
796 struct kvm_vcpu
*vcpu
= vma
->vm_file
->private_data
;
800 page
= virt_to_page(vcpu
->run
);
802 else if (vmf
->pgoff
== KVM_PIO_PAGE_OFFSET
)
803 page
= virt_to_page(vcpu
->arch
.pio_data
);
806 return VM_FAULT_SIGBUS
;
812 static struct vm_operations_struct kvm_vcpu_vm_ops
= {
813 .fault
= kvm_vcpu_fault
,
816 static int kvm_vcpu_mmap(struct file
*file
, struct vm_area_struct
*vma
)
818 vma
->vm_ops
= &kvm_vcpu_vm_ops
;
822 static int kvm_vcpu_release(struct inode
*inode
, struct file
*filp
)
824 struct kvm_vcpu
*vcpu
= filp
->private_data
;
826 kvm_put_kvm(vcpu
->kvm
);
830 static const struct file_operations kvm_vcpu_fops
= {
831 .release
= kvm_vcpu_release
,
832 .unlocked_ioctl
= kvm_vcpu_ioctl
,
833 .compat_ioctl
= kvm_vcpu_ioctl
,
834 .mmap
= kvm_vcpu_mmap
,
838 * Allocates an inode for the vcpu.
840 static int create_vcpu_fd(struct kvm_vcpu
*vcpu
)
842 int fd
= anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops
, vcpu
);
844 kvm_put_kvm(vcpu
->kvm
);
849 * Creates some virtual cpus. Good luck creating more than one.
851 static int kvm_vm_ioctl_create_vcpu(struct kvm
*kvm
, int n
)
854 struct kvm_vcpu
*vcpu
;
859 vcpu
= kvm_arch_vcpu_create(kvm
, n
);
861 return PTR_ERR(vcpu
);
863 preempt_notifier_init(&vcpu
->preempt_notifier
, &kvm_preempt_ops
);
865 r
= kvm_arch_vcpu_setup(vcpu
);
869 mutex_lock(&kvm
->lock
);
872 mutex_unlock(&kvm
->lock
);
875 kvm
->vcpus
[n
] = vcpu
;
876 mutex_unlock(&kvm
->lock
);
878 /* Now it's all set up, let userspace reach it */
880 r
= create_vcpu_fd(vcpu
);
886 mutex_lock(&kvm
->lock
);
887 kvm
->vcpus
[n
] = NULL
;
888 mutex_unlock(&kvm
->lock
);
890 kvm_arch_vcpu_destroy(vcpu
);
894 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu
*vcpu
, sigset_t
*sigset
)
897 sigdelsetmask(sigset
, sigmask(SIGKILL
)|sigmask(SIGSTOP
));
898 vcpu
->sigset_active
= 1;
899 vcpu
->sigset
= *sigset
;
901 vcpu
->sigset_active
= 0;
905 static long kvm_vcpu_ioctl(struct file
*filp
,
906 unsigned int ioctl
, unsigned long arg
)
908 struct kvm_vcpu
*vcpu
= filp
->private_data
;
909 void __user
*argp
= (void __user
*)arg
;
912 if (vcpu
->kvm
->mm
!= current
->mm
)
919 r
= kvm_arch_vcpu_ioctl_run(vcpu
, vcpu
->run
);
922 struct kvm_regs
*kvm_regs
;
925 kvm_regs
= kzalloc(sizeof(struct kvm_regs
), GFP_KERNEL
);
928 r
= kvm_arch_vcpu_ioctl_get_regs(vcpu
, kvm_regs
);
932 if (copy_to_user(argp
, kvm_regs
, sizeof(struct kvm_regs
)))
940 struct kvm_regs
*kvm_regs
;
943 kvm_regs
= kzalloc(sizeof(struct kvm_regs
), GFP_KERNEL
);
947 if (copy_from_user(kvm_regs
, argp
, sizeof(struct kvm_regs
)))
949 r
= kvm_arch_vcpu_ioctl_set_regs(vcpu
, kvm_regs
);
957 case KVM_GET_SREGS
: {
958 struct kvm_sregs kvm_sregs
;
960 memset(&kvm_sregs
, 0, sizeof kvm_sregs
);
961 r
= kvm_arch_vcpu_ioctl_get_sregs(vcpu
, &kvm_sregs
);
965 if (copy_to_user(argp
, &kvm_sregs
, sizeof kvm_sregs
))
970 case KVM_SET_SREGS
: {
971 struct kvm_sregs kvm_sregs
;
974 if (copy_from_user(&kvm_sregs
, argp
, sizeof kvm_sregs
))
976 r
= kvm_arch_vcpu_ioctl_set_sregs(vcpu
, &kvm_sregs
);
982 case KVM_GET_MP_STATE
: {
983 struct kvm_mp_state mp_state
;
985 r
= kvm_arch_vcpu_ioctl_get_mpstate(vcpu
, &mp_state
);
989 if (copy_to_user(argp
, &mp_state
, sizeof mp_state
))
994 case KVM_SET_MP_STATE
: {
995 struct kvm_mp_state mp_state
;
998 if (copy_from_user(&mp_state
, argp
, sizeof mp_state
))
1000 r
= kvm_arch_vcpu_ioctl_set_mpstate(vcpu
, &mp_state
);
1006 case KVM_TRANSLATE
: {
1007 struct kvm_translation tr
;
1010 if (copy_from_user(&tr
, argp
, sizeof tr
))
1012 r
= kvm_arch_vcpu_ioctl_translate(vcpu
, &tr
);
1016 if (copy_to_user(argp
, &tr
, sizeof tr
))
1021 case KVM_DEBUG_GUEST
: {
1022 struct kvm_debug_guest dbg
;
1025 if (copy_from_user(&dbg
, argp
, sizeof dbg
))
1027 r
= kvm_arch_vcpu_ioctl_debug_guest(vcpu
, &dbg
);
1033 case KVM_SET_SIGNAL_MASK
: {
1034 struct kvm_signal_mask __user
*sigmask_arg
= argp
;
1035 struct kvm_signal_mask kvm_sigmask
;
1036 sigset_t sigset
, *p
;
1041 if (copy_from_user(&kvm_sigmask
, argp
,
1042 sizeof kvm_sigmask
))
1045 if (kvm_sigmask
.len
!= sizeof sigset
)
1048 if (copy_from_user(&sigset
, sigmask_arg
->sigset
,
1053 r
= kvm_vcpu_ioctl_set_sigmask(vcpu
, &sigset
);
1059 memset(&fpu
, 0, sizeof fpu
);
1060 r
= kvm_arch_vcpu_ioctl_get_fpu(vcpu
, &fpu
);
1064 if (copy_to_user(argp
, &fpu
, sizeof fpu
))
1073 if (copy_from_user(&fpu
, argp
, sizeof fpu
))
1075 r
= kvm_arch_vcpu_ioctl_set_fpu(vcpu
, &fpu
);
1082 r
= kvm_arch_vcpu_ioctl(filp
, ioctl
, arg
);
1088 static long kvm_vm_ioctl(struct file
*filp
,
1089 unsigned int ioctl
, unsigned long arg
)
1091 struct kvm
*kvm
= filp
->private_data
;
1092 void __user
*argp
= (void __user
*)arg
;
1095 if (kvm
->mm
!= current
->mm
)
1098 case KVM_CREATE_VCPU
:
1099 r
= kvm_vm_ioctl_create_vcpu(kvm
, arg
);
1103 case KVM_SET_USER_MEMORY_REGION
: {
1104 struct kvm_userspace_memory_region kvm_userspace_mem
;
1107 if (copy_from_user(&kvm_userspace_mem
, argp
,
1108 sizeof kvm_userspace_mem
))
1111 r
= kvm_vm_ioctl_set_memory_region(kvm
, &kvm_userspace_mem
, 1);
1116 case KVM_GET_DIRTY_LOG
: {
1117 struct kvm_dirty_log log
;
1120 if (copy_from_user(&log
, argp
, sizeof log
))
1122 r
= kvm_vm_ioctl_get_dirty_log(kvm
, &log
);
1128 r
= kvm_arch_vm_ioctl(filp
, ioctl
, arg
);
1134 static int kvm_vm_fault(struct vm_area_struct
*vma
, struct vm_fault
*vmf
)
1136 struct kvm
*kvm
= vma
->vm_file
->private_data
;
1139 if (!kvm_is_visible_gfn(kvm
, vmf
->pgoff
))
1140 return VM_FAULT_SIGBUS
;
1141 page
= gfn_to_page(kvm
, vmf
->pgoff
);
1142 if (is_error_page(page
)) {
1143 kvm_release_page_clean(page
);
1144 return VM_FAULT_SIGBUS
;
1150 static struct vm_operations_struct kvm_vm_vm_ops
= {
1151 .fault
= kvm_vm_fault
,
1154 static int kvm_vm_mmap(struct file
*file
, struct vm_area_struct
*vma
)
1156 vma
->vm_ops
= &kvm_vm_vm_ops
;
1160 static const struct file_operations kvm_vm_fops
= {
1161 .release
= kvm_vm_release
,
1162 .unlocked_ioctl
= kvm_vm_ioctl
,
1163 .compat_ioctl
= kvm_vm_ioctl
,
1164 .mmap
= kvm_vm_mmap
,
1167 static int kvm_dev_ioctl_create_vm(void)
1172 kvm
= kvm_create_vm();
1174 return PTR_ERR(kvm
);
1175 fd
= anon_inode_getfd("kvm-vm", &kvm_vm_fops
, kvm
);
1182 static long kvm_dev_ioctl(struct file
*filp
,
1183 unsigned int ioctl
, unsigned long arg
)
1185 void __user
*argp
= (void __user
*)arg
;
1189 case KVM_GET_API_VERSION
:
1193 r
= KVM_API_VERSION
;
1199 r
= kvm_dev_ioctl_create_vm();
1201 case KVM_CHECK_EXTENSION
:
1202 r
= kvm_dev_ioctl_check_extension((long)argp
);
1204 case KVM_GET_VCPU_MMAP_SIZE
:
1208 r
= PAGE_SIZE
; /* struct kvm_run */
1210 r
+= PAGE_SIZE
; /* pio data page */
1213 case KVM_TRACE_ENABLE
:
1214 case KVM_TRACE_PAUSE
:
1215 case KVM_TRACE_DISABLE
:
1216 r
= kvm_trace_ioctl(ioctl
, arg
);
1219 return kvm_arch_dev_ioctl(filp
, ioctl
, arg
);
1225 static struct file_operations kvm_chardev_ops
= {
1226 .unlocked_ioctl
= kvm_dev_ioctl
,
1227 .compat_ioctl
= kvm_dev_ioctl
,
1230 static struct miscdevice kvm_dev
= {
1236 static void hardware_enable(void *junk
)
1238 int cpu
= raw_smp_processor_id();
1240 if (cpu_isset(cpu
, cpus_hardware_enabled
))
1242 cpu_set(cpu
, cpus_hardware_enabled
);
1243 kvm_arch_hardware_enable(NULL
);
1246 static void hardware_disable(void *junk
)
1248 int cpu
= raw_smp_processor_id();
1250 if (!cpu_isset(cpu
, cpus_hardware_enabled
))
1252 cpu_clear(cpu
, cpus_hardware_enabled
);
1253 decache_vcpus_on_cpu(cpu
);
1254 kvm_arch_hardware_disable(NULL
);
1257 static int kvm_cpu_hotplug(struct notifier_block
*notifier
, unsigned long val
,
1262 val
&= ~CPU_TASKS_FROZEN
;
1265 printk(KERN_INFO
"kvm: disabling virtualization on CPU%d\n",
1267 hardware_disable(NULL
);
1269 case CPU_UP_CANCELED
:
1270 printk(KERN_INFO
"kvm: disabling virtualization on CPU%d\n",
1272 smp_call_function_single(cpu
, hardware_disable
, NULL
, 0, 1);
1275 printk(KERN_INFO
"kvm: enabling virtualization on CPU%d\n",
1277 smp_call_function_single(cpu
, hardware_enable
, NULL
, 0, 1);
1283 static int kvm_reboot(struct notifier_block
*notifier
, unsigned long val
,
1286 if (val
== SYS_RESTART
) {
1288 * Some (well, at least mine) BIOSes hang on reboot if
1291 printk(KERN_INFO
"kvm: exiting hardware virtualization\n");
1292 on_each_cpu(hardware_disable
, NULL
, 0, 1);
1297 static struct notifier_block kvm_reboot_notifier
= {
1298 .notifier_call
= kvm_reboot
,
1302 void kvm_io_bus_init(struct kvm_io_bus
*bus
)
1304 memset(bus
, 0, sizeof(*bus
));
1307 void kvm_io_bus_destroy(struct kvm_io_bus
*bus
)
1311 for (i
= 0; i
< bus
->dev_count
; i
++) {
1312 struct kvm_io_device
*pos
= bus
->devs
[i
];
1314 kvm_iodevice_destructor(pos
);
1318 struct kvm_io_device
*kvm_io_bus_find_dev(struct kvm_io_bus
*bus
, gpa_t addr
)
1322 for (i
= 0; i
< bus
->dev_count
; i
++) {
1323 struct kvm_io_device
*pos
= bus
->devs
[i
];
1325 if (pos
->in_range(pos
, addr
))
1332 void kvm_io_bus_register_dev(struct kvm_io_bus
*bus
, struct kvm_io_device
*dev
)
1334 BUG_ON(bus
->dev_count
> (NR_IOBUS_DEVS
-1));
1336 bus
->devs
[bus
->dev_count
++] = dev
;
1339 static struct notifier_block kvm_cpu_notifier
= {
1340 .notifier_call
= kvm_cpu_hotplug
,
1341 .priority
= 20, /* must be > scheduler priority */
1344 static int vm_stat_get(void *_offset
, u64
*val
)
1346 unsigned offset
= (long)_offset
;
1350 spin_lock(&kvm_lock
);
1351 list_for_each_entry(kvm
, &vm_list
, vm_list
)
1352 *val
+= *(u32
*)((void *)kvm
+ offset
);
1353 spin_unlock(&kvm_lock
);
1357 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops
, vm_stat_get
, NULL
, "%llu\n");
1359 static int vcpu_stat_get(void *_offset
, u64
*val
)
1361 unsigned offset
= (long)_offset
;
1363 struct kvm_vcpu
*vcpu
;
1367 spin_lock(&kvm_lock
);
1368 list_for_each_entry(kvm
, &vm_list
, vm_list
)
1369 for (i
= 0; i
< KVM_MAX_VCPUS
; ++i
) {
1370 vcpu
= kvm
->vcpus
[i
];
1372 *val
+= *(u32
*)((void *)vcpu
+ offset
);
1374 spin_unlock(&kvm_lock
);
1378 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops
, vcpu_stat_get
, NULL
, "%llu\n");
1380 static struct file_operations
*stat_fops
[] = {
1381 [KVM_STAT_VCPU
] = &vcpu_stat_fops
,
1382 [KVM_STAT_VM
] = &vm_stat_fops
,
1385 static void kvm_init_debug(void)
1387 struct kvm_stats_debugfs_item
*p
;
1389 kvm_debugfs_dir
= debugfs_create_dir("kvm", NULL
);
1390 for (p
= debugfs_entries
; p
->name
; ++p
)
1391 p
->dentry
= debugfs_create_file(p
->name
, 0444, kvm_debugfs_dir
,
1392 (void *)(long)p
->offset
,
1393 stat_fops
[p
->kind
]);
1396 static void kvm_exit_debug(void)
1398 struct kvm_stats_debugfs_item
*p
;
1400 for (p
= debugfs_entries
; p
->name
; ++p
)
1401 debugfs_remove(p
->dentry
);
1402 debugfs_remove(kvm_debugfs_dir
);
1405 static int kvm_suspend(struct sys_device
*dev
, pm_message_t state
)
1407 hardware_disable(NULL
);
1411 static int kvm_resume(struct sys_device
*dev
)
1413 hardware_enable(NULL
);
1417 static struct sysdev_class kvm_sysdev_class
= {
1419 .suspend
= kvm_suspend
,
1420 .resume
= kvm_resume
,
1423 static struct sys_device kvm_sysdev
= {
1425 .cls
= &kvm_sysdev_class
,
1428 struct page
*bad_page
;
1432 struct kvm_vcpu
*preempt_notifier_to_vcpu(struct preempt_notifier
*pn
)
1434 return container_of(pn
, struct kvm_vcpu
, preempt_notifier
);
1437 static void kvm_sched_in(struct preempt_notifier
*pn
, int cpu
)
1439 struct kvm_vcpu
*vcpu
= preempt_notifier_to_vcpu(pn
);
1441 kvm_arch_vcpu_load(vcpu
, cpu
);
1444 static void kvm_sched_out(struct preempt_notifier
*pn
,
1445 struct task_struct
*next
)
1447 struct kvm_vcpu
*vcpu
= preempt_notifier_to_vcpu(pn
);
1449 kvm_arch_vcpu_put(vcpu
);
1452 int kvm_init(void *opaque
, unsigned int vcpu_size
,
1453 struct module
*module
)
1460 r
= kvm_arch_init(opaque
);
1464 bad_page
= alloc_page(GFP_KERNEL
| __GFP_ZERO
);
1466 if (bad_page
== NULL
) {
1471 bad_pfn
= page_to_pfn(bad_page
);
1473 r
= kvm_arch_hardware_setup();
1477 for_each_online_cpu(cpu
) {
1478 smp_call_function_single(cpu
,
1479 kvm_arch_check_processor_compat
,
1485 on_each_cpu(hardware_enable
, NULL
, 0, 1);
1486 r
= register_cpu_notifier(&kvm_cpu_notifier
);
1489 register_reboot_notifier(&kvm_reboot_notifier
);
1491 r
= sysdev_class_register(&kvm_sysdev_class
);
1495 r
= sysdev_register(&kvm_sysdev
);
1499 /* A kmem cache lets us meet the alignment requirements of fx_save. */
1500 kvm_vcpu_cache
= kmem_cache_create("kvm_vcpu", vcpu_size
,
1501 __alignof__(struct kvm_vcpu
),
1503 if (!kvm_vcpu_cache
) {
1508 kvm_chardev_ops
.owner
= module
;
1510 r
= misc_register(&kvm_dev
);
1512 printk(KERN_ERR
"kvm: misc device register failed\n");
1516 kvm_preempt_ops
.sched_in
= kvm_sched_in
;
1517 kvm_preempt_ops
.sched_out
= kvm_sched_out
;
1522 kmem_cache_destroy(kvm_vcpu_cache
);
1524 sysdev_unregister(&kvm_sysdev
);
1526 sysdev_class_unregister(&kvm_sysdev_class
);
1528 unregister_reboot_notifier(&kvm_reboot_notifier
);
1529 unregister_cpu_notifier(&kvm_cpu_notifier
);
1531 on_each_cpu(hardware_disable
, NULL
, 0, 1);
1533 kvm_arch_hardware_unsetup();
1535 __free_page(bad_page
);
1542 EXPORT_SYMBOL_GPL(kvm_init
);
1546 kvm_trace_cleanup();
1547 misc_deregister(&kvm_dev
);
1548 kmem_cache_destroy(kvm_vcpu_cache
);
1549 sysdev_unregister(&kvm_sysdev
);
1550 sysdev_class_unregister(&kvm_sysdev_class
);
1551 unregister_reboot_notifier(&kvm_reboot_notifier
);
1552 unregister_cpu_notifier(&kvm_cpu_notifier
);
1553 on_each_cpu(hardware_disable
, NULL
, 0, 1);
1554 kvm_arch_hardware_unsetup();
1557 __free_page(bad_page
);
1559 EXPORT_SYMBOL_GPL(kvm_exit
);