?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
lockdep: fix overflow in the hlock shrinkage code
[linux-2.6/kvm.git] / virt / kvm / kvm_main.c
bloba845890b680062365930b89a63ba4536c0fa6670
1 /*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * Copyright (C) 2006 Qumranet, Inc.
8 *
9 * Authors:
10 * Avi Kivity <avi@qumranet.com>
11 * Yaniv Kamay <yaniv@qumranet.com>
12 *
13 * This work is licensed under the terms of the GNU GPL, version 2. See
14 * the COPYING file in the top-level directory.
15 *
16 */
17
18 #include "iodev.h"
19
20 #include <linux/kvm_host.h>
21 #include <linux/kvm.h>
22 #include <linux/module.h>
23 #include <linux/errno.h>
24 #include <linux/percpu.h>
25 #include <linux/gfp.h>
26 #include <linux/mm.h>
27 #include <linux/miscdevice.h>
28 #include <linux/vmalloc.h>
29 #include <linux/reboot.h>
30 #include <linux/debugfs.h>
31 #include <linux/highmem.h>
32 #include <linux/file.h>
33 #include <linux/sysdev.h>
34 #include <linux/cpu.h>
35 #include <linux/sched.h>
36 #include <linux/cpumask.h>
37 #include <linux/smp.h>
38 #include <linux/anon_inodes.h>
39 #include <linux/profile.h>
40 #include <linux/kvm_para.h>
41 #include <linux/pagemap.h>
42 #include <linux/mman.h>
43 #include <linux/swap.h>
44
45 #include <asm/processor.h>
46 #include <asm/io.h>
47 #include <asm/uaccess.h>
48 #include <asm/pgtable.h>
49
50 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
51 #include "coalesced_mmio.h"
52 #endif
53
54 MODULE_AUTHOR("Qumranet");
55 MODULE_LICENSE("GPL");
56
57 DEFINE_SPINLOCK(kvm_lock);
58 LIST_HEAD(vm_list);
59
60 static cpumask_t cpus_hardware_enabled;
61
62 struct kmem_cache *kvm_vcpu_cache;
63 EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
64
65 static __read_mostly struct preempt_ops kvm_preempt_ops;
66
67 struct dentry *kvm_debugfs_dir;
68
69 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
70 unsigned long arg);
71
72 bool kvm_rebooting;
73
74 static inline int valid_vcpu(int n)
75 {
76 return likely(n >= 0 && n < KVM_MAX_VCPUS);
77 }
78
79 /*
80 * Switches to specified vcpu, until a matching vcpu_put()
81 */
82 void vcpu_load(struct kvm_vcpu *vcpu)
83 {
84 int cpu;
85
86 mutex_lock(&vcpu->mutex);
87 cpu = get_cpu();
88 preempt_notifier_register(&vcpu->preempt_notifier);
89 kvm_arch_vcpu_load(vcpu, cpu);
90 put_cpu();
91 }
92
93 void vcpu_put(struct kvm_vcpu *vcpu)
94 {
95 preempt_disable();
96 kvm_arch_vcpu_put(vcpu);
97 preempt_notifier_unregister(&vcpu->preempt_notifier);
98 preempt_enable();
99 mutex_unlock(&vcpu->mutex);
100 }
101
102 static void ack_flush(void *_completed)
103 {
104 }
105
106 void kvm_flush_remote_tlbs(struct kvm *kvm)
107 {
108 int i, cpu, me;
109 cpumask_t cpus;
110 struct kvm_vcpu *vcpu;
111
112 me = get_cpu();
113 cpus_clear(cpus);
114 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
115 vcpu = kvm->vcpus[i];
116 if (!vcpu)
117 continue;
118 if (test_and_set_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests))
119 continue;
120 cpu = vcpu->cpu;
121 if (cpu != -1 && cpu != me)
122 cpu_set(cpu, cpus);
123 }
124 if (cpus_empty(cpus))
125 goto out;
126 ++kvm->stat.remote_tlb_flush;
127 smp_call_function_mask(cpus, ack_flush, NULL, 1);
128 out:
129 put_cpu();
130 }
131
132 void kvm_reload_remote_mmus(struct kvm *kvm)
133 {
134 int i, cpu, me;
135 cpumask_t cpus;
136 struct kvm_vcpu *vcpu;
137
138 me = get_cpu();
139 cpus_clear(cpus);
140 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
141 vcpu = kvm->vcpus[i];
142 if (!vcpu)
143 continue;
144 if (test_and_set_bit(KVM_REQ_MMU_RELOAD, &vcpu->requests))
145 continue;
146 cpu = vcpu->cpu;
147 if (cpu != -1 && cpu != me)
148 cpu_set(cpu, cpus);
149 }
150 if (cpus_empty(cpus))
151 goto out;
152 smp_call_function_mask(cpus, ack_flush, NULL, 1);
153 out:
154 put_cpu();
155 }
156
157
158 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
159 {
160 struct page *page;
161 int r;
162
163 mutex_init(&vcpu->mutex);
164 vcpu->cpu = -1;
165 vcpu->kvm = kvm;
166 vcpu->vcpu_id = id;
167 init_waitqueue_head(&vcpu->wq);
168
169 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
170 if (!page) {
171 r = -ENOMEM;
172 goto fail;
173 }
174 vcpu->run = page_address(page);
175
176 r = kvm_arch_vcpu_init(vcpu);
177 if (r < 0)
178 goto fail_free_run;
179 return 0;
180
181 fail_free_run:
182 free_page((unsigned long)vcpu->run);
183 fail:
184 return r;
185 }
186 EXPORT_SYMBOL_GPL(kvm_vcpu_init);
187
188 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
189 {
190 kvm_arch_vcpu_uninit(vcpu);
191 free_page((unsigned long)vcpu->run);
192 }
193 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
194
195 static struct kvm *kvm_create_vm(void)
196 {
197 struct kvm *kvm = kvm_arch_create_vm();
198 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
199 struct page *page;
200 #endif
201
202 if (IS_ERR(kvm))
203 goto out;
204
205 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
206 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
207 if (!page) {
208 kfree(kvm);
209 return ERR_PTR(-ENOMEM);
210 }
211 kvm->coalesced_mmio_ring =
212 (struct kvm_coalesced_mmio_ring *)page_address(page);
213 #endif
214
215 kvm->mm = current->mm;
216 atomic_inc(&kvm->mm->mm_count);
217 spin_lock_init(&kvm->mmu_lock);
218 kvm_io_bus_init(&kvm->pio_bus);
219 mutex_init(&kvm->lock);
220 kvm_io_bus_init(&kvm->mmio_bus);
221 init_rwsem(&kvm->slots_lock);
222 atomic_set(&kvm->users_count, 1);
223 spin_lock(&kvm_lock);
224 list_add(&kvm->vm_list, &vm_list);
225 spin_unlock(&kvm_lock);
226 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
227 kvm_coalesced_mmio_init(kvm);
228 #endif
229 out:
230 return kvm;
231 }
232
233 /*
234 * Free any memory in @free but not in @dont.
235 */
236 static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
237 struct kvm_memory_slot *dont)
238 {
239 if (!dont || free->rmap != dont->rmap)
240 vfree(free->rmap);
241
242 if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
243 vfree(free->dirty_bitmap);
244
245 if (!dont || free->lpage_info != dont->lpage_info)
246 vfree(free->lpage_info);
247
248 free->npages = 0;
249 free->dirty_bitmap = NULL;
250 free->rmap = NULL;
251 free->lpage_info = NULL;
252 }
253
254 void kvm_free_physmem(struct kvm *kvm)
255 {
256 int i;
257
258 for (i = 0; i < kvm->nmemslots; ++i)
259 kvm_free_physmem_slot(&kvm->memslots[i], NULL);
260 }
261
262 static void kvm_destroy_vm(struct kvm *kvm)
263 {
264 struct mm_struct *mm = kvm->mm;
265
266 spin_lock(&kvm_lock);
267 list_del(&kvm->vm_list);
268 spin_unlock(&kvm_lock);
269 kvm_io_bus_destroy(&kvm->pio_bus);
270 kvm_io_bus_destroy(&kvm->mmio_bus);
271 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
272 if (kvm->coalesced_mmio_ring != NULL)
273 free_page((unsigned long)kvm->coalesced_mmio_ring);
274 #endif
275 kvm_arch_destroy_vm(kvm);
276 mmdrop(mm);
277 }
278
279 void kvm_get_kvm(struct kvm *kvm)
280 {
281 atomic_inc(&kvm->users_count);
282 }
283 EXPORT_SYMBOL_GPL(kvm_get_kvm);
284
285 void kvm_put_kvm(struct kvm *kvm)
286 {
287 if (atomic_dec_and_test(&kvm->users_count))
288 kvm_destroy_vm(kvm);
289 }
290 EXPORT_SYMBOL_GPL(kvm_put_kvm);
291
292
293 static int kvm_vm_release(struct inode *inode, struct file *filp)
294 {
295 struct kvm *kvm = filp->private_data;
296
297 kvm_put_kvm(kvm);
298 return 0;
299 }
300
301 /*
302 * Allocate some memory and give it an address in the guest physical address
303 * space.
304 *
305 * Discontiguous memory is allowed, mostly for framebuffers.
306 *
307 * Must be called holding mmap_sem for write.
308 */
309 int __kvm_set_memory_region(struct kvm *kvm,
310 struct kvm_userspace_memory_region *mem,
311 int user_alloc)
312 {
313 int r;
314 gfn_t base_gfn;
315 unsigned long npages;
316 unsigned long i;
317 struct kvm_memory_slot *memslot;
318 struct kvm_memory_slot old, new;
319
320 r = -EINVAL;
321 /* General sanity checks */
322 if (mem->memory_size & (PAGE_SIZE - 1))
323 goto out;
324 if (mem->guest_phys_addr & (PAGE_SIZE - 1))
325 goto out;
326 if (mem->slot >= KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS)
327 goto out;
328 if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
329 goto out;
330
331 memslot = &kvm->memslots[mem->slot];
332 base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
333 npages = mem->memory_size >> PAGE_SHIFT;
334
335 if (!npages)
336 mem->flags &= ~KVM_MEM_LOG_DIRTY_PAGES;
337
338 new = old = *memslot;
339
340 new.base_gfn = base_gfn;
341 new.npages = npages;
342 new.flags = mem->flags;
343
344 /* Disallow changing a memory slot's size. */
345 r = -EINVAL;
346 if (npages && old.npages && npages != old.npages)
347 goto out_free;
348
349 /* Check for overlaps */
350 r = -EEXIST;
351 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
352 struct kvm_memory_slot *s = &kvm->memslots[i];
353
354 if (s == memslot)
355 continue;
356 if (!((base_gfn + npages <= s->base_gfn) ||
357 (base_gfn >= s->base_gfn + s->npages)))
358 goto out_free;
359 }
360
361 /* Free page dirty bitmap if unneeded */
362 if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
363 new.dirty_bitmap = NULL;
364
365 r = -ENOMEM;
366
367 /* Allocate if a slot is being created */
368 #ifndef CONFIG_S390
369 if (npages && !new.rmap) {
370 new.rmap = vmalloc(npages * sizeof(struct page *));
371
372 if (!new.rmap)
373 goto out_free;
374
375 memset(new.rmap, 0, npages * sizeof(*new.rmap));
376
377 new.user_alloc = user_alloc;
378 new.userspace_addr = mem->userspace_addr;
379 }
380 if (npages && !new.lpage_info) {
381 int largepages = npages / KVM_PAGES_PER_HPAGE;
382 if (npages % KVM_PAGES_PER_HPAGE)
383 largepages++;
384 if (base_gfn % KVM_PAGES_PER_HPAGE)
385 largepages++;
386
387 new.lpage_info = vmalloc(largepages * sizeof(*new.lpage_info));
388
389 if (!new.lpage_info)
390 goto out_free;
391
392 memset(new.lpage_info, 0, largepages * sizeof(*new.lpage_info));
393
394 if (base_gfn % KVM_PAGES_PER_HPAGE)
395 new.lpage_info[0].write_count = 1;
396 if ((base_gfn+npages) % KVM_PAGES_PER_HPAGE)
397 new.lpage_info[largepages-1].write_count = 1;
398 }
399
400 /* Allocate page dirty bitmap if needed */
401 if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
402 unsigned dirty_bytes = ALIGN(npages, BITS_PER_LONG) / 8;
403
404 new.dirty_bitmap = vmalloc(dirty_bytes);
405 if (!new.dirty_bitmap)
406 goto out_free;
407 memset(new.dirty_bitmap, 0, dirty_bytes);
408 }
409 #endif /* not defined CONFIG_S390 */
410
411 if (mem->slot >= kvm->nmemslots)
412 kvm->nmemslots = mem->slot + 1;
413
414 if (!npages)
415 kvm_arch_flush_shadow(kvm);
416
417 *memslot = new;
418
419 r = kvm_arch_set_memory_region(kvm, mem, old, user_alloc);
420 if (r) {
421 *memslot = old;
422 goto out_free;
423 }
424
425 kvm_free_physmem_slot(&old, &new);
426 return 0;
427
428 out_free:
429 kvm_free_physmem_slot(&new, &old);
430 out:
431 return r;
432
433 }
434 EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
435
436 int kvm_set_memory_region(struct kvm *kvm,
437 struct kvm_userspace_memory_region *mem,
438 int user_alloc)
439 {
440 int r;
441
442 down_write(&kvm->slots_lock);
443 r = __kvm_set_memory_region(kvm, mem, user_alloc);
444 up_write(&kvm->slots_lock);
445 return r;
446 }
447 EXPORT_SYMBOL_GPL(kvm_set_memory_region);
448
449 int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
450 struct
451 kvm_userspace_memory_region *mem,
452 int user_alloc)
453 {
454 if (mem->slot >= KVM_MEMORY_SLOTS)
455 return -EINVAL;
456 return kvm_set_memory_region(kvm, mem, user_alloc);
457 }
458
459 int kvm_get_dirty_log(struct kvm *kvm,
460 struct kvm_dirty_log *log, int *is_dirty)
461 {
462 struct kvm_memory_slot *memslot;
463 int r, i;
464 int n;
465 unsigned long any = 0;
466
467 r = -EINVAL;
468 if (log->slot >= KVM_MEMORY_SLOTS)
469 goto out;
470
471 memslot = &kvm->memslots[log->slot];
472 r = -ENOENT;
473 if (!memslot->dirty_bitmap)
474 goto out;
475
476 n = ALIGN(memslot->npages, BITS_PER_LONG) / 8;
477
478 for (i = 0; !any && i < n/sizeof(long); ++i)
479 any = memslot->dirty_bitmap[i];
480
481 r = -EFAULT;
482 if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
483 goto out;
484
485 if (any)
486 *is_dirty = 1;
487
488 r = 0;
489 out:
490 return r;
491 }
492
493 int is_error_page(struct page *page)
494 {
495 return page == bad_page;
496 }
497 EXPORT_SYMBOL_GPL(is_error_page);
498
499 int is_error_pfn(pfn_t pfn)
500 {
501 return pfn == bad_pfn;
502 }
503 EXPORT_SYMBOL_GPL(is_error_pfn);
504
505 static inline unsigned long bad_hva(void)
506 {
507 return PAGE_OFFSET;
508 }
509
510 int kvm_is_error_hva(unsigned long addr)
511 {
512 return addr == bad_hva();
513 }
514 EXPORT_SYMBOL_GPL(kvm_is_error_hva);
515
516 static struct kvm_memory_slot *__gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
517 {
518 int i;
519
520 for (i = 0; i < kvm->nmemslots; ++i) {
521 struct kvm_memory_slot *memslot = &kvm->memslots[i];
522
523 if (gfn >= memslot->base_gfn
524 && gfn < memslot->base_gfn + memslot->npages)
525 return memslot;
526 }
527 return NULL;
528 }
529
530 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
531 {
532 gfn = unalias_gfn(kvm, gfn);
533 return __gfn_to_memslot(kvm, gfn);
534 }
535
536 int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
537 {
538 int i;
539
540 gfn = unalias_gfn(kvm, gfn);
541 for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
542 struct kvm_memory_slot *memslot = &kvm->memslots[i];
543
544 if (gfn >= memslot->base_gfn
545 && gfn < memslot->base_gfn + memslot->npages)
546 return 1;
547 }
548 return 0;
549 }
550 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
551
552 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
553 {
554 struct kvm_memory_slot *slot;
555
556 gfn = unalias_gfn(kvm, gfn);
557 slot = __gfn_to_memslot(kvm, gfn);
558 if (!slot)
559 return bad_hva();
560 return (slot->userspace_addr + (gfn - slot->base_gfn) * PAGE_SIZE);
561 }
562 EXPORT_SYMBOL_GPL(gfn_to_hva);
563
564 /*
565 * Requires current->mm->mmap_sem to be held
566 */
567 pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
568 {
569 struct page *page[1];
570 unsigned long addr;
571 int npages;
572 pfn_t pfn;
573
574 might_sleep();
575
576 addr = gfn_to_hva(kvm, gfn);
577 if (kvm_is_error_hva(addr)) {
578 get_page(bad_page);
579 return page_to_pfn(bad_page);
580 }
581
582 npages = get_user_pages(current, current->mm, addr, 1, 1, 1, page,
583 NULL);
584
585 if (unlikely(npages != 1)) {
586 struct vm_area_struct *vma;
587
588 vma = find_vma(current->mm, addr);
589 if (vma == NULL || addr < vma->vm_start ||
590 !(vma->vm_flags & VM_PFNMAP)) {
591 get_page(bad_page);
592 return page_to_pfn(bad_page);
593 }
594
595 pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
596 BUG_ON(pfn_valid(pfn));
597 } else
598 pfn = page_to_pfn(page[0]);
599
600 return pfn;
601 }
602
603 EXPORT_SYMBOL_GPL(gfn_to_pfn);
604
605 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
606 {
607 pfn_t pfn;
608
609 pfn = gfn_to_pfn(kvm, gfn);
610 if (pfn_valid(pfn))
611 return pfn_to_page(pfn);
612
613 WARN_ON(!pfn_valid(pfn));
614
615 get_page(bad_page);
616 return bad_page;
617 }
618
619 EXPORT_SYMBOL_GPL(gfn_to_page);
620
621 void kvm_release_page_clean(struct page *page)
622 {
623 kvm_release_pfn_clean(page_to_pfn(page));
624 }
625 EXPORT_SYMBOL_GPL(kvm_release_page_clean);
626
627 void kvm_release_pfn_clean(pfn_t pfn)
628 {
629 if (pfn_valid(pfn))
630 put_page(pfn_to_page(pfn));
631 }
632 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
633
634 void kvm_release_page_dirty(struct page *page)
635 {
636 kvm_release_pfn_dirty(page_to_pfn(page));
637 }
638 EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
639
640 void kvm_release_pfn_dirty(pfn_t pfn)
641 {
642 kvm_set_pfn_dirty(pfn);
643 kvm_release_pfn_clean(pfn);
644 }
645 EXPORT_SYMBOL_GPL(kvm_release_pfn_dirty);
646
647 void kvm_set_page_dirty(struct page *page)
648 {
649 kvm_set_pfn_dirty(page_to_pfn(page));
650 }
651 EXPORT_SYMBOL_GPL(kvm_set_page_dirty);
652
653 void kvm_set_pfn_dirty(pfn_t pfn)
654 {
655 if (pfn_valid(pfn)) {
656 struct page *page = pfn_to_page(pfn);
657 if (!PageReserved(page))
658 SetPageDirty(page);
659 }
660 }
661 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);
662
663 void kvm_set_pfn_accessed(pfn_t pfn)
664 {
665 if (pfn_valid(pfn))
666 mark_page_accessed(pfn_to_page(pfn));
667 }
668 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);
669
670 void kvm_get_pfn(pfn_t pfn)
671 {
672 if (pfn_valid(pfn))
673 get_page(pfn_to_page(pfn));
674 }
675 EXPORT_SYMBOL_GPL(kvm_get_pfn);
676
677 static int next_segment(unsigned long len, int offset)
678 {
679 if (len > PAGE_SIZE - offset)
680 return PAGE_SIZE - offset;
681 else
682 return len;
683 }
684
685 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
686 int len)
687 {
688 int r;
689 unsigned long addr;
690
691 addr = gfn_to_hva(kvm, gfn);
692 if (kvm_is_error_hva(addr))
693 return -EFAULT;
694 r = copy_from_user(data, (void __user *)addr + offset, len);
695 if (r)
696 return -EFAULT;
697 return 0;
698 }
699 EXPORT_SYMBOL_GPL(kvm_read_guest_page);
700
701 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
702 {
703 gfn_t gfn = gpa >> PAGE_SHIFT;
704 int seg;
705 int offset = offset_in_page(gpa);
706 int ret;
707
708 while ((seg = next_segment(len, offset)) != 0) {
709 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
710 if (ret < 0)
711 return ret;
712 offset = 0;
713 len -= seg;
714 data += seg;
715 ++gfn;
716 }
717 return 0;
718 }
719 EXPORT_SYMBOL_GPL(kvm_read_guest);
720
721 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
722 unsigned long len)
723 {
724 int r;
725 unsigned long addr;
726 gfn_t gfn = gpa >> PAGE_SHIFT;
727 int offset = offset_in_page(gpa);
728
729 addr = gfn_to_hva(kvm, gfn);
730 if (kvm_is_error_hva(addr))
731 return -EFAULT;
732 pagefault_disable();
733 r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
734 pagefault_enable();
735 if (r)
736 return -EFAULT;
737 return 0;
738 }
739 EXPORT_SYMBOL(kvm_read_guest_atomic);
740
741 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn, const void *data,
742 int offset, int len)
743 {
744 int r;
745 unsigned long addr;
746
747 addr = gfn_to_hva(kvm, gfn);
748 if (kvm_is_error_hva(addr))
749 return -EFAULT;
750 r = copy_to_user((void __user *)addr + offset, data, len);
751 if (r)
752 return -EFAULT;
753 mark_page_dirty(kvm, gfn);
754 return 0;
755 }
756 EXPORT_SYMBOL_GPL(kvm_write_guest_page);
757
758 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
759 unsigned long len)
760 {
761 gfn_t gfn = gpa >> PAGE_SHIFT;
762 int seg;
763 int offset = offset_in_page(gpa);
764 int ret;
765
766 while ((seg = next_segment(len, offset)) != 0) {
767 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
768 if (ret < 0)
769 return ret;
770 offset = 0;
771 len -= seg;
772 data += seg;
773 ++gfn;
774 }
775 return 0;
776 }
777
778 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
779 {
780 return kvm_write_guest_page(kvm, gfn, empty_zero_page, offset, len);
781 }
782 EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
783
784 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
785 {
786 gfn_t gfn = gpa >> PAGE_SHIFT;
787 int seg;
788 int offset = offset_in_page(gpa);
789 int ret;
790
791 while ((seg = next_segment(len, offset)) != 0) {
792 ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
793 if (ret < 0)
794 return ret;
795 offset = 0;
796 len -= seg;
797 ++gfn;
798 }
799 return 0;
800 }
801 EXPORT_SYMBOL_GPL(kvm_clear_guest);
802
803 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
804 {
805 struct kvm_memory_slot *memslot;
806
807 gfn = unalias_gfn(kvm, gfn);
808 memslot = __gfn_to_memslot(kvm, gfn);
809 if (memslot && memslot->dirty_bitmap) {
810 unsigned long rel_gfn = gfn - memslot->base_gfn;
811
812 /* avoid RMW */
813 if (!test_bit(rel_gfn, memslot->dirty_bitmap))
814 set_bit(rel_gfn, memslot->dirty_bitmap);
815 }
816 }
817
818 /*
819 * The vCPU has executed a HLT instruction with in-kernel mode enabled.
820 */
821 void kvm_vcpu_block(struct kvm_vcpu *vcpu)
822 {
823 DEFINE_WAIT(wait);
824
825 for (;;) {
826 prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
827
828 if (kvm_cpu_has_interrupt(vcpu))
829 break;
830 if (kvm_cpu_has_pending_timer(vcpu))
831 break;
832 if (kvm_arch_vcpu_runnable(vcpu))
833 break;
834 if (signal_pending(current))
835 break;
836
837 vcpu_put(vcpu);
838 schedule();
839 vcpu_load(vcpu);
840 }
841
842 finish_wait(&vcpu->wq, &wait);
843 }
844
845 void kvm_resched(struct kvm_vcpu *vcpu)
846 {
847 if (!need_resched())
848 return;
849 cond_resched();
850 }
851 EXPORT_SYMBOL_GPL(kvm_resched);
852
853 static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
854 {
855 struct kvm_vcpu *vcpu = vma->vm_file->private_data;
856 struct page *page;
857
858 if (vmf->pgoff == 0)
859 page = virt_to_page(vcpu->run);
860 #ifdef CONFIG_X86
861 else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
862 page = virt_to_page(vcpu->arch.pio_data);
863 #endif
864 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
865 else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
866 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
867 #endif
868 else
869 return VM_FAULT_SIGBUS;
870 get_page(page);
871 vmf->page = page;
872 return 0;
873 }
874
875 static struct vm_operations_struct kvm_vcpu_vm_ops = {
876 .fault = kvm_vcpu_fault,
877 };
878
879 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
880 {
881 vma->vm_ops = &kvm_vcpu_vm_ops;
882 return 0;
883 }
884
885 static int kvm_vcpu_release(struct inode *inode, struct file *filp)
886 {
887 struct kvm_vcpu *vcpu = filp->private_data;
888
889 kvm_put_kvm(vcpu->kvm);
890 return 0;
891 }
892
893 static const struct file_operations kvm_vcpu_fops = {
894 .release = kvm_vcpu_release,
895 .unlocked_ioctl = kvm_vcpu_ioctl,
896 .compat_ioctl = kvm_vcpu_ioctl,
897 .mmap = kvm_vcpu_mmap,
898 };
899
900 /*
901 * Allocates an inode for the vcpu.
902 */
903 static int create_vcpu_fd(struct kvm_vcpu *vcpu)
904 {
905 int fd = anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, 0);
906 if (fd < 0)
907 kvm_put_kvm(vcpu->kvm);
908 return fd;
909 }
910
911 /*
912 * Creates some virtual cpus. Good luck creating more than one.
913 */
914 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, int n)
915 {
916 int r;
917 struct kvm_vcpu *vcpu;
918
919 if (!valid_vcpu(n))
920 return -EINVAL;
921
922 vcpu = kvm_arch_vcpu_create(kvm, n);
923 if (IS_ERR(vcpu))
924 return PTR_ERR(vcpu);
925
926 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
927
928 r = kvm_arch_vcpu_setup(vcpu);
929 if (r)
930 goto vcpu_destroy;
931
932 mutex_lock(&kvm->lock);
933 if (kvm->vcpus[n]) {
934 r = -EEXIST;
935 mutex_unlock(&kvm->lock);
936 goto vcpu_destroy;
937 }
938 kvm->vcpus[n] = vcpu;
939 mutex_unlock(&kvm->lock);
940
941 /* Now it's all set up, let userspace reach it */
942 kvm_get_kvm(kvm);
943 r = create_vcpu_fd(vcpu);
944 if (r < 0)
945 goto unlink;
946 return r;
947
948 unlink:
949 mutex_lock(&kvm->lock);
950 kvm->vcpus[n] = NULL;
951 mutex_unlock(&kvm->lock);
952 vcpu_destroy:
953 kvm_arch_vcpu_destroy(vcpu);
954 return r;
955 }
956
957 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
958 {
959 if (sigset) {
960 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
961 vcpu->sigset_active = 1;
962 vcpu->sigset = *sigset;
963 } else
964 vcpu->sigset_active = 0;
965 return 0;
966 }
967
968 static long kvm_vcpu_ioctl(struct file *filp,
969 unsigned int ioctl, unsigned long arg)
970 {
971 struct kvm_vcpu *vcpu = filp->private_data;
972 void __user *argp = (void __user *)arg;
973 int r;
974
975 if (vcpu->kvm->mm != current->mm)
976 return -EIO;
977 switch (ioctl) {
978 case KVM_RUN:
979 r = -EINVAL;
980 if (arg)
981 goto out;
982 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
983 break;
984 case KVM_GET_REGS: {
985 struct kvm_regs *kvm_regs;
986
987 r = -ENOMEM;
988 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
989 if (!kvm_regs)
990 goto out;
991 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
992 if (r)
993 goto out_free1;
994 r = -EFAULT;
995 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
996 goto out_free1;
997 r = 0;
998 out_free1:
999 kfree(kvm_regs);
1000 break;
1001 }
1002 case KVM_SET_REGS: {
1003 struct kvm_regs *kvm_regs;
1004
1005 r = -ENOMEM;
1006 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
1007 if (!kvm_regs)
1008 goto out;
1009 r = -EFAULT;
1010 if (copy_from_user(kvm_regs, argp, sizeof(struct kvm_regs)))
1011 goto out_free2;
1012 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
1013 if (r)
1014 goto out_free2;
1015 r = 0;
1016 out_free2:
1017 kfree(kvm_regs);
1018 break;
1019 }
1020 case KVM_GET_SREGS: {
1021 struct kvm_sregs kvm_sregs;
1022
1023 memset(&kvm_sregs, 0, sizeof kvm_sregs);
1024 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, &kvm_sregs);
1025 if (r)
1026 goto out;
1027 r = -EFAULT;
1028 if (copy_to_user(argp, &kvm_sregs, sizeof kvm_sregs))
1029 goto out;
1030 r = 0;
1031 break;
1032 }
1033 case KVM_SET_SREGS: {
1034 struct kvm_sregs kvm_sregs;
1035
1036 r = -EFAULT;
1037 if (copy_from_user(&kvm_sregs, argp, sizeof kvm_sregs))
1038 goto out;
1039 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, &kvm_sregs);
1040 if (r)
1041 goto out;
1042 r = 0;
1043 break;
1044 }
1045 case KVM_GET_MP_STATE: {
1046 struct kvm_mp_state mp_state;
1047
1048 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
1049 if (r)
1050 goto out;
1051 r = -EFAULT;
1052 if (copy_to_user(argp, &mp_state, sizeof mp_state))
1053 goto out;
1054 r = 0;
1055 break;
1056 }
1057 case KVM_SET_MP_STATE: {
1058 struct kvm_mp_state mp_state;
1059
1060 r = -EFAULT;
1061 if (copy_from_user(&mp_state, argp, sizeof mp_state))
1062 goto out;
1063 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
1064 if (r)
1065 goto out;
1066 r = 0;
1067 break;
1068 }
1069 case KVM_TRANSLATE: {
1070 struct kvm_translation tr;
1071
1072 r = -EFAULT;
1073 if (copy_from_user(&tr, argp, sizeof tr))
1074 goto out;
1075 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
1076 if (r)
1077 goto out;
1078 r = -EFAULT;
1079 if (copy_to_user(argp, &tr, sizeof tr))
1080 goto out;
1081 r = 0;
1082 break;
1083 }
1084 case KVM_DEBUG_GUEST: {
1085 struct kvm_debug_guest dbg;
1086
1087 r = -EFAULT;
1088 if (copy_from_user(&dbg, argp, sizeof dbg))
1089 goto out;
1090 r = kvm_arch_vcpu_ioctl_debug_guest(vcpu, &dbg);
1091 if (r)
1092 goto out;
1093 r = 0;
1094 break;
1095 }
1096 case KVM_SET_SIGNAL_MASK: {
1097 struct kvm_signal_mask __user *sigmask_arg = argp;
1098 struct kvm_signal_mask kvm_sigmask;
1099 sigset_t sigset, *p;
1100
1101 p = NULL;
1102 if (argp) {
1103 r = -EFAULT;
1104 if (copy_from_user(&kvm_sigmask, argp,
1105 sizeof kvm_sigmask))
1106 goto out;
1107 r = -EINVAL;
1108 if (kvm_sigmask.len != sizeof sigset)
1109 goto out;
1110 r = -EFAULT;
1111 if (copy_from_user(&sigset, sigmask_arg->sigset,
1112 sizeof sigset))
1113 goto out;
1114 p = &sigset;
1115 }
1116 r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
1117 break;
1118 }
1119 case KVM_GET_FPU: {
1120 struct kvm_fpu fpu;
1121
1122 memset(&fpu, 0, sizeof fpu);
1123 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, &fpu);
1124 if (r)
1125 goto out;
1126 r = -EFAULT;
1127 if (copy_to_user(argp, &fpu, sizeof fpu))
1128 goto out;
1129 r = 0;
1130 break;
1131 }
1132 case KVM_SET_FPU: {
1133 struct kvm_fpu fpu;
1134
1135 r = -EFAULT;
1136 if (copy_from_user(&fpu, argp, sizeof fpu))
1137 goto out;
1138 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, &fpu);
1139 if (r)
1140 goto out;
1141 r = 0;
1142 break;
1143 }
1144 default:
1145 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
1146 }
1147 out:
1148 return r;
1149 }
1150
1151 static long kvm_vm_ioctl(struct file *filp,
1152 unsigned int ioctl, unsigned long arg)
1153 {
1154 struct kvm *kvm = filp->private_data;
1155 void __user *argp = (void __user *)arg;
1156 int r;
1157
1158 if (kvm->mm != current->mm)
1159 return -EIO;
1160 switch (ioctl) {
1161 case KVM_CREATE_VCPU:
1162 r = kvm_vm_ioctl_create_vcpu(kvm, arg);
1163 if (r < 0)
1164 goto out;
1165 break;
1166 case KVM_SET_USER_MEMORY_REGION: {
1167 struct kvm_userspace_memory_region kvm_userspace_mem;
1168
1169 r = -EFAULT;
1170 if (copy_from_user(&kvm_userspace_mem, argp,
1171 sizeof kvm_userspace_mem))
1172 goto out;
1173
1174 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem, 1);
1175 if (r)
1176 goto out;
1177 break;
1178 }
1179 case KVM_GET_DIRTY_LOG: {
1180 struct kvm_dirty_log log;
1181
1182 r = -EFAULT;
1183 if (copy_from_user(&log, argp, sizeof log))
1184 goto out;
1185 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
1186 if (r)
1187 goto out;
1188 break;
1189 }
1190 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
1191 case KVM_REGISTER_COALESCED_MMIO: {
1192 struct kvm_coalesced_mmio_zone zone;
1193 r = -EFAULT;
1194 if (copy_from_user(&zone, argp, sizeof zone))
1195 goto out;
1196 r = -ENXIO;
1197 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
1198 if (r)
1199 goto out;
1200 r = 0;
1201 break;
1202 }
1203 case KVM_UNREGISTER_COALESCED_MMIO: {
1204 struct kvm_coalesced_mmio_zone zone;
1205 r = -EFAULT;
1206 if (copy_from_user(&zone, argp, sizeof zone))
1207 goto out;
1208 r = -ENXIO;
1209 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
1210 if (r)
1211 goto out;
1212 r = 0;
1213 break;
1214 }
1215 #endif
1216 default:
1217 r = kvm_arch_vm_ioctl(filp, ioctl, arg);
1218 }
1219 out:
1220 return r;
1221 }
1222
1223 static int kvm_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1224 {
1225 struct kvm *kvm = vma->vm_file->private_data;
1226 struct page *page;
1227
1228 if (!kvm_is_visible_gfn(kvm, vmf->pgoff))
1229 return VM_FAULT_SIGBUS;
1230 page = gfn_to_page(kvm, vmf->pgoff);
1231 if (is_error_page(page)) {
1232 kvm_release_page_clean(page);
1233 return VM_FAULT_SIGBUS;
1234 }
1235 vmf->page = page;
1236 return 0;
1237 }
1238
1239 static struct vm_operations_struct kvm_vm_vm_ops = {
1240 .fault = kvm_vm_fault,
1241 };
1242
1243 static int kvm_vm_mmap(struct file *file, struct vm_area_struct *vma)
1244 {
1245 vma->vm_ops = &kvm_vm_vm_ops;
1246 return 0;
1247 }
1248
1249 static const struct file_operations kvm_vm_fops = {
1250 .release = kvm_vm_release,
1251 .unlocked_ioctl = kvm_vm_ioctl,
1252 .compat_ioctl = kvm_vm_ioctl,
1253 .mmap = kvm_vm_mmap,
1254 };
1255
1256 static int kvm_dev_ioctl_create_vm(void)
1257 {
1258 int fd;
1259 struct kvm *kvm;
1260
1261 kvm = kvm_create_vm();
1262 if (IS_ERR(kvm))
1263 return PTR_ERR(kvm);
1264 fd = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, 0);
1265 if (fd < 0)
1266 kvm_put_kvm(kvm);
1267
1268 return fd;
1269 }
1270
1271 static long kvm_dev_ioctl(struct file *filp,
1272 unsigned int ioctl, unsigned long arg)
1273 {
1274 long r = -EINVAL;
1275
1276 switch (ioctl) {
1277 case KVM_GET_API_VERSION:
1278 r = -EINVAL;
1279 if (arg)
1280 goto out;
1281 r = KVM_API_VERSION;
1282 break;
1283 case KVM_CREATE_VM:
1284 r = -EINVAL;
1285 if (arg)
1286 goto out;
1287 r = kvm_dev_ioctl_create_vm();
1288 break;
1289 case KVM_CHECK_EXTENSION:
1290 r = kvm_dev_ioctl_check_extension(arg);
1291 break;
1292 case KVM_GET_VCPU_MMAP_SIZE:
1293 r = -EINVAL;
1294 if (arg)
1295 goto out;
1296 r = PAGE_SIZE; /* struct kvm_run */
1297 #ifdef CONFIG_X86
1298 r += PAGE_SIZE; /* pio data page */
1299 #endif
1300 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
1301 r += PAGE_SIZE; /* coalesced mmio ring page */
1302 #endif
1303 break;
1304 case KVM_TRACE_ENABLE:
1305 case KVM_TRACE_PAUSE:
1306 case KVM_TRACE_DISABLE:
1307 r = kvm_trace_ioctl(ioctl, arg);
1308 break;
1309 default:
1310 return kvm_arch_dev_ioctl(filp, ioctl, arg);
1311 }
1312 out:
1313 return r;
1314 }
1315
1316 static struct file_operations kvm_chardev_ops = {
1317 .unlocked_ioctl = kvm_dev_ioctl,
1318 .compat_ioctl = kvm_dev_ioctl,
1319 };
1320
1321 static struct miscdevice kvm_dev = {
1322 KVM_MINOR,
1323 "kvm",
1324 &kvm_chardev_ops,
1325 };
1326
1327 static void hardware_enable(void *junk)
1328 {
1329 int cpu = raw_smp_processor_id();
1330
1331 if (cpu_isset(cpu, cpus_hardware_enabled))
1332 return;
1333 cpu_set(cpu, cpus_hardware_enabled);
1334 kvm_arch_hardware_enable(NULL);
1335 }
1336
1337 static void hardware_disable(void *junk)
1338 {
1339 int cpu = raw_smp_processor_id();
1340
1341 if (!cpu_isset(cpu, cpus_hardware_enabled))
1342 return;
1343 cpu_clear(cpu, cpus_hardware_enabled);
1344 kvm_arch_hardware_disable(NULL);
1345 }
1346
1347 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
1348 void *v)
1349 {
1350 int cpu = (long)v;
1351
1352 val &= ~CPU_TASKS_FROZEN;
1353 switch (val) {
1354 case CPU_DYING:
1355 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
1356 cpu);
1357 hardware_disable(NULL);
1358 break;
1359 case CPU_UP_CANCELED:
1360 printk(KERN_INFO "kvm: disabling virtualization on CPU%d\n",
1361 cpu);
1362 smp_call_function_single(cpu, hardware_disable, NULL, 1);
1363 break;
1364 case CPU_ONLINE:
1365 printk(KERN_INFO "kvm: enabling virtualization on CPU%d\n",
1366 cpu);
1367 smp_call_function_single(cpu, hardware_enable, NULL, 1);
1368 break;
1369 }
1370 return NOTIFY_OK;
1371 }
1372
1373
1374 asmlinkage void kvm_handle_fault_on_reboot(void)
1375 {
1376 if (kvm_rebooting)
1377 /* spin while reset goes on */
1378 while (true)
1379 ;
1380 /* Fault while not rebooting. We want the trace. */
1381 BUG();
1382 }
1383 EXPORT_SYMBOL_GPL(kvm_handle_fault_on_reboot);
1384
1385 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
1386 void *v)
1387 {
1388 if (val == SYS_RESTART) {
1389 /*
1390 * Some (well, at least mine) BIOSes hang on reboot if
1391 * in vmx root mode.
1392 */
1393 printk(KERN_INFO "kvm: exiting hardware virtualization\n");
1394 kvm_rebooting = true;
1395 on_each_cpu(hardware_disable, NULL, 1);
1396 }
1397 return NOTIFY_OK;
1398 }
1399
1400 static struct notifier_block kvm_reboot_notifier = {
1401 .notifier_call = kvm_reboot,
1402 .priority = 0,
1403 };
1404
1405 void kvm_io_bus_init(struct kvm_io_bus *bus)
1406 {
1407 memset(bus, 0, sizeof(*bus));
1408 }
1409
1410 void kvm_io_bus_destroy(struct kvm_io_bus *bus)
1411 {
1412 int i;
1413
1414 for (i = 0; i < bus->dev_count; i++) {
1415 struct kvm_io_device *pos = bus->devs[i];
1416
1417 kvm_iodevice_destructor(pos);
1418 }
1419 }
1420
1421 struct kvm_io_device *kvm_io_bus_find_dev(struct kvm_io_bus *bus,
1422 gpa_t addr, int len, int is_write)
1423 {
1424 int i;
1425
1426 for (i = 0; i < bus->dev_count; i++) {
1427 struct kvm_io_device *pos = bus->devs[i];
1428
1429 if (pos->in_range(pos, addr, len, is_write))
1430 return pos;
1431 }
1432
1433 return NULL;
1434 }
1435
1436 void kvm_io_bus_register_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev)
1437 {
1438 BUG_ON(bus->dev_count > (NR_IOBUS_DEVS-1));
1439
1440 bus->devs[bus->dev_count++] = dev;
1441 }
1442
1443 static struct notifier_block kvm_cpu_notifier = {
1444 .notifier_call = kvm_cpu_hotplug,
1445 .priority = 20, /* must be > scheduler priority */
1446 };
1447
1448 static int vm_stat_get(void *_offset, u64 *val)
1449 {
1450 unsigned offset = (long)_offset;
1451 struct kvm *kvm;
1452
1453 *val = 0;
1454 spin_lock(&kvm_lock);
1455 list_for_each_entry(kvm, &vm_list, vm_list)
1456 *val += *(u32 *)((void *)kvm + offset);
1457 spin_unlock(&kvm_lock);
1458 return 0;
1459 }
1460
1461 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");
1462
1463 static int vcpu_stat_get(void *_offset, u64 *val)
1464 {
1465 unsigned offset = (long)_offset;
1466 struct kvm *kvm;
1467 struct kvm_vcpu *vcpu;
1468 int i;
1469
1470 *val = 0;
1471 spin_lock(&kvm_lock);
1472 list_for_each_entry(kvm, &vm_list, vm_list)
1473 for (i = 0; i < KVM_MAX_VCPUS; ++i) {
1474 vcpu = kvm->vcpus[i];
1475 if (vcpu)
1476 *val += *(u32 *)((void *)vcpu + offset);
1477 }
1478 spin_unlock(&kvm_lock);
1479 return 0;
1480 }
1481
1482 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");
1483
1484 static struct file_operations *stat_fops[] = {
1485 [KVM_STAT_VCPU] = &vcpu_stat_fops,
1486 [KVM_STAT_VM] = &vm_stat_fops,
1487 };
1488
1489 static void kvm_init_debug(void)
1490 {
1491 struct kvm_stats_debugfs_item *p;
1492
1493 kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
1494 for (p = debugfs_entries; p->name; ++p)
1495 p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
1496 (void *)(long)p->offset,
1497 stat_fops[p->kind]);
1498 }
1499
1500 static void kvm_exit_debug(void)
1501 {
1502 struct kvm_stats_debugfs_item *p;
1503
1504 for (p = debugfs_entries; p->name; ++p)
1505 debugfs_remove(p->dentry);
1506 debugfs_remove(kvm_debugfs_dir);
1507 }
1508
1509 static int kvm_suspend(struct sys_device *dev, pm_message_t state)
1510 {
1511 hardware_disable(NULL);
1512 return 0;
1513 }
1514
1515 static int kvm_resume(struct sys_device *dev)
1516 {
1517 hardware_enable(NULL);
1518 return 0;
1519 }
1520
1521 static struct sysdev_class kvm_sysdev_class = {
1522 .name = "kvm",
1523 .suspend = kvm_suspend,
1524 .resume = kvm_resume,
1525 };
1526
1527 static struct sys_device kvm_sysdev = {
1528 .id = 0,
1529 .cls = &kvm_sysdev_class,
1530 };
1531
1532 struct page *bad_page;
1533 pfn_t bad_pfn;
1534
1535 static inline
1536 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
1537 {
1538 return container_of(pn, struct kvm_vcpu, preempt_notifier);
1539 }
1540
1541 static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
1542 {
1543 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
1544
1545 kvm_arch_vcpu_load(vcpu, cpu);
1546 }
1547
1548 static void kvm_sched_out(struct preempt_notifier *pn,
1549 struct task_struct *next)
1550 {
1551 struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
1552
1553 kvm_arch_vcpu_put(vcpu);
1554 }
1555
1556 int kvm_init(void *opaque, unsigned int vcpu_size,
1557 struct module *module)
1558 {
1559 int r;
1560 int cpu;
1561
1562 kvm_init_debug();
1563
1564 r = kvm_arch_init(opaque);
1565 if (r)
1566 goto out_fail;
1567
1568 bad_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
1569
1570 if (bad_page == NULL) {
1571 r = -ENOMEM;
1572 goto out;
1573 }
1574
1575 bad_pfn = page_to_pfn(bad_page);
1576
1577 r = kvm_arch_hardware_setup();
1578 if (r < 0)
1579 goto out_free_0;
1580
1581 for_each_online_cpu(cpu) {
1582 smp_call_function_single(cpu,
1583 kvm_arch_check_processor_compat,
1584 &r, 1);
1585 if (r < 0)
1586 goto out_free_1;
1587 }
1588
1589 on_each_cpu(hardware_enable, NULL, 1);
1590 r = register_cpu_notifier(&kvm_cpu_notifier);
1591 if (r)
1592 goto out_free_2;
1593 register_reboot_notifier(&kvm_reboot_notifier);
1594
1595 r = sysdev_class_register(&kvm_sysdev_class);
1596 if (r)
1597 goto out_free_3;
1598
1599 r = sysdev_register(&kvm_sysdev);
1600 if (r)
1601 goto out_free_4;
1602
1603 /* A kmem cache lets us meet the alignment requirements of fx_save. */
1604 kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size,
1605 __alignof__(struct kvm_vcpu),
1606 0, NULL);
1607 if (!kvm_vcpu_cache) {
1608 r = -ENOMEM;
1609 goto out_free_5;
1610 }
1611
1612 kvm_chardev_ops.owner = module;
1613
1614 r = misc_register(&kvm_dev);
1615 if (r) {
1616 printk(KERN_ERR "kvm: misc device register failed\n");
1617 goto out_free;
1618 }
1619
1620 kvm_preempt_ops.sched_in = kvm_sched_in;
1621 kvm_preempt_ops.sched_out = kvm_sched_out;
1622
1623 return 0;
1624
1625 out_free:
1626 kmem_cache_destroy(kvm_vcpu_cache);
1627 out_free_5:
1628 sysdev_unregister(&kvm_sysdev);
1629 out_free_4:
1630 sysdev_class_unregister(&kvm_sysdev_class);
1631 out_free_3:
1632 unregister_reboot_notifier(&kvm_reboot_notifier);
1633 unregister_cpu_notifier(&kvm_cpu_notifier);
1634 out_free_2:
1635 on_each_cpu(hardware_disable, NULL, 1);
1636 out_free_1:
1637 kvm_arch_hardware_unsetup();
1638 out_free_0:
1639 __free_page(bad_page);
1640 out:
1641 kvm_arch_exit();
1642 kvm_exit_debug();
1643 out_fail:
1644 return r;
1645 }
1646 EXPORT_SYMBOL_GPL(kvm_init);
1647
1648 void kvm_exit(void)
1649 {
1650 kvm_trace_cleanup();
1651 misc_deregister(&kvm_dev);
1652 kmem_cache_destroy(kvm_vcpu_cache);
1653 sysdev_unregister(&kvm_sysdev);
1654 sysdev_class_unregister(&kvm_sysdev_class);
1655 unregister_reboot_notifier(&kvm_reboot_notifier);
1656 unregister_cpu_notifier(&kvm_cpu_notifier);
1657 on_each_cpu(hardware_disable, NULL, 1);
1658 kvm_arch_hardware_unsetup();
1659 kvm_arch_exit();
1660 kvm_exit_debug();
1661 __free_page(bad_page);
1662 }
1663 EXPORT_SYMBOL_GPL(kvm_exit);
kernel-based virtual machine