?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
ACPI: thinkpad-acpi: keep track of module state
[linux-2.6/linux-acpi-2.6/ibm-acpi-2.6.git] / drivers / kvm / vmx.c
blob916da2983c669e636016826d5b0ae73fd4847cc1
1 /*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * Copyright (C) 2006 Qumranet, Inc.
8 *
9 * Authors:
10 * Avi Kivity <avi@qumranet.com>
11 * Yaniv Kamay <yaniv@qumranet.com>
12 *
13 * This work is licensed under the terms of the GNU GPL, version 2. See
14 * the COPYING file in the top-level directory.
15 *
16 */
17
18 #include "kvm.h"
19 #include "vmx.h"
20 #include "segment_descriptor.h"
21
22 #include <linux/module.h>
23 #include <linux/kernel.h>
24 #include <linux/mm.h>
25 #include <linux/highmem.h>
26 #include <linux/profile.h>
27 #include <linux/sched.h>
28
29 #include <asm/io.h>
30 #include <asm/desc.h>
31
32 MODULE_AUTHOR("Qumranet");
33 MODULE_LICENSE("GPL");
34
35 static int init_rmode_tss(struct kvm *kvm);
36
37 static DEFINE_PER_CPU(struct vmcs *, vmxarea);
38 static DEFINE_PER_CPU(struct vmcs *, current_vmcs);
39
40 static struct page *vmx_io_bitmap_a;
41 static struct page *vmx_io_bitmap_b;
42
43 #ifdef CONFIG_X86_64
44 #define HOST_IS_64 1
45 #else
46 #define HOST_IS_64 0
47 #endif
48 #define EFER_SAVE_RESTORE_BITS ((u64)EFER_SCE)
49
50 static struct vmcs_descriptor {
51 int size;
52 int order;
53 u32 revision_id;
54 } vmcs_descriptor;
55
56 #define VMX_SEGMENT_FIELD(seg) \
57 [VCPU_SREG_##seg] = { \
58 .selector = GUEST_##seg##_SELECTOR, \
59 .base = GUEST_##seg##_BASE, \
60 .limit = GUEST_##seg##_LIMIT, \
61 .ar_bytes = GUEST_##seg##_AR_BYTES, \
62 }
63
64 static struct kvm_vmx_segment_field {
65 unsigned selector;
66 unsigned base;
67 unsigned limit;
68 unsigned ar_bytes;
69 } kvm_vmx_segment_fields[] = {
70 VMX_SEGMENT_FIELD(CS),
71 VMX_SEGMENT_FIELD(DS),
72 VMX_SEGMENT_FIELD(ES),
73 VMX_SEGMENT_FIELD(FS),
74 VMX_SEGMENT_FIELD(GS),
75 VMX_SEGMENT_FIELD(SS),
76 VMX_SEGMENT_FIELD(TR),
77 VMX_SEGMENT_FIELD(LDTR),
78 };
79
80 /*
81 * Keep MSR_K6_STAR at the end, as setup_msrs() will try to optimize it
82 * away by decrementing the array size.
83 */
84 static const u32 vmx_msr_index[] = {
85 #ifdef CONFIG_X86_64
86 MSR_SYSCALL_MASK, MSR_LSTAR, MSR_CSTAR, MSR_KERNEL_GS_BASE,
87 #endif
88 MSR_EFER, MSR_K6_STAR,
89 };
90 #define NR_VMX_MSR ARRAY_SIZE(vmx_msr_index)
91
92 static inline u64 msr_efer_save_restore_bits(struct vmx_msr_entry msr)
93 {
94 return (u64)msr.data & EFER_SAVE_RESTORE_BITS;
95 }
96
97 static inline int msr_efer_need_save_restore(struct kvm_vcpu *vcpu)
98 {
99 int efer_offset = vcpu->msr_offset_efer;
100 return msr_efer_save_restore_bits(vcpu->host_msrs[efer_offset]) !=
101 msr_efer_save_restore_bits(vcpu->guest_msrs[efer_offset]);
102 }
103
104 static inline int is_page_fault(u32 intr_info)
105 {
106 return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
107 INTR_INFO_VALID_MASK)) ==
108 (INTR_TYPE_EXCEPTION | PF_VECTOR | INTR_INFO_VALID_MASK);
109 }
110
111 static inline int is_no_device(u32 intr_info)
112 {
113 return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK |
114 INTR_INFO_VALID_MASK)) ==
115 (INTR_TYPE_EXCEPTION | NM_VECTOR | INTR_INFO_VALID_MASK);
116 }
117
118 static inline int is_external_interrupt(u32 intr_info)
119 {
120 return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VALID_MASK))
121 == (INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK);
122 }
123
124 static int __find_msr_index(struct kvm_vcpu *vcpu, u32 msr)
125 {
126 int i;
127
128 for (i = 0; i < vcpu->nmsrs; ++i)
129 if (vcpu->guest_msrs[i].index == msr)
130 return i;
131 return -1;
132 }
133
134 static struct vmx_msr_entry *find_msr_entry(struct kvm_vcpu *vcpu, u32 msr)
135 {
136 int i;
137
138 i = __find_msr_index(vcpu, msr);
139 if (i >= 0)
140 return &vcpu->guest_msrs[i];
141 return NULL;
142 }
143
144 static void vmcs_clear(struct vmcs *vmcs)
145 {
146 u64 phys_addr = __pa(vmcs);
147 u8 error;
148
149 asm volatile (ASM_VMX_VMCLEAR_RAX "; setna %0"
150 : "=g"(error) : "a"(&phys_addr), "m"(phys_addr)
151 : "cc", "memory");
152 if (error)
153 printk(KERN_ERR "kvm: vmclear fail: %p/%llx\n",
154 vmcs, phys_addr);
155 }
156
157 static void __vcpu_clear(void *arg)
158 {
159 struct kvm_vcpu *vcpu = arg;
160 int cpu = raw_smp_processor_id();
161
162 if (vcpu->cpu == cpu)
163 vmcs_clear(vcpu->vmcs);
164 if (per_cpu(current_vmcs, cpu) == vcpu->vmcs)
165 per_cpu(current_vmcs, cpu) = NULL;
166 rdtscll(vcpu->host_tsc);
167 }
168
169 static void vcpu_clear(struct kvm_vcpu *vcpu)
170 {
171 if (vcpu->cpu != raw_smp_processor_id() && vcpu->cpu != -1)
172 smp_call_function_single(vcpu->cpu, __vcpu_clear, vcpu, 0, 1);
173 else
174 __vcpu_clear(vcpu);
175 vcpu->launched = 0;
176 }
177
178 static unsigned long vmcs_readl(unsigned long field)
179 {
180 unsigned long value;
181
182 asm volatile (ASM_VMX_VMREAD_RDX_RAX
183 : "=a"(value) : "d"(field) : "cc");
184 return value;
185 }
186
187 static u16 vmcs_read16(unsigned long field)
188 {
189 return vmcs_readl(field);
190 }
191
192 static u32 vmcs_read32(unsigned long field)
193 {
194 return vmcs_readl(field);
195 }
196
197 static u64 vmcs_read64(unsigned long field)
198 {
199 #ifdef CONFIG_X86_64
200 return vmcs_readl(field);
201 #else
202 return vmcs_readl(field) | ((u64)vmcs_readl(field+1) << 32);
203 #endif
204 }
205
206 static noinline void vmwrite_error(unsigned long field, unsigned long value)
207 {
208 printk(KERN_ERR "vmwrite error: reg %lx value %lx (err %d)\n",
209 field, value, vmcs_read32(VM_INSTRUCTION_ERROR));
210 dump_stack();
211 }
212
213 static void vmcs_writel(unsigned long field, unsigned long value)
214 {
215 u8 error;
216
217 asm volatile (ASM_VMX_VMWRITE_RAX_RDX "; setna %0"
218 : "=q"(error) : "a"(value), "d"(field) : "cc" );
219 if (unlikely(error))
220 vmwrite_error(field, value);
221 }
222
223 static void vmcs_write16(unsigned long field, u16 value)
224 {
225 vmcs_writel(field, value);
226 }
227
228 static void vmcs_write32(unsigned long field, u32 value)
229 {
230 vmcs_writel(field, value);
231 }
232
233 static void vmcs_write64(unsigned long field, u64 value)
234 {
235 #ifdef CONFIG_X86_64
236 vmcs_writel(field, value);
237 #else
238 vmcs_writel(field, value);
239 asm volatile ("");
240 vmcs_writel(field+1, value >> 32);
241 #endif
242 }
243
244 static void vmcs_clear_bits(unsigned long field, u32 mask)
245 {
246 vmcs_writel(field, vmcs_readl(field) & ~mask);
247 }
248
249 static void vmcs_set_bits(unsigned long field, u32 mask)
250 {
251 vmcs_writel(field, vmcs_readl(field) | mask);
252 }
253
254 static void update_exception_bitmap(struct kvm_vcpu *vcpu)
255 {
256 u32 eb;
257
258 eb = 1u << PF_VECTOR;
259 if (!vcpu->fpu_active)
260 eb |= 1u << NM_VECTOR;
261 if (vcpu->guest_debug.enabled)
262 eb |= 1u << 1;
263 if (vcpu->rmode.active)
264 eb = ~0;
265 vmcs_write32(EXCEPTION_BITMAP, eb);
266 }
267
268 static void reload_tss(void)
269 {
270 #ifndef CONFIG_X86_64
271
272 /*
273 * VT restores TR but not its size. Useless.
274 */
275 struct descriptor_table gdt;
276 struct segment_descriptor *descs;
277
278 get_gdt(&gdt);
279 descs = (void *)gdt.base;
280 descs[GDT_ENTRY_TSS].type = 9; /* available TSS */
281 load_TR_desc();
282 #endif
283 }
284
285 static void load_transition_efer(struct kvm_vcpu *vcpu)
286 {
287 u64 trans_efer;
288 int efer_offset = vcpu->msr_offset_efer;
289
290 trans_efer = vcpu->host_msrs[efer_offset].data;
291 trans_efer &= ~EFER_SAVE_RESTORE_BITS;
292 trans_efer |= msr_efer_save_restore_bits(
293 vcpu->guest_msrs[efer_offset]);
294 wrmsrl(MSR_EFER, trans_efer);
295 vcpu->stat.efer_reload++;
296 }
297
298 static void vmx_save_host_state(struct kvm_vcpu *vcpu)
299 {
300 struct vmx_host_state *hs = &vcpu->vmx_host_state;
301
302 if (hs->loaded)
303 return;
304
305 hs->loaded = 1;
306 /*
307 * Set host fs and gs selectors. Unfortunately, 22.2.3 does not
308 * allow segment selectors with cpl > 0 or ti == 1.
309 */
310 hs->ldt_sel = read_ldt();
311 hs->fs_gs_ldt_reload_needed = hs->ldt_sel;
312 hs->fs_sel = read_fs();
313 if (!(hs->fs_sel & 7))
314 vmcs_write16(HOST_FS_SELECTOR, hs->fs_sel);
315 else {
316 vmcs_write16(HOST_FS_SELECTOR, 0);
317 hs->fs_gs_ldt_reload_needed = 1;
318 }
319 hs->gs_sel = read_gs();
320 if (!(hs->gs_sel & 7))
321 vmcs_write16(HOST_GS_SELECTOR, hs->gs_sel);
322 else {
323 vmcs_write16(HOST_GS_SELECTOR, 0);
324 hs->fs_gs_ldt_reload_needed = 1;
325 }
326
327 #ifdef CONFIG_X86_64
328 vmcs_writel(HOST_FS_BASE, read_msr(MSR_FS_BASE));
329 vmcs_writel(HOST_GS_BASE, read_msr(MSR_GS_BASE));
330 #else
331 vmcs_writel(HOST_FS_BASE, segment_base(hs->fs_sel));
332 vmcs_writel(HOST_GS_BASE, segment_base(hs->gs_sel));
333 #endif
334
335 #ifdef CONFIG_X86_64
336 if (is_long_mode(vcpu)) {
337 save_msrs(vcpu->host_msrs + vcpu->msr_offset_kernel_gs_base, 1);
338 }
339 #endif
340 load_msrs(vcpu->guest_msrs, vcpu->save_nmsrs);
341 if (msr_efer_need_save_restore(vcpu))
342 load_transition_efer(vcpu);
343 }
344
345 static void vmx_load_host_state(struct kvm_vcpu *vcpu)
346 {
347 struct vmx_host_state *hs = &vcpu->vmx_host_state;
348
349 if (!hs->loaded)
350 return;
351
352 hs->loaded = 0;
353 if (hs->fs_gs_ldt_reload_needed) {
354 load_ldt(hs->ldt_sel);
355 load_fs(hs->fs_sel);
356 /*
357 * If we have to reload gs, we must take care to
358 * preserve our gs base.
359 */
360 local_irq_disable();
361 load_gs(hs->gs_sel);
362 #ifdef CONFIG_X86_64
363 wrmsrl(MSR_GS_BASE, vmcs_readl(HOST_GS_BASE));
364 #endif
365 local_irq_enable();
366
367 reload_tss();
368 }
369 save_msrs(vcpu->guest_msrs, vcpu->save_nmsrs);
370 load_msrs(vcpu->host_msrs, vcpu->save_nmsrs);
371 if (msr_efer_need_save_restore(vcpu))
372 load_msrs(vcpu->host_msrs + vcpu->msr_offset_efer, 1);
373 }
374
375 /*
376 * Switches to specified vcpu, until a matching vcpu_put(), but assumes
377 * vcpu mutex is already taken.
378 */
379 static void vmx_vcpu_load(struct kvm_vcpu *vcpu)
380 {
381 u64 phys_addr = __pa(vcpu->vmcs);
382 int cpu;
383 u64 tsc_this, delta;
384
385 cpu = get_cpu();
386
387 if (vcpu->cpu != cpu)
388 vcpu_clear(vcpu);
389
390 if (per_cpu(current_vmcs, cpu) != vcpu->vmcs) {
391 u8 error;
392
393 per_cpu(current_vmcs, cpu) = vcpu->vmcs;
394 asm volatile (ASM_VMX_VMPTRLD_RAX "; setna %0"
395 : "=g"(error) : "a"(&phys_addr), "m"(phys_addr)
396 : "cc");
397 if (error)
398 printk(KERN_ERR "kvm: vmptrld %p/%llx fail\n",
399 vcpu->vmcs, phys_addr);
400 }
401
402 if (vcpu->cpu != cpu) {
403 struct descriptor_table dt;
404 unsigned long sysenter_esp;
405
406 vcpu->cpu = cpu;
407 /*
408 * Linux uses per-cpu TSS and GDT, so set these when switching
409 * processors.
410 */
411 vmcs_writel(HOST_TR_BASE, read_tr_base()); /* 22.2.4 */
412 get_gdt(&dt);
413 vmcs_writel(HOST_GDTR_BASE, dt.base); /* 22.2.4 */
414
415 rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp);
416 vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */
417
418 /*
419 * Make sure the time stamp counter is monotonous.
420 */
421 rdtscll(tsc_this);
422 delta = vcpu->host_tsc - tsc_this;
423 vmcs_write64(TSC_OFFSET, vmcs_read64(TSC_OFFSET) + delta);
424 }
425 }
426
427 static void vmx_vcpu_put(struct kvm_vcpu *vcpu)
428 {
429 vmx_load_host_state(vcpu);
430 kvm_put_guest_fpu(vcpu);
431 put_cpu();
432 }
433
434 static void vmx_fpu_activate(struct kvm_vcpu *vcpu)
435 {
436 if (vcpu->fpu_active)
437 return;
438 vcpu->fpu_active = 1;
439 vmcs_clear_bits(GUEST_CR0, CR0_TS_MASK);
440 if (vcpu->cr0 & CR0_TS_MASK)
441 vmcs_set_bits(GUEST_CR0, CR0_TS_MASK);
442 update_exception_bitmap(vcpu);
443 }
444
445 static void vmx_fpu_deactivate(struct kvm_vcpu *vcpu)
446 {
447 if (!vcpu->fpu_active)
448 return;
449 vcpu->fpu_active = 0;
450 vmcs_set_bits(GUEST_CR0, CR0_TS_MASK);
451 update_exception_bitmap(vcpu);
452 }
453
454 static void vmx_vcpu_decache(struct kvm_vcpu *vcpu)
455 {
456 vcpu_clear(vcpu);
457 }
458
459 static unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
460 {
461 return vmcs_readl(GUEST_RFLAGS);
462 }
463
464 static void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
465 {
466 if (vcpu->rmode.active)
467 rflags |= IOPL_MASK | X86_EFLAGS_VM;
468 vmcs_writel(GUEST_RFLAGS, rflags);
469 }
470
471 static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
472 {
473 unsigned long rip;
474 u32 interruptibility;
475
476 rip = vmcs_readl(GUEST_RIP);
477 rip += vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
478 vmcs_writel(GUEST_RIP, rip);
479
480 /*
481 * We emulated an instruction, so temporary interrupt blocking
482 * should be removed, if set.
483 */
484 interruptibility = vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
485 if (interruptibility & 3)
486 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO,
487 interruptibility & ~3);
488 vcpu->interrupt_window_open = 1;
489 }
490
491 static void vmx_inject_gp(struct kvm_vcpu *vcpu, unsigned error_code)
492 {
493 printk(KERN_DEBUG "inject_general_protection: rip 0x%lx\n",
494 vmcs_readl(GUEST_RIP));
495 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code);
496 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
497 GP_VECTOR |
498 INTR_TYPE_EXCEPTION |
499 INTR_INFO_DELIEVER_CODE_MASK |
500 INTR_INFO_VALID_MASK);
501 }
502
503 /*
504 * Swap MSR entry in host/guest MSR entry array.
505 */
506 void move_msr_up(struct kvm_vcpu *vcpu, int from, int to)
507 {
508 struct vmx_msr_entry tmp;
509 tmp = vcpu->guest_msrs[to];
510 vcpu->guest_msrs[to] = vcpu->guest_msrs[from];
511 vcpu->guest_msrs[from] = tmp;
512 tmp = vcpu->host_msrs[to];
513 vcpu->host_msrs[to] = vcpu->host_msrs[from];
514 vcpu->host_msrs[from] = tmp;
515 }
516
517 /*
518 * Set up the vmcs to automatically save and restore system
519 * msrs. Don't touch the 64-bit msrs if the guest is in legacy
520 * mode, as fiddling with msrs is very expensive.
521 */
522 static void setup_msrs(struct kvm_vcpu *vcpu)
523 {
524 int save_nmsrs;
525
526 save_nmsrs = 0;
527 #ifdef CONFIG_X86_64
528 if (is_long_mode(vcpu)) {
529 int index;
530
531 index = __find_msr_index(vcpu, MSR_SYSCALL_MASK);
532 if (index >= 0)
533 move_msr_up(vcpu, index, save_nmsrs++);
534 index = __find_msr_index(vcpu, MSR_LSTAR);
535 if (index >= 0)
536 move_msr_up(vcpu, index, save_nmsrs++);
537 index = __find_msr_index(vcpu, MSR_CSTAR);
538 if (index >= 0)
539 move_msr_up(vcpu, index, save_nmsrs++);
540 index = __find_msr_index(vcpu, MSR_KERNEL_GS_BASE);
541 if (index >= 0)
542 move_msr_up(vcpu, index, save_nmsrs++);
543 /*
544 * MSR_K6_STAR is only needed on long mode guests, and only
545 * if efer.sce is enabled.
546 */
547 index = __find_msr_index(vcpu, MSR_K6_STAR);
548 if ((index >= 0) && (vcpu->shadow_efer & EFER_SCE))
549 move_msr_up(vcpu, index, save_nmsrs++);
550 }
551 #endif
552 vcpu->save_nmsrs = save_nmsrs;
553
554 #ifdef CONFIG_X86_64
555 vcpu->msr_offset_kernel_gs_base =
556 __find_msr_index(vcpu, MSR_KERNEL_GS_BASE);
557 #endif
558 vcpu->msr_offset_efer = __find_msr_index(vcpu, MSR_EFER);
559 }
560
561 /*
562 * reads and returns guest's timestamp counter "register"
563 * guest_tsc = host_tsc + tsc_offset -- 21.3
564 */
565 static u64 guest_read_tsc(void)
566 {
567 u64 host_tsc, tsc_offset;
568
569 rdtscll(host_tsc);
570 tsc_offset = vmcs_read64(TSC_OFFSET);
571 return host_tsc + tsc_offset;
572 }
573
574 /*
575 * writes 'guest_tsc' into guest's timestamp counter "register"
576 * guest_tsc = host_tsc + tsc_offset ==> tsc_offset = guest_tsc - host_tsc
577 */
578 static void guest_write_tsc(u64 guest_tsc)
579 {
580 u64 host_tsc;
581
582 rdtscll(host_tsc);
583 vmcs_write64(TSC_OFFSET, guest_tsc - host_tsc);
584 }
585
586 /*
587 * Reads an msr value (of 'msr_index') into 'pdata'.
588 * Returns 0 on success, non-0 otherwise.
589 * Assumes vcpu_load() was already called.
590 */
591 static int vmx_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata)
592 {
593 u64 data;
594 struct vmx_msr_entry *msr;
595
596 if (!pdata) {
597 printk(KERN_ERR "BUG: get_msr called with NULL pdata\n");
598 return -EINVAL;
599 }
600
601 switch (msr_index) {
602 #ifdef CONFIG_X86_64
603 case MSR_FS_BASE:
604 data = vmcs_readl(GUEST_FS_BASE);
605 break;
606 case MSR_GS_BASE:
607 data = vmcs_readl(GUEST_GS_BASE);
608 break;
609 case MSR_EFER:
610 return kvm_get_msr_common(vcpu, msr_index, pdata);
611 #endif
612 case MSR_IA32_TIME_STAMP_COUNTER:
613 data = guest_read_tsc();
614 break;
615 case MSR_IA32_SYSENTER_CS:
616 data = vmcs_read32(GUEST_SYSENTER_CS);
617 break;
618 case MSR_IA32_SYSENTER_EIP:
619 data = vmcs_readl(GUEST_SYSENTER_EIP);
620 break;
621 case MSR_IA32_SYSENTER_ESP:
622 data = vmcs_readl(GUEST_SYSENTER_ESP);
623 break;
624 default:
625 msr = find_msr_entry(vcpu, msr_index);
626 if (msr) {
627 data = msr->data;
628 break;
629 }
630 return kvm_get_msr_common(vcpu, msr_index, pdata);
631 }
632
633 *pdata = data;
634 return 0;
635 }
636
637 /*
638 * Writes msr value into into the appropriate "register".
639 * Returns 0 on success, non-0 otherwise.
640 * Assumes vcpu_load() was already called.
641 */
642 static int vmx_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
643 {
644 struct vmx_msr_entry *msr;
645 int ret = 0;
646
647 switch (msr_index) {
648 #ifdef CONFIG_X86_64
649 case MSR_EFER:
650 ret = kvm_set_msr_common(vcpu, msr_index, data);
651 if (vcpu->vmx_host_state.loaded)
652 load_transition_efer(vcpu);
653 break;
654 case MSR_FS_BASE:
655 vmcs_writel(GUEST_FS_BASE, data);
656 break;
657 case MSR_GS_BASE:
658 vmcs_writel(GUEST_GS_BASE, data);
659 break;
660 #endif
661 case MSR_IA32_SYSENTER_CS:
662 vmcs_write32(GUEST_SYSENTER_CS, data);
663 break;
664 case MSR_IA32_SYSENTER_EIP:
665 vmcs_writel(GUEST_SYSENTER_EIP, data);
666 break;
667 case MSR_IA32_SYSENTER_ESP:
668 vmcs_writel(GUEST_SYSENTER_ESP, data);
669 break;
670 case MSR_IA32_TIME_STAMP_COUNTER:
671 guest_write_tsc(data);
672 break;
673 default:
674 msr = find_msr_entry(vcpu, msr_index);
675 if (msr) {
676 msr->data = data;
677 if (vcpu->vmx_host_state.loaded)
678 load_msrs(vcpu->guest_msrs, vcpu->save_nmsrs);
679 break;
680 }
681 ret = kvm_set_msr_common(vcpu, msr_index, data);
682 }
683
684 return ret;
685 }
686
687 /*
688 * Sync the rsp and rip registers into the vcpu structure. This allows
689 * registers to be accessed by indexing vcpu->regs.
690 */
691 static void vcpu_load_rsp_rip(struct kvm_vcpu *vcpu)
692 {
693 vcpu->regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP);
694 vcpu->rip = vmcs_readl(GUEST_RIP);
695 }
696
697 /*
698 * Syncs rsp and rip back into the vmcs. Should be called after possible
699 * modification.
700 */
701 static void vcpu_put_rsp_rip(struct kvm_vcpu *vcpu)
702 {
703 vmcs_writel(GUEST_RSP, vcpu->regs[VCPU_REGS_RSP]);
704 vmcs_writel(GUEST_RIP, vcpu->rip);
705 }
706
707 static int set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_debug_guest *dbg)
708 {
709 unsigned long dr7 = 0x400;
710 int old_singlestep;
711
712 old_singlestep = vcpu->guest_debug.singlestep;
713
714 vcpu->guest_debug.enabled = dbg->enabled;
715 if (vcpu->guest_debug.enabled) {
716 int i;
717
718 dr7 |= 0x200; /* exact */
719 for (i = 0; i < 4; ++i) {
720 if (!dbg->breakpoints[i].enabled)
721 continue;
722 vcpu->guest_debug.bp[i] = dbg->breakpoints[i].address;
723 dr7 |= 2 << (i*2); /* global enable */
724 dr7 |= 0 << (i*4+16); /* execution breakpoint */
725 }
726
727 vcpu->guest_debug.singlestep = dbg->singlestep;
728 } else
729 vcpu->guest_debug.singlestep = 0;
730
731 if (old_singlestep && !vcpu->guest_debug.singlestep) {
732 unsigned long flags;
733
734 flags = vmcs_readl(GUEST_RFLAGS);
735 flags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
736 vmcs_writel(GUEST_RFLAGS, flags);
737 }
738
739 update_exception_bitmap(vcpu);
740 vmcs_writel(GUEST_DR7, dr7);
741
742 return 0;
743 }
744
745 static __init int cpu_has_kvm_support(void)
746 {
747 unsigned long ecx = cpuid_ecx(1);
748 return test_bit(5, &ecx); /* CPUID.1:ECX.VMX[bit 5] -> VT */
749 }
750
751 static __init int vmx_disabled_by_bios(void)
752 {
753 u64 msr;
754
755 rdmsrl(MSR_IA32_FEATURE_CONTROL, msr);
756 return (msr & 5) == 1; /* locked but not enabled */
757 }
758
759 static void hardware_enable(void *garbage)
760 {
761 int cpu = raw_smp_processor_id();
762 u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
763 u64 old;
764
765 rdmsrl(MSR_IA32_FEATURE_CONTROL, old);
766 if ((old & 5) != 5)
767 /* enable and lock */
768 wrmsrl(MSR_IA32_FEATURE_CONTROL, old | 5);
769 write_cr4(read_cr4() | CR4_VMXE); /* FIXME: not cpu hotplug safe */
770 asm volatile (ASM_VMX_VMXON_RAX : : "a"(&phys_addr), "m"(phys_addr)
771 : "memory", "cc");
772 }
773
774 static void hardware_disable(void *garbage)
775 {
776 asm volatile (ASM_VMX_VMXOFF : : : "cc");
777 }
778
779 static __init void setup_vmcs_descriptor(void)
780 {
781 u32 vmx_msr_low, vmx_msr_high;
782
783 rdmsr(MSR_IA32_VMX_BASIC, vmx_msr_low, vmx_msr_high);
784 vmcs_descriptor.size = vmx_msr_high & 0x1fff;
785 vmcs_descriptor.order = get_order(vmcs_descriptor.size);
786 vmcs_descriptor.revision_id = vmx_msr_low;
787 }
788
789 static struct vmcs *alloc_vmcs_cpu(int cpu)
790 {
791 int node = cpu_to_node(cpu);
792 struct page *pages;
793 struct vmcs *vmcs;
794
795 pages = alloc_pages_node(node, GFP_KERNEL, vmcs_descriptor.order);
796 if (!pages)
797 return NULL;
798 vmcs = page_address(pages);
799 memset(vmcs, 0, vmcs_descriptor.size);
800 vmcs->revision_id = vmcs_descriptor.revision_id; /* vmcs revision id */
801 return vmcs;
802 }
803
804 static struct vmcs *alloc_vmcs(void)
805 {
806 return alloc_vmcs_cpu(raw_smp_processor_id());
807 }
808
809 static void free_vmcs(struct vmcs *vmcs)
810 {
811 free_pages((unsigned long)vmcs, vmcs_descriptor.order);
812 }
813
814 static void free_kvm_area(void)
815 {
816 int cpu;
817
818 for_each_online_cpu(cpu)
819 free_vmcs(per_cpu(vmxarea, cpu));
820 }
821
822 extern struct vmcs *alloc_vmcs_cpu(int cpu);
823
824 static __init int alloc_kvm_area(void)
825 {
826 int cpu;
827
828 for_each_online_cpu(cpu) {
829 struct vmcs *vmcs;
830
831 vmcs = alloc_vmcs_cpu(cpu);
832 if (!vmcs) {
833 free_kvm_area();
834 return -ENOMEM;
835 }
836
837 per_cpu(vmxarea, cpu) = vmcs;
838 }
839 return 0;
840 }
841
842 static __init int hardware_setup(void)
843 {
844 setup_vmcs_descriptor();
845 return alloc_kvm_area();
846 }
847
848 static __exit void hardware_unsetup(void)
849 {
850 free_kvm_area();
851 }
852
853 static void fix_pmode_dataseg(int seg, struct kvm_save_segment *save)
854 {
855 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
856
857 if (vmcs_readl(sf->base) == save->base && (save->base & AR_S_MASK)) {
858 vmcs_write16(sf->selector, save->selector);
859 vmcs_writel(sf->base, save->base);
860 vmcs_write32(sf->limit, save->limit);
861 vmcs_write32(sf->ar_bytes, save->ar);
862 } else {
863 u32 dpl = (vmcs_read16(sf->selector) & SELECTOR_RPL_MASK)
864 << AR_DPL_SHIFT;
865 vmcs_write32(sf->ar_bytes, 0x93 | dpl);
866 }
867 }
868
869 static void enter_pmode(struct kvm_vcpu *vcpu)
870 {
871 unsigned long flags;
872
873 vcpu->rmode.active = 0;
874
875 vmcs_writel(GUEST_TR_BASE, vcpu->rmode.tr.base);
876 vmcs_write32(GUEST_TR_LIMIT, vcpu->rmode.tr.limit);
877 vmcs_write32(GUEST_TR_AR_BYTES, vcpu->rmode.tr.ar);
878
879 flags = vmcs_readl(GUEST_RFLAGS);
880 flags &= ~(IOPL_MASK | X86_EFLAGS_VM);
881 flags |= (vcpu->rmode.save_iopl << IOPL_SHIFT);
882 vmcs_writel(GUEST_RFLAGS, flags);
883
884 vmcs_writel(GUEST_CR4, (vmcs_readl(GUEST_CR4) & ~CR4_VME_MASK) |
885 (vmcs_readl(CR4_READ_SHADOW) & CR4_VME_MASK));
886
887 update_exception_bitmap(vcpu);
888
889 fix_pmode_dataseg(VCPU_SREG_ES, &vcpu->rmode.es);
890 fix_pmode_dataseg(VCPU_SREG_DS, &vcpu->rmode.ds);
891 fix_pmode_dataseg(VCPU_SREG_GS, &vcpu->rmode.gs);
892 fix_pmode_dataseg(VCPU_SREG_FS, &vcpu->rmode.fs);
893
894 vmcs_write16(GUEST_SS_SELECTOR, 0);
895 vmcs_write32(GUEST_SS_AR_BYTES, 0x93);
896
897 vmcs_write16(GUEST_CS_SELECTOR,
898 vmcs_read16(GUEST_CS_SELECTOR) & ~SELECTOR_RPL_MASK);
899 vmcs_write32(GUEST_CS_AR_BYTES, 0x9b);
900 }
901
902 static int rmode_tss_base(struct kvm* kvm)
903 {
904 gfn_t base_gfn = kvm->memslots[0].base_gfn + kvm->memslots[0].npages - 3;
905 return base_gfn << PAGE_SHIFT;
906 }
907
908 static void fix_rmode_seg(int seg, struct kvm_save_segment *save)
909 {
910 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
911
912 save->selector = vmcs_read16(sf->selector);
913 save->base = vmcs_readl(sf->base);
914 save->limit = vmcs_read32(sf->limit);
915 save->ar = vmcs_read32(sf->ar_bytes);
916 vmcs_write16(sf->selector, vmcs_readl(sf->base) >> 4);
917 vmcs_write32(sf->limit, 0xffff);
918 vmcs_write32(sf->ar_bytes, 0xf3);
919 }
920
921 static void enter_rmode(struct kvm_vcpu *vcpu)
922 {
923 unsigned long flags;
924
925 vcpu->rmode.active = 1;
926
927 vcpu->rmode.tr.base = vmcs_readl(GUEST_TR_BASE);
928 vmcs_writel(GUEST_TR_BASE, rmode_tss_base(vcpu->kvm));
929
930 vcpu->rmode.tr.limit = vmcs_read32(GUEST_TR_LIMIT);
931 vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1);
932
933 vcpu->rmode.tr.ar = vmcs_read32(GUEST_TR_AR_BYTES);
934 vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
935
936 flags = vmcs_readl(GUEST_RFLAGS);
937 vcpu->rmode.save_iopl = (flags & IOPL_MASK) >> IOPL_SHIFT;
938
939 flags |= IOPL_MASK | X86_EFLAGS_VM;
940
941 vmcs_writel(GUEST_RFLAGS, flags);
942 vmcs_writel(GUEST_CR4, vmcs_readl(GUEST_CR4) | CR4_VME_MASK);
943 update_exception_bitmap(vcpu);
944
945 vmcs_write16(GUEST_SS_SELECTOR, vmcs_readl(GUEST_SS_BASE) >> 4);
946 vmcs_write32(GUEST_SS_LIMIT, 0xffff);
947 vmcs_write32(GUEST_SS_AR_BYTES, 0xf3);
948
949 vmcs_write32(GUEST_CS_AR_BYTES, 0xf3);
950 vmcs_write32(GUEST_CS_LIMIT, 0xffff);
951 if (vmcs_readl(GUEST_CS_BASE) == 0xffff0000)
952 vmcs_writel(GUEST_CS_BASE, 0xf0000);
953 vmcs_write16(GUEST_CS_SELECTOR, vmcs_readl(GUEST_CS_BASE) >> 4);
954
955 fix_rmode_seg(VCPU_SREG_ES, &vcpu->rmode.es);
956 fix_rmode_seg(VCPU_SREG_DS, &vcpu->rmode.ds);
957 fix_rmode_seg(VCPU_SREG_GS, &vcpu->rmode.gs);
958 fix_rmode_seg(VCPU_SREG_FS, &vcpu->rmode.fs);
959
960 kvm_mmu_reset_context(vcpu);
961 init_rmode_tss(vcpu->kvm);
962 }
963
964 #ifdef CONFIG_X86_64
965
966 static void enter_lmode(struct kvm_vcpu *vcpu)
967 {
968 u32 guest_tr_ar;
969
970 guest_tr_ar = vmcs_read32(GUEST_TR_AR_BYTES);
971 if ((guest_tr_ar & AR_TYPE_MASK) != AR_TYPE_BUSY_64_TSS) {
972 printk(KERN_DEBUG "%s: tss fixup for long mode. \n",
973 __FUNCTION__);
974 vmcs_write32(GUEST_TR_AR_BYTES,
975 (guest_tr_ar & ~AR_TYPE_MASK)
976 | AR_TYPE_BUSY_64_TSS);
977 }
978
979 vcpu->shadow_efer |= EFER_LMA;
980
981 find_msr_entry(vcpu, MSR_EFER)->data |= EFER_LMA | EFER_LME;
982 vmcs_write32(VM_ENTRY_CONTROLS,
983 vmcs_read32(VM_ENTRY_CONTROLS)
984 | VM_ENTRY_CONTROLS_IA32E_MASK);
985 }
986
987 static void exit_lmode(struct kvm_vcpu *vcpu)
988 {
989 vcpu->shadow_efer &= ~EFER_LMA;
990
991 vmcs_write32(VM_ENTRY_CONTROLS,
992 vmcs_read32(VM_ENTRY_CONTROLS)
993 & ~VM_ENTRY_CONTROLS_IA32E_MASK);
994 }
995
996 #endif
997
998 static void vmx_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
999 {
1000 vcpu->cr4 &= KVM_GUEST_CR4_MASK;
1001 vcpu->cr4 |= vmcs_readl(GUEST_CR4) & ~KVM_GUEST_CR4_MASK;
1002 }
1003
1004 static void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1005 {
1006 vmx_fpu_deactivate(vcpu);
1007
1008 if (vcpu->rmode.active && (cr0 & CR0_PE_MASK))
1009 enter_pmode(vcpu);
1010
1011 if (!vcpu->rmode.active && !(cr0 & CR0_PE_MASK))
1012 enter_rmode(vcpu);
1013
1014 #ifdef CONFIG_X86_64
1015 if (vcpu->shadow_efer & EFER_LME) {
1016 if (!is_paging(vcpu) && (cr0 & CR0_PG_MASK))
1017 enter_lmode(vcpu);
1018 if (is_paging(vcpu) && !(cr0 & CR0_PG_MASK))
1019 exit_lmode(vcpu);
1020 }
1021 #endif
1022
1023 vmcs_writel(CR0_READ_SHADOW, cr0);
1024 vmcs_writel(GUEST_CR0,
1025 (cr0 & ~KVM_GUEST_CR0_MASK) | KVM_VM_CR0_ALWAYS_ON);
1026 vcpu->cr0 = cr0;
1027
1028 if (!(cr0 & CR0_TS_MASK) || !(cr0 & CR0_PE_MASK))
1029 vmx_fpu_activate(vcpu);
1030 }
1031
1032 static void vmx_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
1033 {
1034 vmcs_writel(GUEST_CR3, cr3);
1035 if (vcpu->cr0 & CR0_PE_MASK)
1036 vmx_fpu_deactivate(vcpu);
1037 }
1038
1039 static void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1040 {
1041 vmcs_writel(CR4_READ_SHADOW, cr4);
1042 vmcs_writel(GUEST_CR4, cr4 | (vcpu->rmode.active ?
1043 KVM_RMODE_VM_CR4_ALWAYS_ON : KVM_PMODE_VM_CR4_ALWAYS_ON));
1044 vcpu->cr4 = cr4;
1045 }
1046
1047 #ifdef CONFIG_X86_64
1048
1049 static void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer)
1050 {
1051 struct vmx_msr_entry *msr = find_msr_entry(vcpu, MSR_EFER);
1052
1053 vcpu->shadow_efer = efer;
1054 if (efer & EFER_LMA) {
1055 vmcs_write32(VM_ENTRY_CONTROLS,
1056 vmcs_read32(VM_ENTRY_CONTROLS) |
1057 VM_ENTRY_CONTROLS_IA32E_MASK);
1058 msr->data = efer;
1059
1060 } else {
1061 vmcs_write32(VM_ENTRY_CONTROLS,
1062 vmcs_read32(VM_ENTRY_CONTROLS) &
1063 ~VM_ENTRY_CONTROLS_IA32E_MASK);
1064
1065 msr->data = efer & ~EFER_LME;
1066 }
1067 setup_msrs(vcpu);
1068 }
1069
1070 #endif
1071
1072 static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg)
1073 {
1074 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1075
1076 return vmcs_readl(sf->base);
1077 }
1078
1079 static void vmx_get_segment(struct kvm_vcpu *vcpu,
1080 struct kvm_segment *var, int seg)
1081 {
1082 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1083 u32 ar;
1084
1085 var->base = vmcs_readl(sf->base);
1086 var->limit = vmcs_read32(sf->limit);
1087 var->selector = vmcs_read16(sf->selector);
1088 ar = vmcs_read32(sf->ar_bytes);
1089 if (ar & AR_UNUSABLE_MASK)
1090 ar = 0;
1091 var->type = ar & 15;
1092 var->s = (ar >> 4) & 1;
1093 var->dpl = (ar >> 5) & 3;
1094 var->present = (ar >> 7) & 1;
1095 var->avl = (ar >> 12) & 1;
1096 var->l = (ar >> 13) & 1;
1097 var->db = (ar >> 14) & 1;
1098 var->g = (ar >> 15) & 1;
1099 var->unusable = (ar >> 16) & 1;
1100 }
1101
1102 static u32 vmx_segment_access_rights(struct kvm_segment *var)
1103 {
1104 u32 ar;
1105
1106 if (var->unusable)
1107 ar = 1 << 16;
1108 else {
1109 ar = var->type & 15;
1110 ar |= (var->s & 1) << 4;
1111 ar |= (var->dpl & 3) << 5;
1112 ar |= (var->present & 1) << 7;
1113 ar |= (var->avl & 1) << 12;
1114 ar |= (var->l & 1) << 13;
1115 ar |= (var->db & 1) << 14;
1116 ar |= (var->g & 1) << 15;
1117 }
1118 if (ar == 0) /* a 0 value means unusable */
1119 ar = AR_UNUSABLE_MASK;
1120
1121 return ar;
1122 }
1123
1124 static void vmx_set_segment(struct kvm_vcpu *vcpu,
1125 struct kvm_segment *var, int seg)
1126 {
1127 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1128 u32 ar;
1129
1130 if (vcpu->rmode.active && seg == VCPU_SREG_TR) {
1131 vcpu->rmode.tr.selector = var->selector;
1132 vcpu->rmode.tr.base = var->base;
1133 vcpu->rmode.tr.limit = var->limit;
1134 vcpu->rmode.tr.ar = vmx_segment_access_rights(var);
1135 return;
1136 }
1137 vmcs_writel(sf->base, var->base);
1138 vmcs_write32(sf->limit, var->limit);
1139 vmcs_write16(sf->selector, var->selector);
1140 if (vcpu->rmode.active && var->s) {
1141 /*
1142 * Hack real-mode segments into vm86 compatibility.
1143 */
1144 if (var->base == 0xffff0000 && var->selector == 0xf000)
1145 vmcs_writel(sf->base, 0xf0000);
1146 ar = 0xf3;
1147 } else
1148 ar = vmx_segment_access_rights(var);
1149 vmcs_write32(sf->ar_bytes, ar);
1150 }
1151
1152 static void vmx_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l)
1153 {
1154 u32 ar = vmcs_read32(GUEST_CS_AR_BYTES);
1155
1156 *db = (ar >> 14) & 1;
1157 *l = (ar >> 13) & 1;
1158 }
1159
1160 static void vmx_get_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
1161 {
1162 dt->limit = vmcs_read32(GUEST_IDTR_LIMIT);
1163 dt->base = vmcs_readl(GUEST_IDTR_BASE);
1164 }
1165
1166 static void vmx_set_idt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
1167 {
1168 vmcs_write32(GUEST_IDTR_LIMIT, dt->limit);
1169 vmcs_writel(GUEST_IDTR_BASE, dt->base);
1170 }
1171
1172 static void vmx_get_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
1173 {
1174 dt->limit = vmcs_read32(GUEST_GDTR_LIMIT);
1175 dt->base = vmcs_readl(GUEST_GDTR_BASE);
1176 }
1177
1178 static void vmx_set_gdt(struct kvm_vcpu *vcpu, struct descriptor_table *dt)
1179 {
1180 vmcs_write32(GUEST_GDTR_LIMIT, dt->limit);
1181 vmcs_writel(GUEST_GDTR_BASE, dt->base);
1182 }
1183
1184 static int init_rmode_tss(struct kvm* kvm)
1185 {
1186 struct page *p1, *p2, *p3;
1187 gfn_t fn = rmode_tss_base(kvm) >> PAGE_SHIFT;
1188 char *page;
1189
1190 p1 = gfn_to_page(kvm, fn++);
1191 p2 = gfn_to_page(kvm, fn++);
1192 p3 = gfn_to_page(kvm, fn);
1193
1194 if (!p1 || !p2 || !p3) {
1195 kvm_printf(kvm,"%s: gfn_to_page failed\n", __FUNCTION__);
1196 return 0;
1197 }
1198
1199 page = kmap_atomic(p1, KM_USER0);
1200 clear_page(page);
1201 *(u16*)(page + 0x66) = TSS_BASE_SIZE + TSS_REDIRECTION_SIZE;
1202 kunmap_atomic(page, KM_USER0);
1203
1204 page = kmap_atomic(p2, KM_USER0);
1205 clear_page(page);
1206 kunmap_atomic(page, KM_USER0);
1207
1208 page = kmap_atomic(p3, KM_USER0);
1209 clear_page(page);
1210 *(page + RMODE_TSS_SIZE - 2 * PAGE_SIZE - 1) = ~0;
1211 kunmap_atomic(page, KM_USER0);
1212
1213 return 1;
1214 }
1215
1216 static void vmcs_write32_fixedbits(u32 msr, u32 vmcs_field, u32 val)
1217 {
1218 u32 msr_high, msr_low;
1219
1220 rdmsr(msr, msr_low, msr_high);
1221
1222 val &= msr_high;
1223 val |= msr_low;
1224 vmcs_write32(vmcs_field, val);
1225 }
1226
1227 static void seg_setup(int seg)
1228 {
1229 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
1230
1231 vmcs_write16(sf->selector, 0);
1232 vmcs_writel(sf->base, 0);
1233 vmcs_write32(sf->limit, 0xffff);
1234 vmcs_write32(sf->ar_bytes, 0x93);
1235 }
1236
1237 /*
1238 * Sets up the vmcs for emulated real mode.
1239 */
1240 static int vmx_vcpu_setup(struct kvm_vcpu *vcpu)
1241 {
1242 u32 host_sysenter_cs;
1243 u32 junk;
1244 unsigned long a;
1245 struct descriptor_table dt;
1246 int i;
1247 int ret = 0;
1248 unsigned long kvm_vmx_return;
1249
1250 if (!init_rmode_tss(vcpu->kvm)) {
1251 ret = -ENOMEM;
1252 goto out;
1253 }
1254
1255 memset(vcpu->regs, 0, sizeof(vcpu->regs));
1256 vcpu->regs[VCPU_REGS_RDX] = get_rdx_init_val();
1257 vcpu->cr8 = 0;
1258 vcpu->apic_base = 0xfee00000 | MSR_IA32_APICBASE_ENABLE;
1259 if (vcpu == &vcpu->kvm->vcpus[0])
1260 vcpu->apic_base |= MSR_IA32_APICBASE_BSP;
1261
1262 fx_init(vcpu);
1263
1264 /*
1265 * GUEST_CS_BASE should really be 0xffff0000, but VT vm86 mode
1266 * insists on having GUEST_CS_BASE == GUEST_CS_SELECTOR << 4. Sigh.
1267 */
1268 vmcs_write16(GUEST_CS_SELECTOR, 0xf000);
1269 vmcs_writel(GUEST_CS_BASE, 0x000f0000);
1270 vmcs_write32(GUEST_CS_LIMIT, 0xffff);
1271 vmcs_write32(GUEST_CS_AR_BYTES, 0x9b);
1272
1273 seg_setup(VCPU_SREG_DS);
1274 seg_setup(VCPU_SREG_ES);
1275 seg_setup(VCPU_SREG_FS);
1276 seg_setup(VCPU_SREG_GS);
1277 seg_setup(VCPU_SREG_SS);
1278
1279 vmcs_write16(GUEST_TR_SELECTOR, 0);
1280 vmcs_writel(GUEST_TR_BASE, 0);
1281 vmcs_write32(GUEST_TR_LIMIT, 0xffff);
1282 vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
1283
1284 vmcs_write16(GUEST_LDTR_SELECTOR, 0);
1285 vmcs_writel(GUEST_LDTR_BASE, 0);
1286 vmcs_write32(GUEST_LDTR_LIMIT, 0xffff);
1287 vmcs_write32(GUEST_LDTR_AR_BYTES, 0x00082);
1288
1289 vmcs_write32(GUEST_SYSENTER_CS, 0);
1290 vmcs_writel(GUEST_SYSENTER_ESP, 0);
1291 vmcs_writel(GUEST_SYSENTER_EIP, 0);
1292
1293 vmcs_writel(GUEST_RFLAGS, 0x02);
1294 vmcs_writel(GUEST_RIP, 0xfff0);
1295 vmcs_writel(GUEST_RSP, 0);
1296
1297 //todo: dr0 = dr1 = dr2 = dr3 = 0; dr6 = 0xffff0ff0
1298 vmcs_writel(GUEST_DR7, 0x400);
1299
1300 vmcs_writel(GUEST_GDTR_BASE, 0);
1301 vmcs_write32(GUEST_GDTR_LIMIT, 0xffff);
1302
1303 vmcs_writel(GUEST_IDTR_BASE, 0);
1304 vmcs_write32(GUEST_IDTR_LIMIT, 0xffff);
1305
1306 vmcs_write32(GUEST_ACTIVITY_STATE, 0);
1307 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
1308 vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0);
1309
1310 /* I/O */
1311 vmcs_write64(IO_BITMAP_A, page_to_phys(vmx_io_bitmap_a));
1312 vmcs_write64(IO_BITMAP_B, page_to_phys(vmx_io_bitmap_b));
1313
1314 guest_write_tsc(0);
1315
1316 vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */
1317
1318 /* Special registers */
1319 vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
1320
1321 /* Control */
1322 vmcs_write32_fixedbits(MSR_IA32_VMX_PINBASED_CTLS,
1323 PIN_BASED_VM_EXEC_CONTROL,
1324 PIN_BASED_EXT_INTR_MASK /* 20.6.1 */
1325 | PIN_BASED_NMI_EXITING /* 20.6.1 */
1326 );
1327 vmcs_write32_fixedbits(MSR_IA32_VMX_PROCBASED_CTLS,
1328 CPU_BASED_VM_EXEC_CONTROL,
1329 CPU_BASED_HLT_EXITING /* 20.6.2 */
1330 | CPU_BASED_CR8_LOAD_EXITING /* 20.6.2 */
1331 | CPU_BASED_CR8_STORE_EXITING /* 20.6.2 */
1332 | CPU_BASED_ACTIVATE_IO_BITMAP /* 20.6.2 */
1333 | CPU_BASED_MOV_DR_EXITING
1334 | CPU_BASED_USE_TSC_OFFSETING /* 21.3 */
1335 );
1336
1337 vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0);
1338 vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0);
1339 vmcs_write32(CR3_TARGET_COUNT, 0); /* 22.2.1 */
1340
1341 vmcs_writel(HOST_CR0, read_cr0()); /* 22.2.3 */
1342 vmcs_writel(HOST_CR4, read_cr4()); /* 22.2.3, 22.2.5 */
1343 vmcs_writel(HOST_CR3, read_cr3()); /* 22.2.3 FIXME: shadow tables */
1344
1345 vmcs_write16(HOST_CS_SELECTOR, __KERNEL_CS); /* 22.2.4 */
1346 vmcs_write16(HOST_DS_SELECTOR, __KERNEL_DS); /* 22.2.4 */
1347 vmcs_write16(HOST_ES_SELECTOR, __KERNEL_DS); /* 22.2.4 */
1348 vmcs_write16(HOST_FS_SELECTOR, read_fs()); /* 22.2.4 */
1349 vmcs_write16(HOST_GS_SELECTOR, read_gs()); /* 22.2.4 */
1350 vmcs_write16(HOST_SS_SELECTOR, __KERNEL_DS); /* 22.2.4 */
1351 #ifdef CONFIG_X86_64
1352 rdmsrl(MSR_FS_BASE, a);
1353 vmcs_writel(HOST_FS_BASE, a); /* 22.2.4 */
1354 rdmsrl(MSR_GS_BASE, a);
1355 vmcs_writel(HOST_GS_BASE, a); /* 22.2.4 */
1356 #else
1357 vmcs_writel(HOST_FS_BASE, 0); /* 22.2.4 */
1358 vmcs_writel(HOST_GS_BASE, 0); /* 22.2.4 */
1359 #endif
1360
1361 vmcs_write16(HOST_TR_SELECTOR, GDT_ENTRY_TSS*8); /* 22.2.4 */
1362
1363 get_idt(&dt);
1364 vmcs_writel(HOST_IDTR_BASE, dt.base); /* 22.2.4 */
1365
1366 asm ("mov $.Lkvm_vmx_return, %0" : "=r"(kvm_vmx_return));
1367 vmcs_writel(HOST_RIP, kvm_vmx_return); /* 22.2.5 */
1368 vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
1369 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
1370 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
1371
1372 rdmsr(MSR_IA32_SYSENTER_CS, host_sysenter_cs, junk);
1373 vmcs_write32(HOST_IA32_SYSENTER_CS, host_sysenter_cs);
1374 rdmsrl(MSR_IA32_SYSENTER_ESP, a);
1375 vmcs_writel(HOST_IA32_SYSENTER_ESP, a); /* 22.2.3 */
1376 rdmsrl(MSR_IA32_SYSENTER_EIP, a);
1377 vmcs_writel(HOST_IA32_SYSENTER_EIP, a); /* 22.2.3 */
1378
1379 for (i = 0; i < NR_VMX_MSR; ++i) {
1380 u32 index = vmx_msr_index[i];
1381 u32 data_low, data_high;
1382 u64 data;
1383 int j = vcpu->nmsrs;
1384
1385 if (rdmsr_safe(index, &data_low, &data_high) < 0)
1386 continue;
1387 if (wrmsr_safe(index, data_low, data_high) < 0)
1388 continue;
1389 data = data_low | ((u64)data_high << 32);
1390 vcpu->host_msrs[j].index = index;
1391 vcpu->host_msrs[j].reserved = 0;
1392 vcpu->host_msrs[j].data = data;
1393 vcpu->guest_msrs[j] = vcpu->host_msrs[j];
1394 ++vcpu->nmsrs;
1395 }
1396
1397 setup_msrs(vcpu);
1398
1399 vmcs_write32_fixedbits(MSR_IA32_VMX_EXIT_CTLS, VM_EXIT_CONTROLS,
1400 (HOST_IS_64 << 9)); /* 22.2,1, 20.7.1 */
1401
1402 /* 22.2.1, 20.8.1 */
1403 vmcs_write32_fixedbits(MSR_IA32_VMX_ENTRY_CTLS,
1404 VM_ENTRY_CONTROLS, 0);
1405 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0); /* 22.2.1 */
1406
1407 #ifdef CONFIG_X86_64
1408 vmcs_writel(VIRTUAL_APIC_PAGE_ADDR, 0);
1409 vmcs_writel(TPR_THRESHOLD, 0);
1410 #endif
1411
1412 vmcs_writel(CR0_GUEST_HOST_MASK, ~0UL);
1413 vmcs_writel(CR4_GUEST_HOST_MASK, KVM_GUEST_CR4_MASK);
1414
1415 vcpu->cr0 = 0x60000010;
1416 vmx_set_cr0(vcpu, vcpu->cr0); // enter rmode
1417 vmx_set_cr4(vcpu, 0);
1418 #ifdef CONFIG_X86_64
1419 vmx_set_efer(vcpu, 0);
1420 #endif
1421 vmx_fpu_activate(vcpu);
1422 update_exception_bitmap(vcpu);
1423
1424 return 0;
1425
1426 out:
1427 return ret;
1428 }
1429
1430 static void inject_rmode_irq(struct kvm_vcpu *vcpu, int irq)
1431 {
1432 u16 ent[2];
1433 u16 cs;
1434 u16 ip;
1435 unsigned long flags;
1436 unsigned long ss_base = vmcs_readl(GUEST_SS_BASE);
1437 u16 sp = vmcs_readl(GUEST_RSP);
1438 u32 ss_limit = vmcs_read32(GUEST_SS_LIMIT);
1439
1440 if (sp > ss_limit || sp < 6 ) {
1441 vcpu_printf(vcpu, "%s: #SS, rsp 0x%lx ss 0x%lx limit 0x%x\n",
1442 __FUNCTION__,
1443 vmcs_readl(GUEST_RSP),
1444 vmcs_readl(GUEST_SS_BASE),
1445 vmcs_read32(GUEST_SS_LIMIT));
1446 return;
1447 }
1448
1449 if (kvm_read_guest(vcpu, irq * sizeof(ent), sizeof(ent), &ent) !=
1450 sizeof(ent)) {
1451 vcpu_printf(vcpu, "%s: read guest err\n", __FUNCTION__);
1452 return;
1453 }
1454
1455 flags = vmcs_readl(GUEST_RFLAGS);
1456 cs = vmcs_readl(GUEST_CS_BASE) >> 4;
1457 ip = vmcs_readl(GUEST_RIP);
1458
1459
1460 if (kvm_write_guest(vcpu, ss_base + sp - 2, 2, &flags) != 2 ||
1461 kvm_write_guest(vcpu, ss_base + sp - 4, 2, &cs) != 2 ||
1462 kvm_write_guest(vcpu, ss_base + sp - 6, 2, &ip) != 2) {
1463 vcpu_printf(vcpu, "%s: write guest err\n", __FUNCTION__);
1464 return;
1465 }
1466
1467 vmcs_writel(GUEST_RFLAGS, flags &
1468 ~( X86_EFLAGS_IF | X86_EFLAGS_AC | X86_EFLAGS_TF));
1469 vmcs_write16(GUEST_CS_SELECTOR, ent[1]) ;
1470 vmcs_writel(GUEST_CS_BASE, ent[1] << 4);
1471 vmcs_writel(GUEST_RIP, ent[0]);
1472 vmcs_writel(GUEST_RSP, (vmcs_readl(GUEST_RSP) & ~0xffff) | (sp - 6));
1473 }
1474
1475 static void kvm_do_inject_irq(struct kvm_vcpu *vcpu)
1476 {
1477 int word_index = __ffs(vcpu->irq_summary);
1478 int bit_index = __ffs(vcpu->irq_pending[word_index]);
1479 int irq = word_index * BITS_PER_LONG + bit_index;
1480
1481 clear_bit(bit_index, &vcpu->irq_pending[word_index]);
1482 if (!vcpu->irq_pending[word_index])
1483 clear_bit(word_index, &vcpu->irq_summary);
1484
1485 if (vcpu->rmode.active) {
1486 inject_rmode_irq(vcpu, irq);
1487 return;
1488 }
1489 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
1490 irq | INTR_TYPE_EXT_INTR | INTR_INFO_VALID_MASK);
1491 }
1492
1493
1494 static void do_interrupt_requests(struct kvm_vcpu *vcpu,
1495 struct kvm_run *kvm_run)
1496 {
1497 u32 cpu_based_vm_exec_control;
1498
1499 vcpu->interrupt_window_open =
1500 ((vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) &&
1501 (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0);
1502
1503 if (vcpu->interrupt_window_open &&
1504 vcpu->irq_summary &&
1505 !(vmcs_read32(VM_ENTRY_INTR_INFO_FIELD) & INTR_INFO_VALID_MASK))
1506 /*
1507 * If interrupts enabled, and not blocked by sti or mov ss. Good.
1508 */
1509 kvm_do_inject_irq(vcpu);
1510
1511 cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
1512 if (!vcpu->interrupt_window_open &&
1513 (vcpu->irq_summary || kvm_run->request_interrupt_window))
1514 /*
1515 * Interrupts blocked. Wait for unblock.
1516 */
1517 cpu_based_vm_exec_control |= CPU_BASED_VIRTUAL_INTR_PENDING;
1518 else
1519 cpu_based_vm_exec_control &= ~CPU_BASED_VIRTUAL_INTR_PENDING;
1520 vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
1521 }
1522
1523 static void kvm_guest_debug_pre(struct kvm_vcpu *vcpu)
1524 {
1525 struct kvm_guest_debug *dbg = &vcpu->guest_debug;
1526
1527 set_debugreg(dbg->bp[0], 0);
1528 set_debugreg(dbg->bp[1], 1);
1529 set_debugreg(dbg->bp[2], 2);
1530 set_debugreg(dbg->bp[3], 3);
1531
1532 if (dbg->singlestep) {
1533 unsigned long flags;
1534
1535 flags = vmcs_readl(GUEST_RFLAGS);
1536 flags |= X86_EFLAGS_TF | X86_EFLAGS_RF;
1537 vmcs_writel(GUEST_RFLAGS, flags);
1538 }
1539 }
1540
1541 static int handle_rmode_exception(struct kvm_vcpu *vcpu,
1542 int vec, u32 err_code)
1543 {
1544 if (!vcpu->rmode.active)
1545 return 0;
1546
1547 /*
1548 * Instruction with address size override prefix opcode 0x67
1549 * Cause the #SS fault with 0 error code in VM86 mode.
1550 */
1551 if (((vec == GP_VECTOR) || (vec == SS_VECTOR)) && err_code == 0)
1552 if (emulate_instruction(vcpu, NULL, 0, 0) == EMULATE_DONE)
1553 return 1;
1554 return 0;
1555 }
1556
1557 static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1558 {
1559 u32 intr_info, error_code;
1560 unsigned long cr2, rip;
1561 u32 vect_info;
1562 enum emulation_result er;
1563 int r;
1564
1565 vect_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
1566 intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
1567
1568 if ((vect_info & VECTORING_INFO_VALID_MASK) &&
1569 !is_page_fault(intr_info)) {
1570 printk(KERN_ERR "%s: unexpected, vectoring info 0x%x "
1571 "intr info 0x%x\n", __FUNCTION__, vect_info, intr_info);
1572 }
1573
1574 if (is_external_interrupt(vect_info)) {
1575 int irq = vect_info & VECTORING_INFO_VECTOR_MASK;
1576 set_bit(irq, vcpu->irq_pending);
1577 set_bit(irq / BITS_PER_LONG, &vcpu->irq_summary);
1578 }
1579
1580 if ((intr_info & INTR_INFO_INTR_TYPE_MASK) == 0x200) { /* nmi */
1581 asm ("int $2");
1582 return 1;
1583 }
1584
1585 if (is_no_device(intr_info)) {
1586 vmx_fpu_activate(vcpu);
1587 return 1;
1588 }
1589
1590 error_code = 0;
1591 rip = vmcs_readl(GUEST_RIP);
1592 if (intr_info & INTR_INFO_DELIEVER_CODE_MASK)
1593 error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
1594 if (is_page_fault(intr_info)) {
1595 cr2 = vmcs_readl(EXIT_QUALIFICATION);
1596
1597 spin_lock(&vcpu->kvm->lock);
1598 r = kvm_mmu_page_fault(vcpu, cr2, error_code);
1599 if (r < 0) {
1600 spin_unlock(&vcpu->kvm->lock);
1601 return r;
1602 }
1603 if (!r) {
1604 spin_unlock(&vcpu->kvm->lock);
1605 return 1;
1606 }
1607
1608 er = emulate_instruction(vcpu, kvm_run, cr2, error_code);
1609 spin_unlock(&vcpu->kvm->lock);
1610
1611 switch (er) {
1612 case EMULATE_DONE:
1613 return 1;
1614 case EMULATE_DO_MMIO:
1615 ++vcpu->stat.mmio_exits;
1616 kvm_run->exit_reason = KVM_EXIT_MMIO;
1617 return 0;
1618 case EMULATE_FAIL:
1619 vcpu_printf(vcpu, "%s: emulate fail\n", __FUNCTION__);
1620 break;
1621 default:
1622 BUG();
1623 }
1624 }
1625
1626 if (vcpu->rmode.active &&
1627 handle_rmode_exception(vcpu, intr_info & INTR_INFO_VECTOR_MASK,
1628 error_code)) {
1629 if (vcpu->halt_request) {
1630 vcpu->halt_request = 0;
1631 return kvm_emulate_halt(vcpu);
1632 }
1633 return 1;
1634 }
1635
1636 if ((intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK)) == (INTR_TYPE_EXCEPTION | 1)) {
1637 kvm_run->exit_reason = KVM_EXIT_DEBUG;
1638 return 0;
1639 }
1640 kvm_run->exit_reason = KVM_EXIT_EXCEPTION;
1641 kvm_run->ex.exception = intr_info & INTR_INFO_VECTOR_MASK;
1642 kvm_run->ex.error_code = error_code;
1643 return 0;
1644 }
1645
1646 static int handle_external_interrupt(struct kvm_vcpu *vcpu,
1647 struct kvm_run *kvm_run)
1648 {
1649 ++vcpu->stat.irq_exits;
1650 return 1;
1651 }
1652
1653 static int handle_triple_fault(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1654 {
1655 kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
1656 return 0;
1657 }
1658
1659 static int get_io_count(struct kvm_vcpu *vcpu, unsigned long *count)
1660 {
1661 u64 inst;
1662 gva_t rip;
1663 int countr_size;
1664 int i, n;
1665
1666 if ((vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_VM)) {
1667 countr_size = 2;
1668 } else {
1669 u32 cs_ar = vmcs_read32(GUEST_CS_AR_BYTES);
1670
1671 countr_size = (cs_ar & AR_L_MASK) ? 8:
1672 (cs_ar & AR_DB_MASK) ? 4: 2;
1673 }
1674
1675 rip = vmcs_readl(GUEST_RIP);
1676 if (countr_size != 8)
1677 rip += vmcs_readl(GUEST_CS_BASE);
1678
1679 n = kvm_read_guest(vcpu, rip, sizeof(inst), &inst);
1680
1681 for (i = 0; i < n; i++) {
1682 switch (((u8*)&inst)[i]) {
1683 case 0xf0:
1684 case 0xf2:
1685 case 0xf3:
1686 case 0x2e:
1687 case 0x36:
1688 case 0x3e:
1689 case 0x26:
1690 case 0x64:
1691 case 0x65:
1692 case 0x66:
1693 break;
1694 case 0x67:
1695 countr_size = (countr_size == 2) ? 4: (countr_size >> 1);
1696 default:
1697 goto done;
1698 }
1699 }
1700 return 0;
1701 done:
1702 countr_size *= 8;
1703 *count = vcpu->regs[VCPU_REGS_RCX] & (~0ULL >> (64 - countr_size));
1704 //printk("cx: %lx\n", vcpu->regs[VCPU_REGS_RCX]);
1705 return 1;
1706 }
1707
1708 static int handle_io(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1709 {
1710 u64 exit_qualification;
1711 int size, down, in, string, rep;
1712 unsigned port;
1713 unsigned long count;
1714 gva_t address;
1715
1716 ++vcpu->stat.io_exits;
1717 exit_qualification = vmcs_read64(EXIT_QUALIFICATION);
1718 in = (exit_qualification & 8) != 0;
1719 size = (exit_qualification & 7) + 1;
1720 string = (exit_qualification & 16) != 0;
1721 down = (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_DF) != 0;
1722 count = 1;
1723 rep = (exit_qualification & 32) != 0;
1724 port = exit_qualification >> 16;
1725 address = 0;
1726 if (string) {
1727 if (rep && !get_io_count(vcpu, &count))
1728 return 1;
1729 address = vmcs_readl(GUEST_LINEAR_ADDRESS);
1730 }
1731 return kvm_setup_pio(vcpu, kvm_run, in, size, count, string, down,
1732 address, rep, port);
1733 }
1734
1735 static void
1736 vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
1737 {
1738 /*
1739 * Patch in the VMCALL instruction:
1740 */
1741 hypercall[0] = 0x0f;
1742 hypercall[1] = 0x01;
1743 hypercall[2] = 0xc1;
1744 hypercall[3] = 0xc3;
1745 }
1746
1747 static int handle_cr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1748 {
1749 u64 exit_qualification;
1750 int cr;
1751 int reg;
1752
1753 exit_qualification = vmcs_read64(EXIT_QUALIFICATION);
1754 cr = exit_qualification & 15;
1755 reg = (exit_qualification >> 8) & 15;
1756 switch ((exit_qualification >> 4) & 3) {
1757 case 0: /* mov to cr */
1758 switch (cr) {
1759 case 0:
1760 vcpu_load_rsp_rip(vcpu);
1761 set_cr0(vcpu, vcpu->regs[reg]);
1762 skip_emulated_instruction(vcpu);
1763 return 1;
1764 case 3:
1765 vcpu_load_rsp_rip(vcpu);
1766 set_cr3(vcpu, vcpu->regs[reg]);
1767 skip_emulated_instruction(vcpu);
1768 return 1;
1769 case 4:
1770 vcpu_load_rsp_rip(vcpu);
1771 set_cr4(vcpu, vcpu->regs[reg]);
1772 skip_emulated_instruction(vcpu);
1773 return 1;
1774 case 8:
1775 vcpu_load_rsp_rip(vcpu);
1776 set_cr8(vcpu, vcpu->regs[reg]);
1777 skip_emulated_instruction(vcpu);
1778 return 1;
1779 };
1780 break;
1781 case 2: /* clts */
1782 vcpu_load_rsp_rip(vcpu);
1783 vmx_fpu_deactivate(vcpu);
1784 vcpu->cr0 &= ~CR0_TS_MASK;
1785 vmcs_writel(CR0_READ_SHADOW, vcpu->cr0);
1786 vmx_fpu_activate(vcpu);
1787 skip_emulated_instruction(vcpu);
1788 return 1;
1789 case 1: /*mov from cr*/
1790 switch (cr) {
1791 case 3:
1792 vcpu_load_rsp_rip(vcpu);
1793 vcpu->regs[reg] = vcpu->cr3;
1794 vcpu_put_rsp_rip(vcpu);
1795 skip_emulated_instruction(vcpu);
1796 return 1;
1797 case 8:
1798 vcpu_load_rsp_rip(vcpu);
1799 vcpu->regs[reg] = vcpu->cr8;
1800 vcpu_put_rsp_rip(vcpu);
1801 skip_emulated_instruction(vcpu);
1802 return 1;
1803 }
1804 break;
1805 case 3: /* lmsw */
1806 lmsw(vcpu, (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f);
1807
1808 skip_emulated_instruction(vcpu);
1809 return 1;
1810 default:
1811 break;
1812 }
1813 kvm_run->exit_reason = 0;
1814 printk(KERN_ERR "kvm: unhandled control register: op %d cr %d\n",
1815 (int)(exit_qualification >> 4) & 3, cr);
1816 return 0;
1817 }
1818
1819 static int handle_dr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1820 {
1821 u64 exit_qualification;
1822 unsigned long val;
1823 int dr, reg;
1824
1825 /*
1826 * FIXME: this code assumes the host is debugging the guest.
1827 * need to deal with guest debugging itself too.
1828 */
1829 exit_qualification = vmcs_read64(EXIT_QUALIFICATION);
1830 dr = exit_qualification & 7;
1831 reg = (exit_qualification >> 8) & 15;
1832 vcpu_load_rsp_rip(vcpu);
1833 if (exit_qualification & 16) {
1834 /* mov from dr */
1835 switch (dr) {
1836 case 6:
1837 val = 0xffff0ff0;
1838 break;
1839 case 7:
1840 val = 0x400;
1841 break;
1842 default:
1843 val = 0;
1844 }
1845 vcpu->regs[reg] = val;
1846 } else {
1847 /* mov to dr */
1848 }
1849 vcpu_put_rsp_rip(vcpu);
1850 skip_emulated_instruction(vcpu);
1851 return 1;
1852 }
1853
1854 static int handle_cpuid(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1855 {
1856 kvm_emulate_cpuid(vcpu);
1857 return 1;
1858 }
1859
1860 static int handle_rdmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1861 {
1862 u32 ecx = vcpu->regs[VCPU_REGS_RCX];
1863 u64 data;
1864
1865 if (vmx_get_msr(vcpu, ecx, &data)) {
1866 vmx_inject_gp(vcpu, 0);
1867 return 1;
1868 }
1869
1870 /* FIXME: handling of bits 32:63 of rax, rdx */
1871 vcpu->regs[VCPU_REGS_RAX] = data & -1u;
1872 vcpu->regs[VCPU_REGS_RDX] = (data >> 32) & -1u;
1873 skip_emulated_instruction(vcpu);
1874 return 1;
1875 }
1876
1877 static int handle_wrmsr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1878 {
1879 u32 ecx = vcpu->regs[VCPU_REGS_RCX];
1880 u64 data = (vcpu->regs[VCPU_REGS_RAX] & -1u)
1881 | ((u64)(vcpu->regs[VCPU_REGS_RDX] & -1u) << 32);
1882
1883 if (vmx_set_msr(vcpu, ecx, data) != 0) {
1884 vmx_inject_gp(vcpu, 0);
1885 return 1;
1886 }
1887
1888 skip_emulated_instruction(vcpu);
1889 return 1;
1890 }
1891
1892 static void post_kvm_run_save(struct kvm_vcpu *vcpu,
1893 struct kvm_run *kvm_run)
1894 {
1895 kvm_run->if_flag = (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF) != 0;
1896 kvm_run->cr8 = vcpu->cr8;
1897 kvm_run->apic_base = vcpu->apic_base;
1898 kvm_run->ready_for_interrupt_injection = (vcpu->interrupt_window_open &&
1899 vcpu->irq_summary == 0);
1900 }
1901
1902 static int handle_interrupt_window(struct kvm_vcpu *vcpu,
1903 struct kvm_run *kvm_run)
1904 {
1905 /*
1906 * If the user space waits to inject interrupts, exit as soon as
1907 * possible
1908 */
1909 if (kvm_run->request_interrupt_window &&
1910 !vcpu->irq_summary) {
1911 kvm_run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
1912 ++vcpu->stat.irq_window_exits;
1913 return 0;
1914 }
1915 return 1;
1916 }
1917
1918 static int handle_halt(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1919 {
1920 skip_emulated_instruction(vcpu);
1921 return kvm_emulate_halt(vcpu);
1922 }
1923
1924 static int handle_vmcall(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1925 {
1926 skip_emulated_instruction(vcpu);
1927 return kvm_hypercall(vcpu, kvm_run);
1928 }
1929
1930 /*
1931 * The exit handlers return 1 if the exit was handled fully and guest execution
1932 * may resume. Otherwise they set the kvm_run parameter to indicate what needs
1933 * to be done to userspace and return 0.
1934 */
1935 static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu,
1936 struct kvm_run *kvm_run) = {
1937 [EXIT_REASON_EXCEPTION_NMI] = handle_exception,
1938 [EXIT_REASON_EXTERNAL_INTERRUPT] = handle_external_interrupt,
1939 [EXIT_REASON_TRIPLE_FAULT] = handle_triple_fault,
1940 [EXIT_REASON_IO_INSTRUCTION] = handle_io,
1941 [EXIT_REASON_CR_ACCESS] = handle_cr,
1942 [EXIT_REASON_DR_ACCESS] = handle_dr,
1943 [EXIT_REASON_CPUID] = handle_cpuid,
1944 [EXIT_REASON_MSR_READ] = handle_rdmsr,
1945 [EXIT_REASON_MSR_WRITE] = handle_wrmsr,
1946 [EXIT_REASON_PENDING_INTERRUPT] = handle_interrupt_window,
1947 [EXIT_REASON_HLT] = handle_halt,
1948 [EXIT_REASON_VMCALL] = handle_vmcall,
1949 };
1950
1951 static const int kvm_vmx_max_exit_handlers =
1952 ARRAY_SIZE(kvm_vmx_exit_handlers);
1953
1954 /*
1955 * The guest has exited. See if we can fix it or if we need userspace
1956 * assistance.
1957 */
1958 static int kvm_handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
1959 {
1960 u32 vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
1961 u32 exit_reason = vmcs_read32(VM_EXIT_REASON);
1962
1963 if ( (vectoring_info & VECTORING_INFO_VALID_MASK) &&
1964 exit_reason != EXIT_REASON_EXCEPTION_NMI )
1965 printk(KERN_WARNING "%s: unexpected, valid vectoring info and "
1966 "exit reason is 0x%x\n", __FUNCTION__, exit_reason);
1967 if (exit_reason < kvm_vmx_max_exit_handlers
1968 && kvm_vmx_exit_handlers[exit_reason])
1969 return kvm_vmx_exit_handlers[exit_reason](vcpu, kvm_run);
1970 else {
1971 kvm_run->exit_reason = KVM_EXIT_UNKNOWN;
1972 kvm_run->hw.hardware_exit_reason = exit_reason;
1973 }
1974 return 0;
1975 }
1976
1977 /*
1978 * Check if userspace requested an interrupt window, and that the
1979 * interrupt window is open.
1980 *
1981 * No need to exit to userspace if we already have an interrupt queued.
1982 */
1983 static int dm_request_for_irq_injection(struct kvm_vcpu *vcpu,
1984 struct kvm_run *kvm_run)
1985 {
1986 return (!vcpu->irq_summary &&
1987 kvm_run->request_interrupt_window &&
1988 vcpu->interrupt_window_open &&
1989 (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_IF));
1990 }
1991
1992 static void vmx_flush_tlb(struct kvm_vcpu *vcpu)
1993 {
1994 }
1995
1996 static int vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
1997 {
1998 u8 fail;
1999 int r;
2000
2001 preempted:
2002 if (vcpu->guest_debug.enabled)
2003 kvm_guest_debug_pre(vcpu);
2004
2005 again:
2006 if (!vcpu->mmio_read_completed)
2007 do_interrupt_requests(vcpu, kvm_run);
2008
2009 vmx_save_host_state(vcpu);
2010 kvm_load_guest_fpu(vcpu);
2011
2012 r = kvm_mmu_reload(vcpu);
2013 if (unlikely(r))
2014 goto out;
2015
2016 /*
2017 * Loading guest fpu may have cleared host cr0.ts
2018 */
2019 vmcs_writel(HOST_CR0, read_cr0());
2020
2021 local_irq_disable();
2022
2023 vcpu->guest_mode = 1;
2024 if (vcpu->requests)
2025 if (test_and_clear_bit(KVM_TLB_FLUSH, &vcpu->requests))
2026 vmx_flush_tlb(vcpu);
2027
2028 asm (
2029 /* Store host registers */
2030 #ifdef CONFIG_X86_64
2031 "push %%rax; push %%rbx; push %%rdx;"
2032 "push %%rsi; push %%rdi; push %%rbp;"
2033 "push %%r8; push %%r9; push %%r10; push %%r11;"
2034 "push %%r12; push %%r13; push %%r14; push %%r15;"
2035 "push %%rcx \n\t"
2036 ASM_VMX_VMWRITE_RSP_RDX "\n\t"
2037 #else
2038 "pusha; push %%ecx \n\t"
2039 ASM_VMX_VMWRITE_RSP_RDX "\n\t"
2040 #endif
2041 /* Check if vmlaunch of vmresume is needed */
2042 "cmp $0, %1 \n\t"
2043 /* Load guest registers. Don't clobber flags. */
2044 #ifdef CONFIG_X86_64
2045 "mov %c[cr2](%3), %%rax \n\t"
2046 "mov %%rax, %%cr2 \n\t"
2047 "mov %c[rax](%3), %%rax \n\t"
2048 "mov %c[rbx](%3), %%rbx \n\t"
2049 "mov %c[rdx](%3), %%rdx \n\t"
2050 "mov %c[rsi](%3), %%rsi \n\t"
2051 "mov %c[rdi](%3), %%rdi \n\t"
2052 "mov %c[rbp](%3), %%rbp \n\t"
2053 "mov %c[r8](%3), %%r8 \n\t"
2054 "mov %c[r9](%3), %%r9 \n\t"
2055 "mov %c[r10](%3), %%r10 \n\t"
2056 "mov %c[r11](%3), %%r11 \n\t"
2057 "mov %c[r12](%3), %%r12 \n\t"
2058 "mov %c[r13](%3), %%r13 \n\t"
2059 "mov %c[r14](%3), %%r14 \n\t"
2060 "mov %c[r15](%3), %%r15 \n\t"
2061 "mov %c[rcx](%3), %%rcx \n\t" /* kills %3 (rcx) */
2062 #else
2063 "mov %c[cr2](%3), %%eax \n\t"
2064 "mov %%eax, %%cr2 \n\t"
2065 "mov %c[rax](%3), %%eax \n\t"
2066 "mov %c[rbx](%3), %%ebx \n\t"
2067 "mov %c[rdx](%3), %%edx \n\t"
2068 "mov %c[rsi](%3), %%esi \n\t"
2069 "mov %c[rdi](%3), %%edi \n\t"
2070 "mov %c[rbp](%3), %%ebp \n\t"
2071 "mov %c[rcx](%3), %%ecx \n\t" /* kills %3 (ecx) */
2072 #endif
2073 /* Enter guest mode */
2074 "jne .Llaunched \n\t"
2075 ASM_VMX_VMLAUNCH "\n\t"
2076 "jmp .Lkvm_vmx_return \n\t"
2077 ".Llaunched: " ASM_VMX_VMRESUME "\n\t"
2078 ".Lkvm_vmx_return: "
2079 /* Save guest registers, load host registers, keep flags */
2080 #ifdef CONFIG_X86_64
2081 "xchg %3, (%%rsp) \n\t"
2082 "mov %%rax, %c[rax](%3) \n\t"
2083 "mov %%rbx, %c[rbx](%3) \n\t"
2084 "pushq (%%rsp); popq %c[rcx](%3) \n\t"
2085 "mov %%rdx, %c[rdx](%3) \n\t"
2086 "mov %%rsi, %c[rsi](%3) \n\t"
2087 "mov %%rdi, %c[rdi](%3) \n\t"
2088 "mov %%rbp, %c[rbp](%3) \n\t"
2089 "mov %%r8, %c[r8](%3) \n\t"
2090 "mov %%r9, %c[r9](%3) \n\t"
2091 "mov %%r10, %c[r10](%3) \n\t"
2092 "mov %%r11, %c[r11](%3) \n\t"
2093 "mov %%r12, %c[r12](%3) \n\t"
2094 "mov %%r13, %c[r13](%3) \n\t"
2095 "mov %%r14, %c[r14](%3) \n\t"
2096 "mov %%r15, %c[r15](%3) \n\t"
2097 "mov %%cr2, %%rax \n\t"
2098 "mov %%rax, %c[cr2](%3) \n\t"
2099 "mov (%%rsp), %3 \n\t"
2100
2101 "pop %%rcx; pop %%r15; pop %%r14; pop %%r13; pop %%r12;"
2102 "pop %%r11; pop %%r10; pop %%r9; pop %%r8;"
2103 "pop %%rbp; pop %%rdi; pop %%rsi;"
2104 "pop %%rdx; pop %%rbx; pop %%rax \n\t"
2105 #else
2106 "xchg %3, (%%esp) \n\t"
2107 "mov %%eax, %c[rax](%3) \n\t"
2108 "mov %%ebx, %c[rbx](%3) \n\t"
2109 "pushl (%%esp); popl %c[rcx](%3) \n\t"
2110 "mov %%edx, %c[rdx](%3) \n\t"
2111 "mov %%esi, %c[rsi](%3) \n\t"
2112 "mov %%edi, %c[rdi](%3) \n\t"
2113 "mov %%ebp, %c[rbp](%3) \n\t"
2114 "mov %%cr2, %%eax \n\t"
2115 "mov %%eax, %c[cr2](%3) \n\t"
2116 "mov (%%esp), %3 \n\t"
2117
2118 "pop %%ecx; popa \n\t"
2119 #endif
2120 "setbe %0 \n\t"
2121 : "=q" (fail)
2122 : "r"(vcpu->launched), "d"((unsigned long)HOST_RSP),
2123 "c"(vcpu),
2124 [rax]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RAX])),
2125 [rbx]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RBX])),
2126 [rcx]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RCX])),
2127 [rdx]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RDX])),
2128 [rsi]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RSI])),
2129 [rdi]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RDI])),
2130 [rbp]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_RBP])),
2131 #ifdef CONFIG_X86_64
2132 [r8 ]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R8 ])),
2133 [r9 ]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R9 ])),
2134 [r10]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R10])),
2135 [r11]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R11])),
2136 [r12]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R12])),
2137 [r13]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R13])),
2138 [r14]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R14])),
2139 [r15]"i"(offsetof(struct kvm_vcpu, regs[VCPU_REGS_R15])),
2140 #endif
2141 [cr2]"i"(offsetof(struct kvm_vcpu, cr2))
2142 : "cc", "memory" );
2143
2144 vcpu->guest_mode = 0;
2145 local_irq_enable();
2146
2147 ++vcpu->stat.exits;
2148
2149 vcpu->interrupt_window_open = (vmcs_read32(GUEST_INTERRUPTIBILITY_INFO) & 3) == 0;
2150
2151 asm ("mov %0, %%ds; mov %0, %%es" : : "r"(__USER_DS));
2152
2153 if (unlikely(fail)) {
2154 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
2155 kvm_run->fail_entry.hardware_entry_failure_reason
2156 = vmcs_read32(VM_INSTRUCTION_ERROR);
2157 r = 0;
2158 goto out;
2159 }
2160 /*
2161 * Profile KVM exit RIPs:
2162 */
2163 if (unlikely(prof_on == KVM_PROFILING))
2164 profile_hit(KVM_PROFILING, (void *)vmcs_readl(GUEST_RIP));
2165
2166 vcpu->launched = 1;
2167 r = kvm_handle_exit(kvm_run, vcpu);
2168 if (r > 0) {
2169 /* Give scheduler a change to reschedule. */
2170 if (signal_pending(current)) {
2171 r = -EINTR;
2172 kvm_run->exit_reason = KVM_EXIT_INTR;
2173 ++vcpu->stat.signal_exits;
2174 goto out;
2175 }
2176
2177 if (dm_request_for_irq_injection(vcpu, kvm_run)) {
2178 r = -EINTR;
2179 kvm_run->exit_reason = KVM_EXIT_INTR;
2180 ++vcpu->stat.request_irq_exits;
2181 goto out;
2182 }
2183 if (!need_resched()) {
2184 ++vcpu->stat.light_exits;
2185 goto again;
2186 }
2187 }
2188
2189 out:
2190 if (r > 0) {
2191 kvm_resched(vcpu);
2192 goto preempted;
2193 }
2194
2195 post_kvm_run_save(vcpu, kvm_run);
2196 return r;
2197 }
2198
2199 static void vmx_inject_page_fault(struct kvm_vcpu *vcpu,
2200 unsigned long addr,
2201 u32 err_code)
2202 {
2203 u32 vect_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
2204
2205 ++vcpu->stat.pf_guest;
2206
2207 if (is_page_fault(vect_info)) {
2208 printk(KERN_DEBUG "inject_page_fault: "
2209 "double fault 0x%lx @ 0x%lx\n",
2210 addr, vmcs_readl(GUEST_RIP));
2211 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, 0);
2212 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
2213 DF_VECTOR |
2214 INTR_TYPE_EXCEPTION |
2215 INTR_INFO_DELIEVER_CODE_MASK |
2216 INTR_INFO_VALID_MASK);
2217 return;
2218 }
2219 vcpu->cr2 = addr;
2220 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, err_code);
2221 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
2222 PF_VECTOR |
2223 INTR_TYPE_EXCEPTION |
2224 INTR_INFO_DELIEVER_CODE_MASK |
2225 INTR_INFO_VALID_MASK);
2226
2227 }
2228
2229 static void vmx_free_vmcs(struct kvm_vcpu *vcpu)
2230 {
2231 if (vcpu->vmcs) {
2232 on_each_cpu(__vcpu_clear, vcpu, 0, 1);
2233 free_vmcs(vcpu->vmcs);
2234 vcpu->vmcs = NULL;
2235 }
2236 }
2237
2238 static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
2239 {
2240 vmx_free_vmcs(vcpu);
2241 }
2242
2243 static int vmx_create_vcpu(struct kvm_vcpu *vcpu)
2244 {
2245 struct vmcs *vmcs;
2246
2247 vcpu->guest_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL);
2248 if (!vcpu->guest_msrs)
2249 return -ENOMEM;
2250
2251 vcpu->host_msrs = kmalloc(PAGE_SIZE, GFP_KERNEL);
2252 if (!vcpu->host_msrs)
2253 goto out_free_guest_msrs;
2254
2255 vmcs = alloc_vmcs();
2256 if (!vmcs)
2257 goto out_free_msrs;
2258
2259 vmcs_clear(vmcs);
2260 vcpu->vmcs = vmcs;
2261 vcpu->launched = 0;
2262
2263 return 0;
2264
2265 out_free_msrs:
2266 kfree(vcpu->host_msrs);
2267 vcpu->host_msrs = NULL;
2268
2269 out_free_guest_msrs:
2270 kfree(vcpu->guest_msrs);
2271 vcpu->guest_msrs = NULL;
2272
2273 return -ENOMEM;
2274 }
2275
2276 static struct kvm_arch_ops vmx_arch_ops = {
2277 .cpu_has_kvm_support = cpu_has_kvm_support,
2278 .disabled_by_bios = vmx_disabled_by_bios,
2279 .hardware_setup = hardware_setup,
2280 .hardware_unsetup = hardware_unsetup,
2281 .hardware_enable = hardware_enable,
2282 .hardware_disable = hardware_disable,
2283
2284 .vcpu_create = vmx_create_vcpu,
2285 .vcpu_free = vmx_free_vcpu,
2286
2287 .vcpu_load = vmx_vcpu_load,
2288 .vcpu_put = vmx_vcpu_put,
2289 .vcpu_decache = vmx_vcpu_decache,
2290
2291 .set_guest_debug = set_guest_debug,
2292 .get_msr = vmx_get_msr,
2293 .set_msr = vmx_set_msr,
2294 .get_segment_base = vmx_get_segment_base,
2295 .get_segment = vmx_get_segment,
2296 .set_segment = vmx_set_segment,
2297 .get_cs_db_l_bits = vmx_get_cs_db_l_bits,
2298 .decache_cr4_guest_bits = vmx_decache_cr4_guest_bits,
2299 .set_cr0 = vmx_set_cr0,
2300 .set_cr3 = vmx_set_cr3,
2301 .set_cr4 = vmx_set_cr4,
2302 #ifdef CONFIG_X86_64
2303 .set_efer = vmx_set_efer,
2304 #endif
2305 .get_idt = vmx_get_idt,
2306 .set_idt = vmx_set_idt,
2307 .get_gdt = vmx_get_gdt,
2308 .set_gdt = vmx_set_gdt,
2309 .cache_regs = vcpu_load_rsp_rip,
2310 .decache_regs = vcpu_put_rsp_rip,
2311 .get_rflags = vmx_get_rflags,
2312 .set_rflags = vmx_set_rflags,
2313
2314 .tlb_flush = vmx_flush_tlb,
2315 .inject_page_fault = vmx_inject_page_fault,
2316
2317 .inject_gp = vmx_inject_gp,
2318
2319 .run = vmx_vcpu_run,
2320 .skip_emulated_instruction = skip_emulated_instruction,
2321 .vcpu_setup = vmx_vcpu_setup,
2322 .patch_hypercall = vmx_patch_hypercall,
2323 };
2324
2325 static int __init vmx_init(void)
2326 {
2327 void *iova;
2328 int r;
2329
2330 vmx_io_bitmap_a = alloc_page(GFP_KERNEL | __GFP_HIGHMEM);
2331 if (!vmx_io_bitmap_a)
2332 return -ENOMEM;
2333
2334 vmx_io_bitmap_b = alloc_page(GFP_KERNEL | __GFP_HIGHMEM);
2335 if (!vmx_io_bitmap_b) {
2336 r = -ENOMEM;
2337 goto out;
2338 }
2339
2340 /*
2341 * Allow direct access to the PC debug port (it is often used for I/O
2342 * delays, but the vmexits simply slow things down).
2343 */
2344 iova = kmap(vmx_io_bitmap_a);
2345 memset(iova, 0xff, PAGE_SIZE);
2346 clear_bit(0x80, iova);
2347 kunmap(vmx_io_bitmap_a);
2348
2349 iova = kmap(vmx_io_bitmap_b);
2350 memset(iova, 0xff, PAGE_SIZE);
2351 kunmap(vmx_io_bitmap_b);
2352
2353 r = kvm_init_arch(&vmx_arch_ops, THIS_MODULE);
2354 if (r)
2355 goto out1;
2356
2357 return 0;
2358
2359 out1:
2360 __free_page(vmx_io_bitmap_b);
2361 out:
2362 __free_page(vmx_io_bitmap_a);
2363 return r;
2364 }
2365
2366 static void __exit vmx_exit(void)
2367 {
2368 __free_page(vmx_io_bitmap_b);
2369 __free_page(vmx_io_bitmap_a);
2370
2371 kvm_exit_arch();
2372 }
2373
2374 module_init(vmx_init)
2375 module_exit(vmx_exit)
Linux 2.6 ibm-acpi/thinkpad-acpi driver git tree