2 * VMI specific paravirt-ops implementation
4 * Copyright (C) 2005, VMware, Inc.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
14 * NON INFRINGEMENT. See the GNU General Public License for more
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21 * Send feedback to zach@vmware.com
25 #include <linux/module.h>
26 #include <linux/cpu.h>
27 #include <linux/bootmem.h>
29 #include <linux/highmem.h>
30 #include <linux/sched.h>
33 #include <asm/fixmap.h>
34 #include <asm/apicdef.h>
36 #include <asm/processor.h>
37 #include <asm/timer.h>
38 #include <asm/vmi_time.h>
39 #include <asm/kmap_types.h>
41 /* Convenient for calling VMI functions indirectly in the ROM */
42 typedef u32
__attribute__((regparm(1))) (VROMFUNC
)(void);
43 typedef u64
__attribute__((regparm(2))) (VROMLONGFUNC
)(int);
45 #define call_vrom_func(rom,func) \
46 (((VROMFUNC *)(rom->func))())
48 #define call_vrom_long_func(rom,func,arg) \
49 (((VROMLONGFUNC *)(rom->func)) (arg))
51 static struct vrom_header
*vmi_rom
;
52 static int disable_pge
;
53 static int disable_pse
;
54 static int disable_sep
;
55 static int disable_tsc
;
56 static int disable_mtrr
;
57 static int disable_noidle
;
58 static int disable_vmi_timer
;
60 /* Cached VMI operations */
62 void (*cpuid
)(void /* non-c */);
63 void (*_set_ldt
)(u32 selector
);
64 void (*set_tr
)(u32 selector
);
65 void (*write_idt_entry
)(struct desc_struct
*, int, u32
, u32
);
66 void (*write_gdt_entry
)(struct desc_struct
*, int, u32
, u32
);
67 void (*write_ldt_entry
)(struct desc_struct
*, int, u32
, u32
);
68 void (*set_kernel_stack
)(u32 selector
, u32 sp0
);
69 void (*allocate_page
)(u32
, u32
, u32
, u32
, u32
);
70 void (*release_page
)(u32
, u32
);
71 void (*set_pte
)(pte_t
, pte_t
*, unsigned);
72 void (*update_pte
)(pte_t
*, unsigned);
73 void (*set_linear_mapping
)(int, void *, u32
, u32
);
74 void (*_flush_tlb
)(int);
75 void (*set_initial_ap_state
)(int, int);
77 void (*set_lazy_mode
)(int mode
);
80 /* Cached VMI operations */
81 struct vmi_timer_ops vmi_timer_ops
;
84 * VMI patching routines.
86 #define MNEM_CALL 0xe8
90 #define IRQ_PATCH_INT_MASK 0
91 #define IRQ_PATCH_DISABLE 5
93 static inline void patch_offset(void *insnbuf
,
94 unsigned long ip
, unsigned long dest
)
96 *(unsigned long *)(insnbuf
+1) = dest
-ip
-5;
99 static unsigned patch_internal(int call
, unsigned len
, void *insnbuf
,
103 struct vmi_relocation_info
*const rel
= (struct vmi_relocation_info
*)&reloc
;
104 reloc
= call_vrom_long_func(vmi_rom
, get_reloc
, call
);
106 case VMI_RELOCATION_CALL_REL
:
108 *(char *)insnbuf
= MNEM_CALL
;
109 patch_offset(insnbuf
, ip
, (unsigned long)rel
->eip
);
112 case VMI_RELOCATION_JUMP_REL
:
114 *(char *)insnbuf
= MNEM_JMP
;
115 patch_offset(insnbuf
, ip
, (unsigned long)rel
->eip
);
118 case VMI_RELOCATION_NOP
:
119 /* obliterate the whole thing */
122 case VMI_RELOCATION_NONE
:
123 /* leave native code in place */
133 * Apply patch if appropriate, return length of new instruction
134 * sequence. The callee does nop padding for us.
136 static unsigned vmi_patch(u8 type
, u16 clobbers
, void *insns
,
137 unsigned long ip
, unsigned len
)
140 case PARAVIRT_PATCH(pv_irq_ops
.irq_disable
):
141 return patch_internal(VMI_CALL_DisableInterrupts
, len
,
143 case PARAVIRT_PATCH(pv_irq_ops
.irq_enable
):
144 return patch_internal(VMI_CALL_EnableInterrupts
, len
,
146 case PARAVIRT_PATCH(pv_irq_ops
.restore_fl
):
147 return patch_internal(VMI_CALL_SetInterruptMask
, len
,
149 case PARAVIRT_PATCH(pv_irq_ops
.save_fl
):
150 return patch_internal(VMI_CALL_GetInterruptMask
, len
,
152 case PARAVIRT_PATCH(pv_cpu_ops
.iret
):
153 return patch_internal(VMI_CALL_IRET
, len
, insns
, ip
);
154 case PARAVIRT_PATCH(pv_cpu_ops
.irq_enable_syscall_ret
):
155 return patch_internal(VMI_CALL_SYSEXIT
, len
, insns
, ip
);
162 /* CPUID has non-C semantics, and paravirt-ops API doesn't match hardware ISA */
163 static void vmi_cpuid(unsigned int *ax
, unsigned int *bx
,
164 unsigned int *cx
, unsigned int *dx
)
169 asm volatile ("call *%6"
174 : "0" (*ax
), "2" (*cx
), "r" (vmi_ops
.cpuid
));
177 *dx
&= ~X86_FEATURE_PSE
;
179 *dx
&= ~X86_FEATURE_PGE
;
181 *dx
&= ~X86_FEATURE_SEP
;
183 *dx
&= ~X86_FEATURE_TSC
;
185 *dx
&= ~X86_FEATURE_MTRR
;
189 static inline void vmi_maybe_load_tls(struct desc_struct
*gdt
, int nr
, struct desc_struct
*new)
191 if (gdt
[nr
].a
!= new->a
|| gdt
[nr
].b
!= new->b
)
192 write_gdt_entry(gdt
, nr
, new, 0);
195 static void vmi_load_tls(struct thread_struct
*t
, unsigned int cpu
)
197 struct desc_struct
*gdt
= get_cpu_gdt_table(cpu
);
198 vmi_maybe_load_tls(gdt
, GDT_ENTRY_TLS_MIN
+ 0, &t
->tls_array
[0]);
199 vmi_maybe_load_tls(gdt
, GDT_ENTRY_TLS_MIN
+ 1, &t
->tls_array
[1]);
200 vmi_maybe_load_tls(gdt
, GDT_ENTRY_TLS_MIN
+ 2, &t
->tls_array
[2]);
203 static void vmi_set_ldt(const void *addr
, unsigned entries
)
205 unsigned cpu
= smp_processor_id();
206 struct desc_struct desc
;
208 pack_descriptor(&desc
, (unsigned long)addr
,
209 entries
* sizeof(struct desc_struct
) - 1,
211 write_gdt_entry(get_cpu_gdt_table(cpu
), GDT_ENTRY_LDT
, &desc
, DESC_LDT
);
212 vmi_ops
._set_ldt(entries
? GDT_ENTRY_LDT
*sizeof(struct desc_struct
) : 0);
215 static void vmi_set_tr(void)
217 vmi_ops
.set_tr(GDT_ENTRY_TSS
*sizeof(struct desc_struct
));
220 static void vmi_write_idt_entry(gate_desc
*dt
, int entry
, const gate_desc
*g
)
222 u32
*idt_entry
= (u32
*)g
;
223 vmi_ops
.write_idt_entry(dt
, entry
, idt_entry
[0], idt_entry
[2]);
226 static void vmi_write_gdt_entry(struct desc_struct
*dt
, int entry
,
227 const void *desc
, int type
)
229 u32
*gdt_entry
= (u32
*)desc
;
230 vmi_ops
.write_gdt_entry(dt
, entry
, gdt_entry
[0], gdt_entry
[2]);
233 static void vmi_write_ldt_entry(struct desc_struct
*dt
, int entry
,
236 u32
*ldt_entry
= (u32
*)desc
;
237 vmi_ops
.write_idt_entry(dt
, entry
, ldt_entry
[0], ldt_entry
[2]);
240 static void vmi_load_sp0(struct tss_struct
*tss
,
241 struct thread_struct
*thread
)
243 tss
->x86_tss
.sp0
= thread
->sp0
;
245 /* This can only happen when SEP is enabled, no need to test "SEP"arately */
246 if (unlikely(tss
->x86_tss
.ss1
!= thread
->sysenter_cs
)) {
247 tss
->x86_tss
.ss1
= thread
->sysenter_cs
;
248 wrmsr(MSR_IA32_SYSENTER_CS
, thread
->sysenter_cs
, 0);
250 vmi_ops
.set_kernel_stack(__KERNEL_DS
, tss
->x86_tss
.sp0
);
253 static void vmi_flush_tlb_user(void)
255 vmi_ops
._flush_tlb(VMI_FLUSH_TLB
);
258 static void vmi_flush_tlb_kernel(void)
260 vmi_ops
._flush_tlb(VMI_FLUSH_TLB
| VMI_FLUSH_GLOBAL
);
263 /* Stub to do nothing at all; used for delays and unimplemented calls */
264 static void vmi_nop(void)
268 #ifdef CONFIG_DEBUG_PAGE_TYPE
270 #ifdef CONFIG_X86_PAE
271 #define MAX_BOOT_PTS (2048+4+1)
273 #define MAX_BOOT_PTS (1024+1)
277 * During boot, mem_map is not yet available in paging_init, so stash
278 * all the boot page allocations here.
283 } boot_page_allocations
[MAX_BOOT_PTS
];
284 static int num_boot_page_allocations
;
285 static int boot_allocations_applied
;
287 void vmi_apply_boot_page_allocations(void)
291 for (i
= 0; i
< num_boot_page_allocations
; i
++) {
292 struct page
*page
= pfn_to_page(boot_page_allocations
[i
].pfn
);
293 page
->type
= boot_page_allocations
[i
].type
;
294 page
->type
= boot_page_allocations
[i
].type
&
295 ~(VMI_PAGE_ZEROED
| VMI_PAGE_CLONE
);
297 boot_allocations_applied
= 1;
300 static void record_page_type(u32 pfn
, int type
)
302 BUG_ON(num_boot_page_allocations
>= MAX_BOOT_PTS
);
303 boot_page_allocations
[num_boot_page_allocations
].pfn
= pfn
;
304 boot_page_allocations
[num_boot_page_allocations
].type
= type
;
305 num_boot_page_allocations
++;
308 static void check_zeroed_page(u32 pfn
, int type
, struct page
*page
)
312 int limit
= PAGE_SIZE
/ sizeof(int);
314 if (page_address(page
))
315 ptr
= (u32
*)page_address(page
);
317 ptr
= (u32
*)__va(pfn
<< PAGE_SHIFT
);
319 * When cloning the root in non-PAE mode, only the userspace
320 * pdes need to be zeroed.
322 if (type
& VMI_PAGE_CLONE
)
323 limit
= USER_PTRS_PER_PGD
;
324 for (i
= 0; i
< limit
; i
++)
329 * We stash the page type into struct page so we can verify the page
330 * types are used properly.
332 static void vmi_set_page_type(u32 pfn
, int type
)
334 /* PAE can have multiple roots per page - don't track */
335 if (PTRS_PER_PMD
> 1 && (type
& VMI_PAGE_PDP
))
338 if (boot_allocations_applied
) {
339 struct page
*page
= pfn_to_page(pfn
);
340 if (type
!= VMI_PAGE_NORMAL
)
343 BUG_ON(page
->type
== VMI_PAGE_NORMAL
);
344 page
->type
= type
& ~(VMI_PAGE_ZEROED
| VMI_PAGE_CLONE
);
345 if (type
& VMI_PAGE_ZEROED
)
346 check_zeroed_page(pfn
, type
, page
);
348 record_page_type(pfn
, type
);
352 static void vmi_check_page_type(u32 pfn
, int type
)
354 /* PAE can have multiple roots per page - skip checks */
355 if (PTRS_PER_PMD
> 1 && (type
& VMI_PAGE_PDP
))
358 type
&= ~(VMI_PAGE_ZEROED
| VMI_PAGE_CLONE
);
359 if (boot_allocations_applied
) {
360 struct page
*page
= pfn_to_page(pfn
);
361 BUG_ON((page
->type
^ type
) & VMI_PAGE_PAE
);
362 BUG_ON(type
== VMI_PAGE_NORMAL
&& page
->type
);
363 BUG_ON((type
& page
->type
) == 0);
367 #define vmi_set_page_type(p,t) do { } while (0)
368 #define vmi_check_page_type(p,t) do { } while (0)
371 #ifdef CONFIG_HIGHPTE
372 static void *vmi_kmap_atomic_pte(struct page
*page
, enum km_type type
)
374 void *va
= kmap_atomic(page
, type
);
377 * Internally, the VMI ROM must map virtual addresses to physical
378 * addresses for processing MMU updates. By the time MMU updates
379 * are issued, this information is typically already lost.
380 * Fortunately, the VMI provides a cache of mapping slots for active
383 * We use slot zero for the linear mapping of physical memory, and
384 * in HIGHPTE kernels, slot 1 and 2 for KM_PTE0 and KM_PTE1.
386 * args: SLOT VA COUNT PFN
388 BUG_ON(type
!= KM_PTE0
&& type
!= KM_PTE1
);
389 vmi_ops
.set_linear_mapping((type
- KM_PTE0
)+1, va
, 1, page_to_pfn(page
));
395 static void vmi_allocate_pt(struct mm_struct
*mm
, u32 pfn
)
397 vmi_set_page_type(pfn
, VMI_PAGE_L1
);
398 vmi_ops
.allocate_page(pfn
, VMI_PAGE_L1
, 0, 0, 0);
401 static void vmi_allocate_pd(struct mm_struct
*mm
, u32 pfn
)
404 * This call comes in very early, before mem_map is setup.
405 * It is called only for swapper_pg_dir, which already has
408 vmi_set_page_type(pfn
, VMI_PAGE_L2
);
409 vmi_ops
.allocate_page(pfn
, VMI_PAGE_L2
, 0, 0, 0);
412 static void vmi_allocate_pd_clone(u32 pfn
, u32 clonepfn
, u32 start
, u32 count
)
414 vmi_set_page_type(pfn
, VMI_PAGE_L2
| VMI_PAGE_CLONE
);
415 vmi_check_page_type(clonepfn
, VMI_PAGE_L2
);
416 vmi_ops
.allocate_page(pfn
, VMI_PAGE_L2
| VMI_PAGE_CLONE
, clonepfn
, start
, count
);
419 static void vmi_release_pt(u32 pfn
)
421 vmi_ops
.release_page(pfn
, VMI_PAGE_L1
);
422 vmi_set_page_type(pfn
, VMI_PAGE_NORMAL
);
425 static void vmi_release_pd(u32 pfn
)
427 vmi_ops
.release_page(pfn
, VMI_PAGE_L2
);
428 vmi_set_page_type(pfn
, VMI_PAGE_NORMAL
);
432 * Helper macros for MMU update flags. We can defer updates until a flush
433 * or page invalidation only if the update is to the current address space
434 * (otherwise, there is no flush). We must check against init_mm, since
435 * this could be a kernel update, which usually passes init_mm, although
436 * sometimes this check can be skipped if we know the particular function
437 * is only called on user mode PTEs. We could change the kernel to pass
438 * current->active_mm here, but in particular, I was unsure if changing
439 * mm/highmem.c to do this would still be correct on other architectures.
441 #define is_current_as(mm, mustbeuser) ((mm) == current->active_mm || \
442 (!mustbeuser && (mm) == &init_mm))
443 #define vmi_flags_addr(mm, addr, level, user) \
444 ((level) | (is_current_as(mm, user) ? \
445 (VMI_PAGE_CURRENT_AS | ((addr) & VMI_PAGE_VA_MASK)) : 0))
446 #define vmi_flags_addr_defer(mm, addr, level, user) \
447 ((level) | (is_current_as(mm, user) ? \
448 (VMI_PAGE_DEFER | VMI_PAGE_CURRENT_AS | ((addr) & VMI_PAGE_VA_MASK)) : 0))
450 static void vmi_update_pte(struct mm_struct
*mm
, unsigned long addr
, pte_t
*ptep
)
452 vmi_check_page_type(__pa(ptep
) >> PAGE_SHIFT
, VMI_PAGE_PTE
);
453 vmi_ops
.update_pte(ptep
, vmi_flags_addr(mm
, addr
, VMI_PAGE_PT
, 0));
456 static void vmi_update_pte_defer(struct mm_struct
*mm
, unsigned long addr
, pte_t
*ptep
)
458 vmi_check_page_type(__pa(ptep
) >> PAGE_SHIFT
, VMI_PAGE_PTE
);
459 vmi_ops
.update_pte(ptep
, vmi_flags_addr_defer(mm
, addr
, VMI_PAGE_PT
, 0));
462 static void vmi_set_pte(pte_t
*ptep
, pte_t pte
)
464 /* XXX because of set_pmd_pte, this can be called on PT or PD layers */
465 vmi_check_page_type(__pa(ptep
) >> PAGE_SHIFT
, VMI_PAGE_PTE
| VMI_PAGE_PD
);
466 vmi_ops
.set_pte(pte
, ptep
, VMI_PAGE_PT
);
469 static void vmi_set_pte_at(struct mm_struct
*mm
, unsigned long addr
, pte_t
*ptep
, pte_t pte
)
471 vmi_check_page_type(__pa(ptep
) >> PAGE_SHIFT
, VMI_PAGE_PTE
);
472 vmi_ops
.set_pte(pte
, ptep
, vmi_flags_addr(mm
, addr
, VMI_PAGE_PT
, 0));
475 static void vmi_set_pmd(pmd_t
*pmdp
, pmd_t pmdval
)
477 #ifdef CONFIG_X86_PAE
478 const pte_t pte
= { .pte
= pmdval
.pmd
};
479 vmi_check_page_type(__pa(pmdp
) >> PAGE_SHIFT
, VMI_PAGE_PMD
);
481 const pte_t pte
= { pmdval
.pud
.pgd
.pgd
};
482 vmi_check_page_type(__pa(pmdp
) >> PAGE_SHIFT
, VMI_PAGE_PGD
);
484 vmi_ops
.set_pte(pte
, (pte_t
*)pmdp
, VMI_PAGE_PD
);
487 #ifdef CONFIG_X86_PAE
489 static void vmi_set_pte_atomic(pte_t
*ptep
, pte_t pteval
)
492 * XXX This is called from set_pmd_pte, but at both PT
493 * and PD layers so the VMI_PAGE_PT flag is wrong. But
494 * it is only called for large page mapping changes,
495 * the Xen backend, doesn't support large pages, and the
496 * ESX backend doesn't depend on the flag.
498 set_64bit((unsigned long long *)ptep
,pte_val(pteval
));
499 vmi_ops
.update_pte(ptep
, VMI_PAGE_PT
);
502 static void vmi_set_pte_present(struct mm_struct
*mm
, unsigned long addr
, pte_t
*ptep
, pte_t pte
)
504 vmi_check_page_type(__pa(ptep
) >> PAGE_SHIFT
, VMI_PAGE_PTE
);
505 vmi_ops
.set_pte(pte
, ptep
, vmi_flags_addr_defer(mm
, addr
, VMI_PAGE_PT
, 1));
508 static void vmi_set_pud(pud_t
*pudp
, pud_t pudval
)
511 const pte_t pte
= { .pte
= pudval
.pgd
.pgd
};
512 vmi_check_page_type(__pa(pudp
) >> PAGE_SHIFT
, VMI_PAGE_PGD
);
513 vmi_ops
.set_pte(pte
, (pte_t
*)pudp
, VMI_PAGE_PDP
);
516 static void vmi_pte_clear(struct mm_struct
*mm
, unsigned long addr
, pte_t
*ptep
)
518 const pte_t pte
= { .pte
= 0 };
519 vmi_check_page_type(__pa(ptep
) >> PAGE_SHIFT
, VMI_PAGE_PTE
);
520 vmi_ops
.set_pte(pte
, ptep
, vmi_flags_addr(mm
, addr
, VMI_PAGE_PT
, 0));
523 static void vmi_pmd_clear(pmd_t
*pmd
)
525 const pte_t pte
= { .pte
= 0 };
526 vmi_check_page_type(__pa(pmd
) >> PAGE_SHIFT
, VMI_PAGE_PMD
);
527 vmi_ops
.set_pte(pte
, (pte_t
*)pmd
, VMI_PAGE_PD
);
532 static void __devinit
533 vmi_startup_ipi_hook(int phys_apicid
, unsigned long start_eip
,
534 unsigned long start_esp
)
536 struct vmi_ap_state ap
;
538 /* Default everything to zero. This is fine for most GPRs. */
539 memset(&ap
, 0, sizeof(struct vmi_ap_state
));
541 ap
.gdtr_limit
= GDT_SIZE
- 1;
542 ap
.gdtr_base
= (unsigned long) get_cpu_gdt_table(phys_apicid
);
544 ap
.idtr_limit
= IDT_ENTRIES
* 8 - 1;
545 ap
.idtr_base
= (unsigned long) idt_table
;
550 ap
.eip
= (unsigned long) start_eip
;
552 ap
.esp
= (unsigned long) start_esp
;
556 ap
.fs
= __KERNEL_PERCPU
;
561 #ifdef CONFIG_X86_PAE
562 /* efer should match BSP efer. */
565 rdmsr(MSR_EFER
, l
, h
);
566 ap
.efer
= (unsigned long long) h
<< 32 | l
;
570 ap
.cr3
= __pa(swapper_pg_dir
);
571 /* Protected mode, paging, AM, WP, NE, MP. */
573 ap
.cr4
= mmu_cr4_features
;
574 vmi_ops
.set_initial_ap_state((u32
)&ap
, phys_apicid
);
578 static void vmi_enter_lazy_cpu(void)
580 paravirt_enter_lazy_cpu();
581 vmi_ops
.set_lazy_mode(2);
584 static void vmi_enter_lazy_mmu(void)
586 paravirt_enter_lazy_mmu();
587 vmi_ops
.set_lazy_mode(1);
590 static void vmi_leave_lazy(void)
592 paravirt_leave_lazy(paravirt_get_lazy_mode());
593 vmi_ops
.set_lazy_mode(0);
596 static inline int __init
check_vmi_rom(struct vrom_header
*rom
)
598 struct pci_header
*pci
;
599 struct pnp_header
*pnp
;
600 const char *manufacturer
= "UNKNOWN";
601 const char *product
= "UNKNOWN";
602 const char *license
= "unspecified";
604 if (rom
->rom_signature
!= 0xaa55)
606 if (rom
->vrom_signature
!= VMI_SIGNATURE
)
608 if (rom
->api_version_maj
!= VMI_API_REV_MAJOR
||
609 rom
->api_version_min
+1 < VMI_API_REV_MINOR
+1) {
610 printk(KERN_WARNING
"VMI: Found mismatched rom version %d.%d\n",
611 rom
->api_version_maj
,
612 rom
->api_version_min
);
617 * Relying on the VMI_SIGNATURE field is not 100% safe, so check
618 * the PCI header and device type to make sure this is really a
621 if (!rom
->pci_header_offs
) {
622 printk(KERN_WARNING
"VMI: ROM does not contain PCI header.\n");
626 pci
= (struct pci_header
*)((char *)rom
+rom
->pci_header_offs
);
627 if (pci
->vendorID
!= PCI_VENDOR_ID_VMWARE
||
628 pci
->deviceID
!= PCI_DEVICE_ID_VMWARE_VMI
) {
629 /* Allow it to run... anyways, but warn */
630 printk(KERN_WARNING
"VMI: ROM from unknown manufacturer\n");
633 if (rom
->pnp_header_offs
) {
634 pnp
= (struct pnp_header
*)((char *)rom
+rom
->pnp_header_offs
);
635 if (pnp
->manufacturer_offset
)
636 manufacturer
= (const char *)rom
+pnp
->manufacturer_offset
;
637 if (pnp
->product_offset
)
638 product
= (const char *)rom
+pnp
->product_offset
;
641 if (rom
->license_offs
)
642 license
= (char *)rom
+rom
->license_offs
;
644 printk(KERN_INFO
"VMI: Found %s %s, API version %d.%d, ROM version %d.%d\n",
645 manufacturer
, product
,
646 rom
->api_version_maj
, rom
->api_version_min
,
647 pci
->rom_version_maj
, pci
->rom_version_min
);
649 /* Don't allow BSD/MIT here for now because we don't want to end up
650 with any binary only shim layers */
651 if (strcmp(license
, "GPL") && strcmp(license
, "GPL v2")) {
652 printk(KERN_WARNING
"VMI: Non GPL license `%s' found for ROM. Not used.\n",
661 * Probe for the VMI option ROM
663 static inline int __init
probe_vmi_rom(void)
667 /* VMI ROM is in option ROM area, check signature */
668 for (base
= 0xC0000; base
< 0xE0000; base
+= 2048) {
669 struct vrom_header
*romstart
;
670 romstart
= (struct vrom_header
*)isa_bus_to_virt(base
);
671 if (check_vmi_rom(romstart
)) {
680 * VMI setup common to all processors
682 void vmi_bringup(void)
684 /* We must establish the lowmem mapping for MMU ops to work */
685 if (vmi_ops
.set_linear_mapping
)
686 vmi_ops
.set_linear_mapping(0, (void *)__PAGE_OFFSET
, max_low_pfn
, 0);
690 * Return a pointer to a VMI function or NULL if unimplemented
692 static void *vmi_get_function(int vmicall
)
695 const struct vmi_relocation_info
*rel
= (struct vmi_relocation_info
*)&reloc
;
696 reloc
= call_vrom_long_func(vmi_rom
, get_reloc
, vmicall
);
697 BUG_ON(rel
->type
== VMI_RELOCATION_JUMP_REL
);
698 if (rel
->type
== VMI_RELOCATION_CALL_REL
)
699 return (void *)rel
->eip
;
705 * Helper macro for making the VMI paravirt-ops fill code readable.
706 * For unimplemented operations, fall back to default, unless nop
707 * is returned by the ROM.
709 #define para_fill(opname, vmicall) \
711 reloc = call_vrom_long_func(vmi_rom, get_reloc, \
712 VMI_CALL_##vmicall); \
713 if (rel->type == VMI_RELOCATION_CALL_REL) \
714 opname = (void *)rel->eip; \
715 else if (rel->type == VMI_RELOCATION_NOP) \
716 opname = (void *)vmi_nop; \
717 else if (rel->type != VMI_RELOCATION_NONE) \
718 printk(KERN_WARNING "VMI: Unknown relocation " \
719 "type %d for " #vmicall"\n",\
724 * Helper macro for making the VMI paravirt-ops fill code readable.
725 * For cached operations which do not match the VMI ROM ABI and must
726 * go through a tranlation stub. Ignore NOPs, since it is not clear
727 * a NOP * VMI function corresponds to a NOP paravirt-op when the
728 * functions are not in 1-1 correspondence.
730 #define para_wrap(opname, wrapper, cache, vmicall) \
732 reloc = call_vrom_long_func(vmi_rom, get_reloc, \
733 VMI_CALL_##vmicall); \
734 BUG_ON(rel->type == VMI_RELOCATION_JUMP_REL); \
735 if (rel->type == VMI_RELOCATION_CALL_REL) { \
737 vmi_ops.cache = (void *)rel->eip; \
742 * Activate the VMI interface and switch into paravirtualized mode
744 static inline int __init
activate_vmi(void)
748 const struct vmi_relocation_info
*rel
= (struct vmi_relocation_info
*)&reloc
;
750 if (call_vrom_func(vmi_rom
, vmi_init
) != 0) {
751 printk(KERN_ERR
"VMI ROM failed to initialize!");
754 savesegment(cs
, kernel_cs
);
756 pv_info
.paravirt_enabled
= 1;
757 pv_info
.kernel_rpl
= kernel_cs
& SEGMENT_RPL_MASK
;
758 pv_info
.name
= "vmi";
760 pv_init_ops
.patch
= vmi_patch
;
763 * Many of these operations are ABI compatible with VMI.
764 * This means we can fill in the paravirt-ops with direct
765 * pointers into the VMI ROM. If the calling convention for
766 * these operations changes, this code needs to be updated.
769 * CPUID paravirt-op uses pointers, not the native ISA
770 * halt has no VMI equivalent; all VMI halts are "safe"
771 * no MSR support yet - just trap and emulate. VMI uses the
772 * same ABI as the native ISA, but Linux wants exceptions
773 * from bogus MSR read / write handled
774 * rdpmc is not yet used in Linux
777 /* CPUID is special, so very special it gets wrapped like a present */
778 para_wrap(pv_cpu_ops
.cpuid
, vmi_cpuid
, cpuid
, CPUID
);
780 para_fill(pv_cpu_ops
.clts
, CLTS
);
781 para_fill(pv_cpu_ops
.get_debugreg
, GetDR
);
782 para_fill(pv_cpu_ops
.set_debugreg
, SetDR
);
783 para_fill(pv_cpu_ops
.read_cr0
, GetCR0
);
784 para_fill(pv_mmu_ops
.read_cr2
, GetCR2
);
785 para_fill(pv_mmu_ops
.read_cr3
, GetCR3
);
786 para_fill(pv_cpu_ops
.read_cr4
, GetCR4
);
787 para_fill(pv_cpu_ops
.write_cr0
, SetCR0
);
788 para_fill(pv_mmu_ops
.write_cr2
, SetCR2
);
789 para_fill(pv_mmu_ops
.write_cr3
, SetCR3
);
790 para_fill(pv_cpu_ops
.write_cr4
, SetCR4
);
791 para_fill(pv_irq_ops
.save_fl
, GetInterruptMask
);
792 para_fill(pv_irq_ops
.restore_fl
, SetInterruptMask
);
793 para_fill(pv_irq_ops
.irq_disable
, DisableInterrupts
);
794 para_fill(pv_irq_ops
.irq_enable
, EnableInterrupts
);
796 para_fill(pv_cpu_ops
.wbinvd
, WBINVD
);
797 para_fill(pv_cpu_ops
.read_tsc
, RDTSC
);
799 /* The following we emulate with trap and emulate for now */
800 /* paravirt_ops.read_msr = vmi_rdmsr */
801 /* paravirt_ops.write_msr = vmi_wrmsr */
802 /* paravirt_ops.rdpmc = vmi_rdpmc */
804 /* TR interface doesn't pass TR value, wrap */
805 para_wrap(pv_cpu_ops
.load_tr_desc
, vmi_set_tr
, set_tr
, SetTR
);
807 /* LDT is special, too */
808 para_wrap(pv_cpu_ops
.set_ldt
, vmi_set_ldt
, _set_ldt
, SetLDT
);
810 para_fill(pv_cpu_ops
.load_gdt
, SetGDT
);
811 para_fill(pv_cpu_ops
.load_idt
, SetIDT
);
812 para_fill(pv_cpu_ops
.store_gdt
, GetGDT
);
813 para_fill(pv_cpu_ops
.store_idt
, GetIDT
);
814 para_fill(pv_cpu_ops
.store_tr
, GetTR
);
815 pv_cpu_ops
.load_tls
= vmi_load_tls
;
816 para_wrap(pv_cpu_ops
.write_ldt_entry
, vmi_write_ldt_entry
,
817 write_ldt_entry
, WriteLDTEntry
);
818 para_wrap(pv_cpu_ops
.write_gdt_entry
, vmi_write_gdt_entry
,
819 write_gdt_entry
, WriteGDTEntry
);
820 para_wrap(pv_cpu_ops
.write_idt_entry
, vmi_write_idt_entry
,
821 write_idt_entry
, WriteIDTEntry
);
822 para_wrap(pv_cpu_ops
.load_sp0
, vmi_load_sp0
, set_kernel_stack
, UpdateKernelStack
);
823 para_fill(pv_cpu_ops
.set_iopl_mask
, SetIOPLMask
);
824 para_fill(pv_cpu_ops
.io_delay
, IODelay
);
826 para_wrap(pv_cpu_ops
.lazy_mode
.enter
, vmi_enter_lazy_cpu
,
827 set_lazy_mode
, SetLazyMode
);
828 para_wrap(pv_cpu_ops
.lazy_mode
.leave
, vmi_leave_lazy
,
829 set_lazy_mode
, SetLazyMode
);
831 para_wrap(pv_mmu_ops
.lazy_mode
.enter
, vmi_enter_lazy_mmu
,
832 set_lazy_mode
, SetLazyMode
);
833 para_wrap(pv_mmu_ops
.lazy_mode
.leave
, vmi_leave_lazy
,
834 set_lazy_mode
, SetLazyMode
);
836 /* user and kernel flush are just handled with different flags to FlushTLB */
837 para_wrap(pv_mmu_ops
.flush_tlb_user
, vmi_flush_tlb_user
, _flush_tlb
, FlushTLB
);
838 para_wrap(pv_mmu_ops
.flush_tlb_kernel
, vmi_flush_tlb_kernel
, _flush_tlb
, FlushTLB
);
839 para_fill(pv_mmu_ops
.flush_tlb_single
, InvalPage
);
842 * Until a standard flag format can be agreed on, we need to
843 * implement these as wrappers in Linux. Get the VMI ROM
844 * function pointers for the two backend calls.
846 #ifdef CONFIG_X86_PAE
847 vmi_ops
.set_pte
= vmi_get_function(VMI_CALL_SetPxELong
);
848 vmi_ops
.update_pte
= vmi_get_function(VMI_CALL_UpdatePxELong
);
850 vmi_ops
.set_pte
= vmi_get_function(VMI_CALL_SetPxE
);
851 vmi_ops
.update_pte
= vmi_get_function(VMI_CALL_UpdatePxE
);
854 if (vmi_ops
.set_pte
) {
855 pv_mmu_ops
.set_pte
= vmi_set_pte
;
856 pv_mmu_ops
.set_pte_at
= vmi_set_pte_at
;
857 pv_mmu_ops
.set_pmd
= vmi_set_pmd
;
858 #ifdef CONFIG_X86_PAE
859 pv_mmu_ops
.set_pte_atomic
= vmi_set_pte_atomic
;
860 pv_mmu_ops
.set_pte_present
= vmi_set_pte_present
;
861 pv_mmu_ops
.set_pud
= vmi_set_pud
;
862 pv_mmu_ops
.pte_clear
= vmi_pte_clear
;
863 pv_mmu_ops
.pmd_clear
= vmi_pmd_clear
;
867 if (vmi_ops
.update_pte
) {
868 pv_mmu_ops
.pte_update
= vmi_update_pte
;
869 pv_mmu_ops
.pte_update_defer
= vmi_update_pte_defer
;
872 vmi_ops
.allocate_page
= vmi_get_function(VMI_CALL_AllocatePage
);
873 if (vmi_ops
.allocate_page
) {
874 pv_mmu_ops
.alloc_pt
= vmi_allocate_pt
;
875 pv_mmu_ops
.alloc_pd
= vmi_allocate_pd
;
876 pv_mmu_ops
.alloc_pd_clone
= vmi_allocate_pd_clone
;
879 vmi_ops
.release_page
= vmi_get_function(VMI_CALL_ReleasePage
);
880 if (vmi_ops
.release_page
) {
881 pv_mmu_ops
.release_pt
= vmi_release_pt
;
882 pv_mmu_ops
.release_pd
= vmi_release_pd
;
885 /* Set linear is needed in all cases */
886 vmi_ops
.set_linear_mapping
= vmi_get_function(VMI_CALL_SetLinearMapping
);
887 #ifdef CONFIG_HIGHPTE
888 if (vmi_ops
.set_linear_mapping
)
889 pv_mmu_ops
.kmap_atomic_pte
= vmi_kmap_atomic_pte
;
893 * These MUST always be patched. Don't support indirect jumps
894 * through these operations, as the VMI interface may use either
895 * a jump or a call to get to these operations, depending on
896 * the backend. They are performance critical anyway, so requiring
897 * a patch is not a big problem.
899 pv_cpu_ops
.irq_enable_syscall_ret
= (void *)0xfeedbab0;
900 pv_cpu_ops
.iret
= (void *)0xbadbab0;
903 para_wrap(pv_apic_ops
.startup_ipi_hook
, vmi_startup_ipi_hook
, set_initial_ap_state
, SetInitialAPState
);
906 #ifdef CONFIG_X86_LOCAL_APIC
907 para_fill(pv_apic_ops
.apic_read
, APICRead
);
908 para_fill(pv_apic_ops
.apic_write
, APICWrite
);
909 para_fill(pv_apic_ops
.apic_write_atomic
, APICWrite
);
913 * Check for VMI timer functionality by probing for a cycle frequency method
915 reloc
= call_vrom_long_func(vmi_rom
, get_reloc
, VMI_CALL_GetCycleFrequency
);
916 if (!disable_vmi_timer
&& rel
->type
!= VMI_RELOCATION_NONE
) {
917 vmi_timer_ops
.get_cycle_frequency
= (void *)rel
->eip
;
918 vmi_timer_ops
.get_cycle_counter
=
919 vmi_get_function(VMI_CALL_GetCycleCounter
);
920 vmi_timer_ops
.get_wallclock
=
921 vmi_get_function(VMI_CALL_GetWallclockTime
);
922 vmi_timer_ops
.wallclock_updated
=
923 vmi_get_function(VMI_CALL_WallclockUpdated
);
924 vmi_timer_ops
.set_alarm
= vmi_get_function(VMI_CALL_SetAlarm
);
925 vmi_timer_ops
.cancel_alarm
=
926 vmi_get_function(VMI_CALL_CancelAlarm
);
927 pv_time_ops
.time_init
= vmi_time_init
;
928 pv_time_ops
.get_wallclock
= vmi_get_wallclock
;
929 pv_time_ops
.set_wallclock
= vmi_set_wallclock
;
930 #ifdef CONFIG_X86_LOCAL_APIC
931 pv_apic_ops
.setup_boot_clock
= vmi_time_bsp_init
;
932 pv_apic_ops
.setup_secondary_clock
= vmi_time_ap_init
;
934 pv_time_ops
.sched_clock
= vmi_sched_clock
;
935 pv_time_ops
.get_cpu_khz
= vmi_cpu_khz
;
937 /* We have true wallclock functions; disable CMOS clock sync */
938 no_sync_cmos_clock
= 1;
941 disable_vmi_timer
= 1;
944 para_fill(pv_irq_ops
.safe_halt
, Halt
);
947 * Alternative instruction rewriting doesn't happen soon enough
948 * to convert VMI_IRET to a call instead of a jump; so we have
949 * to do this before IRQs get reenabled. Fortunately, it is
952 apply_paravirt(__parainstructions
, __parainstructions_end
);
961 void __init
vmi_init(void)
968 check_vmi_rom(vmi_rom
);
970 /* In case probing for or validating the ROM failed, basil */
974 reserve_top_address(-vmi_rom
->virtual_top
);
976 local_irq_save(flags
);
979 #ifdef CONFIG_X86_IO_APIC
980 /* This is virtual hardware; timer routing is wired correctly */
983 local_irq_restore(flags
& X86_EFLAGS_IF
);
986 static int __init
parse_vmi(char *arg
)
991 if (!strcmp(arg
, "disable_pge")) {
992 clear_cpu_cap(&boot_cpu_data
, X86_FEATURE_PGE
);
994 } else if (!strcmp(arg
, "disable_pse")) {
995 clear_cpu_cap(&boot_cpu_data
, X86_FEATURE_PSE
);
997 } else if (!strcmp(arg
, "disable_sep")) {
998 clear_cpu_cap(&boot_cpu_data
, X86_FEATURE_SEP
);
1000 } else if (!strcmp(arg
, "disable_tsc")) {
1001 clear_cpu_cap(&boot_cpu_data
, X86_FEATURE_TSC
);
1003 } else if (!strcmp(arg
, "disable_mtrr")) {
1004 clear_cpu_cap(&boot_cpu_data
, X86_FEATURE_MTRR
);
1006 } else if (!strcmp(arg
, "disable_timer")) {
1007 disable_vmi_timer
= 1;
1009 } else if (!strcmp(arg
, "disable_noidle"))
1014 early_param("vmi", parse_vmi
);