1 #ifndef __ASM_PARAVIRT_H
2 #define __ASM_PARAVIRT_H
3 /* Various instructions on x86 need to be replaced for
4 * para-virtualization: those hooks are defined here. */
5 #include <linux/linkage.h>
6 #include <linux/stringify.h>
10 /* These are the most performance critical ops, so we want to be able to patch
12 #define PARAVIRT_IRQ_DISABLE 0
13 #define PARAVIRT_IRQ_ENABLE 1
14 #define PARAVIRT_RESTORE_FLAGS 2
15 #define PARAVIRT_SAVE_FLAGS 3
16 #define PARAVIRT_SAVE_FLAGS_IRQ_DISABLE 4
17 #define PARAVIRT_INTERRUPT_RETURN 5
18 #define PARAVIRT_STI_SYSEXIT 6
20 /* Bitmask of what can be clobbered: usually at least eax. */
29 struct Xgt_desc_struct
;
34 unsigned int kernel_rpl
;
39 * Patch may replace one of the defined code sequences with arbitrary
40 * code, subject to the same register constraints. This generally
41 * means the code is not free to clobber any registers other than EAX.
42 * The patch function should return the number of bytes of code
43 * generated, as we nop pad the rest in generic code.
45 unsigned (*patch
)(u8 type
, u16 clobber
, void *firstinsn
, unsigned len
);
47 void (*arch_setup
)(void);
48 char *(*memory_setup
)(void);
49 void (*init_IRQ
)(void);
53 unsigned long (*get_wallclock
)(void);
54 int (*set_wallclock
)(unsigned long);
55 void (*time_init
)(void);
57 /* All the function pointers here are declared as "fastcall"
58 so that we get a specific register-based calling
59 convention. This makes it easier to implement inline
60 assembler replacements. */
62 void (*cpuid
)(unsigned int *eax
, unsigned int *ebx
,
63 unsigned int *ecx
, unsigned int *edx
);
65 unsigned long (*get_debugreg
)(int regno
);
66 void (*set_debugreg
)(int regno
, unsigned long value
);
70 unsigned long (*read_cr0
)(void);
71 void (*write_cr0
)(unsigned long);
73 unsigned long (*read_cr2
)(void);
74 void (*write_cr2
)(unsigned long);
76 unsigned long (*read_cr3
)(void);
77 void (*write_cr3
)(unsigned long);
79 unsigned long (*read_cr4_safe
)(void);
80 unsigned long (*read_cr4
)(void);
81 void (*write_cr4
)(unsigned long);
83 unsigned long (*save_fl
)(void);
84 void (*restore_fl
)(unsigned long);
85 void (*irq_disable
)(void);
86 void (*irq_enable
)(void);
87 void (*safe_halt
)(void);
91 /* err = 0/-EFAULT. wrmsr returns 0/-EFAULT. */
92 u64 (*read_msr
)(unsigned int msr
, int *err
);
93 int (*write_msr
)(unsigned int msr
, u64 val
);
95 u64 (*read_tsc
)(void);
96 u64 (*read_pmc
)(void);
97 u64 (*get_scheduled_cycles
)(void);
99 void (*load_tr_desc
)(void);
100 void (*load_gdt
)(const struct Xgt_desc_struct
*);
101 void (*load_idt
)(const struct Xgt_desc_struct
*);
102 void (*store_gdt
)(struct Xgt_desc_struct
*);
103 void (*store_idt
)(struct Xgt_desc_struct
*);
104 void (*set_ldt
)(const void *desc
, unsigned entries
);
105 unsigned long (*store_tr
)(void);
106 void (*load_tls
)(struct thread_struct
*t
, unsigned int cpu
);
107 void (*write_ldt_entry
)(void *dt
, int entrynum
,
109 void (*write_gdt_entry
)(void *dt
, int entrynum
,
111 void (*write_idt_entry
)(void *dt
, int entrynum
,
113 void (*load_esp0
)(struct tss_struct
*tss
,
114 struct thread_struct
*thread
);
116 void (*set_iopl_mask
)(unsigned mask
);
118 void (*io_delay
)(void);
119 void (*const_udelay
)(unsigned long loops
);
121 #ifdef CONFIG_X86_LOCAL_APIC
122 void (*apic_write
)(unsigned long reg
, unsigned long v
);
123 void (*apic_write_atomic
)(unsigned long reg
, unsigned long v
);
124 unsigned long (*apic_read
)(unsigned long reg
);
125 void (*setup_boot_clock
)(void);
126 void (*setup_secondary_clock
)(void);
129 void (*flush_tlb_user
)(void);
130 void (*flush_tlb_kernel
)(void);
131 void (*flush_tlb_single
)(u32 addr
);
133 void (*alloc_pt
)(u32 pfn
);
134 void (*alloc_pd
)(u32 pfn
);
135 void (*alloc_pd_clone
)(u32 pfn
, u32 clonepfn
, u32 start
, u32 count
);
136 void (*release_pt
)(u32 pfn
);
137 void (*release_pd
)(u32 pfn
);
139 void (*set_pte
)(pte_t
*ptep
, pte_t pteval
);
140 void (*set_pte_at
)(struct mm_struct
*mm
, u32 addr
, pte_t
*ptep
, pte_t pteval
);
141 void (*set_pmd
)(pmd_t
*pmdp
, pmd_t pmdval
);
142 void (*pte_update
)(struct mm_struct
*mm
, u32 addr
, pte_t
*ptep
);
143 void (*pte_update_defer
)(struct mm_struct
*mm
, u32 addr
, pte_t
*ptep
);
144 #ifdef CONFIG_X86_PAE
145 void (*set_pte_atomic
)(pte_t
*ptep
, pte_t pteval
);
146 void (*set_pte_present
)(struct mm_struct
*mm
, unsigned long addr
, pte_t
*ptep
, pte_t pte
);
147 void (*set_pud
)(pud_t
*pudp
, pud_t pudval
);
148 void (*pte_clear
)(struct mm_struct
*mm
, unsigned long addr
, pte_t
*ptep
);
149 void (*pmd_clear
)(pmd_t
*pmdp
);
152 void (*set_lazy_mode
)(int mode
);
154 /* These two are jmp to, not actually called. */
155 void (*irq_enable_sysexit
)(void);
158 void (*startup_ipi_hook
)(int phys_apicid
, unsigned long start_eip
, unsigned long start_esp
);
161 /* Mark a paravirt probe function. */
162 #define paravirt_probe(fn) \
163 static asmlinkage void (*__paravirtprobe_##fn)(void) __attribute_used__ \
164 __attribute__((__section__(".paravirtprobe"))) = fn
166 extern struct paravirt_ops paravirt_ops
;
168 #define paravirt_enabled() (paravirt_ops.paravirt_enabled)
170 static inline void load_esp0(struct tss_struct
*tss
,
171 struct thread_struct
*thread
)
173 paravirt_ops
.load_esp0(tss
, thread
);
176 #define ARCH_SETUP paravirt_ops.arch_setup();
177 static inline unsigned long get_wallclock(void)
179 return paravirt_ops
.get_wallclock();
182 static inline int set_wallclock(unsigned long nowtime
)
184 return paravirt_ops
.set_wallclock(nowtime
);
187 static inline void do_time_init(void)
189 return paravirt_ops
.time_init();
192 /* The paravirtualized CPUID instruction. */
193 static inline void __cpuid(unsigned int *eax
, unsigned int *ebx
,
194 unsigned int *ecx
, unsigned int *edx
)
196 paravirt_ops
.cpuid(eax
, ebx
, ecx
, edx
);
200 * These special macros can be used to get or set a debugging register
202 #define get_debugreg(var, reg) var = paravirt_ops.get_debugreg(reg)
203 #define set_debugreg(val, reg) paravirt_ops.set_debugreg(reg, val)
205 #define clts() paravirt_ops.clts()
207 #define read_cr0() paravirt_ops.read_cr0()
208 #define write_cr0(x) paravirt_ops.write_cr0(x)
210 #define read_cr2() paravirt_ops.read_cr2()
211 #define write_cr2(x) paravirt_ops.write_cr2(x)
213 #define read_cr3() paravirt_ops.read_cr3()
214 #define write_cr3(x) paravirt_ops.write_cr3(x)
216 #define read_cr4() paravirt_ops.read_cr4()
217 #define read_cr4_safe(x) paravirt_ops.read_cr4_safe()
218 #define write_cr4(x) paravirt_ops.write_cr4(x)
220 static inline void raw_safe_halt(void)
222 paravirt_ops
.safe_halt();
225 static inline void halt(void)
227 paravirt_ops
.safe_halt();
229 #define wbinvd() paravirt_ops.wbinvd()
231 #define get_kernel_rpl() (paravirt_ops.kernel_rpl)
233 #define rdmsr(msr,val1,val2) do { \
235 u64 _l = paravirt_ops.read_msr(msr,&_err); \
240 #define wrmsr(msr,val1,val2) do { \
241 u64 _l = ((u64)(val2) << 32) | (val1); \
242 paravirt_ops.write_msr((msr), _l); \
245 #define rdmsrl(msr,val) do { \
247 val = paravirt_ops.read_msr((msr),&_err); \
250 #define wrmsrl(msr,val) (paravirt_ops.write_msr((msr),(val)))
251 #define wrmsr_safe(msr,a,b) ({ \
252 u64 _l = ((u64)(b) << 32) | (a); \
253 paravirt_ops.write_msr((msr),_l); \
256 /* rdmsr with exception handling */
257 #define rdmsr_safe(msr,a,b) ({ \
259 u64 _l = paravirt_ops.read_msr(msr,&_err); \
264 #define rdtsc(low,high) do { \
265 u64 _l = paravirt_ops.read_tsc(); \
270 #define rdtscl(low) do { \
271 u64 _l = paravirt_ops.read_tsc(); \
275 #define rdtscll(val) (val = paravirt_ops.read_tsc())
277 #define get_scheduled_cycles(val) (val = paravirt_ops.get_scheduled_cycles())
279 #define write_tsc(val1,val2) wrmsr(0x10, val1, val2)
281 #define rdpmc(counter,low,high) do { \
282 u64 _l = paravirt_ops.read_pmc(); \
287 #define load_TR_desc() (paravirt_ops.load_tr_desc())
288 #define load_gdt(dtr) (paravirt_ops.load_gdt(dtr))
289 #define load_idt(dtr) (paravirt_ops.load_idt(dtr))
290 #define set_ldt(addr, entries) (paravirt_ops.set_ldt((addr), (entries)))
291 #define store_gdt(dtr) (paravirt_ops.store_gdt(dtr))
292 #define store_idt(dtr) (paravirt_ops.store_idt(dtr))
293 #define store_tr(tr) ((tr) = paravirt_ops.store_tr())
294 #define load_TLS(t,cpu) (paravirt_ops.load_tls((t),(cpu)))
295 #define write_ldt_entry(dt, entry, low, high) \
296 (paravirt_ops.write_ldt_entry((dt), (entry), (low), (high)))
297 #define write_gdt_entry(dt, entry, low, high) \
298 (paravirt_ops.write_gdt_entry((dt), (entry), (low), (high)))
299 #define write_idt_entry(dt, entry, low, high) \
300 (paravirt_ops.write_idt_entry((dt), (entry), (low), (high)))
301 #define set_iopl_mask(mask) (paravirt_ops.set_iopl_mask(mask))
303 /* The paravirtualized I/O functions */
304 static inline void slow_down_io(void) {
305 paravirt_ops
.io_delay();
306 #ifdef REALLY_SLOW_IO
307 paravirt_ops
.io_delay();
308 paravirt_ops
.io_delay();
309 paravirt_ops
.io_delay();
313 #ifdef CONFIG_X86_LOCAL_APIC
315 * Basic functions accessing APICs.
317 static inline void apic_write(unsigned long reg
, unsigned long v
)
319 paravirt_ops
.apic_write(reg
,v
);
322 static inline void apic_write_atomic(unsigned long reg
, unsigned long v
)
324 paravirt_ops
.apic_write_atomic(reg
,v
);
327 static inline unsigned long apic_read(unsigned long reg
)
329 return paravirt_ops
.apic_read(reg
);
332 static inline void setup_boot_clock(void)
334 paravirt_ops
.setup_boot_clock();
337 static inline void setup_secondary_clock(void)
339 paravirt_ops
.setup_secondary_clock();
344 static inline void startup_ipi_hook(int phys_apicid
, unsigned long start_eip
,
345 unsigned long start_esp
)
347 return paravirt_ops
.startup_ipi_hook(phys_apicid
, start_eip
, start_esp
);
351 #define __flush_tlb() paravirt_ops.flush_tlb_user()
352 #define __flush_tlb_global() paravirt_ops.flush_tlb_kernel()
353 #define __flush_tlb_single(addr) paravirt_ops.flush_tlb_single(addr)
355 #define paravirt_alloc_pt(pfn) paravirt_ops.alloc_pt(pfn)
356 #define paravirt_release_pt(pfn) paravirt_ops.release_pt(pfn)
358 #define paravirt_alloc_pd(pfn) paravirt_ops.alloc_pd(pfn)
359 #define paravirt_alloc_pd_clone(pfn, clonepfn, start, count) \
360 paravirt_ops.alloc_pd_clone(pfn, clonepfn, start, count)
361 #define paravirt_release_pd(pfn) paravirt_ops.release_pd(pfn)
363 static inline void set_pte(pte_t
*ptep
, pte_t pteval
)
365 paravirt_ops
.set_pte(ptep
, pteval
);
368 static inline void set_pte_at(struct mm_struct
*mm
, u32 addr
, pte_t
*ptep
, pte_t pteval
)
370 paravirt_ops
.set_pte_at(mm
, addr
, ptep
, pteval
);
373 static inline void set_pmd(pmd_t
*pmdp
, pmd_t pmdval
)
375 paravirt_ops
.set_pmd(pmdp
, pmdval
);
378 static inline void pte_update(struct mm_struct
*mm
, u32 addr
, pte_t
*ptep
)
380 paravirt_ops
.pte_update(mm
, addr
, ptep
);
383 static inline void pte_update_defer(struct mm_struct
*mm
, u32 addr
, pte_t
*ptep
)
385 paravirt_ops
.pte_update_defer(mm
, addr
, ptep
);
388 #ifdef CONFIG_X86_PAE
389 static inline void set_pte_atomic(pte_t
*ptep
, pte_t pteval
)
391 paravirt_ops
.set_pte_atomic(ptep
, pteval
);
394 static inline void set_pte_present(struct mm_struct
*mm
, unsigned long addr
, pte_t
*ptep
, pte_t pte
)
396 paravirt_ops
.set_pte_present(mm
, addr
, ptep
, pte
);
399 static inline void set_pud(pud_t
*pudp
, pud_t pudval
)
401 paravirt_ops
.set_pud(pudp
, pudval
);
404 static inline void pte_clear(struct mm_struct
*mm
, unsigned long addr
, pte_t
*ptep
)
406 paravirt_ops
.pte_clear(mm
, addr
, ptep
);
409 static inline void pmd_clear(pmd_t
*pmdp
)
411 paravirt_ops
.pmd_clear(pmdp
);
415 /* Lazy mode for batching updates / context switch */
416 #define PARAVIRT_LAZY_NONE 0
417 #define PARAVIRT_LAZY_MMU 1
418 #define PARAVIRT_LAZY_CPU 2
420 #define __HAVE_ARCH_ENTER_LAZY_CPU_MODE
421 #define arch_enter_lazy_cpu_mode() paravirt_ops.set_lazy_mode(PARAVIRT_LAZY_CPU)
422 #define arch_leave_lazy_cpu_mode() paravirt_ops.set_lazy_mode(PARAVIRT_LAZY_NONE)
424 #define __HAVE_ARCH_ENTER_LAZY_MMU_MODE
425 #define arch_enter_lazy_mmu_mode() paravirt_ops.set_lazy_mode(PARAVIRT_LAZY_MMU)
426 #define arch_leave_lazy_mmu_mode() paravirt_ops.set_lazy_mode(PARAVIRT_LAZY_NONE)
428 /* These all sit in the .parainstructions section to tell us what to patch. */
429 struct paravirt_patch
{
430 u8
*instr
; /* original instructions */
431 u8 instrtype
; /* type of this instruction */
432 u8 len
; /* length of original instruction */
433 u16 clobbers
; /* what registers you may clobber */
436 #define paravirt_alt(insn_string, typenum, clobber) \
437 "771:\n\t" insn_string "\n" "772:\n" \
438 ".pushsection .parainstructions,\"a\"\n" \
440 " .byte " __stringify(typenum) "\n" \
441 " .byte 772b-771b\n" \
442 " .short " __stringify(clobber) "\n" \
445 static inline unsigned long __raw_local_save_flags(void)
449 __asm__
__volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;"
451 "popl %%edx; popl %%ecx",
452 PARAVIRT_SAVE_FLAGS
, CLBR_NONE
)
453 : "=a"(f
): "m"(paravirt_ops
.save_fl
)
458 static inline void raw_local_irq_restore(unsigned long f
)
460 __asm__
__volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;"
462 "popl %%edx; popl %%ecx",
463 PARAVIRT_RESTORE_FLAGS
, CLBR_EAX
)
464 : "=a"(f
) : "m" (paravirt_ops
.restore_fl
), "0"(f
)
468 static inline void raw_local_irq_disable(void)
470 __asm__
__volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;"
472 "popl %%edx; popl %%ecx",
473 PARAVIRT_IRQ_DISABLE
, CLBR_EAX
)
474 : : "m" (paravirt_ops
.irq_disable
)
475 : "memory", "eax", "cc");
478 static inline void raw_local_irq_enable(void)
480 __asm__
__volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;"
482 "popl %%edx; popl %%ecx",
483 PARAVIRT_IRQ_ENABLE
, CLBR_EAX
)
484 : : "m" (paravirt_ops
.irq_enable
)
485 : "memory", "eax", "cc");
488 static inline unsigned long __raw_local_irq_save(void)
492 __asm__
__volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;"
493 "call *%1; pushl %%eax;"
494 "call *%2; popl %%eax;"
495 "popl %%edx; popl %%ecx",
496 PARAVIRT_SAVE_FLAGS_IRQ_DISABLE
,
499 : "m" (paravirt_ops
.save_fl
),
500 "m" (paravirt_ops
.irq_disable
)
505 #define CLI_STRING paravirt_alt("pushl %%ecx; pushl %%edx;" \
506 "call *paravirt_ops+%c[irq_disable];" \
507 "popl %%edx; popl %%ecx", \
508 PARAVIRT_IRQ_DISABLE, CLBR_EAX)
510 #define STI_STRING paravirt_alt("pushl %%ecx; pushl %%edx;" \
511 "call *paravirt_ops+%c[irq_enable];" \
512 "popl %%edx; popl %%ecx", \
513 PARAVIRT_IRQ_ENABLE, CLBR_EAX)
514 #define CLI_STI_CLOBBERS , "%eax"
515 #define CLI_STI_INPUT_ARGS \
517 [irq_disable] "i" (offsetof(struct paravirt_ops, irq_disable)), \
518 [irq_enable] "i" (offsetof(struct paravirt_ops, irq_enable))
520 #else /* __ASSEMBLY__ */
522 #define PARA_PATCH(ptype, clobbers, ops) \
526 .pushsection .parainstructions,"a"; \
533 #define INTERRUPT_RETURN \
534 PARA_PATCH(PARAVIRT_INTERRUPT_RETURN, CLBR_ANY, \
535 jmp *%cs:paravirt_ops+PARAVIRT_iret)
537 #define DISABLE_INTERRUPTS(clobbers) \
538 PARA_PATCH(PARAVIRT_IRQ_DISABLE, clobbers, \
539 pushl %ecx; pushl %edx; \
540 call *paravirt_ops+PARAVIRT_irq_disable; \
541 popl %edx; popl %ecx) \
543 #define ENABLE_INTERRUPTS(clobbers) \
544 PARA_PATCH(PARAVIRT_IRQ_ENABLE, clobbers, \
545 pushl %ecx; pushl %edx; \
546 call *%cs:paravirt_ops+PARAVIRT_irq_enable; \
547 popl %edx; popl %ecx)
549 #define ENABLE_INTERRUPTS_SYSEXIT \
550 PARA_PATCH(PARAVIRT_STI_SYSEXIT, CLBR_ANY, \
551 jmp *%cs:paravirt_ops+PARAVIRT_irq_enable_sysexit)
553 #define GET_CR0_INTO_EAX \
554 call *paravirt_ops+PARAVIRT_read_cr0
556 #endif /* __ASSEMBLY__ */
557 #endif /* CONFIG_PARAVIRT */
558 #endif /* __ASM_PARAVIRT_H */