kernel/2.6.29.6-aldebaran-rt/arch/x86/include/asm/tlbflush.h

   1 #ifndef _ASM_X86_TLBFLUSH_H
   2 #define _ASM_X86_TLBFLUSH_H
   3
   4 #include <linux/mm.h>
   5 #include <linux/sched.h>
   6
   7 #include <asm/processor.h>
   8 #include <asm/system.h>
   9
  10 /*
  11  * TLB-flush needs to be nonpreemptible on PREEMPT_RT due to the
  12  * following complex race scenario:
  13  *
  14  * if the current task is lazy-TLB and does a TLB flush and
  15  * gets preempted after the movl %%r3, %0 but before the
  16  * movl %0, %%cr3 then its ->active_mm might change and it will
  17  * install the wrong cr3 when it switches back. This is not a
  18  * problem for the lazy-TLB task itself, but if the next task it
  19  * switches to has an ->mm that is also the lazy-TLB task's
  20  * new ->active_mm, then the scheduler will assume that cr3 is
  21  * the new one, while we overwrote it with the old one. The result
  22  * is the wrong cr3 in the new (non-lazy-TLB) task, which typically
  23  * causes an infinite pagefault upon the next userspace access.
  24  */
  25 #ifdef CONFIG_PARAVIRT
  26 #include <asm/paravirt.h>
  27 #else
  28 #define __flush_tlb() __native_flush_tlb()
  29 #define __flush_tlb_global() __native_flush_tlb_global()
  30 #define __flush_tlb_single(addr) __native_flush_tlb_single(addr)
  31 #endif
  32
  33 static inline void __native_flush_tlb(void)
  34 {
  35         preempt_disable();
  36         write_cr3(read_cr3());
  37         preempt_enable();
  38 }
  39
  40 static inline void __native_flush_tlb_global(void)
  41 {
  42         unsigned long flags;
  43         unsigned long cr4;
  44
  45         /*
  46          * Read-modify-write to CR4 - protect it from preemption and
  47          * from interrupts. (Use the raw variant because this code can
  48          * be called from deep inside debugging code.)
  49          */
  50         raw_local_irq_save(flags);
  51
  52         cr4 = read_cr4();
  53         /* clear PGE */
  54         write_cr4(cr4 & ~X86_CR4_PGE);
  55         /* write old PGE again and flush TLBs */
  56         write_cr4(cr4);
  57
  58         raw_local_irq_restore(flags);
  59 }
  60
  61 static inline void __native_flush_tlb_single(unsigned long addr)
  62 {
  63         asm volatile("invlpg (%0)" ::"r" (addr) : "memory");
  64 }
  65
  66 static inline void __flush_tlb_all(void)
  67 {
  68         if (cpu_has_pge)
  69                 __flush_tlb_global();
  70         else
  71                 __flush_tlb();
  72 }
  73
  74 static inline void __flush_tlb_one(unsigned long addr)
  75 {
  76         if (cpu_has_invlpg)
  77                 __flush_tlb_single(addr);
  78         else
  79                 __flush_tlb();
  80 }
  81
  82 #ifdef CONFIG_X86_32
  83 # define TLB_FLUSH_ALL  0xffffffff
  84 #else
  85 # define TLB_FLUSH_ALL  -1ULL
  86 #endif
  87
  88 /*
  89  * TLB flushing:
  90  *
  91  *  - flush_tlb() flushes the current mm struct TLBs
  92  *  - flush_tlb_all() flushes all processes TLBs
  93  *  - flush_tlb_mm(mm) flushes the specified mm context TLB's
  94  *  - flush_tlb_page(vma, vmaddr) flushes one page
  95  *  - flush_tlb_range(vma, start, end) flushes a range of pages
  96  *  - flush_tlb_kernel_range(start, end) flushes a range of kernel pages
  97  *  - flush_tlb_others(cpumask, mm, va) flushes TLBs on other cpus
  98  *
  99  * ..but the i386 has somewhat limited tlb flushing capabilities,
 100  * and page-granular flushes are available only on i486 and up.
 101  *
 102  * x86-64 can only flush individual pages or full VMs. For a range flush
 103  * we always do the full VM. Might be worth trying if for a small
 104  * range a few INVLPGs in a row are a win.
 105  */
 106
 107 #ifndef CONFIG_SMP
 108
 109 #define flush_tlb() __flush_tlb()
 110 #define flush_tlb_all() __flush_tlb_all()
 111 #define local_flush_tlb() __flush_tlb()
 112
 113 static inline void flush_tlb_mm(struct mm_struct *mm)
 114 {
 115         /*
 116          * This is safe on PREEMPT_RT because if we preempt
 117          * right after the check but before the __flush_tlb(),
 118          * and if ->active_mm changes, then we might miss a
 119          * TLB flush, but that TLB flush happened already when
 120          * ->active_mm was changed:
 121          */
 122         if (mm == current->active_mm)
 123                 __flush_tlb();
 124 }
 125
 126 static inline void flush_tlb_page(struct vm_area_struct *vma,
 127                                   unsigned long addr)
 128 {
 129         if (vma->vm_mm == current->active_mm)
 130                 __flush_tlb_one(addr);
 131 }
 132
 133 static inline void flush_tlb_range(struct vm_area_struct *vma,
 134                                    unsigned long start, unsigned long end)
 135 {
 136         if (vma->vm_mm == current->active_mm)
 137                 __flush_tlb();
 138 }
 139
 140 static inline void native_flush_tlb_others(const struct cpumask *cpumask,
 141                                            struct mm_struct *mm,
 142                                            unsigned long va)
 143 {
 144 }
 145
 146 static inline void reset_lazy_tlbstate(void)
 147 {
 148 }
 149
 150 #else  /* SMP */
 151
 152 #include <asm/smp.h>
 153
 154 #define local_flush_tlb() __flush_tlb()
 155
 156 extern void flush_tlb_all(void);
 157 extern void flush_tlb_current_task(void);
 158 extern void flush_tlb_mm(struct mm_struct *);
 159 extern void flush_tlb_page(struct vm_area_struct *, unsigned long);
 160
 161 #define flush_tlb()     flush_tlb_current_task()
 162
 163 static inline void flush_tlb_range(struct vm_area_struct *vma,
 164                                    unsigned long start, unsigned long end)
 165 {
 166         flush_tlb_mm(vma->vm_mm);
 167 }
 168
 169 void native_flush_tlb_others(const struct cpumask *cpumask,
 170                              struct mm_struct *mm, unsigned long va);
 171
 172 #define TLBSTATE_OK     1
 173 #define TLBSTATE_LAZY   2
 174
 175 struct tlb_state {
 176         struct mm_struct *active_mm;
 177         int state;
 178 };
 179 DECLARE_PER_CPU(struct tlb_state, cpu_tlbstate);
 180
 181 static inline void reset_lazy_tlbstate(void)
 182 {
 183         percpu_write(cpu_tlbstate.state, 0);
 184         percpu_write(cpu_tlbstate.active_mm, &init_mm);
 185 }
 186
 187 #endif  /* SMP */
 188
 189 #ifndef CONFIG_PARAVIRT
 190 #define flush_tlb_others(mask, mm, va)  native_flush_tlb_others(mask, mm, va)
 191 #endif
 192
 193 static inline void flush_tlb_kernel_range(unsigned long start,
 194                                           unsigned long end)
 195 {
 196         flush_tlb_all();
 197 }
 198
 199 extern void zap_low_mappings(void);
 200
 201 #endif /* _ASM_X86_TLBFLUSH_H */