kvm/kernel/preempt.c

   1
   2 #ifdef CONFIG_PREEMPT_NOTIFIERS_COMPAT
   3
   4 #include <linux/sched.h>
   5 #include <linux/percpu.h>
   6
   7 static DEFINE_SPINLOCK(pn_lock);
   8 static LIST_HEAD(pn_list);
   9
  10 #define dprintk(fmt) do {                                               \
  11                 if (0)                                                  \
  12                         printk("%s (%d/%d): " fmt, __FUNCTION__,        \
  13                                current->pid, raw_smp_processor_id());   \
  14         } while (0)
  15
  16 #if !defined(CONFIG_X86_64) && (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,25))
  17 #define debugreg(x) debugreg[x]
  18 #else
  19 #define debugreg(x) debugreg##x
  20 #endif
  21
  22 static void preempt_enable_sched_out_notifiers(void)
  23 {
  24         asm volatile ("mov %0, %%db0" : : "r"(schedule));
  25         asm volatile ("mov %0, %%db7" : : "r"(0x701ul));
  26         current->thread.debugreg(7) = 0ul;
  27 #ifdef TIF_DEBUG
  28         clear_tsk_thread_flag(current, TIF_DEBUG);
  29 #endif
  30 }
  31
  32 static void preempt_enable_sched_in_notifiers(void * addr)
  33 {
  34         asm volatile ("mov %0, %%db0" : : "r"(addr));
  35         asm volatile ("mov %0, %%db7" : : "r"(0x701ul));
  36         current->thread.debugreg(0) = (unsigned long) addr;
  37         current->thread.debugreg(7) = 0x701ul;
  38 #ifdef TIF_DEBUG
  39         set_tsk_thread_flag(current, TIF_DEBUG);
  40 #endif
  41 }
  42
  43 void special_reload_dr7(void)
  44 {
  45         asm volatile ("mov %0, %%db7" : : "r"(0x701ul));
  46 }
  47 EXPORT_SYMBOL_GPL(special_reload_dr7);
  48
  49 static void __preempt_disable_notifiers(void)
  50 {
  51         asm volatile ("mov %0, %%db7" : : "r"(0ul));
  52 }
  53
  54 static void preempt_disable_notifiers(void)
  55 {
  56         __preempt_disable_notifiers();
  57         current->thread.debugreg(7) = 0ul;
  58 #ifdef TIF_DEBUG
  59         clear_tsk_thread_flag(current, TIF_DEBUG);
  60 #endif
  61 }
  62
  63 static void fastcall  __attribute__((used)) preempt_notifier_trigger(void *** ip)
  64 {
  65         struct preempt_notifier *pn;
  66         int cpu = raw_smp_processor_id();
  67         int found = 0;
  68
  69         dprintk(" - in\n");
  70         //dump_stack();
  71         spin_lock(&pn_lock);
  72         list_for_each_entry(pn, &pn_list, link)
  73                 if (pn->tsk == current) {
  74                         found = 1;
  75                         break;
  76                 }
  77         spin_unlock(&pn_lock);
  78
  79         if (found) {
  80                 if ((void *) *ip != schedule) {
  81                         dprintk("sched_in\n");
  82                         preempt_enable_sched_out_notifiers();
  83
  84                         preempt_disable();
  85                         local_irq_enable();
  86                         pn->ops->sched_in(pn, cpu);
  87                         local_irq_disable();
  88                         preempt_enable_no_resched();
  89                 } else {
  90                         void * sched_in_addr;
  91                         dprintk("sched_out\n");
  92 #ifdef CONFIG_X86_64
  93                         sched_in_addr = **(ip+3);
  94 #else
  95                         /* no special debug stack switch on x86 */
  96                         sched_in_addr = (void *) *(ip+3);
  97 #endif
  98                         preempt_enable_sched_in_notifiers(sched_in_addr);
  99
 100                         preempt_disable();
 101                         local_irq_enable();
 102                         pn->ops->sched_out(pn, NULL);
 103                         local_irq_disable();
 104                         preempt_enable_no_resched();
 105                 }
 106         } else
 107                 __preempt_disable_notifiers();
 108         dprintk(" - out\n");
 109 }
 110
 111 unsigned long orig_int1_handler;
 112
 113 #ifdef CONFIG_X86_64
 114
 115 #define SAVE_REGS \
 116         "push %rax; push %rbx; push %rcx; push %rdx; " \
 117         "push %rsi; push %rdi; push %rbp; " \
 118         "push %r8;  push %r9;  push %r10; push %r11; " \
 119         "push %r12; push %r13; push %r14; push %r15"
 120
 121 #define RESTORE_REGS \
 122         "pop %r15; pop %r14; pop %r13; pop %r12; " \
 123         "pop %r11; pop %r10; pop %r9;  pop %r8; " \
 124         "pop %rbp; pop %rdi; pop %rsi; " \
 125         "pop %rdx; pop %rcx; pop %rbx; pop %rax "
 126
 127 #define TMP "%rax"
 128
 129 #else
 130
 131 #define SAVE_REGS "pusha"
 132 #define RESTORE_REGS "popa"
 133 #define TMP "%eax"
 134
 135 #endif
 136
 137 asm ("pn_int1_handler:  \n\t"
 138      "push "  TMP " \n\t"
 139      "mov %db7, " TMP " \n\t"
 140      "cmp $0x701, " TMP " \n\t"
 141      "pop "  TMP " \n\t"
 142      "jnz .Lnotme \n\t"
 143      SAVE_REGS "\n\t"
 144 #ifdef CONFIG_X86_64
 145      "leaq 120(%rsp),%rdi\n\t"
 146 #else
 147      "leal 32(%esp),%eax\n\t"
 148 #endif
 149      "call preempt_notifier_trigger \n\t"
 150      RESTORE_REGS "\n\t"
 151 #ifdef CONFIG_X86_64
 152      "orq $0x10000, 16(%rsp) \n\t"
 153      "iretq \n\t"
 154 #else
 155      "orl $0x10000, 8(%esp) \n\t"
 156      "iret \n\t"
 157 #endif
 158      ".Lnotme: \n\t"
 159 #ifdef CONFIG_X86_64
 160      "jmpq *orig_int1_handler\n\t"
 161 #else
 162      "jmpl *orig_int1_handler\n\t"
 163 #endif
 164         );
 165
 166 void preempt_notifier_register(struct preempt_notifier *notifier)
 167 {
 168         unsigned long flags;
 169
 170         dprintk(" - in\n");
 171         spin_lock_irqsave(&pn_lock, flags);
 172         preempt_enable_sched_out_notifiers();
 173         notifier->tsk = current;
 174         list_add(&notifier->link, &pn_list);
 175         spin_unlock_irqrestore(&pn_lock, flags);
 176         dprintk(" - out\n");
 177 }
 178
 179 void preempt_notifier_unregister(struct preempt_notifier *notifier)
 180 {
 181         unsigned long flags;
 182
 183         dprintk(" - in\n");
 184         spin_lock_irqsave(&pn_lock, flags);
 185         list_del(&notifier->link);
 186         spin_unlock_irqrestore(&pn_lock, flags);
 187         preempt_disable_notifiers();
 188         dprintk(" - out\n");
 189 }
 190
 191 struct intr_gate {
 192         u16 offset0;
 193         u16 segment;
 194         u16 junk;
 195         u16 offset1;
 196 #ifdef CONFIG_X86_64
 197         u32 offset2;
 198         u32 blah;
 199 #endif
 200 } __attribute__((packed));
 201
 202 struct idt_desc {
 203         u16 limit;
 204         struct intr_gate *gates;
 205 } __attribute__((packed));
 206
 207 static struct intr_gate orig_int1_gate;
 208
 209 void pn_int1_handler(void);
 210
 211 void preempt_notifier_sys_init(void)
 212 {
 213         struct idt_desc idt_desc;
 214         struct intr_gate *int1_gate;
 215
 216         printk("kvm: emulating preempt notifiers;"
 217                " do not benchmark on this machine\n");
 218         dprintk("\n");
 219         asm ("sidt %0" : "=m"(idt_desc));
 220         int1_gate = &idt_desc.gates[1];
 221         orig_int1_gate = *int1_gate;
 222         orig_int1_handler = int1_gate->offset0
 223                 | ((u32)int1_gate->offset1 << 16);
 224 #ifdef CONFIG_X86_64
 225         orig_int1_handler |= (u64)int1_gate->offset2 << 32;
 226 #endif
 227         int1_gate->offset0 = (unsigned long)pn_int1_handler;
 228         int1_gate->offset1 = (unsigned long)pn_int1_handler >> 16;
 229 #ifdef CONFIG_X86_64
 230         int1_gate->offset2 = (unsigned long)pn_int1_handler >> 32;
 231 #endif
 232 }
 233
 234 static void do_disable(void *blah)
 235 {
 236 #ifdef TIF_DEBUG
 237         if (!test_tsk_thread_flag(current, TIF_DEBUG))
 238 #else
 239         if (!current->thread.debugreg(7))
 240 #endif
 241                 __preempt_disable_notifiers();
 242 }
 243
 244 void preempt_notifier_sys_exit(void)
 245 {
 246         struct idt_desc idt_desc;
 247
 248         dprintk("\n");
 249         on_each_cpu(do_disable, NULL, 1, 1);
 250         asm ("sidt %0" : "=m"(idt_desc));
 251         idt_desc.gates[1] = orig_int1_gate;
 252 }
 253
 254 #endif