kvm/kernel/x86/preempt.c

   1
   2 #ifdef CONFIG_PREEMPT_NOTIFIERS_COMPAT
   3
   4 #include <linux/sched.h>
   5 #include <linux/percpu.h>
   6
   7 static DEFINE_SPINLOCK(pn_lock);
   8 static LIST_HEAD(pn_list);
   9
  10 #define dprintk(fmt) do {                                               \
  11                 if (0)                                                  \
  12                         printk("%s (%d/%d): " fmt, __FUNCTION__,        \
  13                                current->pid, raw_smp_processor_id());   \
  14         } while (0)
  15
  16 #if !defined(CONFIG_X86_64) && (LINUX_VERSION_CODE < KERNEL_VERSION(2,6,25))
  17 #define debugreg(x) debugreg[x]
  18 #else
  19 #define debugreg(x) debugreg##x
  20 #endif
  21
  22 static void preempt_enable_sched_out_notifiers(void)
  23 {
  24         asm volatile ("mov %0, %%db0" : : "r"(schedule));
  25         asm volatile ("mov %0, %%db7" : : "r"(0x701ul));
  26         current->thread.debugreg(7) = 0ul;
  27 #ifdef TIF_DEBUG
  28         clear_tsk_thread_flag(current, TIF_DEBUG);
  29 #endif
  30 }
  31
  32 static void preempt_enable_sched_in_notifiers(void * addr)
  33 {
  34         asm volatile ("mov %0, %%db0" : : "r"(addr));
  35         asm volatile ("mov %0, %%db7" : : "r"(0x701ul));
  36         current->thread.debugreg(0) = (unsigned long) addr;
  37         current->thread.debugreg(7) = 0x701ul;
  38 #ifdef TIF_DEBUG
  39         set_tsk_thread_flag(current, TIF_DEBUG);
  40 #endif
  41 }
  42
  43 static void __preempt_disable_notifiers(void)
  44 {
  45         asm volatile ("mov %0, %%db7" : : "r"(0ul));
  46 }
  47
  48 static void preempt_disable_notifiers(void)
  49 {
  50         __preempt_disable_notifiers();
  51         current->thread.debugreg(7) = 0ul;
  52 #ifdef TIF_DEBUG
  53         clear_tsk_thread_flag(current, TIF_DEBUG);
  54 #endif
  55 }
  56
  57 static void fastcall  __attribute__((used)) preempt_notifier_trigger(void *** ip)
  58 {
  59         struct preempt_notifier *pn;
  60         int cpu = raw_smp_processor_id();
  61         int found = 0;
  62
  63         dprintk(" - in\n");
  64         //dump_stack();
  65         spin_lock(&pn_lock);
  66         list_for_each_entry(pn, &pn_list, link)
  67                 if (pn->tsk == current) {
  68                         found = 1;
  69                         break;
  70                 }
  71         spin_unlock(&pn_lock);
  72
  73         if (found) {
  74                 if ((void *) *ip != schedule) {
  75                         dprintk("sched_in\n");
  76                         preempt_enable_sched_out_notifiers();
  77
  78                         preempt_disable();
  79                         local_irq_enable();
  80                         pn->ops->sched_in(pn, cpu);
  81                         local_irq_disable();
  82                         preempt_enable_no_resched();
  83                 } else {
  84                         void * sched_in_addr;
  85                         dprintk("sched_out\n");
  86 #ifdef CONFIG_X86_64
  87                         sched_in_addr = **(ip+3);
  88 #else
  89                         /* no special debug stack switch on x86 */
  90                         sched_in_addr = (void *) *(ip+3);
  91 #endif
  92                         preempt_enable_sched_in_notifiers(sched_in_addr);
  93
  94                         preempt_disable();
  95                         local_irq_enable();
  96                         pn->ops->sched_out(pn, NULL);
  97                         local_irq_disable();
  98                         preempt_enable_no_resched();
  99                 }
 100         } else
 101                 __preempt_disable_notifiers();
 102         dprintk(" - out\n");
 103 }
 104
 105 unsigned long orig_int1_handler;
 106
 107 #ifdef CONFIG_X86_64
 108
 109 #define SAVE_REGS \
 110         "push %rax; push %rbx; push %rcx; push %rdx; " \
 111         "push %rsi; push %rdi; push %rbp; " \
 112         "push %r8;  push %r9;  push %r10; push %r11; " \
 113         "push %r12; push %r13; push %r14; push %r15"
 114
 115 #define RESTORE_REGS \
 116         "pop %r15; pop %r14; pop %r13; pop %r12; " \
 117         "pop %r11; pop %r10; pop %r9;  pop %r8; " \
 118         "pop %rbp; pop %rdi; pop %rsi; " \
 119         "pop %rdx; pop %rcx; pop %rbx; pop %rax "
 120
 121 #define TMP "%rax"
 122
 123 #else
 124
 125 #define SAVE_REGS "pusha"
 126 #define RESTORE_REGS "popa"
 127 #define TMP "%eax"
 128
 129 #endif
 130
 131 asm ("pn_int1_handler:  \n\t"
 132      "push "  TMP " \n\t"
 133      "mov %db7, " TMP " \n\t"
 134      "cmp $0x701, " TMP " \n\t"
 135      "pop "  TMP " \n\t"
 136      "jnz .Lnotme \n\t"
 137      "push "  TMP " \n\t"
 138      "mov %db6, " TMP " \n\t"
 139      "test $0x1, " TMP " \n\t"
 140      "pop "  TMP " \n\t"
 141      "jz .Lnotme \n\t"
 142      SAVE_REGS "\n\t"
 143 #ifdef CONFIG_X86_64
 144      "leaq 120(%rsp),%rdi\n\t"
 145 #else
 146      "leal 32(%esp),%eax\n\t"
 147 #endif
 148      "call preempt_notifier_trigger \n\t"
 149      RESTORE_REGS "\n\t"
 150 #ifdef CONFIG_X86_64
 151      "orq $0x10000, 16(%rsp) \n\t"
 152      "iretq \n\t"
 153 #else
 154      "orl $0x10000, 8(%esp) \n\t"
 155      "iret \n\t"
 156 #endif
 157      ".Lnotme: \n\t"
 158 #ifdef CONFIG_X86_64
 159      "jmpq *orig_int1_handler\n\t"
 160 #else
 161      "jmpl *orig_int1_handler\n\t"
 162 #endif
 163         );
 164
 165 void preempt_notifier_register(struct preempt_notifier *notifier)
 166 {
 167         unsigned long flags;
 168
 169         dprintk(" - in\n");
 170         spin_lock_irqsave(&pn_lock, flags);
 171         preempt_enable_sched_out_notifiers();
 172         notifier->tsk = current;
 173         list_add(&notifier->link, &pn_list);
 174         spin_unlock_irqrestore(&pn_lock, flags);
 175         dprintk(" - out\n");
 176 }
 177
 178 void preempt_notifier_unregister(struct preempt_notifier *notifier)
 179 {
 180         unsigned long flags;
 181
 182         dprintk(" - in\n");
 183         spin_lock_irqsave(&pn_lock, flags);
 184         list_del(&notifier->link);
 185         spin_unlock_irqrestore(&pn_lock, flags);
 186         preempt_disable_notifiers();
 187         dprintk(" - out\n");
 188 }
 189
 190 struct intr_gate {
 191         u16 offset0;
 192         u16 segment;
 193         u16 junk;
 194         u16 offset1;
 195 #ifdef CONFIG_X86_64
 196         u32 offset2;
 197         u32 blah;
 198 #endif
 199 } __attribute__((packed));
 200
 201 struct idt_desc {
 202         u16 limit;
 203         struct intr_gate *gates;
 204 } __attribute__((packed));
 205
 206 static struct intr_gate orig_int1_gate;
 207
 208 void pn_int1_handler(void);
 209
 210 void preempt_notifier_sys_init(void)
 211 {
 212         struct idt_desc idt_desc;
 213         struct intr_gate *int1_gate;
 214
 215         printk("kvm: emulating preempt notifiers;"
 216                " do not benchmark on this machine\n");
 217         dprintk("\n");
 218         asm ("sidt %0" : "=m"(idt_desc));
 219         int1_gate = &idt_desc.gates[1];
 220         orig_int1_gate = *int1_gate;
 221         orig_int1_handler = int1_gate->offset0
 222                 | ((u32)int1_gate->offset1 << 16);
 223 #ifdef CONFIG_X86_64
 224         orig_int1_handler |= (u64)int1_gate->offset2 << 32;
 225 #endif
 226         int1_gate->offset0 = (unsigned long)pn_int1_handler;
 227         int1_gate->offset1 = (unsigned long)pn_int1_handler >> 16;
 228 #ifdef CONFIG_X86_64
 229         int1_gate->offset2 = (unsigned long)pn_int1_handler >> 32;
 230 #endif
 231 }
 232
 233 static void do_disable(void *blah)
 234 {
 235 #ifdef TIF_DEBUG
 236         if (!test_tsk_thread_flag(current, TIF_DEBUG))
 237 #else
 238         if (!current->thread.debugreg(7))
 239 #endif
 240                 __preempt_disable_notifiers();
 241 }
 242
 243 void preempt_notifier_sys_exit(void)
 244 {
 245         struct idt_desc idt_desc;
 246
 247         dprintk("\n");
 248         kvm_on_each_cpu(do_disable, NULL, 1);
 249         asm ("sidt %0" : "=m"(idt_desc));
 250         idt_desc.gates[1] = orig_int1_gate;
 251 }
 252
 253 #endif