?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
x86: properly initialize temp insn buffer for paravirt patching
[linux-2.6/sactl.git] / arch / i386 / kernel / alternative.c
blob9f4ac8b02de407f756bf69e7756ac5422f5cb0a7
1 #include <linux/module.h>
2 #include <linux/sched.h>
3 #include <linux/spinlock.h>
4 #include <linux/list.h>
5 #include <linux/kprobes.h>
6 #include <linux/mm.h>
7 #include <linux/vmalloc.h>
8 #include <asm/alternative.h>
9 #include <asm/sections.h>
10 #include <asm/pgtable.h>
11 #include <asm/mce.h>
12 #include <asm/nmi.h>
13
14 #define MAX_PATCH_LEN (255-1)
15
16 #ifdef CONFIG_HOTPLUG_CPU
17 static int smp_alt_once;
18
19 static int __init bootonly(char *str)
20 {
21 smp_alt_once = 1;
22 return 1;
23 }
24 __setup("smp-alt-boot", bootonly);
25 #else
26 #define smp_alt_once 1
27 #endif
28
29 static int debug_alternative;
30
31 static int __init debug_alt(char *str)
32 {
33 debug_alternative = 1;
34 return 1;
35 }
36 __setup("debug-alternative", debug_alt);
37
38 static int noreplace_smp;
39
40 static int __init setup_noreplace_smp(char *str)
41 {
42 noreplace_smp = 1;
43 return 1;
44 }
45 __setup("noreplace-smp", setup_noreplace_smp);
46
47 #ifdef CONFIG_PARAVIRT
48 static int noreplace_paravirt = 0;
49
50 static int __init setup_noreplace_paravirt(char *str)
51 {
52 noreplace_paravirt = 1;
53 return 1;
54 }
55 __setup("noreplace-paravirt", setup_noreplace_paravirt);
56 #endif
57
58 #define DPRINTK(fmt, args...) if (debug_alternative) \
59 printk(KERN_DEBUG fmt, args)
60
61 #ifdef GENERIC_NOP1
62 /* Use inline assembly to define this because the nops are defined
63 as inline assembly strings in the include files and we cannot
64 get them easily into strings. */
65 asm("\t.data\nintelnops: "
66 GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6
67 GENERIC_NOP7 GENERIC_NOP8);
68 extern unsigned char intelnops[];
69 static unsigned char *intel_nops[ASM_NOP_MAX+1] = {
70 NULL,
71 intelnops,
72 intelnops + 1,
73 intelnops + 1 + 2,
74 intelnops + 1 + 2 + 3,
75 intelnops + 1 + 2 + 3 + 4,
76 intelnops + 1 + 2 + 3 + 4 + 5,
77 intelnops + 1 + 2 + 3 + 4 + 5 + 6,
78 intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
79 };
80 #endif
81
82 #ifdef K8_NOP1
83 asm("\t.data\nk8nops: "
84 K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
85 K8_NOP7 K8_NOP8);
86 extern unsigned char k8nops[];
87 static unsigned char *k8_nops[ASM_NOP_MAX+1] = {
88 NULL,
89 k8nops,
90 k8nops + 1,
91 k8nops + 1 + 2,
92 k8nops + 1 + 2 + 3,
93 k8nops + 1 + 2 + 3 + 4,
94 k8nops + 1 + 2 + 3 + 4 + 5,
95 k8nops + 1 + 2 + 3 + 4 + 5 + 6,
96 k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
97 };
98 #endif
99
100 #ifdef K7_NOP1
101 asm("\t.data\nk7nops: "
102 K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
103 K7_NOP7 K7_NOP8);
104 extern unsigned char k7nops[];
105 static unsigned char *k7_nops[ASM_NOP_MAX+1] = {
106 NULL,
107 k7nops,
108 k7nops + 1,
109 k7nops + 1 + 2,
110 k7nops + 1 + 2 + 3,
111 k7nops + 1 + 2 + 3 + 4,
112 k7nops + 1 + 2 + 3 + 4 + 5,
113 k7nops + 1 + 2 + 3 + 4 + 5 + 6,
114 k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
115 };
116 #endif
117
118 #ifdef CONFIG_X86_64
119
120 extern char __vsyscall_0;
121 static inline unsigned char** find_nop_table(void)
122 {
123 return k8_nops;
124 }
125
126 #else /* CONFIG_X86_64 */
127
128 static struct nop {
129 int cpuid;
130 unsigned char **noptable;
131 } noptypes[] = {
132 { X86_FEATURE_K8, k8_nops },
133 { X86_FEATURE_K7, k7_nops },
134 { -1, NULL }
135 };
136
137 static unsigned char** find_nop_table(void)
138 {
139 unsigned char **noptable = intel_nops;
140 int i;
141
142 for (i = 0; noptypes[i].cpuid >= 0; i++) {
143 if (boot_cpu_has(noptypes[i].cpuid)) {
144 noptable = noptypes[i].noptable;
145 break;
146 }
147 }
148 return noptable;
149 }
150
151 #endif /* CONFIG_X86_64 */
152
153 /* Use this to add nops to a buffer, then text_poke the whole buffer. */
154 static void add_nops(void *insns, unsigned int len)
155 {
156 unsigned char **noptable = find_nop_table();
157
158 while (len > 0) {
159 unsigned int noplen = len;
160 if (noplen > ASM_NOP_MAX)
161 noplen = ASM_NOP_MAX;
162 memcpy(insns, noptable[noplen], noplen);
163 insns += noplen;
164 len -= noplen;
165 }
166 }
167
168 extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
169 extern u8 *__smp_locks[], *__smp_locks_end[];
170
171 /* Replace instructions with better alternatives for this CPU type.
172 This runs before SMP is initialized to avoid SMP problems with
173 self modifying code. This implies that assymetric systems where
174 APs have less capabilities than the boot processor are not handled.
175 Tough. Make sure you disable such features by hand. */
176
177 void apply_alternatives(struct alt_instr *start, struct alt_instr *end)
178 {
179 struct alt_instr *a;
180 char insnbuf[MAX_PATCH_LEN];
181
182 DPRINTK("%s: alt table %p -> %p\n", __FUNCTION__, start, end);
183 for (a = start; a < end; a++) {
184 u8 *instr = a->instr;
185 BUG_ON(a->replacementlen > a->instrlen);
186 BUG_ON(a->instrlen > sizeof(insnbuf));
187 if (!boot_cpu_has(a->cpuid))
188 continue;
189 #ifdef CONFIG_X86_64
190 /* vsyscall code is not mapped yet. resolve it manually. */
191 if (instr >= (u8 *)VSYSCALL_START && instr < (u8*)VSYSCALL_END) {
192 instr = __va(instr - (u8*)VSYSCALL_START + (u8*)__pa_symbol(&__vsyscall_0));
193 DPRINTK("%s: vsyscall fixup: %p => %p\n",
194 __FUNCTION__, a->instr, instr);
195 }
196 #endif
197 memcpy(insnbuf, a->replacement, a->replacementlen);
198 add_nops(insnbuf + a->replacementlen,
199 a->instrlen - a->replacementlen);
200 text_poke(instr, insnbuf, a->instrlen);
201 }
202 }
203
204 #ifdef CONFIG_SMP
205
206 static void alternatives_smp_lock(u8 **start, u8 **end, u8 *text, u8 *text_end)
207 {
208 u8 **ptr;
209
210 for (ptr = start; ptr < end; ptr++) {
211 if (*ptr < text)
212 continue;
213 if (*ptr > text_end)
214 continue;
215 text_poke(*ptr, ((unsigned char []){0xf0}), 1); /* add lock prefix */
216 };
217 }
218
219 static void alternatives_smp_unlock(u8 **start, u8 **end, u8 *text, u8 *text_end)
220 {
221 u8 **ptr;
222 char insn[1];
223
224 if (noreplace_smp)
225 return;
226
227 add_nops(insn, 1);
228 for (ptr = start; ptr < end; ptr++) {
229 if (*ptr < text)
230 continue;
231 if (*ptr > text_end)
232 continue;
233 text_poke(*ptr, insn, 1);
234 };
235 }
236
237 struct smp_alt_module {
238 /* what is this ??? */
239 struct module *mod;
240 char *name;
241
242 /* ptrs to lock prefixes */
243 u8 **locks;
244 u8 **locks_end;
245
246 /* .text segment, needed to avoid patching init code ;) */
247 u8 *text;
248 u8 *text_end;
249
250 struct list_head next;
251 };
252 static LIST_HEAD(smp_alt_modules);
253 static DEFINE_SPINLOCK(smp_alt);
254
255 void alternatives_smp_module_add(struct module *mod, char *name,
256 void *locks, void *locks_end,
257 void *text, void *text_end)
258 {
259 struct smp_alt_module *smp;
260 unsigned long flags;
261
262 if (noreplace_smp)
263 return;
264
265 if (smp_alt_once) {
266 if (boot_cpu_has(X86_FEATURE_UP))
267 alternatives_smp_unlock(locks, locks_end,
268 text, text_end);
269 return;
270 }
271
272 smp = kzalloc(sizeof(*smp), GFP_KERNEL);
273 if (NULL == smp)
274 return; /* we'll run the (safe but slow) SMP code then ... */
275
276 smp->mod = mod;
277 smp->name = name;
278 smp->locks = locks;
279 smp->locks_end = locks_end;
280 smp->text = text;
281 smp->text_end = text_end;
282 DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n",
283 __FUNCTION__, smp->locks, smp->locks_end,
284 smp->text, smp->text_end, smp->name);
285
286 spin_lock_irqsave(&smp_alt, flags);
287 list_add_tail(&smp->next, &smp_alt_modules);
288 if (boot_cpu_has(X86_FEATURE_UP))
289 alternatives_smp_unlock(smp->locks, smp->locks_end,
290 smp->text, smp->text_end);
291 spin_unlock_irqrestore(&smp_alt, flags);
292 }
293
294 void alternatives_smp_module_del(struct module *mod)
295 {
296 struct smp_alt_module *item;
297 unsigned long flags;
298
299 if (smp_alt_once || noreplace_smp)
300 return;
301
302 spin_lock_irqsave(&smp_alt, flags);
303 list_for_each_entry(item, &smp_alt_modules, next) {
304 if (mod != item->mod)
305 continue;
306 list_del(&item->next);
307 spin_unlock_irqrestore(&smp_alt, flags);
308 DPRINTK("%s: %s\n", __FUNCTION__, item->name);
309 kfree(item);
310 return;
311 }
312 spin_unlock_irqrestore(&smp_alt, flags);
313 }
314
315 void alternatives_smp_switch(int smp)
316 {
317 struct smp_alt_module *mod;
318 unsigned long flags;
319
320 #ifdef CONFIG_LOCKDEP
321 /*
322 * A not yet fixed binutils section handling bug prevents
323 * alternatives-replacement from working reliably, so turn
324 * it off:
325 */
326 printk("lockdep: not fixing up alternatives.\n");
327 return;
328 #endif
329
330 if (noreplace_smp || smp_alt_once)
331 return;
332 BUG_ON(!smp && (num_online_cpus() > 1));
333
334 spin_lock_irqsave(&smp_alt, flags);
335 if (smp) {
336 printk(KERN_INFO "SMP alternatives: switching to SMP code\n");
337 clear_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
338 clear_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
339 list_for_each_entry(mod, &smp_alt_modules, next)
340 alternatives_smp_lock(mod->locks, mod->locks_end,
341 mod->text, mod->text_end);
342 } else {
343 printk(KERN_INFO "SMP alternatives: switching to UP code\n");
344 set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
345 set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
346 list_for_each_entry(mod, &smp_alt_modules, next)
347 alternatives_smp_unlock(mod->locks, mod->locks_end,
348 mod->text, mod->text_end);
349 }
350 spin_unlock_irqrestore(&smp_alt, flags);
351 }
352
353 #endif
354
355 #ifdef CONFIG_PARAVIRT
356 void apply_paravirt(struct paravirt_patch_site *start,
357 struct paravirt_patch_site *end)
358 {
359 struct paravirt_patch_site *p;
360 char insnbuf[MAX_PATCH_LEN];
361
362 if (noreplace_paravirt)
363 return;
364
365 for (p = start; p < end; p++) {
366 unsigned int used;
367
368 BUG_ON(p->len > MAX_PATCH_LEN);
369 /* prep the buffer with the original instructions */
370 memcpy(insnbuf, p->instr, p->len);
371 used = paravirt_ops.patch(p->instrtype, p->clobbers, insnbuf,
372 (unsigned long)p->instr, p->len);
373
374 BUG_ON(used > p->len);
375
376 /* Pad the rest with nops */
377 add_nops(insnbuf + used, p->len - used);
378 text_poke(p->instr, insnbuf, p->len);
379 }
380 }
381 extern struct paravirt_patch_site __start_parainstructions[],
382 __stop_parainstructions[];
383 #endif /* CONFIG_PARAVIRT */
384
385 void __init alternative_instructions(void)
386 {
387 unsigned long flags;
388
389 /* The patching is not fully atomic, so try to avoid local interruptions
390 that might execute the to be patched code.
391 Other CPUs are not running. */
392 stop_nmi();
393 #ifdef CONFIG_X86_MCE
394 stop_mce();
395 #endif
396
397 local_irq_save(flags);
398 apply_alternatives(__alt_instructions, __alt_instructions_end);
399
400 /* switch to patch-once-at-boottime-only mode and free the
401 * tables in case we know the number of CPUs will never ever
402 * change */
403 #ifdef CONFIG_HOTPLUG_CPU
404 if (num_possible_cpus() < 2)
405 smp_alt_once = 1;
406 #endif
407
408 #ifdef CONFIG_SMP
409 if (smp_alt_once) {
410 if (1 == num_possible_cpus()) {
411 printk(KERN_INFO "SMP alternatives: switching to UP code\n");
412 set_bit(X86_FEATURE_UP, boot_cpu_data.x86_capability);
413 set_bit(X86_FEATURE_UP, cpu_data[0].x86_capability);
414 alternatives_smp_unlock(__smp_locks, __smp_locks_end,
415 _text, _etext);
416 }
417 free_init_pages("SMP alternatives",
418 (unsigned long)__smp_locks,
419 (unsigned long)__smp_locks_end);
420 } else {
421 alternatives_smp_module_add(NULL, "core kernel",
422 __smp_locks, __smp_locks_end,
423 _text, _etext);
424 alternatives_smp_switch(0);
425 }
426 #endif
427 apply_paravirt(__parainstructions, __parainstructions_end);
428 local_irq_restore(flags);
429
430 restart_nmi();
431 #ifdef CONFIG_X86_MCE
432 restart_mce();
433 #endif
434 }
435
436 /*
437 * Warning:
438 * When you use this code to patch more than one byte of an instruction
439 * you need to make sure that other CPUs cannot execute this code in parallel.
440 * Also no thread must be currently preempted in the middle of these instructions.
441 * And on the local CPU you need to be protected again NMI or MCE handlers
442 * seeing an inconsistent instruction while you patch.
443 */
444 void __kprobes text_poke(void *addr, unsigned char *opcode, int len)
445 {
446 memcpy(addr, opcode, len);
447 sync_core();
448 /* Not strictly needed, but can speed CPU recovery up. Ignore cross cacheline
449 case. */
450 if (cpu_has_clflush)
451 asm("clflush (%0) " :: "r" (addr) : "memory");
452 }
SocketAccessConTroL development