1 #include <linux/module.h>
2 #include <linux/sched.h>
3 #include <linux/mutex.h>
4 #include <linux/list.h>
5 #include <linux/stringify.h>
6 #include <linux/kprobes.h>
8 #include <linux/vmalloc.h>
9 #include <linux/memory.h>
10 #include <linux/stop_machine.h>
11 #include <asm/alternative.h>
12 #include <asm/sections.h>
13 #include <asm/pgtable.h>
16 #include <asm/vsyscall.h>
17 #include <asm/cacheflush.h>
18 #include <asm/tlbflush.h>
20 #include <asm/fixmap.h>
22 #define MAX_PATCH_LEN (255-1)
24 #ifdef CONFIG_HOTPLUG_CPU
25 static int smp_alt_once
;
27 static int __init
bootonly(char *str
)
32 __setup("smp-alt-boot", bootonly
);
34 #define smp_alt_once 1
37 static int __initdata_or_module debug_alternative
;
39 static int __init
debug_alt(char *str
)
41 debug_alternative
= 1;
44 __setup("debug-alternative", debug_alt
);
46 static int noreplace_smp
;
48 static int __init
setup_noreplace_smp(char *str
)
53 __setup("noreplace-smp", setup_noreplace_smp
);
55 #ifdef CONFIG_PARAVIRT
56 static int __initdata_or_module noreplace_paravirt
= 0;
58 static int __init
setup_noreplace_paravirt(char *str
)
60 noreplace_paravirt
= 1;
63 __setup("noreplace-paravirt", setup_noreplace_paravirt
);
66 #define DPRINTK(fmt, args...) if (debug_alternative) \
67 printk(KERN_DEBUG fmt, args)
69 #if defined(GENERIC_NOP1) && !defined(CONFIG_X86_64)
70 /* Use inline assembly to define this because the nops are defined
71 as inline assembly strings in the include files and we cannot
72 get them easily into strings. */
73 asm("\t" __stringify(__INITRODATA_OR_MODULE
) "\nintelnops: "
74 GENERIC_NOP1 GENERIC_NOP2 GENERIC_NOP3 GENERIC_NOP4 GENERIC_NOP5 GENERIC_NOP6
75 GENERIC_NOP7 GENERIC_NOP8
77 extern const unsigned char intelnops
[];
78 static const unsigned char *const __initconst_or_module
79 intel_nops
[ASM_NOP_MAX
+1] = {
84 intelnops
+ 1 + 2 + 3,
85 intelnops
+ 1 + 2 + 3 + 4,
86 intelnops
+ 1 + 2 + 3 + 4 + 5,
87 intelnops
+ 1 + 2 + 3 + 4 + 5 + 6,
88 intelnops
+ 1 + 2 + 3 + 4 + 5 + 6 + 7,
93 asm("\t" __stringify(__INITRODATA_OR_MODULE
) "\nk8nops: "
94 K8_NOP1 K8_NOP2 K8_NOP3 K8_NOP4 K8_NOP5 K8_NOP6
97 extern const unsigned char k8nops
[];
98 static const unsigned char *const __initconst_or_module
99 k8_nops
[ASM_NOP_MAX
+1] = {
105 k8nops
+ 1 + 2 + 3 + 4,
106 k8nops
+ 1 + 2 + 3 + 4 + 5,
107 k8nops
+ 1 + 2 + 3 + 4 + 5 + 6,
108 k8nops
+ 1 + 2 + 3 + 4 + 5 + 6 + 7,
112 #if defined(K7_NOP1) && !defined(CONFIG_X86_64)
113 asm("\t" __stringify(__INITRODATA_OR_MODULE
) "\nk7nops: "
114 K7_NOP1 K7_NOP2 K7_NOP3 K7_NOP4 K7_NOP5 K7_NOP6
117 extern const unsigned char k7nops
[];
118 static const unsigned char *const __initconst_or_module
119 k7_nops
[ASM_NOP_MAX
+1] = {
125 k7nops
+ 1 + 2 + 3 + 4,
126 k7nops
+ 1 + 2 + 3 + 4 + 5,
127 k7nops
+ 1 + 2 + 3 + 4 + 5 + 6,
128 k7nops
+ 1 + 2 + 3 + 4 + 5 + 6 + 7,
133 asm("\t" __stringify(__INITRODATA_OR_MODULE
) "\np6nops: "
134 P6_NOP1 P6_NOP2 P6_NOP3 P6_NOP4 P6_NOP5 P6_NOP6
137 extern const unsigned char p6nops
[];
138 static const unsigned char *const __initconst_or_module
139 p6_nops
[ASM_NOP_MAX
+1] = {
145 p6nops
+ 1 + 2 + 3 + 4,
146 p6nops
+ 1 + 2 + 3 + 4 + 5,
147 p6nops
+ 1 + 2 + 3 + 4 + 5 + 6,
148 p6nops
+ 1 + 2 + 3 + 4 + 5 + 6 + 7,
154 extern char __vsyscall_0
;
155 static const unsigned char *const *__init_or_module
find_nop_table(void)
157 if (boot_cpu_data
.x86_vendor
== X86_VENDOR_INTEL
&&
158 boot_cpu_has(X86_FEATURE_NOPL
))
164 #else /* CONFIG_X86_64 */
166 static const unsigned char *const *__init_or_module
find_nop_table(void)
168 if (boot_cpu_has(X86_FEATURE_K8
))
170 else if (boot_cpu_has(X86_FEATURE_K7
))
172 else if (boot_cpu_has(X86_FEATURE_NOPL
))
178 #endif /* CONFIG_X86_64 */
180 /* Use this to add nops to a buffer, then text_poke the whole buffer. */
181 static void __init_or_module
add_nops(void *insns
, unsigned int len
)
183 const unsigned char *const *noptable
= find_nop_table();
186 unsigned int noplen
= len
;
187 if (noplen
> ASM_NOP_MAX
)
188 noplen
= ASM_NOP_MAX
;
189 memcpy(insns
, noptable
[noplen
], noplen
);
195 extern struct alt_instr __alt_instructions
[], __alt_instructions_end
[];
196 extern u8
*__smp_locks
[], *__smp_locks_end
[];
197 static void *text_poke_early(void *addr
, const void *opcode
, size_t len
);
199 /* Replace instructions with better alternatives for this CPU type.
200 This runs before SMP is initialized to avoid SMP problems with
201 self modifying code. This implies that assymetric systems where
202 APs have less capabilities than the boot processor are not handled.
203 Tough. Make sure you disable such features by hand. */
205 void __init_or_module
apply_alternatives(struct alt_instr
*start
,
206 struct alt_instr
*end
)
209 u8 insnbuf
[MAX_PATCH_LEN
];
211 DPRINTK("%s: alt table %p -> %p\n", __func__
, start
, end
);
212 for (a
= start
; a
< end
; a
++) {
213 u8
*instr
= a
->instr
;
214 BUG_ON(a
->replacementlen
> a
->instrlen
);
215 BUG_ON(a
->instrlen
> sizeof(insnbuf
));
216 if (!boot_cpu_has(a
->cpuid
))
219 /* vsyscall code is not mapped yet. resolve it manually. */
220 if (instr
>= (u8
*)VSYSCALL_START
&& instr
< (u8
*)VSYSCALL_END
) {
221 instr
= __va(instr
- (u8
*)VSYSCALL_START
+ (u8
*)__pa_symbol(&__vsyscall_0
));
222 DPRINTK("%s: vsyscall fixup: %p => %p\n",
223 __func__
, a
->instr
, instr
);
226 memcpy(insnbuf
, a
->replacement
, a
->replacementlen
);
227 if (*insnbuf
== 0xe8 && a
->replacementlen
== 5)
228 *(s32
*)(insnbuf
+ 1) += a
->replacement
- a
->instr
;
229 add_nops(insnbuf
+ a
->replacementlen
,
230 a
->instrlen
- a
->replacementlen
);
231 text_poke_early(instr
, insnbuf
, a
->instrlen
);
237 static void alternatives_smp_lock(u8
**start
, u8
**end
, u8
*text
, u8
*text_end
)
241 mutex_lock(&text_mutex
);
242 for (ptr
= start
; ptr
< end
; ptr
++) {
247 /* turn DS segment override prefix into lock prefix */
248 text_poke(*ptr
, ((unsigned char []){0xf0}), 1);
250 mutex_unlock(&text_mutex
);
253 static void alternatives_smp_unlock(u8
**start
, u8
**end
, u8
*text
, u8
*text_end
)
260 mutex_lock(&text_mutex
);
261 for (ptr
= start
; ptr
< end
; ptr
++) {
266 /* turn lock prefix into DS segment override prefix */
267 text_poke(*ptr
, ((unsigned char []){0x3E}), 1);
269 mutex_unlock(&text_mutex
);
272 struct smp_alt_module
{
273 /* what is this ??? */
277 /* ptrs to lock prefixes */
281 /* .text segment, needed to avoid patching init code ;) */
285 struct list_head next
;
287 static LIST_HEAD(smp_alt_modules
);
288 static DEFINE_MUTEX(smp_alt
);
289 static int smp_mode
= 1; /* protected by smp_alt */
291 void __init_or_module
alternatives_smp_module_add(struct module
*mod
,
293 void *locks
, void *locks_end
,
294 void *text
, void *text_end
)
296 struct smp_alt_module
*smp
;
302 if (boot_cpu_has(X86_FEATURE_UP
))
303 alternatives_smp_unlock(locks
, locks_end
,
308 smp
= kzalloc(sizeof(*smp
), GFP_KERNEL
);
310 return; /* we'll run the (safe but slow) SMP code then ... */
315 smp
->locks_end
= locks_end
;
317 smp
->text_end
= text_end
;
318 DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n",
319 __func__
, smp
->locks
, smp
->locks_end
,
320 smp
->text
, smp
->text_end
, smp
->name
);
322 mutex_lock(&smp_alt
);
323 list_add_tail(&smp
->next
, &smp_alt_modules
);
324 if (boot_cpu_has(X86_FEATURE_UP
))
325 alternatives_smp_unlock(smp
->locks
, smp
->locks_end
,
326 smp
->text
, smp
->text_end
);
327 mutex_unlock(&smp_alt
);
330 void __init_or_module
alternatives_smp_module_del(struct module
*mod
)
332 struct smp_alt_module
*item
;
334 if (smp_alt_once
|| noreplace_smp
)
337 mutex_lock(&smp_alt
);
338 list_for_each_entry(item
, &smp_alt_modules
, next
) {
339 if (mod
!= item
->mod
)
341 list_del(&item
->next
);
342 mutex_unlock(&smp_alt
);
343 DPRINTK("%s: %s\n", __func__
, item
->name
);
347 mutex_unlock(&smp_alt
);
350 void alternatives_smp_switch(int smp
)
352 struct smp_alt_module
*mod
;
354 #ifdef CONFIG_LOCKDEP
356 * Older binutils section handling bug prevented
357 * alternatives-replacement from working reliably.
359 * If this still occurs then you should see a hang
360 * or crash shortly after this line:
362 printk("lockdep: fixing up alternatives.\n");
365 if (noreplace_smp
|| smp_alt_once
)
367 BUG_ON(!smp
&& (num_online_cpus() > 1));
369 mutex_lock(&smp_alt
);
372 * Avoid unnecessary switches because it forces JIT based VMs to
373 * throw away all cached translations, which can be quite costly.
375 if (smp
== smp_mode
) {
378 printk(KERN_INFO
"SMP alternatives: switching to SMP code\n");
379 clear_cpu_cap(&boot_cpu_data
, X86_FEATURE_UP
);
380 clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP
);
381 list_for_each_entry(mod
, &smp_alt_modules
, next
)
382 alternatives_smp_lock(mod
->locks
, mod
->locks_end
,
383 mod
->text
, mod
->text_end
);
385 printk(KERN_INFO
"SMP alternatives: switching to UP code\n");
386 set_cpu_cap(&boot_cpu_data
, X86_FEATURE_UP
);
387 set_cpu_cap(&cpu_data(0), X86_FEATURE_UP
);
388 list_for_each_entry(mod
, &smp_alt_modules
, next
)
389 alternatives_smp_unlock(mod
->locks
, mod
->locks_end
,
390 mod
->text
, mod
->text_end
);
393 mutex_unlock(&smp_alt
);
396 /* Return 1 if the address range is reserved for smp-alternatives */
397 int alternatives_text_reserved(void *start
, void *end
)
399 struct smp_alt_module
*mod
;
401 u8
*text_start
= start
;
404 list_for_each_entry(mod
, &smp_alt_modules
, next
) {
405 if (mod
->text
> text_end
|| mod
->text_end
< text_start
)
407 for (ptr
= mod
->locks
; ptr
< mod
->locks_end
; ptr
++)
408 if (text_start
<= *ptr
&& text_end
>= *ptr
)
416 #ifdef CONFIG_PARAVIRT
417 void __init_or_module
apply_paravirt(struct paravirt_patch_site
*start
,
418 struct paravirt_patch_site
*end
)
420 struct paravirt_patch_site
*p
;
421 char insnbuf
[MAX_PATCH_LEN
];
423 if (noreplace_paravirt
)
426 for (p
= start
; p
< end
; p
++) {
429 BUG_ON(p
->len
> MAX_PATCH_LEN
);
430 /* prep the buffer with the original instructions */
431 memcpy(insnbuf
, p
->instr
, p
->len
);
432 used
= pv_init_ops
.patch(p
->instrtype
, p
->clobbers
, insnbuf
,
433 (unsigned long)p
->instr
, p
->len
);
435 BUG_ON(used
> p
->len
);
437 /* Pad the rest with nops */
438 add_nops(insnbuf
+ used
, p
->len
- used
);
439 text_poke_early(p
->instr
, insnbuf
, p
->len
);
442 extern struct paravirt_patch_site __start_parainstructions
[],
443 __stop_parainstructions
[];
444 #endif /* CONFIG_PARAVIRT */
446 void __init
alternative_instructions(void)
448 /* The patching is not fully atomic, so try to avoid local interruptions
449 that might execute the to be patched code.
450 Other CPUs are not running. */
454 * Don't stop machine check exceptions while patching.
455 * MCEs only happen when something got corrupted and in this
456 * case we must do something about the corruption.
457 * Ignoring it is worse than a unlikely patching race.
458 * Also machine checks tend to be broadcast and if one CPU
459 * goes into machine check the others follow quickly, so we don't
460 * expect a machine check to cause undue problems during to code
464 apply_alternatives(__alt_instructions
, __alt_instructions_end
);
466 /* switch to patch-once-at-boottime-only mode and free the
467 * tables in case we know the number of CPUs will never ever
469 #ifdef CONFIG_HOTPLUG_CPU
470 if (num_possible_cpus() < 2)
476 if (1 == num_possible_cpus()) {
477 printk(KERN_INFO
"SMP alternatives: switching to UP code\n");
478 set_cpu_cap(&boot_cpu_data
, X86_FEATURE_UP
);
479 set_cpu_cap(&cpu_data(0), X86_FEATURE_UP
);
481 alternatives_smp_unlock(__smp_locks
, __smp_locks_end
,
485 alternatives_smp_module_add(NULL
, "core kernel",
486 __smp_locks
, __smp_locks_end
,
489 /* Only switch to UP mode if we don't immediately boot others */
490 if (num_present_cpus() == 1 || setup_max_cpus
<= 1)
491 alternatives_smp_switch(0);
494 apply_paravirt(__parainstructions
, __parainstructions_end
);
497 free_init_pages("SMP alternatives",
498 (unsigned long)__smp_locks
,
499 (unsigned long)__smp_locks_end
);
505 * text_poke_early - Update instructions on a live kernel at boot time
506 * @addr: address to modify
507 * @opcode: source of the copy
508 * @len: length to copy
510 * When you use this code to patch more than one byte of an instruction
511 * you need to make sure that other CPUs cannot execute this code in parallel.
512 * Also no thread must be currently preempted in the middle of these
513 * instructions. And on the local CPU you need to be protected again NMI or MCE
514 * handlers seeing an inconsistent instruction while you patch.
516 static void *__init_or_module
text_poke_early(void *addr
, const void *opcode
,
520 local_irq_save(flags
);
521 memcpy(addr
, opcode
, len
);
523 local_irq_restore(flags
);
524 /* Could also do a CLFLUSH here to speed up CPU recovery; but
525 that causes hangs on some VIA CPUs. */
530 * text_poke - Update instructions on a live kernel
531 * @addr: address to modify
532 * @opcode: source of the copy
533 * @len: length to copy
535 * Only atomic text poke/set should be allowed when not doing early patching.
536 * It means the size must be writable atomically and the address must be aligned
537 * in a way that permits an atomic write. It also makes sure we fit on a single
540 * Note: Must be called under text_mutex.
542 void *__kprobes
text_poke(void *addr
, const void *opcode
, size_t len
)
546 struct page
*pages
[2];
549 if (!core_kernel_text((unsigned long)addr
)) {
550 pages
[0] = vmalloc_to_page(addr
);
551 pages
[1] = vmalloc_to_page(addr
+ PAGE_SIZE
);
553 pages
[0] = virt_to_page(addr
);
554 WARN_ON(!PageReserved(pages
[0]));
555 pages
[1] = virt_to_page(addr
+ PAGE_SIZE
);
558 local_irq_save(flags
);
559 set_fixmap(FIX_TEXT_POKE0
, page_to_phys(pages
[0]));
561 set_fixmap(FIX_TEXT_POKE1
, page_to_phys(pages
[1]));
562 vaddr
= (char *)fix_to_virt(FIX_TEXT_POKE0
);
563 memcpy(&vaddr
[(unsigned long)addr
& ~PAGE_MASK
], opcode
, len
);
564 clear_fixmap(FIX_TEXT_POKE0
);
566 clear_fixmap(FIX_TEXT_POKE1
);
569 /* Could also do a CLFLUSH here to speed up CPU recovery; but
570 that causes hangs on some VIA CPUs. */
571 for (i
= 0; i
< len
; i
++)
572 BUG_ON(((char *)addr
)[i
] != ((char *)opcode
)[i
]);
573 local_irq_restore(flags
);
578 * Cross-modifying kernel text with stop_machine().
579 * This code originally comes from immediate value.
581 static atomic_t stop_machine_first
;
582 static int wrote_text
;
584 struct text_poke_params
{
590 static int __kprobes
stop_machine_text_poke(void *data
)
592 struct text_poke_params
*tpp
= data
;
594 if (atomic_dec_and_test(&stop_machine_first
)) {
595 text_poke(tpp
->addr
, tpp
->opcode
, tpp
->len
);
596 smp_wmb(); /* Make sure other cpus see that this has run */
601 smp_mb(); /* Load wrote_text before following execution */
604 flush_icache_range((unsigned long)tpp
->addr
,
605 (unsigned long)tpp
->addr
+ tpp
->len
);
610 * text_poke_smp - Update instructions on a live kernel on SMP
611 * @addr: address to modify
612 * @opcode: source of the copy
613 * @len: length to copy
615 * Modify multi-byte instruction by using stop_machine() on SMP. This allows
616 * user to poke/set multi-byte text on SMP. Only non-NMI/MCE code modifying
617 * should be allowed, since stop_machine() does _not_ protect code against
620 * Note: Must be called under get_online_cpus() and text_mutex.
622 void *__kprobes
text_poke_smp(void *addr
, const void *opcode
, size_t len
)
624 struct text_poke_params tpp
;
629 atomic_set(&stop_machine_first
, 1);
631 stop_machine(stop_machine_text_poke
, (void *)&tpp
, NULL
);