2 * Code for replacing ftrace calls with jumps.
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
6 * Thanks goes to Ingo Molnar, for suggesting the idea.
7 * Mathieu Desnoyers, for suggesting postponing the modifications.
8 * Arjan van de Ven, for keeping me straight, and explaining to me
9 * the dangers of modifying code on the run.
12 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
14 #include <linux/spinlock.h>
15 #include <linux/hardirq.h>
16 #include <linux/uaccess.h>
17 #include <linux/ftrace.h>
18 #include <linux/percpu.h>
19 #include <linux/sched.h>
20 #include <linux/init.h>
21 #include <linux/list.h>
22 #include <linux/module.h>
24 #include <trace/syscall.h>
26 #include <asm/cacheflush.h>
27 #include <asm/kprobes.h>
28 #include <asm/ftrace.h>
31 #ifdef CONFIG_DYNAMIC_FTRACE
33 int ftrace_arch_code_modify_prepare(void)
36 set_all_modules_text_rw();
40 int ftrace_arch_code_modify_post_process(void)
42 set_all_modules_text_ro();
47 union ftrace_code_union
{
48 char code
[MCOUNT_INSN_SIZE
];
52 } __attribute__((packed
));
55 static int ftrace_calc_offset(long ip
, long addr
)
57 return (int)(addr
- ip
);
60 static unsigned char *ftrace_call_replace(unsigned long ip
, unsigned long addr
)
62 static union ftrace_code_union calc
;
65 calc
.offset
= ftrace_calc_offset(ip
+ MCOUNT_INSN_SIZE
, addr
);
68 * No locking needed, this must be called via kstop_machine
69 * which in essence is like running on a uniprocessor machine.
75 within(unsigned long addr
, unsigned long start
, unsigned long end
)
77 return addr
>= start
&& addr
< end
;
81 do_ftrace_mod_code(unsigned long ip
, const void *new_code
)
84 * On x86_64, kernel text mappings are mapped read-only with
85 * CONFIG_DEBUG_RODATA. So we use the kernel identity mapping instead
86 * of the kernel text mapping to modify the kernel text.
88 * For 32bit kernels, these mappings are same and we can use
89 * kernel identity mapping to modify code.
91 if (within(ip
, (unsigned long)_text
, (unsigned long)_etext
))
92 ip
= (unsigned long)__va(__pa_symbol(ip
));
94 return probe_kernel_write((void *)ip
, new_code
, MCOUNT_INSN_SIZE
);
97 static const unsigned char *ftrace_nop_replace(void)
99 return ideal_nops
[NOP_ATOMIC5
];
103 ftrace_modify_code_direct(unsigned long ip
, unsigned const char *old_code
,
104 unsigned const char *new_code
)
106 unsigned char replaced
[MCOUNT_INSN_SIZE
];
109 * Note: Due to modules and __init, code can
110 * disappear and change, we need to protect against faulting
111 * as well as code changing. We do this by using the
112 * probe_kernel_* functions.
114 * No real locking needed, this code is run through
115 * kstop_machine, or before SMP starts.
118 /* read the text we want to modify */
119 if (probe_kernel_read(replaced
, (void *)ip
, MCOUNT_INSN_SIZE
))
122 /* Make sure it is what we expect it to be */
123 if (memcmp(replaced
, old_code
, MCOUNT_INSN_SIZE
) != 0)
126 /* replace the text with the new text */
127 if (do_ftrace_mod_code(ip
, new_code
))
135 int ftrace_make_nop(struct module
*mod
,
136 struct dyn_ftrace
*rec
, unsigned long addr
)
138 unsigned const char *new, *old
;
139 unsigned long ip
= rec
->ip
;
141 old
= ftrace_call_replace(ip
, addr
);
142 new = ftrace_nop_replace();
145 * On boot up, and when modules are loaded, the MCOUNT_ADDR
146 * is converted to a nop, and will never become MCOUNT_ADDR
147 * again. This code is either running before SMP (on boot up)
148 * or before the code will ever be executed (module load).
149 * We do not want to use the breakpoint version in this case,
150 * just modify the code directly.
152 if (addr
== MCOUNT_ADDR
)
153 return ftrace_modify_code_direct(rec
->ip
, old
, new);
155 /* Normal cases use add_brk_on_nop */
156 WARN_ONCE(1, "invalid use of ftrace_make_nop");
160 int ftrace_make_call(struct dyn_ftrace
*rec
, unsigned long addr
)
162 unsigned const char *new, *old
;
163 unsigned long ip
= rec
->ip
;
165 old
= ftrace_nop_replace();
166 new = ftrace_call_replace(ip
, addr
);
168 /* Should only be called when module is loaded */
169 return ftrace_modify_code_direct(rec
->ip
, old
, new);
173 * The modifying_ftrace_code is used to tell the breakpoint
174 * handler to call ftrace_int3_handler(). If it fails to
175 * call this handler for a breakpoint added by ftrace, then
176 * the kernel may crash.
178 * As atomic_writes on x86 do not need a barrier, we do not
179 * need to add smp_mb()s for this to work. It is also considered
180 * that we can not read the modifying_ftrace_code before
181 * executing the breakpoint. That would be quite remarkable if
182 * it could do that. Here's the flow that is required:
188 * <trap-int3> // implicit (r)mb
189 * if (atomic_read(mfc))
190 * call ftrace_int3_handler()
192 * Then when we are finished:
196 * If we hit a breakpoint that was not set by ftrace, it does not
197 * matter if ftrace_int3_handler() is called or not. It will
198 * simply be ignored. But it is crucial that a ftrace nop/caller
199 * breakpoint is handled. No other user should ever place a
200 * breakpoint on an ftrace nop/caller location. It must only
201 * be done by this code.
203 atomic_t modifying_ftrace_code __read_mostly
;
206 ftrace_modify_code(unsigned long ip
, unsigned const char *old_code
,
207 unsigned const char *new_code
);
210 * Should never be called:
211 * As it is only called by __ftrace_replace_code() which is called by
212 * ftrace_replace_code() that x86 overrides, and by ftrace_update_code()
213 * which is called to turn mcount into nops or nops into function calls
214 * but not to convert a function from not using regs to one that uses
215 * regs, which ftrace_modify_call() is for.
217 int ftrace_modify_call(struct dyn_ftrace
*rec
, unsigned long old_addr
,
224 int ftrace_update_ftrace_func(ftrace_func_t func
)
226 unsigned long ip
= (unsigned long)(&ftrace_call
);
227 unsigned char old
[MCOUNT_INSN_SIZE
], *new;
230 memcpy(old
, &ftrace_call
, MCOUNT_INSN_SIZE
);
231 new = ftrace_call_replace(ip
, (unsigned long)func
);
233 /* See comment above by declaration of modifying_ftrace_code */
234 atomic_inc(&modifying_ftrace_code
);
236 ret
= ftrace_modify_code(ip
, old
, new);
238 /* Also update the regs callback function */
240 ip
= (unsigned long)(&ftrace_regs_call
);
241 memcpy(old
, &ftrace_regs_call
, MCOUNT_INSN_SIZE
);
242 new = ftrace_call_replace(ip
, (unsigned long)func
);
243 ret
= ftrace_modify_code(ip
, old
, new);
246 atomic_dec(&modifying_ftrace_code
);
252 * A breakpoint was added to the code address we are about to
253 * modify, and this is the handle that will just skip over it.
254 * We are either changing a nop into a trace call, or a trace
255 * call to a nop. While the change is taking place, we treat
256 * it just like it was a nop.
258 int ftrace_int3_handler(struct pt_regs
*regs
)
260 if (WARN_ON_ONCE(!regs
))
263 if (!ftrace_location(regs
->ip
- 1))
266 regs
->ip
+= MCOUNT_INSN_SIZE
- 1;
271 static int ftrace_write(unsigned long ip
, const char *val
, int size
)
274 * On x86_64, kernel text mappings are mapped read-only with
275 * CONFIG_DEBUG_RODATA. So we use the kernel identity mapping instead
276 * of the kernel text mapping to modify the kernel text.
278 * For 32bit kernels, these mappings are same and we can use
279 * kernel identity mapping to modify code.
281 if (within(ip
, (unsigned long)_text
, (unsigned long)_etext
))
282 ip
= (unsigned long)__va(__pa_symbol(ip
));
284 return probe_kernel_write((void *)ip
, val
, size
);
287 static int add_break(unsigned long ip
, const char *old
)
289 unsigned char replaced
[MCOUNT_INSN_SIZE
];
290 unsigned char brk
= BREAKPOINT_INSTRUCTION
;
292 if (probe_kernel_read(replaced
, (void *)ip
, MCOUNT_INSN_SIZE
))
295 /* Make sure it is what we expect it to be */
296 if (memcmp(replaced
, old
, MCOUNT_INSN_SIZE
) != 0)
299 if (ftrace_write(ip
, &brk
, 1))
305 static int add_brk_on_call(struct dyn_ftrace
*rec
, unsigned long addr
)
307 unsigned const char *old
;
308 unsigned long ip
= rec
->ip
;
310 old
= ftrace_call_replace(ip
, addr
);
312 return add_break(rec
->ip
, old
);
316 static int add_brk_on_nop(struct dyn_ftrace
*rec
)
318 unsigned const char *old
;
320 old
= ftrace_nop_replace();
322 return add_break(rec
->ip
, old
);
326 * If the record has the FTRACE_FL_REGS set, that means that it
327 * wants to convert to a callback that saves all regs. If FTRACE_FL_REGS
328 * is not not set, then it wants to convert to the normal callback.
330 static unsigned long get_ftrace_addr(struct dyn_ftrace
*rec
)
332 if (rec
->flags
& FTRACE_FL_REGS
)
333 return (unsigned long)FTRACE_REGS_ADDR
;
335 return (unsigned long)FTRACE_ADDR
;
339 * The FTRACE_FL_REGS_EN is set when the record already points to
340 * a function that saves all the regs. Basically the '_EN' version
341 * represents the current state of the function.
343 static unsigned long get_ftrace_old_addr(struct dyn_ftrace
*rec
)
345 if (rec
->flags
& FTRACE_FL_REGS_EN
)
346 return (unsigned long)FTRACE_REGS_ADDR
;
348 return (unsigned long)FTRACE_ADDR
;
351 static int add_breakpoints(struct dyn_ftrace
*rec
, int enable
)
353 unsigned long ftrace_addr
;
356 ret
= ftrace_test_record(rec
, enable
);
358 ftrace_addr
= get_ftrace_addr(rec
);
361 case FTRACE_UPDATE_IGNORE
:
364 case FTRACE_UPDATE_MAKE_CALL
:
365 /* converting nop to call */
366 return add_brk_on_nop(rec
);
368 case FTRACE_UPDATE_MODIFY_CALL_REGS
:
369 case FTRACE_UPDATE_MODIFY_CALL
:
370 ftrace_addr
= get_ftrace_old_addr(rec
);
372 case FTRACE_UPDATE_MAKE_NOP
:
373 /* converting a call to a nop */
374 return add_brk_on_call(rec
, ftrace_addr
);
380 * On error, we need to remove breakpoints. This needs to
381 * be done caefully. If the address does not currently have a
382 * breakpoint, we know we are done. Otherwise, we look at the
383 * remaining 4 bytes of the instruction. If it matches a nop
384 * we replace the breakpoint with the nop. Otherwise we replace
385 * it with the call instruction.
387 static int remove_breakpoint(struct dyn_ftrace
*rec
)
389 unsigned char ins
[MCOUNT_INSN_SIZE
];
390 unsigned char brk
= BREAKPOINT_INSTRUCTION
;
391 const unsigned char *nop
;
392 unsigned long ftrace_addr
;
393 unsigned long ip
= rec
->ip
;
395 /* If we fail the read, just give up */
396 if (probe_kernel_read(ins
, (void *)ip
, MCOUNT_INSN_SIZE
))
399 /* If this does not have a breakpoint, we are done */
403 nop
= ftrace_nop_replace();
406 * If the last 4 bytes of the instruction do not match
407 * a nop, then we assume that this is a call to ftrace_addr.
409 if (memcmp(&ins
[1], &nop
[1], MCOUNT_INSN_SIZE
- 1) != 0) {
411 * For extra paranoidism, we check if the breakpoint is on
412 * a call that would actually jump to the ftrace_addr.
413 * If not, don't touch the breakpoint, we make just create
416 ftrace_addr
= get_ftrace_addr(rec
);
417 nop
= ftrace_call_replace(ip
, ftrace_addr
);
419 if (memcmp(&ins
[1], &nop
[1], MCOUNT_INSN_SIZE
- 1) == 0)
422 /* Check both ftrace_addr and ftrace_old_addr */
423 ftrace_addr
= get_ftrace_old_addr(rec
);
424 nop
= ftrace_call_replace(ip
, ftrace_addr
);
426 if (memcmp(&ins
[1], &nop
[1], MCOUNT_INSN_SIZE
- 1) != 0)
431 return probe_kernel_write((void *)ip
, &nop
[0], 1);
434 static int add_update_code(unsigned long ip
, unsigned const char *new)
436 /* skip breakpoint */
439 if (ftrace_write(ip
, new, MCOUNT_INSN_SIZE
- 1))
444 static int add_update_call(struct dyn_ftrace
*rec
, unsigned long addr
)
446 unsigned long ip
= rec
->ip
;
447 unsigned const char *new;
449 new = ftrace_call_replace(ip
, addr
);
450 return add_update_code(ip
, new);
453 static int add_update_nop(struct dyn_ftrace
*rec
)
455 unsigned long ip
= rec
->ip
;
456 unsigned const char *new;
458 new = ftrace_nop_replace();
459 return add_update_code(ip
, new);
462 static int add_update(struct dyn_ftrace
*rec
, int enable
)
464 unsigned long ftrace_addr
;
467 ret
= ftrace_test_record(rec
, enable
);
469 ftrace_addr
= get_ftrace_addr(rec
);
472 case FTRACE_UPDATE_IGNORE
:
475 case FTRACE_UPDATE_MODIFY_CALL_REGS
:
476 case FTRACE_UPDATE_MODIFY_CALL
:
477 case FTRACE_UPDATE_MAKE_CALL
:
478 /* converting nop to call */
479 return add_update_call(rec
, ftrace_addr
);
481 case FTRACE_UPDATE_MAKE_NOP
:
482 /* converting a call to a nop */
483 return add_update_nop(rec
);
489 static int finish_update_call(struct dyn_ftrace
*rec
, unsigned long addr
)
491 unsigned long ip
= rec
->ip
;
492 unsigned const char *new;
494 new = ftrace_call_replace(ip
, addr
);
496 if (ftrace_write(ip
, new, 1))
502 static int finish_update_nop(struct dyn_ftrace
*rec
)
504 unsigned long ip
= rec
->ip
;
505 unsigned const char *new;
507 new = ftrace_nop_replace();
509 if (ftrace_write(ip
, new, 1))
514 static int finish_update(struct dyn_ftrace
*rec
, int enable
)
516 unsigned long ftrace_addr
;
519 ret
= ftrace_update_record(rec
, enable
);
521 ftrace_addr
= get_ftrace_addr(rec
);
524 case FTRACE_UPDATE_IGNORE
:
527 case FTRACE_UPDATE_MODIFY_CALL_REGS
:
528 case FTRACE_UPDATE_MODIFY_CALL
:
529 case FTRACE_UPDATE_MAKE_CALL
:
530 /* converting nop to call */
531 return finish_update_call(rec
, ftrace_addr
);
533 case FTRACE_UPDATE_MAKE_NOP
:
534 /* converting a call to a nop */
535 return finish_update_nop(rec
);
541 static void do_sync_core(void *data
)
546 static void run_sync(void)
548 int enable_irqs
= irqs_disabled();
550 /* We may be called with interrupts disbled (on bootup). */
553 on_each_cpu(do_sync_core
, NULL
, 1);
558 void ftrace_replace_code(int enable
)
560 struct ftrace_rec_iter
*iter
;
561 struct dyn_ftrace
*rec
;
562 const char *report
= "adding breakpoints";
566 for_ftrace_rec_iter(iter
) {
567 rec
= ftrace_rec_iter_record(iter
);
569 ret
= add_breakpoints(rec
, enable
);
571 goto remove_breakpoints
;
577 report
= "updating code";
579 for_ftrace_rec_iter(iter
) {
580 rec
= ftrace_rec_iter_record(iter
);
582 ret
= add_update(rec
, enable
);
584 goto remove_breakpoints
;
589 report
= "removing breakpoints";
591 for_ftrace_rec_iter(iter
) {
592 rec
= ftrace_rec_iter_record(iter
);
594 ret
= finish_update(rec
, enable
);
596 goto remove_breakpoints
;
604 ftrace_bug(ret
, rec
? rec
->ip
: 0);
605 printk(KERN_WARNING
"Failed on %s (%d):\n", report
, count
);
606 for_ftrace_rec_iter(iter
) {
607 rec
= ftrace_rec_iter_record(iter
);
608 remove_breakpoint(rec
);
613 ftrace_modify_code(unsigned long ip
, unsigned const char *old_code
,
614 unsigned const char *new_code
)
618 ret
= add_break(ip
, old_code
);
624 ret
= add_update_code(ip
, new_code
);
630 ret
= ftrace_write(ip
, new_code
, 1);
640 probe_kernel_write((void *)ip
, &old_code
[0], 1);
644 void arch_ftrace_update_code(int command
)
646 /* See comment above by declaration of modifying_ftrace_code */
647 atomic_inc(&modifying_ftrace_code
);
649 ftrace_modify_all_code(command
);
651 atomic_dec(&modifying_ftrace_code
);
654 int __init
ftrace_dyn_arch_init(void *data
)
656 /* The return code is retured via data */
657 *(unsigned long *)data
= 0;
663 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
665 #ifdef CONFIG_DYNAMIC_FTRACE
666 extern void ftrace_graph_call(void);
668 static int ftrace_mod_jmp(unsigned long ip
,
669 int old_offset
, int new_offset
)
671 unsigned char code
[MCOUNT_INSN_SIZE
];
673 if (probe_kernel_read(code
, (void *)ip
, MCOUNT_INSN_SIZE
))
676 if (code
[0] != 0xe9 || old_offset
!= *(int *)(&code
[1]))
679 *(int *)(&code
[1]) = new_offset
;
681 if (do_ftrace_mod_code(ip
, &code
))
687 int ftrace_enable_ftrace_graph_caller(void)
689 unsigned long ip
= (unsigned long)(&ftrace_graph_call
);
690 int old_offset
, new_offset
;
692 old_offset
= (unsigned long)(&ftrace_stub
) - (ip
+ MCOUNT_INSN_SIZE
);
693 new_offset
= (unsigned long)(&ftrace_graph_caller
) - (ip
+ MCOUNT_INSN_SIZE
);
695 return ftrace_mod_jmp(ip
, old_offset
, new_offset
);
698 int ftrace_disable_ftrace_graph_caller(void)
700 unsigned long ip
= (unsigned long)(&ftrace_graph_call
);
701 int old_offset
, new_offset
;
703 old_offset
= (unsigned long)(&ftrace_graph_caller
) - (ip
+ MCOUNT_INSN_SIZE
);
704 new_offset
= (unsigned long)(&ftrace_stub
) - (ip
+ MCOUNT_INSN_SIZE
);
706 return ftrace_mod_jmp(ip
, old_offset
, new_offset
);
709 #endif /* !CONFIG_DYNAMIC_FTRACE */
712 * Hook the return address and push it in the stack of return addrs
713 * in current thread info.
715 void prepare_ftrace_return(unsigned long *parent
, unsigned long self_addr
,
716 unsigned long frame_pointer
)
720 struct ftrace_graph_ent trace
;
721 unsigned long return_hooker
= (unsigned long)
724 if (unlikely(atomic_read(¤t
->tracing_graph_pause
)))
728 * Protect against fault, even if it shouldn't
729 * happen. This tool is too much intrusive to
730 * ignore such a protection.
733 "1: " _ASM_MOV
" (%[parent]), %[old]\n"
734 "2: " _ASM_MOV
" %[return_hooker], (%[parent])\n"
735 " movl $0, %[faulted]\n"
738 ".section .fixup, \"ax\"\n"
739 "4: movl $1, %[faulted]\n"
746 : [old
] "=&r" (old
), [faulted
] "=r" (faulted
)
747 : [parent
] "r" (parent
), [return_hooker
] "r" (return_hooker
)
751 if (unlikely(faulted
)) {
757 trace
.func
= self_addr
;
758 trace
.depth
= current
->curr_ret_stack
+ 1;
760 /* Only trace if the calling function expects to */
761 if (!ftrace_graph_entry(&trace
)) {
766 if (ftrace_push_return_trace(old
, self_addr
, &trace
.depth
,
767 frame_pointer
) == -EBUSY
) {
772 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */