2 * Code for replacing ftrace calls with jumps.
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2009, 2010 DSLab, Lanzhou University, China
6 * Author: Wu Zhangjin <wuzhangjin@gmail.com>
8 * Thanks goes to Steven Rostedt for writing the original x86 version.
11 #include <linux/uaccess.h>
12 #include <linux/init.h>
13 #include <linux/ftrace.h>
16 #include <asm/asm-offsets.h>
17 #include <asm/cacheflush.h>
21 static inline int in_module(unsigned long ip
)
23 return ip
& 0x40000000;
26 #ifdef CONFIG_DYNAMIC_FTRACE
28 #define JAL 0x0c000000 /* jump & link: ip --> ra, jump to target */
29 #define ADDR_MASK 0x03ffffff /* op_code|addr : 31...26|25 ....0 */
30 #define JUMP_RANGE_MASK ((1UL << 28) - 1)
32 #define INSN_B_1F_4 0x10000004 /* b 1f; offset = 4 */
33 #define INSN_B_1F_5 0x10000005 /* b 1f; offset = 5 */
34 #define INSN_NOP 0x00000000 /* nop */
35 #define INSN_JAL(addr) \
36 ((unsigned int)(JAL | (((addr) >> 2) & ADDR_MASK)))
38 static unsigned int insn_jal_ftrace_caller __read_mostly
;
39 static unsigned int insn_lui_v1_hi16_mcount __read_mostly
;
40 static unsigned int insn_j_ftrace_graph_caller __maybe_unused __read_mostly
;
42 static inline void ftrace_dyn_arch_init_insns(void)
47 /* lui v1, hi16_mcount */
49 buf
= (u32
*)&insn_lui_v1_hi16_mcount
;
50 UASM_i_LA_mostly(&buf
, v1
, MCOUNT_ADDR
);
52 /* jal (ftrace_caller + 8), jump over the first two instruction */
53 buf
= (u32
*)&insn_jal_ftrace_caller
;
54 uasm_i_jal(&buf
, (FTRACE_ADDR
+ 8) & JUMP_RANGE_MASK
);
56 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
57 /* j ftrace_graph_caller */
58 buf
= (u32
*)&insn_j_ftrace_graph_caller
;
59 uasm_i_j(&buf
, (unsigned long)ftrace_graph_caller
& JUMP_RANGE_MASK
);
63 static int ftrace_modify_code(unsigned long ip
, unsigned int new_code
)
67 /* *(unsigned int *)ip = new_code; */
68 safe_store_code(new_code
, ip
, faulted
);
70 if (unlikely(faulted
))
73 flush_icache_range(ip
, ip
+ 8);
78 int ftrace_make_nop(struct module
*mod
,
79 struct dyn_ftrace
*rec
, unsigned long addr
)
82 unsigned long ip
= rec
->ip
;
85 * We have compiled module with -mlong-calls, but compiled the kernel
86 * without it, we need to cope with them respectively.
89 #if defined(KBUILD_MCOUNT_RA_ADDRESS) && defined(CONFIG_32BIT)
91 * lui v1, hi_16bit_of_mcount --> b 1f (0x10000005)
92 * addiu v1, v1, low_16bit_of_mcount
94 * move $12, ra_address
97 * 1: offset = 5 instructions
102 * lui v1, hi_16bit_of_mcount --> b 1f (0x10000004)
103 * addiu v1, v1, low_16bit_of_mcount
106 * nop | move $12, ra_address | sub sp, sp, 8
107 * 1: offset = 4 instructions
114 * jal _mcount --> nop
118 return ftrace_modify_code(ip
, new);
121 int ftrace_make_call(struct dyn_ftrace
*rec
, unsigned long addr
)
124 unsigned long ip
= rec
->ip
;
126 /* ip, module: 0xc0000000, kernel: 0x80000000 */
127 new = in_module(ip
) ? insn_lui_v1_hi16_mcount
: insn_jal_ftrace_caller
;
129 return ftrace_modify_code(ip
, new);
132 #define FTRACE_CALL_IP ((unsigned long)(&ftrace_call))
134 int ftrace_update_ftrace_func(ftrace_func_t func
)
138 new = INSN_JAL((unsigned long)func
);
140 return ftrace_modify_code(FTRACE_CALL_IP
, new);
143 int __init
ftrace_dyn_arch_init(void *data
)
145 /* Encode the instructions when booting */
146 ftrace_dyn_arch_init_insns();
148 /* Remove "b ftrace_stub" to ensure ftrace_caller() is executed */
149 ftrace_modify_code(MCOUNT_ADDR
, INSN_NOP
);
151 /* The return code is retured via data */
152 *(unsigned long *)data
= 0;
156 #endif /* CONFIG_DYNAMIC_FTRACE */
158 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
160 #ifdef CONFIG_DYNAMIC_FTRACE
162 extern void ftrace_graph_call(void);
163 #define FTRACE_GRAPH_CALL_IP ((unsigned long)(&ftrace_graph_call))
165 int ftrace_enable_ftrace_graph_caller(void)
167 return ftrace_modify_code(FTRACE_GRAPH_CALL_IP
,
168 insn_j_ftrace_graph_caller
);
171 int ftrace_disable_ftrace_graph_caller(void)
173 return ftrace_modify_code(FTRACE_GRAPH_CALL_IP
, INSN_NOP
);
176 #endif /* CONFIG_DYNAMIC_FTRACE */
178 #ifndef KBUILD_MCOUNT_RA_ADDRESS
180 #define S_RA_SP (0xafbf << 16) /* s{d,w} ra, offset(sp) */
181 #define S_R_SP (0xafb0 << 16) /* s{d,w} R, offset(sp) */
182 #define OFFSET_MASK 0xffff /* stack offset range: 0 ~ PT_SIZE */
184 unsigned long ftrace_get_parent_addr(unsigned long self_addr
,
185 unsigned long parent
,
186 unsigned long parent_addr
,
189 unsigned long sp
, ip
, ra
;
194 * For module, move the ip from calling site of mcount to the
195 * instruction "lui v1, hi_16bit_of_mcount"(offset is 20), but for
196 * kernel, move to the instruction "move ra, at"(offset is 12)
198 ip
= self_addr
- (in_module(self_addr
) ? 20 : 12);
201 * search the text until finding the non-store instruction or "s{d,w}
202 * ra, offset(sp)" instruction
207 /* get the code at "ip": code = *(unsigned int *)ip; */
208 safe_load_code(code
, ip
, faulted
);
210 if (unlikely(faulted
))
213 * If we hit the non-store instruction before finding where the
214 * ra is stored, then this is a leaf function and it does not
215 * store the ra on the stack
217 if ((code
& S_R_SP
) != S_R_SP
)
220 } while (((code
& S_RA_SP
) != S_RA_SP
));
222 sp
= fp
+ (code
& OFFSET_MASK
);
224 /* ra = *(unsigned long *)sp; */
225 safe_load_stack(ra
, sp
, faulted
);
226 if (unlikely(faulted
))
234 #endif /* !KBUILD_MCOUNT_RA_ADDRESS */
237 * Hook the return address and push it in the stack of return addrs
238 * in current thread info.
240 void prepare_ftrace_return(unsigned long *parent
, unsigned long self_addr
,
244 struct ftrace_graph_ent trace
;
245 unsigned long return_hooker
= (unsigned long)
249 if (unlikely(atomic_read(¤t
->tracing_graph_pause
)))
253 * "parent" is the stack address saved the return address of the caller
256 * if the gcc < 4.5, a leaf function does not save the return address
257 * in the stack address, so, we "emulate" one in _mcount's stack space,
258 * and hijack it directly, but for a non-leaf function, it save the
259 * return address to the its own stack space, we can not hijack it
260 * directly, but need to find the real stack address,
261 * ftrace_get_parent_addr() does it!
263 * if gcc>= 4.5, with the new -mmcount-ra-address option, for a
264 * non-leaf function, the location of the return address will be saved
265 * to $12 for us, and for a leaf function, only put a zero into $12. we
266 * do it in ftrace_graph_caller of mcount.S.
270 safe_load_stack(old
, parent
, faulted
);
271 if (unlikely(faulted
))
273 #ifndef KBUILD_MCOUNT_RA_ADDRESS
274 parent
= (unsigned long *)ftrace_get_parent_addr(self_addr
, old
,
275 (unsigned long)parent
, fp
);
277 * If fails when getting the stack address of the non-leaf function's
278 * ra, stop function graph tracer and return
283 /* *parent = return_hooker; */
284 safe_store_stack(return_hooker
, parent
, faulted
);
285 if (unlikely(faulted
))
288 if (ftrace_push_return_trace(old
, self_addr
, &trace
.depth
, fp
) ==
294 trace
.func
= self_addr
;
296 /* Only trace if the calling function expects to */
297 if (!ftrace_graph_entry(&trace
)) {
298 current
->curr_ret_stack
--;
306 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */