2 * arch/arm/kernel/kprobes.c
6 * Abhishek Sagar <sagar.abhishek@gmail.com>
7 * Copyright (C) 2006, 2007 Motorola Inc.
9 * Nicolas Pitre <nico@marvell.com>
10 * Copyright (C) 2007 Marvell Ltd.
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License version 2 as
14 * published by the Free Software Foundation.
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 * General Public License for more details.
22 #include <linux/kernel.h>
23 #include <linux/kprobes.h>
24 #include <linux/module.h>
25 #include <linux/stringify.h>
26 #include <asm/traps.h>
27 #include <asm/cacheflush.h>
29 #define MIN_STACK_SIZE(addr) \
30 min((unsigned long)MAX_STACK_SIZE, \
31 (unsigned long)current_thread_info() + THREAD_START_SP - (addr))
33 #define flush_insns(addr, cnt) \
34 flush_icache_range((unsigned long)(addr), \
35 (unsigned long)(addr) + \
36 sizeof(kprobe_opcode_t) * (cnt))
38 /* Used as a marker in ARM_pc to note when we're in a jprobe. */
39 #define JPROBE_MAGIC_ADDR 0xffffffff
41 DEFINE_PER_CPU(struct kprobe
*, current_kprobe
) = NULL
;
42 DEFINE_PER_CPU(struct kprobe_ctlblk
, kprobe_ctlblk
);
45 int __kprobes
arch_prepare_kprobe(struct kprobe
*p
)
48 kprobe_opcode_t tmp_insn
[MAX_INSN_SIZE
];
49 unsigned long addr
= (unsigned long)p
->addr
;
52 if (addr
& 0x3 || in_exception_text(addr
))
57 p
->ainsn
.insn
= tmp_insn
;
59 switch (arm_kprobe_decode_insn(insn
, &p
->ainsn
)) {
60 case INSN_REJECTED
: /* not supported */
63 case INSN_GOOD
: /* instruction uses slot */
64 p
->ainsn
.insn
= get_insn_slot();
67 for (is
= 0; is
< MAX_INSN_SIZE
; ++is
)
68 p
->ainsn
.insn
[is
] = tmp_insn
[is
];
69 flush_insns(p
->ainsn
.insn
, MAX_INSN_SIZE
);
72 case INSN_GOOD_NO_SLOT
: /* instruction doesn't need insn slot */
80 void __kprobes
arch_arm_kprobe(struct kprobe
*p
)
82 *p
->addr
= KPROBE_BREAKPOINT_INSTRUCTION
;
83 flush_insns(p
->addr
, 1);
86 void __kprobes
arch_disarm_kprobe(struct kprobe
*p
)
89 flush_insns(p
->addr
, 1);
92 void __kprobes
arch_remove_kprobe(struct kprobe
*p
)
95 mutex_lock(&kprobe_mutex
);
96 free_insn_slot(p
->ainsn
.insn
, 0);
97 mutex_unlock(&kprobe_mutex
);
102 static void __kprobes
save_previous_kprobe(struct kprobe_ctlblk
*kcb
)
104 kcb
->prev_kprobe
.kp
= kprobe_running();
105 kcb
->prev_kprobe
.status
= kcb
->kprobe_status
;
108 static void __kprobes
restore_previous_kprobe(struct kprobe_ctlblk
*kcb
)
110 __get_cpu_var(current_kprobe
) = kcb
->prev_kprobe
.kp
;
111 kcb
->kprobe_status
= kcb
->prev_kprobe
.status
;
114 static void __kprobes
set_current_kprobe(struct kprobe
*p
)
116 __get_cpu_var(current_kprobe
) = p
;
119 static void __kprobes
singlestep(struct kprobe
*p
, struct pt_regs
*regs
,
120 struct kprobe_ctlblk
*kcb
)
123 p
->ainsn
.insn_handler(p
, regs
);
127 * Called with IRQs disabled. IRQs must remain disabled from that point
128 * all the way until processing this kprobe is complete. The current
129 * kprobes implementation cannot process more than one nested level of
130 * kprobe, and that level is reserved for user kprobe handlers, so we can't
131 * risk encountering a new kprobe in an interrupt handler.
133 void __kprobes
kprobe_handler(struct pt_regs
*regs
)
135 struct kprobe
*p
, *cur
;
136 struct kprobe_ctlblk
*kcb
;
137 kprobe_opcode_t
*addr
= (kprobe_opcode_t
*)regs
->ARM_pc
;
139 kcb
= get_kprobe_ctlblk();
140 cur
= kprobe_running();
141 p
= get_kprobe(addr
);
145 /* Kprobe is pending, so we're recursing. */
146 switch (kcb
->kprobe_status
) {
147 case KPROBE_HIT_ACTIVE
:
148 case KPROBE_HIT_SSDONE
:
149 /* A pre- or post-handler probe got us here. */
150 kprobes_inc_nmissed_count(p
);
151 save_previous_kprobe(kcb
);
152 set_current_kprobe(p
);
153 kcb
->kprobe_status
= KPROBE_REENTER
;
154 singlestep(p
, regs
, kcb
);
155 restore_previous_kprobe(kcb
);
158 /* impossible cases */
162 set_current_kprobe(p
);
163 kcb
->kprobe_status
= KPROBE_HIT_ACTIVE
;
166 * If we have no pre-handler or it returned 0, we
167 * continue with normal processing. If we have a
168 * pre-handler and it returned non-zero, it prepped
169 * for calling the break_handler below on re-entry,
170 * so get out doing nothing more here.
172 if (!p
->pre_handler
|| !p
->pre_handler(p
, regs
)) {
173 kcb
->kprobe_status
= KPROBE_HIT_SS
;
174 singlestep(p
, regs
, kcb
);
175 if (p
->post_handler
) {
176 kcb
->kprobe_status
= KPROBE_HIT_SSDONE
;
177 p
->post_handler(p
, regs
, 0);
179 reset_current_kprobe();
183 /* We probably hit a jprobe. Call its break handler. */
184 if (cur
->break_handler
&& cur
->break_handler(cur
, regs
)) {
185 kcb
->kprobe_status
= KPROBE_HIT_SS
;
186 singlestep(cur
, regs
, kcb
);
187 if (cur
->post_handler
) {
188 kcb
->kprobe_status
= KPROBE_HIT_SSDONE
;
189 cur
->post_handler(cur
, regs
, 0);
192 reset_current_kprobe();
195 * The probe was removed and a race is in progress.
196 * There is nothing we can do about it. Let's restart
197 * the instruction. By the time we can restart, the
198 * real instruction will be there.
203 int kprobe_trap_handler(struct pt_regs
*regs
, unsigned int instr
)
205 kprobe_handler(regs
);
209 int __kprobes
kprobe_fault_handler(struct pt_regs
*regs
, unsigned int fsr
)
211 struct kprobe
*cur
= kprobe_running();
212 struct kprobe_ctlblk
*kcb
= get_kprobe_ctlblk();
214 switch (kcb
->kprobe_status
) {
218 * We are here because the instruction being single
219 * stepped caused a page fault. We reset the current
220 * kprobe and the PC to point back to the probe address
221 * and allow the page fault handler to continue as a
224 regs
->ARM_pc
= (long)cur
->addr
;
225 if (kcb
->kprobe_status
== KPROBE_REENTER
) {
226 restore_previous_kprobe(kcb
);
228 reset_current_kprobe();
232 case KPROBE_HIT_ACTIVE
:
233 case KPROBE_HIT_SSDONE
:
235 * We increment the nmissed count for accounting,
236 * we can also use npre/npostfault count for accounting
237 * these specific fault cases.
239 kprobes_inc_nmissed_count(cur
);
242 * We come here because instructions in the pre/post
243 * handler caused the page_fault, this could happen
244 * if handler tries to access user space by
245 * copy_from_user(), get_user() etc. Let the
246 * user-specified handler try to fix it.
248 if (cur
->fault_handler
&& cur
->fault_handler(cur
, regs
, fsr
))
259 int __kprobes
kprobe_exceptions_notify(struct notifier_block
*self
,
260 unsigned long val
, void *data
)
263 * notify_die() is currently never called on ARM,
264 * so this callback is currently empty.
270 * When a retprobed function returns, trampoline_handler() is called,
271 * calling the kretprobe's handler. We construct a struct pt_regs to
272 * give a view of registers r0-r11 to the user return-handler. This is
273 * not a complete pt_regs structure, but that should be plenty sufficient
274 * for kretprobe handlers which should normally be interested in r0 only
277 void __naked __kprobes
kretprobe_trampoline(void)
279 __asm__
__volatile__ (
280 "stmdb sp!, {r0 - r11} \n\t"
282 "bl trampoline_handler \n\t"
284 "ldmia sp!, {r0 - r11} \n\t"
289 /* Called from kretprobe_trampoline */
290 static __used __kprobes
void *trampoline_handler(struct pt_regs
*regs
)
292 struct kretprobe_instance
*ri
= NULL
;
293 struct hlist_head
*head
, empty_rp
;
294 struct hlist_node
*node
, *tmp
;
295 unsigned long flags
, orig_ret_address
= 0;
296 unsigned long trampoline_address
= (unsigned long)&kretprobe_trampoline
;
298 INIT_HLIST_HEAD(&empty_rp
);
299 kretprobe_hash_lock(current
, &head
, &flags
);
302 * It is possible to have multiple instances associated with a given
303 * task either because multiple functions in the call path have
304 * a return probe installed on them, and/or more than one return
305 * probe was registered for a target function.
307 * We can handle this because:
308 * - instances are always inserted at the head of the list
309 * - when multiple return probes are registered for the same
310 * function, the first instance's ret_addr will point to the
311 * real return address, and all the rest will point to
312 * kretprobe_trampoline
314 hlist_for_each_entry_safe(ri
, node
, tmp
, head
, hlist
) {
315 if (ri
->task
!= current
)
316 /* another task is sharing our hash bucket */
319 if (ri
->rp
&& ri
->rp
->handler
) {
320 __get_cpu_var(current_kprobe
) = &ri
->rp
->kp
;
321 get_kprobe_ctlblk()->kprobe_status
= KPROBE_HIT_ACTIVE
;
322 ri
->rp
->handler(ri
, regs
);
323 __get_cpu_var(current_kprobe
) = NULL
;
326 orig_ret_address
= (unsigned long)ri
->ret_addr
;
327 recycle_rp_inst(ri
, &empty_rp
);
329 if (orig_ret_address
!= trampoline_address
)
331 * This is the real return address. Any other
332 * instances associated with this task are for
333 * other calls deeper on the call stack
338 kretprobe_assert(ri
, orig_ret_address
, trampoline_address
);
339 kretprobe_hash_unlock(current
, &flags
);
341 hlist_for_each_entry_safe(ri
, node
, tmp
, &empty_rp
, hlist
) {
342 hlist_del(&ri
->hlist
);
346 return (void *)orig_ret_address
;
349 void __kprobes
arch_prepare_kretprobe(struct kretprobe_instance
*ri
,
350 struct pt_regs
*regs
)
352 ri
->ret_addr
= (kprobe_opcode_t
*)regs
->ARM_lr
;
354 /* Replace the return addr with trampoline addr. */
355 regs
->ARM_lr
= (unsigned long)&kretprobe_trampoline
;
358 int __kprobes
setjmp_pre_handler(struct kprobe
*p
, struct pt_regs
*regs
)
360 struct jprobe
*jp
= container_of(p
, struct jprobe
, kp
);
361 struct kprobe_ctlblk
*kcb
= get_kprobe_ctlblk();
362 long sp_addr
= regs
->ARM_sp
;
364 kcb
->jprobe_saved_regs
= *regs
;
365 memcpy(kcb
->jprobes_stack
, (void *)sp_addr
, MIN_STACK_SIZE(sp_addr
));
366 regs
->ARM_pc
= (long)jp
->entry
;
367 regs
->ARM_cpsr
|= PSR_I_BIT
;
372 void __kprobes
jprobe_return(void)
374 struct kprobe_ctlblk
*kcb
= get_kprobe_ctlblk();
376 __asm__
__volatile__ (
378 * Setup an empty pt_regs. Fill SP and PC fields as
379 * they're needed by longjmp_break_handler.
381 "sub sp, %0, %1 \n\t"
382 "ldr r0, ="__stringify(JPROBE_MAGIC_ADDR
)"\n\t"
383 "str %0, [sp, %2] \n\t"
384 "str r0, [sp, %3] \n\t"
386 "bl kprobe_handler \n\t"
389 * Return to the context saved by setjmp_pre_handler
390 * and restored by longjmp_break_handler.
392 "ldr r0, [sp, %4] \n\t"
393 "msr cpsr_cxsf, r0 \n\t"
394 "ldmia sp, {r0 - pc} \n\t"
396 : "r" (kcb
->jprobe_saved_regs
.ARM_sp
),
397 "I" (sizeof(struct pt_regs
)),
398 "J" (offsetof(struct pt_regs
, ARM_sp
)),
399 "J" (offsetof(struct pt_regs
, ARM_pc
)),
400 "J" (offsetof(struct pt_regs
, ARM_cpsr
))
404 int __kprobes
longjmp_break_handler(struct kprobe
*p
, struct pt_regs
*regs
)
406 struct kprobe_ctlblk
*kcb
= get_kprobe_ctlblk();
407 long stack_addr
= kcb
->jprobe_saved_regs
.ARM_sp
;
408 long orig_sp
= regs
->ARM_sp
;
409 struct jprobe
*jp
= container_of(p
, struct jprobe
, kp
);
411 if (regs
->ARM_pc
== JPROBE_MAGIC_ADDR
) {
412 if (orig_sp
!= stack_addr
) {
413 struct pt_regs
*saved_regs
=
414 (struct pt_regs
*)kcb
->jprobe_saved_regs
.ARM_sp
;
415 printk("current sp %lx does not match saved sp %lx\n",
416 orig_sp
, stack_addr
);
417 printk("Saved registers for jprobe %p\n", jp
);
418 show_regs(saved_regs
);
419 printk("Current registers\n");
423 *regs
= kcb
->jprobe_saved_regs
;
424 memcpy((void *)stack_addr
, kcb
->jprobes_stack
,
425 MIN_STACK_SIZE(stack_addr
));
426 preempt_enable_no_resched();
432 int __kprobes
arch_trampoline_kprobe(struct kprobe
*p
)
437 static struct undef_hook kprobes_break_hook
= {
438 .instr_mask
= 0xffffffff,
439 .instr_val
= KPROBE_BREAKPOINT_INSTRUCTION
,
440 .cpsr_mask
= MODE_MASK
,
441 .cpsr_val
= SVC_MODE
,
442 .fn
= kprobe_trap_handler
,
445 int __init
arch_init_kprobes()
447 arm_kprobe_decode_init();
448 register_undef_hook(&kprobes_break_hook
);