2 * Kprobes-based tracing events
4 * Created by Masami Hiramatsu <mhiramat@redhat.com>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 #include <linux/module.h>
21 #include <linux/uaccess.h>
22 #include <linux/kprobes.h>
23 #include <linux/seq_file.h>
24 #include <linux/slab.h>
25 #include <linux/smp.h>
26 #include <linux/debugfs.h>
27 #include <linux/types.h>
28 #include <linux/string.h>
29 #include <linux/ctype.h>
30 #include <linux/ptrace.h>
31 #include <linux/perf_event.h>
32 #include <linux/stringify.h>
33 #include <asm/bitsperlong.h>
36 #include "trace_output.h"
38 #define MAX_TRACE_ARGS 128
39 #define MAX_ARGSTR_LEN 63
40 #define MAX_EVENT_NAME_LEN 64
41 #define KPROBE_EVENT_SYSTEM "kprobes"
43 /* Reserved field names */
44 #define FIELD_STRING_IP "__probe_ip"
45 #define FIELD_STRING_RETIP "__probe_ret_ip"
46 #define FIELD_STRING_FUNC "__probe_func"
48 const char *reserved_field_names
[] = {
51 "common_preempt_count",
60 /* Printing function type */
61 typedef int (*print_type_func_t
)(struct trace_seq
*, const char *, void *);
62 #define PRINT_TYPE_FUNC_NAME(type) print_type_##type
63 #define PRINT_TYPE_FMT_NAME(type) print_type_format_##type
65 /* Printing in basic type function template */
66 #define DEFINE_BASIC_PRINT_TYPE_FUNC(type, fmt, cast) \
67 static __kprobes int PRINT_TYPE_FUNC_NAME(type)(struct trace_seq *s, \
68 const char *name, void *data)\
70 return trace_seq_printf(s, " %s=" fmt, name, (cast)*(type *)data);\
72 static const char PRINT_TYPE_FMT_NAME(type)[] = fmt;
74 DEFINE_BASIC_PRINT_TYPE_FUNC(u8
, "%x", unsigned int)
75 DEFINE_BASIC_PRINT_TYPE_FUNC(u16
, "%x", unsigned int)
76 DEFINE_BASIC_PRINT_TYPE_FUNC(u32
, "%lx", unsigned long)
77 DEFINE_BASIC_PRINT_TYPE_FUNC(u64
, "%llx", unsigned long long)
78 DEFINE_BASIC_PRINT_TYPE_FUNC(s8
, "%d", int)
79 DEFINE_BASIC_PRINT_TYPE_FUNC(s16
, "%d", int)
80 DEFINE_BASIC_PRINT_TYPE_FUNC(s32
, "%ld", long)
81 DEFINE_BASIC_PRINT_TYPE_FUNC(s64
, "%lld", long long)
83 /* Data fetch function type */
84 typedef void (*fetch_func_t
)(struct pt_regs
*, void *, void *);
91 static __kprobes
void call_fetch(struct fetch_param
*fprm
,
92 struct pt_regs
*regs
, void *dest
)
94 return fprm
->fn(regs
, fprm
->data
, dest
);
97 #define FETCH_FUNC_NAME(kind, type) fetch_##kind##_##type
99 * Define macro for basic types - we don't need to define s* types, because
100 * we have to care only about bitwidth at recording time.
102 #define DEFINE_BASIC_FETCH_FUNCS(kind) \
103 DEFINE_FETCH_##kind(u8) \
104 DEFINE_FETCH_##kind(u16) \
105 DEFINE_FETCH_##kind(u32) \
106 DEFINE_FETCH_##kind(u64)
108 #define CHECK_BASIC_FETCH_FUNCS(kind, fn) \
109 ((FETCH_FUNC_NAME(kind, u8) == fn) || \
110 (FETCH_FUNC_NAME(kind, u16) == fn) || \
111 (FETCH_FUNC_NAME(kind, u32) == fn) || \
112 (FETCH_FUNC_NAME(kind, u64) == fn))
114 /* Data fetch function templates */
115 #define DEFINE_FETCH_reg(type) \
116 static __kprobes void FETCH_FUNC_NAME(reg, type)(struct pt_regs *regs, \
117 void *offset, void *dest) \
119 *(type *)dest = (type)regs_get_register(regs, \
120 (unsigned int)((unsigned long)offset)); \
122 DEFINE_BASIC_FETCH_FUNCS(reg
)
124 #define DEFINE_FETCH_stack(type) \
125 static __kprobes void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs,\
126 void *offset, void *dest) \
128 *(type *)dest = (type)regs_get_kernel_stack_nth(regs, \
129 (unsigned int)((unsigned long)offset)); \
131 DEFINE_BASIC_FETCH_FUNCS(stack
)
133 #define DEFINE_FETCH_retval(type) \
134 static __kprobes void FETCH_FUNC_NAME(retval, type)(struct pt_regs *regs,\
135 void *dummy, void *dest) \
137 *(type *)dest = (type)regs_return_value(regs); \
139 DEFINE_BASIC_FETCH_FUNCS(retval
)
141 #define DEFINE_FETCH_memory(type) \
142 static __kprobes void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs,\
143 void *addr, void *dest) \
146 if (probe_kernel_address(addr, retval)) \
149 *(type *)dest = retval; \
151 DEFINE_BASIC_FETCH_FUNCS(memory
)
153 /* Memory fetching by symbol */
154 struct symbol_cache
{
160 static unsigned long update_symbol_cache(struct symbol_cache
*sc
)
162 sc
->addr
= (unsigned long)kallsyms_lookup_name(sc
->symbol
);
164 sc
->addr
+= sc
->offset
;
168 static void free_symbol_cache(struct symbol_cache
*sc
)
174 static struct symbol_cache
*alloc_symbol_cache(const char *sym
, long offset
)
176 struct symbol_cache
*sc
;
178 if (!sym
|| strlen(sym
) == 0)
180 sc
= kzalloc(sizeof(struct symbol_cache
), GFP_KERNEL
);
184 sc
->symbol
= kstrdup(sym
, GFP_KERNEL
);
191 update_symbol_cache(sc
);
195 #define DEFINE_FETCH_symbol(type) \
196 static __kprobes void FETCH_FUNC_NAME(symbol, type)(struct pt_regs *regs,\
197 void *data, void *dest) \
199 struct symbol_cache *sc = data; \
201 fetch_memory_##type(regs, (void *)sc->addr, dest); \
205 DEFINE_BASIC_FETCH_FUNCS(symbol
)
207 /* Dereference memory access function */
208 struct deref_fetch_param
{
209 struct fetch_param orig
;
213 #define DEFINE_FETCH_deref(type) \
214 static __kprobes void FETCH_FUNC_NAME(deref, type)(struct pt_regs *regs,\
215 void *data, void *dest) \
217 struct deref_fetch_param *dprm = data; \
218 unsigned long addr; \
219 call_fetch(&dprm->orig, regs, &addr); \
221 addr += dprm->offset; \
222 fetch_memory_##type(regs, (void *)addr, dest); \
226 DEFINE_BASIC_FETCH_FUNCS(deref
)
228 static __kprobes
void free_deref_fetch_param(struct deref_fetch_param
*data
)
230 if (CHECK_BASIC_FETCH_FUNCS(deref
, data
->orig
.fn
))
231 free_deref_fetch_param(data
->orig
.data
);
232 else if (CHECK_BASIC_FETCH_FUNCS(symbol
, data
->orig
.fn
))
233 free_symbol_cache(data
->orig
.data
);
237 /* Default (unsigned long) fetch type */
238 #define __DEFAULT_FETCH_TYPE(t) u##t
239 #define _DEFAULT_FETCH_TYPE(t) __DEFAULT_FETCH_TYPE(t)
240 #define DEFAULT_FETCH_TYPE _DEFAULT_FETCH_TYPE(BITS_PER_LONG)
241 #define DEFAULT_FETCH_TYPE_STR __stringify(DEFAULT_FETCH_TYPE)
243 #define ASSIGN_FETCH_FUNC(kind, type) \
244 .kind = FETCH_FUNC_NAME(kind, type)
246 #define ASSIGN_FETCH_TYPE(ptype, ftype, sign) \
248 .size = sizeof(ftype), \
250 .print = PRINT_TYPE_FUNC_NAME(ptype), \
251 .fmt = PRINT_TYPE_FMT_NAME(ptype), \
252 ASSIGN_FETCH_FUNC(reg, ftype), \
253 ASSIGN_FETCH_FUNC(stack, ftype), \
254 ASSIGN_FETCH_FUNC(retval, ftype), \
255 ASSIGN_FETCH_FUNC(memory, ftype), \
256 ASSIGN_FETCH_FUNC(symbol, ftype), \
257 ASSIGN_FETCH_FUNC(deref, ftype), \
260 /* Fetch type information table */
261 static const struct fetch_type
{
262 const char *name
; /* Name of type */
263 size_t size
; /* Byte size of type */
264 int is_signed
; /* Signed flag */
265 print_type_func_t print
; /* Print functions */
266 const char *fmt
; /* Fromat string */
267 /* Fetch functions */
274 } fetch_type_table
[] = {
275 ASSIGN_FETCH_TYPE(u8
, u8
, 0),
276 ASSIGN_FETCH_TYPE(u16
, u16
, 0),
277 ASSIGN_FETCH_TYPE(u32
, u32
, 0),
278 ASSIGN_FETCH_TYPE(u64
, u64
, 0),
279 ASSIGN_FETCH_TYPE(s8
, u8
, 1),
280 ASSIGN_FETCH_TYPE(s16
, u16
, 1),
281 ASSIGN_FETCH_TYPE(s32
, u32
, 1),
282 ASSIGN_FETCH_TYPE(s64
, u64
, 1),
285 static const struct fetch_type
*find_fetch_type(const char *type
)
290 type
= DEFAULT_FETCH_TYPE_STR
;
292 for (i
= 0; i
< ARRAY_SIZE(fetch_type_table
); i
++)
293 if (strcmp(type
, fetch_type_table
[i
].name
) == 0)
294 return &fetch_type_table
[i
];
298 /* Special function : only accept unsigned long */
299 static __kprobes
void fetch_stack_address(struct pt_regs
*regs
,
300 void *dummy
, void *dest
)
302 *(unsigned long *)dest
= kernel_stack_pointer(regs
);
306 * Kprobe event core functions
310 struct fetch_param fetch
;
311 unsigned int offset
; /* Offset from argument entry */
312 const char *name
; /* Name of this argument */
313 const char *comm
; /* Command of this argument */
314 const struct fetch_type
*type
; /* Type of this argument */
317 /* Flags for trace_probe */
318 #define TP_FLAG_TRACE 1
319 #define TP_FLAG_PROFILE 2
322 struct list_head list
;
323 struct kretprobe rp
; /* Use rp.kp for kprobe use */
325 unsigned int flags
; /* For TP_FLAG_* */
326 const char *symbol
; /* symbol name */
327 struct ftrace_event_call call
;
328 struct trace_event event
;
329 ssize_t size
; /* trace entry size */
330 unsigned int nr_args
;
331 struct probe_arg args
[];
334 #define SIZEOF_TRACE_PROBE(n) \
335 (offsetof(struct trace_probe, args) + \
336 (sizeof(struct probe_arg) * (n)))
339 static __kprobes
int probe_is_return(struct trace_probe
*tp
)
341 return tp
->rp
.handler
!= NULL
;
344 static __kprobes
const char *probe_symbol(struct trace_probe
*tp
)
346 return tp
->symbol
? tp
->symbol
: "unknown";
349 static int register_probe_event(struct trace_probe
*tp
);
350 static void unregister_probe_event(struct trace_probe
*tp
);
352 static DEFINE_MUTEX(probe_lock
);
353 static LIST_HEAD(probe_list
);
355 static int kprobe_dispatcher(struct kprobe
*kp
, struct pt_regs
*regs
);
356 static int kretprobe_dispatcher(struct kretprobe_instance
*ri
,
357 struct pt_regs
*regs
);
359 /* Check the name is good for event/group */
360 static int check_event_name(const char *name
)
362 if (!isalpha(*name
) && *name
!= '_')
364 while (*++name
!= '\0') {
365 if (!isalpha(*name
) && !isdigit(*name
) && *name
!= '_')
372 * Allocate new trace_probe and initialize it (including kprobes).
374 static struct trace_probe
*alloc_trace_probe(const char *group
,
379 int nargs
, int is_return
)
381 struct trace_probe
*tp
;
384 tp
= kzalloc(SIZEOF_TRACE_PROBE(nargs
), GFP_KERNEL
);
389 tp
->symbol
= kstrdup(symbol
, GFP_KERNEL
);
392 tp
->rp
.kp
.symbol_name
= tp
->symbol
;
393 tp
->rp
.kp
.offset
= offs
;
395 tp
->rp
.kp
.addr
= addr
;
398 tp
->rp
.handler
= kretprobe_dispatcher
;
400 tp
->rp
.kp
.pre_handler
= kprobe_dispatcher
;
402 if (!event
|| !check_event_name(event
)) {
407 tp
->call
.name
= kstrdup(event
, GFP_KERNEL
);
411 if (!group
|| !check_event_name(group
)) {
416 tp
->call
.system
= kstrdup(group
, GFP_KERNEL
);
417 if (!tp
->call
.system
)
420 INIT_LIST_HEAD(&tp
->list
);
423 kfree(tp
->call
.name
);
429 static void free_probe_arg(struct probe_arg
*arg
)
431 if (CHECK_BASIC_FETCH_FUNCS(deref
, arg
->fetch
.fn
))
432 free_deref_fetch_param(arg
->fetch
.data
);
433 else if (CHECK_BASIC_FETCH_FUNCS(symbol
, arg
->fetch
.fn
))
434 free_symbol_cache(arg
->fetch
.data
);
439 static void free_trace_probe(struct trace_probe
*tp
)
443 for (i
= 0; i
< tp
->nr_args
; i
++)
444 free_probe_arg(&tp
->args
[i
]);
446 kfree(tp
->call
.system
);
447 kfree(tp
->call
.name
);
452 static struct trace_probe
*find_probe_event(const char *event
,
455 struct trace_probe
*tp
;
457 list_for_each_entry(tp
, &probe_list
, list
)
458 if (strcmp(tp
->call
.name
, event
) == 0 &&
459 strcmp(tp
->call
.system
, group
) == 0)
464 /* Unregister a trace_probe and probe_event: call with locking probe_lock */
465 static void unregister_trace_probe(struct trace_probe
*tp
)
467 if (probe_is_return(tp
))
468 unregister_kretprobe(&tp
->rp
);
470 unregister_kprobe(&tp
->rp
.kp
);
472 unregister_probe_event(tp
);
475 /* Register a trace_probe and probe_event */
476 static int register_trace_probe(struct trace_probe
*tp
)
478 struct trace_probe
*old_tp
;
481 mutex_lock(&probe_lock
);
483 /* register as an event */
484 old_tp
= find_probe_event(tp
->call
.name
, tp
->call
.system
);
486 /* delete old event */
487 unregister_trace_probe(old_tp
);
488 free_trace_probe(old_tp
);
490 ret
= register_probe_event(tp
);
492 pr_warning("Faild to register probe event(%d)\n", ret
);
496 tp
->rp
.kp
.flags
|= KPROBE_FLAG_DISABLED
;
497 if (probe_is_return(tp
))
498 ret
= register_kretprobe(&tp
->rp
);
500 ret
= register_kprobe(&tp
->rp
.kp
);
503 pr_warning("Could not insert probe(%d)\n", ret
);
504 if (ret
== -EILSEQ
) {
505 pr_warning("Probing address(0x%p) is not an "
506 "instruction boundary.\n",
510 unregister_probe_event(tp
);
512 list_add_tail(&tp
->list
, &probe_list
);
514 mutex_unlock(&probe_lock
);
518 /* Split symbol and offset. */
519 static int split_symbol_offset(char *symbol
, unsigned long *offset
)
527 tmp
= strchr(symbol
, '+');
529 /* skip sign because strict_strtol doesn't accept '+' */
530 ret
= strict_strtoul(tmp
+ 1, 0, offset
);
539 #define PARAM_MAX_ARGS 16
540 #define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long))
542 static int parse_probe_vars(char *arg
, const struct fetch_type
*t
,
543 struct fetch_param
*f
, int is_return
)
548 if (strcmp(arg
, "retval") == 0) {
553 } else if (strncmp(arg
, "stack", 5) == 0) {
554 if (arg
[5] == '\0') {
555 if (strcmp(t
->name
, DEFAULT_FETCH_TYPE_STR
) == 0)
556 f
->fn
= fetch_stack_address
;
559 } else if (isdigit(arg
[5])) {
560 ret
= strict_strtoul(arg
+ 5, 10, ¶m
);
561 if (ret
|| param
> PARAM_MAX_STACK
)
565 f
->data
= (void *)param
;
574 /* Recursive argument parser */
575 static int __parse_probe_arg(char *arg
, const struct fetch_type
*t
,
576 struct fetch_param
*f
, int is_return
)
585 ret
= parse_probe_vars(arg
+ 1, t
, f
, is_return
);
587 case '%': /* named register */
588 ret
= regs_query_register_offset(arg
+ 1);
591 f
->data
= (void *)(unsigned long)ret
;
595 case '@': /* memory or symbol */
596 if (isdigit(arg
[1])) {
597 ret
= strict_strtoul(arg
+ 1, 0, ¶m
);
601 f
->data
= (void *)param
;
603 ret
= split_symbol_offset(arg
+ 1, &offset
);
606 f
->data
= alloc_symbol_cache(arg
+ 1, offset
);
611 case '+': /* deref memory */
613 tmp
= strchr(arg
, '(');
617 ret
= strict_strtol(arg
+ 1, 0, &offset
);
623 tmp
= strrchr(arg
, ')');
625 struct deref_fetch_param
*dprm
;
626 const struct fetch_type
*t2
= find_fetch_type(NULL
);
628 dprm
= kzalloc(sizeof(struct deref_fetch_param
),
632 dprm
->offset
= offset
;
633 ret
= __parse_probe_arg(arg
, t2
, &dprm
->orig
,
639 f
->data
= (void *)dprm
;
649 /* String length checking wrapper */
650 static int parse_probe_arg(char *arg
, struct trace_probe
*tp
,
651 struct probe_arg
*parg
, int is_return
)
655 if (strlen(arg
) > MAX_ARGSTR_LEN
) {
656 pr_info("Argument is too long.: %s\n", arg
);
659 parg
->comm
= kstrdup(arg
, GFP_KERNEL
);
661 pr_info("Failed to allocate memory for command '%s'.\n", arg
);
664 t
= strchr(parg
->comm
, ':');
666 arg
[t
- parg
->comm
] = '\0';
669 parg
->type
= find_fetch_type(t
);
671 pr_info("Unsupported type: %s\n", t
);
674 parg
->offset
= tp
->size
;
675 tp
->size
+= parg
->type
->size
;
676 return __parse_probe_arg(arg
, parg
->type
, &parg
->fetch
, is_return
);
679 /* Return 1 if name is reserved or already used by another argument */
680 static int conflict_field_name(const char *name
,
681 struct probe_arg
*args
, int narg
)
684 for (i
= 0; i
< ARRAY_SIZE(reserved_field_names
); i
++)
685 if (strcmp(reserved_field_names
[i
], name
) == 0)
687 for (i
= 0; i
< narg
; i
++)
688 if (strcmp(args
[i
].name
, name
) == 0)
693 static int create_trace_probe(int argc
, char **argv
)
697 * - Add kprobe: p[:[GRP/]EVENT] KSYM[+OFFS]|KADDR [FETCHARGS]
698 * - Add kretprobe: r[:[GRP/]EVENT] KSYM[+0] [FETCHARGS]
700 * $retval : fetch return value
701 * $stack : fetch stack address
702 * $stackN : fetch Nth of stack (N:0-)
703 * @ADDR : fetch memory at ADDR (ADDR should be in kernel)
704 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)
705 * %REG : fetch register REG
706 * Dereferencing memory fetch:
707 * +|-offs(ARG) : fetch memory at ARG +|- offs address.
708 * Alias name of args:
709 * NAME=FETCHARG : set NAME as alias of FETCHARG.
711 * FETCHARG:TYPE : use TYPE instead of unsigned long.
713 struct trace_probe
*tp
;
715 int is_return
= 0, is_delete
= 0;
716 char *symbol
= NULL
, *event
= NULL
, *group
= NULL
;
718 unsigned long offset
= 0;
720 char buf
[MAX_EVENT_NAME_LEN
];
722 /* argc must be >= 1 */
723 if (argv
[0][0] == 'p')
725 else if (argv
[0][0] == 'r')
727 else if (argv
[0][0] == '-')
730 pr_info("Probe definition must be started with 'p', 'r' or"
735 if (argv
[0][1] == ':') {
737 if (strchr(event
, '/')) {
739 event
= strchr(group
, '/') + 1;
741 if (strlen(group
) == 0) {
742 pr_info("Group name is not specified\n");
746 if (strlen(event
) == 0) {
747 pr_info("Event name is not specified\n");
752 group
= KPROBE_EVENT_SYSTEM
;
756 pr_info("Delete command needs an event name.\n");
759 tp
= find_probe_event(event
, group
);
761 pr_info("Event %s/%s doesn't exist.\n", group
, event
);
764 /* delete an event */
765 unregister_trace_probe(tp
);
766 free_trace_probe(tp
);
771 pr_info("Probe point is not specified.\n");
774 if (isdigit(argv
[1][0])) {
776 pr_info("Return probe point must be a symbol.\n");
779 /* an address specified */
780 ret
= strict_strtoul(&argv
[1][0], 0, (unsigned long *)&addr
);
782 pr_info("Failed to parse address.\n");
786 /* a symbol specified */
788 /* TODO: support .init module functions */
789 ret
= split_symbol_offset(symbol
, &offset
);
791 pr_info("Failed to parse symbol.\n");
794 if (offset
&& is_return
) {
795 pr_info("Return probe must be used without offset.\n");
799 argc
-= 2; argv
+= 2;
803 /* Make a new event name */
805 snprintf(buf
, MAX_EVENT_NAME_LEN
, "%c_%s_%ld",
806 is_return
? 'r' : 'p', symbol
, offset
);
808 snprintf(buf
, MAX_EVENT_NAME_LEN
, "%c_0x%p",
809 is_return
? 'r' : 'p', addr
);
812 tp
= alloc_trace_probe(group
, event
, addr
, symbol
, offset
, argc
,
815 pr_info("Failed to allocate trace_probe.(%d)\n",
820 /* parse arguments */
822 for (i
= 0; i
< argc
&& i
< MAX_TRACE_ARGS
; i
++) {
823 /* Parse argument name */
824 arg
= strchr(argv
[i
], '=');
830 tp
->args
[i
].name
= kstrdup(argv
[i
], GFP_KERNEL
);
831 if (!tp
->args
[i
].name
) {
832 pr_info("Failed to allocate argument%d name '%s'.\n",
837 tmp
= strchr(tp
->args
[i
].name
, ':');
839 *tmp
= '_'; /* convert : to _ */
841 if (conflict_field_name(tp
->args
[i
].name
, tp
->args
, i
)) {
842 pr_info("Argument%d name '%s' conflicts with "
843 "another field.\n", i
, argv
[i
]);
848 /* Parse fetch argument */
849 ret
= parse_probe_arg(arg
, tp
, &tp
->args
[i
], is_return
);
851 pr_info("Parse error at argument%d. (%d)\n", i
, ret
);
852 kfree(tp
->args
[i
].name
);
859 ret
= register_trace_probe(tp
);
865 free_trace_probe(tp
);
869 static void cleanup_all_probes(void)
871 struct trace_probe
*tp
;
873 mutex_lock(&probe_lock
);
874 /* TODO: Use batch unregistration */
875 while (!list_empty(&probe_list
)) {
876 tp
= list_entry(probe_list
.next
, struct trace_probe
, list
);
877 unregister_trace_probe(tp
);
878 free_trace_probe(tp
);
880 mutex_unlock(&probe_lock
);
884 /* Probes listing interfaces */
885 static void *probes_seq_start(struct seq_file
*m
, loff_t
*pos
)
887 mutex_lock(&probe_lock
);
888 return seq_list_start(&probe_list
, *pos
);
891 static void *probes_seq_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
893 return seq_list_next(v
, &probe_list
, pos
);
896 static void probes_seq_stop(struct seq_file
*m
, void *v
)
898 mutex_unlock(&probe_lock
);
901 static int probes_seq_show(struct seq_file
*m
, void *v
)
903 struct trace_probe
*tp
= v
;
906 seq_printf(m
, "%c", probe_is_return(tp
) ? 'r' : 'p');
907 seq_printf(m
, ":%s/%s", tp
->call
.system
, tp
->call
.name
);
910 seq_printf(m
, " 0x%p", tp
->rp
.kp
.addr
);
911 else if (tp
->rp
.kp
.offset
)
912 seq_printf(m
, " %s+%u", probe_symbol(tp
), tp
->rp
.kp
.offset
);
914 seq_printf(m
, " %s", probe_symbol(tp
));
916 for (i
= 0; i
< tp
->nr_args
; i
++)
917 seq_printf(m
, " %s=%s", tp
->args
[i
].name
, tp
->args
[i
].comm
);
923 static const struct seq_operations probes_seq_op
= {
924 .start
= probes_seq_start
,
925 .next
= probes_seq_next
,
926 .stop
= probes_seq_stop
,
927 .show
= probes_seq_show
930 static int probes_open(struct inode
*inode
, struct file
*file
)
932 if ((file
->f_mode
& FMODE_WRITE
) &&
933 (file
->f_flags
& O_TRUNC
))
934 cleanup_all_probes();
936 return seq_open(file
, &probes_seq_op
);
939 static int command_trace_probe(const char *buf
)
942 int argc
= 0, ret
= 0;
944 argv
= argv_split(GFP_KERNEL
, buf
, &argc
);
949 ret
= create_trace_probe(argc
, argv
);
955 #define WRITE_BUFSIZE 128
957 static ssize_t
probes_write(struct file
*file
, const char __user
*buffer
,
958 size_t count
, loff_t
*ppos
)
965 kbuf
= kmalloc(WRITE_BUFSIZE
, GFP_KERNEL
);
970 while (done
< count
) {
972 if (size
>= WRITE_BUFSIZE
)
973 size
= WRITE_BUFSIZE
- 1;
974 if (copy_from_user(kbuf
, buffer
+ done
, size
)) {
979 tmp
= strchr(kbuf
, '\n');
982 size
= tmp
- kbuf
+ 1;
983 } else if (done
+ size
< count
) {
984 pr_warning("Line length is too long: "
985 "Should be less than %d.", WRITE_BUFSIZE
);
990 /* Remove comments */
991 tmp
= strchr(kbuf
, '#');
995 ret
= command_trace_probe(kbuf
);
1005 static const struct file_operations kprobe_events_ops
= {
1006 .owner
= THIS_MODULE
,
1007 .open
= probes_open
,
1009 .llseek
= seq_lseek
,
1010 .release
= seq_release
,
1011 .write
= probes_write
,
1014 /* Probes profiling interfaces */
1015 static int probes_profile_seq_show(struct seq_file
*m
, void *v
)
1017 struct trace_probe
*tp
= v
;
1019 seq_printf(m
, " %-44s %15lu %15lu\n", tp
->call
.name
, tp
->nhit
,
1025 static const struct seq_operations profile_seq_op
= {
1026 .start
= probes_seq_start
,
1027 .next
= probes_seq_next
,
1028 .stop
= probes_seq_stop
,
1029 .show
= probes_profile_seq_show
1032 static int profile_open(struct inode
*inode
, struct file
*file
)
1034 return seq_open(file
, &profile_seq_op
);
1037 static const struct file_operations kprobe_profile_ops
= {
1038 .owner
= THIS_MODULE
,
1039 .open
= profile_open
,
1041 .llseek
= seq_lseek
,
1042 .release
= seq_release
,
1045 /* Kprobe handler */
1046 static __kprobes
void kprobe_trace_func(struct kprobe
*kp
, struct pt_regs
*regs
)
1048 struct trace_probe
*tp
= container_of(kp
, struct trace_probe
, rp
.kp
);
1049 struct kprobe_trace_entry_head
*entry
;
1050 struct ring_buffer_event
*event
;
1051 struct ring_buffer
*buffer
;
1054 unsigned long irq_flags
;
1055 struct ftrace_event_call
*call
= &tp
->call
;
1059 local_save_flags(irq_flags
);
1060 pc
= preempt_count();
1062 size
= sizeof(*entry
) + tp
->size
;
1064 event
= trace_current_buffer_lock_reserve(&buffer
, call
->id
, size
,
1069 entry
= ring_buffer_event_data(event
);
1070 entry
->ip
= (unsigned long)kp
->addr
;
1071 data
= (u8
*)&entry
[1];
1072 for (i
= 0; i
< tp
->nr_args
; i
++)
1073 call_fetch(&tp
->args
[i
].fetch
, regs
, data
+ tp
->args
[i
].offset
);
1075 if (!filter_current_check_discard(buffer
, call
, entry
, event
))
1076 trace_nowake_buffer_unlock_commit(buffer
, event
, irq_flags
, pc
);
1079 /* Kretprobe handler */
1080 static __kprobes
void kretprobe_trace_func(struct kretprobe_instance
*ri
,
1081 struct pt_regs
*regs
)
1083 struct trace_probe
*tp
= container_of(ri
->rp
, struct trace_probe
, rp
);
1084 struct kretprobe_trace_entry_head
*entry
;
1085 struct ring_buffer_event
*event
;
1086 struct ring_buffer
*buffer
;
1089 unsigned long irq_flags
;
1090 struct ftrace_event_call
*call
= &tp
->call
;
1092 local_save_flags(irq_flags
);
1093 pc
= preempt_count();
1095 size
= sizeof(*entry
) + tp
->size
;
1097 event
= trace_current_buffer_lock_reserve(&buffer
, call
->id
, size
,
1102 entry
= ring_buffer_event_data(event
);
1103 entry
->func
= (unsigned long)tp
->rp
.kp
.addr
;
1104 entry
->ret_ip
= (unsigned long)ri
->ret_addr
;
1105 data
= (u8
*)&entry
[1];
1106 for (i
= 0; i
< tp
->nr_args
; i
++)
1107 call_fetch(&tp
->args
[i
].fetch
, regs
, data
+ tp
->args
[i
].offset
);
1109 if (!filter_current_check_discard(buffer
, call
, entry
, event
))
1110 trace_nowake_buffer_unlock_commit(buffer
, event
, irq_flags
, pc
);
1113 /* Event entry printers */
1115 print_kprobe_event(struct trace_iterator
*iter
, int flags
)
1117 struct kprobe_trace_entry_head
*field
;
1118 struct trace_seq
*s
= &iter
->seq
;
1119 struct trace_event
*event
;
1120 struct trace_probe
*tp
;
1124 field
= (struct kprobe_trace_entry_head
*)iter
->ent
;
1125 event
= ftrace_find_event(field
->ent
.type
);
1126 tp
= container_of(event
, struct trace_probe
, event
);
1128 if (!trace_seq_printf(s
, "%s: (", tp
->call
.name
))
1131 if (!seq_print_ip_sym(s
, field
->ip
, flags
| TRACE_ITER_SYM_OFFSET
))
1134 if (!trace_seq_puts(s
, ")"))
1137 data
= (u8
*)&field
[1];
1138 for (i
= 0; i
< tp
->nr_args
; i
++)
1139 if (!tp
->args
[i
].type
->print(s
, tp
->args
[i
].name
,
1140 data
+ tp
->args
[i
].offset
))
1143 if (!trace_seq_puts(s
, "\n"))
1146 return TRACE_TYPE_HANDLED
;
1148 return TRACE_TYPE_PARTIAL_LINE
;
1152 print_kretprobe_event(struct trace_iterator
*iter
, int flags
)
1154 struct kretprobe_trace_entry_head
*field
;
1155 struct trace_seq
*s
= &iter
->seq
;
1156 struct trace_event
*event
;
1157 struct trace_probe
*tp
;
1161 field
= (struct kretprobe_trace_entry_head
*)iter
->ent
;
1162 event
= ftrace_find_event(field
->ent
.type
);
1163 tp
= container_of(event
, struct trace_probe
, event
);
1165 if (!trace_seq_printf(s
, "%s: (", tp
->call
.name
))
1168 if (!seq_print_ip_sym(s
, field
->ret_ip
, flags
| TRACE_ITER_SYM_OFFSET
))
1171 if (!trace_seq_puts(s
, " <- "))
1174 if (!seq_print_ip_sym(s
, field
->func
, flags
& ~TRACE_ITER_SYM_OFFSET
))
1177 if (!trace_seq_puts(s
, ")"))
1180 data
= (u8
*)&field
[1];
1181 for (i
= 0; i
< tp
->nr_args
; i
++)
1182 if (!tp
->args
[i
].type
->print(s
, tp
->args
[i
].name
,
1183 data
+ tp
->args
[i
].offset
))
1186 if (!trace_seq_puts(s
, "\n"))
1189 return TRACE_TYPE_HANDLED
;
1191 return TRACE_TYPE_PARTIAL_LINE
;
1194 static int probe_event_enable(struct ftrace_event_call
*call
)
1196 struct trace_probe
*tp
= (struct trace_probe
*)call
->data
;
1198 tp
->flags
|= TP_FLAG_TRACE
;
1199 if (probe_is_return(tp
))
1200 return enable_kretprobe(&tp
->rp
);
1202 return enable_kprobe(&tp
->rp
.kp
);
1205 static void probe_event_disable(struct ftrace_event_call
*call
)
1207 struct trace_probe
*tp
= (struct trace_probe
*)call
->data
;
1209 tp
->flags
&= ~TP_FLAG_TRACE
;
1210 if (!(tp
->flags
& (TP_FLAG_TRACE
| TP_FLAG_PROFILE
))) {
1211 if (probe_is_return(tp
))
1212 disable_kretprobe(&tp
->rp
);
1214 disable_kprobe(&tp
->rp
.kp
);
1218 static int probe_event_raw_init(struct ftrace_event_call
*event_call
)
1220 INIT_LIST_HEAD(&event_call
->fields
);
1226 #define DEFINE_FIELD(type, item, name, is_signed) \
1228 ret = trace_define_field(event_call, #type, name, \
1229 offsetof(typeof(field), item), \
1230 sizeof(field.item), is_signed, \
1236 static int kprobe_event_define_fields(struct ftrace_event_call
*event_call
)
1239 struct kprobe_trace_entry_head field
;
1240 struct trace_probe
*tp
= (struct trace_probe
*)event_call
->data
;
1242 DEFINE_FIELD(unsigned long, ip
, FIELD_STRING_IP
, 0);
1243 /* Set argument names as fields */
1244 for (i
= 0; i
< tp
->nr_args
; i
++) {
1245 ret
= trace_define_field(event_call
, tp
->args
[i
].type
->name
,
1247 sizeof(field
) + tp
->args
[i
].offset
,
1248 tp
->args
[i
].type
->size
,
1249 tp
->args
[i
].type
->is_signed
,
1257 static int kretprobe_event_define_fields(struct ftrace_event_call
*event_call
)
1260 struct kretprobe_trace_entry_head field
;
1261 struct trace_probe
*tp
= (struct trace_probe
*)event_call
->data
;
1263 DEFINE_FIELD(unsigned long, func
, FIELD_STRING_FUNC
, 0);
1264 DEFINE_FIELD(unsigned long, ret_ip
, FIELD_STRING_RETIP
, 0);
1265 /* Set argument names as fields */
1266 for (i
= 0; i
< tp
->nr_args
; i
++) {
1267 ret
= trace_define_field(event_call
, tp
->args
[i
].type
->name
,
1269 sizeof(field
) + tp
->args
[i
].offset
,
1270 tp
->args
[i
].type
->size
,
1271 tp
->args
[i
].type
->is_signed
,
1279 static int __set_print_fmt(struct trace_probe
*tp
, char *buf
, int len
)
1284 const char *fmt
, *arg
;
1286 if (!probe_is_return(tp
)) {
1288 arg
= "REC->" FIELD_STRING_IP
;
1290 fmt
= "(%lx <- %lx)";
1291 arg
= "REC->" FIELD_STRING_FUNC
", REC->" FIELD_STRING_RETIP
;
1294 /* When len=0, we just calculate the needed length */
1295 #define LEN_OR_ZERO (len ? len - pos : 0)
1297 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
, "\"%s", fmt
);
1299 for (i
= 0; i
< tp
->nr_args
; i
++) {
1300 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
, " %s=%s",
1301 tp
->args
[i
].name
, tp
->args
[i
].type
->fmt
);
1304 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
, "\", %s", arg
);
1306 for (i
= 0; i
< tp
->nr_args
; i
++) {
1307 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
, ", REC->%s",
1313 /* return the length of print_fmt */
1317 static int set_print_fmt(struct trace_probe
*tp
)
1322 /* First: called with 0 length to calculate the needed length */
1323 len
= __set_print_fmt(tp
, NULL
, 0);
1324 print_fmt
= kmalloc(len
+ 1, GFP_KERNEL
);
1328 /* Second: actually write the @print_fmt */
1329 __set_print_fmt(tp
, print_fmt
, len
+ 1);
1330 tp
->call
.print_fmt
= print_fmt
;
1335 #ifdef CONFIG_PERF_EVENTS
1337 /* Kprobe profile handler */
1338 static __kprobes
void kprobe_perf_func(struct kprobe
*kp
,
1339 struct pt_regs
*regs
)
1341 struct trace_probe
*tp
= container_of(kp
, struct trace_probe
, rp
.kp
);
1342 struct ftrace_event_call
*call
= &tp
->call
;
1343 struct kprobe_trace_entry_head
*entry
;
1345 int size
, __size
, i
;
1346 unsigned long irq_flags
;
1349 __size
= sizeof(*entry
) + tp
->size
;
1350 size
= ALIGN(__size
+ sizeof(u32
), sizeof(u64
));
1351 size
-= sizeof(u32
);
1352 if (WARN_ONCE(size
> PERF_MAX_TRACE_SIZE
,
1353 "profile buffer not large enough"))
1356 entry
= perf_trace_buf_prepare(size
, call
->id
, &rctx
, &irq_flags
);
1360 entry
->ip
= (unsigned long)kp
->addr
;
1361 data
= (u8
*)&entry
[1];
1362 for (i
= 0; i
< tp
->nr_args
; i
++)
1363 call_fetch(&tp
->args
[i
].fetch
, regs
, data
+ tp
->args
[i
].offset
);
1365 perf_trace_buf_submit(entry
, size
, rctx
, entry
->ip
, 1, irq_flags
, regs
);
1368 /* Kretprobe profile handler */
1369 static __kprobes
void kretprobe_perf_func(struct kretprobe_instance
*ri
,
1370 struct pt_regs
*regs
)
1372 struct trace_probe
*tp
= container_of(ri
->rp
, struct trace_probe
, rp
);
1373 struct ftrace_event_call
*call
= &tp
->call
;
1374 struct kretprobe_trace_entry_head
*entry
;
1376 int size
, __size
, i
;
1377 unsigned long irq_flags
;
1380 __size
= sizeof(*entry
) + tp
->size
;
1381 size
= ALIGN(__size
+ sizeof(u32
), sizeof(u64
));
1382 size
-= sizeof(u32
);
1383 if (WARN_ONCE(size
> PERF_MAX_TRACE_SIZE
,
1384 "profile buffer not large enough"))
1387 entry
= perf_trace_buf_prepare(size
, call
->id
, &rctx
, &irq_flags
);
1391 entry
->func
= (unsigned long)tp
->rp
.kp
.addr
;
1392 entry
->ret_ip
= (unsigned long)ri
->ret_addr
;
1393 data
= (u8
*)&entry
[1];
1394 for (i
= 0; i
< tp
->nr_args
; i
++)
1395 call_fetch(&tp
->args
[i
].fetch
, regs
, data
+ tp
->args
[i
].offset
);
1397 perf_trace_buf_submit(entry
, size
, rctx
, entry
->ret_ip
, 1,
1401 static int probe_perf_enable(struct ftrace_event_call
*call
)
1403 struct trace_probe
*tp
= (struct trace_probe
*)call
->data
;
1405 tp
->flags
|= TP_FLAG_PROFILE
;
1407 if (probe_is_return(tp
))
1408 return enable_kretprobe(&tp
->rp
);
1410 return enable_kprobe(&tp
->rp
.kp
);
1413 static void probe_perf_disable(struct ftrace_event_call
*call
)
1415 struct trace_probe
*tp
= (struct trace_probe
*)call
->data
;
1417 tp
->flags
&= ~TP_FLAG_PROFILE
;
1419 if (!(tp
->flags
& TP_FLAG_TRACE
)) {
1420 if (probe_is_return(tp
))
1421 disable_kretprobe(&tp
->rp
);
1423 disable_kprobe(&tp
->rp
.kp
);
1426 #endif /* CONFIG_PERF_EVENTS */
1430 int kprobe_dispatcher(struct kprobe
*kp
, struct pt_regs
*regs
)
1432 struct trace_probe
*tp
= container_of(kp
, struct trace_probe
, rp
.kp
);
1434 if (tp
->flags
& TP_FLAG_TRACE
)
1435 kprobe_trace_func(kp
, regs
);
1436 #ifdef CONFIG_PERF_EVENTS
1437 if (tp
->flags
& TP_FLAG_PROFILE
)
1438 kprobe_perf_func(kp
, regs
);
1440 return 0; /* We don't tweek kernel, so just return 0 */
1444 int kretprobe_dispatcher(struct kretprobe_instance
*ri
, struct pt_regs
*regs
)
1446 struct trace_probe
*tp
= container_of(ri
->rp
, struct trace_probe
, rp
);
1448 if (tp
->flags
& TP_FLAG_TRACE
)
1449 kretprobe_trace_func(ri
, regs
);
1450 #ifdef CONFIG_PERF_EVENTS
1451 if (tp
->flags
& TP_FLAG_PROFILE
)
1452 kretprobe_perf_func(ri
, regs
);
1454 return 0; /* We don't tweek kernel, so just return 0 */
1457 static int register_probe_event(struct trace_probe
*tp
)
1459 struct ftrace_event_call
*call
= &tp
->call
;
1462 /* Initialize ftrace_event_call */
1463 if (probe_is_return(tp
)) {
1464 tp
->event
.trace
= print_kretprobe_event
;
1465 call
->raw_init
= probe_event_raw_init
;
1466 call
->define_fields
= kretprobe_event_define_fields
;
1468 tp
->event
.trace
= print_kprobe_event
;
1469 call
->raw_init
= probe_event_raw_init
;
1470 call
->define_fields
= kprobe_event_define_fields
;
1472 if (set_print_fmt(tp
) < 0)
1474 call
->event
= &tp
->event
;
1475 call
->id
= register_ftrace_event(&tp
->event
);
1477 kfree(call
->print_fmt
);
1481 call
->regfunc
= probe_event_enable
;
1482 call
->unregfunc
= probe_event_disable
;
1484 #ifdef CONFIG_PERF_EVENTS
1485 call
->perf_event_enable
= probe_perf_enable
;
1486 call
->perf_event_disable
= probe_perf_disable
;
1489 ret
= trace_add_event_call(call
);
1491 pr_info("Failed to register kprobe event: %s\n", call
->name
);
1492 kfree(call
->print_fmt
);
1493 unregister_ftrace_event(&tp
->event
);
1498 static void unregister_probe_event(struct trace_probe
*tp
)
1500 /* tp->event is unregistered in trace_remove_event_call() */
1501 trace_remove_event_call(&tp
->call
);
1502 kfree(tp
->call
.print_fmt
);
1505 /* Make a debugfs interface for controling probe points */
1506 static __init
int init_kprobe_trace(void)
1508 struct dentry
*d_tracer
;
1509 struct dentry
*entry
;
1511 d_tracer
= tracing_init_dentry();
1515 entry
= debugfs_create_file("kprobe_events", 0644, d_tracer
,
1516 NULL
, &kprobe_events_ops
);
1518 /* Event list interface */
1520 pr_warning("Could not create debugfs "
1521 "'kprobe_events' entry\n");
1523 /* Profile interface */
1524 entry
= debugfs_create_file("kprobe_profile", 0444, d_tracer
,
1525 NULL
, &kprobe_profile_ops
);
1528 pr_warning("Could not create debugfs "
1529 "'kprobe_profile' entry\n");
1532 fs_initcall(init_kprobe_trace
);
1535 #ifdef CONFIG_FTRACE_STARTUP_TEST
1537 static int kprobe_trace_selftest_target(int a1
, int a2
, int a3
,
1538 int a4
, int a5
, int a6
)
1540 return a1
+ a2
+ a3
+ a4
+ a5
+ a6
;
1543 static __init
int kprobe_trace_self_tests_init(void)
1546 int (*target
)(int, int, int, int, int, int);
1547 struct trace_probe
*tp
;
1549 target
= kprobe_trace_selftest_target
;
1551 pr_info("Testing kprobe tracing: ");
1553 ret
= command_trace_probe("p:testprobe kprobe_trace_selftest_target "
1554 "$stack $stack0 +0($stack)");
1555 if (WARN_ON_ONCE(ret
)) {
1556 pr_warning("error on probing function entry.\n");
1559 /* Enable trace point */
1560 tp
= find_probe_event("testprobe", KPROBE_EVENT_SYSTEM
);
1561 if (WARN_ON_ONCE(tp
== NULL
)) {
1562 pr_warning("error on getting new probe.\n");
1565 probe_event_enable(&tp
->call
);
1568 ret
= command_trace_probe("r:testprobe2 kprobe_trace_selftest_target "
1570 if (WARN_ON_ONCE(ret
)) {
1571 pr_warning("error on probing function return.\n");
1574 /* Enable trace point */
1575 tp
= find_probe_event("testprobe2", KPROBE_EVENT_SYSTEM
);
1576 if (WARN_ON_ONCE(tp
== NULL
)) {
1577 pr_warning("error on getting new probe.\n");
1580 probe_event_enable(&tp
->call
);
1586 ret
= target(1, 2, 3, 4, 5, 6);
1588 ret
= command_trace_probe("-:testprobe");
1589 if (WARN_ON_ONCE(ret
)) {
1590 pr_warning("error on deleting a probe.\n");
1594 ret
= command_trace_probe("-:testprobe2");
1595 if (WARN_ON_ONCE(ret
)) {
1596 pr_warning("error on deleting a probe.\n");
1601 cleanup_all_probes();
1603 pr_cont("NG: Some tests are failed. Please check them.\n");
1609 late_initcall(kprobe_trace_self_tests_init
);