2 * Kprobes-based tracing events
4 * Created by Masami Hiramatsu <mhiramat@redhat.com>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 #include <linux/module.h>
21 #include <linux/uaccess.h>
22 #include <linux/kprobes.h>
23 #include <linux/seq_file.h>
24 #include <linux/slab.h>
25 #include <linux/smp.h>
26 #include <linux/debugfs.h>
27 #include <linux/types.h>
28 #include <linux/string.h>
29 #include <linux/ctype.h>
30 #include <linux/ptrace.h>
31 #include <linux/perf_event.h>
32 #include <linux/stringify.h>
33 #include <linux/limits.h>
34 #include <linux/uaccess.h>
35 #include <asm/bitsperlong.h>
38 #include "trace_output.h"
40 #define MAX_TRACE_ARGS 128
41 #define MAX_ARGSTR_LEN 63
42 #define MAX_EVENT_NAME_LEN 64
43 #define MAX_STRING_SIZE PATH_MAX
44 #define KPROBE_EVENT_SYSTEM "kprobes"
46 /* Reserved field names */
47 #define FIELD_STRING_IP "__probe_ip"
48 #define FIELD_STRING_RETIP "__probe_ret_ip"
49 #define FIELD_STRING_FUNC "__probe_func"
51 const char *reserved_field_names
[] = {
54 "common_preempt_count",
63 /* Printing function type */
64 typedef int (*print_type_func_t
)(struct trace_seq
*, const char *, void *,
66 #define PRINT_TYPE_FUNC_NAME(type) print_type_##type
67 #define PRINT_TYPE_FMT_NAME(type) print_type_format_##type
69 /* Printing in basic type function template */
70 #define DEFINE_BASIC_PRINT_TYPE_FUNC(type, fmt, cast) \
71 static __kprobes int PRINT_TYPE_FUNC_NAME(type)(struct trace_seq *s, \
73 void *data, void *ent)\
75 return trace_seq_printf(s, " %s=" fmt, name, (cast)*(type *)data);\
77 static const char PRINT_TYPE_FMT_NAME(type)[] = fmt;
79 DEFINE_BASIC_PRINT_TYPE_FUNC(u8
, "%x", unsigned int)
80 DEFINE_BASIC_PRINT_TYPE_FUNC(u16
, "%x", unsigned int)
81 DEFINE_BASIC_PRINT_TYPE_FUNC(u32
, "%lx", unsigned long)
82 DEFINE_BASIC_PRINT_TYPE_FUNC(u64
, "%llx", unsigned long long)
83 DEFINE_BASIC_PRINT_TYPE_FUNC(s8
, "%d", int)
84 DEFINE_BASIC_PRINT_TYPE_FUNC(s16
, "%d", int)
85 DEFINE_BASIC_PRINT_TYPE_FUNC(s32
, "%ld", long)
86 DEFINE_BASIC_PRINT_TYPE_FUNC(s64
, "%lld", long long)
88 /* data_rloc: data relative location, compatible with u32 */
89 #define make_data_rloc(len, roffs) \
90 (((u32)(len) << 16) | ((u32)(roffs) & 0xffff))
91 #define get_rloc_len(dl) ((u32)(dl) >> 16)
92 #define get_rloc_offs(dl) ((u32)(dl) & 0xffff)
94 static inline void *get_rloc_data(u32
*dl
)
96 return (u8
*)dl
+ get_rloc_offs(*dl
);
99 /* For data_loc conversion */
100 static inline void *get_loc_data(u32
*dl
, void *ent
)
102 return (u8
*)ent
+ get_rloc_offs(*dl
);
106 * Convert data_rloc to data_loc:
107 * data_rloc stores the offset from data_rloc itself, but data_loc
108 * stores the offset from event entry.
110 #define convert_rloc_to_loc(dl, offs) ((u32)(dl) + (offs))
112 /* For defining macros, define string/string_size types */
114 typedef u32 string_size
;
116 /* Print type function for string type */
117 static __kprobes
int PRINT_TYPE_FUNC_NAME(string
)(struct trace_seq
*s
,
119 void *data
, void *ent
)
121 int len
= *(u32
*)data
>> 16;
124 return trace_seq_printf(s
, " %s=(fault)", name
);
126 return trace_seq_printf(s
, " %s=\"%s\"", name
,
127 (const char *)get_loc_data(data
, ent
));
129 static const char PRINT_TYPE_FMT_NAME(string
)[] = "\\\"%s\\\"";
131 /* Data fetch function type */
132 typedef void (*fetch_func_t
)(struct pt_regs
*, void *, void *);
139 static __kprobes
void call_fetch(struct fetch_param
*fprm
,
140 struct pt_regs
*regs
, void *dest
)
142 return fprm
->fn(regs
, fprm
->data
, dest
);
145 #define FETCH_FUNC_NAME(method, type) fetch_##method##_##type
147 * Define macro for basic types - we don't need to define s* types, because
148 * we have to care only about bitwidth at recording time.
150 #define DEFINE_BASIC_FETCH_FUNCS(method) \
151 DEFINE_FETCH_##method(u8) \
152 DEFINE_FETCH_##method(u16) \
153 DEFINE_FETCH_##method(u32) \
154 DEFINE_FETCH_##method(u64)
156 #define CHECK_FETCH_FUNCS(method, fn) \
157 (((FETCH_FUNC_NAME(method, u8) == fn) || \
158 (FETCH_FUNC_NAME(method, u16) == fn) || \
159 (FETCH_FUNC_NAME(method, u32) == fn) || \
160 (FETCH_FUNC_NAME(method, u64) == fn) || \
161 (FETCH_FUNC_NAME(method, string) == fn) || \
162 (FETCH_FUNC_NAME(method, string_size) == fn)) \
165 /* Data fetch function templates */
166 #define DEFINE_FETCH_reg(type) \
167 static __kprobes void FETCH_FUNC_NAME(reg, type)(struct pt_regs *regs, \
168 void *offset, void *dest) \
170 *(type *)dest = (type)regs_get_register(regs, \
171 (unsigned int)((unsigned long)offset)); \
173 DEFINE_BASIC_FETCH_FUNCS(reg
)
174 /* No string on the register */
175 #define fetch_reg_string NULL
176 #define fetch_reg_string_size NULL
178 #define DEFINE_FETCH_stack(type) \
179 static __kprobes void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs,\
180 void *offset, void *dest) \
182 *(type *)dest = (type)regs_get_kernel_stack_nth(regs, \
183 (unsigned int)((unsigned long)offset)); \
185 DEFINE_BASIC_FETCH_FUNCS(stack
)
186 /* No string on the stack entry */
187 #define fetch_stack_string NULL
188 #define fetch_stack_string_size NULL
190 #define DEFINE_FETCH_retval(type) \
191 static __kprobes void FETCH_FUNC_NAME(retval, type)(struct pt_regs *regs,\
192 void *dummy, void *dest) \
194 *(type *)dest = (type)regs_return_value(regs); \
196 DEFINE_BASIC_FETCH_FUNCS(retval
)
197 /* No string on the retval */
198 #define fetch_retval_string NULL
199 #define fetch_retval_string_size NULL
201 #define DEFINE_FETCH_memory(type) \
202 static __kprobes void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs,\
203 void *addr, void *dest) \
206 if (probe_kernel_address(addr, retval)) \
209 *(type *)dest = retval; \
211 DEFINE_BASIC_FETCH_FUNCS(memory
)
213 * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max
214 * length and relative data location.
216 static __kprobes
void FETCH_FUNC_NAME(memory
, string
)(struct pt_regs
*regs
,
217 void *addr
, void *dest
)
220 int maxlen
= get_rloc_len(*(u32
*)dest
);
221 u8
*dst
= get_rloc_data(dest
);
223 mm_segment_t old_fs
= get_fs();
227 * Try to get string again, since the string can be changed while
233 ret
= __copy_from_user_inatomic(dst
++, src
++, 1);
234 while (dst
[-1] && ret
== 0 && src
- (u8
*)addr
< maxlen
);
239 if (ret
< 0) { /* Failed to fetch string */
240 ((u8
*)get_rloc_data(dest
))[0] = '\0';
241 *(u32
*)dest
= make_data_rloc(0, get_rloc_offs(*(u32
*)dest
));
243 *(u32
*)dest
= make_data_rloc(src
- (u8
*)addr
,
244 get_rloc_offs(*(u32
*)dest
));
246 /* Return the length of string -- including null terminal byte */
247 static __kprobes
void FETCH_FUNC_NAME(memory
, string_size
)(struct pt_regs
*regs
,
248 void *addr
, void *dest
)
252 mm_segment_t old_fs
= get_fs();
257 ret
= __copy_from_user_inatomic(&c
, (u8
*)addr
+ len
, 1);
259 } while (c
&& ret
== 0 && len
< MAX_STRING_SIZE
);
263 if (ret
< 0) /* Failed to check the length */
269 /* Memory fetching by symbol */
270 struct symbol_cache
{
276 static unsigned long update_symbol_cache(struct symbol_cache
*sc
)
278 sc
->addr
= (unsigned long)kallsyms_lookup_name(sc
->symbol
);
280 sc
->addr
+= sc
->offset
;
284 static void free_symbol_cache(struct symbol_cache
*sc
)
290 static struct symbol_cache
*alloc_symbol_cache(const char *sym
, long offset
)
292 struct symbol_cache
*sc
;
294 if (!sym
|| strlen(sym
) == 0)
296 sc
= kzalloc(sizeof(struct symbol_cache
), GFP_KERNEL
);
300 sc
->symbol
= kstrdup(sym
, GFP_KERNEL
);
307 update_symbol_cache(sc
);
311 #define DEFINE_FETCH_symbol(type) \
312 static __kprobes void FETCH_FUNC_NAME(symbol, type)(struct pt_regs *regs,\
313 void *data, void *dest) \
315 struct symbol_cache *sc = data; \
317 fetch_memory_##type(regs, (void *)sc->addr, dest); \
321 DEFINE_BASIC_FETCH_FUNCS(symbol
)
322 DEFINE_FETCH_symbol(string
)
323 DEFINE_FETCH_symbol(string_size
)
325 /* Dereference memory access function */
326 struct deref_fetch_param
{
327 struct fetch_param orig
;
331 #define DEFINE_FETCH_deref(type) \
332 static __kprobes void FETCH_FUNC_NAME(deref, type)(struct pt_regs *regs,\
333 void *data, void *dest) \
335 struct deref_fetch_param *dprm = data; \
336 unsigned long addr; \
337 call_fetch(&dprm->orig, regs, &addr); \
339 addr += dprm->offset; \
340 fetch_memory_##type(regs, (void *)addr, dest); \
344 DEFINE_BASIC_FETCH_FUNCS(deref
)
345 DEFINE_FETCH_deref(string
)
346 DEFINE_FETCH_deref(string_size
)
348 static __kprobes
void free_deref_fetch_param(struct deref_fetch_param
*data
)
350 if (CHECK_FETCH_FUNCS(deref
, data
->orig
.fn
))
351 free_deref_fetch_param(data
->orig
.data
);
352 else if (CHECK_FETCH_FUNCS(symbol
, data
->orig
.fn
))
353 free_symbol_cache(data
->orig
.data
);
357 /* Default (unsigned long) fetch type */
358 #define __DEFAULT_FETCH_TYPE(t) u##t
359 #define _DEFAULT_FETCH_TYPE(t) __DEFAULT_FETCH_TYPE(t)
360 #define DEFAULT_FETCH_TYPE _DEFAULT_FETCH_TYPE(BITS_PER_LONG)
361 #define DEFAULT_FETCH_TYPE_STR __stringify(DEFAULT_FETCH_TYPE)
374 #define ASSIGN_FETCH_FUNC(method, type) \
375 [FETCH_MTD_##method] = FETCH_FUNC_NAME(method, type)
377 #define __ASSIGN_FETCH_TYPE(_name, ptype, ftype, _size, sign, _fmttype) \
381 .print = PRINT_TYPE_FUNC_NAME(ptype), \
382 .fmt = PRINT_TYPE_FMT_NAME(ptype), \
383 .fmttype = _fmttype, \
385 ASSIGN_FETCH_FUNC(reg, ftype), \
386 ASSIGN_FETCH_FUNC(stack, ftype), \
387 ASSIGN_FETCH_FUNC(retval, ftype), \
388 ASSIGN_FETCH_FUNC(memory, ftype), \
389 ASSIGN_FETCH_FUNC(symbol, ftype), \
390 ASSIGN_FETCH_FUNC(deref, ftype), \
394 #define ASSIGN_FETCH_TYPE(ptype, ftype, sign) \
395 __ASSIGN_FETCH_TYPE(#ptype, ptype, ftype, sizeof(ftype), sign, #ptype)
397 #define FETCH_TYPE_STRING 0
398 #define FETCH_TYPE_STRSIZE 1
400 /* Fetch type information table */
401 static const struct fetch_type
{
402 const char *name
; /* Name of type */
403 size_t size
; /* Byte size of type */
404 int is_signed
; /* Signed flag */
405 print_type_func_t print
; /* Print functions */
406 const char *fmt
; /* Fromat string */
407 const char *fmttype
; /* Name in format file */
408 /* Fetch functions */
409 fetch_func_t fetch
[FETCH_MTD_END
];
410 } fetch_type_table
[] = {
412 [FETCH_TYPE_STRING
] = __ASSIGN_FETCH_TYPE("string", string
, string
,
413 sizeof(u32
), 1, "__data_loc char[]"),
414 [FETCH_TYPE_STRSIZE
] = __ASSIGN_FETCH_TYPE("string_size", u32
,
415 string_size
, sizeof(u32
), 0, "u32"),
417 ASSIGN_FETCH_TYPE(u8
, u8
, 0),
418 ASSIGN_FETCH_TYPE(u16
, u16
, 0),
419 ASSIGN_FETCH_TYPE(u32
, u32
, 0),
420 ASSIGN_FETCH_TYPE(u64
, u64
, 0),
421 ASSIGN_FETCH_TYPE(s8
, u8
, 1),
422 ASSIGN_FETCH_TYPE(s16
, u16
, 1),
423 ASSIGN_FETCH_TYPE(s32
, u32
, 1),
424 ASSIGN_FETCH_TYPE(s64
, u64
, 1),
427 static const struct fetch_type
*find_fetch_type(const char *type
)
432 type
= DEFAULT_FETCH_TYPE_STR
;
434 for (i
= 0; i
< ARRAY_SIZE(fetch_type_table
); i
++)
435 if (strcmp(type
, fetch_type_table
[i
].name
) == 0)
436 return &fetch_type_table
[i
];
440 /* Special function : only accept unsigned long */
441 static __kprobes
void fetch_stack_address(struct pt_regs
*regs
,
442 void *dummy
, void *dest
)
444 *(unsigned long *)dest
= kernel_stack_pointer(regs
);
447 static fetch_func_t
get_fetch_size_function(const struct fetch_type
*type
,
448 fetch_func_t orig_fn
)
452 if (type
!= &fetch_type_table
[FETCH_TYPE_STRING
])
453 return NULL
; /* Only string type needs size function */
454 for (i
= 0; i
< FETCH_MTD_END
; i
++)
455 if (type
->fetch
[i
] == orig_fn
)
456 return fetch_type_table
[FETCH_TYPE_STRSIZE
].fetch
[i
];
458 WARN_ON(1); /* This should not happen */
463 * Kprobe event core functions
467 struct fetch_param fetch
;
468 struct fetch_param fetch_size
;
469 unsigned int offset
; /* Offset from argument entry */
470 const char *name
; /* Name of this argument */
471 const char *comm
; /* Command of this argument */
472 const struct fetch_type
*type
; /* Type of this argument */
475 /* Flags for trace_probe */
476 #define TP_FLAG_TRACE 1
477 #define TP_FLAG_PROFILE 2
480 struct list_head list
;
481 struct kretprobe rp
; /* Use rp.kp for kprobe use */
483 unsigned int flags
; /* For TP_FLAG_* */
484 const char *symbol
; /* symbol name */
485 struct ftrace_event_class
class;
486 struct ftrace_event_call call
;
487 ssize_t size
; /* trace entry size */
488 unsigned int nr_args
;
489 struct probe_arg args
[];
492 #define SIZEOF_TRACE_PROBE(n) \
493 (offsetof(struct trace_probe, args) + \
494 (sizeof(struct probe_arg) * (n)))
497 static __kprobes
int probe_is_return(struct trace_probe
*tp
)
499 return tp
->rp
.handler
!= NULL
;
502 static __kprobes
const char *probe_symbol(struct trace_probe
*tp
)
504 return tp
->symbol
? tp
->symbol
: "unknown";
507 static int register_probe_event(struct trace_probe
*tp
);
508 static void unregister_probe_event(struct trace_probe
*tp
);
510 static DEFINE_MUTEX(probe_lock
);
511 static LIST_HEAD(probe_list
);
513 static int kprobe_dispatcher(struct kprobe
*kp
, struct pt_regs
*regs
);
514 static int kretprobe_dispatcher(struct kretprobe_instance
*ri
,
515 struct pt_regs
*regs
);
517 /* Check the name is good for event/group/fields */
518 static int is_good_name(const char *name
)
520 if (!isalpha(*name
) && *name
!= '_')
522 while (*++name
!= '\0') {
523 if (!isalpha(*name
) && !isdigit(*name
) && *name
!= '_')
530 * Allocate new trace_probe and initialize it (including kprobes).
532 static struct trace_probe
*alloc_trace_probe(const char *group
,
537 int nargs
, int is_return
)
539 struct trace_probe
*tp
;
542 tp
= kzalloc(SIZEOF_TRACE_PROBE(nargs
), GFP_KERNEL
);
547 tp
->symbol
= kstrdup(symbol
, GFP_KERNEL
);
550 tp
->rp
.kp
.symbol_name
= tp
->symbol
;
551 tp
->rp
.kp
.offset
= offs
;
553 tp
->rp
.kp
.addr
= addr
;
556 tp
->rp
.handler
= kretprobe_dispatcher
;
558 tp
->rp
.kp
.pre_handler
= kprobe_dispatcher
;
560 if (!event
|| !is_good_name(event
)) {
565 tp
->call
.class = &tp
->class;
566 tp
->call
.name
= kstrdup(event
, GFP_KERNEL
);
570 if (!group
|| !is_good_name(group
)) {
575 tp
->class.system
= kstrdup(group
, GFP_KERNEL
);
576 if (!tp
->class.system
)
579 INIT_LIST_HEAD(&tp
->list
);
582 kfree(tp
->call
.name
);
588 static void free_probe_arg(struct probe_arg
*arg
)
590 if (CHECK_FETCH_FUNCS(deref
, arg
->fetch
.fn
))
591 free_deref_fetch_param(arg
->fetch
.data
);
592 else if (CHECK_FETCH_FUNCS(symbol
, arg
->fetch
.fn
))
593 free_symbol_cache(arg
->fetch
.data
);
598 static void free_trace_probe(struct trace_probe
*tp
)
602 for (i
= 0; i
< tp
->nr_args
; i
++)
603 free_probe_arg(&tp
->args
[i
]);
605 kfree(tp
->call
.class->system
);
606 kfree(tp
->call
.name
);
611 static struct trace_probe
*find_probe_event(const char *event
,
614 struct trace_probe
*tp
;
616 list_for_each_entry(tp
, &probe_list
, list
)
617 if (strcmp(tp
->call
.name
, event
) == 0 &&
618 strcmp(tp
->call
.class->system
, group
) == 0)
623 /* Unregister a trace_probe and probe_event: call with locking probe_lock */
624 static void unregister_trace_probe(struct trace_probe
*tp
)
626 if (probe_is_return(tp
))
627 unregister_kretprobe(&tp
->rp
);
629 unregister_kprobe(&tp
->rp
.kp
);
631 unregister_probe_event(tp
);
634 /* Register a trace_probe and probe_event */
635 static int register_trace_probe(struct trace_probe
*tp
)
637 struct trace_probe
*old_tp
;
640 mutex_lock(&probe_lock
);
642 /* register as an event */
643 old_tp
= find_probe_event(tp
->call
.name
, tp
->call
.class->system
);
645 /* delete old event */
646 unregister_trace_probe(old_tp
);
647 free_trace_probe(old_tp
);
649 ret
= register_probe_event(tp
);
651 pr_warning("Faild to register probe event(%d)\n", ret
);
655 tp
->rp
.kp
.flags
|= KPROBE_FLAG_DISABLED
;
656 if (probe_is_return(tp
))
657 ret
= register_kretprobe(&tp
->rp
);
659 ret
= register_kprobe(&tp
->rp
.kp
);
662 pr_warning("Could not insert probe(%d)\n", ret
);
663 if (ret
== -EILSEQ
) {
664 pr_warning("Probing address(0x%p) is not an "
665 "instruction boundary.\n",
669 unregister_probe_event(tp
);
671 list_add_tail(&tp
->list
, &probe_list
);
673 mutex_unlock(&probe_lock
);
677 /* Split symbol and offset. */
678 static int split_symbol_offset(char *symbol
, unsigned long *offset
)
686 tmp
= strchr(symbol
, '+');
688 /* skip sign because strict_strtol doesn't accept '+' */
689 ret
= strict_strtoul(tmp
+ 1, 0, offset
);
698 #define PARAM_MAX_ARGS 16
699 #define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long))
701 static int parse_probe_vars(char *arg
, const struct fetch_type
*t
,
702 struct fetch_param
*f
, int is_return
)
707 if (strcmp(arg
, "retval") == 0) {
709 f
->fn
= t
->fetch
[FETCH_MTD_retval
];
712 } else if (strncmp(arg
, "stack", 5) == 0) {
713 if (arg
[5] == '\0') {
714 if (strcmp(t
->name
, DEFAULT_FETCH_TYPE_STR
) == 0)
715 f
->fn
= fetch_stack_address
;
718 } else if (isdigit(arg
[5])) {
719 ret
= strict_strtoul(arg
+ 5, 10, ¶m
);
720 if (ret
|| param
> PARAM_MAX_STACK
)
723 f
->fn
= t
->fetch
[FETCH_MTD_stack
];
724 f
->data
= (void *)param
;
733 /* Recursive argument parser */
734 static int __parse_probe_arg(char *arg
, const struct fetch_type
*t
,
735 struct fetch_param
*f
, int is_return
)
744 ret
= parse_probe_vars(arg
+ 1, t
, f
, is_return
);
746 case '%': /* named register */
747 ret
= regs_query_register_offset(arg
+ 1);
749 f
->fn
= t
->fetch
[FETCH_MTD_reg
];
750 f
->data
= (void *)(unsigned long)ret
;
754 case '@': /* memory or symbol */
755 if (isdigit(arg
[1])) {
756 ret
= strict_strtoul(arg
+ 1, 0, ¶m
);
759 f
->fn
= t
->fetch
[FETCH_MTD_memory
];
760 f
->data
= (void *)param
;
762 ret
= split_symbol_offset(arg
+ 1, &offset
);
765 f
->data
= alloc_symbol_cache(arg
+ 1, offset
);
767 f
->fn
= t
->fetch
[FETCH_MTD_symbol
];
770 case '+': /* deref memory */
772 tmp
= strchr(arg
, '(');
776 ret
= strict_strtol(arg
+ 1, 0, &offset
);
782 tmp
= strrchr(arg
, ')');
784 struct deref_fetch_param
*dprm
;
785 const struct fetch_type
*t2
= find_fetch_type(NULL
);
787 dprm
= kzalloc(sizeof(struct deref_fetch_param
),
791 dprm
->offset
= offset
;
792 ret
= __parse_probe_arg(arg
, t2
, &dprm
->orig
,
797 f
->fn
= t
->fetch
[FETCH_MTD_deref
];
798 f
->data
= (void *)dprm
;
803 if (!ret
&& !f
->fn
) { /* Parsed, but do not find fetch method */
804 pr_info("%s type has no corresponding fetch method.\n",
811 /* String length checking wrapper */
812 static int parse_probe_arg(char *arg
, struct trace_probe
*tp
,
813 struct probe_arg
*parg
, int is_return
)
818 if (strlen(arg
) > MAX_ARGSTR_LEN
) {
819 pr_info("Argument is too long.: %s\n", arg
);
822 parg
->comm
= kstrdup(arg
, GFP_KERNEL
);
824 pr_info("Failed to allocate memory for command '%s'.\n", arg
);
827 t
= strchr(parg
->comm
, ':');
829 arg
[t
- parg
->comm
] = '\0';
832 parg
->type
= find_fetch_type(t
);
834 pr_info("Unsupported type: %s\n", t
);
837 parg
->offset
= tp
->size
;
838 tp
->size
+= parg
->type
->size
;
839 ret
= __parse_probe_arg(arg
, parg
->type
, &parg
->fetch
, is_return
);
841 parg
->fetch_size
.fn
= get_fetch_size_function(parg
->type
,
843 parg
->fetch_size
.data
= parg
->fetch
.data
;
848 /* Return 1 if name is reserved or already used by another argument */
849 static int conflict_field_name(const char *name
,
850 struct probe_arg
*args
, int narg
)
853 for (i
= 0; i
< ARRAY_SIZE(reserved_field_names
); i
++)
854 if (strcmp(reserved_field_names
[i
], name
) == 0)
856 for (i
= 0; i
< narg
; i
++)
857 if (strcmp(args
[i
].name
, name
) == 0)
862 static int create_trace_probe(int argc
, char **argv
)
866 * - Add kprobe: p[:[GRP/]EVENT] KSYM[+OFFS]|KADDR [FETCHARGS]
867 * - Add kretprobe: r[:[GRP/]EVENT] KSYM[+0] [FETCHARGS]
869 * $retval : fetch return value
870 * $stack : fetch stack address
871 * $stackN : fetch Nth of stack (N:0-)
872 * @ADDR : fetch memory at ADDR (ADDR should be in kernel)
873 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)
874 * %REG : fetch register REG
875 * Dereferencing memory fetch:
876 * +|-offs(ARG) : fetch memory at ARG +|- offs address.
877 * Alias name of args:
878 * NAME=FETCHARG : set NAME as alias of FETCHARG.
880 * FETCHARG:TYPE : use TYPE instead of unsigned long.
882 struct trace_probe
*tp
;
884 int is_return
= 0, is_delete
= 0;
885 char *symbol
= NULL
, *event
= NULL
, *group
= NULL
;
887 unsigned long offset
= 0;
889 char buf
[MAX_EVENT_NAME_LEN
];
891 /* argc must be >= 1 */
892 if (argv
[0][0] == 'p')
894 else if (argv
[0][0] == 'r')
896 else if (argv
[0][0] == '-')
899 pr_info("Probe definition must be started with 'p', 'r' or"
904 if (argv
[0][1] == ':') {
906 if (strchr(event
, '/')) {
908 event
= strchr(group
, '/') + 1;
910 if (strlen(group
) == 0) {
911 pr_info("Group name is not specified\n");
915 if (strlen(event
) == 0) {
916 pr_info("Event name is not specified\n");
921 group
= KPROBE_EVENT_SYSTEM
;
925 pr_info("Delete command needs an event name.\n");
928 mutex_lock(&probe_lock
);
929 tp
= find_probe_event(event
, group
);
931 mutex_unlock(&probe_lock
);
932 pr_info("Event %s/%s doesn't exist.\n", group
, event
);
935 /* delete an event */
936 unregister_trace_probe(tp
);
937 free_trace_probe(tp
);
938 mutex_unlock(&probe_lock
);
943 pr_info("Probe point is not specified.\n");
946 if (isdigit(argv
[1][0])) {
948 pr_info("Return probe point must be a symbol.\n");
951 /* an address specified */
952 ret
= strict_strtoul(&argv
[1][0], 0, (unsigned long *)&addr
);
954 pr_info("Failed to parse address.\n");
958 /* a symbol specified */
960 /* TODO: support .init module functions */
961 ret
= split_symbol_offset(symbol
, &offset
);
963 pr_info("Failed to parse symbol.\n");
966 if (offset
&& is_return
) {
967 pr_info("Return probe must be used without offset.\n");
971 argc
-= 2; argv
+= 2;
975 /* Make a new event name */
977 snprintf(buf
, MAX_EVENT_NAME_LEN
, "%c_%s_%ld",
978 is_return
? 'r' : 'p', symbol
, offset
);
980 snprintf(buf
, MAX_EVENT_NAME_LEN
, "%c_0x%p",
981 is_return
? 'r' : 'p', addr
);
984 tp
= alloc_trace_probe(group
, event
, addr
, symbol
, offset
, argc
,
987 pr_info("Failed to allocate trace_probe.(%d)\n",
992 /* parse arguments */
994 for (i
= 0; i
< argc
&& i
< MAX_TRACE_ARGS
; i
++) {
995 /* Increment count for freeing args in error case */
998 /* Parse argument name */
999 arg
= strchr(argv
[i
], '=');
1002 tp
->args
[i
].name
= kstrdup(argv
[i
], GFP_KERNEL
);
1005 /* If argument name is omitted, set "argN" */
1006 snprintf(buf
, MAX_EVENT_NAME_LEN
, "arg%d", i
+ 1);
1007 tp
->args
[i
].name
= kstrdup(buf
, GFP_KERNEL
);
1010 if (!tp
->args
[i
].name
) {
1011 pr_info("Failed to allocate argument[%d] name.\n", i
);
1016 if (!is_good_name(tp
->args
[i
].name
)) {
1017 pr_info("Invalid argument[%d] name: %s\n",
1018 i
, tp
->args
[i
].name
);
1023 if (conflict_field_name(tp
->args
[i
].name
, tp
->args
, i
)) {
1024 pr_info("Argument[%d] name '%s' conflicts with "
1025 "another field.\n", i
, argv
[i
]);
1030 /* Parse fetch argument */
1031 ret
= parse_probe_arg(arg
, tp
, &tp
->args
[i
], is_return
);
1033 pr_info("Parse error at argument[%d]. (%d)\n", i
, ret
);
1038 ret
= register_trace_probe(tp
);
1044 free_trace_probe(tp
);
1048 static void cleanup_all_probes(void)
1050 struct trace_probe
*tp
;
1052 mutex_lock(&probe_lock
);
1053 /* TODO: Use batch unregistration */
1054 while (!list_empty(&probe_list
)) {
1055 tp
= list_entry(probe_list
.next
, struct trace_probe
, list
);
1056 unregister_trace_probe(tp
);
1057 free_trace_probe(tp
);
1059 mutex_unlock(&probe_lock
);
1063 /* Probes listing interfaces */
1064 static void *probes_seq_start(struct seq_file
*m
, loff_t
*pos
)
1066 mutex_lock(&probe_lock
);
1067 return seq_list_start(&probe_list
, *pos
);
1070 static void *probes_seq_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
1072 return seq_list_next(v
, &probe_list
, pos
);
1075 static void probes_seq_stop(struct seq_file
*m
, void *v
)
1077 mutex_unlock(&probe_lock
);
1080 static int probes_seq_show(struct seq_file
*m
, void *v
)
1082 struct trace_probe
*tp
= v
;
1085 seq_printf(m
, "%c", probe_is_return(tp
) ? 'r' : 'p');
1086 seq_printf(m
, ":%s/%s", tp
->call
.class->system
, tp
->call
.name
);
1089 seq_printf(m
, " 0x%p", tp
->rp
.kp
.addr
);
1090 else if (tp
->rp
.kp
.offset
)
1091 seq_printf(m
, " %s+%u", probe_symbol(tp
), tp
->rp
.kp
.offset
);
1093 seq_printf(m
, " %s", probe_symbol(tp
));
1095 for (i
= 0; i
< tp
->nr_args
; i
++)
1096 seq_printf(m
, " %s=%s", tp
->args
[i
].name
, tp
->args
[i
].comm
);
1097 seq_printf(m
, "\n");
1102 static const struct seq_operations probes_seq_op
= {
1103 .start
= probes_seq_start
,
1104 .next
= probes_seq_next
,
1105 .stop
= probes_seq_stop
,
1106 .show
= probes_seq_show
1109 static int probes_open(struct inode
*inode
, struct file
*file
)
1111 if ((file
->f_mode
& FMODE_WRITE
) &&
1112 (file
->f_flags
& O_TRUNC
))
1113 cleanup_all_probes();
1115 return seq_open(file
, &probes_seq_op
);
1118 static int command_trace_probe(const char *buf
)
1121 int argc
= 0, ret
= 0;
1123 argv
= argv_split(GFP_KERNEL
, buf
, &argc
);
1128 ret
= create_trace_probe(argc
, argv
);
1134 #define WRITE_BUFSIZE 128
1136 static ssize_t
probes_write(struct file
*file
, const char __user
*buffer
,
1137 size_t count
, loff_t
*ppos
)
1144 kbuf
= kmalloc(WRITE_BUFSIZE
, GFP_KERNEL
);
1149 while (done
< count
) {
1150 size
= count
- done
;
1151 if (size
>= WRITE_BUFSIZE
)
1152 size
= WRITE_BUFSIZE
- 1;
1153 if (copy_from_user(kbuf
, buffer
+ done
, size
)) {
1158 tmp
= strchr(kbuf
, '\n');
1161 size
= tmp
- kbuf
+ 1;
1162 } else if (done
+ size
< count
) {
1163 pr_warning("Line length is too long: "
1164 "Should be less than %d.", WRITE_BUFSIZE
);
1169 /* Remove comments */
1170 tmp
= strchr(kbuf
, '#');
1174 ret
= command_trace_probe(kbuf
);
1184 static const struct file_operations kprobe_events_ops
= {
1185 .owner
= THIS_MODULE
,
1186 .open
= probes_open
,
1188 .llseek
= seq_lseek
,
1189 .release
= seq_release
,
1190 .write
= probes_write
,
1193 /* Probes profiling interfaces */
1194 static int probes_profile_seq_show(struct seq_file
*m
, void *v
)
1196 struct trace_probe
*tp
= v
;
1198 seq_printf(m
, " %-44s %15lu %15lu\n", tp
->call
.name
, tp
->nhit
,
1204 static const struct seq_operations profile_seq_op
= {
1205 .start
= probes_seq_start
,
1206 .next
= probes_seq_next
,
1207 .stop
= probes_seq_stop
,
1208 .show
= probes_profile_seq_show
1211 static int profile_open(struct inode
*inode
, struct file
*file
)
1213 return seq_open(file
, &profile_seq_op
);
1216 static const struct file_operations kprobe_profile_ops
= {
1217 .owner
= THIS_MODULE
,
1218 .open
= profile_open
,
1220 .llseek
= seq_lseek
,
1221 .release
= seq_release
,
1224 /* Sum up total data length for dynamic arraies (strings) */
1225 static __kprobes
int __get_data_size(struct trace_probe
*tp
,
1226 struct pt_regs
*regs
)
1231 for (i
= 0; i
< tp
->nr_args
; i
++)
1232 if (unlikely(tp
->args
[i
].fetch_size
.fn
)) {
1233 call_fetch(&tp
->args
[i
].fetch_size
, regs
, &len
);
1240 /* Store the value of each argument */
1241 static __kprobes
void store_trace_args(int ent_size
, struct trace_probe
*tp
,
1242 struct pt_regs
*regs
,
1243 u8
*data
, int maxlen
)
1247 u32
*dl
; /* Data (relative) location */
1249 for (i
= 0; i
< tp
->nr_args
; i
++) {
1250 if (unlikely(tp
->args
[i
].fetch_size
.fn
)) {
1252 * First, we set the relative location and
1253 * maximum data length to *dl
1255 dl
= (u32
*)(data
+ tp
->args
[i
].offset
);
1256 *dl
= make_data_rloc(maxlen
, end
- tp
->args
[i
].offset
);
1257 /* Then try to fetch string or dynamic array data */
1258 call_fetch(&tp
->args
[i
].fetch
, regs
, dl
);
1259 /* Reduce maximum length */
1260 end
+= get_rloc_len(*dl
);
1261 maxlen
-= get_rloc_len(*dl
);
1262 /* Trick here, convert data_rloc to data_loc */
1263 *dl
= convert_rloc_to_loc(*dl
,
1264 ent_size
+ tp
->args
[i
].offset
);
1266 /* Just fetching data normally */
1267 call_fetch(&tp
->args
[i
].fetch
, regs
,
1268 data
+ tp
->args
[i
].offset
);
1272 /* Kprobe handler */
1273 static __kprobes
void kprobe_trace_func(struct kprobe
*kp
, struct pt_regs
*regs
)
1275 struct trace_probe
*tp
= container_of(kp
, struct trace_probe
, rp
.kp
);
1276 struct kprobe_trace_entry_head
*entry
;
1277 struct ring_buffer_event
*event
;
1278 struct ring_buffer
*buffer
;
1279 int size
, dsize
, pc
;
1280 unsigned long irq_flags
;
1281 struct ftrace_event_call
*call
= &tp
->call
;
1285 local_save_flags(irq_flags
);
1286 pc
= preempt_count();
1288 dsize
= __get_data_size(tp
, regs
);
1289 size
= sizeof(*entry
) + tp
->size
+ dsize
;
1291 event
= trace_current_buffer_lock_reserve(&buffer
, call
->event
.type
,
1292 size
, irq_flags
, pc
);
1296 entry
= ring_buffer_event_data(event
);
1297 entry
->ip
= (unsigned long)kp
->addr
;
1298 store_trace_args(sizeof(*entry
), tp
, regs
, (u8
*)&entry
[1], dsize
);
1300 if (!filter_current_check_discard(buffer
, call
, entry
, event
))
1301 trace_nowake_buffer_unlock_commit(buffer
, event
, irq_flags
, pc
);
1304 /* Kretprobe handler */
1305 static __kprobes
void kretprobe_trace_func(struct kretprobe_instance
*ri
,
1306 struct pt_regs
*regs
)
1308 struct trace_probe
*tp
= container_of(ri
->rp
, struct trace_probe
, rp
);
1309 struct kretprobe_trace_entry_head
*entry
;
1310 struct ring_buffer_event
*event
;
1311 struct ring_buffer
*buffer
;
1312 int size
, pc
, dsize
;
1313 unsigned long irq_flags
;
1314 struct ftrace_event_call
*call
= &tp
->call
;
1316 local_save_flags(irq_flags
);
1317 pc
= preempt_count();
1319 dsize
= __get_data_size(tp
, regs
);
1320 size
= sizeof(*entry
) + tp
->size
+ dsize
;
1322 event
= trace_current_buffer_lock_reserve(&buffer
, call
->event
.type
,
1323 size
, irq_flags
, pc
);
1327 entry
= ring_buffer_event_data(event
);
1328 entry
->func
= (unsigned long)tp
->rp
.kp
.addr
;
1329 entry
->ret_ip
= (unsigned long)ri
->ret_addr
;
1330 store_trace_args(sizeof(*entry
), tp
, regs
, (u8
*)&entry
[1], dsize
);
1332 if (!filter_current_check_discard(buffer
, call
, entry
, event
))
1333 trace_nowake_buffer_unlock_commit(buffer
, event
, irq_flags
, pc
);
1336 /* Event entry printers */
1338 print_kprobe_event(struct trace_iterator
*iter
, int flags
,
1339 struct trace_event
*event
)
1341 struct kprobe_trace_entry_head
*field
;
1342 struct trace_seq
*s
= &iter
->seq
;
1343 struct trace_probe
*tp
;
1347 field
= (struct kprobe_trace_entry_head
*)iter
->ent
;
1348 tp
= container_of(event
, struct trace_probe
, call
.event
);
1350 if (!trace_seq_printf(s
, "%s: (", tp
->call
.name
))
1353 if (!seq_print_ip_sym(s
, field
->ip
, flags
| TRACE_ITER_SYM_OFFSET
))
1356 if (!trace_seq_puts(s
, ")"))
1359 data
= (u8
*)&field
[1];
1360 for (i
= 0; i
< tp
->nr_args
; i
++)
1361 if (!tp
->args
[i
].type
->print(s
, tp
->args
[i
].name
,
1362 data
+ tp
->args
[i
].offset
, field
))
1365 if (!trace_seq_puts(s
, "\n"))
1368 return TRACE_TYPE_HANDLED
;
1370 return TRACE_TYPE_PARTIAL_LINE
;
1374 print_kretprobe_event(struct trace_iterator
*iter
, int flags
,
1375 struct trace_event
*event
)
1377 struct kretprobe_trace_entry_head
*field
;
1378 struct trace_seq
*s
= &iter
->seq
;
1379 struct trace_probe
*tp
;
1383 field
= (struct kretprobe_trace_entry_head
*)iter
->ent
;
1384 tp
= container_of(event
, struct trace_probe
, call
.event
);
1386 if (!trace_seq_printf(s
, "%s: (", tp
->call
.name
))
1389 if (!seq_print_ip_sym(s
, field
->ret_ip
, flags
| TRACE_ITER_SYM_OFFSET
))
1392 if (!trace_seq_puts(s
, " <- "))
1395 if (!seq_print_ip_sym(s
, field
->func
, flags
& ~TRACE_ITER_SYM_OFFSET
))
1398 if (!trace_seq_puts(s
, ")"))
1401 data
= (u8
*)&field
[1];
1402 for (i
= 0; i
< tp
->nr_args
; i
++)
1403 if (!tp
->args
[i
].type
->print(s
, tp
->args
[i
].name
,
1404 data
+ tp
->args
[i
].offset
, field
))
1407 if (!trace_seq_puts(s
, "\n"))
1410 return TRACE_TYPE_HANDLED
;
1412 return TRACE_TYPE_PARTIAL_LINE
;
1415 static int probe_event_enable(struct ftrace_event_call
*call
)
1417 struct trace_probe
*tp
= (struct trace_probe
*)call
->data
;
1419 tp
->flags
|= TP_FLAG_TRACE
;
1420 if (probe_is_return(tp
))
1421 return enable_kretprobe(&tp
->rp
);
1423 return enable_kprobe(&tp
->rp
.kp
);
1426 static void probe_event_disable(struct ftrace_event_call
*call
)
1428 struct trace_probe
*tp
= (struct trace_probe
*)call
->data
;
1430 tp
->flags
&= ~TP_FLAG_TRACE
;
1431 if (!(tp
->flags
& (TP_FLAG_TRACE
| TP_FLAG_PROFILE
))) {
1432 if (probe_is_return(tp
))
1433 disable_kretprobe(&tp
->rp
);
1435 disable_kprobe(&tp
->rp
.kp
);
1440 #define DEFINE_FIELD(type, item, name, is_signed) \
1442 ret = trace_define_field(event_call, #type, name, \
1443 offsetof(typeof(field), item), \
1444 sizeof(field.item), is_signed, \
1450 static int kprobe_event_define_fields(struct ftrace_event_call
*event_call
)
1453 struct kprobe_trace_entry_head field
;
1454 struct trace_probe
*tp
= (struct trace_probe
*)event_call
->data
;
1456 DEFINE_FIELD(unsigned long, ip
, FIELD_STRING_IP
, 0);
1457 /* Set argument names as fields */
1458 for (i
= 0; i
< tp
->nr_args
; i
++) {
1459 ret
= trace_define_field(event_call
, tp
->args
[i
].type
->fmttype
,
1461 sizeof(field
) + tp
->args
[i
].offset
,
1462 tp
->args
[i
].type
->size
,
1463 tp
->args
[i
].type
->is_signed
,
1471 static int kretprobe_event_define_fields(struct ftrace_event_call
*event_call
)
1474 struct kretprobe_trace_entry_head field
;
1475 struct trace_probe
*tp
= (struct trace_probe
*)event_call
->data
;
1477 DEFINE_FIELD(unsigned long, func
, FIELD_STRING_FUNC
, 0);
1478 DEFINE_FIELD(unsigned long, ret_ip
, FIELD_STRING_RETIP
, 0);
1479 /* Set argument names as fields */
1480 for (i
= 0; i
< tp
->nr_args
; i
++) {
1481 ret
= trace_define_field(event_call
, tp
->args
[i
].type
->fmttype
,
1483 sizeof(field
) + tp
->args
[i
].offset
,
1484 tp
->args
[i
].type
->size
,
1485 tp
->args
[i
].type
->is_signed
,
1493 static int __set_print_fmt(struct trace_probe
*tp
, char *buf
, int len
)
1498 const char *fmt
, *arg
;
1500 if (!probe_is_return(tp
)) {
1502 arg
= "REC->" FIELD_STRING_IP
;
1504 fmt
= "(%lx <- %lx)";
1505 arg
= "REC->" FIELD_STRING_FUNC
", REC->" FIELD_STRING_RETIP
;
1508 /* When len=0, we just calculate the needed length */
1509 #define LEN_OR_ZERO (len ? len - pos : 0)
1511 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
, "\"%s", fmt
);
1513 for (i
= 0; i
< tp
->nr_args
; i
++) {
1514 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
, " %s=%s",
1515 tp
->args
[i
].name
, tp
->args
[i
].type
->fmt
);
1518 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
, "\", %s", arg
);
1520 for (i
= 0; i
< tp
->nr_args
; i
++) {
1521 if (strcmp(tp
->args
[i
].type
->name
, "string") == 0)
1522 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
,
1526 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
, ", REC->%s",
1532 /* return the length of print_fmt */
1536 static int set_print_fmt(struct trace_probe
*tp
)
1541 /* First: called with 0 length to calculate the needed length */
1542 len
= __set_print_fmt(tp
, NULL
, 0);
1543 print_fmt
= kmalloc(len
+ 1, GFP_KERNEL
);
1547 /* Second: actually write the @print_fmt */
1548 __set_print_fmt(tp
, print_fmt
, len
+ 1);
1549 tp
->call
.print_fmt
= print_fmt
;
1554 #ifdef CONFIG_PERF_EVENTS
1556 /* Kprobe profile handler */
1557 static __kprobes
void kprobe_perf_func(struct kprobe
*kp
,
1558 struct pt_regs
*regs
)
1560 struct trace_probe
*tp
= container_of(kp
, struct trace_probe
, rp
.kp
);
1561 struct ftrace_event_call
*call
= &tp
->call
;
1562 struct kprobe_trace_entry_head
*entry
;
1563 struct hlist_head
*head
;
1564 int size
, __size
, dsize
;
1567 dsize
= __get_data_size(tp
, regs
);
1568 __size
= sizeof(*entry
) + tp
->size
+ dsize
;
1569 size
= ALIGN(__size
+ sizeof(u32
), sizeof(u64
));
1570 size
-= sizeof(u32
);
1571 if (WARN_ONCE(size
> PERF_MAX_TRACE_SIZE
,
1572 "profile buffer not large enough"))
1575 entry
= perf_trace_buf_prepare(size
, call
->event
.type
, regs
, &rctx
);
1579 entry
->ip
= (unsigned long)kp
->addr
;
1580 memset(&entry
[1], 0, dsize
);
1581 store_trace_args(sizeof(*entry
), tp
, regs
, (u8
*)&entry
[1], dsize
);
1583 head
= this_cpu_ptr(call
->perf_events
);
1584 perf_trace_buf_submit(entry
, size
, rctx
, entry
->ip
, 1, regs
, head
);
1587 /* Kretprobe profile handler */
1588 static __kprobes
void kretprobe_perf_func(struct kretprobe_instance
*ri
,
1589 struct pt_regs
*regs
)
1591 struct trace_probe
*tp
= container_of(ri
->rp
, struct trace_probe
, rp
);
1592 struct ftrace_event_call
*call
= &tp
->call
;
1593 struct kretprobe_trace_entry_head
*entry
;
1594 struct hlist_head
*head
;
1595 int size
, __size
, dsize
;
1598 dsize
= __get_data_size(tp
, regs
);
1599 __size
= sizeof(*entry
) + tp
->size
+ dsize
;
1600 size
= ALIGN(__size
+ sizeof(u32
), sizeof(u64
));
1601 size
-= sizeof(u32
);
1602 if (WARN_ONCE(size
> PERF_MAX_TRACE_SIZE
,
1603 "profile buffer not large enough"))
1606 entry
= perf_trace_buf_prepare(size
, call
->event
.type
, regs
, &rctx
);
1610 entry
->func
= (unsigned long)tp
->rp
.kp
.addr
;
1611 entry
->ret_ip
= (unsigned long)ri
->ret_addr
;
1612 store_trace_args(sizeof(*entry
), tp
, regs
, (u8
*)&entry
[1], dsize
);
1614 head
= this_cpu_ptr(call
->perf_events
);
1615 perf_trace_buf_submit(entry
, size
, rctx
, entry
->ret_ip
, 1, regs
, head
);
1618 static int probe_perf_enable(struct ftrace_event_call
*call
)
1620 struct trace_probe
*tp
= (struct trace_probe
*)call
->data
;
1622 tp
->flags
|= TP_FLAG_PROFILE
;
1624 if (probe_is_return(tp
))
1625 return enable_kretprobe(&tp
->rp
);
1627 return enable_kprobe(&tp
->rp
.kp
);
1630 static void probe_perf_disable(struct ftrace_event_call
*call
)
1632 struct trace_probe
*tp
= (struct trace_probe
*)call
->data
;
1634 tp
->flags
&= ~TP_FLAG_PROFILE
;
1636 if (!(tp
->flags
& TP_FLAG_TRACE
)) {
1637 if (probe_is_return(tp
))
1638 disable_kretprobe(&tp
->rp
);
1640 disable_kprobe(&tp
->rp
.kp
);
1643 #endif /* CONFIG_PERF_EVENTS */
1646 int kprobe_register(struct ftrace_event_call
*event
, enum trace_reg type
)
1649 case TRACE_REG_REGISTER
:
1650 return probe_event_enable(event
);
1651 case TRACE_REG_UNREGISTER
:
1652 probe_event_disable(event
);
1655 #ifdef CONFIG_PERF_EVENTS
1656 case TRACE_REG_PERF_REGISTER
:
1657 return probe_perf_enable(event
);
1658 case TRACE_REG_PERF_UNREGISTER
:
1659 probe_perf_disable(event
);
1667 int kprobe_dispatcher(struct kprobe
*kp
, struct pt_regs
*regs
)
1669 struct trace_probe
*tp
= container_of(kp
, struct trace_probe
, rp
.kp
);
1671 if (tp
->flags
& TP_FLAG_TRACE
)
1672 kprobe_trace_func(kp
, regs
);
1673 #ifdef CONFIG_PERF_EVENTS
1674 if (tp
->flags
& TP_FLAG_PROFILE
)
1675 kprobe_perf_func(kp
, regs
);
1677 return 0; /* We don't tweek kernel, so just return 0 */
1681 int kretprobe_dispatcher(struct kretprobe_instance
*ri
, struct pt_regs
*regs
)
1683 struct trace_probe
*tp
= container_of(ri
->rp
, struct trace_probe
, rp
);
1685 if (tp
->flags
& TP_FLAG_TRACE
)
1686 kretprobe_trace_func(ri
, regs
);
1687 #ifdef CONFIG_PERF_EVENTS
1688 if (tp
->flags
& TP_FLAG_PROFILE
)
1689 kretprobe_perf_func(ri
, regs
);
1691 return 0; /* We don't tweek kernel, so just return 0 */
1694 static struct trace_event_functions kretprobe_funcs
= {
1695 .trace
= print_kretprobe_event
1698 static struct trace_event_functions kprobe_funcs
= {
1699 .trace
= print_kprobe_event
1702 static int register_probe_event(struct trace_probe
*tp
)
1704 struct ftrace_event_call
*call
= &tp
->call
;
1707 /* Initialize ftrace_event_call */
1708 INIT_LIST_HEAD(&call
->class->fields
);
1709 if (probe_is_return(tp
)) {
1710 call
->event
.funcs
= &kretprobe_funcs
;
1711 call
->class->define_fields
= kretprobe_event_define_fields
;
1713 call
->event
.funcs
= &kprobe_funcs
;
1714 call
->class->define_fields
= kprobe_event_define_fields
;
1716 if (set_print_fmt(tp
) < 0)
1718 ret
= register_ftrace_event(&call
->event
);
1720 kfree(call
->print_fmt
);
1724 call
->class->reg
= kprobe_register
;
1726 ret
= trace_add_event_call(call
);
1728 pr_info("Failed to register kprobe event: %s\n", call
->name
);
1729 kfree(call
->print_fmt
);
1730 unregister_ftrace_event(&call
->event
);
1735 static void unregister_probe_event(struct trace_probe
*tp
)
1737 /* tp->event is unregistered in trace_remove_event_call() */
1738 trace_remove_event_call(&tp
->call
);
1739 kfree(tp
->call
.print_fmt
);
1742 /* Make a debugfs interface for controling probe points */
1743 static __init
int init_kprobe_trace(void)
1745 struct dentry
*d_tracer
;
1746 struct dentry
*entry
;
1748 d_tracer
= tracing_init_dentry();
1752 entry
= debugfs_create_file("kprobe_events", 0644, d_tracer
,
1753 NULL
, &kprobe_events_ops
);
1755 /* Event list interface */
1757 pr_warning("Could not create debugfs "
1758 "'kprobe_events' entry\n");
1760 /* Profile interface */
1761 entry
= debugfs_create_file("kprobe_profile", 0444, d_tracer
,
1762 NULL
, &kprobe_profile_ops
);
1765 pr_warning("Could not create debugfs "
1766 "'kprobe_profile' entry\n");
1769 fs_initcall(init_kprobe_trace
);
1772 #ifdef CONFIG_FTRACE_STARTUP_TEST
1774 static int kprobe_trace_selftest_target(int a1
, int a2
, int a3
,
1775 int a4
, int a5
, int a6
)
1777 return a1
+ a2
+ a3
+ a4
+ a5
+ a6
;
1780 static __init
int kprobe_trace_self_tests_init(void)
1783 int (*target
)(int, int, int, int, int, int);
1784 struct trace_probe
*tp
;
1786 target
= kprobe_trace_selftest_target
;
1788 pr_info("Testing kprobe tracing: ");
1790 ret
= command_trace_probe("p:testprobe kprobe_trace_selftest_target "
1791 "$stack $stack0 +0($stack)");
1792 if (WARN_ON_ONCE(ret
)) {
1793 pr_warning("error on probing function entry.\n");
1796 /* Enable trace point */
1797 tp
= find_probe_event("testprobe", KPROBE_EVENT_SYSTEM
);
1798 if (WARN_ON_ONCE(tp
== NULL
)) {
1799 pr_warning("error on getting new probe.\n");
1802 probe_event_enable(&tp
->call
);
1805 ret
= command_trace_probe("r:testprobe2 kprobe_trace_selftest_target "
1807 if (WARN_ON_ONCE(ret
)) {
1808 pr_warning("error on probing function return.\n");
1811 /* Enable trace point */
1812 tp
= find_probe_event("testprobe2", KPROBE_EVENT_SYSTEM
);
1813 if (WARN_ON_ONCE(tp
== NULL
)) {
1814 pr_warning("error on getting new probe.\n");
1817 probe_event_enable(&tp
->call
);
1823 ret
= target(1, 2, 3, 4, 5, 6);
1825 ret
= command_trace_probe("-:testprobe");
1826 if (WARN_ON_ONCE(ret
)) {
1827 pr_warning("error on deleting a probe.\n");
1831 ret
= command_trace_probe("-:testprobe2");
1832 if (WARN_ON_ONCE(ret
)) {
1833 pr_warning("error on deleting a probe.\n");
1838 cleanup_all_probes();
1840 pr_cont("NG: Some tests are failed. Please check them.\n");
1846 late_initcall(kprobe_trace_self_tests_init
);