2 * Kprobes-based tracing events
4 * Created by Masami Hiramatsu <mhiramat@redhat.com>
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 #include <linux/module.h>
21 #include <linux/uaccess.h>
22 #include <linux/kprobes.h>
23 #include <linux/seq_file.h>
24 #include <linux/slab.h>
25 #include <linux/smp.h>
26 #include <linux/debugfs.h>
27 #include <linux/types.h>
28 #include <linux/string.h>
29 #include <linux/ctype.h>
30 #include <linux/ptrace.h>
31 #include <linux/perf_event.h>
32 #include <linux/stringify.h>
33 #include <linux/limits.h>
34 #include <asm/bitsperlong.h>
37 #include "trace_output.h"
39 #define MAX_TRACE_ARGS 128
40 #define MAX_ARGSTR_LEN 63
41 #define MAX_EVENT_NAME_LEN 64
42 #define MAX_STRING_SIZE PATH_MAX
43 #define KPROBE_EVENT_SYSTEM "kprobes"
45 /* Reserved field names */
46 #define FIELD_STRING_IP "__probe_ip"
47 #define FIELD_STRING_RETIP "__probe_ret_ip"
48 #define FIELD_STRING_FUNC "__probe_func"
50 const char *reserved_field_names
[] = {
53 "common_preempt_count",
61 /* Printing function type */
62 typedef int (*print_type_func_t
)(struct trace_seq
*, const char *, void *,
64 #define PRINT_TYPE_FUNC_NAME(type) print_type_##type
65 #define PRINT_TYPE_FMT_NAME(type) print_type_format_##type
67 /* Printing in basic type function template */
68 #define DEFINE_BASIC_PRINT_TYPE_FUNC(type, fmt, cast) \
69 static __kprobes int PRINT_TYPE_FUNC_NAME(type)(struct trace_seq *s, \
71 void *data, void *ent)\
73 return trace_seq_printf(s, " %s=" fmt, name, (cast)*(type *)data);\
75 static const char PRINT_TYPE_FMT_NAME(type)[] = fmt;
77 DEFINE_BASIC_PRINT_TYPE_FUNC(u8
, "%x", unsigned int)
78 DEFINE_BASIC_PRINT_TYPE_FUNC(u16
, "%x", unsigned int)
79 DEFINE_BASIC_PRINT_TYPE_FUNC(u32
, "%lx", unsigned long)
80 DEFINE_BASIC_PRINT_TYPE_FUNC(u64
, "%llx", unsigned long long)
81 DEFINE_BASIC_PRINT_TYPE_FUNC(s8
, "%d", int)
82 DEFINE_BASIC_PRINT_TYPE_FUNC(s16
, "%d", int)
83 DEFINE_BASIC_PRINT_TYPE_FUNC(s32
, "%ld", long)
84 DEFINE_BASIC_PRINT_TYPE_FUNC(s64
, "%lld", long long)
86 /* data_rloc: data relative location, compatible with u32 */
87 #define make_data_rloc(len, roffs) \
88 (((u32)(len) << 16) | ((u32)(roffs) & 0xffff))
89 #define get_rloc_len(dl) ((u32)(dl) >> 16)
90 #define get_rloc_offs(dl) ((u32)(dl) & 0xffff)
92 static inline void *get_rloc_data(u32
*dl
)
94 return (u8
*)dl
+ get_rloc_offs(*dl
);
97 /* For data_loc conversion */
98 static inline void *get_loc_data(u32
*dl
, void *ent
)
100 return (u8
*)ent
+ get_rloc_offs(*dl
);
104 * Convert data_rloc to data_loc:
105 * data_rloc stores the offset from data_rloc itself, but data_loc
106 * stores the offset from event entry.
108 #define convert_rloc_to_loc(dl, offs) ((u32)(dl) + (offs))
110 /* For defining macros, define string/string_size types */
112 typedef u32 string_size
;
114 /* Print type function for string type */
115 static __kprobes
int PRINT_TYPE_FUNC_NAME(string
)(struct trace_seq
*s
,
117 void *data
, void *ent
)
119 int len
= *(u32
*)data
>> 16;
122 return trace_seq_printf(s
, " %s=(fault)", name
);
124 return trace_seq_printf(s
, " %s=\"%s\"", name
,
125 (const char *)get_loc_data(data
, ent
));
127 static const char PRINT_TYPE_FMT_NAME(string
)[] = "\\\"%s\\\"";
129 /* Data fetch function type */
130 typedef void (*fetch_func_t
)(struct pt_regs
*, void *, void *);
137 static __kprobes
void call_fetch(struct fetch_param
*fprm
,
138 struct pt_regs
*regs
, void *dest
)
140 return fprm
->fn(regs
, fprm
->data
, dest
);
143 #define FETCH_FUNC_NAME(method, type) fetch_##method##_##type
145 * Define macro for basic types - we don't need to define s* types, because
146 * we have to care only about bitwidth at recording time.
148 #define DEFINE_BASIC_FETCH_FUNCS(method) \
149 DEFINE_FETCH_##method(u8) \
150 DEFINE_FETCH_##method(u16) \
151 DEFINE_FETCH_##method(u32) \
152 DEFINE_FETCH_##method(u64)
154 #define CHECK_FETCH_FUNCS(method, fn) \
155 (((FETCH_FUNC_NAME(method, u8) == fn) || \
156 (FETCH_FUNC_NAME(method, u16) == fn) || \
157 (FETCH_FUNC_NAME(method, u32) == fn) || \
158 (FETCH_FUNC_NAME(method, u64) == fn) || \
159 (FETCH_FUNC_NAME(method, string) == fn) || \
160 (FETCH_FUNC_NAME(method, string_size) == fn)) \
163 /* Data fetch function templates */
164 #define DEFINE_FETCH_reg(type) \
165 static __kprobes void FETCH_FUNC_NAME(reg, type)(struct pt_regs *regs, \
166 void *offset, void *dest) \
168 *(type *)dest = (type)regs_get_register(regs, \
169 (unsigned int)((unsigned long)offset)); \
171 DEFINE_BASIC_FETCH_FUNCS(reg
)
172 /* No string on the register */
173 #define fetch_reg_string NULL
174 #define fetch_reg_string_size NULL
176 #define DEFINE_FETCH_stack(type) \
177 static __kprobes void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs,\
178 void *offset, void *dest) \
180 *(type *)dest = (type)regs_get_kernel_stack_nth(regs, \
181 (unsigned int)((unsigned long)offset)); \
183 DEFINE_BASIC_FETCH_FUNCS(stack
)
184 /* No string on the stack entry */
185 #define fetch_stack_string NULL
186 #define fetch_stack_string_size NULL
188 #define DEFINE_FETCH_retval(type) \
189 static __kprobes void FETCH_FUNC_NAME(retval, type)(struct pt_regs *regs,\
190 void *dummy, void *dest) \
192 *(type *)dest = (type)regs_return_value(regs); \
194 DEFINE_BASIC_FETCH_FUNCS(retval
)
195 /* No string on the retval */
196 #define fetch_retval_string NULL
197 #define fetch_retval_string_size NULL
199 #define DEFINE_FETCH_memory(type) \
200 static __kprobes void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs,\
201 void *addr, void *dest) \
204 if (probe_kernel_address(addr, retval)) \
207 *(type *)dest = retval; \
209 DEFINE_BASIC_FETCH_FUNCS(memory
)
211 * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max
212 * length and relative data location.
214 static __kprobes
void FETCH_FUNC_NAME(memory
, string
)(struct pt_regs
*regs
,
215 void *addr
, void *dest
)
218 int maxlen
= get_rloc_len(*(u32
*)dest
);
219 u8
*dst
= get_rloc_data(dest
);
221 mm_segment_t old_fs
= get_fs();
225 * Try to get string again, since the string can be changed while
231 ret
= __copy_from_user_inatomic(dst
++, src
++, 1);
232 while (dst
[-1] && ret
== 0 && src
- (u8
*)addr
< maxlen
);
237 if (ret
< 0) { /* Failed to fetch string */
238 ((u8
*)get_rloc_data(dest
))[0] = '\0';
239 *(u32
*)dest
= make_data_rloc(0, get_rloc_offs(*(u32
*)dest
));
241 *(u32
*)dest
= make_data_rloc(src
- (u8
*)addr
,
242 get_rloc_offs(*(u32
*)dest
));
244 /* Return the length of string -- including null terminal byte */
245 static __kprobes
void FETCH_FUNC_NAME(memory
, string_size
)(struct pt_regs
*regs
,
246 void *addr
, void *dest
)
250 mm_segment_t old_fs
= get_fs();
255 ret
= __copy_from_user_inatomic(&c
, (u8
*)addr
+ len
, 1);
257 } while (c
&& ret
== 0 && len
< MAX_STRING_SIZE
);
261 if (ret
< 0) /* Failed to check the length */
267 /* Memory fetching by symbol */
268 struct symbol_cache
{
274 static unsigned long update_symbol_cache(struct symbol_cache
*sc
)
276 sc
->addr
= (unsigned long)kallsyms_lookup_name(sc
->symbol
);
278 sc
->addr
+= sc
->offset
;
282 static void free_symbol_cache(struct symbol_cache
*sc
)
288 static struct symbol_cache
*alloc_symbol_cache(const char *sym
, long offset
)
290 struct symbol_cache
*sc
;
292 if (!sym
|| strlen(sym
) == 0)
294 sc
= kzalloc(sizeof(struct symbol_cache
), GFP_KERNEL
);
298 sc
->symbol
= kstrdup(sym
, GFP_KERNEL
);
305 update_symbol_cache(sc
);
309 #define DEFINE_FETCH_symbol(type) \
310 static __kprobes void FETCH_FUNC_NAME(symbol, type)(struct pt_regs *regs,\
311 void *data, void *dest) \
313 struct symbol_cache *sc = data; \
315 fetch_memory_##type(regs, (void *)sc->addr, dest); \
319 DEFINE_BASIC_FETCH_FUNCS(symbol
)
320 DEFINE_FETCH_symbol(string
)
321 DEFINE_FETCH_symbol(string_size
)
323 /* Dereference memory access function */
324 struct deref_fetch_param
{
325 struct fetch_param orig
;
329 #define DEFINE_FETCH_deref(type) \
330 static __kprobes void FETCH_FUNC_NAME(deref, type)(struct pt_regs *regs,\
331 void *data, void *dest) \
333 struct deref_fetch_param *dprm = data; \
334 unsigned long addr; \
335 call_fetch(&dprm->orig, regs, &addr); \
337 addr += dprm->offset; \
338 fetch_memory_##type(regs, (void *)addr, dest); \
342 DEFINE_BASIC_FETCH_FUNCS(deref
)
343 DEFINE_FETCH_deref(string
)
344 DEFINE_FETCH_deref(string_size
)
346 static __kprobes
void free_deref_fetch_param(struct deref_fetch_param
*data
)
348 if (CHECK_FETCH_FUNCS(deref
, data
->orig
.fn
))
349 free_deref_fetch_param(data
->orig
.data
);
350 else if (CHECK_FETCH_FUNCS(symbol
, data
->orig
.fn
))
351 free_symbol_cache(data
->orig
.data
);
355 /* Bitfield fetch function */
356 struct bitfield_fetch_param
{
357 struct fetch_param orig
;
358 unsigned char hi_shift
;
359 unsigned char low_shift
;
362 #define DEFINE_FETCH_bitfield(type) \
363 static __kprobes void FETCH_FUNC_NAME(bitfield, type)(struct pt_regs *regs,\
364 void *data, void *dest) \
366 struct bitfield_fetch_param *bprm = data; \
368 call_fetch(&bprm->orig, regs, &buf); \
370 buf <<= bprm->hi_shift; \
371 buf >>= bprm->low_shift; \
373 *(type *)dest = buf; \
375 DEFINE_BASIC_FETCH_FUNCS(bitfield
)
376 #define fetch_bitfield_string NULL
377 #define fetch_bitfield_string_size NULL
379 static __kprobes
void
380 free_bitfield_fetch_param(struct bitfield_fetch_param
*data
)
383 * Don't check the bitfield itself, because this must be the
384 * last fetch function.
386 if (CHECK_FETCH_FUNCS(deref
, data
->orig
.fn
))
387 free_deref_fetch_param(data
->orig
.data
);
388 else if (CHECK_FETCH_FUNCS(symbol
, data
->orig
.fn
))
389 free_symbol_cache(data
->orig
.data
);
392 /* Default (unsigned long) fetch type */
393 #define __DEFAULT_FETCH_TYPE(t) u##t
394 #define _DEFAULT_FETCH_TYPE(t) __DEFAULT_FETCH_TYPE(t)
395 #define DEFAULT_FETCH_TYPE _DEFAULT_FETCH_TYPE(BITS_PER_LONG)
396 #define DEFAULT_FETCH_TYPE_STR __stringify(DEFAULT_FETCH_TYPE)
410 #define ASSIGN_FETCH_FUNC(method, type) \
411 [FETCH_MTD_##method] = FETCH_FUNC_NAME(method, type)
413 #define __ASSIGN_FETCH_TYPE(_name, ptype, ftype, _size, sign, _fmttype) \
417 .print = PRINT_TYPE_FUNC_NAME(ptype), \
418 .fmt = PRINT_TYPE_FMT_NAME(ptype), \
419 .fmttype = _fmttype, \
421 ASSIGN_FETCH_FUNC(reg, ftype), \
422 ASSIGN_FETCH_FUNC(stack, ftype), \
423 ASSIGN_FETCH_FUNC(retval, ftype), \
424 ASSIGN_FETCH_FUNC(memory, ftype), \
425 ASSIGN_FETCH_FUNC(symbol, ftype), \
426 ASSIGN_FETCH_FUNC(deref, ftype), \
427 ASSIGN_FETCH_FUNC(bitfield, ftype), \
431 #define ASSIGN_FETCH_TYPE(ptype, ftype, sign) \
432 __ASSIGN_FETCH_TYPE(#ptype, ptype, ftype, sizeof(ftype), sign, #ptype)
434 #define FETCH_TYPE_STRING 0
435 #define FETCH_TYPE_STRSIZE 1
437 /* Fetch type information table */
438 static const struct fetch_type
{
439 const char *name
; /* Name of type */
440 size_t size
; /* Byte size of type */
441 int is_signed
; /* Signed flag */
442 print_type_func_t print
; /* Print functions */
443 const char *fmt
; /* Fromat string */
444 const char *fmttype
; /* Name in format file */
445 /* Fetch functions */
446 fetch_func_t fetch
[FETCH_MTD_END
];
447 } fetch_type_table
[] = {
449 [FETCH_TYPE_STRING
] = __ASSIGN_FETCH_TYPE("string", string
, string
,
450 sizeof(u32
), 1, "__data_loc char[]"),
451 [FETCH_TYPE_STRSIZE
] = __ASSIGN_FETCH_TYPE("string_size", u32
,
452 string_size
, sizeof(u32
), 0, "u32"),
454 ASSIGN_FETCH_TYPE(u8
, u8
, 0),
455 ASSIGN_FETCH_TYPE(u16
, u16
, 0),
456 ASSIGN_FETCH_TYPE(u32
, u32
, 0),
457 ASSIGN_FETCH_TYPE(u64
, u64
, 0),
458 ASSIGN_FETCH_TYPE(s8
, u8
, 1),
459 ASSIGN_FETCH_TYPE(s16
, u16
, 1),
460 ASSIGN_FETCH_TYPE(s32
, u32
, 1),
461 ASSIGN_FETCH_TYPE(s64
, u64
, 1),
464 static const struct fetch_type
*find_fetch_type(const char *type
)
469 type
= DEFAULT_FETCH_TYPE_STR
;
471 /* Special case: bitfield */
474 type
= strchr(type
, '/');
478 if (strict_strtoul(type
, 0, &bs
))
482 return find_fetch_type("u8");
484 return find_fetch_type("u16");
486 return find_fetch_type("u32");
488 return find_fetch_type("u64");
494 for (i
= 0; i
< ARRAY_SIZE(fetch_type_table
); i
++)
495 if (strcmp(type
, fetch_type_table
[i
].name
) == 0)
496 return &fetch_type_table
[i
];
501 /* Special function : only accept unsigned long */
502 static __kprobes
void fetch_stack_address(struct pt_regs
*regs
,
503 void *dummy
, void *dest
)
505 *(unsigned long *)dest
= kernel_stack_pointer(regs
);
508 static fetch_func_t
get_fetch_size_function(const struct fetch_type
*type
,
509 fetch_func_t orig_fn
)
513 if (type
!= &fetch_type_table
[FETCH_TYPE_STRING
])
514 return NULL
; /* Only string type needs size function */
515 for (i
= 0; i
< FETCH_MTD_END
; i
++)
516 if (type
->fetch
[i
] == orig_fn
)
517 return fetch_type_table
[FETCH_TYPE_STRSIZE
].fetch
[i
];
519 WARN_ON(1); /* This should not happen */
524 * Kprobe event core functions
528 struct fetch_param fetch
;
529 struct fetch_param fetch_size
;
530 unsigned int offset
; /* Offset from argument entry */
531 const char *name
; /* Name of this argument */
532 const char *comm
; /* Command of this argument */
533 const struct fetch_type
*type
; /* Type of this argument */
536 /* Flags for trace_probe */
537 #define TP_FLAG_TRACE 1
538 #define TP_FLAG_PROFILE 2
541 struct list_head list
;
542 struct kretprobe rp
; /* Use rp.kp for kprobe use */
544 unsigned int flags
; /* For TP_FLAG_* */
545 const char *symbol
; /* symbol name */
546 struct ftrace_event_class
class;
547 struct ftrace_event_call call
;
548 ssize_t size
; /* trace entry size */
549 unsigned int nr_args
;
550 struct probe_arg args
[];
553 #define SIZEOF_TRACE_PROBE(n) \
554 (offsetof(struct trace_probe, args) + \
555 (sizeof(struct probe_arg) * (n)))
558 static __kprobes
int trace_probe_is_return(struct trace_probe
*tp
)
560 return tp
->rp
.handler
!= NULL
;
563 static __kprobes
const char *trace_probe_symbol(struct trace_probe
*tp
)
565 return tp
->symbol
? tp
->symbol
: "unknown";
568 static int register_probe_event(struct trace_probe
*tp
);
569 static void unregister_probe_event(struct trace_probe
*tp
);
571 static DEFINE_MUTEX(probe_lock
);
572 static LIST_HEAD(probe_list
);
574 static int kprobe_dispatcher(struct kprobe
*kp
, struct pt_regs
*regs
);
575 static int kretprobe_dispatcher(struct kretprobe_instance
*ri
,
576 struct pt_regs
*regs
);
578 /* Check the name is good for event/group/fields */
579 static int is_good_name(const char *name
)
581 if (!isalpha(*name
) && *name
!= '_')
583 while (*++name
!= '\0') {
584 if (!isalpha(*name
) && !isdigit(*name
) && *name
!= '_')
591 * Allocate new trace_probe and initialize it (including kprobes).
593 static struct trace_probe
*alloc_trace_probe(const char *group
,
598 int nargs
, int is_return
)
600 struct trace_probe
*tp
;
603 tp
= kzalloc(SIZEOF_TRACE_PROBE(nargs
), GFP_KERNEL
);
608 tp
->symbol
= kstrdup(symbol
, GFP_KERNEL
);
611 tp
->rp
.kp
.symbol_name
= tp
->symbol
;
612 tp
->rp
.kp
.offset
= offs
;
614 tp
->rp
.kp
.addr
= addr
;
617 tp
->rp
.handler
= kretprobe_dispatcher
;
619 tp
->rp
.kp
.pre_handler
= kprobe_dispatcher
;
621 if (!event
|| !is_good_name(event
)) {
626 tp
->call
.class = &tp
->class;
627 tp
->call
.name
= kstrdup(event
, GFP_KERNEL
);
631 if (!group
|| !is_good_name(group
)) {
636 tp
->class.system
= kstrdup(group
, GFP_KERNEL
);
637 if (!tp
->class.system
)
640 INIT_LIST_HEAD(&tp
->list
);
643 kfree(tp
->call
.name
);
649 static void free_probe_arg(struct probe_arg
*arg
)
651 if (CHECK_FETCH_FUNCS(bitfield
, arg
->fetch
.fn
))
652 free_bitfield_fetch_param(arg
->fetch
.data
);
653 else if (CHECK_FETCH_FUNCS(deref
, arg
->fetch
.fn
))
654 free_deref_fetch_param(arg
->fetch
.data
);
655 else if (CHECK_FETCH_FUNCS(symbol
, arg
->fetch
.fn
))
656 free_symbol_cache(arg
->fetch
.data
);
661 static void free_trace_probe(struct trace_probe
*tp
)
665 for (i
= 0; i
< tp
->nr_args
; i
++)
666 free_probe_arg(&tp
->args
[i
]);
668 kfree(tp
->call
.class->system
);
669 kfree(tp
->call
.name
);
674 static struct trace_probe
*find_trace_probe(const char *event
,
677 struct trace_probe
*tp
;
679 list_for_each_entry(tp
, &probe_list
, list
)
680 if (strcmp(tp
->call
.name
, event
) == 0 &&
681 strcmp(tp
->call
.class->system
, group
) == 0)
686 /* Unregister a trace_probe and probe_event: call with locking probe_lock */
687 static void unregister_trace_probe(struct trace_probe
*tp
)
689 if (trace_probe_is_return(tp
))
690 unregister_kretprobe(&tp
->rp
);
692 unregister_kprobe(&tp
->rp
.kp
);
694 unregister_probe_event(tp
);
697 /* Register a trace_probe and probe_event */
698 static int register_trace_probe(struct trace_probe
*tp
)
700 struct trace_probe
*old_tp
;
703 mutex_lock(&probe_lock
);
705 /* register as an event */
706 old_tp
= find_trace_probe(tp
->call
.name
, tp
->call
.class->system
);
708 /* delete old event */
709 unregister_trace_probe(old_tp
);
710 free_trace_probe(old_tp
);
712 ret
= register_probe_event(tp
);
714 pr_warning("Failed to register probe event(%d)\n", ret
);
718 tp
->rp
.kp
.flags
|= KPROBE_FLAG_DISABLED
;
719 if (trace_probe_is_return(tp
))
720 ret
= register_kretprobe(&tp
->rp
);
722 ret
= register_kprobe(&tp
->rp
.kp
);
725 pr_warning("Could not insert probe(%d)\n", ret
);
726 if (ret
== -EILSEQ
) {
727 pr_warning("Probing address(0x%p) is not an "
728 "instruction boundary.\n",
732 unregister_probe_event(tp
);
734 list_add_tail(&tp
->list
, &probe_list
);
736 mutex_unlock(&probe_lock
);
740 /* Split symbol and offset. */
741 static int split_symbol_offset(char *symbol
, unsigned long *offset
)
749 tmp
= strchr(symbol
, '+');
751 /* skip sign because strict_strtol doesn't accept '+' */
752 ret
= strict_strtoul(tmp
+ 1, 0, offset
);
761 #define PARAM_MAX_ARGS 16
762 #define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long))
764 static int parse_probe_vars(char *arg
, const struct fetch_type
*t
,
765 struct fetch_param
*f
, int is_return
)
770 if (strcmp(arg
, "retval") == 0) {
772 f
->fn
= t
->fetch
[FETCH_MTD_retval
];
775 } else if (strncmp(arg
, "stack", 5) == 0) {
776 if (arg
[5] == '\0') {
777 if (strcmp(t
->name
, DEFAULT_FETCH_TYPE_STR
) == 0)
778 f
->fn
= fetch_stack_address
;
781 } else if (isdigit(arg
[5])) {
782 ret
= strict_strtoul(arg
+ 5, 10, ¶m
);
783 if (ret
|| param
> PARAM_MAX_STACK
)
786 f
->fn
= t
->fetch
[FETCH_MTD_stack
];
787 f
->data
= (void *)param
;
796 /* Recursive argument parser */
797 static int __parse_probe_arg(char *arg
, const struct fetch_type
*t
,
798 struct fetch_param
*f
, int is_return
)
807 ret
= parse_probe_vars(arg
+ 1, t
, f
, is_return
);
809 case '%': /* named register */
810 ret
= regs_query_register_offset(arg
+ 1);
812 f
->fn
= t
->fetch
[FETCH_MTD_reg
];
813 f
->data
= (void *)(unsigned long)ret
;
817 case '@': /* memory or symbol */
818 if (isdigit(arg
[1])) {
819 ret
= strict_strtoul(arg
+ 1, 0, ¶m
);
822 f
->fn
= t
->fetch
[FETCH_MTD_memory
];
823 f
->data
= (void *)param
;
825 ret
= split_symbol_offset(arg
+ 1, &offset
);
828 f
->data
= alloc_symbol_cache(arg
+ 1, offset
);
830 f
->fn
= t
->fetch
[FETCH_MTD_symbol
];
833 case '+': /* deref memory */
834 arg
++; /* Skip '+', because strict_strtol() rejects it. */
836 tmp
= strchr(arg
, '(');
840 ret
= strict_strtol(arg
, 0, &offset
);
844 tmp
= strrchr(arg
, ')');
846 struct deref_fetch_param
*dprm
;
847 const struct fetch_type
*t2
= find_fetch_type(NULL
);
849 dprm
= kzalloc(sizeof(struct deref_fetch_param
),
853 dprm
->offset
= offset
;
854 ret
= __parse_probe_arg(arg
, t2
, &dprm
->orig
,
859 f
->fn
= t
->fetch
[FETCH_MTD_deref
];
860 f
->data
= (void *)dprm
;
865 if (!ret
&& !f
->fn
) { /* Parsed, but do not find fetch method */
866 pr_info("%s type has no corresponding fetch method.\n",
873 #define BYTES_TO_BITS(nb) ((BITS_PER_LONG * (nb)) / sizeof(long))
875 /* Bitfield type needs to be parsed into a fetch function */
876 static int __parse_bitfield_probe_arg(const char *bf
,
877 const struct fetch_type
*t
,
878 struct fetch_param
*f
)
880 struct bitfield_fetch_param
*bprm
;
881 unsigned long bw
, bo
;
887 bprm
= kzalloc(sizeof(*bprm
), GFP_KERNEL
);
891 f
->fn
= t
->fetch
[FETCH_MTD_bitfield
];
892 f
->data
= (void *)bprm
;
894 bw
= simple_strtoul(bf
+ 1, &tail
, 0); /* Use simple one */
895 if (bw
== 0 || *tail
!= '@')
899 bo
= simple_strtoul(bf
, &tail
, 0);
900 if (tail
== bf
|| *tail
!= '/')
903 bprm
->hi_shift
= BYTES_TO_BITS(t
->size
) - (bw
+ bo
);
904 bprm
->low_shift
= bprm
->hi_shift
+ bo
;
905 return (BYTES_TO_BITS(t
->size
) < (bw
+ bo
)) ? -EINVAL
: 0;
908 /* String length checking wrapper */
909 static int parse_probe_arg(char *arg
, struct trace_probe
*tp
,
910 struct probe_arg
*parg
, int is_return
)
915 if (strlen(arg
) > MAX_ARGSTR_LEN
) {
916 pr_info("Argument is too long.: %s\n", arg
);
919 parg
->comm
= kstrdup(arg
, GFP_KERNEL
);
921 pr_info("Failed to allocate memory for command '%s'.\n", arg
);
924 t
= strchr(parg
->comm
, ':');
926 arg
[t
- parg
->comm
] = '\0';
929 parg
->type
= find_fetch_type(t
);
931 pr_info("Unsupported type: %s\n", t
);
934 parg
->offset
= tp
->size
;
935 tp
->size
+= parg
->type
->size
;
936 ret
= __parse_probe_arg(arg
, parg
->type
, &parg
->fetch
, is_return
);
937 if (ret
>= 0 && t
!= NULL
)
938 ret
= __parse_bitfield_probe_arg(t
, parg
->type
, &parg
->fetch
);
940 parg
->fetch_size
.fn
= get_fetch_size_function(parg
->type
,
942 parg
->fetch_size
.data
= parg
->fetch
.data
;
947 /* Return 1 if name is reserved or already used by another argument */
948 static int conflict_field_name(const char *name
,
949 struct probe_arg
*args
, int narg
)
952 for (i
= 0; i
< ARRAY_SIZE(reserved_field_names
); i
++)
953 if (strcmp(reserved_field_names
[i
], name
) == 0)
955 for (i
= 0; i
< narg
; i
++)
956 if (strcmp(args
[i
].name
, name
) == 0)
961 static int create_trace_probe(int argc
, char **argv
)
965 * - Add kprobe: p[:[GRP/]EVENT] KSYM[+OFFS]|KADDR [FETCHARGS]
966 * - Add kretprobe: r[:[GRP/]EVENT] KSYM[+0] [FETCHARGS]
968 * $retval : fetch return value
969 * $stack : fetch stack address
970 * $stackN : fetch Nth of stack (N:0-)
971 * @ADDR : fetch memory at ADDR (ADDR should be in kernel)
972 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)
973 * %REG : fetch register REG
974 * Dereferencing memory fetch:
975 * +|-offs(ARG) : fetch memory at ARG +|- offs address.
976 * Alias name of args:
977 * NAME=FETCHARG : set NAME as alias of FETCHARG.
979 * FETCHARG:TYPE : use TYPE instead of unsigned long.
981 struct trace_probe
*tp
;
983 int is_return
= 0, is_delete
= 0;
984 char *symbol
= NULL
, *event
= NULL
, *group
= NULL
;
986 unsigned long offset
= 0;
988 char buf
[MAX_EVENT_NAME_LEN
];
990 /* argc must be >= 1 */
991 if (argv
[0][0] == 'p')
993 else if (argv
[0][0] == 'r')
995 else if (argv
[0][0] == '-')
998 pr_info("Probe definition must be started with 'p', 'r' or"
1003 if (argv
[0][1] == ':') {
1004 event
= &argv
[0][2];
1005 if (strchr(event
, '/')) {
1007 event
= strchr(group
, '/') + 1;
1009 if (strlen(group
) == 0) {
1010 pr_info("Group name is not specified\n");
1014 if (strlen(event
) == 0) {
1015 pr_info("Event name is not specified\n");
1020 group
= KPROBE_EVENT_SYSTEM
;
1024 pr_info("Delete command needs an event name.\n");
1027 mutex_lock(&probe_lock
);
1028 tp
= find_trace_probe(event
, group
);
1030 mutex_unlock(&probe_lock
);
1031 pr_info("Event %s/%s doesn't exist.\n", group
, event
);
1034 /* delete an event */
1035 unregister_trace_probe(tp
);
1036 free_trace_probe(tp
);
1037 mutex_unlock(&probe_lock
);
1042 pr_info("Probe point is not specified.\n");
1045 if (isdigit(argv
[1][0])) {
1047 pr_info("Return probe point must be a symbol.\n");
1050 /* an address specified */
1051 ret
= strict_strtoul(&argv
[1][0], 0, (unsigned long *)&addr
);
1053 pr_info("Failed to parse address.\n");
1057 /* a symbol specified */
1059 /* TODO: support .init module functions */
1060 ret
= split_symbol_offset(symbol
, &offset
);
1062 pr_info("Failed to parse symbol.\n");
1065 if (offset
&& is_return
) {
1066 pr_info("Return probe must be used without offset.\n");
1070 argc
-= 2; argv
+= 2;
1074 /* Make a new event name */
1076 snprintf(buf
, MAX_EVENT_NAME_LEN
, "%c_%s_%ld",
1077 is_return
? 'r' : 'p', symbol
, offset
);
1079 snprintf(buf
, MAX_EVENT_NAME_LEN
, "%c_0x%p",
1080 is_return
? 'r' : 'p', addr
);
1083 tp
= alloc_trace_probe(group
, event
, addr
, symbol
, offset
, argc
,
1086 pr_info("Failed to allocate trace_probe.(%d)\n",
1091 /* parse arguments */
1093 for (i
= 0; i
< argc
&& i
< MAX_TRACE_ARGS
; i
++) {
1094 /* Increment count for freeing args in error case */
1097 /* Parse argument name */
1098 arg
= strchr(argv
[i
], '=');
1101 tp
->args
[i
].name
= kstrdup(argv
[i
], GFP_KERNEL
);
1104 /* If argument name is omitted, set "argN" */
1105 snprintf(buf
, MAX_EVENT_NAME_LEN
, "arg%d", i
+ 1);
1106 tp
->args
[i
].name
= kstrdup(buf
, GFP_KERNEL
);
1109 if (!tp
->args
[i
].name
) {
1110 pr_info("Failed to allocate argument[%d] name.\n", i
);
1115 if (!is_good_name(tp
->args
[i
].name
)) {
1116 pr_info("Invalid argument[%d] name: %s\n",
1117 i
, tp
->args
[i
].name
);
1122 if (conflict_field_name(tp
->args
[i
].name
, tp
->args
, i
)) {
1123 pr_info("Argument[%d] name '%s' conflicts with "
1124 "another field.\n", i
, argv
[i
]);
1129 /* Parse fetch argument */
1130 ret
= parse_probe_arg(arg
, tp
, &tp
->args
[i
], is_return
);
1132 pr_info("Parse error at argument[%d]. (%d)\n", i
, ret
);
1137 ret
= register_trace_probe(tp
);
1143 free_trace_probe(tp
);
1147 static void release_all_trace_probes(void)
1149 struct trace_probe
*tp
;
1151 mutex_lock(&probe_lock
);
1152 /* TODO: Use batch unregistration */
1153 while (!list_empty(&probe_list
)) {
1154 tp
= list_entry(probe_list
.next
, struct trace_probe
, list
);
1155 unregister_trace_probe(tp
);
1156 free_trace_probe(tp
);
1158 mutex_unlock(&probe_lock
);
1162 /* Probes listing interfaces */
1163 static void *probes_seq_start(struct seq_file
*m
, loff_t
*pos
)
1165 mutex_lock(&probe_lock
);
1166 return seq_list_start(&probe_list
, *pos
);
1169 static void *probes_seq_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
1171 return seq_list_next(v
, &probe_list
, pos
);
1174 static void probes_seq_stop(struct seq_file
*m
, void *v
)
1176 mutex_unlock(&probe_lock
);
1179 static int probes_seq_show(struct seq_file
*m
, void *v
)
1181 struct trace_probe
*tp
= v
;
1184 seq_printf(m
, "%c", trace_probe_is_return(tp
) ? 'r' : 'p');
1185 seq_printf(m
, ":%s/%s", tp
->call
.class->system
, tp
->call
.name
);
1188 seq_printf(m
, " 0x%p", tp
->rp
.kp
.addr
);
1189 else if (tp
->rp
.kp
.offset
)
1190 seq_printf(m
, " %s+%u", trace_probe_symbol(tp
),
1193 seq_printf(m
, " %s", trace_probe_symbol(tp
));
1195 for (i
= 0; i
< tp
->nr_args
; i
++)
1196 seq_printf(m
, " %s=%s", tp
->args
[i
].name
, tp
->args
[i
].comm
);
1197 seq_printf(m
, "\n");
1202 static const struct seq_operations probes_seq_op
= {
1203 .start
= probes_seq_start
,
1204 .next
= probes_seq_next
,
1205 .stop
= probes_seq_stop
,
1206 .show
= probes_seq_show
1209 static int probes_open(struct inode
*inode
, struct file
*file
)
1211 if ((file
->f_mode
& FMODE_WRITE
) &&
1212 (file
->f_flags
& O_TRUNC
))
1213 release_all_trace_probes();
1215 return seq_open(file
, &probes_seq_op
);
1218 static int command_trace_probe(const char *buf
)
1221 int argc
= 0, ret
= 0;
1223 argv
= argv_split(GFP_KERNEL
, buf
, &argc
);
1228 ret
= create_trace_probe(argc
, argv
);
1234 #define WRITE_BUFSIZE 4096
1236 static ssize_t
probes_write(struct file
*file
, const char __user
*buffer
,
1237 size_t count
, loff_t
*ppos
)
1244 kbuf
= kmalloc(WRITE_BUFSIZE
, GFP_KERNEL
);
1249 while (done
< count
) {
1250 size
= count
- done
;
1251 if (size
>= WRITE_BUFSIZE
)
1252 size
= WRITE_BUFSIZE
- 1;
1253 if (copy_from_user(kbuf
, buffer
+ done
, size
)) {
1258 tmp
= strchr(kbuf
, '\n');
1261 size
= tmp
- kbuf
+ 1;
1262 } else if (done
+ size
< count
) {
1263 pr_warning("Line length is too long: "
1264 "Should be less than %d.", WRITE_BUFSIZE
);
1269 /* Remove comments */
1270 tmp
= strchr(kbuf
, '#');
1274 ret
= command_trace_probe(kbuf
);
1284 static const struct file_operations kprobe_events_ops
= {
1285 .owner
= THIS_MODULE
,
1286 .open
= probes_open
,
1288 .llseek
= seq_lseek
,
1289 .release
= seq_release
,
1290 .write
= probes_write
,
1293 /* Probes profiling interfaces */
1294 static int probes_profile_seq_show(struct seq_file
*m
, void *v
)
1296 struct trace_probe
*tp
= v
;
1298 seq_printf(m
, " %-44s %15lu %15lu\n", tp
->call
.name
, tp
->nhit
,
1304 static const struct seq_operations profile_seq_op
= {
1305 .start
= probes_seq_start
,
1306 .next
= probes_seq_next
,
1307 .stop
= probes_seq_stop
,
1308 .show
= probes_profile_seq_show
1311 static int profile_open(struct inode
*inode
, struct file
*file
)
1313 return seq_open(file
, &profile_seq_op
);
1316 static const struct file_operations kprobe_profile_ops
= {
1317 .owner
= THIS_MODULE
,
1318 .open
= profile_open
,
1320 .llseek
= seq_lseek
,
1321 .release
= seq_release
,
1324 /* Sum up total data length for dynamic arraies (strings) */
1325 static __kprobes
int __get_data_size(struct trace_probe
*tp
,
1326 struct pt_regs
*regs
)
1331 for (i
= 0; i
< tp
->nr_args
; i
++)
1332 if (unlikely(tp
->args
[i
].fetch_size
.fn
)) {
1333 call_fetch(&tp
->args
[i
].fetch_size
, regs
, &len
);
1340 /* Store the value of each argument */
1341 static __kprobes
void store_trace_args(int ent_size
, struct trace_probe
*tp
,
1342 struct pt_regs
*regs
,
1343 u8
*data
, int maxlen
)
1347 u32
*dl
; /* Data (relative) location */
1349 for (i
= 0; i
< tp
->nr_args
; i
++) {
1350 if (unlikely(tp
->args
[i
].fetch_size
.fn
)) {
1352 * First, we set the relative location and
1353 * maximum data length to *dl
1355 dl
= (u32
*)(data
+ tp
->args
[i
].offset
);
1356 *dl
= make_data_rloc(maxlen
, end
- tp
->args
[i
].offset
);
1357 /* Then try to fetch string or dynamic array data */
1358 call_fetch(&tp
->args
[i
].fetch
, regs
, dl
);
1359 /* Reduce maximum length */
1360 end
+= get_rloc_len(*dl
);
1361 maxlen
-= get_rloc_len(*dl
);
1362 /* Trick here, convert data_rloc to data_loc */
1363 *dl
= convert_rloc_to_loc(*dl
,
1364 ent_size
+ tp
->args
[i
].offset
);
1366 /* Just fetching data normally */
1367 call_fetch(&tp
->args
[i
].fetch
, regs
,
1368 data
+ tp
->args
[i
].offset
);
1372 /* Kprobe handler */
1373 static __kprobes
void kprobe_trace_func(struct kprobe
*kp
, struct pt_regs
*regs
)
1375 struct trace_probe
*tp
= container_of(kp
, struct trace_probe
, rp
.kp
);
1376 struct kprobe_trace_entry_head
*entry
;
1377 struct ring_buffer_event
*event
;
1378 struct ring_buffer
*buffer
;
1379 int size
, dsize
, pc
;
1380 unsigned long irq_flags
;
1381 struct ftrace_event_call
*call
= &tp
->call
;
1385 local_save_flags(irq_flags
);
1386 pc
= preempt_count();
1388 dsize
= __get_data_size(tp
, regs
);
1389 size
= sizeof(*entry
) + tp
->size
+ dsize
;
1391 event
= trace_current_buffer_lock_reserve(&buffer
, call
->event
.type
,
1392 size
, irq_flags
, pc
);
1396 entry
= ring_buffer_event_data(event
);
1397 entry
->ip
= (unsigned long)kp
->addr
;
1398 store_trace_args(sizeof(*entry
), tp
, regs
, (u8
*)&entry
[1], dsize
);
1400 if (!filter_current_check_discard(buffer
, call
, entry
, event
))
1401 trace_nowake_buffer_unlock_commit_regs(buffer
, event
,
1402 irq_flags
, pc
, regs
);
1405 /* Kretprobe handler */
1406 static __kprobes
void kretprobe_trace_func(struct kretprobe_instance
*ri
,
1407 struct pt_regs
*regs
)
1409 struct trace_probe
*tp
= container_of(ri
->rp
, struct trace_probe
, rp
);
1410 struct kretprobe_trace_entry_head
*entry
;
1411 struct ring_buffer_event
*event
;
1412 struct ring_buffer
*buffer
;
1413 int size
, pc
, dsize
;
1414 unsigned long irq_flags
;
1415 struct ftrace_event_call
*call
= &tp
->call
;
1417 local_save_flags(irq_flags
);
1418 pc
= preempt_count();
1420 dsize
= __get_data_size(tp
, regs
);
1421 size
= sizeof(*entry
) + tp
->size
+ dsize
;
1423 event
= trace_current_buffer_lock_reserve(&buffer
, call
->event
.type
,
1424 size
, irq_flags
, pc
);
1428 entry
= ring_buffer_event_data(event
);
1429 entry
->func
= (unsigned long)tp
->rp
.kp
.addr
;
1430 entry
->ret_ip
= (unsigned long)ri
->ret_addr
;
1431 store_trace_args(sizeof(*entry
), tp
, regs
, (u8
*)&entry
[1], dsize
);
1433 if (!filter_current_check_discard(buffer
, call
, entry
, event
))
1434 trace_nowake_buffer_unlock_commit_regs(buffer
, event
,
1435 irq_flags
, pc
, regs
);
1438 /* Event entry printers */
1440 print_kprobe_event(struct trace_iterator
*iter
, int flags
,
1441 struct trace_event
*event
)
1443 struct kprobe_trace_entry_head
*field
;
1444 struct trace_seq
*s
= &iter
->seq
;
1445 struct trace_probe
*tp
;
1449 field
= (struct kprobe_trace_entry_head
*)iter
->ent
;
1450 tp
= container_of(event
, struct trace_probe
, call
.event
);
1452 if (!trace_seq_printf(s
, "%s: (", tp
->call
.name
))
1455 if (!seq_print_ip_sym(s
, field
->ip
, flags
| TRACE_ITER_SYM_OFFSET
))
1458 if (!trace_seq_puts(s
, ")"))
1461 data
= (u8
*)&field
[1];
1462 for (i
= 0; i
< tp
->nr_args
; i
++)
1463 if (!tp
->args
[i
].type
->print(s
, tp
->args
[i
].name
,
1464 data
+ tp
->args
[i
].offset
, field
))
1467 if (!trace_seq_puts(s
, "\n"))
1470 return TRACE_TYPE_HANDLED
;
1472 return TRACE_TYPE_PARTIAL_LINE
;
1476 print_kretprobe_event(struct trace_iterator
*iter
, int flags
,
1477 struct trace_event
*event
)
1479 struct kretprobe_trace_entry_head
*field
;
1480 struct trace_seq
*s
= &iter
->seq
;
1481 struct trace_probe
*tp
;
1485 field
= (struct kretprobe_trace_entry_head
*)iter
->ent
;
1486 tp
= container_of(event
, struct trace_probe
, call
.event
);
1488 if (!trace_seq_printf(s
, "%s: (", tp
->call
.name
))
1491 if (!seq_print_ip_sym(s
, field
->ret_ip
, flags
| TRACE_ITER_SYM_OFFSET
))
1494 if (!trace_seq_puts(s
, " <- "))
1497 if (!seq_print_ip_sym(s
, field
->func
, flags
& ~TRACE_ITER_SYM_OFFSET
))
1500 if (!trace_seq_puts(s
, ")"))
1503 data
= (u8
*)&field
[1];
1504 for (i
= 0; i
< tp
->nr_args
; i
++)
1505 if (!tp
->args
[i
].type
->print(s
, tp
->args
[i
].name
,
1506 data
+ tp
->args
[i
].offset
, field
))
1509 if (!trace_seq_puts(s
, "\n"))
1512 return TRACE_TYPE_HANDLED
;
1514 return TRACE_TYPE_PARTIAL_LINE
;
1517 static int probe_event_enable(struct ftrace_event_call
*call
)
1519 struct trace_probe
*tp
= (struct trace_probe
*)call
->data
;
1521 tp
->flags
|= TP_FLAG_TRACE
;
1522 if (trace_probe_is_return(tp
))
1523 return enable_kretprobe(&tp
->rp
);
1525 return enable_kprobe(&tp
->rp
.kp
);
1528 static void probe_event_disable(struct ftrace_event_call
*call
)
1530 struct trace_probe
*tp
= (struct trace_probe
*)call
->data
;
1532 tp
->flags
&= ~TP_FLAG_TRACE
;
1533 if (!(tp
->flags
& (TP_FLAG_TRACE
| TP_FLAG_PROFILE
))) {
1534 if (trace_probe_is_return(tp
))
1535 disable_kretprobe(&tp
->rp
);
1537 disable_kprobe(&tp
->rp
.kp
);
1542 #define DEFINE_FIELD(type, item, name, is_signed) \
1544 ret = trace_define_field(event_call, #type, name, \
1545 offsetof(typeof(field), item), \
1546 sizeof(field.item), is_signed, \
1552 static int kprobe_event_define_fields(struct ftrace_event_call
*event_call
)
1555 struct kprobe_trace_entry_head field
;
1556 struct trace_probe
*tp
= (struct trace_probe
*)event_call
->data
;
1558 DEFINE_FIELD(unsigned long, ip
, FIELD_STRING_IP
, 0);
1559 /* Set argument names as fields */
1560 for (i
= 0; i
< tp
->nr_args
; i
++) {
1561 ret
= trace_define_field(event_call
, tp
->args
[i
].type
->fmttype
,
1563 sizeof(field
) + tp
->args
[i
].offset
,
1564 tp
->args
[i
].type
->size
,
1565 tp
->args
[i
].type
->is_signed
,
1573 static int kretprobe_event_define_fields(struct ftrace_event_call
*event_call
)
1576 struct kretprobe_trace_entry_head field
;
1577 struct trace_probe
*tp
= (struct trace_probe
*)event_call
->data
;
1579 DEFINE_FIELD(unsigned long, func
, FIELD_STRING_FUNC
, 0);
1580 DEFINE_FIELD(unsigned long, ret_ip
, FIELD_STRING_RETIP
, 0);
1581 /* Set argument names as fields */
1582 for (i
= 0; i
< tp
->nr_args
; i
++) {
1583 ret
= trace_define_field(event_call
, tp
->args
[i
].type
->fmttype
,
1585 sizeof(field
) + tp
->args
[i
].offset
,
1586 tp
->args
[i
].type
->size
,
1587 tp
->args
[i
].type
->is_signed
,
1595 static int __set_print_fmt(struct trace_probe
*tp
, char *buf
, int len
)
1600 const char *fmt
, *arg
;
1602 if (!trace_probe_is_return(tp
)) {
1604 arg
= "REC->" FIELD_STRING_IP
;
1606 fmt
= "(%lx <- %lx)";
1607 arg
= "REC->" FIELD_STRING_FUNC
", REC->" FIELD_STRING_RETIP
;
1610 /* When len=0, we just calculate the needed length */
1611 #define LEN_OR_ZERO (len ? len - pos : 0)
1613 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
, "\"%s", fmt
);
1615 for (i
= 0; i
< tp
->nr_args
; i
++) {
1616 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
, " %s=%s",
1617 tp
->args
[i
].name
, tp
->args
[i
].type
->fmt
);
1620 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
, "\", %s", arg
);
1622 for (i
= 0; i
< tp
->nr_args
; i
++) {
1623 if (strcmp(tp
->args
[i
].type
->name
, "string") == 0)
1624 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
,
1628 pos
+= snprintf(buf
+ pos
, LEN_OR_ZERO
, ", REC->%s",
1634 /* return the length of print_fmt */
1638 static int set_print_fmt(struct trace_probe
*tp
)
1643 /* First: called with 0 length to calculate the needed length */
1644 len
= __set_print_fmt(tp
, NULL
, 0);
1645 print_fmt
= kmalloc(len
+ 1, GFP_KERNEL
);
1649 /* Second: actually write the @print_fmt */
1650 __set_print_fmt(tp
, print_fmt
, len
+ 1);
1651 tp
->call
.print_fmt
= print_fmt
;
1656 #ifdef CONFIG_PERF_EVENTS
1658 /* Kprobe profile handler */
1659 static __kprobes
void kprobe_perf_func(struct kprobe
*kp
,
1660 struct pt_regs
*regs
)
1662 struct trace_probe
*tp
= container_of(kp
, struct trace_probe
, rp
.kp
);
1663 struct ftrace_event_call
*call
= &tp
->call
;
1664 struct kprobe_trace_entry_head
*entry
;
1665 struct hlist_head
*head
;
1666 int size
, __size
, dsize
;
1669 dsize
= __get_data_size(tp
, regs
);
1670 __size
= sizeof(*entry
) + tp
->size
+ dsize
;
1671 size
= ALIGN(__size
+ sizeof(u32
), sizeof(u64
));
1672 size
-= sizeof(u32
);
1673 if (WARN_ONCE(size
> PERF_MAX_TRACE_SIZE
,
1674 "profile buffer not large enough"))
1677 entry
= perf_trace_buf_prepare(size
, call
->event
.type
, regs
, &rctx
);
1681 entry
->ip
= (unsigned long)kp
->addr
;
1682 memset(&entry
[1], 0, dsize
);
1683 store_trace_args(sizeof(*entry
), tp
, regs
, (u8
*)&entry
[1], dsize
);
1685 head
= this_cpu_ptr(call
->perf_events
);
1686 perf_trace_buf_submit(entry
, size
, rctx
, entry
->ip
, 1, regs
, head
);
1689 /* Kretprobe profile handler */
1690 static __kprobes
void kretprobe_perf_func(struct kretprobe_instance
*ri
,
1691 struct pt_regs
*regs
)
1693 struct trace_probe
*tp
= container_of(ri
->rp
, struct trace_probe
, rp
);
1694 struct ftrace_event_call
*call
= &tp
->call
;
1695 struct kretprobe_trace_entry_head
*entry
;
1696 struct hlist_head
*head
;
1697 int size
, __size
, dsize
;
1700 dsize
= __get_data_size(tp
, regs
);
1701 __size
= sizeof(*entry
) + tp
->size
+ dsize
;
1702 size
= ALIGN(__size
+ sizeof(u32
), sizeof(u64
));
1703 size
-= sizeof(u32
);
1704 if (WARN_ONCE(size
> PERF_MAX_TRACE_SIZE
,
1705 "profile buffer not large enough"))
1708 entry
= perf_trace_buf_prepare(size
, call
->event
.type
, regs
, &rctx
);
1712 entry
->func
= (unsigned long)tp
->rp
.kp
.addr
;
1713 entry
->ret_ip
= (unsigned long)ri
->ret_addr
;
1714 store_trace_args(sizeof(*entry
), tp
, regs
, (u8
*)&entry
[1], dsize
);
1716 head
= this_cpu_ptr(call
->perf_events
);
1717 perf_trace_buf_submit(entry
, size
, rctx
, entry
->ret_ip
, 1, regs
, head
);
1720 static int probe_perf_enable(struct ftrace_event_call
*call
)
1722 struct trace_probe
*tp
= (struct trace_probe
*)call
->data
;
1724 tp
->flags
|= TP_FLAG_PROFILE
;
1726 if (trace_probe_is_return(tp
))
1727 return enable_kretprobe(&tp
->rp
);
1729 return enable_kprobe(&tp
->rp
.kp
);
1732 static void probe_perf_disable(struct ftrace_event_call
*call
)
1734 struct trace_probe
*tp
= (struct trace_probe
*)call
->data
;
1736 tp
->flags
&= ~TP_FLAG_PROFILE
;
1738 if (!(tp
->flags
& TP_FLAG_TRACE
)) {
1739 if (trace_probe_is_return(tp
))
1740 disable_kretprobe(&tp
->rp
);
1742 disable_kprobe(&tp
->rp
.kp
);
1745 #endif /* CONFIG_PERF_EVENTS */
1748 int kprobe_register(struct ftrace_event_call
*event
, enum trace_reg type
)
1751 case TRACE_REG_REGISTER
:
1752 return probe_event_enable(event
);
1753 case TRACE_REG_UNREGISTER
:
1754 probe_event_disable(event
);
1757 #ifdef CONFIG_PERF_EVENTS
1758 case TRACE_REG_PERF_REGISTER
:
1759 return probe_perf_enable(event
);
1760 case TRACE_REG_PERF_UNREGISTER
:
1761 probe_perf_disable(event
);
1769 int kprobe_dispatcher(struct kprobe
*kp
, struct pt_regs
*regs
)
1771 struct trace_probe
*tp
= container_of(kp
, struct trace_probe
, rp
.kp
);
1773 if (tp
->flags
& TP_FLAG_TRACE
)
1774 kprobe_trace_func(kp
, regs
);
1775 #ifdef CONFIG_PERF_EVENTS
1776 if (tp
->flags
& TP_FLAG_PROFILE
)
1777 kprobe_perf_func(kp
, regs
);
1779 return 0; /* We don't tweek kernel, so just return 0 */
1783 int kretprobe_dispatcher(struct kretprobe_instance
*ri
, struct pt_regs
*regs
)
1785 struct trace_probe
*tp
= container_of(ri
->rp
, struct trace_probe
, rp
);
1787 if (tp
->flags
& TP_FLAG_TRACE
)
1788 kretprobe_trace_func(ri
, regs
);
1789 #ifdef CONFIG_PERF_EVENTS
1790 if (tp
->flags
& TP_FLAG_PROFILE
)
1791 kretprobe_perf_func(ri
, regs
);
1793 return 0; /* We don't tweek kernel, so just return 0 */
1796 static struct trace_event_functions kretprobe_funcs
= {
1797 .trace
= print_kretprobe_event
1800 static struct trace_event_functions kprobe_funcs
= {
1801 .trace
= print_kprobe_event
1804 static int register_probe_event(struct trace_probe
*tp
)
1806 struct ftrace_event_call
*call
= &tp
->call
;
1809 /* Initialize ftrace_event_call */
1810 INIT_LIST_HEAD(&call
->class->fields
);
1811 if (trace_probe_is_return(tp
)) {
1812 call
->event
.funcs
= &kretprobe_funcs
;
1813 call
->class->define_fields
= kretprobe_event_define_fields
;
1815 call
->event
.funcs
= &kprobe_funcs
;
1816 call
->class->define_fields
= kprobe_event_define_fields
;
1818 if (set_print_fmt(tp
) < 0)
1820 ret
= register_ftrace_event(&call
->event
);
1822 kfree(call
->print_fmt
);
1826 call
->class->reg
= kprobe_register
;
1828 ret
= trace_add_event_call(call
);
1830 pr_info("Failed to register kprobe event: %s\n", call
->name
);
1831 kfree(call
->print_fmt
);
1832 unregister_ftrace_event(&call
->event
);
1837 static void unregister_probe_event(struct trace_probe
*tp
)
1839 /* tp->event is unregistered in trace_remove_event_call() */
1840 trace_remove_event_call(&tp
->call
);
1841 kfree(tp
->call
.print_fmt
);
1844 /* Make a debugfs interface for controlling probe points */
1845 static __init
int init_kprobe_trace(void)
1847 struct dentry
*d_tracer
;
1848 struct dentry
*entry
;
1850 d_tracer
= tracing_init_dentry();
1854 entry
= debugfs_create_file("kprobe_events", 0644, d_tracer
,
1855 NULL
, &kprobe_events_ops
);
1857 /* Event list interface */
1859 pr_warning("Could not create debugfs "
1860 "'kprobe_events' entry\n");
1862 /* Profile interface */
1863 entry
= debugfs_create_file("kprobe_profile", 0444, d_tracer
,
1864 NULL
, &kprobe_profile_ops
);
1867 pr_warning("Could not create debugfs "
1868 "'kprobe_profile' entry\n");
1871 fs_initcall(init_kprobe_trace
);
1874 #ifdef CONFIG_FTRACE_STARTUP_TEST
1877 * The "__used" keeps gcc from removing the function symbol
1878 * from the kallsyms table.
1880 static __used
int kprobe_trace_selftest_target(int a1
, int a2
, int a3
,
1881 int a4
, int a5
, int a6
)
1883 return a1
+ a2
+ a3
+ a4
+ a5
+ a6
;
1886 static __init
int kprobe_trace_self_tests_init(void)
1889 int (*target
)(int, int, int, int, int, int);
1890 struct trace_probe
*tp
;
1892 target
= kprobe_trace_selftest_target
;
1894 pr_info("Testing kprobe tracing: ");
1896 ret
= command_trace_probe("p:testprobe kprobe_trace_selftest_target "
1897 "$stack $stack0 +0($stack)");
1898 if (WARN_ON_ONCE(ret
)) {
1899 pr_warning("error on probing function entry.\n");
1902 /* Enable trace point */
1903 tp
= find_trace_probe("testprobe", KPROBE_EVENT_SYSTEM
);
1904 if (WARN_ON_ONCE(tp
== NULL
)) {
1905 pr_warning("error on getting new probe.\n");
1908 probe_event_enable(&tp
->call
);
1911 ret
= command_trace_probe("r:testprobe2 kprobe_trace_selftest_target "
1913 if (WARN_ON_ONCE(ret
)) {
1914 pr_warning("error on probing function return.\n");
1917 /* Enable trace point */
1918 tp
= find_trace_probe("testprobe2", KPROBE_EVENT_SYSTEM
);
1919 if (WARN_ON_ONCE(tp
== NULL
)) {
1920 pr_warning("error on getting new probe.\n");
1923 probe_event_enable(&tp
->call
);
1929 ret
= target(1, 2, 3, 4, 5, 6);
1931 ret
= command_trace_probe("-:testprobe");
1932 if (WARN_ON_ONCE(ret
)) {
1933 pr_warning("error on deleting a probe.\n");
1937 ret
= command_trace_probe("-:testprobe2");
1938 if (WARN_ON_ONCE(ret
)) {
1939 pr_warning("error on deleting a probe.\n");
1944 release_all_trace_probes();
1946 pr_cont("NG: Some tests are failed. Please check them.\n");
1952 late_initcall(kprobe_trace_self_tests_init
);