2 * Infrastructure for profiling code inserted by 'gcc -pg'.
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
7 * Originally ported from the -rt patch by:
8 * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code in the latency_tracer, that is:
12 * Copyright (C) 2004-2006 Ingo Molnar
13 * Copyright (C) 2004 William Lee Irwin III
16 #include <linux/stop_machine.h>
17 #include <linux/clocksource.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/suspend.h>
21 #include <linux/debugfs.h>
22 #include <linux/hardirq.h>
23 #include <linux/kthread.h>
24 #include <linux/uaccess.h>
25 #include <linux/kprobes.h>
26 #include <linux/ftrace.h>
27 #include <linux/sysctl.h>
28 #include <linux/ctype.h>
29 #include <linux/list.h>
30 #include <linux/hash.h>
32 #include <trace/sched.h>
34 #include <asm/ftrace.h>
36 #include "trace_output.h"
37 #include "trace_stat.h"
39 #define FTRACE_WARN_ON(cond) \
45 #define FTRACE_WARN_ON_ONCE(cond) \
47 if (WARN_ON_ONCE(cond)) \
51 /* hash bits for specific function selection */
52 #define FTRACE_HASH_BITS 7
53 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
55 /* ftrace_enabled is a method to turn ftrace on or off */
56 int ftrace_enabled __read_mostly
;
57 static int last_ftrace_enabled
;
59 /* Quick disabling of function tracer. */
60 int function_trace_stop
;
63 * ftrace_disabled is set when an anomaly is discovered.
64 * ftrace_disabled is much stronger than ftrace_enabled.
66 static int ftrace_disabled __read_mostly
;
68 static DEFINE_MUTEX(ftrace_lock
);
70 static struct ftrace_ops ftrace_list_end __read_mostly
=
75 static struct ftrace_ops
*ftrace_list __read_mostly
= &ftrace_list_end
;
76 ftrace_func_t ftrace_trace_function __read_mostly
= ftrace_stub
;
77 ftrace_func_t __ftrace_trace_function __read_mostly
= ftrace_stub
;
78 ftrace_func_t ftrace_pid_function __read_mostly
= ftrace_stub
;
80 static void ftrace_list_func(unsigned long ip
, unsigned long parent_ip
)
82 struct ftrace_ops
*op
= ftrace_list
;
84 /* in case someone actually ports this to alpha! */
85 read_barrier_depends();
87 while (op
!= &ftrace_list_end
) {
89 read_barrier_depends();
90 op
->func(ip
, parent_ip
);
95 static void ftrace_pid_func(unsigned long ip
, unsigned long parent_ip
)
97 if (!test_tsk_trace_trace(current
))
100 ftrace_pid_function(ip
, parent_ip
);
103 static void set_ftrace_pid_function(ftrace_func_t func
)
105 /* do not set ftrace_pid_function to itself! */
106 if (func
!= ftrace_pid_func
)
107 ftrace_pid_function
= func
;
111 * clear_ftrace_function - reset the ftrace function
113 * This NULLs the ftrace function and in essence stops
114 * tracing. There may be lag
116 void clear_ftrace_function(void)
118 ftrace_trace_function
= ftrace_stub
;
119 __ftrace_trace_function
= ftrace_stub
;
120 ftrace_pid_function
= ftrace_stub
;
123 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
125 * For those archs that do not test ftrace_trace_stop in their
126 * mcount call site, we need to do it from C.
128 static void ftrace_test_stop_func(unsigned long ip
, unsigned long parent_ip
)
130 if (function_trace_stop
)
133 __ftrace_trace_function(ip
, parent_ip
);
137 static int __register_ftrace_function(struct ftrace_ops
*ops
)
139 ops
->next
= ftrace_list
;
141 * We are entering ops into the ftrace_list but another
142 * CPU might be walking that list. We need to make sure
143 * the ops->next pointer is valid before another CPU sees
144 * the ops pointer included into the ftrace_list.
149 if (ftrace_enabled
) {
152 if (ops
->next
== &ftrace_list_end
)
155 func
= ftrace_list_func
;
157 if (ftrace_pid_trace
) {
158 set_ftrace_pid_function(func
);
159 func
= ftrace_pid_func
;
163 * For one func, simply call it directly.
164 * For more than one func, call the chain.
166 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
167 ftrace_trace_function
= func
;
169 __ftrace_trace_function
= func
;
170 ftrace_trace_function
= ftrace_test_stop_func
;
177 static int __unregister_ftrace_function(struct ftrace_ops
*ops
)
179 struct ftrace_ops
**p
;
182 * If we are removing the last function, then simply point
183 * to the ftrace_stub.
185 if (ftrace_list
== ops
&& ops
->next
== &ftrace_list_end
) {
186 ftrace_trace_function
= ftrace_stub
;
187 ftrace_list
= &ftrace_list_end
;
191 for (p
= &ftrace_list
; *p
!= &ftrace_list_end
; p
= &(*p
)->next
)
200 if (ftrace_enabled
) {
201 /* If we only have one func left, then call that directly */
202 if (ftrace_list
->next
== &ftrace_list_end
) {
203 ftrace_func_t func
= ftrace_list
->func
;
205 if (ftrace_pid_trace
) {
206 set_ftrace_pid_function(func
);
207 func
= ftrace_pid_func
;
209 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
210 ftrace_trace_function
= func
;
212 __ftrace_trace_function
= func
;
220 static void ftrace_update_pid_func(void)
224 if (ftrace_trace_function
== ftrace_stub
)
227 func
= ftrace_trace_function
;
229 if (ftrace_pid_trace
) {
230 set_ftrace_pid_function(func
);
231 func
= ftrace_pid_func
;
233 if (func
== ftrace_pid_func
)
234 func
= ftrace_pid_function
;
237 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
238 ftrace_trace_function
= func
;
240 __ftrace_trace_function
= func
;
244 #ifdef CONFIG_FUNCTION_PROFILER
245 struct ftrace_profile
{
246 struct hlist_node node
;
248 unsigned long counter
;
249 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
250 unsigned long long time
;
254 struct ftrace_profile_page
{
255 struct ftrace_profile_page
*next
;
257 struct ftrace_profile records
[];
260 struct ftrace_profile_stat
{
262 struct hlist_head
*hash
;
263 struct ftrace_profile_page
*pages
;
264 struct ftrace_profile_page
*start
;
265 struct tracer_stat stat
;
268 #define PROFILE_RECORDS_SIZE \
269 (PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
271 #define PROFILES_PER_PAGE \
272 (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
274 static int ftrace_profile_bits
;
275 static int ftrace_profile_enabled
;
276 static DEFINE_MUTEX(ftrace_profile_lock
);
278 static DEFINE_PER_CPU(struct ftrace_profile_stat
, ftrace_profile_stats
);
280 #define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */
283 function_stat_next(void *v
, int idx
)
285 struct ftrace_profile
*rec
= v
;
286 struct ftrace_profile_page
*pg
;
288 pg
= (struct ftrace_profile_page
*)((unsigned long)rec
& PAGE_MASK
);
292 if ((void *)rec
>= (void *)&pg
->records
[pg
->index
]) {
296 rec
= &pg
->records
[0];
304 static void *function_stat_start(struct tracer_stat
*trace
)
306 struct ftrace_profile_stat
*stat
=
307 container_of(trace
, struct ftrace_profile_stat
, stat
);
309 if (!stat
|| !stat
->start
)
312 return function_stat_next(&stat
->start
->records
[0], 0);
315 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
316 /* function graph compares on total time */
317 static int function_stat_cmp(void *p1
, void *p2
)
319 struct ftrace_profile
*a
= p1
;
320 struct ftrace_profile
*b
= p2
;
322 if (a
->time
< b
->time
)
324 if (a
->time
> b
->time
)
330 /* not function graph compares against hits */
331 static int function_stat_cmp(void *p1
, void *p2
)
333 struct ftrace_profile
*a
= p1
;
334 struct ftrace_profile
*b
= p2
;
336 if (a
->counter
< b
->counter
)
338 if (a
->counter
> b
->counter
)
345 static int function_stat_headers(struct seq_file
*m
)
347 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
348 seq_printf(m
, " Function "
353 seq_printf(m
, " Function Hit\n"
359 static int function_stat_show(struct seq_file
*m
, void *v
)
361 struct ftrace_profile
*rec
= v
;
362 char str
[KSYM_SYMBOL_LEN
];
363 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
364 static DEFINE_MUTEX(mutex
);
365 static struct trace_seq s
;
366 unsigned long long avg
;
369 kallsyms_lookup(rec
->ip
, NULL
, NULL
, NULL
, str
);
370 seq_printf(m
, " %-30.30s %10lu", str
, rec
->counter
);
372 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
376 do_div(avg
, rec
->counter
);
380 trace_print_graph_duration(rec
->time
, &s
);
381 trace_seq_puts(&s
, " ");
382 trace_print_graph_duration(avg
, &s
);
383 trace_print_seq(m
, &s
);
384 mutex_unlock(&mutex
);
391 static void ftrace_profile_reset(struct ftrace_profile_stat
*stat
)
393 struct ftrace_profile_page
*pg
;
395 pg
= stat
->pages
= stat
->start
;
398 memset(pg
->records
, 0, PROFILE_RECORDS_SIZE
);
403 memset(stat
->hash
, 0,
404 FTRACE_PROFILE_HASH_SIZE
* sizeof(struct hlist_head
));
407 int ftrace_profile_pages_init(struct ftrace_profile_stat
*stat
)
409 struct ftrace_profile_page
*pg
;
414 /* If we already allocated, do nothing */
418 stat
->pages
= (void *)get_zeroed_page(GFP_KERNEL
);
422 #ifdef CONFIG_DYNAMIC_FTRACE
423 functions
= ftrace_update_tot_cnt
;
426 * We do not know the number of functions that exist because
427 * dynamic tracing is what counts them. With past experience
428 * we have around 20K functions. That should be more than enough.
429 * It is highly unlikely we will execute every function in
435 pg
= stat
->start
= stat
->pages
;
437 pages
= DIV_ROUND_UP(functions
, PROFILES_PER_PAGE
);
439 for (i
= 0; i
< pages
; i
++) {
440 pg
->next
= (void *)get_zeroed_page(GFP_KERNEL
);
451 unsigned long tmp
= (unsigned long)pg
;
457 free_page((unsigned long)stat
->pages
);
464 static int ftrace_profile_init_cpu(int cpu
)
466 struct ftrace_profile_stat
*stat
;
469 stat
= &per_cpu(ftrace_profile_stats
, cpu
);
472 /* If the profile is already created, simply reset it */
473 ftrace_profile_reset(stat
);
478 * We are profiling all functions, but usually only a few thousand
479 * functions are hit. We'll make a hash of 1024 items.
481 size
= FTRACE_PROFILE_HASH_SIZE
;
483 stat
->hash
= kzalloc(sizeof(struct hlist_head
) * size
, GFP_KERNEL
);
488 if (!ftrace_profile_bits
) {
491 for (; size
; size
>>= 1)
492 ftrace_profile_bits
++;
495 /* Preallocate the function profiling pages */
496 if (ftrace_profile_pages_init(stat
) < 0) {
505 static int ftrace_profile_init(void)
510 for_each_online_cpu(cpu
) {
511 ret
= ftrace_profile_init_cpu(cpu
);
519 /* interrupts must be disabled */
520 static struct ftrace_profile
*
521 ftrace_find_profiled_func(struct ftrace_profile_stat
*stat
, unsigned long ip
)
523 struct ftrace_profile
*rec
;
524 struct hlist_head
*hhd
;
525 struct hlist_node
*n
;
528 key
= hash_long(ip
, ftrace_profile_bits
);
529 hhd
= &stat
->hash
[key
];
531 if (hlist_empty(hhd
))
534 hlist_for_each_entry_rcu(rec
, n
, hhd
, node
) {
542 static void ftrace_add_profile(struct ftrace_profile_stat
*stat
,
543 struct ftrace_profile
*rec
)
547 key
= hash_long(rec
->ip
, ftrace_profile_bits
);
548 hlist_add_head_rcu(&rec
->node
, &stat
->hash
[key
]);
552 * The memory is already allocated, this simply finds a new record to use.
554 static struct ftrace_profile
*
555 ftrace_profile_alloc(struct ftrace_profile_stat
*stat
, unsigned long ip
)
557 struct ftrace_profile
*rec
= NULL
;
559 /* prevent recursion (from NMIs) */
560 if (atomic_inc_return(&stat
->disabled
) != 1)
564 * Try to find the function again since an NMI
565 * could have added it
567 rec
= ftrace_find_profiled_func(stat
, ip
);
571 if (stat
->pages
->index
== PROFILES_PER_PAGE
) {
572 if (!stat
->pages
->next
)
574 stat
->pages
= stat
->pages
->next
;
577 rec
= &stat
->pages
->records
[stat
->pages
->index
++];
579 ftrace_add_profile(stat
, rec
);
582 atomic_dec(&stat
->disabled
);
588 function_profile_call(unsigned long ip
, unsigned long parent_ip
)
590 struct ftrace_profile_stat
*stat
;
591 struct ftrace_profile
*rec
;
594 if (!ftrace_profile_enabled
)
597 local_irq_save(flags
);
599 stat
= &__get_cpu_var(ftrace_profile_stats
);
600 if (!stat
->hash
|| !ftrace_profile_enabled
)
603 rec
= ftrace_find_profiled_func(stat
, ip
);
605 rec
= ftrace_profile_alloc(stat
, ip
);
612 local_irq_restore(flags
);
615 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
616 static int profile_graph_entry(struct ftrace_graph_ent
*trace
)
618 function_profile_call(trace
->func
, 0);
622 static void profile_graph_return(struct ftrace_graph_ret
*trace
)
624 struct ftrace_profile_stat
*stat
;
625 unsigned long long calltime
;
626 struct ftrace_profile
*rec
;
629 local_irq_save(flags
);
630 stat
= &__get_cpu_var(ftrace_profile_stats
);
631 if (!stat
->hash
|| !ftrace_profile_enabled
)
634 calltime
= trace
->rettime
- trace
->calltime
;
636 if (!(trace_flags
& TRACE_ITER_GRAPH_TIME
)) {
639 index
= trace
->depth
;
641 /* Append this call time to the parent time to subtract */
643 current
->ret_stack
[index
- 1].subtime
+= calltime
;
645 if (current
->ret_stack
[index
].subtime
< calltime
)
646 calltime
-= current
->ret_stack
[index
].subtime
;
651 rec
= ftrace_find_profiled_func(stat
, trace
->func
);
653 rec
->time
+= calltime
;
656 local_irq_restore(flags
);
659 static int register_ftrace_profiler(void)
661 return register_ftrace_graph(&profile_graph_return
,
662 &profile_graph_entry
);
665 static void unregister_ftrace_profiler(void)
667 unregister_ftrace_graph();
670 static struct ftrace_ops ftrace_profile_ops __read_mostly
=
672 .func
= function_profile_call
,
675 static int register_ftrace_profiler(void)
677 return register_ftrace_function(&ftrace_profile_ops
);
680 static void unregister_ftrace_profiler(void)
682 unregister_ftrace_function(&ftrace_profile_ops
);
684 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
687 ftrace_profile_write(struct file
*filp
, const char __user
*ubuf
,
688 size_t cnt
, loff_t
*ppos
)
694 if (cnt
>= sizeof(buf
))
697 if (copy_from_user(&buf
, ubuf
, cnt
))
702 ret
= strict_strtoul(buf
, 10, &val
);
708 mutex_lock(&ftrace_profile_lock
);
709 if (ftrace_profile_enabled
^ val
) {
711 ret
= ftrace_profile_init();
717 ret
= register_ftrace_profiler();
722 ftrace_profile_enabled
= 1;
724 ftrace_profile_enabled
= 0;
726 * unregister_ftrace_profiler calls stop_machine
727 * so this acts like an synchronize_sched.
729 unregister_ftrace_profiler();
733 mutex_unlock(&ftrace_profile_lock
);
741 ftrace_profile_read(struct file
*filp
, char __user
*ubuf
,
742 size_t cnt
, loff_t
*ppos
)
747 r
= sprintf(buf
, "%u\n", ftrace_profile_enabled
);
748 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
751 static const struct file_operations ftrace_profile_fops
= {
752 .open
= tracing_open_generic
,
753 .read
= ftrace_profile_read
,
754 .write
= ftrace_profile_write
,
757 /* used to initialize the real stat files */
758 static struct tracer_stat function_stats __initdata
= {
760 .stat_start
= function_stat_start
,
761 .stat_next
= function_stat_next
,
762 .stat_cmp
= function_stat_cmp
,
763 .stat_headers
= function_stat_headers
,
764 .stat_show
= function_stat_show
767 static void ftrace_profile_debugfs(struct dentry
*d_tracer
)
769 struct ftrace_profile_stat
*stat
;
770 struct dentry
*entry
;
775 for_each_possible_cpu(cpu
) {
776 stat
= &per_cpu(ftrace_profile_stats
, cpu
);
778 /* allocate enough for function name + cpu number */
779 name
= kmalloc(32, GFP_KERNEL
);
782 * The files created are permanent, if something happens
783 * we still do not free memory.
787 "Could not allocate stat file for cpu %d\n",
791 stat
->stat
= function_stats
;
792 snprintf(name
, 32, "function%d", cpu
);
793 stat
->stat
.name
= name
;
794 ret
= register_stat_tracer(&stat
->stat
);
797 "Could not register function stat for cpu %d\n",
804 entry
= debugfs_create_file("function_profile_enabled", 0644,
805 d_tracer
, NULL
, &ftrace_profile_fops
);
807 pr_warning("Could not create debugfs "
808 "'function_profile_enabled' entry\n");
811 #else /* CONFIG_FUNCTION_PROFILER */
812 static void ftrace_profile_debugfs(struct dentry
*d_tracer
)
815 #endif /* CONFIG_FUNCTION_PROFILER */
817 /* set when tracing only a pid */
818 struct pid
*ftrace_pid_trace
;
819 static struct pid
* const ftrace_swapper_pid
= &init_struct_pid
;
821 #ifdef CONFIG_DYNAMIC_FTRACE
823 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
824 # error Dynamic ftrace depends on MCOUNT_RECORD
827 static struct hlist_head ftrace_func_hash
[FTRACE_FUNC_HASHSIZE
] __read_mostly
;
829 struct ftrace_func_probe
{
830 struct hlist_node node
;
831 struct ftrace_probe_ops
*ops
;
839 FTRACE_ENABLE_CALLS
= (1 << 0),
840 FTRACE_DISABLE_CALLS
= (1 << 1),
841 FTRACE_UPDATE_TRACE_FUNC
= (1 << 2),
842 FTRACE_ENABLE_MCOUNT
= (1 << 3),
843 FTRACE_DISABLE_MCOUNT
= (1 << 4),
844 FTRACE_START_FUNC_RET
= (1 << 5),
845 FTRACE_STOP_FUNC_RET
= (1 << 6),
848 static int ftrace_filtered
;
850 static struct dyn_ftrace
*ftrace_new_addrs
;
852 static DEFINE_MUTEX(ftrace_regex_lock
);
855 struct ftrace_page
*next
;
857 struct dyn_ftrace records
[];
860 #define ENTRIES_PER_PAGE \
861 ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
863 /* estimate from running different kernels */
864 #define NR_TO_INIT 10000
866 static struct ftrace_page
*ftrace_pages_start
;
867 static struct ftrace_page
*ftrace_pages
;
869 static struct dyn_ftrace
*ftrace_free_records
;
872 * This is a double for. Do not use 'break' to break out of the loop,
873 * you must use a goto.
875 #define do_for_each_ftrace_rec(pg, rec) \
876 for (pg = ftrace_pages_start; pg; pg = pg->next) { \
878 for (_____i = 0; _____i < pg->index; _____i++) { \
879 rec = &pg->records[_____i];
881 #define while_for_each_ftrace_rec() \
885 #ifdef CONFIG_KPROBES
887 static int frozen_record_count
;
889 static inline void freeze_record(struct dyn_ftrace
*rec
)
891 if (!(rec
->flags
& FTRACE_FL_FROZEN
)) {
892 rec
->flags
|= FTRACE_FL_FROZEN
;
893 frozen_record_count
++;
897 static inline void unfreeze_record(struct dyn_ftrace
*rec
)
899 if (rec
->flags
& FTRACE_FL_FROZEN
) {
900 rec
->flags
&= ~FTRACE_FL_FROZEN
;
901 frozen_record_count
--;
905 static inline int record_frozen(struct dyn_ftrace
*rec
)
907 return rec
->flags
& FTRACE_FL_FROZEN
;
910 # define freeze_record(rec) ({ 0; })
911 # define unfreeze_record(rec) ({ 0; })
912 # define record_frozen(rec) ({ 0; })
913 #endif /* CONFIG_KPROBES */
915 static void ftrace_free_rec(struct dyn_ftrace
*rec
)
917 rec
->freelist
= ftrace_free_records
;
918 ftrace_free_records
= rec
;
919 rec
->flags
|= FTRACE_FL_FREE
;
922 void ftrace_release(void *start
, unsigned long size
)
924 struct dyn_ftrace
*rec
;
925 struct ftrace_page
*pg
;
926 unsigned long s
= (unsigned long)start
;
927 unsigned long e
= s
+ size
;
929 if (ftrace_disabled
|| !start
)
932 mutex_lock(&ftrace_lock
);
933 do_for_each_ftrace_rec(pg
, rec
) {
934 if ((rec
->ip
>= s
) && (rec
->ip
< e
) &&
935 !(rec
->flags
& FTRACE_FL_FREE
))
936 ftrace_free_rec(rec
);
937 } while_for_each_ftrace_rec();
938 mutex_unlock(&ftrace_lock
);
941 static struct dyn_ftrace
*ftrace_alloc_dyn_node(unsigned long ip
)
943 struct dyn_ftrace
*rec
;
945 /* First check for freed records */
946 if (ftrace_free_records
) {
947 rec
= ftrace_free_records
;
949 if (unlikely(!(rec
->flags
& FTRACE_FL_FREE
))) {
950 FTRACE_WARN_ON_ONCE(1);
951 ftrace_free_records
= NULL
;
955 ftrace_free_records
= rec
->freelist
;
956 memset(rec
, 0, sizeof(*rec
));
960 if (ftrace_pages
->index
== ENTRIES_PER_PAGE
) {
961 if (!ftrace_pages
->next
) {
962 /* allocate another page */
964 (void *)get_zeroed_page(GFP_KERNEL
);
965 if (!ftrace_pages
->next
)
968 ftrace_pages
= ftrace_pages
->next
;
971 return &ftrace_pages
->records
[ftrace_pages
->index
++];
974 static struct dyn_ftrace
*
975 ftrace_record_ip(unsigned long ip
)
977 struct dyn_ftrace
*rec
;
982 rec
= ftrace_alloc_dyn_node(ip
);
987 rec
->newlist
= ftrace_new_addrs
;
988 ftrace_new_addrs
= rec
;
993 static void print_ip_ins(const char *fmt
, unsigned char *p
)
997 printk(KERN_CONT
"%s", fmt
);
999 for (i
= 0; i
< MCOUNT_INSN_SIZE
; i
++)
1000 printk(KERN_CONT
"%s%02x", i
? ":" : "", p
[i
]);
1003 static void ftrace_bug(int failed
, unsigned long ip
)
1007 FTRACE_WARN_ON_ONCE(1);
1008 pr_info("ftrace faulted on modifying ");
1012 FTRACE_WARN_ON_ONCE(1);
1013 pr_info("ftrace failed to modify ");
1015 print_ip_ins(" actual: ", (unsigned char *)ip
);
1016 printk(KERN_CONT
"\n");
1019 FTRACE_WARN_ON_ONCE(1);
1020 pr_info("ftrace faulted on writing ");
1024 FTRACE_WARN_ON_ONCE(1);
1025 pr_info("ftrace faulted on unknown error ");
1032 __ftrace_replace_code(struct dyn_ftrace
*rec
, int enable
)
1034 unsigned long ftrace_addr
;
1035 unsigned long ip
, fl
;
1037 ftrace_addr
= (unsigned long)FTRACE_ADDR
;
1042 * If this record is not to be traced and
1043 * it is not enabled then do nothing.
1045 * If this record is not to be traced and
1046 * it is enabled then disable it.
1049 if (rec
->flags
& FTRACE_FL_NOTRACE
) {
1050 if (rec
->flags
& FTRACE_FL_ENABLED
)
1051 rec
->flags
&= ~FTRACE_FL_ENABLED
;
1055 } else if (ftrace_filtered
&& enable
) {
1060 fl
= rec
->flags
& (FTRACE_FL_FILTER
| FTRACE_FL_ENABLED
);
1062 /* Record is filtered and enabled, do nothing */
1063 if (fl
== (FTRACE_FL_FILTER
| FTRACE_FL_ENABLED
))
1066 /* Record is not filtered or enabled, do nothing */
1070 /* Record is not filtered but enabled, disable it */
1071 if (fl
== FTRACE_FL_ENABLED
)
1072 rec
->flags
&= ~FTRACE_FL_ENABLED
;
1074 /* Otherwise record is filtered but not enabled, enable it */
1075 rec
->flags
|= FTRACE_FL_ENABLED
;
1077 /* Disable or not filtered */
1080 /* if record is enabled, do nothing */
1081 if (rec
->flags
& FTRACE_FL_ENABLED
)
1084 rec
->flags
|= FTRACE_FL_ENABLED
;
1088 /* if record is not enabled, do nothing */
1089 if (!(rec
->flags
& FTRACE_FL_ENABLED
))
1092 rec
->flags
&= ~FTRACE_FL_ENABLED
;
1096 if (rec
->flags
& FTRACE_FL_ENABLED
)
1097 return ftrace_make_call(rec
, ftrace_addr
);
1099 return ftrace_make_nop(NULL
, rec
, ftrace_addr
);
1102 static void ftrace_replace_code(int enable
)
1104 struct dyn_ftrace
*rec
;
1105 struct ftrace_page
*pg
;
1108 do_for_each_ftrace_rec(pg
, rec
) {
1110 * Skip over free records, records that have
1111 * failed and not converted.
1113 if (rec
->flags
& FTRACE_FL_FREE
||
1114 rec
->flags
& FTRACE_FL_FAILED
||
1115 !(rec
->flags
& FTRACE_FL_CONVERTED
))
1118 /* ignore updates to this record's mcount site */
1119 if (get_kprobe((void *)rec
->ip
)) {
1123 unfreeze_record(rec
);
1126 failed
= __ftrace_replace_code(rec
, enable
);
1128 rec
->flags
|= FTRACE_FL_FAILED
;
1129 if ((system_state
== SYSTEM_BOOTING
) ||
1130 !core_kernel_text(rec
->ip
)) {
1131 ftrace_free_rec(rec
);
1133 ftrace_bug(failed
, rec
->ip
);
1134 /* Stop processing */
1138 } while_for_each_ftrace_rec();
1142 ftrace_code_disable(struct module
*mod
, struct dyn_ftrace
*rec
)
1149 ret
= ftrace_make_nop(mod
, rec
, MCOUNT_ADDR
);
1151 ftrace_bug(ret
, ip
);
1152 rec
->flags
|= FTRACE_FL_FAILED
;
1159 * archs can override this function if they must do something
1160 * before the modifying code is performed.
1162 int __weak
ftrace_arch_code_modify_prepare(void)
1168 * archs can override this function if they must do something
1169 * after the modifying code is performed.
1171 int __weak
ftrace_arch_code_modify_post_process(void)
1176 static int __ftrace_modify_code(void *data
)
1178 int *command
= data
;
1180 if (*command
& FTRACE_ENABLE_CALLS
)
1181 ftrace_replace_code(1);
1182 else if (*command
& FTRACE_DISABLE_CALLS
)
1183 ftrace_replace_code(0);
1185 if (*command
& FTRACE_UPDATE_TRACE_FUNC
)
1186 ftrace_update_ftrace_func(ftrace_trace_function
);
1188 if (*command
& FTRACE_START_FUNC_RET
)
1189 ftrace_enable_ftrace_graph_caller();
1190 else if (*command
& FTRACE_STOP_FUNC_RET
)
1191 ftrace_disable_ftrace_graph_caller();
1196 static void ftrace_run_update_code(int command
)
1200 ret
= ftrace_arch_code_modify_prepare();
1201 FTRACE_WARN_ON(ret
);
1205 stop_machine(__ftrace_modify_code
, &command
, NULL
);
1207 ret
= ftrace_arch_code_modify_post_process();
1208 FTRACE_WARN_ON(ret
);
1211 static ftrace_func_t saved_ftrace_func
;
1212 static int ftrace_start_up
;
1214 static void ftrace_startup_enable(int command
)
1216 if (saved_ftrace_func
!= ftrace_trace_function
) {
1217 saved_ftrace_func
= ftrace_trace_function
;
1218 command
|= FTRACE_UPDATE_TRACE_FUNC
;
1221 if (!command
|| !ftrace_enabled
)
1224 ftrace_run_update_code(command
);
1227 static void ftrace_startup(int command
)
1229 if (unlikely(ftrace_disabled
))
1233 command
|= FTRACE_ENABLE_CALLS
;
1235 ftrace_startup_enable(command
);
1238 static void ftrace_shutdown(int command
)
1240 if (unlikely(ftrace_disabled
))
1244 if (!ftrace_start_up
)
1245 command
|= FTRACE_DISABLE_CALLS
;
1247 if (saved_ftrace_func
!= ftrace_trace_function
) {
1248 saved_ftrace_func
= ftrace_trace_function
;
1249 command
|= FTRACE_UPDATE_TRACE_FUNC
;
1252 if (!command
|| !ftrace_enabled
)
1255 ftrace_run_update_code(command
);
1258 static void ftrace_startup_sysctl(void)
1260 int command
= FTRACE_ENABLE_MCOUNT
;
1262 if (unlikely(ftrace_disabled
))
1265 /* Force update next time */
1266 saved_ftrace_func
= NULL
;
1267 /* ftrace_start_up is true if we want ftrace running */
1268 if (ftrace_start_up
)
1269 command
|= FTRACE_ENABLE_CALLS
;
1271 ftrace_run_update_code(command
);
1274 static void ftrace_shutdown_sysctl(void)
1276 int command
= FTRACE_DISABLE_MCOUNT
;
1278 if (unlikely(ftrace_disabled
))
1281 /* ftrace_start_up is true if ftrace is running */
1282 if (ftrace_start_up
)
1283 command
|= FTRACE_DISABLE_CALLS
;
1285 ftrace_run_update_code(command
);
1288 static cycle_t ftrace_update_time
;
1289 static unsigned long ftrace_update_cnt
;
1290 unsigned long ftrace_update_tot_cnt
;
1292 static int ftrace_update_code(struct module
*mod
)
1294 struct dyn_ftrace
*p
;
1295 cycle_t start
, stop
;
1297 start
= ftrace_now(raw_smp_processor_id());
1298 ftrace_update_cnt
= 0;
1300 while (ftrace_new_addrs
) {
1302 /* If something went wrong, bail without enabling anything */
1303 if (unlikely(ftrace_disabled
))
1306 p
= ftrace_new_addrs
;
1307 ftrace_new_addrs
= p
->newlist
;
1310 /* convert record (i.e, patch mcount-call with NOP) */
1311 if (ftrace_code_disable(mod
, p
)) {
1312 p
->flags
|= FTRACE_FL_CONVERTED
;
1313 ftrace_update_cnt
++;
1318 stop
= ftrace_now(raw_smp_processor_id());
1319 ftrace_update_time
= stop
- start
;
1320 ftrace_update_tot_cnt
+= ftrace_update_cnt
;
1325 static int __init
ftrace_dyn_table_alloc(unsigned long num_to_init
)
1327 struct ftrace_page
*pg
;
1331 /* allocate a few pages */
1332 ftrace_pages_start
= (void *)get_zeroed_page(GFP_KERNEL
);
1333 if (!ftrace_pages_start
)
1337 * Allocate a few more pages.
1339 * TODO: have some parser search vmlinux before
1340 * final linking to find all calls to ftrace.
1342 * a) know how many pages to allocate.
1344 * b) set up the table then.
1346 * The dynamic code is still necessary for
1350 pg
= ftrace_pages
= ftrace_pages_start
;
1352 cnt
= num_to_init
/ ENTRIES_PER_PAGE
;
1353 pr_info("ftrace: allocating %ld entries in %d pages\n",
1354 num_to_init
, cnt
+ 1);
1356 for (i
= 0; i
< cnt
; i
++) {
1357 pg
->next
= (void *)get_zeroed_page(GFP_KERNEL
);
1359 /* If we fail, we'll try later anyway */
1370 FTRACE_ITER_FILTER
= (1 << 0),
1371 FTRACE_ITER_CONT
= (1 << 1),
1372 FTRACE_ITER_NOTRACE
= (1 << 2),
1373 FTRACE_ITER_FAILURES
= (1 << 3),
1374 FTRACE_ITER_PRINTALL
= (1 << 4),
1375 FTRACE_ITER_HASH
= (1 << 5),
1378 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
1380 struct ftrace_iterator
{
1381 struct ftrace_page
*pg
;
1385 unsigned char buffer
[FTRACE_BUFF_MAX
+1];
1386 unsigned buffer_idx
;
1391 t_hash_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
1393 struct ftrace_iterator
*iter
= m
->private;
1394 struct hlist_node
*hnd
= v
;
1395 struct hlist_head
*hhd
;
1397 WARN_ON(!(iter
->flags
& FTRACE_ITER_HASH
));
1402 if (iter
->hidx
>= FTRACE_FUNC_HASHSIZE
)
1405 hhd
= &ftrace_func_hash
[iter
->hidx
];
1407 if (hlist_empty(hhd
)) {
1426 static void *t_hash_start(struct seq_file
*m
, loff_t
*pos
)
1428 struct ftrace_iterator
*iter
= m
->private;
1431 iter
->flags
|= FTRACE_ITER_HASH
;
1433 return t_hash_next(m
, p
, pos
);
1436 static int t_hash_show(struct seq_file
*m
, void *v
)
1438 struct ftrace_func_probe
*rec
;
1439 struct hlist_node
*hnd
= v
;
1440 char str
[KSYM_SYMBOL_LEN
];
1442 rec
= hlist_entry(hnd
, struct ftrace_func_probe
, node
);
1444 if (rec
->ops
->print
)
1445 return rec
->ops
->print(m
, rec
->ip
, rec
->ops
, rec
->data
);
1447 kallsyms_lookup(rec
->ip
, NULL
, NULL
, NULL
, str
);
1448 seq_printf(m
, "%s:", str
);
1450 kallsyms_lookup((unsigned long)rec
->ops
->func
, NULL
, NULL
, NULL
, str
);
1451 seq_printf(m
, "%s", str
);
1454 seq_printf(m
, ":%p", rec
->data
);
1461 t_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
1463 struct ftrace_iterator
*iter
= m
->private;
1464 struct dyn_ftrace
*rec
= NULL
;
1466 if (iter
->flags
& FTRACE_ITER_HASH
)
1467 return t_hash_next(m
, v
, pos
);
1471 if (iter
->flags
& FTRACE_ITER_PRINTALL
)
1475 if (iter
->idx
>= iter
->pg
->index
) {
1476 if (iter
->pg
->next
) {
1477 iter
->pg
= iter
->pg
->next
;
1484 rec
= &iter
->pg
->records
[iter
->idx
++];
1485 if ((rec
->flags
& FTRACE_FL_FREE
) ||
1487 (!(iter
->flags
& FTRACE_ITER_FAILURES
) &&
1488 (rec
->flags
& FTRACE_FL_FAILED
)) ||
1490 ((iter
->flags
& FTRACE_ITER_FAILURES
) &&
1491 !(rec
->flags
& FTRACE_FL_FAILED
)) ||
1493 ((iter
->flags
& FTRACE_ITER_FILTER
) &&
1494 !(rec
->flags
& FTRACE_FL_FILTER
)) ||
1496 ((iter
->flags
& FTRACE_ITER_NOTRACE
) &&
1497 !(rec
->flags
& FTRACE_FL_NOTRACE
))) {
1506 static void *t_start(struct seq_file
*m
, loff_t
*pos
)
1508 struct ftrace_iterator
*iter
= m
->private;
1511 mutex_lock(&ftrace_lock
);
1513 * For set_ftrace_filter reading, if we have the filter
1514 * off, we can short cut and just print out that all
1515 * functions are enabled.
1517 if (iter
->flags
& FTRACE_ITER_FILTER
&& !ftrace_filtered
) {
1519 return t_hash_start(m
, pos
);
1520 iter
->flags
|= FTRACE_ITER_PRINTALL
;
1525 if (iter
->flags
& FTRACE_ITER_HASH
)
1526 return t_hash_start(m
, pos
);
1535 p
= t_next(m
, p
, pos
);
1538 return t_hash_start(m
, pos
);
1543 static void t_stop(struct seq_file
*m
, void *p
)
1545 mutex_unlock(&ftrace_lock
);
1548 static int t_show(struct seq_file
*m
, void *v
)
1550 struct ftrace_iterator
*iter
= m
->private;
1551 struct dyn_ftrace
*rec
= v
;
1552 char str
[KSYM_SYMBOL_LEN
];
1554 if (iter
->flags
& FTRACE_ITER_HASH
)
1555 return t_hash_show(m
, v
);
1557 if (iter
->flags
& FTRACE_ITER_PRINTALL
) {
1558 seq_printf(m
, "#### all functions enabled ####\n");
1565 kallsyms_lookup(rec
->ip
, NULL
, NULL
, NULL
, str
);
1567 seq_printf(m
, "%s\n", str
);
1572 static struct seq_operations show_ftrace_seq_ops
= {
1580 ftrace_avail_open(struct inode
*inode
, struct file
*file
)
1582 struct ftrace_iterator
*iter
;
1585 if (unlikely(ftrace_disabled
))
1588 iter
= kzalloc(sizeof(*iter
), GFP_KERNEL
);
1592 iter
->pg
= ftrace_pages_start
;
1594 ret
= seq_open(file
, &show_ftrace_seq_ops
);
1596 struct seq_file
*m
= file
->private_data
;
1606 int ftrace_avail_release(struct inode
*inode
, struct file
*file
)
1608 struct seq_file
*m
= (struct seq_file
*)file
->private_data
;
1609 struct ftrace_iterator
*iter
= m
->private;
1611 seq_release(inode
, file
);
1618 ftrace_failures_open(struct inode
*inode
, struct file
*file
)
1622 struct ftrace_iterator
*iter
;
1624 ret
= ftrace_avail_open(inode
, file
);
1626 m
= (struct seq_file
*)file
->private_data
;
1627 iter
= (struct ftrace_iterator
*)m
->private;
1628 iter
->flags
= FTRACE_ITER_FAILURES
;
1635 static void ftrace_filter_reset(int enable
)
1637 struct ftrace_page
*pg
;
1638 struct dyn_ftrace
*rec
;
1639 unsigned long type
= enable
? FTRACE_FL_FILTER
: FTRACE_FL_NOTRACE
;
1641 mutex_lock(&ftrace_lock
);
1643 ftrace_filtered
= 0;
1644 do_for_each_ftrace_rec(pg
, rec
) {
1645 if (rec
->flags
& FTRACE_FL_FAILED
)
1647 rec
->flags
&= ~type
;
1648 } while_for_each_ftrace_rec();
1649 mutex_unlock(&ftrace_lock
);
1653 ftrace_regex_open(struct inode
*inode
, struct file
*file
, int enable
)
1655 struct ftrace_iterator
*iter
;
1658 if (unlikely(ftrace_disabled
))
1661 iter
= kzalloc(sizeof(*iter
), GFP_KERNEL
);
1665 mutex_lock(&ftrace_regex_lock
);
1666 if ((file
->f_mode
& FMODE_WRITE
) &&
1667 !(file
->f_flags
& O_APPEND
))
1668 ftrace_filter_reset(enable
);
1670 if (file
->f_mode
& FMODE_READ
) {
1671 iter
->pg
= ftrace_pages_start
;
1672 iter
->flags
= enable
? FTRACE_ITER_FILTER
:
1673 FTRACE_ITER_NOTRACE
;
1675 ret
= seq_open(file
, &show_ftrace_seq_ops
);
1677 struct seq_file
*m
= file
->private_data
;
1682 file
->private_data
= iter
;
1683 mutex_unlock(&ftrace_regex_lock
);
1689 ftrace_filter_open(struct inode
*inode
, struct file
*file
)
1691 return ftrace_regex_open(inode
, file
, 1);
1695 ftrace_notrace_open(struct inode
*inode
, struct file
*file
)
1697 return ftrace_regex_open(inode
, file
, 0);
1701 ftrace_regex_lseek(struct file
*file
, loff_t offset
, int origin
)
1705 if (file
->f_mode
& FMODE_READ
)
1706 ret
= seq_lseek(file
, offset
, origin
);
1708 file
->f_pos
= ret
= 1;
1721 * (static function - no need for kernel doc)
1723 * Pass in a buffer containing a glob and this function will
1724 * set search to point to the search part of the buffer and
1725 * return the type of search it is (see enum above).
1726 * This does modify buff.
1728 * Returns enum type.
1729 * search returns the pointer to use for comparison.
1730 * not returns 1 if buff started with a '!'
1734 ftrace_setup_glob(char *buff
, int len
, char **search
, int *not)
1736 int type
= MATCH_FULL
;
1739 if (buff
[0] == '!') {
1748 for (i
= 0; i
< len
; i
++) {
1749 if (buff
[i
] == '*') {
1752 type
= MATCH_END_ONLY
;
1754 if (type
== MATCH_END_ONLY
)
1755 type
= MATCH_MIDDLE_ONLY
;
1757 type
= MATCH_FRONT_ONLY
;
1767 static int ftrace_match(char *str
, char *regex
, int len
, int type
)
1774 if (strcmp(str
, regex
) == 0)
1777 case MATCH_FRONT_ONLY
:
1778 if (strncmp(str
, regex
, len
) == 0)
1781 case MATCH_MIDDLE_ONLY
:
1782 if (strstr(str
, regex
))
1785 case MATCH_END_ONLY
:
1786 ptr
= strstr(str
, regex
);
1787 if (ptr
&& (ptr
[len
] == 0))
1796 ftrace_match_record(struct dyn_ftrace
*rec
, char *regex
, int len
, int type
)
1798 char str
[KSYM_SYMBOL_LEN
];
1800 kallsyms_lookup(rec
->ip
, NULL
, NULL
, NULL
, str
);
1801 return ftrace_match(str
, regex
, len
, type
);
1804 static void ftrace_match_records(char *buff
, int len
, int enable
)
1806 unsigned int search_len
;
1807 struct ftrace_page
*pg
;
1808 struct dyn_ftrace
*rec
;
1814 flag
= enable
? FTRACE_FL_FILTER
: FTRACE_FL_NOTRACE
;
1815 type
= ftrace_setup_glob(buff
, len
, &search
, ¬);
1817 search_len
= strlen(search
);
1819 mutex_lock(&ftrace_lock
);
1820 do_for_each_ftrace_rec(pg
, rec
) {
1822 if (rec
->flags
& FTRACE_FL_FAILED
)
1825 if (ftrace_match_record(rec
, search
, search_len
, type
)) {
1827 rec
->flags
&= ~flag
;
1832 * Only enable filtering if we have a function that
1835 if (enable
&& (rec
->flags
& FTRACE_FL_FILTER
))
1836 ftrace_filtered
= 1;
1837 } while_for_each_ftrace_rec();
1838 mutex_unlock(&ftrace_lock
);
1842 ftrace_match_module_record(struct dyn_ftrace
*rec
, char *mod
,
1843 char *regex
, int len
, int type
)
1845 char str
[KSYM_SYMBOL_LEN
];
1848 kallsyms_lookup(rec
->ip
, NULL
, NULL
, &modname
, str
);
1850 if (!modname
|| strcmp(modname
, mod
))
1853 /* blank search means to match all funcs in the mod */
1855 return ftrace_match(str
, regex
, len
, type
);
1860 static void ftrace_match_module_records(char *buff
, char *mod
, int enable
)
1862 unsigned search_len
= 0;
1863 struct ftrace_page
*pg
;
1864 struct dyn_ftrace
*rec
;
1865 int type
= MATCH_FULL
;
1866 char *search
= buff
;
1870 flag
= enable
? FTRACE_FL_FILTER
: FTRACE_FL_NOTRACE
;
1872 /* blank or '*' mean the same */
1873 if (strcmp(buff
, "*") == 0)
1876 /* handle the case of 'dont filter this module' */
1877 if (strcmp(buff
, "!") == 0 || strcmp(buff
, "!*") == 0) {
1883 type
= ftrace_setup_glob(buff
, strlen(buff
), &search
, ¬);
1884 search_len
= strlen(search
);
1887 mutex_lock(&ftrace_lock
);
1888 do_for_each_ftrace_rec(pg
, rec
) {
1890 if (rec
->flags
& FTRACE_FL_FAILED
)
1893 if (ftrace_match_module_record(rec
, mod
,
1894 search
, search_len
, type
)) {
1896 rec
->flags
&= ~flag
;
1900 if (enable
&& (rec
->flags
& FTRACE_FL_FILTER
))
1901 ftrace_filtered
= 1;
1903 } while_for_each_ftrace_rec();
1904 mutex_unlock(&ftrace_lock
);
1908 * We register the module command as a template to show others how
1909 * to register the a command as well.
1913 ftrace_mod_callback(char *func
, char *cmd
, char *param
, int enable
)
1918 * cmd == 'mod' because we only registered this func
1919 * for the 'mod' ftrace_func_command.
1920 * But if you register one func with multiple commands,
1921 * you can tell which command was used by the cmd
1925 /* we must have a module name */
1929 mod
= strsep(¶m
, ":");
1933 ftrace_match_module_records(func
, mod
, enable
);
1937 static struct ftrace_func_command ftrace_mod_cmd
= {
1939 .func
= ftrace_mod_callback
,
1942 static int __init
ftrace_mod_cmd_init(void)
1944 return register_ftrace_command(&ftrace_mod_cmd
);
1946 device_initcall(ftrace_mod_cmd_init
);
1949 function_trace_probe_call(unsigned long ip
, unsigned long parent_ip
)
1951 struct ftrace_func_probe
*entry
;
1952 struct hlist_head
*hhd
;
1953 struct hlist_node
*n
;
1957 key
= hash_long(ip
, FTRACE_HASH_BITS
);
1959 hhd
= &ftrace_func_hash
[key
];
1961 if (hlist_empty(hhd
))
1965 * Disable preemption for these calls to prevent a RCU grace
1966 * period. This syncs the hash iteration and freeing of items
1967 * on the hash. rcu_read_lock is too dangerous here.
1969 resched
= ftrace_preempt_disable();
1970 hlist_for_each_entry_rcu(entry
, n
, hhd
, node
) {
1971 if (entry
->ip
== ip
)
1972 entry
->ops
->func(ip
, parent_ip
, &entry
->data
);
1974 ftrace_preempt_enable(resched
);
1977 static struct ftrace_ops trace_probe_ops __read_mostly
=
1979 .func
= function_trace_probe_call
,
1982 static int ftrace_probe_registered
;
1984 static void __enable_ftrace_function_probe(void)
1988 if (ftrace_probe_registered
)
1991 for (i
= 0; i
< FTRACE_FUNC_HASHSIZE
; i
++) {
1992 struct hlist_head
*hhd
= &ftrace_func_hash
[i
];
1996 /* Nothing registered? */
1997 if (i
== FTRACE_FUNC_HASHSIZE
)
2000 __register_ftrace_function(&trace_probe_ops
);
2002 ftrace_probe_registered
= 1;
2005 static void __disable_ftrace_function_probe(void)
2009 if (!ftrace_probe_registered
)
2012 for (i
= 0; i
< FTRACE_FUNC_HASHSIZE
; i
++) {
2013 struct hlist_head
*hhd
= &ftrace_func_hash
[i
];
2018 /* no more funcs left */
2019 __unregister_ftrace_function(&trace_probe_ops
);
2021 ftrace_probe_registered
= 0;
2025 static void ftrace_free_entry_rcu(struct rcu_head
*rhp
)
2027 struct ftrace_func_probe
*entry
=
2028 container_of(rhp
, struct ftrace_func_probe
, rcu
);
2030 if (entry
->ops
->free
)
2031 entry
->ops
->free(&entry
->data
);
2037 register_ftrace_function_probe(char *glob
, struct ftrace_probe_ops
*ops
,
2040 struct ftrace_func_probe
*entry
;
2041 struct ftrace_page
*pg
;
2042 struct dyn_ftrace
*rec
;
2048 type
= ftrace_setup_glob(glob
, strlen(glob
), &search
, ¬);
2049 len
= strlen(search
);
2051 /* we do not support '!' for function probes */
2055 mutex_lock(&ftrace_lock
);
2056 do_for_each_ftrace_rec(pg
, rec
) {
2058 if (rec
->flags
& FTRACE_FL_FAILED
)
2061 if (!ftrace_match_record(rec
, search
, len
, type
))
2064 entry
= kmalloc(sizeof(*entry
), GFP_KERNEL
);
2066 /* If we did not process any, then return error */
2077 * The caller might want to do something special
2078 * for each function we find. We call the callback
2079 * to give the caller an opportunity to do so.
2081 if (ops
->callback
) {
2082 if (ops
->callback(rec
->ip
, &entry
->data
) < 0) {
2083 /* caller does not like this func */
2090 entry
->ip
= rec
->ip
;
2092 key
= hash_long(entry
->ip
, FTRACE_HASH_BITS
);
2093 hlist_add_head_rcu(&entry
->node
, &ftrace_func_hash
[key
]);
2095 } while_for_each_ftrace_rec();
2096 __enable_ftrace_function_probe();
2099 mutex_unlock(&ftrace_lock
);
2105 PROBE_TEST_FUNC
= 1,
2110 __unregister_ftrace_function_probe(char *glob
, struct ftrace_probe_ops
*ops
,
2111 void *data
, int flags
)
2113 struct ftrace_func_probe
*entry
;
2114 struct hlist_node
*n
, *tmp
;
2115 char str
[KSYM_SYMBOL_LEN
];
2116 int type
= MATCH_FULL
;
2120 if (glob
&& (strcmp(glob
, "*") || !strlen(glob
)))
2125 type
= ftrace_setup_glob(glob
, strlen(glob
), &search
, ¬);
2126 len
= strlen(search
);
2128 /* we do not support '!' for function probes */
2133 mutex_lock(&ftrace_lock
);
2134 for (i
= 0; i
< FTRACE_FUNC_HASHSIZE
; i
++) {
2135 struct hlist_head
*hhd
= &ftrace_func_hash
[i
];
2137 hlist_for_each_entry_safe(entry
, n
, tmp
, hhd
, node
) {
2139 /* break up if statements for readability */
2140 if ((flags
& PROBE_TEST_FUNC
) && entry
->ops
!= ops
)
2143 if ((flags
& PROBE_TEST_DATA
) && entry
->data
!= data
)
2146 /* do this last, since it is the most expensive */
2148 kallsyms_lookup(entry
->ip
, NULL
, NULL
,
2150 if (!ftrace_match(str
, glob
, len
, type
))
2154 hlist_del(&entry
->node
);
2155 call_rcu(&entry
->rcu
, ftrace_free_entry_rcu
);
2158 __disable_ftrace_function_probe();
2159 mutex_unlock(&ftrace_lock
);
2163 unregister_ftrace_function_probe(char *glob
, struct ftrace_probe_ops
*ops
,
2166 __unregister_ftrace_function_probe(glob
, ops
, data
,
2167 PROBE_TEST_FUNC
| PROBE_TEST_DATA
);
2171 unregister_ftrace_function_probe_func(char *glob
, struct ftrace_probe_ops
*ops
)
2173 __unregister_ftrace_function_probe(glob
, ops
, NULL
, PROBE_TEST_FUNC
);
2176 void unregister_ftrace_function_probe_all(char *glob
)
2178 __unregister_ftrace_function_probe(glob
, NULL
, NULL
, 0);
2181 static LIST_HEAD(ftrace_commands
);
2182 static DEFINE_MUTEX(ftrace_cmd_mutex
);
2184 int register_ftrace_command(struct ftrace_func_command
*cmd
)
2186 struct ftrace_func_command
*p
;
2189 mutex_lock(&ftrace_cmd_mutex
);
2190 list_for_each_entry(p
, &ftrace_commands
, list
) {
2191 if (strcmp(cmd
->name
, p
->name
) == 0) {
2196 list_add(&cmd
->list
, &ftrace_commands
);
2198 mutex_unlock(&ftrace_cmd_mutex
);
2203 int unregister_ftrace_command(struct ftrace_func_command
*cmd
)
2205 struct ftrace_func_command
*p
, *n
;
2208 mutex_lock(&ftrace_cmd_mutex
);
2209 list_for_each_entry_safe(p
, n
, &ftrace_commands
, list
) {
2210 if (strcmp(cmd
->name
, p
->name
) == 0) {
2212 list_del_init(&p
->list
);
2217 mutex_unlock(&ftrace_cmd_mutex
);
2222 static int ftrace_process_regex(char *buff
, int len
, int enable
)
2224 char *func
, *command
, *next
= buff
;
2225 struct ftrace_func_command
*p
;
2228 func
= strsep(&next
, ":");
2231 ftrace_match_records(func
, len
, enable
);
2237 command
= strsep(&next
, ":");
2239 mutex_lock(&ftrace_cmd_mutex
);
2240 list_for_each_entry(p
, &ftrace_commands
, list
) {
2241 if (strcmp(p
->name
, command
) == 0) {
2242 ret
= p
->func(func
, command
, next
, enable
);
2247 mutex_unlock(&ftrace_cmd_mutex
);
2253 ftrace_regex_write(struct file
*file
, const char __user
*ubuf
,
2254 size_t cnt
, loff_t
*ppos
, int enable
)
2256 struct ftrace_iterator
*iter
;
2261 if (!cnt
|| cnt
< 0)
2264 mutex_lock(&ftrace_regex_lock
);
2266 if (file
->f_mode
& FMODE_READ
) {
2267 struct seq_file
*m
= file
->private_data
;
2270 iter
= file
->private_data
;
2273 iter
->flags
&= ~FTRACE_ITER_CONT
;
2274 iter
->buffer_idx
= 0;
2277 ret
= get_user(ch
, ubuf
++);
2283 if (!(iter
->flags
& ~FTRACE_ITER_CONT
)) {
2284 /* skip white space */
2285 while (cnt
&& isspace(ch
)) {
2286 ret
= get_user(ch
, ubuf
++);
2294 file
->f_pos
+= read
;
2299 iter
->buffer_idx
= 0;
2302 while (cnt
&& !isspace(ch
)) {
2303 if (iter
->buffer_idx
< FTRACE_BUFF_MAX
)
2304 iter
->buffer
[iter
->buffer_idx
++] = ch
;
2309 ret
= get_user(ch
, ubuf
++);
2318 iter
->buffer
[iter
->buffer_idx
] = 0;
2319 ret
= ftrace_process_regex(iter
->buffer
,
2320 iter
->buffer_idx
, enable
);
2323 iter
->buffer_idx
= 0;
2325 iter
->flags
|= FTRACE_ITER_CONT
;
2328 file
->f_pos
+= read
;
2332 mutex_unlock(&ftrace_regex_lock
);
2338 ftrace_filter_write(struct file
*file
, const char __user
*ubuf
,
2339 size_t cnt
, loff_t
*ppos
)
2341 return ftrace_regex_write(file
, ubuf
, cnt
, ppos
, 1);
2345 ftrace_notrace_write(struct file
*file
, const char __user
*ubuf
,
2346 size_t cnt
, loff_t
*ppos
)
2348 return ftrace_regex_write(file
, ubuf
, cnt
, ppos
, 0);
2352 ftrace_set_regex(unsigned char *buf
, int len
, int reset
, int enable
)
2354 if (unlikely(ftrace_disabled
))
2357 mutex_lock(&ftrace_regex_lock
);
2359 ftrace_filter_reset(enable
);
2361 ftrace_match_records(buf
, len
, enable
);
2362 mutex_unlock(&ftrace_regex_lock
);
2366 * ftrace_set_filter - set a function to filter on in ftrace
2367 * @buf - the string that holds the function filter text.
2368 * @len - the length of the string.
2369 * @reset - non zero to reset all filters before applying this filter.
2371 * Filters denote which functions should be enabled when tracing is enabled.
2372 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
2374 void ftrace_set_filter(unsigned char *buf
, int len
, int reset
)
2376 ftrace_set_regex(buf
, len
, reset
, 1);
2380 * ftrace_set_notrace - set a function to not trace in ftrace
2381 * @buf - the string that holds the function notrace text.
2382 * @len - the length of the string.
2383 * @reset - non zero to reset all filters before applying this filter.
2385 * Notrace Filters denote which functions should not be enabled when tracing
2386 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
2389 void ftrace_set_notrace(unsigned char *buf
, int len
, int reset
)
2391 ftrace_set_regex(buf
, len
, reset
, 0);
2395 ftrace_regex_release(struct inode
*inode
, struct file
*file
, int enable
)
2397 struct seq_file
*m
= (struct seq_file
*)file
->private_data
;
2398 struct ftrace_iterator
*iter
;
2400 mutex_lock(&ftrace_regex_lock
);
2401 if (file
->f_mode
& FMODE_READ
) {
2404 seq_release(inode
, file
);
2406 iter
= file
->private_data
;
2408 if (iter
->buffer_idx
) {
2410 iter
->buffer
[iter
->buffer_idx
] = 0;
2411 ftrace_match_records(iter
->buffer
, iter
->buffer_idx
, enable
);
2414 mutex_lock(&ftrace_lock
);
2415 if (ftrace_start_up
&& ftrace_enabled
)
2416 ftrace_run_update_code(FTRACE_ENABLE_CALLS
);
2417 mutex_unlock(&ftrace_lock
);
2420 mutex_unlock(&ftrace_regex_lock
);
2425 ftrace_filter_release(struct inode
*inode
, struct file
*file
)
2427 return ftrace_regex_release(inode
, file
, 1);
2431 ftrace_notrace_release(struct inode
*inode
, struct file
*file
)
2433 return ftrace_regex_release(inode
, file
, 0);
2436 static const struct file_operations ftrace_avail_fops
= {
2437 .open
= ftrace_avail_open
,
2439 .llseek
= seq_lseek
,
2440 .release
= ftrace_avail_release
,
2443 static const struct file_operations ftrace_failures_fops
= {
2444 .open
= ftrace_failures_open
,
2446 .llseek
= seq_lseek
,
2447 .release
= ftrace_avail_release
,
2450 static const struct file_operations ftrace_filter_fops
= {
2451 .open
= ftrace_filter_open
,
2453 .write
= ftrace_filter_write
,
2454 .llseek
= ftrace_regex_lseek
,
2455 .release
= ftrace_filter_release
,
2458 static const struct file_operations ftrace_notrace_fops
= {
2459 .open
= ftrace_notrace_open
,
2461 .write
= ftrace_notrace_write
,
2462 .llseek
= ftrace_regex_lseek
,
2463 .release
= ftrace_notrace_release
,
2466 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2468 static DEFINE_MUTEX(graph_lock
);
2470 int ftrace_graph_count
;
2471 unsigned long ftrace_graph_funcs
[FTRACE_GRAPH_MAX_FUNCS
] __read_mostly
;
2474 g_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
2476 unsigned long *array
= m
->private;
2481 if (index
>= ftrace_graph_count
)
2484 return &array
[index
];
2487 static void *g_start(struct seq_file
*m
, loff_t
*pos
)
2491 mutex_lock(&graph_lock
);
2493 /* Nothing, tell g_show to print all functions are enabled */
2494 if (!ftrace_graph_count
&& !*pos
)
2497 p
= g_next(m
, p
, pos
);
2502 static void g_stop(struct seq_file
*m
, void *p
)
2504 mutex_unlock(&graph_lock
);
2507 static int g_show(struct seq_file
*m
, void *v
)
2509 unsigned long *ptr
= v
;
2510 char str
[KSYM_SYMBOL_LEN
];
2515 if (ptr
== (unsigned long *)1) {
2516 seq_printf(m
, "#### all functions enabled ####\n");
2520 kallsyms_lookup(*ptr
, NULL
, NULL
, NULL
, str
);
2522 seq_printf(m
, "%s\n", str
);
2527 static struct seq_operations ftrace_graph_seq_ops
= {
2535 ftrace_graph_open(struct inode
*inode
, struct file
*file
)
2539 if (unlikely(ftrace_disabled
))
2542 mutex_lock(&graph_lock
);
2543 if ((file
->f_mode
& FMODE_WRITE
) &&
2544 !(file
->f_flags
& O_APPEND
)) {
2545 ftrace_graph_count
= 0;
2546 memset(ftrace_graph_funcs
, 0, sizeof(ftrace_graph_funcs
));
2549 if (file
->f_mode
& FMODE_READ
) {
2550 ret
= seq_open(file
, &ftrace_graph_seq_ops
);
2552 struct seq_file
*m
= file
->private_data
;
2553 m
->private = ftrace_graph_funcs
;
2556 file
->private_data
= ftrace_graph_funcs
;
2557 mutex_unlock(&graph_lock
);
2563 ftrace_set_func(unsigned long *array
, int *idx
, char *buffer
)
2565 struct dyn_ftrace
*rec
;
2566 struct ftrace_page
*pg
;
2574 if (ftrace_disabled
)
2578 type
= ftrace_setup_glob(buffer
, strlen(buffer
), &search
, ¬);
2582 search_len
= strlen(search
);
2584 mutex_lock(&ftrace_lock
);
2585 do_for_each_ftrace_rec(pg
, rec
) {
2587 if (*idx
>= FTRACE_GRAPH_MAX_FUNCS
)
2590 if (rec
->flags
& (FTRACE_FL_FAILED
| FTRACE_FL_FREE
))
2593 if (ftrace_match_record(rec
, search
, search_len
, type
)) {
2594 /* ensure it is not already in the array */
2596 for (i
= 0; i
< *idx
; i
++)
2597 if (array
[i
] == rec
->ip
) {
2602 array
[(*idx
)++] = rec
->ip
;
2606 } while_for_each_ftrace_rec();
2608 mutex_unlock(&ftrace_lock
);
2610 return found
? 0 : -EINVAL
;
2614 ftrace_graph_write(struct file
*file
, const char __user
*ubuf
,
2615 size_t cnt
, loff_t
*ppos
)
2617 unsigned char buffer
[FTRACE_BUFF_MAX
+1];
2618 unsigned long *array
;
2624 if (!cnt
|| cnt
< 0)
2627 mutex_lock(&graph_lock
);
2629 if (ftrace_graph_count
>= FTRACE_GRAPH_MAX_FUNCS
) {
2634 if (file
->f_mode
& FMODE_READ
) {
2635 struct seq_file
*m
= file
->private_data
;
2638 array
= file
->private_data
;
2640 ret
= get_user(ch
, ubuf
++);
2646 /* skip white space */
2647 while (cnt
&& isspace(ch
)) {
2648 ret
= get_user(ch
, ubuf
++);
2661 while (cnt
&& !isspace(ch
)) {
2662 if (index
< FTRACE_BUFF_MAX
)
2663 buffer
[index
++] = ch
;
2668 ret
= get_user(ch
, ubuf
++);
2676 /* we allow only one expression at a time */
2677 ret
= ftrace_set_func(array
, &ftrace_graph_count
, buffer
);
2681 file
->f_pos
+= read
;
2685 mutex_unlock(&graph_lock
);
2690 static const struct file_operations ftrace_graph_fops
= {
2691 .open
= ftrace_graph_open
,
2693 .write
= ftrace_graph_write
,
2695 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2697 static __init
int ftrace_init_dyn_debugfs(struct dentry
*d_tracer
)
2699 struct dentry
*entry
;
2701 entry
= debugfs_create_file("available_filter_functions", 0444,
2702 d_tracer
, NULL
, &ftrace_avail_fops
);
2704 pr_warning("Could not create debugfs "
2705 "'available_filter_functions' entry\n");
2707 entry
= debugfs_create_file("failures", 0444,
2708 d_tracer
, NULL
, &ftrace_failures_fops
);
2710 pr_warning("Could not create debugfs 'failures' entry\n");
2712 entry
= debugfs_create_file("set_ftrace_filter", 0644, d_tracer
,
2713 NULL
, &ftrace_filter_fops
);
2715 pr_warning("Could not create debugfs "
2716 "'set_ftrace_filter' entry\n");
2718 entry
= debugfs_create_file("set_ftrace_notrace", 0644, d_tracer
,
2719 NULL
, &ftrace_notrace_fops
);
2721 pr_warning("Could not create debugfs "
2722 "'set_ftrace_notrace' entry\n");
2724 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2725 entry
= debugfs_create_file("set_graph_function", 0444, d_tracer
,
2727 &ftrace_graph_fops
);
2729 pr_warning("Could not create debugfs "
2730 "'set_graph_function' entry\n");
2731 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2736 static int ftrace_convert_nops(struct module
*mod
,
2737 unsigned long *start
,
2742 unsigned long flags
;
2744 mutex_lock(&ftrace_lock
);
2747 addr
= ftrace_call_adjust(*p
++);
2749 * Some architecture linkers will pad between
2750 * the different mcount_loc sections of different
2751 * object files to satisfy alignments.
2752 * Skip any NULL pointers.
2756 ftrace_record_ip(addr
);
2759 /* disable interrupts to prevent kstop machine */
2760 local_irq_save(flags
);
2761 ftrace_update_code(mod
);
2762 local_irq_restore(flags
);
2763 mutex_unlock(&ftrace_lock
);
2768 void ftrace_init_module(struct module
*mod
,
2769 unsigned long *start
, unsigned long *end
)
2771 if (ftrace_disabled
|| start
== end
)
2773 ftrace_convert_nops(mod
, start
, end
);
2776 extern unsigned long __start_mcount_loc
[];
2777 extern unsigned long __stop_mcount_loc
[];
2779 void __init
ftrace_init(void)
2781 unsigned long count
, addr
, flags
;
2784 /* Keep the ftrace pointer to the stub */
2785 addr
= (unsigned long)ftrace_stub
;
2787 local_irq_save(flags
);
2788 ftrace_dyn_arch_init(&addr
);
2789 local_irq_restore(flags
);
2791 /* ftrace_dyn_arch_init places the return code in addr */
2795 count
= __stop_mcount_loc
- __start_mcount_loc
;
2797 ret
= ftrace_dyn_table_alloc(count
);
2801 last_ftrace_enabled
= ftrace_enabled
= 1;
2803 ret
= ftrace_convert_nops(NULL
,
2809 ftrace_disabled
= 1;
2814 static int __init
ftrace_nodyn_init(void)
2819 device_initcall(ftrace_nodyn_init
);
2821 static inline int ftrace_init_dyn_debugfs(struct dentry
*d_tracer
) { return 0; }
2822 static inline void ftrace_startup_enable(int command
) { }
2823 /* Keep as macros so we do not need to define the commands */
2824 # define ftrace_startup(command) do { } while (0)
2825 # define ftrace_shutdown(command) do { } while (0)
2826 # define ftrace_startup_sysctl() do { } while (0)
2827 # define ftrace_shutdown_sysctl() do { } while (0)
2828 #endif /* CONFIG_DYNAMIC_FTRACE */
2831 ftrace_pid_read(struct file
*file
, char __user
*ubuf
,
2832 size_t cnt
, loff_t
*ppos
)
2837 if (ftrace_pid_trace
== ftrace_swapper_pid
)
2838 r
= sprintf(buf
, "swapper tasks\n");
2839 else if (ftrace_pid_trace
)
2840 r
= sprintf(buf
, "%u\n", pid_vnr(ftrace_pid_trace
));
2842 r
= sprintf(buf
, "no pid\n");
2844 return simple_read_from_buffer(ubuf
, cnt
, ppos
, buf
, r
);
2847 static void clear_ftrace_swapper(void)
2849 struct task_struct
*p
;
2853 for_each_online_cpu(cpu
) {
2855 clear_tsk_trace_trace(p
);
2860 static void set_ftrace_swapper(void)
2862 struct task_struct
*p
;
2866 for_each_online_cpu(cpu
) {
2868 set_tsk_trace_trace(p
);
2873 static void clear_ftrace_pid(struct pid
*pid
)
2875 struct task_struct
*p
;
2878 do_each_pid_task(pid
, PIDTYPE_PID
, p
) {
2879 clear_tsk_trace_trace(p
);
2880 } while_each_pid_task(pid
, PIDTYPE_PID
, p
);
2886 static void set_ftrace_pid(struct pid
*pid
)
2888 struct task_struct
*p
;
2891 do_each_pid_task(pid
, PIDTYPE_PID
, p
) {
2892 set_tsk_trace_trace(p
);
2893 } while_each_pid_task(pid
, PIDTYPE_PID
, p
);
2897 static void clear_ftrace_pid_task(struct pid
**pid
)
2899 if (*pid
== ftrace_swapper_pid
)
2900 clear_ftrace_swapper();
2902 clear_ftrace_pid(*pid
);
2907 static void set_ftrace_pid_task(struct pid
*pid
)
2909 if (pid
== ftrace_swapper_pid
)
2910 set_ftrace_swapper();
2912 set_ftrace_pid(pid
);
2916 ftrace_pid_write(struct file
*filp
, const char __user
*ubuf
,
2917 size_t cnt
, loff_t
*ppos
)
2924 if (cnt
>= sizeof(buf
))
2927 if (copy_from_user(&buf
, ubuf
, cnt
))
2932 ret
= strict_strtol(buf
, 10, &val
);
2936 mutex_lock(&ftrace_lock
);
2938 /* disable pid tracing */
2939 if (!ftrace_pid_trace
)
2942 clear_ftrace_pid_task(&ftrace_pid_trace
);
2945 /* swapper task is special */
2947 pid
= ftrace_swapper_pid
;
2948 if (pid
== ftrace_pid_trace
)
2951 pid
= find_get_pid(val
);
2953 if (pid
== ftrace_pid_trace
) {
2959 if (ftrace_pid_trace
)
2960 clear_ftrace_pid_task(&ftrace_pid_trace
);
2965 ftrace_pid_trace
= pid
;
2967 set_ftrace_pid_task(ftrace_pid_trace
);
2970 /* update the function call */
2971 ftrace_update_pid_func();
2972 ftrace_startup_enable(0);
2975 mutex_unlock(&ftrace_lock
);
2980 static const struct file_operations ftrace_pid_fops
= {
2981 .read
= ftrace_pid_read
,
2982 .write
= ftrace_pid_write
,
2985 static __init
int ftrace_init_debugfs(void)
2987 struct dentry
*d_tracer
;
2988 struct dentry
*entry
;
2990 d_tracer
= tracing_init_dentry();
2994 ftrace_init_dyn_debugfs(d_tracer
);
2996 entry
= debugfs_create_file("set_ftrace_pid", 0644, d_tracer
,
2997 NULL
, &ftrace_pid_fops
);
2999 pr_warning("Could not create debugfs "
3000 "'set_ftrace_pid' entry\n");
3002 ftrace_profile_debugfs(d_tracer
);
3006 fs_initcall(ftrace_init_debugfs
);
3009 * ftrace_kill - kill ftrace
3011 * This function should be used by panic code. It stops ftrace
3012 * but in a not so nice way. If you need to simply kill ftrace
3013 * from a non-atomic section, use ftrace_kill.
3015 void ftrace_kill(void)
3017 ftrace_disabled
= 1;
3019 clear_ftrace_function();
3023 * register_ftrace_function - register a function for profiling
3024 * @ops - ops structure that holds the function for profiling.
3026 * Register a function to be called by all functions in the
3029 * Note: @ops->func and all the functions it calls must be labeled
3030 * with "notrace", otherwise it will go into a
3033 int register_ftrace_function(struct ftrace_ops
*ops
)
3037 if (unlikely(ftrace_disabled
))
3040 mutex_lock(&ftrace_lock
);
3042 ret
= __register_ftrace_function(ops
);
3045 mutex_unlock(&ftrace_lock
);
3050 * unregister_ftrace_function - unregister a function for profiling.
3051 * @ops - ops structure that holds the function to unregister
3053 * Unregister a function that was added to be called by ftrace profiling.
3055 int unregister_ftrace_function(struct ftrace_ops
*ops
)
3059 mutex_lock(&ftrace_lock
);
3060 ret
= __unregister_ftrace_function(ops
);
3062 mutex_unlock(&ftrace_lock
);
3068 ftrace_enable_sysctl(struct ctl_table
*table
, int write
,
3069 struct file
*file
, void __user
*buffer
, size_t *lenp
,
3074 if (unlikely(ftrace_disabled
))
3077 mutex_lock(&ftrace_lock
);
3079 ret
= proc_dointvec(table
, write
, file
, buffer
, lenp
, ppos
);
3081 if (ret
|| !write
|| (last_ftrace_enabled
== ftrace_enabled
))
3084 last_ftrace_enabled
= ftrace_enabled
;
3086 if (ftrace_enabled
) {
3088 ftrace_startup_sysctl();
3090 /* we are starting ftrace again */
3091 if (ftrace_list
!= &ftrace_list_end
) {
3092 if (ftrace_list
->next
== &ftrace_list_end
)
3093 ftrace_trace_function
= ftrace_list
->func
;
3095 ftrace_trace_function
= ftrace_list_func
;
3099 /* stopping ftrace calls (just send to ftrace_stub) */
3100 ftrace_trace_function
= ftrace_stub
;
3102 ftrace_shutdown_sysctl();
3106 mutex_unlock(&ftrace_lock
);
3110 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3112 static atomic_t ftrace_graph_active
;
3113 static struct notifier_block ftrace_suspend_notifier
;
3115 int ftrace_graph_entry_stub(struct ftrace_graph_ent
*trace
)
3120 /* The callbacks that hook a function */
3121 trace_func_graph_ret_t ftrace_graph_return
=
3122 (trace_func_graph_ret_t
)ftrace_stub
;
3123 trace_func_graph_ent_t ftrace_graph_entry
= ftrace_graph_entry_stub
;
3125 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
3126 static int alloc_retstack_tasklist(struct ftrace_ret_stack
**ret_stack_list
)
3130 unsigned long flags
;
3131 int start
= 0, end
= FTRACE_RETSTACK_ALLOC_SIZE
;
3132 struct task_struct
*g
, *t
;
3134 for (i
= 0; i
< FTRACE_RETSTACK_ALLOC_SIZE
; i
++) {
3135 ret_stack_list
[i
] = kmalloc(FTRACE_RETFUNC_DEPTH
3136 * sizeof(struct ftrace_ret_stack
),
3138 if (!ret_stack_list
[i
]) {
3146 read_lock_irqsave(&tasklist_lock
, flags
);
3147 do_each_thread(g
, t
) {
3153 if (t
->ret_stack
== NULL
) {
3154 t
->curr_ret_stack
= -1;
3155 /* Make sure IRQs see the -1 first: */
3157 t
->ret_stack
= ret_stack_list
[start
++];
3158 atomic_set(&t
->tracing_graph_pause
, 0);
3159 atomic_set(&t
->trace_overrun
, 0);
3161 } while_each_thread(g
, t
);
3164 read_unlock_irqrestore(&tasklist_lock
, flags
);
3166 for (i
= start
; i
< end
; i
++)
3167 kfree(ret_stack_list
[i
]);
3172 ftrace_graph_probe_sched_switch(struct rq
*__rq
, struct task_struct
*prev
,
3173 struct task_struct
*next
)
3175 unsigned long long timestamp
;
3179 * Does the user want to count the time a function was asleep.
3180 * If so, do not update the time stamps.
3182 if (trace_flags
& TRACE_ITER_SLEEP_TIME
)
3185 timestamp
= trace_clock_local();
3187 prev
->ftrace_timestamp
= timestamp
;
3189 /* only process tasks that we timestamped */
3190 if (!next
->ftrace_timestamp
)
3194 * Update all the counters in next to make up for the
3195 * time next was sleeping.
3197 timestamp
-= next
->ftrace_timestamp
;
3199 for (index
= next
->curr_ret_stack
; index
>= 0; index
--)
3200 next
->ret_stack
[index
].calltime
+= timestamp
;
3203 /* Allocate a return stack for each task */
3204 static int start_graph_tracing(void)
3206 struct ftrace_ret_stack
**ret_stack_list
;
3209 ret_stack_list
= kmalloc(FTRACE_RETSTACK_ALLOC_SIZE
*
3210 sizeof(struct ftrace_ret_stack
*),
3213 if (!ret_stack_list
)
3216 /* The cpu_boot init_task->ret_stack will never be freed */
3217 for_each_online_cpu(cpu
)
3218 ftrace_graph_init_task(idle_task(cpu
));
3221 ret
= alloc_retstack_tasklist(ret_stack_list
);
3222 } while (ret
== -EAGAIN
);
3225 ret
= register_trace_sched_switch(ftrace_graph_probe_sched_switch
);
3227 pr_info("ftrace_graph: Couldn't activate tracepoint"
3228 " probe to kernel_sched_switch\n");
3231 kfree(ret_stack_list
);
3236 * Hibernation protection.
3237 * The state of the current task is too much unstable during
3238 * suspend/restore to disk. We want to protect against that.
3241 ftrace_suspend_notifier_call(struct notifier_block
*bl
, unsigned long state
,
3245 case PM_HIBERNATION_PREPARE
:
3246 pause_graph_tracing();
3249 case PM_POST_HIBERNATION
:
3250 unpause_graph_tracing();
3256 int register_ftrace_graph(trace_func_graph_ret_t retfunc
,
3257 trace_func_graph_ent_t entryfunc
)
3261 mutex_lock(&ftrace_lock
);
3263 /* we currently allow only one tracer registered at a time */
3264 if (atomic_read(&ftrace_graph_active
)) {
3269 ftrace_suspend_notifier
.notifier_call
= ftrace_suspend_notifier_call
;
3270 register_pm_notifier(&ftrace_suspend_notifier
);
3272 atomic_inc(&ftrace_graph_active
);
3273 ret
= start_graph_tracing();
3275 atomic_dec(&ftrace_graph_active
);
3279 ftrace_graph_return
= retfunc
;
3280 ftrace_graph_entry
= entryfunc
;
3282 ftrace_startup(FTRACE_START_FUNC_RET
);
3285 mutex_unlock(&ftrace_lock
);
3289 void unregister_ftrace_graph(void)
3291 mutex_lock(&ftrace_lock
);
3293 atomic_dec(&ftrace_graph_active
);
3294 unregister_trace_sched_switch(ftrace_graph_probe_sched_switch
);
3295 ftrace_graph_return
= (trace_func_graph_ret_t
)ftrace_stub
;
3296 ftrace_graph_entry
= ftrace_graph_entry_stub
;
3297 ftrace_shutdown(FTRACE_STOP_FUNC_RET
);
3298 unregister_pm_notifier(&ftrace_suspend_notifier
);
3300 mutex_unlock(&ftrace_lock
);
3303 /* Allocate a return stack for newly created task */
3304 void ftrace_graph_init_task(struct task_struct
*t
)
3306 if (atomic_read(&ftrace_graph_active
)) {
3307 t
->ret_stack
= kmalloc(FTRACE_RETFUNC_DEPTH
3308 * sizeof(struct ftrace_ret_stack
),
3312 t
->curr_ret_stack
= -1;
3313 atomic_set(&t
->tracing_graph_pause
, 0);
3314 atomic_set(&t
->trace_overrun
, 0);
3315 t
->ftrace_timestamp
= 0;
3317 t
->ret_stack
= NULL
;
3320 void ftrace_graph_exit_task(struct task_struct
*t
)
3322 struct ftrace_ret_stack
*ret_stack
= t
->ret_stack
;
3324 t
->ret_stack
= NULL
;
3325 /* NULL must become visible to IRQs before we free it: */
3331 void ftrace_graph_stop(void)