2 * Infrastructure for profiling code inserted by 'gcc -pg'.
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
7 * Originally ported from the -rt patch by:
8 * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code in the latency_tracer, that is:
12 * Copyright (C) 2004-2006 Ingo Molnar
13 * Copyright (C) 2004 William Lee Irwin III
16 #include <linux/stop_machine.h>
17 #include <linux/clocksource.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/debugfs.h>
21 #include <linux/hardirq.h>
22 #include <linux/kthread.h>
23 #include <linux/uaccess.h>
24 #include <linux/kprobes.h>
25 #include <linux/ftrace.h>
26 #include <linux/sysctl.h>
27 #include <linux/ctype.h>
28 #include <linux/list.h>
30 #include <asm/ftrace.h>
34 #define FTRACE_WARN_ON(cond) \
40 #define FTRACE_WARN_ON_ONCE(cond) \
42 if (WARN_ON_ONCE(cond)) \
46 /* ftrace_enabled is a method to turn ftrace on or off */
47 int ftrace_enabled __read_mostly
;
48 static int last_ftrace_enabled
;
51 * ftrace_disabled is set when an anomaly is discovered.
52 * ftrace_disabled is much stronger than ftrace_enabled.
54 static int ftrace_disabled __read_mostly
;
56 static DEFINE_SPINLOCK(ftrace_lock
);
57 static DEFINE_MUTEX(ftrace_sysctl_lock
);
59 static struct ftrace_ops ftrace_list_end __read_mostly
=
64 static struct ftrace_ops
*ftrace_list __read_mostly
= &ftrace_list_end
;
65 ftrace_func_t ftrace_trace_function __read_mostly
= ftrace_stub
;
67 static void ftrace_list_func(unsigned long ip
, unsigned long parent_ip
)
69 struct ftrace_ops
*op
= ftrace_list
;
71 /* in case someone actually ports this to alpha! */
72 read_barrier_depends();
74 while (op
!= &ftrace_list_end
) {
76 read_barrier_depends();
77 op
->func(ip
, parent_ip
);
83 * clear_ftrace_function - reset the ftrace function
85 * This NULLs the ftrace function and in essence stops
86 * tracing. There may be lag
88 void clear_ftrace_function(void)
90 ftrace_trace_function
= ftrace_stub
;
93 static int __register_ftrace_function(struct ftrace_ops
*ops
)
95 /* should not be called from interrupt context */
96 spin_lock(&ftrace_lock
);
98 ops
->next
= ftrace_list
;
100 * We are entering ops into the ftrace_list but another
101 * CPU might be walking that list. We need to make sure
102 * the ops->next pointer is valid before another CPU sees
103 * the ops pointer included into the ftrace_list.
108 if (ftrace_enabled
) {
110 * For one func, simply call it directly.
111 * For more than one func, call the chain.
113 if (ops
->next
== &ftrace_list_end
)
114 ftrace_trace_function
= ops
->func
;
116 ftrace_trace_function
= ftrace_list_func
;
119 spin_unlock(&ftrace_lock
);
124 static int __unregister_ftrace_function(struct ftrace_ops
*ops
)
126 struct ftrace_ops
**p
;
129 /* should not be called from interrupt context */
130 spin_lock(&ftrace_lock
);
133 * If we are removing the last function, then simply point
134 * to the ftrace_stub.
136 if (ftrace_list
== ops
&& ops
->next
== &ftrace_list_end
) {
137 ftrace_trace_function
= ftrace_stub
;
138 ftrace_list
= &ftrace_list_end
;
142 for (p
= &ftrace_list
; *p
!= &ftrace_list_end
; p
= &(*p
)->next
)
153 if (ftrace_enabled
) {
154 /* If we only have one func left, then call that directly */
155 if (ftrace_list
== &ftrace_list_end
||
156 ftrace_list
->next
== &ftrace_list_end
)
157 ftrace_trace_function
= ftrace_list
->func
;
161 spin_unlock(&ftrace_lock
);
166 #ifdef CONFIG_DYNAMIC_FTRACE
167 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
168 # error Dynamic ftrace depends on MCOUNT_RECORD
172 * Since MCOUNT_ADDR may point to mcount itself, we do not want
173 * to get it confused by reading a reference in the code as we
174 * are parsing on objcopy output of text. Use a variable for
177 static unsigned long mcount_addr
= MCOUNT_ADDR
;
180 FTRACE_ENABLE_CALLS
= (1 << 0),
181 FTRACE_DISABLE_CALLS
= (1 << 1),
182 FTRACE_UPDATE_TRACE_FUNC
= (1 << 2),
183 FTRACE_ENABLE_MCOUNT
= (1 << 3),
184 FTRACE_DISABLE_MCOUNT
= (1 << 4),
187 static int ftrace_filtered
;
188 static int tracing_on
;
189 static int frozen_record_count
;
191 static LIST_HEAD(ftrace_new_addrs
);
193 static DEFINE_MUTEX(ftrace_regex_lock
);
196 struct ftrace_page
*next
;
198 struct dyn_ftrace records
[];
201 #define ENTRIES_PER_PAGE \
202 ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
204 /* estimate from running different kernels */
205 #define NR_TO_INIT 10000
207 static struct ftrace_page
*ftrace_pages_start
;
208 static struct ftrace_page
*ftrace_pages
;
210 static struct dyn_ftrace
*ftrace_free_records
;
213 #ifdef CONFIG_KPROBES
214 static inline void freeze_record(struct dyn_ftrace
*rec
)
216 if (!(rec
->flags
& FTRACE_FL_FROZEN
)) {
217 rec
->flags
|= FTRACE_FL_FROZEN
;
218 frozen_record_count
++;
222 static inline void unfreeze_record(struct dyn_ftrace
*rec
)
224 if (rec
->flags
& FTRACE_FL_FROZEN
) {
225 rec
->flags
&= ~FTRACE_FL_FROZEN
;
226 frozen_record_count
--;
230 static inline int record_frozen(struct dyn_ftrace
*rec
)
232 return rec
->flags
& FTRACE_FL_FROZEN
;
235 # define freeze_record(rec) ({ 0; })
236 # define unfreeze_record(rec) ({ 0; })
237 # define record_frozen(rec) ({ 0; })
238 #endif /* CONFIG_KPROBES */
240 static void ftrace_free_rec(struct dyn_ftrace
*rec
)
242 rec
->ip
= (unsigned long)ftrace_free_records
;
243 ftrace_free_records
= rec
;
244 rec
->flags
|= FTRACE_FL_FREE
;
247 void ftrace_release(void *start
, unsigned long size
)
249 struct dyn_ftrace
*rec
;
250 struct ftrace_page
*pg
;
251 unsigned long s
= (unsigned long)start
;
252 unsigned long e
= s
+ size
;
255 if (ftrace_disabled
|| !start
)
258 /* should not be called from interrupt context */
259 spin_lock(&ftrace_lock
);
261 for (pg
= ftrace_pages_start
; pg
; pg
= pg
->next
) {
262 for (i
= 0; i
< pg
->index
; i
++) {
263 rec
= &pg
->records
[i
];
265 if ((rec
->ip
>= s
) && (rec
->ip
< e
))
266 ftrace_free_rec(rec
);
269 spin_unlock(&ftrace_lock
);
272 static struct dyn_ftrace
*ftrace_alloc_dyn_node(unsigned long ip
)
274 struct dyn_ftrace
*rec
;
276 /* First check for freed records */
277 if (ftrace_free_records
) {
278 rec
= ftrace_free_records
;
280 if (unlikely(!(rec
->flags
& FTRACE_FL_FREE
))) {
281 FTRACE_WARN_ON_ONCE(1);
282 ftrace_free_records
= NULL
;
286 ftrace_free_records
= (void *)rec
->ip
;
287 memset(rec
, 0, sizeof(*rec
));
291 if (ftrace_pages
->index
== ENTRIES_PER_PAGE
) {
292 if (!ftrace_pages
->next
) {
293 /* allocate another page */
295 (void *)get_zeroed_page(GFP_KERNEL
);
296 if (!ftrace_pages
->next
)
299 ftrace_pages
= ftrace_pages
->next
;
302 return &ftrace_pages
->records
[ftrace_pages
->index
++];
305 static struct dyn_ftrace
*
306 ftrace_record_ip(unsigned long ip
)
308 struct dyn_ftrace
*rec
;
310 if (!ftrace_enabled
|| ftrace_disabled
)
313 rec
= ftrace_alloc_dyn_node(ip
);
319 list_add(&rec
->list
, &ftrace_new_addrs
);
324 #define FTRACE_ADDR ((long)(ftrace_caller))
327 __ftrace_replace_code(struct dyn_ftrace
*rec
,
328 unsigned char *old
, unsigned char *new, int enable
)
330 unsigned long ip
, fl
;
334 if (ftrace_filtered
&& enable
) {
336 * If filtering is on:
338 * If this record is set to be filtered and
339 * is enabled then do nothing.
341 * If this record is set to be filtered and
342 * it is not enabled, enable it.
344 * If this record is not set to be filtered
345 * and it is not enabled do nothing.
347 * If this record is set not to trace then
350 * If this record is set not to trace and
351 * it is enabled then disable it.
353 * If this record is not set to be filtered and
354 * it is enabled, disable it.
357 fl
= rec
->flags
& (FTRACE_FL_FILTER
| FTRACE_FL_NOTRACE
|
360 if ((fl
== (FTRACE_FL_FILTER
| FTRACE_FL_ENABLED
)) ||
361 (fl
== (FTRACE_FL_FILTER
| FTRACE_FL_NOTRACE
)) ||
362 !fl
|| (fl
== FTRACE_FL_NOTRACE
))
366 * If it is enabled disable it,
367 * otherwise enable it!
369 if (fl
& FTRACE_FL_ENABLED
) {
370 /* swap new and old */
372 old
= ftrace_call_replace(ip
, FTRACE_ADDR
);
373 rec
->flags
&= ~FTRACE_FL_ENABLED
;
375 new = ftrace_call_replace(ip
, FTRACE_ADDR
);
376 rec
->flags
|= FTRACE_FL_ENABLED
;
382 * If this record is set not to trace and is
383 * not enabled, do nothing.
385 fl
= rec
->flags
& (FTRACE_FL_NOTRACE
| FTRACE_FL_ENABLED
);
386 if (fl
== FTRACE_FL_NOTRACE
)
389 new = ftrace_call_replace(ip
, FTRACE_ADDR
);
391 old
= ftrace_call_replace(ip
, FTRACE_ADDR
);
394 if (rec
->flags
& FTRACE_FL_ENABLED
)
396 rec
->flags
|= FTRACE_FL_ENABLED
;
398 if (!(rec
->flags
& FTRACE_FL_ENABLED
))
400 rec
->flags
&= ~FTRACE_FL_ENABLED
;
404 return ftrace_modify_code(ip
, old
, new);
407 static void ftrace_replace_code(int enable
)
410 unsigned char *new = NULL
, *old
= NULL
;
411 struct dyn_ftrace
*rec
;
412 struct ftrace_page
*pg
;
415 old
= ftrace_nop_replace();
417 new = ftrace_nop_replace();
419 for (pg
= ftrace_pages_start
; pg
; pg
= pg
->next
) {
420 for (i
= 0; i
< pg
->index
; i
++) {
421 rec
= &pg
->records
[i
];
423 /* don't modify code that has already faulted */
424 if (rec
->flags
& FTRACE_FL_FAILED
)
427 /* ignore updates to this record's mcount site */
428 if (get_kprobe((void *)rec
->ip
)) {
432 unfreeze_record(rec
);
435 failed
= __ftrace_replace_code(rec
, old
, new, enable
);
436 if (failed
&& (rec
->flags
& FTRACE_FL_CONVERTED
)) {
437 rec
->flags
|= FTRACE_FL_FAILED
;
438 if ((system_state
== SYSTEM_BOOTING
) ||
439 !core_kernel_text(rec
->ip
)) {
440 ftrace_free_rec(rec
);
447 static void print_ip_ins(const char *fmt
, unsigned char *p
)
451 printk(KERN_CONT
"%s", fmt
);
453 for (i
= 0; i
< MCOUNT_INSN_SIZE
; i
++)
454 printk(KERN_CONT
"%s%02x", i
? ":" : "", p
[i
]);
458 ftrace_code_disable(struct dyn_ftrace
*rec
)
461 unsigned char *nop
, *call
;
466 nop
= ftrace_nop_replace();
467 call
= ftrace_call_replace(ip
, mcount_addr
);
469 ret
= ftrace_modify_code(ip
, call
, nop
);
473 FTRACE_WARN_ON_ONCE(1);
474 pr_info("ftrace faulted on modifying ");
478 FTRACE_WARN_ON_ONCE(1);
479 pr_info("ftrace failed to modify ");
481 print_ip_ins(" expected: ", call
);
482 print_ip_ins(" actual: ", (unsigned char *)ip
);
483 print_ip_ins(" replace: ", nop
);
484 printk(KERN_CONT
"\n");
487 FTRACE_WARN_ON_ONCE(1);
488 pr_info("ftrace faulted on writing ");
492 FTRACE_WARN_ON_ONCE(1);
493 pr_info("ftrace faulted on unknown error ");
497 rec
->flags
|= FTRACE_FL_FAILED
;
503 static int __ftrace_modify_code(void *data
)
507 if (*command
& FTRACE_ENABLE_CALLS
) {
508 ftrace_replace_code(1);
510 } else if (*command
& FTRACE_DISABLE_CALLS
) {
511 ftrace_replace_code(0);
515 if (*command
& FTRACE_UPDATE_TRACE_FUNC
)
516 ftrace_update_ftrace_func(ftrace_trace_function
);
521 static void ftrace_run_update_code(int command
)
523 stop_machine(__ftrace_modify_code
, &command
, NULL
);
526 static ftrace_func_t saved_ftrace_func
;
527 static int ftrace_start
;
528 static DEFINE_MUTEX(ftrace_start_lock
);
530 static void ftrace_startup(void)
534 if (unlikely(ftrace_disabled
))
537 mutex_lock(&ftrace_start_lock
);
539 if (ftrace_start
== 1)
540 command
|= FTRACE_ENABLE_CALLS
;
542 if (saved_ftrace_func
!= ftrace_trace_function
) {
543 saved_ftrace_func
= ftrace_trace_function
;
544 command
|= FTRACE_UPDATE_TRACE_FUNC
;
547 if (!command
|| !ftrace_enabled
)
550 ftrace_run_update_code(command
);
552 mutex_unlock(&ftrace_start_lock
);
555 static void ftrace_shutdown(void)
559 if (unlikely(ftrace_disabled
))
562 mutex_lock(&ftrace_start_lock
);
565 command
|= FTRACE_DISABLE_CALLS
;
567 if (saved_ftrace_func
!= ftrace_trace_function
) {
568 saved_ftrace_func
= ftrace_trace_function
;
569 command
|= FTRACE_UPDATE_TRACE_FUNC
;
572 if (!command
|| !ftrace_enabled
)
575 ftrace_run_update_code(command
);
577 mutex_unlock(&ftrace_start_lock
);
580 static void ftrace_startup_sysctl(void)
582 int command
= FTRACE_ENABLE_MCOUNT
;
584 if (unlikely(ftrace_disabled
))
587 mutex_lock(&ftrace_start_lock
);
588 /* Force update next time */
589 saved_ftrace_func
= NULL
;
590 /* ftrace_start is true if we want ftrace running */
592 command
|= FTRACE_ENABLE_CALLS
;
594 ftrace_run_update_code(command
);
595 mutex_unlock(&ftrace_start_lock
);
598 static void ftrace_shutdown_sysctl(void)
600 int command
= FTRACE_DISABLE_MCOUNT
;
602 if (unlikely(ftrace_disabled
))
605 mutex_lock(&ftrace_start_lock
);
606 /* ftrace_start is true if ftrace is running */
608 command
|= FTRACE_DISABLE_CALLS
;
610 ftrace_run_update_code(command
);
611 mutex_unlock(&ftrace_start_lock
);
614 static cycle_t ftrace_update_time
;
615 static unsigned long ftrace_update_cnt
;
616 unsigned long ftrace_update_tot_cnt
;
618 static int ftrace_update_code(void)
620 struct dyn_ftrace
*p
, *t
;
623 start
= ftrace_now(raw_smp_processor_id());
624 ftrace_update_cnt
= 0;
626 list_for_each_entry_safe(p
, t
, &ftrace_new_addrs
, list
) {
628 /* If something went wrong, bail without enabling anything */
629 if (unlikely(ftrace_disabled
))
632 list_del_init(&p
->list
);
634 /* convert record (i.e, patch mcount-call with NOP) */
635 if (ftrace_code_disable(p
)) {
636 p
->flags
|= FTRACE_FL_CONVERTED
;
642 stop
= ftrace_now(raw_smp_processor_id());
643 ftrace_update_time
= stop
- start
;
644 ftrace_update_tot_cnt
+= ftrace_update_cnt
;
649 static int __init
ftrace_dyn_table_alloc(unsigned long num_to_init
)
651 struct ftrace_page
*pg
;
655 /* allocate a few pages */
656 ftrace_pages_start
= (void *)get_zeroed_page(GFP_KERNEL
);
657 if (!ftrace_pages_start
)
661 * Allocate a few more pages.
663 * TODO: have some parser search vmlinux before
664 * final linking to find all calls to ftrace.
666 * a) know how many pages to allocate.
668 * b) set up the table then.
670 * The dynamic code is still necessary for
674 pg
= ftrace_pages
= ftrace_pages_start
;
676 cnt
= num_to_init
/ ENTRIES_PER_PAGE
;
677 pr_info("ftrace: allocating %ld entries in %d pages\n",
680 for (i
= 0; i
< cnt
; i
++) {
681 pg
->next
= (void *)get_zeroed_page(GFP_KERNEL
);
683 /* If we fail, we'll try later anyway */
694 FTRACE_ITER_FILTER
= (1 << 0),
695 FTRACE_ITER_CONT
= (1 << 1),
696 FTRACE_ITER_NOTRACE
= (1 << 2),
697 FTRACE_ITER_FAILURES
= (1 << 3),
700 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
702 struct ftrace_iterator
{
704 struct ftrace_page
*pg
;
707 unsigned char buffer
[FTRACE_BUFF_MAX
+1];
713 t_next(struct seq_file
*m
, void *v
, loff_t
*pos
)
715 struct ftrace_iterator
*iter
= m
->private;
716 struct dyn_ftrace
*rec
= NULL
;
720 /* should not be called from interrupt context */
721 spin_lock(&ftrace_lock
);
723 if (iter
->idx
>= iter
->pg
->index
) {
724 if (iter
->pg
->next
) {
725 iter
->pg
= iter
->pg
->next
;
730 rec
= &iter
->pg
->records
[iter
->idx
++];
731 if ((rec
->flags
& FTRACE_FL_FREE
) ||
733 (!(iter
->flags
& FTRACE_ITER_FAILURES
) &&
734 (rec
->flags
& FTRACE_FL_FAILED
)) ||
736 ((iter
->flags
& FTRACE_ITER_FAILURES
) &&
737 !(rec
->flags
& FTRACE_FL_FAILED
)) ||
739 ((iter
->flags
& FTRACE_ITER_NOTRACE
) &&
740 !(rec
->flags
& FTRACE_FL_NOTRACE
))) {
745 spin_unlock(&ftrace_lock
);
752 static void *t_start(struct seq_file
*m
, loff_t
*pos
)
754 struct ftrace_iterator
*iter
= m
->private;
758 if (*pos
!= iter
->pos
) {
759 for (p
= t_next(m
, p
, &l
); p
&& l
< *pos
; p
= t_next(m
, p
, &l
))
763 p
= t_next(m
, p
, &l
);
769 static void t_stop(struct seq_file
*m
, void *p
)
773 static int t_show(struct seq_file
*m
, void *v
)
775 struct dyn_ftrace
*rec
= v
;
776 char str
[KSYM_SYMBOL_LEN
];
781 kallsyms_lookup(rec
->ip
, NULL
, NULL
, NULL
, str
);
783 seq_printf(m
, "%s\n", str
);
788 static struct seq_operations show_ftrace_seq_ops
= {
796 ftrace_avail_open(struct inode
*inode
, struct file
*file
)
798 struct ftrace_iterator
*iter
;
801 if (unlikely(ftrace_disabled
))
804 iter
= kzalloc(sizeof(*iter
), GFP_KERNEL
);
808 iter
->pg
= ftrace_pages_start
;
811 ret
= seq_open(file
, &show_ftrace_seq_ops
);
813 struct seq_file
*m
= file
->private_data
;
823 int ftrace_avail_release(struct inode
*inode
, struct file
*file
)
825 struct seq_file
*m
= (struct seq_file
*)file
->private_data
;
826 struct ftrace_iterator
*iter
= m
->private;
828 seq_release(inode
, file
);
835 ftrace_failures_open(struct inode
*inode
, struct file
*file
)
839 struct ftrace_iterator
*iter
;
841 ret
= ftrace_avail_open(inode
, file
);
843 m
= (struct seq_file
*)file
->private_data
;
844 iter
= (struct ftrace_iterator
*)m
->private;
845 iter
->flags
= FTRACE_ITER_FAILURES
;
852 static void ftrace_filter_reset(int enable
)
854 struct ftrace_page
*pg
;
855 struct dyn_ftrace
*rec
;
856 unsigned long type
= enable
? FTRACE_FL_FILTER
: FTRACE_FL_NOTRACE
;
859 /* should not be called from interrupt context */
860 spin_lock(&ftrace_lock
);
863 pg
= ftrace_pages_start
;
865 for (i
= 0; i
< pg
->index
; i
++) {
866 rec
= &pg
->records
[i
];
867 if (rec
->flags
& FTRACE_FL_FAILED
)
873 spin_unlock(&ftrace_lock
);
877 ftrace_regex_open(struct inode
*inode
, struct file
*file
, int enable
)
879 struct ftrace_iterator
*iter
;
882 if (unlikely(ftrace_disabled
))
885 iter
= kzalloc(sizeof(*iter
), GFP_KERNEL
);
889 mutex_lock(&ftrace_regex_lock
);
890 if ((file
->f_mode
& FMODE_WRITE
) &&
891 !(file
->f_flags
& O_APPEND
))
892 ftrace_filter_reset(enable
);
894 if (file
->f_mode
& FMODE_READ
) {
895 iter
->pg
= ftrace_pages_start
;
897 iter
->flags
= enable
? FTRACE_ITER_FILTER
:
900 ret
= seq_open(file
, &show_ftrace_seq_ops
);
902 struct seq_file
*m
= file
->private_data
;
907 file
->private_data
= iter
;
908 mutex_unlock(&ftrace_regex_lock
);
914 ftrace_filter_open(struct inode
*inode
, struct file
*file
)
916 return ftrace_regex_open(inode
, file
, 1);
920 ftrace_notrace_open(struct inode
*inode
, struct file
*file
)
922 return ftrace_regex_open(inode
, file
, 0);
926 ftrace_regex_read(struct file
*file
, char __user
*ubuf
,
927 size_t cnt
, loff_t
*ppos
)
929 if (file
->f_mode
& FMODE_READ
)
930 return seq_read(file
, ubuf
, cnt
, ppos
);
936 ftrace_regex_lseek(struct file
*file
, loff_t offset
, int origin
)
940 if (file
->f_mode
& FMODE_READ
)
941 ret
= seq_lseek(file
, offset
, origin
);
943 file
->f_pos
= ret
= 1;
956 ftrace_match(unsigned char *buff
, int len
, int enable
)
958 char str
[KSYM_SYMBOL_LEN
];
960 struct ftrace_page
*pg
;
961 struct dyn_ftrace
*rec
;
962 int type
= MATCH_FULL
;
963 unsigned long flag
= enable
? FTRACE_FL_FILTER
: FTRACE_FL_NOTRACE
;
964 unsigned i
, match
= 0, search_len
= 0;
966 for (i
= 0; i
< len
; i
++) {
967 if (buff
[i
] == '*') {
969 search
= buff
+ i
+ 1;
970 type
= MATCH_END_ONLY
;
971 search_len
= len
- (i
+ 1);
973 if (type
== MATCH_END_ONLY
) {
974 type
= MATCH_MIDDLE_ONLY
;
977 type
= MATCH_FRONT_ONLY
;
985 /* should not be called from interrupt context */
986 spin_lock(&ftrace_lock
);
989 pg
= ftrace_pages_start
;
991 for (i
= 0; i
< pg
->index
; i
++) {
995 rec
= &pg
->records
[i
];
996 if (rec
->flags
& FTRACE_FL_FAILED
)
998 kallsyms_lookup(rec
->ip
, NULL
, NULL
, NULL
, str
);
1001 if (strcmp(str
, buff
) == 0)
1004 case MATCH_FRONT_ONLY
:
1005 if (memcmp(str
, buff
, match
) == 0)
1008 case MATCH_MIDDLE_ONLY
:
1009 if (strstr(str
, search
))
1012 case MATCH_END_ONLY
:
1013 ptr
= strstr(str
, search
);
1014 if (ptr
&& (ptr
[search_len
] == 0))
1023 spin_unlock(&ftrace_lock
);
1027 ftrace_regex_write(struct file
*file
, const char __user
*ubuf
,
1028 size_t cnt
, loff_t
*ppos
, int enable
)
1030 struct ftrace_iterator
*iter
;
1035 if (!cnt
|| cnt
< 0)
1038 mutex_lock(&ftrace_regex_lock
);
1040 if (file
->f_mode
& FMODE_READ
) {
1041 struct seq_file
*m
= file
->private_data
;
1044 iter
= file
->private_data
;
1047 iter
->flags
&= ~FTRACE_ITER_CONT
;
1048 iter
->buffer_idx
= 0;
1051 ret
= get_user(ch
, ubuf
++);
1057 if (!(iter
->flags
& ~FTRACE_ITER_CONT
)) {
1058 /* skip white space */
1059 while (cnt
&& isspace(ch
)) {
1060 ret
= get_user(ch
, ubuf
++);
1068 file
->f_pos
+= read
;
1073 iter
->buffer_idx
= 0;
1076 while (cnt
&& !isspace(ch
)) {
1077 if (iter
->buffer_idx
< FTRACE_BUFF_MAX
)
1078 iter
->buffer
[iter
->buffer_idx
++] = ch
;
1083 ret
= get_user(ch
, ubuf
++);
1092 iter
->buffer
[iter
->buffer_idx
] = 0;
1093 ftrace_match(iter
->buffer
, iter
->buffer_idx
, enable
);
1094 iter
->buffer_idx
= 0;
1096 iter
->flags
|= FTRACE_ITER_CONT
;
1099 file
->f_pos
+= read
;
1103 mutex_unlock(&ftrace_regex_lock
);
1109 ftrace_filter_write(struct file
*file
, const char __user
*ubuf
,
1110 size_t cnt
, loff_t
*ppos
)
1112 return ftrace_regex_write(file
, ubuf
, cnt
, ppos
, 1);
1116 ftrace_notrace_write(struct file
*file
, const char __user
*ubuf
,
1117 size_t cnt
, loff_t
*ppos
)
1119 return ftrace_regex_write(file
, ubuf
, cnt
, ppos
, 0);
1123 ftrace_set_regex(unsigned char *buf
, int len
, int reset
, int enable
)
1125 if (unlikely(ftrace_disabled
))
1128 mutex_lock(&ftrace_regex_lock
);
1130 ftrace_filter_reset(enable
);
1132 ftrace_match(buf
, len
, enable
);
1133 mutex_unlock(&ftrace_regex_lock
);
1137 * ftrace_set_filter - set a function to filter on in ftrace
1138 * @buf - the string that holds the function filter text.
1139 * @len - the length of the string.
1140 * @reset - non zero to reset all filters before applying this filter.
1142 * Filters denote which functions should be enabled when tracing is enabled.
1143 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
1145 void ftrace_set_filter(unsigned char *buf
, int len
, int reset
)
1147 ftrace_set_regex(buf
, len
, reset
, 1);
1151 * ftrace_set_notrace - set a function to not trace in ftrace
1152 * @buf - the string that holds the function notrace text.
1153 * @len - the length of the string.
1154 * @reset - non zero to reset all filters before applying this filter.
1156 * Notrace Filters denote which functions should not be enabled when tracing
1157 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
1160 void ftrace_set_notrace(unsigned char *buf
, int len
, int reset
)
1162 ftrace_set_regex(buf
, len
, reset
, 0);
1166 ftrace_regex_release(struct inode
*inode
, struct file
*file
, int enable
)
1168 struct seq_file
*m
= (struct seq_file
*)file
->private_data
;
1169 struct ftrace_iterator
*iter
;
1171 mutex_lock(&ftrace_regex_lock
);
1172 if (file
->f_mode
& FMODE_READ
) {
1175 seq_release(inode
, file
);
1177 iter
= file
->private_data
;
1179 if (iter
->buffer_idx
) {
1181 iter
->buffer
[iter
->buffer_idx
] = 0;
1182 ftrace_match(iter
->buffer
, iter
->buffer_idx
, enable
);
1185 mutex_lock(&ftrace_sysctl_lock
);
1186 mutex_lock(&ftrace_start_lock
);
1187 if (iter
->filtered
&& ftrace_start
&& ftrace_enabled
)
1188 ftrace_run_update_code(FTRACE_ENABLE_CALLS
);
1189 mutex_unlock(&ftrace_start_lock
);
1190 mutex_unlock(&ftrace_sysctl_lock
);
1193 mutex_unlock(&ftrace_regex_lock
);
1198 ftrace_filter_release(struct inode
*inode
, struct file
*file
)
1200 return ftrace_regex_release(inode
, file
, 1);
1204 ftrace_notrace_release(struct inode
*inode
, struct file
*file
)
1206 return ftrace_regex_release(inode
, file
, 0);
1209 static struct file_operations ftrace_avail_fops
= {
1210 .open
= ftrace_avail_open
,
1212 .llseek
= seq_lseek
,
1213 .release
= ftrace_avail_release
,
1216 static struct file_operations ftrace_failures_fops
= {
1217 .open
= ftrace_failures_open
,
1219 .llseek
= seq_lseek
,
1220 .release
= ftrace_avail_release
,
1223 static struct file_operations ftrace_filter_fops
= {
1224 .open
= ftrace_filter_open
,
1225 .read
= ftrace_regex_read
,
1226 .write
= ftrace_filter_write
,
1227 .llseek
= ftrace_regex_lseek
,
1228 .release
= ftrace_filter_release
,
1231 static struct file_operations ftrace_notrace_fops
= {
1232 .open
= ftrace_notrace_open
,
1233 .read
= ftrace_regex_read
,
1234 .write
= ftrace_notrace_write
,
1235 .llseek
= ftrace_regex_lseek
,
1236 .release
= ftrace_notrace_release
,
1239 static __init
int ftrace_init_debugfs(void)
1241 struct dentry
*d_tracer
;
1242 struct dentry
*entry
;
1244 d_tracer
= tracing_init_dentry();
1246 entry
= debugfs_create_file("available_filter_functions", 0444,
1247 d_tracer
, NULL
, &ftrace_avail_fops
);
1249 pr_warning("Could not create debugfs "
1250 "'available_filter_functions' entry\n");
1252 entry
= debugfs_create_file("failures", 0444,
1253 d_tracer
, NULL
, &ftrace_failures_fops
);
1255 pr_warning("Could not create debugfs 'failures' entry\n");
1257 entry
= debugfs_create_file("set_ftrace_filter", 0644, d_tracer
,
1258 NULL
, &ftrace_filter_fops
);
1260 pr_warning("Could not create debugfs "
1261 "'set_ftrace_filter' entry\n");
1263 entry
= debugfs_create_file("set_ftrace_notrace", 0644, d_tracer
,
1264 NULL
, &ftrace_notrace_fops
);
1266 pr_warning("Could not create debugfs "
1267 "'set_ftrace_notrace' entry\n");
1272 fs_initcall(ftrace_init_debugfs
);
1274 static int ftrace_convert_nops(unsigned long *start
,
1279 unsigned long flags
;
1281 mutex_lock(&ftrace_start_lock
);
1284 addr
= ftrace_call_adjust(*p
++);
1285 ftrace_record_ip(addr
);
1288 /* disable interrupts to prevent kstop machine */
1289 local_irq_save(flags
);
1290 ftrace_update_code();
1291 local_irq_restore(flags
);
1292 mutex_unlock(&ftrace_start_lock
);
1297 void ftrace_init_module(unsigned long *start
, unsigned long *end
)
1299 if (ftrace_disabled
|| start
== end
)
1301 ftrace_convert_nops(start
, end
);
1304 extern unsigned long __start_mcount_loc
[];
1305 extern unsigned long __stop_mcount_loc
[];
1307 void __init
ftrace_init(void)
1309 unsigned long count
, addr
, flags
;
1312 /* Keep the ftrace pointer to the stub */
1313 addr
= (unsigned long)ftrace_stub
;
1315 local_irq_save(flags
);
1316 ftrace_dyn_arch_init(&addr
);
1317 local_irq_restore(flags
);
1319 /* ftrace_dyn_arch_init places the return code in addr */
1323 count
= __stop_mcount_loc
- __start_mcount_loc
;
1325 ret
= ftrace_dyn_table_alloc(count
);
1329 last_ftrace_enabled
= ftrace_enabled
= 1;
1331 ret
= ftrace_convert_nops(__start_mcount_loc
,
1336 ftrace_disabled
= 1;
1340 # define ftrace_startup() do { } while (0)
1341 # define ftrace_shutdown() do { } while (0)
1342 # define ftrace_startup_sysctl() do { } while (0)
1343 # define ftrace_shutdown_sysctl() do { } while (0)
1344 #endif /* CONFIG_DYNAMIC_FTRACE */
1347 * ftrace_kill - kill ftrace
1349 * This function should be used by panic code. It stops ftrace
1350 * but in a not so nice way. If you need to simply kill ftrace
1351 * from a non-atomic section, use ftrace_kill.
1353 void ftrace_kill(void)
1355 ftrace_disabled
= 1;
1357 clear_ftrace_function();
1361 * register_ftrace_function - register a function for profiling
1362 * @ops - ops structure that holds the function for profiling.
1364 * Register a function to be called by all functions in the
1367 * Note: @ops->func and all the functions it calls must be labeled
1368 * with "notrace", otherwise it will go into a
1371 int register_ftrace_function(struct ftrace_ops
*ops
)
1375 if (unlikely(ftrace_disabled
))
1378 mutex_lock(&ftrace_sysctl_lock
);
1379 ret
= __register_ftrace_function(ops
);
1381 mutex_unlock(&ftrace_sysctl_lock
);
1387 * unregister_ftrace_function - unresgister a function for profiling.
1388 * @ops - ops structure that holds the function to unregister
1390 * Unregister a function that was added to be called by ftrace profiling.
1392 int unregister_ftrace_function(struct ftrace_ops
*ops
)
1396 mutex_lock(&ftrace_sysctl_lock
);
1397 ret
= __unregister_ftrace_function(ops
);
1399 mutex_unlock(&ftrace_sysctl_lock
);
1405 ftrace_enable_sysctl(struct ctl_table
*table
, int write
,
1406 struct file
*file
, void __user
*buffer
, size_t *lenp
,
1411 if (unlikely(ftrace_disabled
))
1414 mutex_lock(&ftrace_sysctl_lock
);
1416 ret
= proc_dointvec(table
, write
, file
, buffer
, lenp
, ppos
);
1418 if (ret
|| !write
|| (last_ftrace_enabled
== ftrace_enabled
))
1421 last_ftrace_enabled
= ftrace_enabled
;
1423 if (ftrace_enabled
) {
1425 ftrace_startup_sysctl();
1427 /* we are starting ftrace again */
1428 if (ftrace_list
!= &ftrace_list_end
) {
1429 if (ftrace_list
->next
== &ftrace_list_end
)
1430 ftrace_trace_function
= ftrace_list
->func
;
1432 ftrace_trace_function
= ftrace_list_func
;
1436 /* stopping ftrace calls (just send to ftrace_stub) */
1437 ftrace_trace_function
= ftrace_stub
;
1439 ftrace_shutdown_sysctl();
1443 mutex_unlock(&ftrace_sysctl_lock
);