4 * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
6 #include <linux/kallsyms.h>
7 #include <linux/seq_file.h>
8 #include <linux/spinlock.h>
9 #include <linux/irqflags.h>
10 #include <linux/debugfs.h>
11 #include <linux/uaccess.h>
12 #include <linux/module.h>
13 #include <linux/ftrace.h>
14 #include <linux/hash.h>
16 #include <asm/local.h>
19 #include "trace_stat.h"
20 #include "trace_output.h"
22 #ifdef CONFIG_BRANCH_TRACER
24 static struct tracer branch_trace
;
25 static int branch_tracing_enabled __read_mostly
;
26 static DEFINE_MUTEX(branch_tracing_mutex
);
28 static struct trace_array
*branch_tracer
;
31 probe_likely_condition(struct ftrace_branch_data
*f
, int val
, int expect
)
33 struct trace_array
*tr
= branch_tracer
;
34 struct ring_buffer_event
*event
;
35 struct trace_branch
*entry
;
36 unsigned long flags
, irq_flags
;
41 * I would love to save just the ftrace_likely_data pointer, but
42 * this code can also be used by modules. Ugly things can happen
43 * if the module is unloaded, and then we go and read the
44 * pointer. This is slower, but much safer.
50 local_irq_save(flags
);
51 cpu
= raw_smp_processor_id();
52 if (atomic_inc_return(&tr
->data
[cpu
]->disabled
) != 1)
55 event
= ring_buffer_lock_reserve(tr
->buffer
, sizeof(*entry
),
61 entry
= ring_buffer_event_data(event
);
62 tracing_generic_entry_update(&entry
->ent
, flags
, pc
);
63 entry
->ent
.type
= TRACE_BRANCH
;
65 /* Strip off the path, only save the file */
66 p
= f
->file
+ strlen(f
->file
);
67 while (p
>= f
->file
&& *p
!= '/')
71 strncpy(entry
->func
, f
->func
, TRACE_FUNC_SIZE
);
72 strncpy(entry
->file
, p
, TRACE_FILE_SIZE
);
73 entry
->func
[TRACE_FUNC_SIZE
] = 0;
74 entry
->file
[TRACE_FILE_SIZE
] = 0;
75 entry
->line
= f
->line
;
76 entry
->correct
= val
== expect
;
78 ring_buffer_unlock_commit(tr
->buffer
, event
, irq_flags
);
81 atomic_dec(&tr
->data
[cpu
]->disabled
);
82 local_irq_restore(flags
);
86 void trace_likely_condition(struct ftrace_branch_data
*f
, int val
, int expect
)
88 if (!branch_tracing_enabled
)
91 probe_likely_condition(f
, val
, expect
);
94 int enable_branch_tracing(struct trace_array
*tr
)
98 mutex_lock(&branch_tracing_mutex
);
101 * Must be seen before enabling. The reader is a condition
102 * where we do not need a matching rmb()
105 branch_tracing_enabled
++;
106 mutex_unlock(&branch_tracing_mutex
);
111 void disable_branch_tracing(void)
113 mutex_lock(&branch_tracing_mutex
);
115 if (!branch_tracing_enabled
)
118 branch_tracing_enabled
--;
121 mutex_unlock(&branch_tracing_mutex
);
124 static void start_branch_trace(struct trace_array
*tr
)
126 enable_branch_tracing(tr
);
129 static void stop_branch_trace(struct trace_array
*tr
)
131 disable_branch_tracing();
134 static int branch_trace_init(struct trace_array
*tr
)
136 tracing_reset_online_cpus(tr
);
137 start_branch_trace(tr
);
141 static void branch_trace_reset(struct trace_array
*tr
)
143 stop_branch_trace(tr
);
147 trace_print_print(struct trace_seq
*s
, struct trace_entry
*entry
, int flags
)
149 struct print_entry
*field
;
151 trace_assign_type(field
, entry
);
153 if (seq_print_ip_sym(s
, field
->ip
, flags
))
156 if (trace_seq_printf(s
, ": %s", field
->buf
))
160 return TRACE_TYPE_PARTIAL_LINE
;
163 static enum print_line_t
trace_branch_print(struct trace_iterator
*iter
,
166 struct trace_branch
*field
;
168 trace_assign_type(field
, iter
->ent
);
170 if (trace_seq_printf(&iter
->seq
, "[%s] %s:%s:%d\n",
171 field
->correct
? " ok " : " MISS ",
175 return TRACE_TYPE_PARTIAL_LINE
;
177 return TRACE_TYPE_HANDLED
;
181 static struct trace_event trace_branch_event
= {
182 .type
= TRACE_BRANCH
,
183 .trace
= trace_branch_print
,
184 .latency_trace
= trace_branch_print
,
187 static struct tracer branch_trace __read_mostly
=
190 .init
= branch_trace_init
,
191 .reset
= branch_trace_reset
,
192 #ifdef CONFIG_FTRACE_SELFTEST
193 .selftest
= trace_selftest_startup_branch
,
194 #endif /* CONFIG_FTRACE_SELFTEST */
197 __init
static int init_branch_tracer(void)
201 ret
= register_ftrace_event(&trace_branch_event
);
203 printk(KERN_WARNING
"Warning: could not register "
207 return register_tracer(&branch_trace
);
209 device_initcall(init_branch_tracer
);
213 void trace_likely_condition(struct ftrace_branch_data
*f
, int val
, int expect
)
216 #endif /* CONFIG_BRANCH_TRACER */
218 void ftrace_likely_update(struct ftrace_branch_data
*f
, int val
, int expect
)
221 * I would love to have a trace point here instead, but the
222 * trace point code is so inundated with unlikely and likely
223 * conditions that the recursive nightmare that exists is too
224 * much to try to get working. At least for now.
226 trace_likely_condition(f
, val
, expect
);
228 /* FIXME: Make this atomic! */
234 EXPORT_SYMBOL(ftrace_likely_update
);
236 extern unsigned long __start_annotated_branch_profile
[];
237 extern unsigned long __stop_annotated_branch_profile
[];
239 static int annotated_branch_stat_headers(struct seq_file
*m
)
241 seq_printf(m
, " correct incorrect %% ");
242 seq_printf(m
, " Function "
244 " ------- --------- - "
250 static inline long get_incorrect_percent(struct ftrace_branch_data
*p
)
255 percent
= p
->incorrect
* 100;
256 percent
/= p
->correct
+ p
->incorrect
;
258 percent
= p
->incorrect
? 100 : -1;
263 static int branch_stat_show(struct seq_file
*m
, void *v
)
265 struct ftrace_branch_data
*p
= v
;
269 /* Only print the file, not the path */
270 f
= p
->file
+ strlen(p
->file
);
271 while (f
>= p
->file
&& *f
!= '/')
276 * The miss is overlayed on correct, and hit on incorrect.
278 percent
= get_incorrect_percent(p
);
280 seq_printf(m
, "%8lu %8lu ", p
->correct
, p
->incorrect
);
282 seq_printf(m
, " X ");
284 seq_printf(m
, "%3ld ", percent
);
285 seq_printf(m
, "%-30.30s %-20.20s %d\n", p
->func
, f
, p
->line
);
289 static void *annotated_branch_stat_start(void)
291 return __start_annotated_branch_profile
;
295 annotated_branch_stat_next(void *v
, int idx
)
297 struct ftrace_branch_data
*p
= v
;
301 if ((void *)p
>= (void *)__stop_annotated_branch_profile
)
307 static int annotated_branch_stat_cmp(void *p1
, void *p2
)
309 struct ftrace_branch_data
*a
= p1
;
310 struct ftrace_branch_data
*b
= p2
;
312 long percent_a
, percent_b
;
314 percent_a
= get_incorrect_percent(a
);
315 percent_b
= get_incorrect_percent(b
);
317 if (percent_a
< percent_b
)
319 if (percent_a
> percent_b
)
325 static struct tracer_stat annotated_branch_stats
= {
326 .name
= "branch_annotated",
327 .stat_start
= annotated_branch_stat_start
,
328 .stat_next
= annotated_branch_stat_next
,
329 .stat_cmp
= annotated_branch_stat_cmp
,
330 .stat_headers
= annotated_branch_stat_headers
,
331 .stat_show
= branch_stat_show
334 __init
static int init_annotated_branch_stats(void)
338 ret
= register_stat_tracer(&annotated_branch_stats
);
340 printk(KERN_WARNING
"Warning: could not register "
341 "annotated branches stats\n");
346 fs_initcall(init_annotated_branch_stats
);
348 #ifdef CONFIG_PROFILE_ALL_BRANCHES
350 extern unsigned long __start_branch_profile
[];
351 extern unsigned long __stop_branch_profile
[];
353 static int all_branch_stat_headers(struct seq_file
*m
)
355 seq_printf(m
, " miss hit %% ");
356 seq_printf(m
, " Function "
358 " ------- --------- - "
364 static void *all_branch_stat_start(void)
366 return __start_branch_profile
;
370 all_branch_stat_next(void *v
, int idx
)
372 struct ftrace_branch_data
*p
= v
;
376 if ((void *)p
>= (void *)__stop_branch_profile
)
382 static struct tracer_stat all_branch_stats
= {
383 .name
= "branch_all",
384 .stat_start
= all_branch_stat_start
,
385 .stat_next
= all_branch_stat_next
,
386 .stat_headers
= all_branch_stat_headers
,
387 .stat_show
= branch_stat_show
390 __init
static int all_annotated_branch_stats(void)
394 ret
= register_stat_tracer(&all_branch_stats
);
396 printk(KERN_WARNING
"Warning: could not register "
397 "all branches stats\n");
402 fs_initcall(all_annotated_branch_stats
);
403 #endif /* CONFIG_PROFILE_ALL_BRANCHES */