?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
ftrace: Fix regression where ftrace breaks when modules are loaded
[linux-2.6/btrfs-unstable.git] / kernel / trace / ftrace.c
blobef9271b69b4f1759fa6dff8b0f8effaa19a13726
1 /*
2 * Infrastructure for profiling code inserted by 'gcc -pg'.
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally ported from the -rt patch by:
8 * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code in the latency_tracer, that is:
11 *
12 * Copyright (C) 2004-2006 Ingo Molnar
13 * Copyright (C) 2004 William Lee Irwin III
14 */
15
16 #include <linux/stop_machine.h>
17 #include <linux/clocksource.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/suspend.h>
21 #include <linux/debugfs.h>
22 #include <linux/hardirq.h>
23 #include <linux/kthread.h>
24 #include <linux/uaccess.h>
25 #include <linux/ftrace.h>
26 #include <linux/sysctl.h>
27 #include <linux/slab.h>
28 #include <linux/ctype.h>
29 #include <linux/list.h>
30 #include <linux/hash.h>
31 #include <linux/rcupdate.h>
32
33 #include <trace/events/sched.h>
34
35 #include <asm/ftrace.h>
36 #include <asm/setup.h>
37
38 #include "trace_output.h"
39 #include "trace_stat.h"
40
41 #define FTRACE_WARN_ON(cond) \
42 ({ \
43 int ___r = cond; \
44 if (WARN_ON(___r)) \
45 ftrace_kill(); \
46 ___r; \
47 })
48
49 #define FTRACE_WARN_ON_ONCE(cond) \
50 ({ \
51 int ___r = cond; \
52 if (WARN_ON_ONCE(___r)) \
53 ftrace_kill(); \
54 ___r; \
55 })
56
57 /* hash bits for specific function selection */
58 #define FTRACE_HASH_BITS 7
59 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
60 #define FTRACE_HASH_DEFAULT_BITS 10
61 #define FTRACE_HASH_MAX_BITS 12
62
63 /* ftrace_enabled is a method to turn ftrace on or off */
64 int ftrace_enabled __read_mostly;
65 static int last_ftrace_enabled;
66
67 /* Quick disabling of function tracer. */
68 int function_trace_stop;
69
70 /* List for set_ftrace_pid's pids. */
71 LIST_HEAD(ftrace_pids);
72 struct ftrace_pid {
73 struct list_head list;
74 struct pid *pid;
75 };
76
77 /*
78 * ftrace_disabled is set when an anomaly is discovered.
79 * ftrace_disabled is much stronger than ftrace_enabled.
80 */
81 static int ftrace_disabled __read_mostly;
82
83 static DEFINE_MUTEX(ftrace_lock);
84
85 static struct ftrace_ops ftrace_list_end __read_mostly =
86 {
87 .func = ftrace_stub,
88 };
89
90 static struct ftrace_ops *ftrace_global_list __read_mostly = &ftrace_list_end;
91 static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end;
92 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
93 ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
94 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
95 static struct ftrace_ops global_ops;
96
97 static void
98 ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip);
99
100 /*
101 * Traverse the ftrace_global_list, invoking all entries. The reason that we
102 * can use rcu_dereference_raw() is that elements removed from this list
103 * are simply leaked, so there is no need to interact with a grace-period
104 * mechanism. The rcu_dereference_raw() calls are needed to handle
105 * concurrent insertions into the ftrace_global_list.
106 *
107 * Silly Alpha and silly pointer-speculation compiler optimizations!
108 */
109 static void ftrace_global_list_func(unsigned long ip,
110 unsigned long parent_ip)
111 {
112 struct ftrace_ops *op;
113
114 if (unlikely(trace_recursion_test(TRACE_GLOBAL_BIT)))
115 return;
116
117 trace_recursion_set(TRACE_GLOBAL_BIT);
118 op = rcu_dereference_raw(ftrace_global_list); /*see above*/
119 while (op != &ftrace_list_end) {
120 op->func(ip, parent_ip);
121 op = rcu_dereference_raw(op->next); /*see above*/
122 };
123 trace_recursion_clear(TRACE_GLOBAL_BIT);
124 }
125
126 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
127 {
128 if (!test_tsk_trace_trace(current))
129 return;
130
131 ftrace_pid_function(ip, parent_ip);
132 }
133
134 static void set_ftrace_pid_function(ftrace_func_t func)
135 {
136 /* do not set ftrace_pid_function to itself! */
137 if (func != ftrace_pid_func)
138 ftrace_pid_function = func;
139 }
140
141 /**
142 * clear_ftrace_function - reset the ftrace function
143 *
144 * This NULLs the ftrace function and in essence stops
145 * tracing. There may be lag
146 */
147 void clear_ftrace_function(void)
148 {
149 ftrace_trace_function = ftrace_stub;
150 __ftrace_trace_function = ftrace_stub;
151 ftrace_pid_function = ftrace_stub;
152 }
153
154 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
155 /*
156 * For those archs that do not test ftrace_trace_stop in their
157 * mcount call site, we need to do it from C.
158 */
159 static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
160 {
161 if (function_trace_stop)
162 return;
163
164 __ftrace_trace_function(ip, parent_ip);
165 }
166 #endif
167
168 static void update_global_ops(void)
169 {
170 ftrace_func_t func;
171
172 /*
173 * If there's only one function registered, then call that
174 * function directly. Otherwise, we need to iterate over the
175 * registered callers.
176 */
177 if (ftrace_global_list == &ftrace_list_end ||
178 ftrace_global_list->next == &ftrace_list_end)
179 func = ftrace_global_list->func;
180 else
181 func = ftrace_global_list_func;
182
183 /* If we filter on pids, update to use the pid function */
184 if (!list_empty(&ftrace_pids)) {
185 set_ftrace_pid_function(func);
186 func = ftrace_pid_func;
187 }
188
189 global_ops.func = func;
190 }
191
192 static void update_ftrace_function(void)
193 {
194 ftrace_func_t func;
195
196 update_global_ops();
197
198 /*
199 * If we are at the end of the list and this ops is
200 * not dynamic, then have the mcount trampoline call
201 * the function directly
202 */
203 if (ftrace_ops_list == &ftrace_list_end ||
204 (ftrace_ops_list->next == &ftrace_list_end &&
205 !(ftrace_ops_list->flags & FTRACE_OPS_FL_DYNAMIC)))
206 func = ftrace_ops_list->func;
207 else
208 func = ftrace_ops_list_func;
209
210 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
211 ftrace_trace_function = func;
212 #else
213 __ftrace_trace_function = func;
214 ftrace_trace_function = ftrace_test_stop_func;
215 #endif
216 }
217
218 static void add_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
219 {
220 ops->next = *list;
221 /*
222 * We are entering ops into the list but another
223 * CPU might be walking that list. We need to make sure
224 * the ops->next pointer is valid before another CPU sees
225 * the ops pointer included into the list.
226 */
227 rcu_assign_pointer(*list, ops);
228 }
229
230 static int remove_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
231 {
232 struct ftrace_ops **p;
233
234 /*
235 * If we are removing the last function, then simply point
236 * to the ftrace_stub.
237 */
238 if (*list == ops && ops->next == &ftrace_list_end) {
239 *list = &ftrace_list_end;
240 return 0;
241 }
242
243 for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
244 if (*p == ops)
245 break;
246
247 if (*p != ops)
248 return -1;
249
250 *p = (*p)->next;
251 return 0;
252 }
253
254 static int __register_ftrace_function(struct ftrace_ops *ops)
255 {
256 if (ftrace_disabled)
257 return -ENODEV;
258
259 if (FTRACE_WARN_ON(ops == &global_ops))
260 return -EINVAL;
261
262 if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
263 return -EBUSY;
264
265 if (!core_kernel_data((unsigned long)ops))
266 ops->flags |= FTRACE_OPS_FL_DYNAMIC;
267
268 if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
269 int first = ftrace_global_list == &ftrace_list_end;
270 add_ftrace_ops(&ftrace_global_list, ops);
271 ops->flags |= FTRACE_OPS_FL_ENABLED;
272 if (first)
273 add_ftrace_ops(&ftrace_ops_list, &global_ops);
274 } else
275 add_ftrace_ops(&ftrace_ops_list, ops);
276
277 if (ftrace_enabled)
278 update_ftrace_function();
279
280 return 0;
281 }
282
283 static int __unregister_ftrace_function(struct ftrace_ops *ops)
284 {
285 int ret;
286
287 if (ftrace_disabled)
288 return -ENODEV;
289
290 if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
291 return -EBUSY;
292
293 if (FTRACE_WARN_ON(ops == &global_ops))
294 return -EINVAL;
295
296 if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
297 ret = remove_ftrace_ops(&ftrace_global_list, ops);
298 if (!ret && ftrace_global_list == &ftrace_list_end)
299 ret = remove_ftrace_ops(&ftrace_ops_list, &global_ops);
300 if (!ret)
301 ops->flags &= ~FTRACE_OPS_FL_ENABLED;
302 } else
303 ret = remove_ftrace_ops(&ftrace_ops_list, ops);
304
305 if (ret < 0)
306 return ret;
307
308 if (ftrace_enabled)
309 update_ftrace_function();
310
311 /*
312 * Dynamic ops may be freed, we must make sure that all
313 * callers are done before leaving this function.
314 */
315 if (ops->flags & FTRACE_OPS_FL_DYNAMIC)
316 synchronize_sched();
317
318 return 0;
319 }
320
321 static void ftrace_update_pid_func(void)
322 {
323 /* Only do something if we are tracing something */
324 if (ftrace_trace_function == ftrace_stub)
325 return;
326
327 update_ftrace_function();
328 }
329
330 #ifdef CONFIG_FUNCTION_PROFILER
331 struct ftrace_profile {
332 struct hlist_node node;
333 unsigned long ip;
334 unsigned long counter;
335 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
336 unsigned long long time;
337 unsigned long long time_squared;
338 #endif
339 };
340
341 struct ftrace_profile_page {
342 struct ftrace_profile_page *next;
343 unsigned long index;
344 struct ftrace_profile records[];
345 };
346
347 struct ftrace_profile_stat {
348 atomic_t disabled;
349 struct hlist_head *hash;
350 struct ftrace_profile_page *pages;
351 struct ftrace_profile_page *start;
352 struct tracer_stat stat;
353 };
354
355 #define PROFILE_RECORDS_SIZE \
356 (PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
357
358 #define PROFILES_PER_PAGE \
359 (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
360
361 static int ftrace_profile_bits __read_mostly;
362 static int ftrace_profile_enabled __read_mostly;
363
364 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */
365 static DEFINE_MUTEX(ftrace_profile_lock);
366
367 static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
368
369 #define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */
370
371 static void *
372 function_stat_next(void *v, int idx)
373 {
374 struct ftrace_profile *rec = v;
375 struct ftrace_profile_page *pg;
376
377 pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
378
379 again:
380 if (idx != 0)
381 rec++;
382
383 if ((void *)rec >= (void *)&pg->records[pg->index]) {
384 pg = pg->next;
385 if (!pg)
386 return NULL;
387 rec = &pg->records[0];
388 if (!rec->counter)
389 goto again;
390 }
391
392 return rec;
393 }
394
395 static void *function_stat_start(struct tracer_stat *trace)
396 {
397 struct ftrace_profile_stat *stat =
398 container_of(trace, struct ftrace_profile_stat, stat);
399
400 if (!stat || !stat->start)
401 return NULL;
402
403 return function_stat_next(&stat->start->records[0], 0);
404 }
405
406 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
407 /* function graph compares on total time */
408 static int function_stat_cmp(void *p1, void *p2)
409 {
410 struct ftrace_profile *a = p1;
411 struct ftrace_profile *b = p2;
412
413 if (a->time < b->time)
414 return -1;
415 if (a->time > b->time)
416 return 1;
417 else
418 return 0;
419 }
420 #else
421 /* not function graph compares against hits */
422 static int function_stat_cmp(void *p1, void *p2)
423 {
424 struct ftrace_profile *a = p1;
425 struct ftrace_profile *b = p2;
426
427 if (a->counter < b->counter)
428 return -1;
429 if (a->counter > b->counter)
430 return 1;
431 else
432 return 0;
433 }
434 #endif
435
436 static int function_stat_headers(struct seq_file *m)
437 {
438 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
439 seq_printf(m, " Function "
440 "Hit Time Avg s^2\n"
441 " -------- "
442 "--- ---- --- ---\n");
443 #else
444 seq_printf(m, " Function Hit\n"
445 " -------- ---\n");
446 #endif
447 return 0;
448 }
449
450 static int function_stat_show(struct seq_file *m, void *v)
451 {
452 struct ftrace_profile *rec = v;
453 char str[KSYM_SYMBOL_LEN];
454 int ret = 0;
455 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
456 static struct trace_seq s;
457 unsigned long long avg;
458 unsigned long long stddev;
459 #endif
460 mutex_lock(&ftrace_profile_lock);
461
462 /* we raced with function_profile_reset() */
463 if (unlikely(rec->counter == 0)) {
464 ret = -EBUSY;
465 goto out;
466 }
467
468 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
469 seq_printf(m, " %-30.30s %10lu", str, rec->counter);
470
471 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
472 seq_printf(m, " ");
473 avg = rec->time;
474 do_div(avg, rec->counter);
475
476 /* Sample standard deviation (s^2) */
477 if (rec->counter <= 1)
478 stddev = 0;
479 else {
480 stddev = rec->time_squared - rec->counter * avg * avg;
481 /*
482 * Divide only 1000 for ns^2 -> us^2 conversion.
483 * trace_print_graph_duration will divide 1000 again.
484 */
485 do_div(stddev, (rec->counter - 1) * 1000);
486 }
487
488 trace_seq_init(&s);
489 trace_print_graph_duration(rec->time, &s);
490 trace_seq_puts(&s, " ");
491 trace_print_graph_duration(avg, &s);
492 trace_seq_puts(&s, " ");
493 trace_print_graph_duration(stddev, &s);
494 trace_print_seq(m, &s);
495 #endif
496 seq_putc(m, '\n');
497 out:
498 mutex_unlock(&ftrace_profile_lock);
499
500 return ret;
501 }
502
503 static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
504 {
505 struct ftrace_profile_page *pg;
506
507 pg = stat->pages = stat->start;
508
509 while (pg) {
510 memset(pg->records, 0, PROFILE_RECORDS_SIZE);
511 pg->index = 0;
512 pg = pg->next;
513 }
514
515 memset(stat->hash, 0,
516 FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
517 }
518
519 int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
520 {
521 struct ftrace_profile_page *pg;
522 int functions;
523 int pages;
524 int i;
525
526 /* If we already allocated, do nothing */
527 if (stat->pages)
528 return 0;
529
530 stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
531 if (!stat->pages)
532 return -ENOMEM;
533
534 #ifdef CONFIG_DYNAMIC_FTRACE
535 functions = ftrace_update_tot_cnt;
536 #else
537 /*
538 * We do not know the number of functions that exist because
539 * dynamic tracing is what counts them. With past experience
540 * we have around 20K functions. That should be more than enough.
541 * It is highly unlikely we will execute every function in
542 * the kernel.
543 */
544 functions = 20000;
545 #endif
546
547 pg = stat->start = stat->pages;
548
549 pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
550
551 for (i = 0; i < pages; i++) {
552 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
553 if (!pg->next)
554 goto out_free;
555 pg = pg->next;
556 }
557
558 return 0;
559
560 out_free:
561 pg = stat->start;
562 while (pg) {
563 unsigned long tmp = (unsigned long)pg;
564
565 pg = pg->next;
566 free_page(tmp);
567 }
568
569 free_page((unsigned long)stat->pages);
570 stat->pages = NULL;
571 stat->start = NULL;
572
573 return -ENOMEM;
574 }
575
576 static int ftrace_profile_init_cpu(int cpu)
577 {
578 struct ftrace_profile_stat *stat;
579 int size;
580
581 stat = &per_cpu(ftrace_profile_stats, cpu);
582
583 if (stat->hash) {
584 /* If the profile is already created, simply reset it */
585 ftrace_profile_reset(stat);
586 return 0;
587 }
588
589 /*
590 * We are profiling all functions, but usually only a few thousand
591 * functions are hit. We'll make a hash of 1024 items.
592 */
593 size = FTRACE_PROFILE_HASH_SIZE;
594
595 stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
596
597 if (!stat->hash)
598 return -ENOMEM;
599
600 if (!ftrace_profile_bits) {
601 size--;
602
603 for (; size; size >>= 1)
604 ftrace_profile_bits++;
605 }
606
607 /* Preallocate the function profiling pages */
608 if (ftrace_profile_pages_init(stat) < 0) {
609 kfree(stat->hash);
610 stat->hash = NULL;
611 return -ENOMEM;
612 }
613
614 return 0;
615 }
616
617 static int ftrace_profile_init(void)
618 {
619 int cpu;
620 int ret = 0;
621
622 for_each_online_cpu(cpu) {
623 ret = ftrace_profile_init_cpu(cpu);
624 if (ret)
625 break;
626 }
627
628 return ret;
629 }
630
631 /* interrupts must be disabled */
632 static struct ftrace_profile *
633 ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
634 {
635 struct ftrace_profile *rec;
636 struct hlist_head *hhd;
637 struct hlist_node *n;
638 unsigned long key;
639
640 key = hash_long(ip, ftrace_profile_bits);
641 hhd = &stat->hash[key];
642
643 if (hlist_empty(hhd))
644 return NULL;
645
646 hlist_for_each_entry_rcu(rec, n, hhd, node) {
647 if (rec->ip == ip)
648 return rec;
649 }
650
651 return NULL;
652 }
653
654 static void ftrace_add_profile(struct ftrace_profile_stat *stat,
655 struct ftrace_profile *rec)
656 {
657 unsigned long key;
658
659 key = hash_long(rec->ip, ftrace_profile_bits);
660 hlist_add_head_rcu(&rec->node, &stat->hash[key]);
661 }
662
663 /*
664 * The memory is already allocated, this simply finds a new record to use.
665 */
666 static struct ftrace_profile *
667 ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
668 {
669 struct ftrace_profile *rec = NULL;
670
671 /* prevent recursion (from NMIs) */
672 if (atomic_inc_return(&stat->disabled) != 1)
673 goto out;
674
675 /*
676 * Try to find the function again since an NMI
677 * could have added it
678 */
679 rec = ftrace_find_profiled_func(stat, ip);
680 if (rec)
681 goto out;
682
683 if (stat->pages->index == PROFILES_PER_PAGE) {
684 if (!stat->pages->next)
685 goto out;
686 stat->pages = stat->pages->next;
687 }
688
689 rec = &stat->pages->records[stat->pages->index++];
690 rec->ip = ip;
691 ftrace_add_profile(stat, rec);
692
693 out:
694 atomic_dec(&stat->disabled);
695
696 return rec;
697 }
698
699 static void
700 function_profile_call(unsigned long ip, unsigned long parent_ip)
701 {
702 struct ftrace_profile_stat *stat;
703 struct ftrace_profile *rec;
704 unsigned long flags;
705
706 if (!ftrace_profile_enabled)
707 return;
708
709 local_irq_save(flags);
710
711 stat = &__get_cpu_var(ftrace_profile_stats);
712 if (!stat->hash || !ftrace_profile_enabled)
713 goto out;
714
715 rec = ftrace_find_profiled_func(stat, ip);
716 if (!rec) {
717 rec = ftrace_profile_alloc(stat, ip);
718 if (!rec)
719 goto out;
720 }
721
722 rec->counter++;
723 out:
724 local_irq_restore(flags);
725 }
726
727 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
728 static int profile_graph_entry(struct ftrace_graph_ent *trace)
729 {
730 function_profile_call(trace->func, 0);
731 return 1;
732 }
733
734 static void profile_graph_return(struct ftrace_graph_ret *trace)
735 {
736 struct ftrace_profile_stat *stat;
737 unsigned long long calltime;
738 struct ftrace_profile *rec;
739 unsigned long flags;
740
741 local_irq_save(flags);
742 stat = &__get_cpu_var(ftrace_profile_stats);
743 if (!stat->hash || !ftrace_profile_enabled)
744 goto out;
745
746 /* If the calltime was zero'd ignore it */
747 if (!trace->calltime)
748 goto out;
749
750 calltime = trace->rettime - trace->calltime;
751
752 if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) {
753 int index;
754
755 index = trace->depth;
756
757 /* Append this call time to the parent time to subtract */
758 if (index)
759 current->ret_stack[index - 1].subtime += calltime;
760
761 if (current->ret_stack[index].subtime < calltime)
762 calltime -= current->ret_stack[index].subtime;
763 else
764 calltime = 0;
765 }
766
767 rec = ftrace_find_profiled_func(stat, trace->func);
768 if (rec) {
769 rec->time += calltime;
770 rec->time_squared += calltime * calltime;
771 }
772
773 out:
774 local_irq_restore(flags);
775 }
776
777 static int register_ftrace_profiler(void)
778 {
779 return register_ftrace_graph(&profile_graph_return,
780 &profile_graph_entry);
781 }
782
783 static void unregister_ftrace_profiler(void)
784 {
785 unregister_ftrace_graph();
786 }
787 #else
788 static struct ftrace_ops ftrace_profile_ops __read_mostly =
789 {
790 .func = function_profile_call,
791 };
792
793 static int register_ftrace_profiler(void)
794 {
795 return register_ftrace_function(&ftrace_profile_ops);
796 }
797
798 static void unregister_ftrace_profiler(void)
799 {
800 unregister_ftrace_function(&ftrace_profile_ops);
801 }
802 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
803
804 static ssize_t
805 ftrace_profile_write(struct file *filp, const char __user *ubuf,
806 size_t cnt, loff_t *ppos)
807 {
808 unsigned long val;
809 char buf[64]; /* big enough to hold a number */
810 int ret;
811
812 if (cnt >= sizeof(buf))
813 return -EINVAL;
814
815 if (copy_from_user(&buf, ubuf, cnt))
816 return -EFAULT;
817
818 buf[cnt] = 0;
819
820 ret = strict_strtoul(buf, 10, &val);
821 if (ret < 0)
822 return ret;
823
824 val = !!val;
825
826 mutex_lock(&ftrace_profile_lock);
827 if (ftrace_profile_enabled ^ val) {
828 if (val) {
829 ret = ftrace_profile_init();
830 if (ret < 0) {
831 cnt = ret;
832 goto out;
833 }
834
835 ret = register_ftrace_profiler();
836 if (ret < 0) {
837 cnt = ret;
838 goto out;
839 }
840 ftrace_profile_enabled = 1;
841 } else {
842 ftrace_profile_enabled = 0;
843 /*
844 * unregister_ftrace_profiler calls stop_machine
845 * so this acts like an synchronize_sched.
846 */
847 unregister_ftrace_profiler();
848 }
849 }
850 out:
851 mutex_unlock(&ftrace_profile_lock);
852
853 *ppos += cnt;
854
855 return cnt;
856 }
857
858 static ssize_t
859 ftrace_profile_read(struct file *filp, char __user *ubuf,
860 size_t cnt, loff_t *ppos)
861 {
862 char buf[64]; /* big enough to hold a number */
863 int r;
864
865 r = sprintf(buf, "%u\n", ftrace_profile_enabled);
866 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
867 }
868
869 static const struct file_operations ftrace_profile_fops = {
870 .open = tracing_open_generic,
871 .read = ftrace_profile_read,
872 .write = ftrace_profile_write,
873 .llseek = default_llseek,
874 };
875
876 /* used to initialize the real stat files */
877 static struct tracer_stat function_stats __initdata = {
878 .name = "functions",
879 .stat_start = function_stat_start,
880 .stat_next = function_stat_next,
881 .stat_cmp = function_stat_cmp,
882 .stat_headers = function_stat_headers,
883 .stat_show = function_stat_show
884 };
885
886 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
887 {
888 struct ftrace_profile_stat *stat;
889 struct dentry *entry;
890 char *name;
891 int ret;
892 int cpu;
893
894 for_each_possible_cpu(cpu) {
895 stat = &per_cpu(ftrace_profile_stats, cpu);
896
897 /* allocate enough for function name + cpu number */
898 name = kmalloc(32, GFP_KERNEL);
899 if (!name) {
900 /*
901 * The files created are permanent, if something happens
902 * we still do not free memory.
903 */
904 WARN(1,
905 "Could not allocate stat file for cpu %d\n",
906 cpu);
907 return;
908 }
909 stat->stat = function_stats;
910 snprintf(name, 32, "function%d", cpu);
911 stat->stat.name = name;
912 ret = register_stat_tracer(&stat->stat);
913 if (ret) {
914 WARN(1,
915 "Could not register function stat for cpu %d\n",
916 cpu);
917 kfree(name);
918 return;
919 }
920 }
921
922 entry = debugfs_create_file("function_profile_enabled", 0644,
923 d_tracer, NULL, &ftrace_profile_fops);
924 if (!entry)
925 pr_warning("Could not create debugfs "
926 "'function_profile_enabled' entry\n");
927 }
928
929 #else /* CONFIG_FUNCTION_PROFILER */
930 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
931 {
932 }
933 #endif /* CONFIG_FUNCTION_PROFILER */
934
935 static struct pid * const ftrace_swapper_pid = &init_struct_pid;
936
937 #ifdef CONFIG_DYNAMIC_FTRACE
938
939 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
940 # error Dynamic ftrace depends on MCOUNT_RECORD
941 #endif
942
943 static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
944
945 struct ftrace_func_probe {
946 struct hlist_node node;
947 struct ftrace_probe_ops *ops;
948 unsigned long flags;
949 unsigned long ip;
950 void *data;
951 struct rcu_head rcu;
952 };
953
954 enum {
955 FTRACE_ENABLE_CALLS = (1 << 0),
956 FTRACE_DISABLE_CALLS = (1 << 1),
957 FTRACE_UPDATE_TRACE_FUNC = (1 << 2),
958 FTRACE_START_FUNC_RET = (1 << 3),
959 FTRACE_STOP_FUNC_RET = (1 << 4),
960 };
961 struct ftrace_func_entry {
962 struct hlist_node hlist;
963 unsigned long ip;
964 };
965
966 struct ftrace_hash {
967 unsigned long size_bits;
968 struct hlist_head *buckets;
969 unsigned long count;
970 struct rcu_head rcu;
971 };
972
973 /*
974 * We make these constant because no one should touch them,
975 * but they are used as the default "empty hash", to avoid allocating
976 * it all the time. These are in a read only section such that if
977 * anyone does try to modify it, it will cause an exception.
978 */
979 static const struct hlist_head empty_buckets[1];
980 static const struct ftrace_hash empty_hash = {
981 .buckets = (struct hlist_head *)empty_buckets,
982 };
983 #define EMPTY_HASH ((struct ftrace_hash *)&empty_hash)
984
985 static struct ftrace_ops global_ops = {
986 .func = ftrace_stub,
987 .notrace_hash = EMPTY_HASH,
988 .filter_hash = EMPTY_HASH,
989 };
990
991 static struct dyn_ftrace *ftrace_new_addrs;
992
993 static DEFINE_MUTEX(ftrace_regex_lock);
994
995 struct ftrace_page {
996 struct ftrace_page *next;
997 int index;
998 struct dyn_ftrace records[];
999 };
1000
1001 #define ENTRIES_PER_PAGE \
1002 ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
1003
1004 /* estimate from running different kernels */
1005 #define NR_TO_INIT 10000
1006
1007 static struct ftrace_page *ftrace_pages_start;
1008 static struct ftrace_page *ftrace_pages;
1009
1010 static struct dyn_ftrace *ftrace_free_records;
1011
1012 static struct ftrace_func_entry *
1013 ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
1014 {
1015 unsigned long key;
1016 struct ftrace_func_entry *entry;
1017 struct hlist_head *hhd;
1018 struct hlist_node *n;
1019
1020 if (!hash->count)
1021 return NULL;
1022
1023 if (hash->size_bits > 0)
1024 key = hash_long(ip, hash->size_bits);
1025 else
1026 key = 0;
1027
1028 hhd = &hash->buckets[key];
1029
1030 hlist_for_each_entry_rcu(entry, n, hhd, hlist) {
1031 if (entry->ip == ip)
1032 return entry;
1033 }
1034 return NULL;
1035 }
1036
1037 static void __add_hash_entry(struct ftrace_hash *hash,
1038 struct ftrace_func_entry *entry)
1039 {
1040 struct hlist_head *hhd;
1041 unsigned long key;
1042
1043 if (hash->size_bits)
1044 key = hash_long(entry->ip, hash->size_bits);
1045 else
1046 key = 0;
1047
1048 hhd = &hash->buckets[key];
1049 hlist_add_head(&entry->hlist, hhd);
1050 hash->count++;
1051 }
1052
1053 static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
1054 {
1055 struct ftrace_func_entry *entry;
1056
1057 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1058 if (!entry)
1059 return -ENOMEM;
1060
1061 entry->ip = ip;
1062 __add_hash_entry(hash, entry);
1063
1064 return 0;
1065 }
1066
1067 static void
1068 free_hash_entry(struct ftrace_hash *hash,
1069 struct ftrace_func_entry *entry)
1070 {
1071 hlist_del(&entry->hlist);
1072 kfree(entry);
1073 hash->count--;
1074 }
1075
1076 static void
1077 remove_hash_entry(struct ftrace_hash *hash,
1078 struct ftrace_func_entry *entry)
1079 {
1080 hlist_del(&entry->hlist);
1081 hash->count--;
1082 }
1083
1084 static void ftrace_hash_clear(struct ftrace_hash *hash)
1085 {
1086 struct hlist_head *hhd;
1087 struct hlist_node *tp, *tn;
1088 struct ftrace_func_entry *entry;
1089 int size = 1 << hash->size_bits;
1090 int i;
1091
1092 if (!hash->count)
1093 return;
1094
1095 for (i = 0; i < size; i++) {
1096 hhd = &hash->buckets[i];
1097 hlist_for_each_entry_safe(entry, tp, tn, hhd, hlist)
1098 free_hash_entry(hash, entry);
1099 }
1100 FTRACE_WARN_ON(hash->count);
1101 }
1102
1103 static void free_ftrace_hash(struct ftrace_hash *hash)
1104 {
1105 if (!hash || hash == EMPTY_HASH)
1106 return;
1107 ftrace_hash_clear(hash);
1108 kfree(hash->buckets);
1109 kfree(hash);
1110 }
1111
1112 static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
1113 {
1114 struct ftrace_hash *hash;
1115
1116 hash = container_of(rcu, struct ftrace_hash, rcu);
1117 free_ftrace_hash(hash);
1118 }
1119
1120 static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
1121 {
1122 if (!hash || hash == EMPTY_HASH)
1123 return;
1124 call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu);
1125 }
1126
1127 static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
1128 {
1129 struct ftrace_hash *hash;
1130 int size;
1131
1132 hash = kzalloc(sizeof(*hash), GFP_KERNEL);
1133 if (!hash)
1134 return NULL;
1135
1136 size = 1 << size_bits;
1137 hash->buckets = kzalloc(sizeof(*hash->buckets) * size, GFP_KERNEL);
1138
1139 if (!hash->buckets) {
1140 kfree(hash);
1141 return NULL;
1142 }
1143
1144 hash->size_bits = size_bits;
1145
1146 return hash;
1147 }
1148
1149 static struct ftrace_hash *
1150 alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
1151 {
1152 struct ftrace_func_entry *entry;
1153 struct ftrace_hash *new_hash;
1154 struct hlist_node *tp;
1155 int size;
1156 int ret;
1157 int i;
1158
1159 new_hash = alloc_ftrace_hash(size_bits);
1160 if (!new_hash)
1161 return NULL;
1162
1163 /* Empty hash? */
1164 if (!hash || !hash->count)
1165 return new_hash;
1166
1167 size = 1 << hash->size_bits;
1168 for (i = 0; i < size; i++) {
1169 hlist_for_each_entry(entry, tp, &hash->buckets[i], hlist) {
1170 ret = add_hash_entry(new_hash, entry->ip);
1171 if (ret < 0)
1172 goto free_hash;
1173 }
1174 }
1175
1176 FTRACE_WARN_ON(new_hash->count != hash->count);
1177
1178 return new_hash;
1179
1180 free_hash:
1181 free_ftrace_hash(new_hash);
1182 return NULL;
1183 }
1184
1185 static int
1186 ftrace_hash_move(struct ftrace_hash **dst, struct ftrace_hash *src)
1187 {
1188 struct ftrace_func_entry *entry;
1189 struct hlist_node *tp, *tn;
1190 struct hlist_head *hhd;
1191 struct ftrace_hash *old_hash;
1192 struct ftrace_hash *new_hash;
1193 unsigned long key;
1194 int size = src->count;
1195 int bits = 0;
1196 int i;
1197
1198 /*
1199 * If the new source is empty, just free dst and assign it
1200 * the empty_hash.
1201 */
1202 if (!src->count) {
1203 free_ftrace_hash_rcu(*dst);
1204 rcu_assign_pointer(*dst, EMPTY_HASH);
1205 return 0;
1206 }
1207
1208 /*
1209 * Make the hash size about 1/2 the # found
1210 */
1211 for (size /= 2; size; size >>= 1)
1212 bits++;
1213
1214 /* Don't allocate too much */
1215 if (bits > FTRACE_HASH_MAX_BITS)
1216 bits = FTRACE_HASH_MAX_BITS;
1217
1218 new_hash = alloc_ftrace_hash(bits);
1219 if (!new_hash)
1220 return -ENOMEM;
1221
1222 size = 1 << src->size_bits;
1223 for (i = 0; i < size; i++) {
1224 hhd = &src->buckets[i];
1225 hlist_for_each_entry_safe(entry, tp, tn, hhd, hlist) {
1226 if (bits > 0)
1227 key = hash_long(entry->ip, bits);
1228 else
1229 key = 0;
1230 remove_hash_entry(src, entry);
1231 __add_hash_entry(new_hash, entry);
1232 }
1233 }
1234
1235 old_hash = *dst;
1236 rcu_assign_pointer(*dst, new_hash);
1237 free_ftrace_hash_rcu(old_hash);
1238
1239 return 0;
1240 }
1241
1242 /*
1243 * Test the hashes for this ops to see if we want to call
1244 * the ops->func or not.
1245 *
1246 * It's a match if the ip is in the ops->filter_hash or
1247 * the filter_hash does not exist or is empty,
1248 * AND
1249 * the ip is not in the ops->notrace_hash.
1250 *
1251 * This needs to be called with preemption disabled as
1252 * the hashes are freed with call_rcu_sched().
1253 */
1254 static int
1255 ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
1256 {
1257 struct ftrace_hash *filter_hash;
1258 struct ftrace_hash *notrace_hash;
1259 int ret;
1260
1261 filter_hash = rcu_dereference_raw(ops->filter_hash);
1262 notrace_hash = rcu_dereference_raw(ops->notrace_hash);
1263
1264 if ((!filter_hash || !filter_hash->count ||
1265 ftrace_lookup_ip(filter_hash, ip)) &&
1266 (!notrace_hash || !notrace_hash->count ||
1267 !ftrace_lookup_ip(notrace_hash, ip)))
1268 ret = 1;
1269 else
1270 ret = 0;
1271
1272 return ret;
1273 }
1274
1275 /*
1276 * This is a double for. Do not use 'break' to break out of the loop,
1277 * you must use a goto.
1278 */
1279 #define do_for_each_ftrace_rec(pg, rec) \
1280 for (pg = ftrace_pages_start; pg; pg = pg->next) { \
1281 int _____i; \
1282 for (_____i = 0; _____i < pg->index; _____i++) { \
1283 rec = &pg->records[_____i];
1284
1285 #define while_for_each_ftrace_rec() \
1286 } \
1287 }
1288
1289 static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
1290 int filter_hash,
1291 bool inc)
1292 {
1293 struct ftrace_hash *hash;
1294 struct ftrace_hash *other_hash;
1295 struct ftrace_page *pg;
1296 struct dyn_ftrace *rec;
1297 int count = 0;
1298 int all = 0;
1299
1300 /* Only update if the ops has been registered */
1301 if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
1302 return;
1303
1304 /*
1305 * In the filter_hash case:
1306 * If the count is zero, we update all records.
1307 * Otherwise we just update the items in the hash.
1308 *
1309 * In the notrace_hash case:
1310 * We enable the update in the hash.
1311 * As disabling notrace means enabling the tracing,
1312 * and enabling notrace means disabling, the inc variable
1313 * gets inversed.
1314 */
1315 if (filter_hash) {
1316 hash = ops->filter_hash;
1317 other_hash = ops->notrace_hash;
1318 if (!hash || !hash->count)
1319 all = 1;
1320 } else {
1321 inc = !inc;
1322 hash = ops->notrace_hash;
1323 other_hash = ops->filter_hash;
1324 /*
1325 * If the notrace hash has no items,
1326 * then there's nothing to do.
1327 */
1328 if (hash && !hash->count)
1329 return;
1330 }
1331
1332 do_for_each_ftrace_rec(pg, rec) {
1333 int in_other_hash = 0;
1334 int in_hash = 0;
1335 int match = 0;
1336
1337 if (all) {
1338 /*
1339 * Only the filter_hash affects all records.
1340 * Update if the record is not in the notrace hash.
1341 */
1342 if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
1343 match = 1;
1344 } else {
1345 in_hash = hash && !!ftrace_lookup_ip(hash, rec->ip);
1346 in_other_hash = other_hash && !!ftrace_lookup_ip(other_hash, rec->ip);
1347
1348 /*
1349 *
1350 */
1351 if (filter_hash && in_hash && !in_other_hash)
1352 match = 1;
1353 else if (!filter_hash && in_hash &&
1354 (in_other_hash || !other_hash->count))
1355 match = 1;
1356 }
1357 if (!match)
1358 continue;
1359
1360 if (inc) {
1361 rec->flags++;
1362 if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == FTRACE_REF_MAX))
1363 return;
1364 } else {
1365 if (FTRACE_WARN_ON((rec->flags & ~FTRACE_FL_MASK) == 0))
1366 return;
1367 rec->flags--;
1368 }
1369 count++;
1370 /* Shortcut, if we handled all records, we are done. */
1371 if (!all && count == hash->count)
1372 return;
1373 } while_for_each_ftrace_rec();
1374 }
1375
1376 static void ftrace_hash_rec_disable(struct ftrace_ops *ops,
1377 int filter_hash)
1378 {
1379 __ftrace_hash_rec_update(ops, filter_hash, 0);
1380 }
1381
1382 static void ftrace_hash_rec_enable(struct ftrace_ops *ops,
1383 int filter_hash)
1384 {
1385 __ftrace_hash_rec_update(ops, filter_hash, 1);
1386 }
1387
1388 static void ftrace_free_rec(struct dyn_ftrace *rec)
1389 {
1390 rec->freelist = ftrace_free_records;
1391 ftrace_free_records = rec;
1392 rec->flags |= FTRACE_FL_FREE;
1393 }
1394
1395 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
1396 {
1397 struct dyn_ftrace *rec;
1398
1399 /* First check for freed records */
1400 if (ftrace_free_records) {
1401 rec = ftrace_free_records;
1402
1403 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
1404 FTRACE_WARN_ON_ONCE(1);
1405 ftrace_free_records = NULL;
1406 return NULL;
1407 }
1408
1409 ftrace_free_records = rec->freelist;
1410 memset(rec, 0, sizeof(*rec));
1411 return rec;
1412 }
1413
1414 if (ftrace_pages->index == ENTRIES_PER_PAGE) {
1415 if (!ftrace_pages->next) {
1416 /* allocate another page */
1417 ftrace_pages->next =
1418 (void *)get_zeroed_page(GFP_KERNEL);
1419 if (!ftrace_pages->next)
1420 return NULL;
1421 }
1422 ftrace_pages = ftrace_pages->next;
1423 }
1424
1425 return &ftrace_pages->records[ftrace_pages->index++];
1426 }
1427
1428 static struct dyn_ftrace *
1429 ftrace_record_ip(unsigned long ip)
1430 {
1431 struct dyn_ftrace *rec;
1432
1433 if (ftrace_disabled)
1434 return NULL;
1435
1436 rec = ftrace_alloc_dyn_node(ip);
1437 if (!rec)
1438 return NULL;
1439
1440 rec->ip = ip;
1441 rec->newlist = ftrace_new_addrs;
1442 ftrace_new_addrs = rec;
1443
1444 return rec;
1445 }
1446
1447 static void print_ip_ins(const char *fmt, unsigned char *p)
1448 {
1449 int i;
1450
1451 printk(KERN_CONT "%s", fmt);
1452
1453 for (i = 0; i < MCOUNT_INSN_SIZE; i++)
1454 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
1455 }
1456
1457 static void ftrace_bug(int failed, unsigned long ip)
1458 {
1459 switch (failed) {
1460 case -EFAULT:
1461 FTRACE_WARN_ON_ONCE(1);
1462 pr_info("ftrace faulted on modifying ");
1463 print_ip_sym(ip);
1464 break;
1465 case -EINVAL:
1466 FTRACE_WARN_ON_ONCE(1);
1467 pr_info("ftrace failed to modify ");
1468 print_ip_sym(ip);
1469 print_ip_ins(" actual: ", (unsigned char *)ip);
1470 printk(KERN_CONT "\n");
1471 break;
1472 case -EPERM:
1473 FTRACE_WARN_ON_ONCE(1);
1474 pr_info("ftrace faulted on writing ");
1475 print_ip_sym(ip);
1476 break;
1477 default:
1478 FTRACE_WARN_ON_ONCE(1);
1479 pr_info("ftrace faulted on unknown error ");
1480 print_ip_sym(ip);
1481 }
1482 }
1483
1484
1485 /* Return 1 if the address range is reserved for ftrace */
1486 int ftrace_text_reserved(void *start, void *end)
1487 {
1488 struct dyn_ftrace *rec;
1489 struct ftrace_page *pg;
1490
1491 do_for_each_ftrace_rec(pg, rec) {
1492 if (rec->ip <= (unsigned long)end &&
1493 rec->ip + MCOUNT_INSN_SIZE > (unsigned long)start)
1494 return 1;
1495 } while_for_each_ftrace_rec();
1496 return 0;
1497 }
1498
1499
1500 static int
1501 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
1502 {
1503 unsigned long ftrace_addr;
1504 unsigned long flag = 0UL;
1505
1506 ftrace_addr = (unsigned long)FTRACE_ADDR;
1507
1508 /*
1509 * If we are enabling tracing:
1510 *
1511 * If the record has a ref count, then we need to enable it
1512 * because someone is using it.
1513 *
1514 * Otherwise we make sure its disabled.
1515 *
1516 * If we are disabling tracing, then disable all records that
1517 * are enabled.
1518 */
1519 if (enable && (rec->flags & ~FTRACE_FL_MASK))
1520 flag = FTRACE_FL_ENABLED;
1521
1522 /* If the state of this record hasn't changed, then do nothing */
1523 if ((rec->flags & FTRACE_FL_ENABLED) == flag)
1524 return 0;
1525
1526 if (flag) {
1527 rec->flags |= FTRACE_FL_ENABLED;
1528 return ftrace_make_call(rec, ftrace_addr);
1529 }
1530
1531 rec->flags &= ~FTRACE_FL_ENABLED;
1532 return ftrace_make_nop(NULL, rec, ftrace_addr);
1533 }
1534
1535 static void ftrace_replace_code(int enable)
1536 {
1537 struct dyn_ftrace *rec;
1538 struct ftrace_page *pg;
1539 int failed;
1540
1541 if (unlikely(ftrace_disabled))
1542 return;
1543
1544 do_for_each_ftrace_rec(pg, rec) {
1545 /* Skip over free records */
1546 if (rec->flags & FTRACE_FL_FREE)
1547 continue;
1548
1549 failed = __ftrace_replace_code(rec, enable);
1550 if (failed) {
1551 ftrace_bug(failed, rec->ip);
1552 /* Stop processing */
1553 return;
1554 }
1555 } while_for_each_ftrace_rec();
1556 }
1557
1558 static int
1559 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
1560 {
1561 unsigned long ip;
1562 int ret;
1563
1564 ip = rec->ip;
1565
1566 if (unlikely(ftrace_disabled))
1567 return 0;
1568
1569 ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
1570 if (ret) {
1571 ftrace_bug(ret, ip);
1572 return 0;
1573 }
1574 return 1;
1575 }
1576
1577 /*
1578 * archs can override this function if they must do something
1579 * before the modifying code is performed.
1580 */
1581 int __weak ftrace_arch_code_modify_prepare(void)
1582 {
1583 return 0;
1584 }
1585
1586 /*
1587 * archs can override this function if they must do something
1588 * after the modifying code is performed.
1589 */
1590 int __weak ftrace_arch_code_modify_post_process(void)
1591 {
1592 return 0;
1593 }
1594
1595 static int __ftrace_modify_code(void *data)
1596 {
1597 int *command = data;
1598
1599 if (*command & FTRACE_ENABLE_CALLS)
1600 ftrace_replace_code(1);
1601 else if (*command & FTRACE_DISABLE_CALLS)
1602 ftrace_replace_code(0);
1603
1604 if (*command & FTRACE_UPDATE_TRACE_FUNC)
1605 ftrace_update_ftrace_func(ftrace_trace_function);
1606
1607 if (*command & FTRACE_START_FUNC_RET)
1608 ftrace_enable_ftrace_graph_caller();
1609 else if (*command & FTRACE_STOP_FUNC_RET)
1610 ftrace_disable_ftrace_graph_caller();
1611
1612 return 0;
1613 }
1614
1615 static void ftrace_run_update_code(int command)
1616 {
1617 int ret;
1618
1619 ret = ftrace_arch_code_modify_prepare();
1620 FTRACE_WARN_ON(ret);
1621 if (ret)
1622 return;
1623
1624 stop_machine(__ftrace_modify_code, &command, NULL);
1625
1626 ret = ftrace_arch_code_modify_post_process();
1627 FTRACE_WARN_ON(ret);
1628 }
1629
1630 static ftrace_func_t saved_ftrace_func;
1631 static int ftrace_start_up;
1632 static int global_start_up;
1633
1634 static void ftrace_startup_enable(int command)
1635 {
1636 if (saved_ftrace_func != ftrace_trace_function) {
1637 saved_ftrace_func = ftrace_trace_function;
1638 command |= FTRACE_UPDATE_TRACE_FUNC;
1639 }
1640
1641 if (!command || !ftrace_enabled)
1642 return;
1643
1644 ftrace_run_update_code(command);
1645 }
1646
1647 static int ftrace_startup(struct ftrace_ops *ops, int command)
1648 {
1649 bool hash_enable = true;
1650
1651 if (unlikely(ftrace_disabled))
1652 return -ENODEV;
1653
1654 ftrace_start_up++;
1655 command |= FTRACE_ENABLE_CALLS;
1656
1657 /* ops marked global share the filter hashes */
1658 if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
1659 ops = &global_ops;
1660 /* Don't update hash if global is already set */
1661 if (global_start_up)
1662 hash_enable = false;
1663 global_start_up++;
1664 }
1665
1666 ops->flags |= FTRACE_OPS_FL_ENABLED;
1667 if (hash_enable)
1668 ftrace_hash_rec_enable(ops, 1);
1669
1670 ftrace_startup_enable(command);
1671
1672 return 0;
1673 }
1674
1675 static void ftrace_shutdown(struct ftrace_ops *ops, int command)
1676 {
1677 bool hash_disable = true;
1678
1679 if (unlikely(ftrace_disabled))
1680 return;
1681
1682 ftrace_start_up--;
1683 /*
1684 * Just warn in case of unbalance, no need to kill ftrace, it's not
1685 * critical but the ftrace_call callers may be never nopped again after
1686 * further ftrace uses.
1687 */
1688 WARN_ON_ONCE(ftrace_start_up < 0);
1689
1690 if (ops->flags & FTRACE_OPS_FL_GLOBAL) {
1691 ops = &global_ops;
1692 global_start_up--;
1693 WARN_ON_ONCE(global_start_up < 0);
1694 /* Don't update hash if global still has users */
1695 if (global_start_up) {
1696 WARN_ON_ONCE(!ftrace_start_up);
1697 hash_disable = false;
1698 }
1699 }
1700
1701 if (hash_disable)
1702 ftrace_hash_rec_disable(ops, 1);
1703
1704 if (ops != &global_ops || !global_start_up)
1705 ops->flags &= ~FTRACE_OPS_FL_ENABLED;
1706
1707 if (!ftrace_start_up)
1708 command |= FTRACE_DISABLE_CALLS;
1709
1710 if (saved_ftrace_func != ftrace_trace_function) {
1711 saved_ftrace_func = ftrace_trace_function;
1712 command |= FTRACE_UPDATE_TRACE_FUNC;
1713 }
1714
1715 if (!command || !ftrace_enabled)
1716 return;
1717
1718 ftrace_run_update_code(command);
1719 }
1720
1721 static void ftrace_startup_sysctl(void)
1722 {
1723 if (unlikely(ftrace_disabled))
1724 return;
1725
1726 /* Force update next time */
1727 saved_ftrace_func = NULL;
1728 /* ftrace_start_up is true if we want ftrace running */
1729 if (ftrace_start_up)
1730 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
1731 }
1732
1733 static void ftrace_shutdown_sysctl(void)
1734 {
1735 if (unlikely(ftrace_disabled))
1736 return;
1737
1738 /* ftrace_start_up is true if ftrace is running */
1739 if (ftrace_start_up)
1740 ftrace_run_update_code(FTRACE_DISABLE_CALLS);
1741 }
1742
1743 static cycle_t ftrace_update_time;
1744 static unsigned long ftrace_update_cnt;
1745 unsigned long ftrace_update_tot_cnt;
1746
1747 static int ops_traces_mod(struct ftrace_ops *ops)
1748 {
1749 struct ftrace_hash *hash;
1750
1751 hash = ops->filter_hash;
1752 return !!(!hash || !hash->count);
1753 }
1754
1755 static int ftrace_update_code(struct module *mod)
1756 {
1757 struct dyn_ftrace *p;
1758 cycle_t start, stop;
1759 unsigned long ref = 0;
1760
1761 /*
1762 * When adding a module, we need to check if tracers are
1763 * currently enabled and if they are set to trace all functions.
1764 * If they are, we need to enable the module functions as well
1765 * as update the reference counts for those function records.
1766 */
1767 if (mod) {
1768 struct ftrace_ops *ops;
1769
1770 for (ops = ftrace_ops_list;
1771 ops != &ftrace_list_end; ops = ops->next) {
1772 if (ops->flags & FTRACE_OPS_FL_ENABLED &&
1773 ops_traces_mod(ops))
1774 ref++;
1775 }
1776 }
1777
1778 start = ftrace_now(raw_smp_processor_id());
1779 ftrace_update_cnt = 0;
1780
1781 while (ftrace_new_addrs) {
1782
1783 /* If something went wrong, bail without enabling anything */
1784 if (unlikely(ftrace_disabled))
1785 return -1;
1786
1787 p = ftrace_new_addrs;
1788 ftrace_new_addrs = p->newlist;
1789 p->flags = ref;
1790
1791 /*
1792 * Do the initial record conversion from mcount jump
1793 * to the NOP instructions.
1794 */
1795 if (!ftrace_code_disable(mod, p)) {
1796 ftrace_free_rec(p);
1797 /* Game over */
1798 break;
1799 }
1800
1801 ftrace_update_cnt++;
1802
1803 /*
1804 * If the tracing is enabled, go ahead and enable the record.
1805 *
1806 * The reason not to enable the record immediatelly is the
1807 * inherent check of ftrace_make_nop/ftrace_make_call for
1808 * correct previous instructions. Making first the NOP
1809 * conversion puts the module to the correct state, thus
1810 * passing the ftrace_make_call check.
1811 */
1812 if (ftrace_start_up && ref) {
1813 int failed = __ftrace_replace_code(p, 1);
1814 if (failed) {
1815 ftrace_bug(failed, p->ip);
1816 ftrace_free_rec(p);
1817 }
1818 }
1819 }
1820
1821 stop = ftrace_now(raw_smp_processor_id());
1822 ftrace_update_time = stop - start;
1823 ftrace_update_tot_cnt += ftrace_update_cnt;
1824
1825 return 0;
1826 }
1827
1828 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
1829 {
1830 struct ftrace_page *pg;
1831 int cnt;
1832 int i;
1833
1834 /* allocate a few pages */
1835 ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
1836 if (!ftrace_pages_start)
1837 return -1;
1838
1839 /*
1840 * Allocate a few more pages.
1841 *
1842 * TODO: have some parser search vmlinux before
1843 * final linking to find all calls to ftrace.
1844 * Then we can:
1845 * a) know how many pages to allocate.
1846 * and/or
1847 * b) set up the table then.
1848 *
1849 * The dynamic code is still necessary for
1850 * modules.
1851 */
1852
1853 pg = ftrace_pages = ftrace_pages_start;
1854
1855 cnt = num_to_init / ENTRIES_PER_PAGE;
1856 pr_info("ftrace: allocating %ld entries in %d pages\n",
1857 num_to_init, cnt + 1);
1858
1859 for (i = 0; i < cnt; i++) {
1860 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
1861
1862 /* If we fail, we'll try later anyway */
1863 if (!pg->next)
1864 break;
1865
1866 pg = pg->next;
1867 }
1868
1869 return 0;
1870 }
1871
1872 enum {
1873 FTRACE_ITER_FILTER = (1 << 0),
1874 FTRACE_ITER_NOTRACE = (1 << 1),
1875 FTRACE_ITER_PRINTALL = (1 << 2),
1876 FTRACE_ITER_HASH = (1 << 3),
1877 FTRACE_ITER_ENABLED = (1 << 4),
1878 };
1879
1880 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
1881
1882 struct ftrace_iterator {
1883 loff_t pos;
1884 loff_t func_pos;
1885 struct ftrace_page *pg;
1886 struct dyn_ftrace *func;
1887 struct ftrace_func_probe *probe;
1888 struct trace_parser parser;
1889 struct ftrace_hash *hash;
1890 struct ftrace_ops *ops;
1891 int hidx;
1892 int idx;
1893 unsigned flags;
1894 };
1895
1896 static void *
1897 t_hash_next(struct seq_file *m, loff_t *pos)
1898 {
1899 struct ftrace_iterator *iter = m->private;
1900 struct hlist_node *hnd = NULL;
1901 struct hlist_head *hhd;
1902
1903 (*pos)++;
1904 iter->pos = *pos;
1905
1906 if (iter->probe)
1907 hnd = &iter->probe->node;
1908 retry:
1909 if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
1910 return NULL;
1911
1912 hhd = &ftrace_func_hash[iter->hidx];
1913
1914 if (hlist_empty(hhd)) {
1915 iter->hidx++;
1916 hnd = NULL;
1917 goto retry;
1918 }
1919
1920 if (!hnd)
1921 hnd = hhd->first;
1922 else {
1923 hnd = hnd->next;
1924 if (!hnd) {
1925 iter->hidx++;
1926 goto retry;
1927 }
1928 }
1929
1930 if (WARN_ON_ONCE(!hnd))
1931 return NULL;
1932
1933 iter->probe = hlist_entry(hnd, struct ftrace_func_probe, node);
1934
1935 return iter;
1936 }
1937
1938 static void *t_hash_start(struct seq_file *m, loff_t *pos)
1939 {
1940 struct ftrace_iterator *iter = m->private;
1941 void *p = NULL;
1942 loff_t l;
1943
1944 if (iter->func_pos > *pos)
1945 return NULL;
1946
1947 iter->hidx = 0;
1948 for (l = 0; l <= (*pos - iter->func_pos); ) {
1949 p = t_hash_next(m, &l);
1950 if (!p)
1951 break;
1952 }
1953 if (!p)
1954 return NULL;
1955
1956 /* Only set this if we have an item */
1957 iter->flags |= FTRACE_ITER_HASH;
1958
1959 return iter;
1960 }
1961
1962 static int
1963 t_hash_show(struct seq_file *m, struct ftrace_iterator *iter)
1964 {
1965 struct ftrace_func_probe *rec;
1966
1967 rec = iter->probe;
1968 if (WARN_ON_ONCE(!rec))
1969 return -EIO;
1970
1971 if (rec->ops->print)
1972 return rec->ops->print(m, rec->ip, rec->ops, rec->data);
1973
1974 seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func);
1975
1976 if (rec->data)
1977 seq_printf(m, ":%p", rec->data);
1978 seq_putc(m, '\n');
1979
1980 return 0;
1981 }
1982
1983 static void *
1984 t_next(struct seq_file *m, void *v, loff_t *pos)
1985 {
1986 struct ftrace_iterator *iter = m->private;
1987 struct ftrace_ops *ops = &global_ops;
1988 struct dyn_ftrace *rec = NULL;
1989
1990 if (unlikely(ftrace_disabled))
1991 return NULL;
1992
1993 if (iter->flags & FTRACE_ITER_HASH)
1994 return t_hash_next(m, pos);
1995
1996 (*pos)++;
1997 iter->pos = iter->func_pos = *pos;
1998
1999 if (iter->flags & FTRACE_ITER_PRINTALL)
2000 return t_hash_start(m, pos);
2001
2002 retry:
2003 if (iter->idx >= iter->pg->index) {
2004 if (iter->pg->next) {
2005 iter->pg = iter->pg->next;
2006 iter->idx = 0;
2007 goto retry;
2008 }
2009 } else {
2010 rec = &iter->pg->records[iter->idx++];
2011 if ((rec->flags & FTRACE_FL_FREE) ||
2012
2013 ((iter->flags & FTRACE_ITER_FILTER) &&
2014 !(ftrace_lookup_ip(ops->filter_hash, rec->ip))) ||
2015
2016 ((iter->flags & FTRACE_ITER_NOTRACE) &&
2017 !ftrace_lookup_ip(ops->notrace_hash, rec->ip)) ||
2018
2019 ((iter->flags & FTRACE_ITER_ENABLED) &&
2020 !(rec->flags & ~FTRACE_FL_MASK))) {
2021
2022 rec = NULL;
2023 goto retry;
2024 }
2025 }
2026
2027 if (!rec)
2028 return t_hash_start(m, pos);
2029
2030 iter->func = rec;
2031
2032 return iter;
2033 }
2034
2035 static void reset_iter_read(struct ftrace_iterator *iter)
2036 {
2037 iter->pos = 0;
2038 iter->func_pos = 0;
2039 iter->flags &= ~(FTRACE_ITER_PRINTALL & FTRACE_ITER_HASH);
2040 }
2041
2042 static void *t_start(struct seq_file *m, loff_t *pos)
2043 {
2044 struct ftrace_iterator *iter = m->private;
2045 struct ftrace_ops *ops = &global_ops;
2046 void *p = NULL;
2047 loff_t l;
2048
2049 mutex_lock(&ftrace_lock);
2050
2051 if (unlikely(ftrace_disabled))
2052 return NULL;
2053
2054 /*
2055 * If an lseek was done, then reset and start from beginning.
2056 */
2057 if (*pos < iter->pos)
2058 reset_iter_read(iter);
2059
2060 /*
2061 * For set_ftrace_filter reading, if we have the filter
2062 * off, we can short cut and just print out that all
2063 * functions are enabled.
2064 */
2065 if (iter->flags & FTRACE_ITER_FILTER && !ops->filter_hash->count) {
2066 if (*pos > 0)
2067 return t_hash_start(m, pos);
2068 iter->flags |= FTRACE_ITER_PRINTALL;
2069 /* reset in case of seek/pread */
2070 iter->flags &= ~FTRACE_ITER_HASH;
2071 return iter;
2072 }
2073
2074 if (iter->flags & FTRACE_ITER_HASH)
2075 return t_hash_start(m, pos);
2076
2077 /*
2078 * Unfortunately, we need to restart at ftrace_pages_start
2079 * every time we let go of the ftrace_mutex. This is because
2080 * those pointers can change without the lock.
2081 */
2082 iter->pg = ftrace_pages_start;
2083 iter->idx = 0;
2084 for (l = 0; l <= *pos; ) {
2085 p = t_next(m, p, &l);
2086 if (!p)
2087 break;
2088 }
2089
2090 if (!p) {
2091 if (iter->flags & FTRACE_ITER_FILTER)
2092 return t_hash_start(m, pos);
2093
2094 return NULL;
2095 }
2096
2097 return iter;
2098 }
2099
2100 static void t_stop(struct seq_file *m, void *p)
2101 {
2102 mutex_unlock(&ftrace_lock);
2103 }
2104
2105 static int t_show(struct seq_file *m, void *v)
2106 {
2107 struct ftrace_iterator *iter = m->private;
2108 struct dyn_ftrace *rec;
2109
2110 if (iter->flags & FTRACE_ITER_HASH)
2111 return t_hash_show(m, iter);
2112
2113 if (iter->flags & FTRACE_ITER_PRINTALL) {
2114 seq_printf(m, "#### all functions enabled ####\n");
2115 return 0;
2116 }
2117
2118 rec = iter->func;
2119
2120 if (!rec)
2121 return 0;
2122
2123 seq_printf(m, "%ps", (void *)rec->ip);
2124 if (iter->flags & FTRACE_ITER_ENABLED)
2125 seq_printf(m, " (%ld)",
2126 rec->flags & ~FTRACE_FL_MASK);
2127 seq_printf(m, "\n");
2128
2129 return 0;
2130 }
2131
2132 static const struct seq_operations show_ftrace_seq_ops = {
2133 .start = t_start,
2134 .next = t_next,
2135 .stop = t_stop,
2136 .show = t_show,
2137 };
2138
2139 static int
2140 ftrace_avail_open(struct inode *inode, struct file *file)
2141 {
2142 struct ftrace_iterator *iter;
2143 int ret;
2144
2145 if (unlikely(ftrace_disabled))
2146 return -ENODEV;
2147
2148 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2149 if (!iter)
2150 return -ENOMEM;
2151
2152 iter->pg = ftrace_pages_start;
2153
2154 ret = seq_open(file, &show_ftrace_seq_ops);
2155 if (!ret) {
2156 struct seq_file *m = file->private_data;
2157
2158 m->private = iter;
2159 } else {
2160 kfree(iter);
2161 }
2162
2163 return ret;
2164 }
2165
2166 static int
2167 ftrace_enabled_open(struct inode *inode, struct file *file)
2168 {
2169 struct ftrace_iterator *iter;
2170 int ret;
2171
2172 if (unlikely(ftrace_disabled))
2173 return -ENODEV;
2174
2175 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2176 if (!iter)
2177 return -ENOMEM;
2178
2179 iter->pg = ftrace_pages_start;
2180 iter->flags = FTRACE_ITER_ENABLED;
2181
2182 ret = seq_open(file, &show_ftrace_seq_ops);
2183 if (!ret) {
2184 struct seq_file *m = file->private_data;
2185
2186 m->private = iter;
2187 } else {
2188 kfree(iter);
2189 }
2190
2191 return ret;
2192 }
2193
2194 static void ftrace_filter_reset(struct ftrace_hash *hash)
2195 {
2196 mutex_lock(&ftrace_lock);
2197 ftrace_hash_clear(hash);
2198 mutex_unlock(&ftrace_lock);
2199 }
2200
2201 static int
2202 ftrace_regex_open(struct ftrace_ops *ops, int flag,
2203 struct inode *inode, struct file *file)
2204 {
2205 struct ftrace_iterator *iter;
2206 struct ftrace_hash *hash;
2207 int ret = 0;
2208
2209 if (unlikely(ftrace_disabled))
2210 return -ENODEV;
2211
2212 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2213 if (!iter)
2214 return -ENOMEM;
2215
2216 if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
2217 kfree(iter);
2218 return -ENOMEM;
2219 }
2220
2221 if (flag & FTRACE_ITER_NOTRACE)
2222 hash = ops->notrace_hash;
2223 else
2224 hash = ops->filter_hash;
2225
2226 iter->ops = ops;
2227 iter->flags = flag;
2228
2229 if (file->f_mode & FMODE_WRITE) {
2230 mutex_lock(&ftrace_lock);
2231 iter->hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, hash);
2232 mutex_unlock(&ftrace_lock);
2233
2234 if (!iter->hash) {
2235 trace_parser_put(&iter->parser);
2236 kfree(iter);
2237 return -ENOMEM;
2238 }
2239 }
2240
2241 mutex_lock(&ftrace_regex_lock);
2242
2243 if ((file->f_mode & FMODE_WRITE) &&
2244 (file->f_flags & O_TRUNC))
2245 ftrace_filter_reset(iter->hash);
2246
2247 if (file->f_mode & FMODE_READ) {
2248 iter->pg = ftrace_pages_start;
2249
2250 ret = seq_open(file, &show_ftrace_seq_ops);
2251 if (!ret) {
2252 struct seq_file *m = file->private_data;
2253 m->private = iter;
2254 } else {
2255 /* Failed */
2256 free_ftrace_hash(iter->hash);
2257 trace_parser_put(&iter->parser);
2258 kfree(iter);
2259 }
2260 } else
2261 file->private_data = iter;
2262 mutex_unlock(&ftrace_regex_lock);
2263
2264 return ret;
2265 }
2266
2267 static int
2268 ftrace_filter_open(struct inode *inode, struct file *file)
2269 {
2270 return ftrace_regex_open(&global_ops, FTRACE_ITER_FILTER,
2271 inode, file);
2272 }
2273
2274 static int
2275 ftrace_notrace_open(struct inode *inode, struct file *file)
2276 {
2277 return ftrace_regex_open(&global_ops, FTRACE_ITER_NOTRACE,
2278 inode, file);
2279 }
2280
2281 static loff_t
2282 ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
2283 {
2284 loff_t ret;
2285
2286 if (file->f_mode & FMODE_READ)
2287 ret = seq_lseek(file, offset, origin);
2288 else
2289 file->f_pos = ret = 1;
2290
2291 return ret;
2292 }
2293
2294 static int ftrace_match(char *str, char *regex, int len, int type)
2295 {
2296 int matched = 0;
2297 int slen;
2298
2299 switch (type) {
2300 case MATCH_FULL:
2301 if (strcmp(str, regex) == 0)
2302 matched = 1;
2303 break;
2304 case MATCH_FRONT_ONLY:
2305 if (strncmp(str, regex, len) == 0)
2306 matched = 1;
2307 break;
2308 case MATCH_MIDDLE_ONLY:
2309 if (strstr(str, regex))
2310 matched = 1;
2311 break;
2312 case MATCH_END_ONLY:
2313 slen = strlen(str);
2314 if (slen >= len && memcmp(str + slen - len, regex, len) == 0)
2315 matched = 1;
2316 break;
2317 }
2318
2319 return matched;
2320 }
2321
2322 static int
2323 enter_record(struct ftrace_hash *hash, struct dyn_ftrace *rec, int not)
2324 {
2325 struct ftrace_func_entry *entry;
2326 int ret = 0;
2327
2328 entry = ftrace_lookup_ip(hash, rec->ip);
2329 if (not) {
2330 /* Do nothing if it doesn't exist */
2331 if (!entry)
2332 return 0;
2333
2334 free_hash_entry(hash, entry);
2335 } else {
2336 /* Do nothing if it exists */
2337 if (entry)
2338 return 0;
2339
2340 ret = add_hash_entry(hash, rec->ip);
2341 }
2342 return ret;
2343 }
2344
2345 static int
2346 ftrace_match_record(struct dyn_ftrace *rec, char *mod,
2347 char *regex, int len, int type)
2348 {
2349 char str[KSYM_SYMBOL_LEN];
2350 char *modname;
2351
2352 kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
2353
2354 if (mod) {
2355 /* module lookup requires matching the module */
2356 if (!modname || strcmp(modname, mod))
2357 return 0;
2358
2359 /* blank search means to match all funcs in the mod */
2360 if (!len)
2361 return 1;
2362 }
2363
2364 return ftrace_match(str, regex, len, type);
2365 }
2366
2367 static int
2368 match_records(struct ftrace_hash *hash, char *buff,
2369 int len, char *mod, int not)
2370 {
2371 unsigned search_len = 0;
2372 struct ftrace_page *pg;
2373 struct dyn_ftrace *rec;
2374 int type = MATCH_FULL;
2375 char *search = buff;
2376 int found = 0;
2377 int ret;
2378
2379 if (len) {
2380 type = filter_parse_regex(buff, len, &search, &not);
2381 search_len = strlen(search);
2382 }
2383
2384 mutex_lock(&ftrace_lock);
2385
2386 if (unlikely(ftrace_disabled))
2387 goto out_unlock;
2388
2389 do_for_each_ftrace_rec(pg, rec) {
2390
2391 if (ftrace_match_record(rec, mod, search, search_len, type)) {
2392 ret = enter_record(hash, rec, not);
2393 if (ret < 0) {
2394 found = ret;
2395 goto out_unlock;
2396 }
2397 found = 1;
2398 }
2399 } while_for_each_ftrace_rec();
2400 out_unlock:
2401 mutex_unlock(&ftrace_lock);
2402
2403 return found;
2404 }
2405
2406 static int
2407 ftrace_match_records(struct ftrace_hash *hash, char *buff, int len)
2408 {
2409 return match_records(hash, buff, len, NULL, 0);
2410 }
2411
2412 static int
2413 ftrace_match_module_records(struct ftrace_hash *hash, char *buff, char *mod)
2414 {
2415 int not = 0;
2416
2417 /* blank or '*' mean the same */
2418 if (strcmp(buff, "*") == 0)
2419 buff[0] = 0;
2420
2421 /* handle the case of 'dont filter this module' */
2422 if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
2423 buff[0] = 0;
2424 not = 1;
2425 }
2426
2427 return match_records(hash, buff, strlen(buff), mod, not);
2428 }
2429
2430 /*
2431 * We register the module command as a template to show others how
2432 * to register the a command as well.
2433 */
2434
2435 static int
2436 ftrace_mod_callback(struct ftrace_hash *hash,
2437 char *func, char *cmd, char *param, int enable)
2438 {
2439 char *mod;
2440 int ret = -EINVAL;
2441
2442 /*
2443 * cmd == 'mod' because we only registered this func
2444 * for the 'mod' ftrace_func_command.
2445 * But if you register one func with multiple commands,
2446 * you can tell which command was used by the cmd
2447 * parameter.
2448 */
2449
2450 /* we must have a module name */
2451 if (!param)
2452 return ret;
2453
2454 mod = strsep(&param, ":");
2455 if (!strlen(mod))
2456 return ret;
2457
2458 ret = ftrace_match_module_records(hash, func, mod);
2459 if (!ret)
2460 ret = -EINVAL;
2461 if (ret < 0)
2462 return ret;
2463
2464 return 0;
2465 }
2466
2467 static struct ftrace_func_command ftrace_mod_cmd = {
2468 .name = "mod",
2469 .func = ftrace_mod_callback,
2470 };
2471
2472 static int __init ftrace_mod_cmd_init(void)
2473 {
2474 return register_ftrace_command(&ftrace_mod_cmd);
2475 }
2476 device_initcall(ftrace_mod_cmd_init);
2477
2478 static void
2479 function_trace_probe_call(unsigned long ip, unsigned long parent_ip)
2480 {
2481 struct ftrace_func_probe *entry;
2482 struct hlist_head *hhd;
2483 struct hlist_node *n;
2484 unsigned long key;
2485
2486 key = hash_long(ip, FTRACE_HASH_BITS);
2487
2488 hhd = &ftrace_func_hash[key];
2489
2490 if (hlist_empty(hhd))
2491 return;
2492
2493 /*
2494 * Disable preemption for these calls to prevent a RCU grace
2495 * period. This syncs the hash iteration and freeing of items
2496 * on the hash. rcu_read_lock is too dangerous here.
2497 */
2498 preempt_disable_notrace();
2499 hlist_for_each_entry_rcu(entry, n, hhd, node) {
2500 if (entry->ip == ip)
2501 entry->ops->func(ip, parent_ip, &entry->data);
2502 }
2503 preempt_enable_notrace();
2504 }
2505
2506 static struct ftrace_ops trace_probe_ops __read_mostly =
2507 {
2508 .func = function_trace_probe_call,
2509 };
2510
2511 static int ftrace_probe_registered;
2512
2513 static void __enable_ftrace_function_probe(void)
2514 {
2515 int ret;
2516 int i;
2517
2518 if (ftrace_probe_registered)
2519 return;
2520
2521 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2522 struct hlist_head *hhd = &ftrace_func_hash[i];
2523 if (hhd->first)
2524 break;
2525 }
2526 /* Nothing registered? */
2527 if (i == FTRACE_FUNC_HASHSIZE)
2528 return;
2529
2530 ret = __register_ftrace_function(&trace_probe_ops);
2531 if (!ret)
2532 ret = ftrace_startup(&trace_probe_ops, 0);
2533
2534 ftrace_probe_registered = 1;
2535 }
2536
2537 static void __disable_ftrace_function_probe(void)
2538 {
2539 int ret;
2540 int i;
2541
2542 if (!ftrace_probe_registered)
2543 return;
2544
2545 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2546 struct hlist_head *hhd = &ftrace_func_hash[i];
2547 if (hhd->first)
2548 return;
2549 }
2550
2551 /* no more funcs left */
2552 ret = __unregister_ftrace_function(&trace_probe_ops);
2553 if (!ret)
2554 ftrace_shutdown(&trace_probe_ops, 0);
2555
2556 ftrace_probe_registered = 0;
2557 }
2558
2559
2560 static void ftrace_free_entry_rcu(struct rcu_head *rhp)
2561 {
2562 struct ftrace_func_probe *entry =
2563 container_of(rhp, struct ftrace_func_probe, rcu);
2564
2565 if (entry->ops->free)
2566 entry->ops->free(&entry->data);
2567 kfree(entry);
2568 }
2569
2570
2571 int
2572 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2573 void *data)
2574 {
2575 struct ftrace_func_probe *entry;
2576 struct ftrace_page *pg;
2577 struct dyn_ftrace *rec;
2578 int type, len, not;
2579 unsigned long key;
2580 int count = 0;
2581 char *search;
2582
2583 type = filter_parse_regex(glob, strlen(glob), &search, &not);
2584 len = strlen(search);
2585
2586 /* we do not support '!' for function probes */
2587 if (WARN_ON(not))
2588 return -EINVAL;
2589
2590 mutex_lock(&ftrace_lock);
2591
2592 if (unlikely(ftrace_disabled))
2593 goto out_unlock;
2594
2595 do_for_each_ftrace_rec(pg, rec) {
2596
2597 if (!ftrace_match_record(rec, NULL, search, len, type))
2598 continue;
2599
2600 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
2601 if (!entry) {
2602 /* If we did not process any, then return error */
2603 if (!count)
2604 count = -ENOMEM;
2605 goto out_unlock;
2606 }
2607
2608 count++;
2609
2610 entry->data = data;
2611
2612 /*
2613 * The caller might want to do something special
2614 * for each function we find. We call the callback
2615 * to give the caller an opportunity to do so.
2616 */
2617 if (ops->callback) {
2618 if (ops->callback(rec->ip, &entry->data) < 0) {
2619 /* caller does not like this func */
2620 kfree(entry);
2621 continue;
2622 }
2623 }
2624
2625 entry->ops = ops;
2626 entry->ip = rec->ip;
2627
2628 key = hash_long(entry->ip, FTRACE_HASH_BITS);
2629 hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
2630
2631 } while_for_each_ftrace_rec();
2632 __enable_ftrace_function_probe();
2633
2634 out_unlock:
2635 mutex_unlock(&ftrace_lock);
2636
2637 return count;
2638 }
2639
2640 enum {
2641 PROBE_TEST_FUNC = 1,
2642 PROBE_TEST_DATA = 2
2643 };
2644
2645 static void
2646 __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2647 void *data, int flags)
2648 {
2649 struct ftrace_func_probe *entry;
2650 struct hlist_node *n, *tmp;
2651 char str[KSYM_SYMBOL_LEN];
2652 int type = MATCH_FULL;
2653 int i, len = 0;
2654 char *search;
2655
2656 if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
2657 glob = NULL;
2658 else if (glob) {
2659 int not;
2660
2661 type = filter_parse_regex(glob, strlen(glob), &search, &not);
2662 len = strlen(search);
2663
2664 /* we do not support '!' for function probes */
2665 if (WARN_ON(not))
2666 return;
2667 }
2668
2669 mutex_lock(&ftrace_lock);
2670 for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2671 struct hlist_head *hhd = &ftrace_func_hash[i];
2672
2673 hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {
2674
2675 /* break up if statements for readability */
2676 if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
2677 continue;
2678
2679 if ((flags & PROBE_TEST_DATA) && entry->data != data)
2680 continue;
2681
2682 /* do this last, since it is the most expensive */
2683 if (glob) {
2684 kallsyms_lookup(entry->ip, NULL, NULL,
2685 NULL, str);
2686 if (!ftrace_match(str, glob, len, type))
2687 continue;
2688 }
2689
2690 hlist_del(&entry->node);
2691 call_rcu(&entry->rcu, ftrace_free_entry_rcu);
2692 }
2693 }
2694 __disable_ftrace_function_probe();
2695 mutex_unlock(&ftrace_lock);
2696 }
2697
2698 void
2699 unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2700 void *data)
2701 {
2702 __unregister_ftrace_function_probe(glob, ops, data,
2703 PROBE_TEST_FUNC | PROBE_TEST_DATA);
2704 }
2705
2706 void
2707 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
2708 {
2709 __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
2710 }
2711
2712 void unregister_ftrace_function_probe_all(char *glob)
2713 {
2714 __unregister_ftrace_function_probe(glob, NULL, NULL, 0);
2715 }
2716
2717 static LIST_HEAD(ftrace_commands);
2718 static DEFINE_MUTEX(ftrace_cmd_mutex);
2719
2720 int register_ftrace_command(struct ftrace_func_command *cmd)
2721 {
2722 struct ftrace_func_command *p;
2723 int ret = 0;
2724
2725 mutex_lock(&ftrace_cmd_mutex);
2726 list_for_each_entry(p, &ftrace_commands, list) {
2727 if (strcmp(cmd->name, p->name) == 0) {
2728 ret = -EBUSY;
2729 goto out_unlock;
2730 }
2731 }
2732 list_add(&cmd->list, &ftrace_commands);
2733 out_unlock:
2734 mutex_unlock(&ftrace_cmd_mutex);
2735
2736 return ret;
2737 }
2738
2739 int unregister_ftrace_command(struct ftrace_func_command *cmd)
2740 {
2741 struct ftrace_func_command *p, *n;
2742 int ret = -ENODEV;
2743
2744 mutex_lock(&ftrace_cmd_mutex);
2745 list_for_each_entry_safe(p, n, &ftrace_commands, list) {
2746 if (strcmp(cmd->name, p->name) == 0) {
2747 ret = 0;
2748 list_del_init(&p->list);
2749 goto out_unlock;
2750 }
2751 }
2752 out_unlock:
2753 mutex_unlock(&ftrace_cmd_mutex);
2754
2755 return ret;
2756 }
2757
2758 static int ftrace_process_regex(struct ftrace_hash *hash,
2759 char *buff, int len, int enable)
2760 {
2761 char *func, *command, *next = buff;
2762 struct ftrace_func_command *p;
2763 int ret = -EINVAL;
2764
2765 func = strsep(&next, ":");
2766
2767 if (!next) {
2768 ret = ftrace_match_records(hash, func, len);
2769 if (!ret)
2770 ret = -EINVAL;
2771 if (ret < 0)
2772 return ret;
2773 return 0;
2774 }
2775
2776 /* command found */
2777
2778 command = strsep(&next, ":");
2779
2780 mutex_lock(&ftrace_cmd_mutex);
2781 list_for_each_entry(p, &ftrace_commands, list) {
2782 if (strcmp(p->name, command) == 0) {
2783 ret = p->func(hash, func, command, next, enable);
2784 goto out_unlock;
2785 }
2786 }
2787 out_unlock:
2788 mutex_unlock(&ftrace_cmd_mutex);
2789
2790 return ret;
2791 }
2792
2793 static ssize_t
2794 ftrace_regex_write(struct file *file, const char __user *ubuf,
2795 size_t cnt, loff_t *ppos, int enable)
2796 {
2797 struct ftrace_iterator *iter;
2798 struct trace_parser *parser;
2799 ssize_t ret, read;
2800
2801 if (!cnt)
2802 return 0;
2803
2804 mutex_lock(&ftrace_regex_lock);
2805
2806 ret = -ENODEV;
2807 if (unlikely(ftrace_disabled))
2808 goto out_unlock;
2809
2810 if (file->f_mode & FMODE_READ) {
2811 struct seq_file *m = file->private_data;
2812 iter = m->private;
2813 } else
2814 iter = file->private_data;
2815
2816 parser = &iter->parser;
2817 read = trace_get_user(parser, ubuf, cnt, ppos);
2818
2819 if (read >= 0 && trace_parser_loaded(parser) &&
2820 !trace_parser_cont(parser)) {
2821 ret = ftrace_process_regex(iter->hash, parser->buffer,
2822 parser->idx, enable);
2823 trace_parser_clear(parser);
2824 if (ret)
2825 goto out_unlock;
2826 }
2827
2828 ret = read;
2829 out_unlock:
2830 mutex_unlock(&ftrace_regex_lock);
2831
2832 return ret;
2833 }
2834
2835 static ssize_t
2836 ftrace_filter_write(struct file *file, const char __user *ubuf,
2837 size_t cnt, loff_t *ppos)
2838 {
2839 return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
2840 }
2841
2842 static ssize_t
2843 ftrace_notrace_write(struct file *file, const char __user *ubuf,
2844 size_t cnt, loff_t *ppos)
2845 {
2846 return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
2847 }
2848
2849 static int
2850 ftrace_set_regex(struct ftrace_ops *ops, unsigned char *buf, int len,
2851 int reset, int enable)
2852 {
2853 struct ftrace_hash **orig_hash;
2854 struct ftrace_hash *hash;
2855 int ret;
2856
2857 /* All global ops uses the global ops filters */
2858 if (ops->flags & FTRACE_OPS_FL_GLOBAL)
2859 ops = &global_ops;
2860
2861 if (unlikely(ftrace_disabled))
2862 return -ENODEV;
2863
2864 if (enable)
2865 orig_hash = &ops->filter_hash;
2866 else
2867 orig_hash = &ops->notrace_hash;
2868
2869 hash = alloc_and_copy_ftrace_hash(FTRACE_HASH_DEFAULT_BITS, *orig_hash);
2870 if (!hash)
2871 return -ENOMEM;
2872
2873 mutex_lock(&ftrace_regex_lock);
2874 if (reset)
2875 ftrace_filter_reset(hash);
2876 if (buf)
2877 ftrace_match_records(hash, buf, len);
2878
2879 mutex_lock(&ftrace_lock);
2880 ret = ftrace_hash_move(orig_hash, hash);
2881 mutex_unlock(&ftrace_lock);
2882
2883 mutex_unlock(&ftrace_regex_lock);
2884
2885 free_ftrace_hash(hash);
2886 return ret;
2887 }
2888
2889 /**
2890 * ftrace_set_filter - set a function to filter on in ftrace
2891 * @ops - the ops to set the filter with
2892 * @buf - the string that holds the function filter text.
2893 * @len - the length of the string.
2894 * @reset - non zero to reset all filters before applying this filter.
2895 *
2896 * Filters denote which functions should be enabled when tracing is enabled.
2897 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
2898 */
2899 void ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
2900 int len, int reset)
2901 {
2902 ftrace_set_regex(ops, buf, len, reset, 1);
2903 }
2904 EXPORT_SYMBOL_GPL(ftrace_set_filter);
2905
2906 /**
2907 * ftrace_set_notrace - set a function to not trace in ftrace
2908 * @ops - the ops to set the notrace filter with
2909 * @buf - the string that holds the function notrace text.
2910 * @len - the length of the string.
2911 * @reset - non zero to reset all filters before applying this filter.
2912 *
2913 * Notrace Filters denote which functions should not be enabled when tracing
2914 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
2915 * for tracing.
2916 */
2917 void ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
2918 int len, int reset)
2919 {
2920 ftrace_set_regex(ops, buf, len, reset, 0);
2921 }
2922 EXPORT_SYMBOL_GPL(ftrace_set_notrace);
2923 /**
2924 * ftrace_set_filter - set a function to filter on in ftrace
2925 * @ops - the ops to set the filter with
2926 * @buf - the string that holds the function filter text.
2927 * @len - the length of the string.
2928 * @reset - non zero to reset all filters before applying this filter.
2929 *
2930 * Filters denote which functions should be enabled when tracing is enabled.
2931 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
2932 */
2933 void ftrace_set_global_filter(unsigned char *buf, int len, int reset)
2934 {
2935 ftrace_set_regex(&global_ops, buf, len, reset, 1);
2936 }
2937 EXPORT_SYMBOL_GPL(ftrace_set_global_filter);
2938
2939 /**
2940 * ftrace_set_notrace - set a function to not trace in ftrace
2941 * @ops - the ops to set the notrace filter with
2942 * @buf - the string that holds the function notrace text.
2943 * @len - the length of the string.
2944 * @reset - non zero to reset all filters before applying this filter.
2945 *
2946 * Notrace Filters denote which functions should not be enabled when tracing
2947 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
2948 * for tracing.
2949 */
2950 void ftrace_set_global_notrace(unsigned char *buf, int len, int reset)
2951 {
2952 ftrace_set_regex(&global_ops, buf, len, reset, 0);
2953 }
2954 EXPORT_SYMBOL_GPL(ftrace_set_global_notrace);
2955
2956 /*
2957 * command line interface to allow users to set filters on boot up.
2958 */
2959 #define FTRACE_FILTER_SIZE COMMAND_LINE_SIZE
2960 static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
2961 static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
2962
2963 static int __init set_ftrace_notrace(char *str)
2964 {
2965 strncpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
2966 return 1;
2967 }
2968 __setup("ftrace_notrace=", set_ftrace_notrace);
2969
2970 static int __init set_ftrace_filter(char *str)
2971 {
2972 strncpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
2973 return 1;
2974 }
2975 __setup("ftrace_filter=", set_ftrace_filter);
2976
2977 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2978 static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
2979 static int ftrace_set_func(unsigned long *array, int *idx, char *buffer);
2980
2981 static int __init set_graph_function(char *str)
2982 {
2983 strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
2984 return 1;
2985 }
2986 __setup("ftrace_graph_filter=", set_graph_function);
2987
2988 static void __init set_ftrace_early_graph(char *buf)
2989 {
2990 int ret;
2991 char *func;
2992
2993 while (buf) {
2994 func = strsep(&buf, ",");
2995 /* we allow only one expression at a time */
2996 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
2997 func);
2998 if (ret)
2999 printk(KERN_DEBUG "ftrace: function %s not "
3000 "traceable\n", func);
3001 }
3002 }
3003 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3004
3005 static void __init
3006 set_ftrace_early_filter(struct ftrace_ops *ops, char *buf, int enable)
3007 {
3008 char *func;
3009
3010 while (buf) {
3011 func = strsep(&buf, ",");
3012 ftrace_set_regex(ops, func, strlen(func), 0, enable);
3013 }
3014 }
3015
3016 static void __init set_ftrace_early_filters(void)
3017 {
3018 if (ftrace_filter_buf[0])
3019 set_ftrace_early_filter(&global_ops, ftrace_filter_buf, 1);
3020 if (ftrace_notrace_buf[0])
3021 set_ftrace_early_filter(&global_ops, ftrace_notrace_buf, 0);
3022 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3023 if (ftrace_graph_buf[0])
3024 set_ftrace_early_graph(ftrace_graph_buf);
3025 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3026 }
3027
3028 static int
3029 ftrace_regex_release(struct inode *inode, struct file *file)
3030 {
3031 struct seq_file *m = (struct seq_file *)file->private_data;
3032 struct ftrace_iterator *iter;
3033 struct ftrace_hash **orig_hash;
3034 struct trace_parser *parser;
3035 int filter_hash;
3036 int ret;
3037
3038 mutex_lock(&ftrace_regex_lock);
3039 if (file->f_mode & FMODE_READ) {
3040 iter = m->private;
3041
3042 seq_release(inode, file);
3043 } else
3044 iter = file->private_data;
3045
3046 parser = &iter->parser;
3047 if (trace_parser_loaded(parser)) {
3048 parser->buffer[parser->idx] = 0;
3049 ftrace_match_records(iter->hash, parser->buffer, parser->idx);
3050 }
3051
3052 trace_parser_put(parser);
3053
3054 if (file->f_mode & FMODE_WRITE) {
3055 filter_hash = !!(iter->flags & FTRACE_ITER_FILTER);
3056
3057 if (filter_hash)
3058 orig_hash = &iter->ops->filter_hash;
3059 else
3060 orig_hash = &iter->ops->notrace_hash;
3061
3062 mutex_lock(&ftrace_lock);
3063 /*
3064 * Remove the current set, update the hash and add
3065 * them back.
3066 */
3067 ftrace_hash_rec_disable(iter->ops, filter_hash);
3068 ret = ftrace_hash_move(orig_hash, iter->hash);
3069 if (!ret) {
3070 ftrace_hash_rec_enable(iter->ops, filter_hash);
3071 if (iter->ops->flags & FTRACE_OPS_FL_ENABLED
3072 && ftrace_enabled)
3073 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
3074 }
3075 mutex_unlock(&ftrace_lock);
3076 }
3077 free_ftrace_hash(iter->hash);
3078 kfree(iter);
3079
3080 mutex_unlock(&ftrace_regex_lock);
3081 return 0;
3082 }
3083
3084 static const struct file_operations ftrace_avail_fops = {
3085 .open = ftrace_avail_open,
3086 .read = seq_read,
3087 .llseek = seq_lseek,
3088 .release = seq_release_private,
3089 };
3090
3091 static const struct file_operations ftrace_enabled_fops = {
3092 .open = ftrace_enabled_open,
3093 .read = seq_read,
3094 .llseek = seq_lseek,
3095 .release = seq_release_private,
3096 };
3097
3098 static const struct file_operations ftrace_filter_fops = {
3099 .open = ftrace_filter_open,
3100 .read = seq_read,
3101 .write = ftrace_filter_write,
3102 .llseek = ftrace_regex_lseek,
3103 .release = ftrace_regex_release,
3104 };
3105
3106 static const struct file_operations ftrace_notrace_fops = {
3107 .open = ftrace_notrace_open,
3108 .read = seq_read,
3109 .write = ftrace_notrace_write,
3110 .llseek = ftrace_regex_lseek,
3111 .release = ftrace_regex_release,
3112 };
3113
3114 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3115
3116 static DEFINE_MUTEX(graph_lock);
3117
3118 int ftrace_graph_count;
3119 int ftrace_graph_filter_enabled;
3120 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
3121
3122 static void *
3123 __g_next(struct seq_file *m, loff_t *pos)
3124 {
3125 if (*pos >= ftrace_graph_count)
3126 return NULL;
3127 return &ftrace_graph_funcs[*pos];
3128 }
3129
3130 static void *
3131 g_next(struct seq_file *m, void *v, loff_t *pos)
3132 {
3133 (*pos)++;
3134 return __g_next(m, pos);
3135 }
3136
3137 static void *g_start(struct seq_file *m, loff_t *pos)
3138 {
3139 mutex_lock(&graph_lock);
3140
3141 /* Nothing, tell g_show to print all functions are enabled */
3142 if (!ftrace_graph_filter_enabled && !*pos)
3143 return (void *)1;
3144
3145 return __g_next(m, pos);
3146 }
3147
3148 static void g_stop(struct seq_file *m, void *p)
3149 {
3150 mutex_unlock(&graph_lock);
3151 }
3152
3153 static int g_show(struct seq_file *m, void *v)
3154 {
3155 unsigned long *ptr = v;
3156
3157 if (!ptr)
3158 return 0;
3159
3160 if (ptr == (unsigned long *)1) {
3161 seq_printf(m, "#### all functions enabled ####\n");
3162 return 0;
3163 }
3164
3165 seq_printf(m, "%ps\n", (void *)*ptr);
3166
3167 return 0;
3168 }
3169
3170 static const struct seq_operations ftrace_graph_seq_ops = {
3171 .start = g_start,
3172 .next = g_next,
3173 .stop = g_stop,
3174 .show = g_show,
3175 };
3176
3177 static int
3178 ftrace_graph_open(struct inode *inode, struct file *file)
3179 {
3180 int ret = 0;
3181
3182 if (unlikely(ftrace_disabled))
3183 return -ENODEV;
3184
3185 mutex_lock(&graph_lock);
3186 if ((file->f_mode & FMODE_WRITE) &&
3187 (file->f_flags & O_TRUNC)) {
3188 ftrace_graph_filter_enabled = 0;
3189 ftrace_graph_count = 0;
3190 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
3191 }
3192 mutex_unlock(&graph_lock);
3193
3194 if (file->f_mode & FMODE_READ)
3195 ret = seq_open(file, &ftrace_graph_seq_ops);
3196
3197 return ret;
3198 }
3199
3200 static int
3201 ftrace_graph_release(struct inode *inode, struct file *file)
3202 {
3203 if (file->f_mode & FMODE_READ)
3204 seq_release(inode, file);
3205 return 0;
3206 }
3207
3208 static int
3209 ftrace_set_func(unsigned long *array, int *idx, char *buffer)
3210 {
3211 struct dyn_ftrace *rec;
3212 struct ftrace_page *pg;
3213 int search_len;
3214 int fail = 1;
3215 int type, not;
3216 char *search;
3217 bool exists;
3218 int i;
3219
3220 /* decode regex */
3221 type = filter_parse_regex(buffer, strlen(buffer), &search, &not);
3222 if (!not && *idx >= FTRACE_GRAPH_MAX_FUNCS)
3223 return -EBUSY;
3224
3225 search_len = strlen(search);
3226
3227 mutex_lock(&ftrace_lock);
3228
3229 if (unlikely(ftrace_disabled)) {
3230 mutex_unlock(&ftrace_lock);
3231 return -ENODEV;
3232 }
3233
3234 do_for_each_ftrace_rec(pg, rec) {
3235
3236 if (rec->flags & FTRACE_FL_FREE)
3237 continue;
3238
3239 if (ftrace_match_record(rec, NULL, search, search_len, type)) {
3240 /* if it is in the array */
3241 exists = false;
3242 for (i = 0; i < *idx; i++) {
3243 if (array[i] == rec->ip) {
3244 exists = true;
3245 break;
3246 }
3247 }
3248
3249 if (!not) {
3250 fail = 0;
3251 if (!exists) {
3252 array[(*idx)++] = rec->ip;
3253 if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
3254 goto out;
3255 }
3256 } else {
3257 if (exists) {
3258 array[i] = array[--(*idx)];
3259 array[*idx] = 0;
3260 fail = 0;
3261 }
3262 }
3263 }
3264 } while_for_each_ftrace_rec();
3265 out:
3266 mutex_unlock(&ftrace_lock);
3267
3268 if (fail)
3269 return -EINVAL;
3270
3271 ftrace_graph_filter_enabled = 1;
3272 return 0;
3273 }
3274
3275 static ssize_t
3276 ftrace_graph_write(struct file *file, const char __user *ubuf,
3277 size_t cnt, loff_t *ppos)
3278 {
3279 struct trace_parser parser;
3280 ssize_t read, ret;
3281
3282 if (!cnt)
3283 return 0;
3284
3285 mutex_lock(&graph_lock);
3286
3287 if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) {
3288 ret = -ENOMEM;
3289 goto out_unlock;
3290 }
3291
3292 read = trace_get_user(&parser, ubuf, cnt, ppos);
3293
3294 if (read >= 0 && trace_parser_loaded((&parser))) {
3295 parser.buffer[parser.idx] = 0;
3296
3297 /* we allow only one expression at a time */
3298 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
3299 parser.buffer);
3300 if (ret)
3301 goto out_free;
3302 }
3303
3304 ret = read;
3305
3306 out_free:
3307 trace_parser_put(&parser);
3308 out_unlock:
3309 mutex_unlock(&graph_lock);
3310
3311 return ret;
3312 }
3313
3314 static const struct file_operations ftrace_graph_fops = {
3315 .open = ftrace_graph_open,
3316 .read = seq_read,
3317 .write = ftrace_graph_write,
3318 .release = ftrace_graph_release,
3319 .llseek = seq_lseek,
3320 };
3321 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3322
3323 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
3324 {
3325
3326 trace_create_file("available_filter_functions", 0444,
3327 d_tracer, NULL, &ftrace_avail_fops);
3328
3329 trace_create_file("enabled_functions", 0444,
3330 d_tracer, NULL, &ftrace_enabled_fops);
3331
3332 trace_create_file("set_ftrace_filter", 0644, d_tracer,
3333 NULL, &ftrace_filter_fops);
3334
3335 trace_create_file("set_ftrace_notrace", 0644, d_tracer,
3336 NULL, &ftrace_notrace_fops);
3337
3338 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3339 trace_create_file("set_graph_function", 0444, d_tracer,
3340 NULL,
3341 &ftrace_graph_fops);
3342 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
3343
3344 return 0;
3345 }
3346
3347 static int ftrace_process_locs(struct module *mod,
3348 unsigned long *start,
3349 unsigned long *end)
3350 {
3351 unsigned long *p;
3352 unsigned long addr;
3353 unsigned long flags;
3354
3355 mutex_lock(&ftrace_lock);
3356 p = start;
3357 while (p < end) {
3358 addr = ftrace_call_adjust(*p++);
3359 /*
3360 * Some architecture linkers will pad between
3361 * the different mcount_loc sections of different
3362 * object files to satisfy alignments.
3363 * Skip any NULL pointers.
3364 */
3365 if (!addr)
3366 continue;
3367 ftrace_record_ip(addr);
3368 }
3369
3370 /*
3371 * Disable interrupts to prevent interrupts from executing
3372 * code that is being modified.
3373 */
3374 local_irq_save(flags);
3375 ftrace_update_code(mod);
3376 local_irq_restore(flags);
3377 mutex_unlock(&ftrace_lock);
3378
3379 return 0;
3380 }
3381
3382 #ifdef CONFIG_MODULES
3383 void ftrace_release_mod(struct module *mod)
3384 {
3385 struct dyn_ftrace *rec;
3386 struct ftrace_page *pg;
3387
3388 mutex_lock(&ftrace_lock);
3389
3390 if (ftrace_disabled)
3391 goto out_unlock;
3392
3393 do_for_each_ftrace_rec(pg, rec) {
3394 if (within_module_core(rec->ip, mod)) {
3395 /*
3396 * rec->ip is changed in ftrace_free_rec()
3397 * It should not between s and e if record was freed.
3398 */
3399 FTRACE_WARN_ON(rec->flags & FTRACE_FL_FREE);
3400 ftrace_free_rec(rec);
3401 }
3402 } while_for_each_ftrace_rec();
3403 out_unlock:
3404 mutex_unlock(&ftrace_lock);
3405 }
3406
3407 static void ftrace_init_module(struct module *mod,
3408 unsigned long *start, unsigned long *end)
3409 {
3410 if (ftrace_disabled || start == end)
3411 return;
3412 ftrace_process_locs(mod, start, end);
3413 }
3414
3415 static int ftrace_module_notify(struct notifier_block *self,
3416 unsigned long val, void *data)
3417 {
3418 struct module *mod = data;
3419
3420 switch (val) {
3421 case MODULE_STATE_COMING:
3422 ftrace_init_module(mod, mod->ftrace_callsites,
3423 mod->ftrace_callsites +
3424 mod->num_ftrace_callsites);
3425 break;
3426 case MODULE_STATE_GOING:
3427 ftrace_release_mod(mod);
3428 break;
3429 }
3430
3431 return 0;
3432 }
3433 #else
3434 static int ftrace_module_notify(struct notifier_block *self,
3435 unsigned long val, void *data)
3436 {
3437 return 0;
3438 }
3439 #endif /* CONFIG_MODULES */
3440
3441 struct notifier_block ftrace_module_nb = {
3442 .notifier_call = ftrace_module_notify,
3443 .priority = 0,
3444 };
3445
3446 extern unsigned long __start_mcount_loc[];
3447 extern unsigned long __stop_mcount_loc[];
3448
3449 void __init ftrace_init(void)
3450 {
3451 unsigned long count, addr, flags;
3452 int ret;
3453
3454 /* Keep the ftrace pointer to the stub */
3455 addr = (unsigned long)ftrace_stub;
3456
3457 local_irq_save(flags);
3458 ftrace_dyn_arch_init(&addr);
3459 local_irq_restore(flags);
3460
3461 /* ftrace_dyn_arch_init places the return code in addr */
3462 if (addr)
3463 goto failed;
3464
3465 count = __stop_mcount_loc - __start_mcount_loc;
3466
3467 ret = ftrace_dyn_table_alloc(count);
3468 if (ret)
3469 goto failed;
3470
3471 last_ftrace_enabled = ftrace_enabled = 1;
3472
3473 ret = ftrace_process_locs(NULL,
3474 __start_mcount_loc,
3475 __stop_mcount_loc);
3476
3477 ret = register_module_notifier(&ftrace_module_nb);
3478 if (ret)
3479 pr_warning("Failed to register trace ftrace module notifier\n");
3480
3481 set_ftrace_early_filters();
3482
3483 return;
3484 failed:
3485 ftrace_disabled = 1;
3486 }
3487
3488 #else
3489
3490 static struct ftrace_ops global_ops = {
3491 .func = ftrace_stub,
3492 };
3493
3494 static int __init ftrace_nodyn_init(void)
3495 {
3496 ftrace_enabled = 1;
3497 return 0;
3498 }
3499 device_initcall(ftrace_nodyn_init);
3500
3501 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
3502 static inline void ftrace_startup_enable(int command) { }
3503 /* Keep as macros so we do not need to define the commands */
3504 # define ftrace_startup(ops, command) \
3505 ({ \
3506 (ops)->flags |= FTRACE_OPS_FL_ENABLED; \
3507 0; \
3508 })
3509 # define ftrace_shutdown(ops, command) do { } while (0)
3510 # define ftrace_startup_sysctl() do { } while (0)
3511 # define ftrace_shutdown_sysctl() do { } while (0)
3512
3513 static inline int
3514 ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
3515 {
3516 return 1;
3517 }
3518
3519 #endif /* CONFIG_DYNAMIC_FTRACE */
3520
3521 static void
3522 ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip)
3523 {
3524 struct ftrace_ops *op;
3525
3526 if (unlikely(trace_recursion_test(TRACE_INTERNAL_BIT)))
3527 return;
3528
3529 trace_recursion_set(TRACE_INTERNAL_BIT);
3530 /*
3531 * Some of the ops may be dynamically allocated,
3532 * they must be freed after a synchronize_sched().
3533 */
3534 preempt_disable_notrace();
3535 op = rcu_dereference_raw(ftrace_ops_list);
3536 while (op != &ftrace_list_end) {
3537 if (ftrace_ops_test(op, ip))
3538 op->func(ip, parent_ip);
3539 op = rcu_dereference_raw(op->next);
3540 };
3541 preempt_enable_notrace();
3542 trace_recursion_clear(TRACE_INTERNAL_BIT);
3543 }
3544
3545 static void clear_ftrace_swapper(void)
3546 {
3547 struct task_struct *p;
3548 int cpu;
3549
3550 get_online_cpus();
3551 for_each_online_cpu(cpu) {
3552 p = idle_task(cpu);
3553 clear_tsk_trace_trace(p);
3554 }
3555 put_online_cpus();
3556 }
3557
3558 static void set_ftrace_swapper(void)
3559 {
3560 struct task_struct *p;
3561 int cpu;
3562
3563 get_online_cpus();
3564 for_each_online_cpu(cpu) {
3565 p = idle_task(cpu);
3566 set_tsk_trace_trace(p);
3567 }
3568 put_online_cpus();
3569 }
3570
3571 static void clear_ftrace_pid(struct pid *pid)
3572 {
3573 struct task_struct *p;
3574
3575 rcu_read_lock();
3576 do_each_pid_task(pid, PIDTYPE_PID, p) {
3577 clear_tsk_trace_trace(p);
3578 } while_each_pid_task(pid, PIDTYPE_PID, p);
3579 rcu_read_unlock();
3580
3581 put_pid(pid);
3582 }
3583
3584 static void set_ftrace_pid(struct pid *pid)
3585 {
3586 struct task_struct *p;
3587
3588 rcu_read_lock();
3589 do_each_pid_task(pid, PIDTYPE_PID, p) {
3590 set_tsk_trace_trace(p);
3591 } while_each_pid_task(pid, PIDTYPE_PID, p);
3592 rcu_read_unlock();
3593 }
3594
3595 static void clear_ftrace_pid_task(struct pid *pid)
3596 {
3597 if (pid == ftrace_swapper_pid)
3598 clear_ftrace_swapper();
3599 else
3600 clear_ftrace_pid(pid);
3601 }
3602
3603 static void set_ftrace_pid_task(struct pid *pid)
3604 {
3605 if (pid == ftrace_swapper_pid)
3606 set_ftrace_swapper();
3607 else
3608 set_ftrace_pid(pid);
3609 }
3610
3611 static int ftrace_pid_add(int p)
3612 {
3613 struct pid *pid;
3614 struct ftrace_pid *fpid;
3615 int ret = -EINVAL;
3616
3617 mutex_lock(&ftrace_lock);
3618
3619 if (!p)
3620 pid = ftrace_swapper_pid;
3621 else
3622 pid = find_get_pid(p);
3623
3624 if (!pid)
3625 goto out;
3626
3627 ret = 0;
3628
3629 list_for_each_entry(fpid, &ftrace_pids, list)
3630 if (fpid->pid == pid)
3631 goto out_put;
3632
3633 ret = -ENOMEM;
3634
3635 fpid = kmalloc(sizeof(*fpid), GFP_KERNEL);
3636 if (!fpid)
3637 goto out_put;
3638
3639 list_add(&fpid->list, &ftrace_pids);
3640 fpid->pid = pid;
3641
3642 set_ftrace_pid_task(pid);
3643
3644 ftrace_update_pid_func();
3645 ftrace_startup_enable(0);
3646
3647 mutex_unlock(&ftrace_lock);
3648 return 0;
3649
3650 out_put:
3651 if (pid != ftrace_swapper_pid)
3652 put_pid(pid);
3653
3654 out:
3655 mutex_unlock(&ftrace_lock);
3656 return ret;
3657 }
3658
3659 static void ftrace_pid_reset(void)
3660 {
3661 struct ftrace_pid *fpid, *safe;
3662
3663 mutex_lock(&ftrace_lock);
3664 list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) {
3665 struct pid *pid = fpid->pid;
3666
3667 clear_ftrace_pid_task(pid);
3668
3669 list_del(&fpid->list);
3670 kfree(fpid);
3671 }
3672
3673 ftrace_update_pid_func();
3674 ftrace_startup_enable(0);
3675
3676 mutex_unlock(&ftrace_lock);
3677 }
3678
3679 static void *fpid_start(struct seq_file *m, loff_t *pos)
3680 {
3681 mutex_lock(&ftrace_lock);
3682
3683 if (list_empty(&ftrace_pids) && (!*pos))
3684 return (void *) 1;
3685
3686 return seq_list_start(&ftrace_pids, *pos);
3687 }
3688
3689 static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
3690 {
3691 if (v == (void *)1)
3692 return NULL;
3693
3694 return seq_list_next(v, &ftrace_pids, pos);
3695 }
3696
3697 static void fpid_stop(struct seq_file *m, void *p)
3698 {
3699 mutex_unlock(&ftrace_lock);
3700 }
3701
3702 static int fpid_show(struct seq_file *m, void *v)
3703 {
3704 const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list);
3705
3706 if (v == (void *)1) {
3707 seq_printf(m, "no pid\n");
3708 return 0;
3709 }
3710
3711 if (fpid->pid == ftrace_swapper_pid)
3712 seq_printf(m, "swapper tasks\n");
3713 else
3714 seq_printf(m, "%u\n", pid_vnr(fpid->pid));
3715
3716 return 0;
3717 }
3718
3719 static const struct seq_operations ftrace_pid_sops = {
3720 .start = fpid_start,
3721 .next = fpid_next,
3722 .stop = fpid_stop,
3723 .show = fpid_show,
3724 };
3725
3726 static int
3727 ftrace_pid_open(struct inode *inode, struct file *file)
3728 {
3729 int ret = 0;
3730
3731 if ((file->f_mode & FMODE_WRITE) &&
3732 (file->f_flags & O_TRUNC))
3733 ftrace_pid_reset();
3734
3735 if (file->f_mode & FMODE_READ)
3736 ret = seq_open(file, &ftrace_pid_sops);
3737
3738 return ret;
3739 }
3740
3741 static ssize_t
3742 ftrace_pid_write(struct file *filp, const char __user *ubuf,
3743 size_t cnt, loff_t *ppos)
3744 {
3745 char buf[64], *tmp;
3746 long val;
3747 int ret;
3748
3749 if (cnt >= sizeof(buf))
3750 return -EINVAL;
3751
3752 if (copy_from_user(&buf, ubuf, cnt))
3753 return -EFAULT;
3754
3755 buf[cnt] = 0;
3756
3757 /*
3758 * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid"
3759 * to clean the filter quietly.
3760 */
3761 tmp = strstrip(buf);
3762 if (strlen(tmp) == 0)
3763 return 1;
3764
3765 ret = strict_strtol(tmp, 10, &val);
3766 if (ret < 0)
3767 return ret;
3768
3769 ret = ftrace_pid_add(val);
3770
3771 return ret ? ret : cnt;
3772 }
3773
3774 static int
3775 ftrace_pid_release(struct inode *inode, struct file *file)
3776 {
3777 if (file->f_mode & FMODE_READ)
3778 seq_release(inode, file);
3779
3780 return 0;
3781 }
3782
3783 static const struct file_operations ftrace_pid_fops = {
3784 .open = ftrace_pid_open,
3785 .write = ftrace_pid_write,
3786 .read = seq_read,
3787 .llseek = seq_lseek,
3788 .release = ftrace_pid_release,
3789 };
3790
3791 static __init int ftrace_init_debugfs(void)
3792 {
3793 struct dentry *d_tracer;
3794
3795 d_tracer = tracing_init_dentry();
3796 if (!d_tracer)
3797 return 0;
3798
3799 ftrace_init_dyn_debugfs(d_tracer);
3800
3801 trace_create_file("set_ftrace_pid", 0644, d_tracer,
3802 NULL, &ftrace_pid_fops);
3803
3804 ftrace_profile_debugfs(d_tracer);
3805
3806 return 0;
3807 }
3808 fs_initcall(ftrace_init_debugfs);
3809
3810 /**
3811 * ftrace_kill - kill ftrace
3812 *
3813 * This function should be used by panic code. It stops ftrace
3814 * but in a not so nice way. If you need to simply kill ftrace
3815 * from a non-atomic section, use ftrace_kill.
3816 */
3817 void ftrace_kill(void)
3818 {
3819 ftrace_disabled = 1;
3820 ftrace_enabled = 0;
3821 clear_ftrace_function();
3822 }
3823
3824 /**
3825 * register_ftrace_function - register a function for profiling
3826 * @ops - ops structure that holds the function for profiling.
3827 *
3828 * Register a function to be called by all functions in the
3829 * kernel.
3830 *
3831 * Note: @ops->func and all the functions it calls must be labeled
3832 * with "notrace", otherwise it will go into a
3833 * recursive loop.
3834 */
3835 int register_ftrace_function(struct ftrace_ops *ops)
3836 {
3837 int ret = -1;
3838
3839 mutex_lock(&ftrace_lock);
3840
3841 if (unlikely(ftrace_disabled))
3842 goto out_unlock;
3843
3844 ret = __register_ftrace_function(ops);
3845 if (!ret)
3846 ret = ftrace_startup(ops, 0);
3847
3848
3849 out_unlock:
3850 mutex_unlock(&ftrace_lock);
3851 return ret;
3852 }
3853 EXPORT_SYMBOL_GPL(register_ftrace_function);
3854
3855 /**
3856 * unregister_ftrace_function - unregister a function for profiling.
3857 * @ops - ops structure that holds the function to unregister
3858 *
3859 * Unregister a function that was added to be called by ftrace profiling.
3860 */
3861 int unregister_ftrace_function(struct ftrace_ops *ops)
3862 {
3863 int ret;
3864
3865 mutex_lock(&ftrace_lock);
3866 ret = __unregister_ftrace_function(ops);
3867 if (!ret)
3868 ftrace_shutdown(ops, 0);
3869 mutex_unlock(&ftrace_lock);
3870
3871 return ret;
3872 }
3873 EXPORT_SYMBOL_GPL(unregister_ftrace_function);
3874
3875 int
3876 ftrace_enable_sysctl(struct ctl_table *table, int write,
3877 void __user *buffer, size_t *lenp,
3878 loff_t *ppos)
3879 {
3880 int ret = -ENODEV;
3881
3882 mutex_lock(&ftrace_lock);
3883
3884 if (unlikely(ftrace_disabled))
3885 goto out;
3886
3887 ret = proc_dointvec(table, write, buffer, lenp, ppos);
3888
3889 if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
3890 goto out;
3891
3892 last_ftrace_enabled = !!ftrace_enabled;
3893
3894 if (ftrace_enabled) {
3895
3896 ftrace_startup_sysctl();
3897
3898 /* we are starting ftrace again */
3899 if (ftrace_ops_list != &ftrace_list_end) {
3900 if (ftrace_ops_list->next == &ftrace_list_end)
3901 ftrace_trace_function = ftrace_ops_list->func;
3902 else
3903 ftrace_trace_function = ftrace_ops_list_func;
3904 }
3905
3906 } else {
3907 /* stopping ftrace calls (just send to ftrace_stub) */
3908 ftrace_trace_function = ftrace_stub;
3909
3910 ftrace_shutdown_sysctl();
3911 }
3912
3913 out:
3914 mutex_unlock(&ftrace_lock);
3915 return ret;
3916 }
3917
3918 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3919
3920 static int ftrace_graph_active;
3921 static struct notifier_block ftrace_suspend_notifier;
3922
3923 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
3924 {
3925 return 0;
3926 }
3927
3928 /* The callbacks that hook a function */
3929 trace_func_graph_ret_t ftrace_graph_return =
3930 (trace_func_graph_ret_t)ftrace_stub;
3931 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
3932
3933 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
3934 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
3935 {
3936 int i;
3937 int ret = 0;
3938 unsigned long flags;
3939 int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
3940 struct task_struct *g, *t;
3941
3942 for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
3943 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
3944 * sizeof(struct ftrace_ret_stack),
3945 GFP_KERNEL);
3946 if (!ret_stack_list[i]) {
3947 start = 0;
3948 end = i;
3949 ret = -ENOMEM;
3950 goto free;
3951 }
3952 }
3953
3954 read_lock_irqsave(&tasklist_lock, flags);
3955 do_each_thread(g, t) {
3956 if (start == end) {
3957 ret = -EAGAIN;
3958 goto unlock;
3959 }
3960
3961 if (t->ret_stack == NULL) {
3962 atomic_set(&t->tracing_graph_pause, 0);
3963 atomic_set(&t->trace_overrun, 0);
3964 t->curr_ret_stack = -1;
3965 /* Make sure the tasks see the -1 first: */
3966 smp_wmb();
3967 t->ret_stack = ret_stack_list[start++];
3968 }
3969 } while_each_thread(g, t);
3970
3971 unlock:
3972 read_unlock_irqrestore(&tasklist_lock, flags);
3973 free:
3974 for (i = start; i < end; i++)
3975 kfree(ret_stack_list[i]);
3976 return ret;
3977 }
3978
3979 static void
3980 ftrace_graph_probe_sched_switch(void *ignore,
3981 struct task_struct *prev, struct task_struct *next)
3982 {
3983 unsigned long long timestamp;
3984 int index;
3985
3986 /*
3987 * Does the user want to count the time a function was asleep.
3988 * If so, do not update the time stamps.
3989 */
3990 if (trace_flags & TRACE_ITER_SLEEP_TIME)
3991 return;
3992
3993 timestamp = trace_clock_local();
3994
3995 prev->ftrace_timestamp = timestamp;
3996
3997 /* only process tasks that we timestamped */
3998 if (!next->ftrace_timestamp)
3999 return;
4000
4001 /*
4002 * Update all the counters in next to make up for the
4003 * time next was sleeping.
4004 */
4005 timestamp -= next->ftrace_timestamp;
4006
4007 for (index = next->curr_ret_stack; index >= 0; index--)
4008 next->ret_stack[index].calltime += timestamp;
4009 }
4010
4011 /* Allocate a return stack for each task */
4012 static int start_graph_tracing(void)
4013 {
4014 struct ftrace_ret_stack **ret_stack_list;
4015 int ret, cpu;
4016
4017 ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
4018 sizeof(struct ftrace_ret_stack *),
4019 GFP_KERNEL);
4020
4021 if (!ret_stack_list)
4022 return -ENOMEM;
4023
4024 /* The cpu_boot init_task->ret_stack will never be freed */
4025 for_each_online_cpu(cpu) {
4026 if (!idle_task(cpu)->ret_stack)
4027 ftrace_graph_init_idle_task(idle_task(cpu), cpu);
4028 }
4029
4030 do {
4031 ret = alloc_retstack_tasklist(ret_stack_list);
4032 } while (ret == -EAGAIN);
4033
4034 if (!ret) {
4035 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
4036 if (ret)
4037 pr_info("ftrace_graph: Couldn't activate tracepoint"
4038 " probe to kernel_sched_switch\n");
4039 }
4040
4041 kfree(ret_stack_list);
4042 return ret;
4043 }
4044
4045 /*
4046 * Hibernation protection.
4047 * The state of the current task is too much unstable during
4048 * suspend/restore to disk. We want to protect against that.
4049 */
4050 static int
4051 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
4052 void *unused)
4053 {
4054 switch (state) {
4055 case PM_HIBERNATION_PREPARE:
4056 pause_graph_tracing();
4057 break;
4058
4059 case PM_POST_HIBERNATION:
4060 unpause_graph_tracing();
4061 break;
4062 }
4063 return NOTIFY_DONE;
4064 }
4065
4066 int register_ftrace_graph(trace_func_graph_ret_t retfunc,
4067 trace_func_graph_ent_t entryfunc)
4068 {
4069 int ret = 0;
4070
4071 mutex_lock(&ftrace_lock);
4072
4073 /* we currently allow only one tracer registered at a time */
4074 if (ftrace_graph_active) {
4075 ret = -EBUSY;
4076 goto out;
4077 }
4078
4079 ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
4080 register_pm_notifier(&ftrace_suspend_notifier);
4081
4082 ftrace_graph_active++;
4083 ret = start_graph_tracing();
4084 if (ret) {
4085 ftrace_graph_active--;
4086 goto out;
4087 }
4088
4089 ftrace_graph_return = retfunc;
4090 ftrace_graph_entry = entryfunc;
4091
4092 ret = ftrace_startup(&global_ops, FTRACE_START_FUNC_RET);
4093
4094 out:
4095 mutex_unlock(&ftrace_lock);
4096 return ret;
4097 }
4098
4099 void unregister_ftrace_graph(void)
4100 {
4101 mutex_lock(&ftrace_lock);
4102
4103 if (unlikely(!ftrace_graph_active))
4104 goto out;
4105
4106 ftrace_graph_active--;
4107 ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
4108 ftrace_graph_entry = ftrace_graph_entry_stub;
4109 ftrace_shutdown(&global_ops, FTRACE_STOP_FUNC_RET);
4110 unregister_pm_notifier(&ftrace_suspend_notifier);
4111 unregister_trace_sched_switch(ftrace_graph_probe_sched_switch, NULL);
4112
4113 out:
4114 mutex_unlock(&ftrace_lock);
4115 }
4116
4117 static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);
4118
4119 static void
4120 graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
4121 {
4122 atomic_set(&t->tracing_graph_pause, 0);
4123 atomic_set(&t->trace_overrun, 0);
4124 t->ftrace_timestamp = 0;
4125 /* make curr_ret_stack visible before we add the ret_stack */
4126 smp_wmb();
4127 t->ret_stack = ret_stack;
4128 }
4129
4130 /*
4131 * Allocate a return stack for the idle task. May be the first
4132 * time through, or it may be done by CPU hotplug online.
4133 */
4134 void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
4135 {
4136 t->curr_ret_stack = -1;
4137 /*
4138 * The idle task has no parent, it either has its own
4139 * stack or no stack at all.
4140 */
4141 if (t->ret_stack)
4142 WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));
4143
4144 if (ftrace_graph_active) {
4145 struct ftrace_ret_stack *ret_stack;
4146
4147 ret_stack = per_cpu(idle_ret_stack, cpu);
4148 if (!ret_stack) {
4149 ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
4150 * sizeof(struct ftrace_ret_stack),
4151 GFP_KERNEL);
4152 if (!ret_stack)
4153 return;
4154 per_cpu(idle_ret_stack, cpu) = ret_stack;
4155 }
4156 graph_init_task(t, ret_stack);
4157 }
4158 }
4159
4160 /* Allocate a return stack for newly created task */
4161 void ftrace_graph_init_task(struct task_struct *t)
4162 {
4163 /* Make sure we do not use the parent ret_stack */
4164 t->ret_stack = NULL;
4165 t->curr_ret_stack = -1;
4166
4167 if (ftrace_graph_active) {
4168 struct ftrace_ret_stack *ret_stack;
4169
4170 ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
4171 * sizeof(struct ftrace_ret_stack),
4172 GFP_KERNEL);
4173 if (!ret_stack)
4174 return;
4175 graph_init_task(t, ret_stack);
4176 }
4177 }
4178
4179 void ftrace_graph_exit_task(struct task_struct *t)
4180 {
4181 struct ftrace_ret_stack *ret_stack = t->ret_stack;
4182
4183 t->ret_stack = NULL;
4184 /* NULL must become visible to IRQs before we free it: */
4185 barrier();
4186
4187 kfree(ret_stack);
4188 }
4189
4190 void ftrace_graph_stop(void)
4191 {
4192 ftrace_stop();
4193 }
4194 #endif
(deprecated) Btrfs devel tree