kernel/trace/trace_functions_graph.c

   1 /*
   2  *
   3  * Function graph tracer.
   4  * Copyright (c) 2008-2009 Frederic Weisbecker <fweisbec@gmail.com>
   5  * Mostly borrowed from function tracer which
   6  * is Copyright (c) Steven Rostedt <srostedt@redhat.com>
   7  *
   8  */
   9 #include <linux/debugfs.h>
  10 #include <linux/uaccess.h>
  11 #include <linux/ftrace.h>
  12 #include <linux/fs.h>
  13
  14 #include "trace.h"
  15 #include "trace_output.h"
  16
  17 struct fgraph_cpu_data {
  18         pid_t           last_pid;
  19         int             depth;
  20         int             ignore;
  21 };
  22
  23 struct fgraph_data {
  24         struct fgraph_cpu_data          *cpu_data;
  25
  26         /* Place to preserve last processed entry. */
  27         struct ftrace_graph_ent_entry   ent;
  28         struct ftrace_graph_ret_entry   ret;
  29         int                             failed;
  30         int                             cpu;
  31 };
  32
  33 #define TRACE_GRAPH_INDENT      2
  34
  35 /* Flag options */
  36 #define TRACE_GRAPH_PRINT_OVERRUN       0x1
  37 #define TRACE_GRAPH_PRINT_CPU           0x2
  38 #define TRACE_GRAPH_PRINT_OVERHEAD      0x4
  39 #define TRACE_GRAPH_PRINT_PROC          0x8
  40 #define TRACE_GRAPH_PRINT_DURATION      0x10
  41 #define TRACE_GRAPH_PRINT_ABS_TIME      0X20
  42
  43 static struct tracer_opt trace_opts[] = {
  44         /* Display overruns? (for self-debug purpose) */
  45         { TRACER_OPT(funcgraph-overrun, TRACE_GRAPH_PRINT_OVERRUN) },
  46         /* Display CPU ? */
  47         { TRACER_OPT(funcgraph-cpu, TRACE_GRAPH_PRINT_CPU) },
  48         /* Display Overhead ? */
  49         { TRACER_OPT(funcgraph-overhead, TRACE_GRAPH_PRINT_OVERHEAD) },
  50         /* Display proc name/pid */
  51         { TRACER_OPT(funcgraph-proc, TRACE_GRAPH_PRINT_PROC) },
  52         /* Display duration of execution */
  53         { TRACER_OPT(funcgraph-duration, TRACE_GRAPH_PRINT_DURATION) },
  54         /* Display absolute time of an entry */
  55         { TRACER_OPT(funcgraph-abstime, TRACE_GRAPH_PRINT_ABS_TIME) },
  56         { } /* Empty entry */
  57 };
  58
  59 static struct tracer_flags tracer_flags = {
  60         /* Don't display overruns and proc by default */
  61         .val = TRACE_GRAPH_PRINT_CPU | TRACE_GRAPH_PRINT_OVERHEAD |
  62                TRACE_GRAPH_PRINT_DURATION,
  63         .opts = trace_opts
  64 };
  65
  66 static struct trace_array *graph_array;
  67
  68
  69 /* Add a function return address to the trace stack on thread info.*/
  70 int
  71 ftrace_push_return_trace(unsigned long ret, unsigned long func, int *depth,
  72                          unsigned long frame_pointer)
  73 {
  74         unsigned long long calltime;
  75         int index;
  76
  77         if (!current->ret_stack)
  78                 return -EBUSY;
  79
  80         /*
  81          * We must make sure the ret_stack is tested before we read
  82          * anything else.
  83          */
  84         smp_rmb();
  85
  86         /* The return trace stack is full */
  87         if (current->curr_ret_stack == FTRACE_RETFUNC_DEPTH - 1) {
  88                 atomic_inc(&current->trace_overrun);
  89                 return -EBUSY;
  90         }
  91
  92         calltime = trace_clock_local();
  93
  94         index = ++current->curr_ret_stack;
  95         barrier();
  96         current->ret_stack[index].ret = ret;
  97         current->ret_stack[index].func = func;
  98         current->ret_stack[index].calltime = calltime;
  99         current->ret_stack[index].subtime = 0;
 100         current->ret_stack[index].fp = frame_pointer;
 101         *depth = index;
 102
 103         return 0;
 104 }
 105
 106 /* Retrieve a function return address to the trace stack on thread info.*/
 107 static void
 108 ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret,
 109                         unsigned long frame_pointer)
 110 {
 111         int index;
 112
 113         index = current->curr_ret_stack;
 114
 115         if (unlikely(index < 0)) {
 116                 ftrace_graph_stop();
 117                 WARN_ON(1);
 118                 /* Might as well panic, otherwise we have no where to go */
 119                 *ret = (unsigned long)panic;
 120                 return;
 121         }
 122
 123 #ifdef CONFIG_HAVE_FUNCTION_GRAPH_FP_TEST
 124         /*
 125          * The arch may choose to record the frame pointer used
 126          * and check it here to make sure that it is what we expect it
 127          * to be. If gcc does not set the place holder of the return
 128          * address in the frame pointer, and does a copy instead, then
 129          * the function graph trace will fail. This test detects this
 130          * case.
 131          *
 132          * Currently, x86_32 with optimize for size (-Os) makes the latest
 133          * gcc do the above.
 134          */
 135         if (unlikely(current->ret_stack[index].fp != frame_pointer)) {
 136                 ftrace_graph_stop();
 137                 WARN(1, "Bad frame pointer: expected %lx, received %lx\n"
 138                      "  from func %ps return to %lx\n",
 139                      current->ret_stack[index].fp,
 140                      frame_pointer,
 141                      (void *)current->ret_stack[index].func,
 142                      current->ret_stack[index].ret);
 143                 *ret = (unsigned long)panic;
 144                 return;
 145         }
 146 #endif
 147
 148         *ret = current->ret_stack[index].ret;
 149         trace->func = current->ret_stack[index].func;
 150         trace->calltime = current->ret_stack[index].calltime;
 151         trace->overrun = atomic_read(&current->trace_overrun);
 152         trace->depth = index;
 153 }
 154
 155 /*
 156  * Send the trace to the ring-buffer.
 157  * @return the original return address.
 158  */
 159 unsigned long ftrace_return_to_handler(unsigned long frame_pointer)
 160 {
 161         struct ftrace_graph_ret trace;
 162         unsigned long ret;
 163
 164         ftrace_pop_return_trace(&trace, &ret, frame_pointer);
 165         trace.rettime = trace_clock_local();
 166         ftrace_graph_return(&trace);
 167         barrier();
 168         current->curr_ret_stack--;
 169
 170         if (unlikely(!ret)) {
 171                 ftrace_graph_stop();
 172                 WARN_ON(1);
 173                 /* Might as well panic. What else to do? */
 174                 ret = (unsigned long)panic;
 175         }
 176
 177         return ret;
 178 }
 179
 180 static int __trace_graph_entry(struct trace_array *tr,
 181                                 struct ftrace_graph_ent *trace,
 182                                 unsigned long flags,
 183                                 int pc)
 184 {
 185         struct ftrace_event_call *call = &event_funcgraph_entry;
 186         struct ring_buffer_event *event;
 187         struct ring_buffer *buffer = tr->buffer;
 188         struct ftrace_graph_ent_entry *entry;
 189
 190         if (unlikely(__this_cpu_read(per_cpu_var(ftrace_cpu_disabled))))
 191                 return 0;
 192
 193         event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_ENT,
 194                                           sizeof(*entry), flags, pc);
 195         if (!event)
 196                 return 0;
 197         entry   = ring_buffer_event_data(event);
 198         entry->graph_ent                        = *trace;
 199         if (!filter_current_check_discard(buffer, call, entry, event))
 200                 ring_buffer_unlock_commit(buffer, event);
 201
 202         return 1;
 203 }
 204
 205 int trace_graph_entry(struct ftrace_graph_ent *trace)
 206 {
 207         struct trace_array *tr = graph_array;
 208         struct trace_array_cpu *data;
 209         unsigned long flags;
 210         long disabled;
 211         int ret;
 212         int cpu;
 213         int pc;
 214
 215         if (unlikely(!tr))
 216                 return 0;
 217
 218         if (!ftrace_trace_task(current))
 219                 return 0;
 220
 221         if (!ftrace_graph_addr(trace->func))
 222                 return 0;
 223
 224         local_irq_save(flags);
 225         cpu = raw_smp_processor_id();
 226         data = tr->data[cpu];
 227         disabled = atomic_inc_return(&data->disabled);
 228         if (likely(disabled == 1)) {
 229                 pc = preempt_count();
 230                 ret = __trace_graph_entry(tr, trace, flags, pc);
 231         } else {
 232                 ret = 0;
 233         }
 234         /* Only do the atomic if it is not already set */
 235         if (!test_tsk_trace_graph(current))
 236                 set_tsk_trace_graph(current);
 237
 238         atomic_dec(&data->disabled);
 239         local_irq_restore(flags);
 240
 241         return ret;
 242 }
 243
 244 static void __trace_graph_return(struct trace_array *tr,
 245                                 struct ftrace_graph_ret *trace,
 246                                 unsigned long flags,
 247                                 int pc)
 248 {
 249         struct ftrace_event_call *call = &event_funcgraph_exit;
 250         struct ring_buffer_event *event;
 251         struct ring_buffer *buffer = tr->buffer;
 252         struct ftrace_graph_ret_entry *entry;
 253
 254         if (unlikely(__this_cpu_read(per_cpu_var(ftrace_cpu_disabled))))
 255                 return;
 256
 257         event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RET,
 258                                           sizeof(*entry), flags, pc);
 259         if (!event)
 260                 return;
 261         entry   = ring_buffer_event_data(event);
 262         entry->ret                              = *trace;
 263         if (!filter_current_check_discard(buffer, call, entry, event))
 264                 ring_buffer_unlock_commit(buffer, event);
 265 }
 266
 267 void trace_graph_return(struct ftrace_graph_ret *trace)
 268 {
 269         struct trace_array *tr = graph_array;
 270         struct trace_array_cpu *data;
 271         unsigned long flags;
 272         long disabled;
 273         int cpu;
 274         int pc;
 275
 276         local_irq_save(flags);
 277         cpu = raw_smp_processor_id();
 278         data = tr->data[cpu];
 279         disabled = atomic_inc_return(&data->disabled);
 280         if (likely(disabled == 1)) {
 281                 pc = preempt_count();
 282                 __trace_graph_return(tr, trace, flags, pc);
 283         }
 284         if (!trace->depth)
 285                 clear_tsk_trace_graph(current);
 286         atomic_dec(&data->disabled);
 287         local_irq_restore(flags);
 288 }
 289
 290 static int graph_trace_init(struct trace_array *tr)
 291 {
 292         int ret;
 293
 294         graph_array = tr;
 295         ret = register_ftrace_graph(&trace_graph_return,
 296                                     &trace_graph_entry);
 297         if (ret)
 298                 return ret;
 299         tracing_start_cmdline_record();
 300
 301         return 0;
 302 }
 303
 304 void set_graph_array(struct trace_array *tr)
 305 {
 306         graph_array = tr;
 307 }
 308
 309 static void graph_trace_reset(struct trace_array *tr)
 310 {
 311         tracing_stop_cmdline_record();
 312         unregister_ftrace_graph();
 313 }
 314
 315 static int max_bytes_for_cpu;
 316
 317 static enum print_line_t
 318 print_graph_cpu(struct trace_seq *s, int cpu)
 319 {
 320         int ret;
 321
 322         /*
 323          * Start with a space character - to make it stand out
 324          * to the right a bit when trace output is pasted into
 325          * email:
 326          */
 327         ret = trace_seq_printf(s, " %*d) ", max_bytes_for_cpu, cpu);
 328         if (!ret)
 329                 return TRACE_TYPE_PARTIAL_LINE;
 330
 331         return TRACE_TYPE_HANDLED;
 332 }
 333
 334 #define TRACE_GRAPH_PROCINFO_LENGTH     14
 335
 336 static enum print_line_t
 337 print_graph_proc(struct trace_seq *s, pid_t pid)
 338 {
 339         char comm[TASK_COMM_LEN];
 340         /* sign + log10(MAX_INT) + '\0' */
 341         char pid_str[11];
 342         int spaces = 0;
 343         int ret;
 344         int len;
 345         int i;
 346
 347         trace_find_cmdline(pid, comm);
 348         comm[7] = '\0';
 349         sprintf(pid_str, "%d", pid);
 350
 351         /* 1 stands for the "-" character */
 352         len = strlen(comm) + strlen(pid_str) + 1;
 353
 354         if (len < TRACE_GRAPH_PROCINFO_LENGTH)
 355                 spaces = TRACE_GRAPH_PROCINFO_LENGTH - len;
 356
 357         /* First spaces to align center */
 358         for (i = 0; i < spaces / 2; i++) {
 359                 ret = trace_seq_printf(s, " ");
 360                 if (!ret)
 361                         return TRACE_TYPE_PARTIAL_LINE;
 362         }
 363
 364         ret = trace_seq_printf(s, "%s-%s", comm, pid_str);
 365         if (!ret)
 366                 return TRACE_TYPE_PARTIAL_LINE;
 367
 368         /* Last spaces to align center */
 369         for (i = 0; i < spaces - (spaces / 2); i++) {
 370                 ret = trace_seq_printf(s, " ");
 371                 if (!ret)
 372                         return TRACE_TYPE_PARTIAL_LINE;
 373         }
 374         return TRACE_TYPE_HANDLED;
 375 }
 376
 377
 378 static enum print_line_t
 379 print_graph_lat_fmt(struct trace_seq *s, struct trace_entry *entry)
 380 {
 381         if (!trace_seq_putc(s, ' '))
 382                 return 0;
 383
 384         return trace_print_lat_fmt(s, entry);
 385 }
 386
 387 /* If the pid changed since the last trace, output this event */
 388 static enum print_line_t
 389 verif_pid(struct trace_seq *s, pid_t pid, int cpu, struct fgraph_data *data)
 390 {
 391         pid_t prev_pid;
 392         pid_t *last_pid;
 393         int ret;
 394
 395         if (!data)
 396                 return TRACE_TYPE_HANDLED;
 397
 398         last_pid = &(per_cpu_ptr(data->cpu_data, cpu)->last_pid);
 399
 400         if (*last_pid == pid)
 401                 return TRACE_TYPE_HANDLED;
 402
 403         prev_pid = *last_pid;
 404         *last_pid = pid;
 405
 406         if (prev_pid == -1)
 407                 return TRACE_TYPE_HANDLED;
 408 /*
 409  * Context-switch trace line:
 410
 411  ------------------------------------------
 412  | 1)  migration/0--1  =>  sshd-1755
 413  ------------------------------------------
 414
 415  */
 416         ret = trace_seq_printf(s,
 417                 " ------------------------------------------\n");
 418         if (!ret)
 419                 return TRACE_TYPE_PARTIAL_LINE;
 420
 421         ret = print_graph_cpu(s, cpu);
 422         if (ret == TRACE_TYPE_PARTIAL_LINE)
 423                 return TRACE_TYPE_PARTIAL_LINE;
 424
 425         ret = print_graph_proc(s, prev_pid);
 426         if (ret == TRACE_TYPE_PARTIAL_LINE)
 427                 return TRACE_TYPE_PARTIAL_LINE;
 428
 429         ret = trace_seq_printf(s, " => ");
 430         if (!ret)
 431                 return TRACE_TYPE_PARTIAL_LINE;
 432
 433         ret = print_graph_proc(s, pid);
 434         if (ret == TRACE_TYPE_PARTIAL_LINE)
 435                 return TRACE_TYPE_PARTIAL_LINE;
 436
 437         ret = trace_seq_printf(s,
 438                 "\n ------------------------------------------\n\n");
 439         if (!ret)
 440                 return TRACE_TYPE_PARTIAL_LINE;
 441
 442         return TRACE_TYPE_HANDLED;
 443 }
 444
 445 static struct ftrace_graph_ret_entry *
 446 get_return_for_leaf(struct trace_iterator *iter,
 447                 struct ftrace_graph_ent_entry *curr)
 448 {
 449         struct fgraph_data *data = iter->private;
 450         struct ring_buffer_iter *ring_iter = NULL;
 451         struct ring_buffer_event *event;
 452         struct ftrace_graph_ret_entry *next;
 453
 454         /*
 455          * If the previous output failed to write to the seq buffer,
 456          * then we just reuse the data from before.
 457          */
 458         if (data && data->failed) {
 459                 curr = &data->ent;
 460                 next = &data->ret;
 461         } else {
 462
 463                 ring_iter = iter->buffer_iter[iter->cpu];
 464
 465                 /* First peek to compare current entry and the next one */
 466                 if (ring_iter)
 467                         event = ring_buffer_iter_peek(ring_iter, NULL);
 468                 else {
 469                         /*
 470                          * We need to consume the current entry to see
 471                          * the next one.
 472                          */
 473                         ring_buffer_consume(iter->tr->buffer, iter->cpu, NULL);
 474                         event = ring_buffer_peek(iter->tr->buffer, iter->cpu,
 475                                                  NULL);
 476                 }
 477
 478                 if (!event)
 479                         return NULL;
 480
 481                 next = ring_buffer_event_data(event);
 482
 483                 if (data) {
 484                         /*
 485                          * Save current and next entries for later reference
 486                          * if the output fails.
 487                          */
 488                         data->ent = *curr;
 489                         data->ret = *next;
 490                 }
 491         }
 492
 493         if (next->ent.type != TRACE_GRAPH_RET)
 494                 return NULL;
 495
 496         if (curr->ent.pid != next->ent.pid ||
 497                         curr->graph_ent.func != next->ret.func)
 498                 return NULL;
 499
 500         /* this is a leaf, now advance the iterator */
 501         if (ring_iter)
 502                 ring_buffer_read(ring_iter, NULL);
 503
 504         return next;
 505 }
 506
 507 /* Signal a overhead of time execution to the output */
 508 static int
 509 print_graph_overhead(unsigned long long duration, struct trace_seq *s)
 510 {
 511         /* If duration disappear, we don't need anything */
 512         if (!(tracer_flags.val & TRACE_GRAPH_PRINT_DURATION))
 513                 return 1;
 514
 515         /* Non nested entry or return */
 516         if (duration == -1)
 517                 return trace_seq_printf(s, "  ");
 518
 519         if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) {
 520                 /* Duration exceeded 100 msecs */
 521                 if (duration > 100000ULL)
 522                         return trace_seq_printf(s, "! ");
 523
 524                 /* Duration exceeded 10 msecs */
 525                 if (duration > 10000ULL)
 526                         return trace_seq_printf(s, "+ ");
 527         }
 528
 529         return trace_seq_printf(s, "  ");
 530 }
 531
 532 static int print_graph_abs_time(u64 t, struct trace_seq *s)
 533 {
 534         unsigned long usecs_rem;
 535
 536         usecs_rem = do_div(t, NSEC_PER_SEC);
 537         usecs_rem /= 1000;
 538
 539         return trace_seq_printf(s, "%5lu.%06lu |  ",
 540                         (unsigned long)t, usecs_rem);
 541 }
 542
 543 static enum print_line_t
 544 print_graph_irq(struct trace_iterator *iter, unsigned long addr,
 545                 enum trace_type type, int cpu, pid_t pid)
 546 {
 547         int ret;
 548         struct trace_seq *s = &iter->seq;
 549
 550         if (addr < (unsigned long)__irqentry_text_start ||
 551                 addr >= (unsigned long)__irqentry_text_end)
 552                 return TRACE_TYPE_UNHANDLED;
 553
 554         /* Absolute time */
 555         if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME) {
 556                 ret = print_graph_abs_time(iter->ts, s);
 557                 if (!ret)
 558                         return TRACE_TYPE_PARTIAL_LINE;
 559         }
 560
 561         /* Cpu */
 562         if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) {
 563                 ret = print_graph_cpu(s, cpu);
 564                 if (ret == TRACE_TYPE_PARTIAL_LINE)
 565                         return TRACE_TYPE_PARTIAL_LINE;
 566         }
 567
 568         /* Proc */
 569         if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC) {
 570                 ret = print_graph_proc(s, pid);
 571                 if (ret == TRACE_TYPE_PARTIAL_LINE)
 572                         return TRACE_TYPE_PARTIAL_LINE;
 573                 ret = trace_seq_printf(s, " | ");
 574                 if (!ret)
 575                         return TRACE_TYPE_PARTIAL_LINE;
 576         }
 577
 578         /* No overhead */
 579         ret = print_graph_overhead(-1, s);
 580         if (!ret)
 581                 return TRACE_TYPE_PARTIAL_LINE;
 582
 583         if (type == TRACE_GRAPH_ENT)
 584                 ret = trace_seq_printf(s, "==========>");
 585         else
 586                 ret = trace_seq_printf(s, "<==========");
 587
 588         if (!ret)
 589                 return TRACE_TYPE_PARTIAL_LINE;
 590
 591         /* Don't close the duration column if haven't one */
 592         if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION)
 593                 trace_seq_printf(s, " |");
 594         ret = trace_seq_printf(s, "\n");
 595
 596         if (!ret)
 597                 return TRACE_TYPE_PARTIAL_LINE;
 598         return TRACE_TYPE_HANDLED;
 599 }
 600
 601 enum print_line_t
 602 trace_print_graph_duration(unsigned long long duration, struct trace_seq *s)
 603 {
 604         unsigned long nsecs_rem = do_div(duration, 1000);
 605         /* log10(ULONG_MAX) + '\0' */
 606         char msecs_str[21];
 607         char nsecs_str[5];
 608         int ret, len;
 609         int i;
 610
 611         sprintf(msecs_str, "%lu", (unsigned long) duration);
 612
 613         /* Print msecs */
 614         ret = trace_seq_printf(s, "%s", msecs_str);
 615         if (!ret)
 616                 return TRACE_TYPE_PARTIAL_LINE;
 617
 618         len = strlen(msecs_str);
 619
 620         /* Print nsecs (we don't want to exceed 7 numbers) */
 621         if (len < 7) {
 622                 snprintf(nsecs_str, 8 - len, "%03lu", nsecs_rem);
 623                 ret = trace_seq_printf(s, ".%s", nsecs_str);
 624                 if (!ret)
 625                         return TRACE_TYPE_PARTIAL_LINE;
 626                 len += strlen(nsecs_str);
 627         }
 628
 629         ret = trace_seq_printf(s, " us ");
 630         if (!ret)
 631                 return TRACE_TYPE_PARTIAL_LINE;
 632
 633         /* Print remaining spaces to fit the row's width */
 634         for (i = len; i < 7; i++) {
 635                 ret = trace_seq_printf(s, " ");
 636                 if (!ret)
 637                         return TRACE_TYPE_PARTIAL_LINE;
 638         }
 639         return TRACE_TYPE_HANDLED;
 640 }
 641
 642 static enum print_line_t
 643 print_graph_duration(unsigned long long duration, struct trace_seq *s)
 644 {
 645         int ret;
 646
 647         ret = trace_print_graph_duration(duration, s);
 648         if (ret != TRACE_TYPE_HANDLED)
 649                 return ret;
 650
 651         ret = trace_seq_printf(s, "|  ");
 652         if (!ret)
 653                 return TRACE_TYPE_PARTIAL_LINE;
 654
 655         return TRACE_TYPE_HANDLED;
 656 }
 657
 658 /* Case of a leaf function on its call entry */
 659 static enum print_line_t
 660 print_graph_entry_leaf(struct trace_iterator *iter,
 661                 struct ftrace_graph_ent_entry *entry,
 662                 struct ftrace_graph_ret_entry *ret_entry, struct trace_seq *s)
 663 {
 664         struct fgraph_data *data = iter->private;
 665         struct ftrace_graph_ret *graph_ret;
 666         struct ftrace_graph_ent *call;
 667         unsigned long long duration;
 668         int ret;
 669         int i;
 670
 671         graph_ret = &ret_entry->ret;
 672         call = &entry->graph_ent;
 673         duration = graph_ret->rettime - graph_ret->calltime;
 674
 675         if (data) {
 676                 int cpu = iter->cpu;
 677                 int *depth = &(per_cpu_ptr(data->cpu_data, cpu)->depth);
 678
 679                 /*
 680                  * Comments display at + 1 to depth. Since
 681                  * this is a leaf function, keep the comments
 682                  * equal to this depth.
 683                  */
 684                 *depth = call->depth - 1;
 685         }
 686
 687         /* Overhead */
 688         ret = print_graph_overhead(duration, s);
 689         if (!ret)
 690                 return TRACE_TYPE_PARTIAL_LINE;
 691
 692         /* Duration */
 693         if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION) {
 694                 ret = print_graph_duration(duration, s);
 695                 if (ret == TRACE_TYPE_PARTIAL_LINE)
 696                         return TRACE_TYPE_PARTIAL_LINE;
 697         }
 698
 699         /* Function */
 700         for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) {
 701                 ret = trace_seq_printf(s, " ");
 702                 if (!ret)
 703                         return TRACE_TYPE_PARTIAL_LINE;
 704         }
 705
 706         ret = trace_seq_printf(s, "%ps();\n", (void *)call->func);
 707         if (!ret)
 708                 return TRACE_TYPE_PARTIAL_LINE;
 709
 710         return TRACE_TYPE_HANDLED;
 711 }
 712
 713 static enum print_line_t
 714 print_graph_entry_nested(struct trace_iterator *iter,
 715                          struct ftrace_graph_ent_entry *entry,
 716                          struct trace_seq *s, int cpu)
 717 {
 718         struct ftrace_graph_ent *call = &entry->graph_ent;
 719         struct fgraph_data *data = iter->private;
 720         int ret;
 721         int i;
 722
 723         if (data) {
 724                 int cpu = iter->cpu;
 725                 int *depth = &(per_cpu_ptr(data->cpu_data, cpu)->depth);
 726
 727                 *depth = call->depth;
 728         }
 729
 730         /* No overhead */
 731         ret = print_graph_overhead(-1, s);
 732         if (!ret)
 733                 return TRACE_TYPE_PARTIAL_LINE;
 734
 735         /* No time */
 736         if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION) {
 737                 ret = trace_seq_printf(s, "            |  ");
 738                 if (!ret)
 739                         return TRACE_TYPE_PARTIAL_LINE;
 740         }
 741
 742         /* Function */
 743         for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) {
 744                 ret = trace_seq_printf(s, " ");
 745                 if (!ret)
 746                         return TRACE_TYPE_PARTIAL_LINE;
 747         }
 748
 749         ret = trace_seq_printf(s, "%ps() {\n", (void *)call->func);
 750         if (!ret)
 751                 return TRACE_TYPE_PARTIAL_LINE;
 752
 753         /*
 754          * we already consumed the current entry to check the next one
 755          * and see if this is a leaf.
 756          */
 757         return TRACE_TYPE_NO_CONSUME;
 758 }
 759
 760 static enum print_line_t
 761 print_graph_prologue(struct trace_iterator *iter, struct trace_seq *s,
 762                      int type, unsigned long addr)
 763 {
 764         struct fgraph_data *data = iter->private;
 765         struct trace_entry *ent = iter->ent;
 766         int cpu = iter->cpu;
 767         int ret;
 768
 769         /* Pid */
 770         if (verif_pid(s, ent->pid, cpu, data) == TRACE_TYPE_PARTIAL_LINE)
 771                 return TRACE_TYPE_PARTIAL_LINE;
 772
 773         if (type) {
 774                 /* Interrupt */
 775                 ret = print_graph_irq(iter, addr, type, cpu, ent->pid);
 776                 if (ret == TRACE_TYPE_PARTIAL_LINE)
 777                         return TRACE_TYPE_PARTIAL_LINE;
 778         }
 779
 780         /* Absolute time */
 781         if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME) {
 782                 ret = print_graph_abs_time(iter->ts, s);
 783                 if (!ret)
 784                         return TRACE_TYPE_PARTIAL_LINE;
 785         }
 786
 787         /* Cpu */
 788         if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) {
 789                 ret = print_graph_cpu(s, cpu);
 790                 if (ret == TRACE_TYPE_PARTIAL_LINE)
 791                         return TRACE_TYPE_PARTIAL_LINE;
 792         }
 793
 794         /* Proc */
 795         if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC) {
 796                 ret = print_graph_proc(s, ent->pid);
 797                 if (ret == TRACE_TYPE_PARTIAL_LINE)
 798                         return TRACE_TYPE_PARTIAL_LINE;
 799
 800                 ret = trace_seq_printf(s, " | ");
 801                 if (!ret)
 802                         return TRACE_TYPE_PARTIAL_LINE;
 803         }
 804
 805         /* Latency format */
 806         if (trace_flags & TRACE_ITER_LATENCY_FMT) {
 807                 ret = print_graph_lat_fmt(s, ent);
 808                 if (ret == TRACE_TYPE_PARTIAL_LINE)
 809                         return TRACE_TYPE_PARTIAL_LINE;
 810         }
 811
 812         return 0;
 813 }
 814
 815 static enum print_line_t
 816 print_graph_entry(struct ftrace_graph_ent_entry *field, struct trace_seq *s,
 817                         struct trace_iterator *iter)
 818 {
 819         struct fgraph_data *data = iter->private;
 820         struct ftrace_graph_ent *call = &field->graph_ent;
 821         struct ftrace_graph_ret_entry *leaf_ret;
 822         static enum print_line_t ret;
 823         int cpu = iter->cpu;
 824
 825         if (print_graph_prologue(iter, s, TRACE_GRAPH_ENT, call->func))
 826                 return TRACE_TYPE_PARTIAL_LINE;
 827
 828         leaf_ret = get_return_for_leaf(iter, field);
 829         if (leaf_ret)
 830                 ret = print_graph_entry_leaf(iter, field, leaf_ret, s);
 831         else
 832                 ret = print_graph_entry_nested(iter, field, s, cpu);
 833
 834         if (data) {
 835                 /*
 836                  * If we failed to write our output, then we need to make
 837                  * note of it. Because we already consumed our entry.
 838                  */
 839                 if (s->full) {
 840                         data->failed = 1;
 841                         data->cpu = cpu;
 842                 } else
 843                         data->failed = 0;
 844         }
 845
 846         return ret;
 847 }
 848
 849 static enum print_line_t
 850 print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s,
 851                    struct trace_entry *ent, struct trace_iterator *iter)
 852 {
 853         unsigned long long duration = trace->rettime - trace->calltime;
 854         struct fgraph_data *data = iter->private;
 855         pid_t pid = ent->pid;
 856         int cpu = iter->cpu;
 857         int ret;
 858         int i;
 859
 860         if (data) {
 861                 int cpu = iter->cpu;
 862                 int *depth = &(per_cpu_ptr(data->cpu_data, cpu)->depth);
 863
 864                 /*
 865                  * Comments display at + 1 to depth. This is the
 866                  * return from a function, we now want the comments
 867                  * to display at the same level of the bracket.
 868                  */
 869                 *depth = trace->depth - 1;
 870         }
 871
 872         if (print_graph_prologue(iter, s, 0, 0))
 873                 return TRACE_TYPE_PARTIAL_LINE;
 874
 875         /* Overhead */
 876         ret = print_graph_overhead(duration, s);
 877         if (!ret)
 878                 return TRACE_TYPE_PARTIAL_LINE;
 879
 880         /* Duration */
 881         if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION) {
 882                 ret = print_graph_duration(duration, s);
 883                 if (ret == TRACE_TYPE_PARTIAL_LINE)
 884                         return TRACE_TYPE_PARTIAL_LINE;
 885         }
 886
 887         /* Closing brace */
 888         for (i = 0; i < trace->depth * TRACE_GRAPH_INDENT; i++) {
 889                 ret = trace_seq_printf(s, " ");
 890                 if (!ret)
 891                         return TRACE_TYPE_PARTIAL_LINE;
 892         }
 893
 894         ret = trace_seq_printf(s, "}\n");
 895         if (!ret)
 896                 return TRACE_TYPE_PARTIAL_LINE;
 897
 898         /* Overrun */
 899         if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERRUN) {
 900                 ret = trace_seq_printf(s, " (Overruns: %lu)\n",
 901                                         trace->overrun);
 902                 if (!ret)
 903                         return TRACE_TYPE_PARTIAL_LINE;
 904         }
 905
 906         ret = print_graph_irq(iter, trace->func, TRACE_GRAPH_RET, cpu, pid);
 907         if (ret == TRACE_TYPE_PARTIAL_LINE)
 908                 return TRACE_TYPE_PARTIAL_LINE;
 909
 910         return TRACE_TYPE_HANDLED;
 911 }
 912
 913 static enum print_line_t
 914 print_graph_comment(struct trace_seq *s,  struct trace_entry *ent,
 915                     struct trace_iterator *iter)
 916 {
 917         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
 918         struct fgraph_data *data = iter->private;
 919         struct trace_event *event;
 920         int depth = 0;
 921         int ret;
 922         int i;
 923
 924         if (data)
 925                 depth = per_cpu_ptr(data->cpu_data, iter->cpu)->depth;
 926
 927         if (print_graph_prologue(iter, s, 0, 0))
 928                 return TRACE_TYPE_PARTIAL_LINE;
 929
 930         /* No overhead */
 931         ret = print_graph_overhead(-1, s);
 932         if (!ret)
 933                 return TRACE_TYPE_PARTIAL_LINE;
 934
 935         /* No time */
 936         if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION) {
 937                 ret = trace_seq_printf(s, "            |  ");
 938                 if (!ret)
 939                         return TRACE_TYPE_PARTIAL_LINE;
 940         }
 941
 942         /* Indentation */
 943         if (depth > 0)
 944                 for (i = 0; i < (depth + 1) * TRACE_GRAPH_INDENT; i++) {
 945                         ret = trace_seq_printf(s, " ");
 946                         if (!ret)
 947                                 return TRACE_TYPE_PARTIAL_LINE;
 948                 }
 949
 950         /* The comment */
 951         ret = trace_seq_printf(s, "/* ");
 952         if (!ret)
 953                 return TRACE_TYPE_PARTIAL_LINE;
 954
 955         switch (iter->ent->type) {
 956         case TRACE_BPRINT:
 957                 ret = trace_print_bprintk_msg_only(iter);
 958                 if (ret != TRACE_TYPE_HANDLED)
 959                         return ret;
 960                 break;
 961         case TRACE_PRINT:
 962                 ret = trace_print_printk_msg_only(iter);
 963                 if (ret != TRACE_TYPE_HANDLED)
 964                         return ret;
 965                 break;
 966         default:
 967                 event = ftrace_find_event(ent->type);
 968                 if (!event)
 969                         return TRACE_TYPE_UNHANDLED;
 970
 971                 ret = event->trace(iter, sym_flags);
 972                 if (ret != TRACE_TYPE_HANDLED)
 973                         return ret;
 974         }
 975
 976         /* Strip ending newline */
 977         if (s->buffer[s->len - 1] == '\n') {
 978                 s->buffer[s->len - 1] = '\0';
 979                 s->len--;
 980         }
 981
 982         ret = trace_seq_printf(s, " */\n");
 983         if (!ret)
 984                 return TRACE_TYPE_PARTIAL_LINE;
 985
 986         return TRACE_TYPE_HANDLED;
 987 }
 988
 989
 990 enum print_line_t
 991 print_graph_function(struct trace_iterator *iter)
 992 {
 993         struct ftrace_graph_ent_entry *field;
 994         struct fgraph_data *data = iter->private;
 995         struct trace_entry *entry = iter->ent;
 996         struct trace_seq *s = &iter->seq;
 997         int cpu = iter->cpu;
 998         int ret;
 999
1000         if (data && per_cpu_ptr(data->cpu_data, cpu)->ignore) {
1001                 per_cpu_ptr(data->cpu_data, cpu)->ignore = 0;
1002                 return TRACE_TYPE_HANDLED;
1003         }
1004
1005         /*
1006          * If the last output failed, there's a possibility we need
1007          * to print out the missing entry which would never go out.
1008          */
1009         if (data && data->failed) {
1010                 field = &data->ent;
1011                 iter->cpu = data->cpu;
1012                 ret = print_graph_entry(field, s, iter);
1013                 if (ret == TRACE_TYPE_HANDLED && iter->cpu != cpu) {
1014                         per_cpu_ptr(data->cpu_data, iter->cpu)->ignore = 1;
1015                         ret = TRACE_TYPE_NO_CONSUME;
1016                 }
1017                 iter->cpu = cpu;
1018                 return ret;
1019         }
1020
1021         switch (entry->type) {
1022         case TRACE_GRAPH_ENT: {
1023                 /*
1024                  * print_graph_entry() may consume the current event,
1025                  * thus @field may become invalid, so we need to save it.
1026                  * sizeof(struct ftrace_graph_ent_entry) is very small,
1027                  * it can be safely saved at the stack.
1028                  */
1029                 struct ftrace_graph_ent_entry saved;
1030                 trace_assign_type(field, entry);
1031                 saved = *field;
1032                 return print_graph_entry(&saved, s, iter);
1033         }
1034         case TRACE_GRAPH_RET: {
1035                 struct ftrace_graph_ret_entry *field;
1036                 trace_assign_type(field, entry);
1037                 return print_graph_return(&field->ret, s, entry, iter);
1038         }
1039         default:
1040                 return print_graph_comment(s, entry, iter);
1041         }
1042
1043         return TRACE_TYPE_HANDLED;
1044 }
1045
1046 static void print_lat_header(struct seq_file *s)
1047 {
1048         static const char spaces[] = "                " /* 16 spaces */
1049                 "    "                                  /* 4 spaces */
1050                 "                 ";                    /* 17 spaces */
1051         int size = 0;
1052
1053         if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME)
1054                 size += 16;
1055         if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU)
1056                 size += 4;
1057         if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC)
1058                 size += 17;
1059
1060         seq_printf(s, "#%.*s  _-----=> irqs-off        \n", size, spaces);
1061         seq_printf(s, "#%.*s / _----=> need-resched    \n", size, spaces);
1062         seq_printf(s, "#%.*s| / _---=> hardirq/softirq \n", size, spaces);
1063         seq_printf(s, "#%.*s|| / _--=> preempt-depth   \n", size, spaces);
1064         seq_printf(s, "#%.*s||| / _-=> lock-depth      \n", size, spaces);
1065         seq_printf(s, "#%.*s|||| /                     \n", size, spaces);
1066 }
1067
1068 static void print_graph_headers(struct seq_file *s)
1069 {
1070         int lat = trace_flags & TRACE_ITER_LATENCY_FMT;
1071
1072         if (lat)
1073                 print_lat_header(s);
1074
1075         /* 1st line */
1076         seq_printf(s, "#");
1077         if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME)
1078                 seq_printf(s, "     TIME       ");
1079         if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU)
1080                 seq_printf(s, " CPU");
1081         if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC)
1082                 seq_printf(s, "  TASK/PID       ");
1083         if (lat)
1084                 seq_printf(s, "|||||");
1085         if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION)
1086                 seq_printf(s, "  DURATION   ");
1087         seq_printf(s, "               FUNCTION CALLS\n");
1088
1089         /* 2nd line */
1090         seq_printf(s, "#");
1091         if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME)
1092                 seq_printf(s, "      |         ");
1093         if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU)
1094                 seq_printf(s, " |  ");
1095         if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC)
1096                 seq_printf(s, "   |    |        ");
1097         if (lat)
1098                 seq_printf(s, "|||||");
1099         if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION)
1100                 seq_printf(s, "   |   |      ");
1101         seq_printf(s, "               |   |   |   |\n");
1102 }
1103
1104 static void graph_trace_open(struct trace_iterator *iter)
1105 {
1106         /* pid and depth on the last trace processed */
1107         struct fgraph_data *data;
1108         int cpu;
1109
1110         iter->private = NULL;
1111
1112         data = kzalloc(sizeof(*data), GFP_KERNEL);
1113         if (!data)
1114                 goto out_err;
1115
1116         data->cpu_data = alloc_percpu(struct fgraph_cpu_data);
1117         if (!data->cpu_data)
1118                 goto out_err_free;
1119
1120         for_each_possible_cpu(cpu) {
1121                 pid_t *pid = &(per_cpu_ptr(data->cpu_data, cpu)->last_pid);
1122                 int *depth = &(per_cpu_ptr(data->cpu_data, cpu)->depth);
1123                 int *ignore = &(per_cpu_ptr(data->cpu_data, cpu)->ignore);
1124                 *pid = -1;
1125                 *depth = 0;
1126                 *ignore = 0;
1127         }
1128
1129         iter->private = data;
1130
1131         return;
1132
1133  out_err_free:
1134         kfree(data);
1135  out_err:
1136         pr_warning("function graph tracer: not enough memory\n");
1137 }
1138
1139 static void graph_trace_close(struct trace_iterator *iter)
1140 {
1141         struct fgraph_data *data = iter->private;
1142
1143         if (data) {
1144                 free_percpu(data->cpu_data);
1145                 kfree(data);
1146         }
1147 }
1148
1149 static struct tracer graph_trace __read_mostly = {
1150         .name           = "function_graph",
1151         .open           = graph_trace_open,
1152         .pipe_open      = graph_trace_open,
1153         .close          = graph_trace_close,
1154         .pipe_close     = graph_trace_close,
1155         .wait_pipe      = poll_wait_pipe,
1156         .init           = graph_trace_init,
1157         .reset          = graph_trace_reset,
1158         .print_line     = print_graph_function,
1159         .print_header   = print_graph_headers,
1160         .flags          = &tracer_flags,
1161 #ifdef CONFIG_FTRACE_SELFTEST
1162         .selftest       = trace_selftest_startup_function_graph,
1163 #endif
1164 };
1165
1166 static __init int init_graph_trace(void)
1167 {
1168         max_bytes_for_cpu = snprintf(NULL, 0, "%d", nr_cpu_ids - 1);
1169
1170         return register_tracer(&graph_trace);
1171 }
1172
1173 device_initcall(init_graph_trace);