Merge branch 'master' into percpu
[sfrench/cifs-2.6.git] / kernel / trace / trace_functions_graph.c
1 /*
2  *
3  * Function graph tracer.
4  * Copyright (c) 2008-2009 Frederic Weisbecker <fweisbec@gmail.com>
5  * Mostly borrowed from function tracer which
6  * is Copyright (c) Steven Rostedt <srostedt@redhat.com>
7  *
8  */
9 #include <linux/debugfs.h>
10 #include <linux/uaccess.h>
11 #include <linux/ftrace.h>
12 #include <linux/fs.h>
13
14 #include "trace.h"
15 #include "trace_output.h"
16
17 struct fgraph_cpu_data {
18         pid_t           last_pid;
19         int             depth;
20         int             ignore;
21 };
22
23 struct fgraph_data {
24         struct fgraph_cpu_data          *cpu_data;
25
26         /* Place to preserve last processed entry. */
27         struct ftrace_graph_ent_entry   ent;
28         struct ftrace_graph_ret_entry   ret;
29         int                             failed;
30         int                             cpu;
31 };
32
33 #define TRACE_GRAPH_INDENT      2
34
35 /* Flag options */
36 #define TRACE_GRAPH_PRINT_OVERRUN       0x1
37 #define TRACE_GRAPH_PRINT_CPU           0x2
38 #define TRACE_GRAPH_PRINT_OVERHEAD      0x4
39 #define TRACE_GRAPH_PRINT_PROC          0x8
40 #define TRACE_GRAPH_PRINT_DURATION      0x10
41 #define TRACE_GRAPH_PRINT_ABS_TIME      0X20
42
43 static struct tracer_opt trace_opts[] = {
44         /* Display overruns? (for self-debug purpose) */
45         { TRACER_OPT(funcgraph-overrun, TRACE_GRAPH_PRINT_OVERRUN) },
46         /* Display CPU ? */
47         { TRACER_OPT(funcgraph-cpu, TRACE_GRAPH_PRINT_CPU) },
48         /* Display Overhead ? */
49         { TRACER_OPT(funcgraph-overhead, TRACE_GRAPH_PRINT_OVERHEAD) },
50         /* Display proc name/pid */
51         { TRACER_OPT(funcgraph-proc, TRACE_GRAPH_PRINT_PROC) },
52         /* Display duration of execution */
53         { TRACER_OPT(funcgraph-duration, TRACE_GRAPH_PRINT_DURATION) },
54         /* Display absolute time of an entry */
55         { TRACER_OPT(funcgraph-abstime, TRACE_GRAPH_PRINT_ABS_TIME) },
56         { } /* Empty entry */
57 };
58
59 static struct tracer_flags tracer_flags = {
60         /* Don't display overruns and proc by default */
61         .val = TRACE_GRAPH_PRINT_CPU | TRACE_GRAPH_PRINT_OVERHEAD |
62                TRACE_GRAPH_PRINT_DURATION,
63         .opts = trace_opts
64 };
65
66 static struct trace_array *graph_array;
67
68
69 /* Add a function return address to the trace stack on thread info.*/
70 int
71 ftrace_push_return_trace(unsigned long ret, unsigned long func, int *depth,
72                          unsigned long frame_pointer)
73 {
74         unsigned long long calltime;
75         int index;
76
77         if (!current->ret_stack)
78                 return -EBUSY;
79
80         /*
81          * We must make sure the ret_stack is tested before we read
82          * anything else.
83          */
84         smp_rmb();
85
86         /* The return trace stack is full */
87         if (current->curr_ret_stack == FTRACE_RETFUNC_DEPTH - 1) {
88                 atomic_inc(&current->trace_overrun);
89                 return -EBUSY;
90         }
91
92         calltime = trace_clock_local();
93
94         index = ++current->curr_ret_stack;
95         barrier();
96         current->ret_stack[index].ret = ret;
97         current->ret_stack[index].func = func;
98         current->ret_stack[index].calltime = calltime;
99         current->ret_stack[index].subtime = 0;
100         current->ret_stack[index].fp = frame_pointer;
101         *depth = index;
102
103         return 0;
104 }
105
106 /* Retrieve a function return address to the trace stack on thread info.*/
107 static void
108 ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret,
109                         unsigned long frame_pointer)
110 {
111         int index;
112
113         index = current->curr_ret_stack;
114
115         if (unlikely(index < 0)) {
116                 ftrace_graph_stop();
117                 WARN_ON(1);
118                 /* Might as well panic, otherwise we have no where to go */
119                 *ret = (unsigned long)panic;
120                 return;
121         }
122
123 #ifdef CONFIG_HAVE_FUNCTION_GRAPH_FP_TEST
124         /*
125          * The arch may choose to record the frame pointer used
126          * and check it here to make sure that it is what we expect it
127          * to be. If gcc does not set the place holder of the return
128          * address in the frame pointer, and does a copy instead, then
129          * the function graph trace will fail. This test detects this
130          * case.
131          *
132          * Currently, x86_32 with optimize for size (-Os) makes the latest
133          * gcc do the above.
134          */
135         if (unlikely(current->ret_stack[index].fp != frame_pointer)) {
136                 ftrace_graph_stop();
137                 WARN(1, "Bad frame pointer: expected %lx, received %lx\n"
138                      "  from func %ps return to %lx\n",
139                      current->ret_stack[index].fp,
140                      frame_pointer,
141                      (void *)current->ret_stack[index].func,
142                      current->ret_stack[index].ret);
143                 *ret = (unsigned long)panic;
144                 return;
145         }
146 #endif
147
148         *ret = current->ret_stack[index].ret;
149         trace->func = current->ret_stack[index].func;
150         trace->calltime = current->ret_stack[index].calltime;
151         trace->overrun = atomic_read(&current->trace_overrun);
152         trace->depth = index;
153 }
154
155 /*
156  * Send the trace to the ring-buffer.
157  * @return the original return address.
158  */
159 unsigned long ftrace_return_to_handler(unsigned long frame_pointer)
160 {
161         struct ftrace_graph_ret trace;
162         unsigned long ret;
163
164         ftrace_pop_return_trace(&trace, &ret, frame_pointer);
165         trace.rettime = trace_clock_local();
166         ftrace_graph_return(&trace);
167         barrier();
168         current->curr_ret_stack--;
169
170         if (unlikely(!ret)) {
171                 ftrace_graph_stop();
172                 WARN_ON(1);
173                 /* Might as well panic. What else to do? */
174                 ret = (unsigned long)panic;
175         }
176
177         return ret;
178 }
179
180 static int __trace_graph_entry(struct trace_array *tr,
181                                 struct ftrace_graph_ent *trace,
182                                 unsigned long flags,
183                                 int pc)
184 {
185         struct ftrace_event_call *call = &event_funcgraph_entry;
186         struct ring_buffer_event *event;
187         struct ring_buffer *buffer = tr->buffer;
188         struct ftrace_graph_ent_entry *entry;
189
190         if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
191                 return 0;
192
193         event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_ENT,
194                                           sizeof(*entry), flags, pc);
195         if (!event)
196                 return 0;
197         entry   = ring_buffer_event_data(event);
198         entry->graph_ent                        = *trace;
199         if (!filter_current_check_discard(buffer, call, entry, event))
200                 ring_buffer_unlock_commit(buffer, event);
201
202         return 1;
203 }
204
205 int trace_graph_entry(struct ftrace_graph_ent *trace)
206 {
207         struct trace_array *tr = graph_array;
208         struct trace_array_cpu *data;
209         unsigned long flags;
210         long disabled;
211         int ret;
212         int cpu;
213         int pc;
214
215         if (unlikely(!tr))
216                 return 0;
217
218         if (!ftrace_trace_task(current))
219                 return 0;
220
221         if (!ftrace_graph_addr(trace->func))
222                 return 0;
223
224         local_irq_save(flags);
225         cpu = raw_smp_processor_id();
226         data = tr->data[cpu];
227         disabled = atomic_inc_return(&data->disabled);
228         if (likely(disabled == 1)) {
229                 pc = preempt_count();
230                 ret = __trace_graph_entry(tr, trace, flags, pc);
231         } else {
232                 ret = 0;
233         }
234         /* Only do the atomic if it is not already set */
235         if (!test_tsk_trace_graph(current))
236                 set_tsk_trace_graph(current);
237
238         atomic_dec(&data->disabled);
239         local_irq_restore(flags);
240
241         return ret;
242 }
243
244 static void __trace_graph_return(struct trace_array *tr,
245                                 struct ftrace_graph_ret *trace,
246                                 unsigned long flags,
247                                 int pc)
248 {
249         struct ftrace_event_call *call = &event_funcgraph_exit;
250         struct ring_buffer_event *event;
251         struct ring_buffer *buffer = tr->buffer;
252         struct ftrace_graph_ret_entry *entry;
253
254         if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
255                 return;
256
257         event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RET,
258                                           sizeof(*entry), flags, pc);
259         if (!event)
260                 return;
261         entry   = ring_buffer_event_data(event);
262         entry->ret                              = *trace;
263         if (!filter_current_check_discard(buffer, call, entry, event))
264                 ring_buffer_unlock_commit(buffer, event);
265 }
266
267 void trace_graph_return(struct ftrace_graph_ret *trace)
268 {
269         struct trace_array *tr = graph_array;
270         struct trace_array_cpu *data;
271         unsigned long flags;
272         long disabled;
273         int cpu;
274         int pc;
275
276         local_irq_save(flags);
277         cpu = raw_smp_processor_id();
278         data = tr->data[cpu];
279         disabled = atomic_inc_return(&data->disabled);
280         if (likely(disabled == 1)) {
281                 pc = preempt_count();
282                 __trace_graph_return(tr, trace, flags, pc);
283         }
284         if (!trace->depth)
285                 clear_tsk_trace_graph(current);
286         atomic_dec(&data->disabled);
287         local_irq_restore(flags);
288 }
289
290 static int graph_trace_init(struct trace_array *tr)
291 {
292         int ret;
293
294         graph_array = tr;
295         ret = register_ftrace_graph(&trace_graph_return,
296                                     &trace_graph_entry);
297         if (ret)
298                 return ret;
299         tracing_start_cmdline_record();
300
301         return 0;
302 }
303
304 void set_graph_array(struct trace_array *tr)
305 {
306         graph_array = tr;
307 }
308
309 static void graph_trace_reset(struct trace_array *tr)
310 {
311         tracing_stop_cmdline_record();
312         unregister_ftrace_graph();
313 }
314
315 static int max_bytes_for_cpu;
316
317 static enum print_line_t
318 print_graph_cpu(struct trace_seq *s, int cpu)
319 {
320         int ret;
321
322         /*
323          * Start with a space character - to make it stand out
324          * to the right a bit when trace output is pasted into
325          * email:
326          */
327         ret = trace_seq_printf(s, " %*d) ", max_bytes_for_cpu, cpu);
328         if (!ret)
329                 return TRACE_TYPE_PARTIAL_LINE;
330
331         return TRACE_TYPE_HANDLED;
332 }
333
334 #define TRACE_GRAPH_PROCINFO_LENGTH     14
335
336 static enum print_line_t
337 print_graph_proc(struct trace_seq *s, pid_t pid)
338 {
339         char comm[TASK_COMM_LEN];
340         /* sign + log10(MAX_INT) + '\0' */
341         char pid_str[11];
342         int spaces = 0;
343         int ret;
344         int len;
345         int i;
346
347         trace_find_cmdline(pid, comm);
348         comm[7] = '\0';
349         sprintf(pid_str, "%d", pid);
350
351         /* 1 stands for the "-" character */
352         len = strlen(comm) + strlen(pid_str) + 1;
353
354         if (len < TRACE_GRAPH_PROCINFO_LENGTH)
355                 spaces = TRACE_GRAPH_PROCINFO_LENGTH - len;
356
357         /* First spaces to align center */
358         for (i = 0; i < spaces / 2; i++) {
359                 ret = trace_seq_printf(s, " ");
360                 if (!ret)
361                         return TRACE_TYPE_PARTIAL_LINE;
362         }
363
364         ret = trace_seq_printf(s, "%s-%s", comm, pid_str);
365         if (!ret)
366                 return TRACE_TYPE_PARTIAL_LINE;
367
368         /* Last spaces to align center */
369         for (i = 0; i < spaces - (spaces / 2); i++) {
370                 ret = trace_seq_printf(s, " ");
371                 if (!ret)
372                         return TRACE_TYPE_PARTIAL_LINE;
373         }
374         return TRACE_TYPE_HANDLED;
375 }
376
377
378 static enum print_line_t
379 print_graph_lat_fmt(struct trace_seq *s, struct trace_entry *entry)
380 {
381         if (!trace_seq_putc(s, ' '))
382                 return 0;
383
384         return trace_print_lat_fmt(s, entry);
385 }
386
387 /* If the pid changed since the last trace, output this event */
388 static enum print_line_t
389 verif_pid(struct trace_seq *s, pid_t pid, int cpu, struct fgraph_data *data)
390 {
391         pid_t prev_pid;
392         pid_t *last_pid;
393         int ret;
394
395         if (!data)
396                 return TRACE_TYPE_HANDLED;
397
398         last_pid = &(per_cpu_ptr(data->cpu_data, cpu)->last_pid);
399
400         if (*last_pid == pid)
401                 return TRACE_TYPE_HANDLED;
402
403         prev_pid = *last_pid;
404         *last_pid = pid;
405
406         if (prev_pid == -1)
407                 return TRACE_TYPE_HANDLED;
408 /*
409  * Context-switch trace line:
410
411  ------------------------------------------
412  | 1)  migration/0--1  =>  sshd-1755
413  ------------------------------------------
414
415  */
416         ret = trace_seq_printf(s,
417                 " ------------------------------------------\n");
418         if (!ret)
419                 return TRACE_TYPE_PARTIAL_LINE;
420
421         ret = print_graph_cpu(s, cpu);
422         if (ret == TRACE_TYPE_PARTIAL_LINE)
423                 return TRACE_TYPE_PARTIAL_LINE;
424
425         ret = print_graph_proc(s, prev_pid);
426         if (ret == TRACE_TYPE_PARTIAL_LINE)
427                 return TRACE_TYPE_PARTIAL_LINE;
428
429         ret = trace_seq_printf(s, " => ");
430         if (!ret)
431                 return TRACE_TYPE_PARTIAL_LINE;
432
433         ret = print_graph_proc(s, pid);
434         if (ret == TRACE_TYPE_PARTIAL_LINE)
435                 return TRACE_TYPE_PARTIAL_LINE;
436
437         ret = trace_seq_printf(s,
438                 "\n ------------------------------------------\n\n");
439         if (!ret)
440                 return TRACE_TYPE_PARTIAL_LINE;
441
442         return TRACE_TYPE_HANDLED;
443 }
444
445 static struct ftrace_graph_ret_entry *
446 get_return_for_leaf(struct trace_iterator *iter,
447                 struct ftrace_graph_ent_entry *curr)
448 {
449         struct fgraph_data *data = iter->private;
450         struct ring_buffer_iter *ring_iter = NULL;
451         struct ring_buffer_event *event;
452         struct ftrace_graph_ret_entry *next;
453
454         /*
455          * If the previous output failed to write to the seq buffer,
456          * then we just reuse the data from before.
457          */
458         if (data && data->failed) {
459                 curr = &data->ent;
460                 next = &data->ret;
461         } else {
462
463                 ring_iter = iter->buffer_iter[iter->cpu];
464
465                 /* First peek to compare current entry and the next one */
466                 if (ring_iter)
467                         event = ring_buffer_iter_peek(ring_iter, NULL);
468                 else {
469                         /*
470                          * We need to consume the current entry to see
471                          * the next one.
472                          */
473                         ring_buffer_consume(iter->tr->buffer, iter->cpu, NULL);
474                         event = ring_buffer_peek(iter->tr->buffer, iter->cpu,
475                                                  NULL);
476                 }
477
478                 if (!event)
479                         return NULL;
480
481                 next = ring_buffer_event_data(event);
482
483                 if (data) {
484                         /*
485                          * Save current and next entries for later reference
486                          * if the output fails.
487                          */
488                         data->ent = *curr;
489                         data->ret = *next;
490                 }
491         }
492
493         if (next->ent.type != TRACE_GRAPH_RET)
494                 return NULL;
495
496         if (curr->ent.pid != next->ent.pid ||
497                         curr->graph_ent.func != next->ret.func)
498                 return NULL;
499
500         /* this is a leaf, now advance the iterator */
501         if (ring_iter)
502                 ring_buffer_read(ring_iter, NULL);
503
504         return next;
505 }
506
507 /* Signal a overhead of time execution to the output */
508 static int
509 print_graph_overhead(unsigned long long duration, struct trace_seq *s)
510 {
511         /* If duration disappear, we don't need anything */
512         if (!(tracer_flags.val & TRACE_GRAPH_PRINT_DURATION))
513                 return 1;
514
515         /* Non nested entry or return */
516         if (duration == -1)
517                 return trace_seq_printf(s, "  ");
518
519         if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) {
520                 /* Duration exceeded 100 msecs */
521                 if (duration > 100000ULL)
522                         return trace_seq_printf(s, "! ");
523
524                 /* Duration exceeded 10 msecs */
525                 if (duration > 10000ULL)
526                         return trace_seq_printf(s, "+ ");
527         }
528
529         return trace_seq_printf(s, "  ");
530 }
531
532 static int print_graph_abs_time(u64 t, struct trace_seq *s)
533 {
534         unsigned long usecs_rem;
535
536         usecs_rem = do_div(t, NSEC_PER_SEC);
537         usecs_rem /= 1000;
538
539         return trace_seq_printf(s, "%5lu.%06lu |  ",
540                         (unsigned long)t, usecs_rem);
541 }
542
543 static enum print_line_t
544 print_graph_irq(struct trace_iterator *iter, unsigned long addr,
545                 enum trace_type type, int cpu, pid_t pid)
546 {
547         int ret;
548         struct trace_seq *s = &iter->seq;
549
550         if (addr < (unsigned long)__irqentry_text_start ||
551                 addr >= (unsigned long)__irqentry_text_end)
552                 return TRACE_TYPE_UNHANDLED;
553
554         /* Absolute time */
555         if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME) {
556                 ret = print_graph_abs_time(iter->ts, s);
557                 if (!ret)
558                         return TRACE_TYPE_PARTIAL_LINE;
559         }
560
561         /* Cpu */
562         if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) {
563                 ret = print_graph_cpu(s, cpu);
564                 if (ret == TRACE_TYPE_PARTIAL_LINE)
565                         return TRACE_TYPE_PARTIAL_LINE;
566         }
567
568         /* Proc */
569         if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC) {
570                 ret = print_graph_proc(s, pid);
571                 if (ret == TRACE_TYPE_PARTIAL_LINE)
572                         return TRACE_TYPE_PARTIAL_LINE;
573                 ret = trace_seq_printf(s, " | ");
574                 if (!ret)
575                         return TRACE_TYPE_PARTIAL_LINE;
576         }
577
578         /* No overhead */
579         ret = print_graph_overhead(-1, s);
580         if (!ret)
581                 return TRACE_TYPE_PARTIAL_LINE;
582
583         if (type == TRACE_GRAPH_ENT)
584                 ret = trace_seq_printf(s, "==========>");
585         else
586                 ret = trace_seq_printf(s, "<==========");
587
588         if (!ret)
589                 return TRACE_TYPE_PARTIAL_LINE;
590
591         /* Don't close the duration column if haven't one */
592         if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION)
593                 trace_seq_printf(s, " |");
594         ret = trace_seq_printf(s, "\n");
595
596         if (!ret)
597                 return TRACE_TYPE_PARTIAL_LINE;
598         return TRACE_TYPE_HANDLED;
599 }
600
601 enum print_line_t
602 trace_print_graph_duration(unsigned long long duration, struct trace_seq *s)
603 {
604         unsigned long nsecs_rem = do_div(duration, 1000);
605         /* log10(ULONG_MAX) + '\0' */
606         char msecs_str[21];
607         char nsecs_str[5];
608         int ret, len;
609         int i;
610
611         sprintf(msecs_str, "%lu", (unsigned long) duration);
612
613         /* Print msecs */
614         ret = trace_seq_printf(s, "%s", msecs_str);
615         if (!ret)
616                 return TRACE_TYPE_PARTIAL_LINE;
617
618         len = strlen(msecs_str);
619
620         /* Print nsecs (we don't want to exceed 7 numbers) */
621         if (len < 7) {
622                 snprintf(nsecs_str, 8 - len, "%03lu", nsecs_rem);
623                 ret = trace_seq_printf(s, ".%s", nsecs_str);
624                 if (!ret)
625                         return TRACE_TYPE_PARTIAL_LINE;
626                 len += strlen(nsecs_str);
627         }
628
629         ret = trace_seq_printf(s, " us ");
630         if (!ret)
631                 return TRACE_TYPE_PARTIAL_LINE;
632
633         /* Print remaining spaces to fit the row's width */
634         for (i = len; i < 7; i++) {
635                 ret = trace_seq_printf(s, " ");
636                 if (!ret)
637                         return TRACE_TYPE_PARTIAL_LINE;
638         }
639         return TRACE_TYPE_HANDLED;
640 }
641
642 static enum print_line_t
643 print_graph_duration(unsigned long long duration, struct trace_seq *s)
644 {
645         int ret;
646
647         ret = trace_print_graph_duration(duration, s);
648         if (ret != TRACE_TYPE_HANDLED)
649                 return ret;
650
651         ret = trace_seq_printf(s, "|  ");
652         if (!ret)
653                 return TRACE_TYPE_PARTIAL_LINE;
654
655         return TRACE_TYPE_HANDLED;
656 }
657
658 /* Case of a leaf function on its call entry */
659 static enum print_line_t
660 print_graph_entry_leaf(struct trace_iterator *iter,
661                 struct ftrace_graph_ent_entry *entry,
662                 struct ftrace_graph_ret_entry *ret_entry, struct trace_seq *s)
663 {
664         struct fgraph_data *data = iter->private;
665         struct ftrace_graph_ret *graph_ret;
666         struct ftrace_graph_ent *call;
667         unsigned long long duration;
668         int ret;
669         int i;
670
671         graph_ret = &ret_entry->ret;
672         call = &entry->graph_ent;
673         duration = graph_ret->rettime - graph_ret->calltime;
674
675         if (data) {
676                 int cpu = iter->cpu;
677                 int *depth = &(per_cpu_ptr(data->cpu_data, cpu)->depth);
678
679                 /*
680                  * Comments display at + 1 to depth. Since
681                  * this is a leaf function, keep the comments
682                  * equal to this depth.
683                  */
684                 *depth = call->depth - 1;
685         }
686
687         /* Overhead */
688         ret = print_graph_overhead(duration, s);
689         if (!ret)
690                 return TRACE_TYPE_PARTIAL_LINE;
691
692         /* Duration */
693         if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION) {
694                 ret = print_graph_duration(duration, s);
695                 if (ret == TRACE_TYPE_PARTIAL_LINE)
696                         return TRACE_TYPE_PARTIAL_LINE;
697         }
698
699         /* Function */
700         for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) {
701                 ret = trace_seq_printf(s, " ");
702                 if (!ret)
703                         return TRACE_TYPE_PARTIAL_LINE;
704         }
705
706         ret = trace_seq_printf(s, "%ps();\n", (void *)call->func);
707         if (!ret)
708                 return TRACE_TYPE_PARTIAL_LINE;
709
710         return TRACE_TYPE_HANDLED;
711 }
712
713 static enum print_line_t
714 print_graph_entry_nested(struct trace_iterator *iter,
715                          struct ftrace_graph_ent_entry *entry,
716                          struct trace_seq *s, int cpu)
717 {
718         struct ftrace_graph_ent *call = &entry->graph_ent;
719         struct fgraph_data *data = iter->private;
720         int ret;
721         int i;
722
723         if (data) {
724                 int cpu = iter->cpu;
725                 int *depth = &(per_cpu_ptr(data->cpu_data, cpu)->depth);
726
727                 *depth = call->depth;
728         }
729
730         /* No overhead */
731         ret = print_graph_overhead(-1, s);
732         if (!ret)
733                 return TRACE_TYPE_PARTIAL_LINE;
734
735         /* No time */
736         if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION) {
737                 ret = trace_seq_printf(s, "            |  ");
738                 if (!ret)
739                         return TRACE_TYPE_PARTIAL_LINE;
740         }
741
742         /* Function */
743         for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) {
744                 ret = trace_seq_printf(s, " ");
745                 if (!ret)
746                         return TRACE_TYPE_PARTIAL_LINE;
747         }
748
749         ret = trace_seq_printf(s, "%ps() {\n", (void *)call->func);
750         if (!ret)
751                 return TRACE_TYPE_PARTIAL_LINE;
752
753         /*
754          * we already consumed the current entry to check the next one
755          * and see if this is a leaf.
756          */
757         return TRACE_TYPE_NO_CONSUME;
758 }
759
760 static enum print_line_t
761 print_graph_prologue(struct trace_iterator *iter, struct trace_seq *s,
762                      int type, unsigned long addr)
763 {
764         struct fgraph_data *data = iter->private;
765         struct trace_entry *ent = iter->ent;
766         int cpu = iter->cpu;
767         int ret;
768
769         /* Pid */
770         if (verif_pid(s, ent->pid, cpu, data) == TRACE_TYPE_PARTIAL_LINE)
771                 return TRACE_TYPE_PARTIAL_LINE;
772
773         if (type) {
774                 /* Interrupt */
775                 ret = print_graph_irq(iter, addr, type, cpu, ent->pid);
776                 if (ret == TRACE_TYPE_PARTIAL_LINE)
777                         return TRACE_TYPE_PARTIAL_LINE;
778         }
779
780         /* Absolute time */
781         if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME) {
782                 ret = print_graph_abs_time(iter->ts, s);
783                 if (!ret)
784                         return TRACE_TYPE_PARTIAL_LINE;
785         }
786
787         /* Cpu */
788         if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) {
789                 ret = print_graph_cpu(s, cpu);
790                 if (ret == TRACE_TYPE_PARTIAL_LINE)
791                         return TRACE_TYPE_PARTIAL_LINE;
792         }
793
794         /* Proc */
795         if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC) {
796                 ret = print_graph_proc(s, ent->pid);
797                 if (ret == TRACE_TYPE_PARTIAL_LINE)
798                         return TRACE_TYPE_PARTIAL_LINE;
799
800                 ret = trace_seq_printf(s, " | ");
801                 if (!ret)
802                         return TRACE_TYPE_PARTIAL_LINE;
803         }
804
805         /* Latency format */
806         if (trace_flags & TRACE_ITER_LATENCY_FMT) {
807                 ret = print_graph_lat_fmt(s, ent);
808                 if (ret == TRACE_TYPE_PARTIAL_LINE)
809                         return TRACE_TYPE_PARTIAL_LINE;
810         }
811
812         return 0;
813 }
814
815 static enum print_line_t
816 print_graph_entry(struct ftrace_graph_ent_entry *field, struct trace_seq *s,
817                         struct trace_iterator *iter)
818 {
819         struct fgraph_data *data = iter->private;
820         struct ftrace_graph_ent *call = &field->graph_ent;
821         struct ftrace_graph_ret_entry *leaf_ret;
822         static enum print_line_t ret;
823         int cpu = iter->cpu;
824
825         if (print_graph_prologue(iter, s, TRACE_GRAPH_ENT, call->func))
826                 return TRACE_TYPE_PARTIAL_LINE;
827
828         leaf_ret = get_return_for_leaf(iter, field);
829         if (leaf_ret)
830                 ret = print_graph_entry_leaf(iter, field, leaf_ret, s);
831         else
832                 ret = print_graph_entry_nested(iter, field, s, cpu);
833
834         if (data) {
835                 /*
836                  * If we failed to write our output, then we need to make
837                  * note of it. Because we already consumed our entry.
838                  */
839                 if (s->full) {
840                         data->failed = 1;
841                         data->cpu = cpu;
842                 } else
843                         data->failed = 0;
844         }
845
846         return ret;
847 }
848
849 static enum print_line_t
850 print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s,
851                    struct trace_entry *ent, struct trace_iterator *iter)
852 {
853         unsigned long long duration = trace->rettime - trace->calltime;
854         struct fgraph_data *data = iter->private;
855         pid_t pid = ent->pid;
856         int cpu = iter->cpu;
857         int ret;
858         int i;
859
860         if (data) {
861                 int cpu = iter->cpu;
862                 int *depth = &(per_cpu_ptr(data->cpu_data, cpu)->depth);
863
864                 /*
865                  * Comments display at + 1 to depth. This is the
866                  * return from a function, we now want the comments
867                  * to display at the same level of the bracket.
868                  */
869                 *depth = trace->depth - 1;
870         }
871
872         if (print_graph_prologue(iter, s, 0, 0))
873                 return TRACE_TYPE_PARTIAL_LINE;
874
875         /* Overhead */
876         ret = print_graph_overhead(duration, s);
877         if (!ret)
878                 return TRACE_TYPE_PARTIAL_LINE;
879
880         /* Duration */
881         if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION) {
882                 ret = print_graph_duration(duration, s);
883                 if (ret == TRACE_TYPE_PARTIAL_LINE)
884                         return TRACE_TYPE_PARTIAL_LINE;
885         }
886
887         /* Closing brace */
888         for (i = 0; i < trace->depth * TRACE_GRAPH_INDENT; i++) {
889                 ret = trace_seq_printf(s, " ");
890                 if (!ret)
891                         return TRACE_TYPE_PARTIAL_LINE;
892         }
893
894         ret = trace_seq_printf(s, "}\n");
895         if (!ret)
896                 return TRACE_TYPE_PARTIAL_LINE;
897
898         /* Overrun */
899         if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERRUN) {
900                 ret = trace_seq_printf(s, " (Overruns: %lu)\n",
901                                         trace->overrun);
902                 if (!ret)
903                         return TRACE_TYPE_PARTIAL_LINE;
904         }
905
906         ret = print_graph_irq(iter, trace->func, TRACE_GRAPH_RET, cpu, pid);
907         if (ret == TRACE_TYPE_PARTIAL_LINE)
908                 return TRACE_TYPE_PARTIAL_LINE;
909
910         return TRACE_TYPE_HANDLED;
911 }
912
913 static enum print_line_t
914 print_graph_comment(struct trace_seq *s,  struct trace_entry *ent,
915                     struct trace_iterator *iter)
916 {
917         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
918         struct fgraph_data *data = iter->private;
919         struct trace_event *event;
920         int depth = 0;
921         int ret;
922         int i;
923
924         if (data)
925                 depth = per_cpu_ptr(data->cpu_data, iter->cpu)->depth;
926
927         if (print_graph_prologue(iter, s, 0, 0))
928                 return TRACE_TYPE_PARTIAL_LINE;
929
930         /* No overhead */
931         ret = print_graph_overhead(-1, s);
932         if (!ret)
933                 return TRACE_TYPE_PARTIAL_LINE;
934
935         /* No time */
936         if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION) {
937                 ret = trace_seq_printf(s, "            |  ");
938                 if (!ret)
939                         return TRACE_TYPE_PARTIAL_LINE;
940         }
941
942         /* Indentation */
943         if (depth > 0)
944                 for (i = 0; i < (depth + 1) * TRACE_GRAPH_INDENT; i++) {
945                         ret = trace_seq_printf(s, " ");
946                         if (!ret)
947                                 return TRACE_TYPE_PARTIAL_LINE;
948                 }
949
950         /* The comment */
951         ret = trace_seq_printf(s, "/* ");
952         if (!ret)
953                 return TRACE_TYPE_PARTIAL_LINE;
954
955         switch (iter->ent->type) {
956         case TRACE_BPRINT:
957                 ret = trace_print_bprintk_msg_only(iter);
958                 if (ret != TRACE_TYPE_HANDLED)
959                         return ret;
960                 break;
961         case TRACE_PRINT:
962                 ret = trace_print_printk_msg_only(iter);
963                 if (ret != TRACE_TYPE_HANDLED)
964                         return ret;
965                 break;
966         default:
967                 event = ftrace_find_event(ent->type);
968                 if (!event)
969                         return TRACE_TYPE_UNHANDLED;
970
971                 ret = event->trace(iter, sym_flags);
972                 if (ret != TRACE_TYPE_HANDLED)
973                         return ret;
974         }
975
976         /* Strip ending newline */
977         if (s->buffer[s->len - 1] == '\n') {
978                 s->buffer[s->len - 1] = '\0';
979                 s->len--;
980         }
981
982         ret = trace_seq_printf(s, " */\n");
983         if (!ret)
984                 return TRACE_TYPE_PARTIAL_LINE;
985
986         return TRACE_TYPE_HANDLED;
987 }
988
989
990 enum print_line_t
991 print_graph_function(struct trace_iterator *iter)
992 {
993         struct ftrace_graph_ent_entry *field;
994         struct fgraph_data *data = iter->private;
995         struct trace_entry *entry = iter->ent;
996         struct trace_seq *s = &iter->seq;
997         int cpu = iter->cpu;
998         int ret;
999
1000         if (data && per_cpu_ptr(data->cpu_data, cpu)->ignore) {
1001                 per_cpu_ptr(data->cpu_data, cpu)->ignore = 0;
1002                 return TRACE_TYPE_HANDLED;
1003         }
1004
1005         /*
1006          * If the last output failed, there's a possibility we need
1007          * to print out the missing entry which would never go out.
1008          */
1009         if (data && data->failed) {
1010                 field = &data->ent;
1011                 iter->cpu = data->cpu;
1012                 ret = print_graph_entry(field, s, iter);
1013                 if (ret == TRACE_TYPE_HANDLED && iter->cpu != cpu) {
1014                         per_cpu_ptr(data->cpu_data, iter->cpu)->ignore = 1;
1015                         ret = TRACE_TYPE_NO_CONSUME;
1016                 }
1017                 iter->cpu = cpu;
1018                 return ret;
1019         }
1020
1021         switch (entry->type) {
1022         case TRACE_GRAPH_ENT: {
1023                 /*
1024                  * print_graph_entry() may consume the current event,
1025                  * thus @field may become invalid, so we need to save it.
1026                  * sizeof(struct ftrace_graph_ent_entry) is very small,
1027                  * it can be safely saved at the stack.
1028                  */
1029                 struct ftrace_graph_ent_entry saved;
1030                 trace_assign_type(field, entry);
1031                 saved = *field;
1032                 return print_graph_entry(&saved, s, iter);
1033         }
1034         case TRACE_GRAPH_RET: {
1035                 struct ftrace_graph_ret_entry *field;
1036                 trace_assign_type(field, entry);
1037                 return print_graph_return(&field->ret, s, entry, iter);
1038         }
1039         default:
1040                 return print_graph_comment(s, entry, iter);
1041         }
1042
1043         return TRACE_TYPE_HANDLED;
1044 }
1045
1046 static void print_lat_header(struct seq_file *s)
1047 {
1048         static const char spaces[] = "                " /* 16 spaces */
1049                 "    "                                  /* 4 spaces */
1050                 "                 ";                    /* 17 spaces */
1051         int size = 0;
1052
1053         if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME)
1054                 size += 16;
1055         if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU)
1056                 size += 4;
1057         if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC)
1058                 size += 17;
1059
1060         seq_printf(s, "#%.*s  _-----=> irqs-off        \n", size, spaces);
1061         seq_printf(s, "#%.*s / _----=> need-resched    \n", size, spaces);
1062         seq_printf(s, "#%.*s| / _---=> hardirq/softirq \n", size, spaces);
1063         seq_printf(s, "#%.*s|| / _--=> preempt-depth   \n", size, spaces);
1064         seq_printf(s, "#%.*s||| / _-=> lock-depth      \n", size, spaces);
1065         seq_printf(s, "#%.*s|||| /                     \n", size, spaces);
1066 }
1067
1068 static void print_graph_headers(struct seq_file *s)
1069 {
1070         int lat = trace_flags & TRACE_ITER_LATENCY_FMT;
1071
1072         if (lat)
1073                 print_lat_header(s);
1074
1075         /* 1st line */
1076         seq_printf(s, "#");
1077         if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME)
1078                 seq_printf(s, "     TIME       ");
1079         if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU)
1080                 seq_printf(s, " CPU");
1081         if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC)
1082                 seq_printf(s, "  TASK/PID       ");
1083         if (lat)
1084                 seq_printf(s, "|||||");
1085         if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION)
1086                 seq_printf(s, "  DURATION   ");
1087         seq_printf(s, "               FUNCTION CALLS\n");
1088
1089         /* 2nd line */
1090         seq_printf(s, "#");
1091         if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME)
1092                 seq_printf(s, "      |         ");
1093         if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU)
1094                 seq_printf(s, " |  ");
1095         if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC)
1096                 seq_printf(s, "   |    |        ");
1097         if (lat)
1098                 seq_printf(s, "|||||");
1099         if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION)
1100                 seq_printf(s, "   |   |      ");
1101         seq_printf(s, "               |   |   |   |\n");
1102 }
1103
1104 static void graph_trace_open(struct trace_iterator *iter)
1105 {
1106         /* pid and depth on the last trace processed */
1107         struct fgraph_data *data;
1108         int cpu;
1109
1110         iter->private = NULL;
1111
1112         data = kzalloc(sizeof(*data), GFP_KERNEL);
1113         if (!data)
1114                 goto out_err;
1115
1116         data->cpu_data = alloc_percpu(struct fgraph_cpu_data);
1117         if (!data->cpu_data)
1118                 goto out_err_free;
1119
1120         for_each_possible_cpu(cpu) {
1121                 pid_t *pid = &(per_cpu_ptr(data->cpu_data, cpu)->last_pid);
1122                 int *depth = &(per_cpu_ptr(data->cpu_data, cpu)->depth);
1123                 int *ignore = &(per_cpu_ptr(data->cpu_data, cpu)->ignore);
1124                 *pid = -1;
1125                 *depth = 0;
1126                 *ignore = 0;
1127         }
1128
1129         iter->private = data;
1130
1131         return;
1132
1133  out_err_free:
1134         kfree(data);
1135  out_err:
1136         pr_warning("function graph tracer: not enough memory\n");
1137 }
1138
1139 static void graph_trace_close(struct trace_iterator *iter)
1140 {
1141         struct fgraph_data *data = iter->private;
1142
1143         if (data) {
1144                 free_percpu(data->cpu_data);
1145                 kfree(data);
1146         }
1147 }
1148
1149 static struct tracer graph_trace __read_mostly = {
1150         .name           = "function_graph",
1151         .open           = graph_trace_open,
1152         .pipe_open      = graph_trace_open,
1153         .close          = graph_trace_close,
1154         .pipe_close     = graph_trace_close,
1155         .wait_pipe      = poll_wait_pipe,
1156         .init           = graph_trace_init,
1157         .reset          = graph_trace_reset,
1158         .print_line     = print_graph_function,
1159         .print_header   = print_graph_headers,
1160         .flags          = &tracer_flags,
1161 #ifdef CONFIG_FTRACE_SELFTEST
1162         .selftest       = trace_selftest_startup_function_graph,
1163 #endif
1164 };
1165
1166 static __init int init_graph_trace(void)
1167 {
1168         max_bytes_for_cpu = snprintf(NULL, 0, "%d", nr_cpu_ids - 1);
1169
1170         return register_tracer(&graph_trace);
1171 }
1172
1173 device_initcall(init_graph_trace);