Merge branch 'master' into percpu
[sfrench/cifs-2.6.git] / kernel / trace / trace.c
1 /*
2  * ring buffer based function tracer
3  *
4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5  * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6  *
7  * Originally taken from the RT patch by:
8  *    Arnaldo Carvalho de Melo <acme@redhat.com>
9  *
10  * Based on code from the latency_tracer, that is:
11  *  Copyright (C) 2004-2006 Ingo Molnar
12  *  Copyright (C) 2004 William Lee Irwin III
13  */
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/smp_lock.h>
21 #include <linux/notifier.h>
22 #include <linux/irqflags.h>
23 #include <linux/debugfs.h>
24 #include <linux/pagemap.h>
25 #include <linux/hardirq.h>
26 #include <linux/linkage.h>
27 #include <linux/uaccess.h>
28 #include <linux/kprobes.h>
29 #include <linux/ftrace.h>
30 #include <linux/module.h>
31 #include <linux/percpu.h>
32 #include <linux/splice.h>
33 #include <linux/kdebug.h>
34 #include <linux/string.h>
35 #include <linux/ctype.h>
36 #include <linux/init.h>
37 #include <linux/poll.h>
38 #include <linux/gfp.h>
39 #include <linux/fs.h>
40
41 #include "trace.h"
42 #include "trace_output.h"
43
44 #define TRACE_BUFFER_FLAGS      (RB_FL_OVERWRITE)
45
46 /*
47  * On boot up, the ring buffer is set to the minimum size, so that
48  * we do not waste memory on systems that are not using tracing.
49  */
50 int ring_buffer_expanded;
51
52 /*
53  * We need to change this state when a selftest is running.
54  * A selftest will lurk into the ring-buffer to count the
55  * entries inserted during the selftest although some concurrent
56  * insertions into the ring-buffer such as trace_printk could occurred
57  * at the same time, giving false positive or negative results.
58  */
59 static bool __read_mostly tracing_selftest_running;
60
61 /*
62  * If a tracer is running, we do not want to run SELFTEST.
63  */
64 bool __read_mostly tracing_selftest_disabled;
65
66 /* For tracers that don't implement custom flags */
67 static struct tracer_opt dummy_tracer_opt[] = {
68         { }
69 };
70
71 static struct tracer_flags dummy_tracer_flags = {
72         .val = 0,
73         .opts = dummy_tracer_opt
74 };
75
76 static int dummy_set_flag(u32 old_flags, u32 bit, int set)
77 {
78         return 0;
79 }
80
81 /*
82  * Kill all tracing for good (never come back).
83  * It is initialized to 1 but will turn to zero if the initialization
84  * of the tracer is successful. But that is the only place that sets
85  * this back to zero.
86  */
87 static int tracing_disabled = 1;
88
89 DEFINE_PER_CPU(int, ftrace_cpu_disabled);
90
91 static inline void ftrace_disable_cpu(void)
92 {
93         preempt_disable();
94         __this_cpu_inc(ftrace_cpu_disabled);
95 }
96
97 static inline void ftrace_enable_cpu(void)
98 {
99         __this_cpu_dec(ftrace_cpu_disabled);
100         preempt_enable();
101 }
102
103 static cpumask_var_t __read_mostly      tracing_buffer_mask;
104
105 /* Define which cpu buffers are currently read in trace_pipe */
106 static cpumask_var_t                    tracing_reader_cpumask;
107
108 #define for_each_tracing_cpu(cpu)       \
109         for_each_cpu(cpu, tracing_buffer_mask)
110
111 /*
112  * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
113  *
114  * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
115  * is set, then ftrace_dump is called. This will output the contents
116  * of the ftrace buffers to the console.  This is very useful for
117  * capturing traces that lead to crashes and outputing it to a
118  * serial console.
119  *
120  * It is default off, but you can enable it with either specifying
121  * "ftrace_dump_on_oops" in the kernel command line, or setting
122  * /proc/sys/kernel/ftrace_dump_on_oops to true.
123  */
124 int ftrace_dump_on_oops;
125
126 static int tracing_set_tracer(const char *buf);
127
128 #define MAX_TRACER_SIZE         100
129 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
130 static char *default_bootup_tracer;
131
132 static int __init set_cmdline_ftrace(char *str)
133 {
134         strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
135         default_bootup_tracer = bootup_tracer_buf;
136         /* We are using ftrace early, expand it */
137         ring_buffer_expanded = 1;
138         return 1;
139 }
140 __setup("ftrace=", set_cmdline_ftrace);
141
142 static int __init set_ftrace_dump_on_oops(char *str)
143 {
144         ftrace_dump_on_oops = 1;
145         return 1;
146 }
147 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
148
149 unsigned long long ns2usecs(cycle_t nsec)
150 {
151         nsec += 500;
152         do_div(nsec, 1000);
153         return nsec;
154 }
155
156 /*
157  * The global_trace is the descriptor that holds the tracing
158  * buffers for the live tracing. For each CPU, it contains
159  * a link list of pages that will store trace entries. The
160  * page descriptor of the pages in the memory is used to hold
161  * the link list by linking the lru item in the page descriptor
162  * to each of the pages in the buffer per CPU.
163  *
164  * For each active CPU there is a data field that holds the
165  * pages for the buffer for that CPU. Each CPU has the same number
166  * of pages allocated for its buffer.
167  */
168 static struct trace_array       global_trace;
169
170 static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
171
172 int filter_current_check_discard(struct ring_buffer *buffer,
173                                  struct ftrace_event_call *call, void *rec,
174                                  struct ring_buffer_event *event)
175 {
176         return filter_check_discard(call, rec, buffer, event);
177 }
178 EXPORT_SYMBOL_GPL(filter_current_check_discard);
179
180 cycle_t ftrace_now(int cpu)
181 {
182         u64 ts;
183
184         /* Early boot up does not have a buffer yet */
185         if (!global_trace.buffer)
186                 return trace_clock_local();
187
188         ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
189         ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);
190
191         return ts;
192 }
193
194 /*
195  * The max_tr is used to snapshot the global_trace when a maximum
196  * latency is reached. Some tracers will use this to store a maximum
197  * trace while it continues examining live traces.
198  *
199  * The buffers for the max_tr are set up the same as the global_trace.
200  * When a snapshot is taken, the link list of the max_tr is swapped
201  * with the link list of the global_trace and the buffers are reset for
202  * the global_trace so the tracing can continue.
203  */
204 static struct trace_array       max_tr;
205
206 static DEFINE_PER_CPU(struct trace_array_cpu, max_tr_data);
207
208 /* tracer_enabled is used to toggle activation of a tracer */
209 static int                      tracer_enabled = 1;
210
211 /**
212  * tracing_is_enabled - return tracer_enabled status
213  *
214  * This function is used by other tracers to know the status
215  * of the tracer_enabled flag.  Tracers may use this function
216  * to know if it should enable their features when starting
217  * up. See irqsoff tracer for an example (start_irqsoff_tracer).
218  */
219 int tracing_is_enabled(void)
220 {
221         return tracer_enabled;
222 }
223
224 /*
225  * trace_buf_size is the size in bytes that is allocated
226  * for a buffer. Note, the number of bytes is always rounded
227  * to page size.
228  *
229  * This number is purposely set to a low number of 16384.
230  * If the dump on oops happens, it will be much appreciated
231  * to not have to wait for all that output. Anyway this can be
232  * boot time and run time configurable.
233  */
234 #define TRACE_BUF_SIZE_DEFAULT  1441792UL /* 16384 * 88 (sizeof(entry)) */
235
236 static unsigned long            trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
237
238 /* trace_types holds a link list of available tracers. */
239 static struct tracer            *trace_types __read_mostly;
240
241 /* current_trace points to the tracer that is currently active */
242 static struct tracer            *current_trace __read_mostly;
243
244 /*
245  * trace_types_lock is used to protect the trace_types list.
246  * This lock is also used to keep user access serialized.
247  * Accesses from userspace will grab this lock while userspace
248  * activities happen inside the kernel.
249  */
250 static DEFINE_MUTEX(trace_types_lock);
251
252 /* trace_wait is a waitqueue for tasks blocked on trace_poll */
253 static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
254
255 /* trace_flags holds trace_options default values */
256 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
257         TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
258         TRACE_ITER_GRAPH_TIME;
259
260 static int trace_stop_count;
261 static DEFINE_SPINLOCK(tracing_start_lock);
262
263 /**
264  * trace_wake_up - wake up tasks waiting for trace input
265  *
266  * Simply wakes up any task that is blocked on the trace_wait
267  * queue. These is used with trace_poll for tasks polling the trace.
268  */
269 void trace_wake_up(void)
270 {
271         int cpu;
272
273         if (trace_flags & TRACE_ITER_BLOCK)
274                 return;
275         /*
276          * The runqueue_is_locked() can fail, but this is the best we
277          * have for now:
278          */
279         cpu = get_cpu();
280         if (!runqueue_is_locked(cpu))
281                 wake_up(&trace_wait);
282         put_cpu();
283 }
284
285 static int __init set_buf_size(char *str)
286 {
287         unsigned long buf_size;
288
289         if (!str)
290                 return 0;
291         buf_size = memparse(str, &str);
292         /* nr_entries can not be zero */
293         if (buf_size == 0)
294                 return 0;
295         trace_buf_size = buf_size;
296         return 1;
297 }
298 __setup("trace_buf_size=", set_buf_size);
299
300 unsigned long nsecs_to_usecs(unsigned long nsecs)
301 {
302         return nsecs / 1000;
303 }
304
305 /* These must match the bit postions in trace_iterator_flags */
306 static const char *trace_options[] = {
307         "print-parent",
308         "sym-offset",
309         "sym-addr",
310         "verbose",
311         "raw",
312         "hex",
313         "bin",
314         "block",
315         "stacktrace",
316         "trace_printk",
317         "ftrace_preempt",
318         "branch",
319         "annotate",
320         "userstacktrace",
321         "sym-userobj",
322         "printk-msg-only",
323         "context-info",
324         "latency-format",
325         "sleep-time",
326         "graph-time",
327         NULL
328 };
329
330 static struct {
331         u64 (*func)(void);
332         const char *name;
333 } trace_clocks[] = {
334         { trace_clock_local,    "local" },
335         { trace_clock_global,   "global" },
336 };
337
338 int trace_clock_id;
339
340 /*
341  * trace_parser_get_init - gets the buffer for trace parser
342  */
343 int trace_parser_get_init(struct trace_parser *parser, int size)
344 {
345         memset(parser, 0, sizeof(*parser));
346
347         parser->buffer = kmalloc(size, GFP_KERNEL);
348         if (!parser->buffer)
349                 return 1;
350
351         parser->size = size;
352         return 0;
353 }
354
355 /*
356  * trace_parser_put - frees the buffer for trace parser
357  */
358 void trace_parser_put(struct trace_parser *parser)
359 {
360         kfree(parser->buffer);
361 }
362
363 /*
364  * trace_get_user - reads the user input string separated by  space
365  * (matched by isspace(ch))
366  *
367  * For each string found the 'struct trace_parser' is updated,
368  * and the function returns.
369  *
370  * Returns number of bytes read.
371  *
372  * See kernel/trace/trace.h for 'struct trace_parser' details.
373  */
374 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
375         size_t cnt, loff_t *ppos)
376 {
377         char ch;
378         size_t read = 0;
379         ssize_t ret;
380
381         if (!*ppos)
382                 trace_parser_clear(parser);
383
384         ret = get_user(ch, ubuf++);
385         if (ret)
386                 goto out;
387
388         read++;
389         cnt--;
390
391         /*
392          * The parser is not finished with the last write,
393          * continue reading the user input without skipping spaces.
394          */
395         if (!parser->cont) {
396                 /* skip white space */
397                 while (cnt && isspace(ch)) {
398                         ret = get_user(ch, ubuf++);
399                         if (ret)
400                                 goto out;
401                         read++;
402                         cnt--;
403                 }
404
405                 /* only spaces were written */
406                 if (isspace(ch)) {
407                         *ppos += read;
408                         ret = read;
409                         goto out;
410                 }
411
412                 parser->idx = 0;
413         }
414
415         /* read the non-space input */
416         while (cnt && !isspace(ch)) {
417                 if (parser->idx < parser->size - 1)
418                         parser->buffer[parser->idx++] = ch;
419                 else {
420                         ret = -EINVAL;
421                         goto out;
422                 }
423                 ret = get_user(ch, ubuf++);
424                 if (ret)
425                         goto out;
426                 read++;
427                 cnt--;
428         }
429
430         /* We either got finished input or we have to wait for another call. */
431         if (isspace(ch)) {
432                 parser->buffer[parser->idx] = 0;
433                 parser->cont = false;
434         } else {
435                 parser->cont = true;
436                 parser->buffer[parser->idx++] = ch;
437         }
438
439         *ppos += read;
440         ret = read;
441
442 out:
443         return ret;
444 }
445
446 ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
447 {
448         int len;
449         int ret;
450
451         if (!cnt)
452                 return 0;
453
454         if (s->len <= s->readpos)
455                 return -EBUSY;
456
457         len = s->len - s->readpos;
458         if (cnt > len)
459                 cnt = len;
460         ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
461         if (ret == cnt)
462                 return -EFAULT;
463
464         cnt -= ret;
465
466         s->readpos += cnt;
467         return cnt;
468 }
469
470 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
471 {
472         int len;
473         void *ret;
474
475         if (s->len <= s->readpos)
476                 return -EBUSY;
477
478         len = s->len - s->readpos;
479         if (cnt > len)
480                 cnt = len;
481         ret = memcpy(buf, s->buffer + s->readpos, cnt);
482         if (!ret)
483                 return -EFAULT;
484
485         s->readpos += cnt;
486         return cnt;
487 }
488
489 /*
490  * ftrace_max_lock is used to protect the swapping of buffers
491  * when taking a max snapshot. The buffers themselves are
492  * protected by per_cpu spinlocks. But the action of the swap
493  * needs its own lock.
494  *
495  * This is defined as a arch_spinlock_t in order to help
496  * with performance when lockdep debugging is enabled.
497  *
498  * It is also used in other places outside the update_max_tr
499  * so it needs to be defined outside of the
500  * CONFIG_TRACER_MAX_TRACE.
501  */
502 static arch_spinlock_t ftrace_max_lock =
503         (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
504
505 #ifdef CONFIG_TRACER_MAX_TRACE
506 unsigned long __read_mostly     tracing_max_latency;
507 unsigned long __read_mostly     tracing_thresh;
508
509 /*
510  * Copy the new maximum trace into the separate maximum-trace
511  * structure. (this way the maximum trace is permanently saved,
512  * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
513  */
514 static void
515 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
516 {
517         struct trace_array_cpu *data = tr->data[cpu];
518         struct trace_array_cpu *max_data = tr->data[cpu];
519
520         max_tr.cpu = cpu;
521         max_tr.time_start = data->preempt_timestamp;
522
523         max_data = max_tr.data[cpu];
524         max_data->saved_latency = tracing_max_latency;
525         max_data->critical_start = data->critical_start;
526         max_data->critical_end = data->critical_end;
527
528         memcpy(data->comm, tsk->comm, TASK_COMM_LEN);
529         max_data->pid = tsk->pid;
530         max_data->uid = task_uid(tsk);
531         max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
532         max_data->policy = tsk->policy;
533         max_data->rt_priority = tsk->rt_priority;
534
535         /* record this tasks comm */
536         tracing_record_cmdline(tsk);
537 }
538
539 /**
540  * update_max_tr - snapshot all trace buffers from global_trace to max_tr
541  * @tr: tracer
542  * @tsk: the task with the latency
543  * @cpu: The cpu that initiated the trace.
544  *
545  * Flip the buffers between the @tr and the max_tr and record information
546  * about which task was the cause of this latency.
547  */
548 void
549 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
550 {
551         struct ring_buffer *buf = tr->buffer;
552
553         if (trace_stop_count)
554                 return;
555
556         WARN_ON_ONCE(!irqs_disabled());
557         arch_spin_lock(&ftrace_max_lock);
558
559         tr->buffer = max_tr.buffer;
560         max_tr.buffer = buf;
561
562         __update_max_tr(tr, tsk, cpu);
563         arch_spin_unlock(&ftrace_max_lock);
564 }
565
566 /**
567  * update_max_tr_single - only copy one trace over, and reset the rest
568  * @tr - tracer
569  * @tsk - task with the latency
570  * @cpu - the cpu of the buffer to copy.
571  *
572  * Flip the trace of a single CPU buffer between the @tr and the max_tr.
573  */
574 void
575 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
576 {
577         int ret;
578
579         if (trace_stop_count)
580                 return;
581
582         WARN_ON_ONCE(!irqs_disabled());
583         arch_spin_lock(&ftrace_max_lock);
584
585         ftrace_disable_cpu();
586
587         ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
588
589         if (ret == -EBUSY) {
590                 /*
591                  * We failed to swap the buffer due to a commit taking
592                  * place on this CPU. We fail to record, but we reset
593                  * the max trace buffer (no one writes directly to it)
594                  * and flag that it failed.
595                  */
596                 trace_array_printk(&max_tr, _THIS_IP_,
597                         "Failed to swap buffers due to commit in progress\n");
598         }
599
600         ftrace_enable_cpu();
601
602         WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
603
604         __update_max_tr(tr, tsk, cpu);
605         arch_spin_unlock(&ftrace_max_lock);
606 }
607 #endif /* CONFIG_TRACER_MAX_TRACE */
608
609 /**
610  * register_tracer - register a tracer with the ftrace system.
611  * @type - the plugin for the tracer
612  *
613  * Register a new plugin tracer.
614  */
615 int register_tracer(struct tracer *type)
616 __releases(kernel_lock)
617 __acquires(kernel_lock)
618 {
619         struct tracer *t;
620         int ret = 0;
621
622         if (!type->name) {
623                 pr_info("Tracer must have a name\n");
624                 return -1;
625         }
626
627         if (strlen(type->name) > MAX_TRACER_SIZE) {
628                 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
629                 return -1;
630         }
631
632         /*
633          * When this gets called we hold the BKL which means that
634          * preemption is disabled. Various trace selftests however
635          * need to disable and enable preemption for successful tests.
636          * So we drop the BKL here and grab it after the tests again.
637          */
638         unlock_kernel();
639         mutex_lock(&trace_types_lock);
640
641         tracing_selftest_running = true;
642
643         for (t = trace_types; t; t = t->next) {
644                 if (strcmp(type->name, t->name) == 0) {
645                         /* already found */
646                         pr_info("Tracer %s already registered\n",
647                                 type->name);
648                         ret = -1;
649                         goto out;
650                 }
651         }
652
653         if (!type->set_flag)
654                 type->set_flag = &dummy_set_flag;
655         if (!type->flags)
656                 type->flags = &dummy_tracer_flags;
657         else
658                 if (!type->flags->opts)
659                         type->flags->opts = dummy_tracer_opt;
660         if (!type->wait_pipe)
661                 type->wait_pipe = default_wait_pipe;
662
663
664 #ifdef CONFIG_FTRACE_STARTUP_TEST
665         if (type->selftest && !tracing_selftest_disabled) {
666                 struct tracer *saved_tracer = current_trace;
667                 struct trace_array *tr = &global_trace;
668
669                 /*
670                  * Run a selftest on this tracer.
671                  * Here we reset the trace buffer, and set the current
672                  * tracer to be this tracer. The tracer can then run some
673                  * internal tracing to verify that everything is in order.
674                  * If we fail, we do not register this tracer.
675                  */
676                 tracing_reset_online_cpus(tr);
677
678                 current_trace = type;
679                 /* the test is responsible for initializing and enabling */
680                 pr_info("Testing tracer %s: ", type->name);
681                 ret = type->selftest(type, tr);
682                 /* the test is responsible for resetting too */
683                 current_trace = saved_tracer;
684                 if (ret) {
685                         printk(KERN_CONT "FAILED!\n");
686                         goto out;
687                 }
688                 /* Only reset on passing, to avoid touching corrupted buffers */
689                 tracing_reset_online_cpus(tr);
690
691                 printk(KERN_CONT "PASSED\n");
692         }
693 #endif
694
695         type->next = trace_types;
696         trace_types = type;
697
698  out:
699         tracing_selftest_running = false;
700         mutex_unlock(&trace_types_lock);
701
702         if (ret || !default_bootup_tracer)
703                 goto out_unlock;
704
705         if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
706                 goto out_unlock;
707
708         printk(KERN_INFO "Starting tracer '%s'\n", type->name);
709         /* Do we want this tracer to start on bootup? */
710         tracing_set_tracer(type->name);
711         default_bootup_tracer = NULL;
712         /* disable other selftests, since this will break it. */
713         tracing_selftest_disabled = 1;
714 #ifdef CONFIG_FTRACE_STARTUP_TEST
715         printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
716                type->name);
717 #endif
718
719  out_unlock:
720         lock_kernel();
721         return ret;
722 }
723
724 void unregister_tracer(struct tracer *type)
725 {
726         struct tracer **t;
727
728         mutex_lock(&trace_types_lock);
729         for (t = &trace_types; *t; t = &(*t)->next) {
730                 if (*t == type)
731                         goto found;
732         }
733         pr_info("Tracer %s not registered\n", type->name);
734         goto out;
735
736  found:
737         *t = (*t)->next;
738
739         if (type == current_trace && tracer_enabled) {
740                 tracer_enabled = 0;
741                 tracing_stop();
742                 if (current_trace->stop)
743                         current_trace->stop(&global_trace);
744                 current_trace = &nop_trace;
745         }
746 out:
747         mutex_unlock(&trace_types_lock);
748 }
749
750 static void __tracing_reset(struct trace_array *tr, int cpu)
751 {
752         ftrace_disable_cpu();
753         ring_buffer_reset_cpu(tr->buffer, cpu);
754         ftrace_enable_cpu();
755 }
756
757 void tracing_reset(struct trace_array *tr, int cpu)
758 {
759         struct ring_buffer *buffer = tr->buffer;
760
761         ring_buffer_record_disable(buffer);
762
763         /* Make sure all commits have finished */
764         synchronize_sched();
765         __tracing_reset(tr, cpu);
766
767         ring_buffer_record_enable(buffer);
768 }
769
770 void tracing_reset_online_cpus(struct trace_array *tr)
771 {
772         struct ring_buffer *buffer = tr->buffer;
773         int cpu;
774
775         ring_buffer_record_disable(buffer);
776
777         /* Make sure all commits have finished */
778         synchronize_sched();
779
780         tr->time_start = ftrace_now(tr->cpu);
781
782         for_each_online_cpu(cpu)
783                 __tracing_reset(tr, cpu);
784
785         ring_buffer_record_enable(buffer);
786 }
787
788 void tracing_reset_current(int cpu)
789 {
790         tracing_reset(&global_trace, cpu);
791 }
792
793 void tracing_reset_current_online_cpus(void)
794 {
795         tracing_reset_online_cpus(&global_trace);
796 }
797
798 #define SAVED_CMDLINES 128
799 #define NO_CMDLINE_MAP UINT_MAX
800 static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
801 static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
802 static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
803 static int cmdline_idx;
804 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
805
806 /* temporary disable recording */
807 static atomic_t trace_record_cmdline_disabled __read_mostly;
808
809 static void trace_init_cmdlines(void)
810 {
811         memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
812         memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
813         cmdline_idx = 0;
814 }
815
816 int is_tracing_stopped(void)
817 {
818         return trace_stop_count;
819 }
820
821 /**
822  * ftrace_off_permanent - disable all ftrace code permanently
823  *
824  * This should only be called when a serious anomally has
825  * been detected.  This will turn off the function tracing,
826  * ring buffers, and other tracing utilites. It takes no
827  * locks and can be called from any context.
828  */
829 void ftrace_off_permanent(void)
830 {
831         tracing_disabled = 1;
832         ftrace_stop();
833         tracing_off_permanent();
834 }
835
836 /**
837  * tracing_start - quick start of the tracer
838  *
839  * If tracing is enabled but was stopped by tracing_stop,
840  * this will start the tracer back up.
841  */
842 void tracing_start(void)
843 {
844         struct ring_buffer *buffer;
845         unsigned long flags;
846
847         if (tracing_disabled)
848                 return;
849
850         spin_lock_irqsave(&tracing_start_lock, flags);
851         if (--trace_stop_count) {
852                 if (trace_stop_count < 0) {
853                         /* Someone screwed up their debugging */
854                         WARN_ON_ONCE(1);
855                         trace_stop_count = 0;
856                 }
857                 goto out;
858         }
859
860
861         buffer = global_trace.buffer;
862         if (buffer)
863                 ring_buffer_record_enable(buffer);
864
865         buffer = max_tr.buffer;
866         if (buffer)
867                 ring_buffer_record_enable(buffer);
868
869         ftrace_start();
870  out:
871         spin_unlock_irqrestore(&tracing_start_lock, flags);
872 }
873
874 /**
875  * tracing_stop - quick stop of the tracer
876  *
877  * Light weight way to stop tracing. Use in conjunction with
878  * tracing_start.
879  */
880 void tracing_stop(void)
881 {
882         struct ring_buffer *buffer;
883         unsigned long flags;
884
885         ftrace_stop();
886         spin_lock_irqsave(&tracing_start_lock, flags);
887         if (trace_stop_count++)
888                 goto out;
889
890         buffer = global_trace.buffer;
891         if (buffer)
892                 ring_buffer_record_disable(buffer);
893
894         buffer = max_tr.buffer;
895         if (buffer)
896                 ring_buffer_record_disable(buffer);
897
898  out:
899         spin_unlock_irqrestore(&tracing_start_lock, flags);
900 }
901
902 void trace_stop_cmdline_recording(void);
903
904 static void trace_save_cmdline(struct task_struct *tsk)
905 {
906         unsigned pid, idx;
907
908         if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
909                 return;
910
911         /*
912          * It's not the end of the world if we don't get
913          * the lock, but we also don't want to spin
914          * nor do we want to disable interrupts,
915          * so if we miss here, then better luck next time.
916          */
917         if (!arch_spin_trylock(&trace_cmdline_lock))
918                 return;
919
920         idx = map_pid_to_cmdline[tsk->pid];
921         if (idx == NO_CMDLINE_MAP) {
922                 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
923
924                 /*
925                  * Check whether the cmdline buffer at idx has a pid
926                  * mapped. We are going to overwrite that entry so we
927                  * need to clear the map_pid_to_cmdline. Otherwise we
928                  * would read the new comm for the old pid.
929                  */
930                 pid = map_cmdline_to_pid[idx];
931                 if (pid != NO_CMDLINE_MAP)
932                         map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
933
934                 map_cmdline_to_pid[idx] = tsk->pid;
935                 map_pid_to_cmdline[tsk->pid] = idx;
936
937                 cmdline_idx = idx;
938         }
939
940         memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
941
942         arch_spin_unlock(&trace_cmdline_lock);
943 }
944
945 void trace_find_cmdline(int pid, char comm[])
946 {
947         unsigned map;
948
949         if (!pid) {
950                 strcpy(comm, "<idle>");
951                 return;
952         }
953
954         if (pid > PID_MAX_DEFAULT) {
955                 strcpy(comm, "<...>");
956                 return;
957         }
958
959         preempt_disable();
960         arch_spin_lock(&trace_cmdline_lock);
961         map = map_pid_to_cmdline[pid];
962         if (map != NO_CMDLINE_MAP)
963                 strcpy(comm, saved_cmdlines[map]);
964         else
965                 strcpy(comm, "<...>");
966
967         arch_spin_unlock(&trace_cmdline_lock);
968         preempt_enable();
969 }
970
971 void tracing_record_cmdline(struct task_struct *tsk)
972 {
973         if (atomic_read(&trace_record_cmdline_disabled) || !tracer_enabled ||
974             !tracing_is_on())
975                 return;
976
977         trace_save_cmdline(tsk);
978 }
979
980 void
981 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
982                              int pc)
983 {
984         struct task_struct *tsk = current;
985
986         entry->preempt_count            = pc & 0xff;
987         entry->pid                      = (tsk) ? tsk->pid : 0;
988         entry->lock_depth               = (tsk) ? tsk->lock_depth : 0;
989         entry->flags =
990 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
991                 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
992 #else
993                 TRACE_FLAG_IRQS_NOSUPPORT |
994 #endif
995                 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
996                 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
997                 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
998 }
999 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1000
1001 struct ring_buffer_event *
1002 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1003                           int type,
1004                           unsigned long len,
1005                           unsigned long flags, int pc)
1006 {
1007         struct ring_buffer_event *event;
1008
1009         event = ring_buffer_lock_reserve(buffer, len);
1010         if (event != NULL) {
1011                 struct trace_entry *ent = ring_buffer_event_data(event);
1012
1013                 tracing_generic_entry_update(ent, flags, pc);
1014                 ent->type = type;
1015         }
1016
1017         return event;
1018 }
1019
1020 static inline void
1021 __trace_buffer_unlock_commit(struct ring_buffer *buffer,
1022                              struct ring_buffer_event *event,
1023                              unsigned long flags, int pc,
1024                              int wake)
1025 {
1026         ring_buffer_unlock_commit(buffer, event);
1027
1028         ftrace_trace_stack(buffer, flags, 6, pc);
1029         ftrace_trace_userstack(buffer, flags, pc);
1030
1031         if (wake)
1032                 trace_wake_up();
1033 }
1034
1035 void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1036                                 struct ring_buffer_event *event,
1037                                 unsigned long flags, int pc)
1038 {
1039         __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
1040 }
1041
1042 struct ring_buffer_event *
1043 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1044                                   int type, unsigned long len,
1045                                   unsigned long flags, int pc)
1046 {
1047         *current_rb = global_trace.buffer;
1048         return trace_buffer_lock_reserve(*current_rb,
1049                                          type, len, flags, pc);
1050 }
1051 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1052
1053 void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1054                                         struct ring_buffer_event *event,
1055                                         unsigned long flags, int pc)
1056 {
1057         __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
1058 }
1059 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1060
1061 void trace_nowake_buffer_unlock_commit(struct ring_buffer *buffer,
1062                                        struct ring_buffer_event *event,
1063                                        unsigned long flags, int pc)
1064 {
1065         __trace_buffer_unlock_commit(buffer, event, flags, pc, 0);
1066 }
1067 EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit);
1068
1069 void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1070                                          struct ring_buffer_event *event)
1071 {
1072         ring_buffer_discard_commit(buffer, event);
1073 }
1074 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1075
1076 void
1077 trace_function(struct trace_array *tr,
1078                unsigned long ip, unsigned long parent_ip, unsigned long flags,
1079                int pc)
1080 {
1081         struct ftrace_event_call *call = &event_function;
1082         struct ring_buffer *buffer = tr->buffer;
1083         struct ring_buffer_event *event;
1084         struct ftrace_entry *entry;
1085
1086         /* If we are reading the ring buffer, don't trace */
1087         if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1088                 return;
1089
1090         event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1091                                           flags, pc);
1092         if (!event)
1093                 return;
1094         entry   = ring_buffer_event_data(event);
1095         entry->ip                       = ip;
1096         entry->parent_ip                = parent_ip;
1097
1098         if (!filter_check_discard(call, entry, buffer, event))
1099                 ring_buffer_unlock_commit(buffer, event);
1100 }
1101
1102 void
1103 ftrace(struct trace_array *tr, struct trace_array_cpu *data,
1104        unsigned long ip, unsigned long parent_ip, unsigned long flags,
1105        int pc)
1106 {
1107         if (likely(!atomic_read(&data->disabled)))
1108                 trace_function(tr, ip, parent_ip, flags, pc);
1109 }
1110
1111 #ifdef CONFIG_STACKTRACE
1112 static void __ftrace_trace_stack(struct ring_buffer *buffer,
1113                                  unsigned long flags,
1114                                  int skip, int pc)
1115 {
1116         struct ftrace_event_call *call = &event_kernel_stack;
1117         struct ring_buffer_event *event;
1118         struct stack_entry *entry;
1119         struct stack_trace trace;
1120
1121         event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1122                                           sizeof(*entry), flags, pc);
1123         if (!event)
1124                 return;
1125         entry   = ring_buffer_event_data(event);
1126         memset(&entry->caller, 0, sizeof(entry->caller));
1127
1128         trace.nr_entries        = 0;
1129         trace.max_entries       = FTRACE_STACK_ENTRIES;
1130         trace.skip              = skip;
1131         trace.entries           = entry->caller;
1132
1133         save_stack_trace(&trace);
1134         if (!filter_check_discard(call, entry, buffer, event))
1135                 ring_buffer_unlock_commit(buffer, event);
1136 }
1137
1138 void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1139                         int skip, int pc)
1140 {
1141         if (!(trace_flags & TRACE_ITER_STACKTRACE))
1142                 return;
1143
1144         __ftrace_trace_stack(buffer, flags, skip, pc);
1145 }
1146
1147 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1148                    int pc)
1149 {
1150         __ftrace_trace_stack(tr->buffer, flags, skip, pc);
1151 }
1152
1153 /**
1154  * trace_dump_stack - record a stack back trace in the trace buffer
1155  */
1156 void trace_dump_stack(void)
1157 {
1158         unsigned long flags;
1159
1160         if (tracing_disabled || tracing_selftest_running)
1161                 return;
1162
1163         local_save_flags(flags);
1164
1165         /* skipping 3 traces, seems to get us at the caller of this function */
1166         __ftrace_trace_stack(global_trace.buffer, flags, 3, preempt_count());
1167 }
1168
1169 void
1170 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1171 {
1172         struct ftrace_event_call *call = &event_user_stack;
1173         struct ring_buffer_event *event;
1174         struct userstack_entry *entry;
1175         struct stack_trace trace;
1176
1177         if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1178                 return;
1179
1180         event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1181                                           sizeof(*entry), flags, pc);
1182         if (!event)
1183                 return;
1184         entry   = ring_buffer_event_data(event);
1185
1186         entry->tgid             = current->tgid;
1187         memset(&entry->caller, 0, sizeof(entry->caller));
1188
1189         trace.nr_entries        = 0;
1190         trace.max_entries       = FTRACE_STACK_ENTRIES;
1191         trace.skip              = 0;
1192         trace.entries           = entry->caller;
1193
1194         save_stack_trace_user(&trace);
1195         if (!filter_check_discard(call, entry, buffer, event))
1196                 ring_buffer_unlock_commit(buffer, event);
1197 }
1198
1199 #ifdef UNUSED
1200 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1201 {
1202         ftrace_trace_userstack(tr, flags, preempt_count());
1203 }
1204 #endif /* UNUSED */
1205
1206 #endif /* CONFIG_STACKTRACE */
1207
1208 static void
1209 ftrace_trace_special(void *__tr,
1210                      unsigned long arg1, unsigned long arg2, unsigned long arg3,
1211                      int pc)
1212 {
1213         struct ftrace_event_call *call = &event_special;
1214         struct ring_buffer_event *event;
1215         struct trace_array *tr = __tr;
1216         struct ring_buffer *buffer = tr->buffer;
1217         struct special_entry *entry;
1218
1219         event = trace_buffer_lock_reserve(buffer, TRACE_SPECIAL,
1220                                           sizeof(*entry), 0, pc);
1221         if (!event)
1222                 return;
1223         entry   = ring_buffer_event_data(event);
1224         entry->arg1                     = arg1;
1225         entry->arg2                     = arg2;
1226         entry->arg3                     = arg3;
1227
1228         if (!filter_check_discard(call, entry, buffer, event))
1229                 trace_buffer_unlock_commit(buffer, event, 0, pc);
1230 }
1231
1232 void
1233 __trace_special(void *__tr, void *__data,
1234                 unsigned long arg1, unsigned long arg2, unsigned long arg3)
1235 {
1236         ftrace_trace_special(__tr, arg1, arg2, arg3, preempt_count());
1237 }
1238
1239 void
1240 ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
1241 {
1242         struct trace_array *tr = &global_trace;
1243         struct trace_array_cpu *data;
1244         unsigned long flags;
1245         int cpu;
1246         int pc;
1247
1248         if (tracing_disabled)
1249                 return;
1250
1251         pc = preempt_count();
1252         local_irq_save(flags);
1253         cpu = raw_smp_processor_id();
1254         data = tr->data[cpu];
1255
1256         if (likely(atomic_inc_return(&data->disabled) == 1))
1257                 ftrace_trace_special(tr, arg1, arg2, arg3, pc);
1258
1259         atomic_dec(&data->disabled);
1260         local_irq_restore(flags);
1261 }
1262
1263 /**
1264  * trace_vbprintk - write binary msg to tracing buffer
1265  *
1266  */
1267 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1268 {
1269         static arch_spinlock_t trace_buf_lock =
1270                 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
1271         static u32 trace_buf[TRACE_BUF_SIZE];
1272
1273         struct ftrace_event_call *call = &event_bprint;
1274         struct ring_buffer_event *event;
1275         struct ring_buffer *buffer;
1276         struct trace_array *tr = &global_trace;
1277         struct trace_array_cpu *data;
1278         struct bprint_entry *entry;
1279         unsigned long flags;
1280         int disable;
1281         int resched;
1282         int cpu, len = 0, size, pc;
1283
1284         if (unlikely(tracing_selftest_running || tracing_disabled))
1285                 return 0;
1286
1287         /* Don't pollute graph traces with trace_vprintk internals */
1288         pause_graph_tracing();
1289
1290         pc = preempt_count();
1291         resched = ftrace_preempt_disable();
1292         cpu = raw_smp_processor_id();
1293         data = tr->data[cpu];
1294
1295         disable = atomic_inc_return(&data->disabled);
1296         if (unlikely(disable != 1))
1297                 goto out;
1298
1299         /* Lockdep uses trace_printk for lock tracing */
1300         local_irq_save(flags);
1301         arch_spin_lock(&trace_buf_lock);
1302         len = vbin_printf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1303
1304         if (len > TRACE_BUF_SIZE || len < 0)
1305                 goto out_unlock;
1306
1307         size = sizeof(*entry) + sizeof(u32) * len;
1308         buffer = tr->buffer;
1309         event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1310                                           flags, pc);
1311         if (!event)
1312                 goto out_unlock;
1313         entry = ring_buffer_event_data(event);
1314         entry->ip                       = ip;
1315         entry->fmt                      = fmt;
1316
1317         memcpy(entry->buf, trace_buf, sizeof(u32) * len);
1318         if (!filter_check_discard(call, entry, buffer, event))
1319                 ring_buffer_unlock_commit(buffer, event);
1320
1321 out_unlock:
1322         arch_spin_unlock(&trace_buf_lock);
1323         local_irq_restore(flags);
1324
1325 out:
1326         atomic_dec_return(&data->disabled);
1327         ftrace_preempt_enable(resched);
1328         unpause_graph_tracing();
1329
1330         return len;
1331 }
1332 EXPORT_SYMBOL_GPL(trace_vbprintk);
1333
1334 int trace_array_printk(struct trace_array *tr,
1335                        unsigned long ip, const char *fmt, ...)
1336 {
1337         int ret;
1338         va_list ap;
1339
1340         if (!(trace_flags & TRACE_ITER_PRINTK))
1341                 return 0;
1342
1343         va_start(ap, fmt);
1344         ret = trace_array_vprintk(tr, ip, fmt, ap);
1345         va_end(ap);
1346         return ret;
1347 }
1348
1349 int trace_array_vprintk(struct trace_array *tr,
1350                         unsigned long ip, const char *fmt, va_list args)
1351 {
1352         static arch_spinlock_t trace_buf_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1353         static char trace_buf[TRACE_BUF_SIZE];
1354
1355         struct ftrace_event_call *call = &event_print;
1356         struct ring_buffer_event *event;
1357         struct ring_buffer *buffer;
1358         struct trace_array_cpu *data;
1359         int cpu, len = 0, size, pc;
1360         struct print_entry *entry;
1361         unsigned long irq_flags;
1362         int disable;
1363
1364         if (tracing_disabled || tracing_selftest_running)
1365                 return 0;
1366
1367         pc = preempt_count();
1368         preempt_disable_notrace();
1369         cpu = raw_smp_processor_id();
1370         data = tr->data[cpu];
1371
1372         disable = atomic_inc_return(&data->disabled);
1373         if (unlikely(disable != 1))
1374                 goto out;
1375
1376         pause_graph_tracing();
1377         raw_local_irq_save(irq_flags);
1378         arch_spin_lock(&trace_buf_lock);
1379         len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1380
1381         size = sizeof(*entry) + len + 1;
1382         buffer = tr->buffer;
1383         event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1384                                           irq_flags, pc);
1385         if (!event)
1386                 goto out_unlock;
1387         entry = ring_buffer_event_data(event);
1388         entry->ip = ip;
1389
1390         memcpy(&entry->buf, trace_buf, len);
1391         entry->buf[len] = '\0';
1392         if (!filter_check_discard(call, entry, buffer, event))
1393                 ring_buffer_unlock_commit(buffer, event);
1394
1395  out_unlock:
1396         arch_spin_unlock(&trace_buf_lock);
1397         raw_local_irq_restore(irq_flags);
1398         unpause_graph_tracing();
1399  out:
1400         atomic_dec_return(&data->disabled);
1401         preempt_enable_notrace();
1402
1403         return len;
1404 }
1405
1406 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1407 {
1408         return trace_array_vprintk(&global_trace, ip, fmt, args);
1409 }
1410 EXPORT_SYMBOL_GPL(trace_vprintk);
1411
1412 enum trace_file_type {
1413         TRACE_FILE_LAT_FMT      = 1,
1414         TRACE_FILE_ANNOTATE     = 2,
1415 };
1416
1417 static void trace_iterator_increment(struct trace_iterator *iter)
1418 {
1419         /* Don't allow ftrace to trace into the ring buffers */
1420         ftrace_disable_cpu();
1421
1422         iter->idx++;
1423         if (iter->buffer_iter[iter->cpu])
1424                 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1425
1426         ftrace_enable_cpu();
1427 }
1428
1429 static struct trace_entry *
1430 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts)
1431 {
1432         struct ring_buffer_event *event;
1433         struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
1434
1435         /* Don't allow ftrace to trace into the ring buffers */
1436         ftrace_disable_cpu();
1437
1438         if (buf_iter)
1439                 event = ring_buffer_iter_peek(buf_iter, ts);
1440         else
1441                 event = ring_buffer_peek(iter->tr->buffer, cpu, ts);
1442
1443         ftrace_enable_cpu();
1444
1445         return event ? ring_buffer_event_data(event) : NULL;
1446 }
1447
1448 static struct trace_entry *
1449 __find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts)
1450 {
1451         struct ring_buffer *buffer = iter->tr->buffer;
1452         struct trace_entry *ent, *next = NULL;
1453         int cpu_file = iter->cpu_file;
1454         u64 next_ts = 0, ts;
1455         int next_cpu = -1;
1456         int cpu;
1457
1458         /*
1459          * If we are in a per_cpu trace file, don't bother by iterating over
1460          * all cpu and peek directly.
1461          */
1462         if (cpu_file > TRACE_PIPE_ALL_CPU) {
1463                 if (ring_buffer_empty_cpu(buffer, cpu_file))
1464                         return NULL;
1465                 ent = peek_next_entry(iter, cpu_file, ent_ts);
1466                 if (ent_cpu)
1467                         *ent_cpu = cpu_file;
1468
1469                 return ent;
1470         }
1471
1472         for_each_tracing_cpu(cpu) {
1473
1474                 if (ring_buffer_empty_cpu(buffer, cpu))
1475                         continue;
1476
1477                 ent = peek_next_entry(iter, cpu, &ts);
1478
1479                 /*
1480                  * Pick the entry with the smallest timestamp:
1481                  */
1482                 if (ent && (!next || ts < next_ts)) {
1483                         next = ent;
1484                         next_cpu = cpu;
1485                         next_ts = ts;
1486                 }
1487         }
1488
1489         if (ent_cpu)
1490                 *ent_cpu = next_cpu;
1491
1492         if (ent_ts)
1493                 *ent_ts = next_ts;
1494
1495         return next;
1496 }
1497
1498 /* Find the next real entry, without updating the iterator itself */
1499 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
1500                                           int *ent_cpu, u64 *ent_ts)
1501 {
1502         return __find_next_entry(iter, ent_cpu, ent_ts);
1503 }
1504
1505 /* Find the next real entry, and increment the iterator to the next entry */
1506 static void *find_next_entry_inc(struct trace_iterator *iter)
1507 {
1508         iter->ent = __find_next_entry(iter, &iter->cpu, &iter->ts);
1509
1510         if (iter->ent)
1511                 trace_iterator_increment(iter);
1512
1513         return iter->ent ? iter : NULL;
1514 }
1515
1516 static void trace_consume(struct trace_iterator *iter)
1517 {
1518         /* Don't allow ftrace to trace into the ring buffers */
1519         ftrace_disable_cpu();
1520         ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts);
1521         ftrace_enable_cpu();
1522 }
1523
1524 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
1525 {
1526         struct trace_iterator *iter = m->private;
1527         int i = (int)*pos;
1528         void *ent;
1529
1530         WARN_ON_ONCE(iter->leftover);
1531
1532         (*pos)++;
1533
1534         /* can't go backwards */
1535         if (iter->idx > i)
1536                 return NULL;
1537
1538         if (iter->idx < 0)
1539                 ent = find_next_entry_inc(iter);
1540         else
1541                 ent = iter;
1542
1543         while (ent && iter->idx < i)
1544                 ent = find_next_entry_inc(iter);
1545
1546         iter->pos = *pos;
1547
1548         return ent;
1549 }
1550
1551 static void tracing_iter_reset(struct trace_iterator *iter, int cpu)
1552 {
1553         struct trace_array *tr = iter->tr;
1554         struct ring_buffer_event *event;
1555         struct ring_buffer_iter *buf_iter;
1556         unsigned long entries = 0;
1557         u64 ts;
1558
1559         tr->data[cpu]->skipped_entries = 0;
1560
1561         if (!iter->buffer_iter[cpu])
1562                 return;
1563
1564         buf_iter = iter->buffer_iter[cpu];
1565         ring_buffer_iter_reset(buf_iter);
1566
1567         /*
1568          * We could have the case with the max latency tracers
1569          * that a reset never took place on a cpu. This is evident
1570          * by the timestamp being before the start of the buffer.
1571          */
1572         while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
1573                 if (ts >= iter->tr->time_start)
1574                         break;
1575                 entries++;
1576                 ring_buffer_read(buf_iter, NULL);
1577         }
1578
1579         tr->data[cpu]->skipped_entries = entries;
1580 }
1581
1582 /*
1583  * No necessary locking here. The worst thing which can
1584  * happen is loosing events consumed at the same time
1585  * by a trace_pipe reader.
1586  * Other than that, we don't risk to crash the ring buffer
1587  * because it serializes the readers.
1588  *
1589  * The current tracer is copied to avoid a global locking
1590  * all around.
1591  */
1592 static void *s_start(struct seq_file *m, loff_t *pos)
1593 {
1594         struct trace_iterator *iter = m->private;
1595         static struct tracer *old_tracer;
1596         int cpu_file = iter->cpu_file;
1597         void *p = NULL;
1598         loff_t l = 0;
1599         int cpu;
1600
1601         /* copy the tracer to avoid using a global lock all around */
1602         mutex_lock(&trace_types_lock);
1603         if (unlikely(old_tracer != current_trace && current_trace)) {
1604                 old_tracer = current_trace;
1605                 *iter->trace = *current_trace;
1606         }
1607         mutex_unlock(&trace_types_lock);
1608
1609         atomic_inc(&trace_record_cmdline_disabled);
1610
1611         if (*pos != iter->pos) {
1612                 iter->ent = NULL;
1613                 iter->cpu = 0;
1614                 iter->idx = -1;
1615
1616                 ftrace_disable_cpu();
1617
1618                 if (cpu_file == TRACE_PIPE_ALL_CPU) {
1619                         for_each_tracing_cpu(cpu)
1620                                 tracing_iter_reset(iter, cpu);
1621                 } else
1622                         tracing_iter_reset(iter, cpu_file);
1623
1624                 ftrace_enable_cpu();
1625
1626                 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1627                         ;
1628
1629         } else {
1630                 /*
1631                  * If we overflowed the seq_file before, then we want
1632                  * to just reuse the trace_seq buffer again.
1633                  */
1634                 if (iter->leftover)
1635                         p = iter;
1636                 else {
1637                         l = *pos - 1;
1638                         p = s_next(m, p, &l);
1639                 }
1640         }
1641
1642         trace_event_read_lock();
1643         return p;
1644 }
1645
1646 static void s_stop(struct seq_file *m, void *p)
1647 {
1648         atomic_dec(&trace_record_cmdline_disabled);
1649         trace_event_read_unlock();
1650 }
1651
1652 static void print_lat_help_header(struct seq_file *m)
1653 {
1654         seq_puts(m, "#                  _------=> CPU#            \n");
1655         seq_puts(m, "#                 / _-----=> irqs-off        \n");
1656         seq_puts(m, "#                | / _----=> need-resched    \n");
1657         seq_puts(m, "#                || / _---=> hardirq/softirq \n");
1658         seq_puts(m, "#                ||| / _--=> preempt-depth   \n");
1659         seq_puts(m, "#                |||| /_--=> lock-depth       \n");
1660         seq_puts(m, "#                |||||/     delay             \n");
1661         seq_puts(m, "#  cmd     pid   |||||| time  |   caller      \n");
1662         seq_puts(m, "#     \\   /      ||||||   \\   |   /           \n");
1663 }
1664
1665 static void print_func_help_header(struct seq_file *m)
1666 {
1667         seq_puts(m, "#           TASK-PID    CPU#    TIMESTAMP  FUNCTION\n");
1668         seq_puts(m, "#              | |       |          |         |\n");
1669 }
1670
1671
1672 static void
1673 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1674 {
1675         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1676         struct trace_array *tr = iter->tr;
1677         struct trace_array_cpu *data = tr->data[tr->cpu];
1678         struct tracer *type = current_trace;
1679         unsigned long entries = 0;
1680         unsigned long total = 0;
1681         unsigned long count;
1682         const char *name = "preemption";
1683         int cpu;
1684
1685         if (type)
1686                 name = type->name;
1687
1688
1689         for_each_tracing_cpu(cpu) {
1690                 count = ring_buffer_entries_cpu(tr->buffer, cpu);
1691                 /*
1692                  * If this buffer has skipped entries, then we hold all
1693                  * entries for the trace and we need to ignore the
1694                  * ones before the time stamp.
1695                  */
1696                 if (tr->data[cpu]->skipped_entries) {
1697                         count -= tr->data[cpu]->skipped_entries;
1698                         /* total is the same as the entries */
1699                         total += count;
1700                 } else
1701                         total += count +
1702                                 ring_buffer_overrun_cpu(tr->buffer, cpu);
1703                 entries += count;
1704         }
1705
1706         seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
1707                    name, UTS_RELEASE);
1708         seq_puts(m, "# -----------------------------------"
1709                  "---------------------------------\n");
1710         seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
1711                    " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
1712                    nsecs_to_usecs(data->saved_latency),
1713                    entries,
1714                    total,
1715                    tr->cpu,
1716 #if defined(CONFIG_PREEMPT_NONE)
1717                    "server",
1718 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
1719                    "desktop",
1720 #elif defined(CONFIG_PREEMPT)
1721                    "preempt",
1722 #else
1723                    "unknown",
1724 #endif
1725                    /* These are reserved for later use */
1726                    0, 0, 0, 0);
1727 #ifdef CONFIG_SMP
1728         seq_printf(m, " #P:%d)\n", num_online_cpus());
1729 #else
1730         seq_puts(m, ")\n");
1731 #endif
1732         seq_puts(m, "#    -----------------\n");
1733         seq_printf(m, "#    | task: %.16s-%d "
1734                    "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
1735                    data->comm, data->pid, data->uid, data->nice,
1736                    data->policy, data->rt_priority);
1737         seq_puts(m, "#    -----------------\n");
1738
1739         if (data->critical_start) {
1740                 seq_puts(m, "#  => started at: ");
1741                 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
1742                 trace_print_seq(m, &iter->seq);
1743                 seq_puts(m, "\n#  => ended at:   ");
1744                 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
1745                 trace_print_seq(m, &iter->seq);
1746                 seq_puts(m, "\n#\n");
1747         }
1748
1749         seq_puts(m, "#\n");
1750 }
1751
1752 static void test_cpu_buff_start(struct trace_iterator *iter)
1753 {
1754         struct trace_seq *s = &iter->seq;
1755
1756         if (!(trace_flags & TRACE_ITER_ANNOTATE))
1757                 return;
1758
1759         if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
1760                 return;
1761
1762         if (cpumask_test_cpu(iter->cpu, iter->started))
1763                 return;
1764
1765         if (iter->tr->data[iter->cpu]->skipped_entries)
1766                 return;
1767
1768         cpumask_set_cpu(iter->cpu, iter->started);
1769
1770         /* Don't print started cpu buffer for the first entry of the trace */
1771         if (iter->idx > 1)
1772                 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
1773                                 iter->cpu);
1774 }
1775
1776 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
1777 {
1778         struct trace_seq *s = &iter->seq;
1779         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1780         struct trace_entry *entry;
1781         struct trace_event *event;
1782
1783         entry = iter->ent;
1784
1785         test_cpu_buff_start(iter);
1786
1787         event = ftrace_find_event(entry->type);
1788
1789         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1790                 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
1791                         if (!trace_print_lat_context(iter))
1792                                 goto partial;
1793                 } else {
1794                         if (!trace_print_context(iter))
1795                                 goto partial;
1796                 }
1797         }
1798
1799         if (event)
1800                 return event->trace(iter, sym_flags);
1801
1802         if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
1803                 goto partial;
1804
1805         return TRACE_TYPE_HANDLED;
1806 partial:
1807         return TRACE_TYPE_PARTIAL_LINE;
1808 }
1809
1810 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
1811 {
1812         struct trace_seq *s = &iter->seq;
1813         struct trace_entry *entry;
1814         struct trace_event *event;
1815
1816         entry = iter->ent;
1817
1818         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1819                 if (!trace_seq_printf(s, "%d %d %llu ",
1820                                       entry->pid, iter->cpu, iter->ts))
1821                         goto partial;
1822         }
1823
1824         event = ftrace_find_event(entry->type);
1825         if (event)
1826                 return event->raw(iter, 0);
1827
1828         if (!trace_seq_printf(s, "%d ?\n", entry->type))
1829                 goto partial;
1830
1831         return TRACE_TYPE_HANDLED;
1832 partial:
1833         return TRACE_TYPE_PARTIAL_LINE;
1834 }
1835
1836 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
1837 {
1838         struct trace_seq *s = &iter->seq;
1839         unsigned char newline = '\n';
1840         struct trace_entry *entry;
1841         struct trace_event *event;
1842
1843         entry = iter->ent;
1844
1845         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1846                 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
1847                 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
1848                 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
1849         }
1850
1851         event = ftrace_find_event(entry->type);
1852         if (event) {
1853                 enum print_line_t ret = event->hex(iter, 0);
1854                 if (ret != TRACE_TYPE_HANDLED)
1855                         return ret;
1856         }
1857
1858         SEQ_PUT_FIELD_RET(s, newline);
1859
1860         return TRACE_TYPE_HANDLED;
1861 }
1862
1863 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
1864 {
1865         struct trace_seq *s = &iter->seq;
1866         struct trace_entry *entry;
1867         struct trace_event *event;
1868
1869         entry = iter->ent;
1870
1871         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1872                 SEQ_PUT_FIELD_RET(s, entry->pid);
1873                 SEQ_PUT_FIELD_RET(s, iter->cpu);
1874                 SEQ_PUT_FIELD_RET(s, iter->ts);
1875         }
1876
1877         event = ftrace_find_event(entry->type);
1878         return event ? event->binary(iter, 0) : TRACE_TYPE_HANDLED;
1879 }
1880
1881 static int trace_empty(struct trace_iterator *iter)
1882 {
1883         int cpu;
1884
1885         /* If we are looking at one CPU buffer, only check that one */
1886         if (iter->cpu_file != TRACE_PIPE_ALL_CPU) {
1887                 cpu = iter->cpu_file;
1888                 if (iter->buffer_iter[cpu]) {
1889                         if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
1890                                 return 0;
1891                 } else {
1892                         if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
1893                                 return 0;
1894                 }
1895                 return 1;
1896         }
1897
1898         for_each_tracing_cpu(cpu) {
1899                 if (iter->buffer_iter[cpu]) {
1900                         if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
1901                                 return 0;
1902                 } else {
1903                         if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
1904                                 return 0;
1905                 }
1906         }
1907
1908         return 1;
1909 }
1910
1911 /*  Called with trace_event_read_lock() held. */
1912 static enum print_line_t print_trace_line(struct trace_iterator *iter)
1913 {
1914         enum print_line_t ret;
1915
1916         if (iter->trace && iter->trace->print_line) {
1917                 ret = iter->trace->print_line(iter);
1918                 if (ret != TRACE_TYPE_UNHANDLED)
1919                         return ret;
1920         }
1921
1922         if (iter->ent->type == TRACE_BPRINT &&
1923                         trace_flags & TRACE_ITER_PRINTK &&
1924                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
1925                 return trace_print_bprintk_msg_only(iter);
1926
1927         if (iter->ent->type == TRACE_PRINT &&
1928                         trace_flags & TRACE_ITER_PRINTK &&
1929                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
1930                 return trace_print_printk_msg_only(iter);
1931
1932         if (trace_flags & TRACE_ITER_BIN)
1933                 return print_bin_fmt(iter);
1934
1935         if (trace_flags & TRACE_ITER_HEX)
1936                 return print_hex_fmt(iter);
1937
1938         if (trace_flags & TRACE_ITER_RAW)
1939                 return print_raw_fmt(iter);
1940
1941         return print_trace_fmt(iter);
1942 }
1943
1944 static int s_show(struct seq_file *m, void *v)
1945 {
1946         struct trace_iterator *iter = v;
1947         int ret;
1948
1949         if (iter->ent == NULL) {
1950                 if (iter->tr) {
1951                         seq_printf(m, "# tracer: %s\n", iter->trace->name);
1952                         seq_puts(m, "#\n");
1953                 }
1954                 if (iter->trace && iter->trace->print_header)
1955                         iter->trace->print_header(m);
1956                 else if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
1957                         /* print nothing if the buffers are empty */
1958                         if (trace_empty(iter))
1959                                 return 0;
1960                         print_trace_header(m, iter);
1961                         if (!(trace_flags & TRACE_ITER_VERBOSE))
1962                                 print_lat_help_header(m);
1963                 } else {
1964                         if (!(trace_flags & TRACE_ITER_VERBOSE))
1965                                 print_func_help_header(m);
1966                 }
1967         } else if (iter->leftover) {
1968                 /*
1969                  * If we filled the seq_file buffer earlier, we
1970                  * want to just show it now.
1971                  */
1972                 ret = trace_print_seq(m, &iter->seq);
1973
1974                 /* ret should this time be zero, but you never know */
1975                 iter->leftover = ret;
1976
1977         } else {
1978                 print_trace_line(iter);
1979                 ret = trace_print_seq(m, &iter->seq);
1980                 /*
1981                  * If we overflow the seq_file buffer, then it will
1982                  * ask us for this data again at start up.
1983                  * Use that instead.
1984                  *  ret is 0 if seq_file write succeeded.
1985                  *        -1 otherwise.
1986                  */
1987                 iter->leftover = ret;
1988         }
1989
1990         return 0;
1991 }
1992
1993 static const struct seq_operations tracer_seq_ops = {
1994         .start          = s_start,
1995         .next           = s_next,
1996         .stop           = s_stop,
1997         .show           = s_show,
1998 };
1999
2000 static struct trace_iterator *
2001 __tracing_open(struct inode *inode, struct file *file)
2002 {
2003         long cpu_file = (long) inode->i_private;
2004         void *fail_ret = ERR_PTR(-ENOMEM);
2005         struct trace_iterator *iter;
2006         struct seq_file *m;
2007         int cpu, ret;
2008
2009         if (tracing_disabled)
2010                 return ERR_PTR(-ENODEV);
2011
2012         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2013         if (!iter)
2014                 return ERR_PTR(-ENOMEM);
2015
2016         /*
2017          * We make a copy of the current tracer to avoid concurrent
2018          * changes on it while we are reading.
2019          */
2020         mutex_lock(&trace_types_lock);
2021         iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2022         if (!iter->trace)
2023                 goto fail;
2024
2025         if (current_trace)
2026                 *iter->trace = *current_trace;
2027
2028         if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2029                 goto fail;
2030
2031         if (current_trace && current_trace->print_max)
2032                 iter->tr = &max_tr;
2033         else
2034                 iter->tr = &global_trace;
2035         iter->pos = -1;
2036         mutex_init(&iter->mutex);
2037         iter->cpu_file = cpu_file;
2038
2039         /* Notify the tracer early; before we stop tracing. */
2040         if (iter->trace && iter->trace->open)
2041                 iter->trace->open(iter);
2042
2043         /* Annotate start of buffers if we had overruns */
2044         if (ring_buffer_overruns(iter->tr->buffer))
2045                 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2046
2047         /* stop the trace while dumping */
2048         tracing_stop();
2049
2050         if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
2051                 for_each_tracing_cpu(cpu) {
2052
2053                         iter->buffer_iter[cpu] =
2054                                 ring_buffer_read_start(iter->tr->buffer, cpu);
2055                         tracing_iter_reset(iter, cpu);
2056                 }
2057         } else {
2058                 cpu = iter->cpu_file;
2059                 iter->buffer_iter[cpu] =
2060                                 ring_buffer_read_start(iter->tr->buffer, cpu);
2061                 tracing_iter_reset(iter, cpu);
2062         }
2063
2064         ret = seq_open(file, &tracer_seq_ops);
2065         if (ret < 0) {
2066                 fail_ret = ERR_PTR(ret);
2067                 goto fail_buffer;
2068         }
2069
2070         m = file->private_data;
2071         m->private = iter;
2072
2073         mutex_unlock(&trace_types_lock);
2074
2075         return iter;
2076
2077  fail_buffer:
2078         for_each_tracing_cpu(cpu) {
2079                 if (iter->buffer_iter[cpu])
2080                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
2081         }
2082         free_cpumask_var(iter->started);
2083         tracing_start();
2084  fail:
2085         mutex_unlock(&trace_types_lock);
2086         kfree(iter->trace);
2087         kfree(iter);
2088
2089         return fail_ret;
2090 }
2091
2092 int tracing_open_generic(struct inode *inode, struct file *filp)
2093 {
2094         if (tracing_disabled)
2095                 return -ENODEV;
2096
2097         filp->private_data = inode->i_private;
2098         return 0;
2099 }
2100
2101 static int tracing_release(struct inode *inode, struct file *file)
2102 {
2103         struct seq_file *m = (struct seq_file *)file->private_data;
2104         struct trace_iterator *iter;
2105         int cpu;
2106
2107         if (!(file->f_mode & FMODE_READ))
2108                 return 0;
2109
2110         iter = m->private;
2111
2112         mutex_lock(&trace_types_lock);
2113         for_each_tracing_cpu(cpu) {
2114                 if (iter->buffer_iter[cpu])
2115                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
2116         }
2117
2118         if (iter->trace && iter->trace->close)
2119                 iter->trace->close(iter);
2120
2121         /* reenable tracing if it was previously enabled */
2122         tracing_start();
2123         mutex_unlock(&trace_types_lock);
2124
2125         seq_release(inode, file);
2126         mutex_destroy(&iter->mutex);
2127         free_cpumask_var(iter->started);
2128         kfree(iter->trace);
2129         kfree(iter);
2130         return 0;
2131 }
2132
2133 static int tracing_open(struct inode *inode, struct file *file)
2134 {
2135         struct trace_iterator *iter;
2136         int ret = 0;
2137
2138         /* If this file was open for write, then erase contents */
2139         if ((file->f_mode & FMODE_WRITE) &&
2140             (file->f_flags & O_TRUNC)) {
2141                 long cpu = (long) inode->i_private;
2142
2143                 if (cpu == TRACE_PIPE_ALL_CPU)
2144                         tracing_reset_online_cpus(&global_trace);
2145                 else
2146                         tracing_reset(&global_trace, cpu);
2147         }
2148
2149         if (file->f_mode & FMODE_READ) {
2150                 iter = __tracing_open(inode, file);
2151                 if (IS_ERR(iter))
2152                         ret = PTR_ERR(iter);
2153                 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2154                         iter->iter_flags |= TRACE_FILE_LAT_FMT;
2155         }
2156         return ret;
2157 }
2158
2159 static void *
2160 t_next(struct seq_file *m, void *v, loff_t *pos)
2161 {
2162         struct tracer *t = v;
2163
2164         (*pos)++;
2165
2166         if (t)
2167                 t = t->next;
2168
2169         return t;
2170 }
2171
2172 static void *t_start(struct seq_file *m, loff_t *pos)
2173 {
2174         struct tracer *t;
2175         loff_t l = 0;
2176
2177         mutex_lock(&trace_types_lock);
2178         for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
2179                 ;
2180
2181         return t;
2182 }
2183
2184 static void t_stop(struct seq_file *m, void *p)
2185 {
2186         mutex_unlock(&trace_types_lock);
2187 }
2188
2189 static int t_show(struct seq_file *m, void *v)
2190 {
2191         struct tracer *t = v;
2192
2193         if (!t)
2194                 return 0;
2195
2196         seq_printf(m, "%s", t->name);
2197         if (t->next)
2198                 seq_putc(m, ' ');
2199         else
2200                 seq_putc(m, '\n');
2201
2202         return 0;
2203 }
2204
2205 static const struct seq_operations show_traces_seq_ops = {
2206         .start          = t_start,
2207         .next           = t_next,
2208         .stop           = t_stop,
2209         .show           = t_show,
2210 };
2211
2212 static int show_traces_open(struct inode *inode, struct file *file)
2213 {
2214         if (tracing_disabled)
2215                 return -ENODEV;
2216
2217         return seq_open(file, &show_traces_seq_ops);
2218 }
2219
2220 static ssize_t
2221 tracing_write_stub(struct file *filp, const char __user *ubuf,
2222                    size_t count, loff_t *ppos)
2223 {
2224         return count;
2225 }
2226
2227 static const struct file_operations tracing_fops = {
2228         .open           = tracing_open,
2229         .read           = seq_read,
2230         .write          = tracing_write_stub,
2231         .llseek         = seq_lseek,
2232         .release        = tracing_release,
2233 };
2234
2235 static const struct file_operations show_traces_fops = {
2236         .open           = show_traces_open,
2237         .read           = seq_read,
2238         .release        = seq_release,
2239 };
2240
2241 /*
2242  * Only trace on a CPU if the bitmask is set:
2243  */
2244 static cpumask_var_t tracing_cpumask;
2245
2246 /*
2247  * The tracer itself will not take this lock, but still we want
2248  * to provide a consistent cpumask to user-space:
2249  */
2250 static DEFINE_MUTEX(tracing_cpumask_update_lock);
2251
2252 /*
2253  * Temporary storage for the character representation of the
2254  * CPU bitmask (and one more byte for the newline):
2255  */
2256 static char mask_str[NR_CPUS + 1];
2257
2258 static ssize_t
2259 tracing_cpumask_read(struct file *filp, char __user *ubuf,
2260                      size_t count, loff_t *ppos)
2261 {
2262         int len;
2263
2264         mutex_lock(&tracing_cpumask_update_lock);
2265
2266         len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
2267         if (count - len < 2) {
2268                 count = -EINVAL;
2269                 goto out_err;
2270         }
2271         len += sprintf(mask_str + len, "\n");
2272         count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2273
2274 out_err:
2275         mutex_unlock(&tracing_cpumask_update_lock);
2276
2277         return count;
2278 }
2279
2280 static ssize_t
2281 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2282                       size_t count, loff_t *ppos)
2283 {
2284         int err, cpu;
2285         cpumask_var_t tracing_cpumask_new;
2286
2287         if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
2288                 return -ENOMEM;
2289
2290         err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
2291         if (err)
2292                 goto err_unlock;
2293
2294         mutex_lock(&tracing_cpumask_update_lock);
2295
2296         local_irq_disable();
2297         arch_spin_lock(&ftrace_max_lock);
2298         for_each_tracing_cpu(cpu) {
2299                 /*
2300                  * Increase/decrease the disabled counter if we are
2301                  * about to flip a bit in the cpumask:
2302                  */
2303                 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2304                                 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2305                         atomic_inc(&global_trace.data[cpu]->disabled);
2306                 }
2307                 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2308                                 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2309                         atomic_dec(&global_trace.data[cpu]->disabled);
2310                 }
2311         }
2312         arch_spin_unlock(&ftrace_max_lock);
2313         local_irq_enable();
2314
2315         cpumask_copy(tracing_cpumask, tracing_cpumask_new);
2316
2317         mutex_unlock(&tracing_cpumask_update_lock);
2318         free_cpumask_var(tracing_cpumask_new);
2319
2320         return count;
2321
2322 err_unlock:
2323         free_cpumask_var(tracing_cpumask_new);
2324
2325         return err;
2326 }
2327
2328 static const struct file_operations tracing_cpumask_fops = {
2329         .open           = tracing_open_generic,
2330         .read           = tracing_cpumask_read,
2331         .write          = tracing_cpumask_write,
2332 };
2333
2334 static int tracing_trace_options_show(struct seq_file *m, void *v)
2335 {
2336         struct tracer_opt *trace_opts;
2337         u32 tracer_flags;
2338         int i;
2339
2340         mutex_lock(&trace_types_lock);
2341         tracer_flags = current_trace->flags->val;
2342         trace_opts = current_trace->flags->opts;
2343
2344         for (i = 0; trace_options[i]; i++) {
2345                 if (trace_flags & (1 << i))
2346                         seq_printf(m, "%s\n", trace_options[i]);
2347                 else
2348                         seq_printf(m, "no%s\n", trace_options[i]);
2349         }
2350
2351         for (i = 0; trace_opts[i].name; i++) {
2352                 if (tracer_flags & trace_opts[i].bit)
2353                         seq_printf(m, "%s\n", trace_opts[i].name);
2354                 else
2355                         seq_printf(m, "no%s\n", trace_opts[i].name);
2356         }
2357         mutex_unlock(&trace_types_lock);
2358
2359         return 0;
2360 }
2361
2362 static int __set_tracer_option(struct tracer *trace,
2363                                struct tracer_flags *tracer_flags,
2364                                struct tracer_opt *opts, int neg)
2365 {
2366         int ret;
2367
2368         ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
2369         if (ret)
2370                 return ret;
2371
2372         if (neg)
2373                 tracer_flags->val &= ~opts->bit;
2374         else
2375                 tracer_flags->val |= opts->bit;
2376         return 0;
2377 }
2378
2379 /* Try to assign a tracer specific option */
2380 static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
2381 {
2382         struct tracer_flags *tracer_flags = trace->flags;
2383         struct tracer_opt *opts = NULL;
2384         int i;
2385
2386         for (i = 0; tracer_flags->opts[i].name; i++) {
2387                 opts = &tracer_flags->opts[i];
2388
2389                 if (strcmp(cmp, opts->name) == 0)
2390                         return __set_tracer_option(trace, trace->flags,
2391                                                    opts, neg);
2392         }
2393
2394         return -EINVAL;
2395 }
2396
2397 static void set_tracer_flags(unsigned int mask, int enabled)
2398 {
2399         /* do nothing if flag is already set */
2400         if (!!(trace_flags & mask) == !!enabled)
2401                 return;
2402
2403         if (enabled)
2404                 trace_flags |= mask;
2405         else
2406                 trace_flags &= ~mask;
2407 }
2408
2409 static ssize_t
2410 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
2411                         size_t cnt, loff_t *ppos)
2412 {
2413         char buf[64];
2414         char *cmp;
2415         int neg = 0;
2416         int ret;
2417         int i;
2418
2419         if (cnt >= sizeof(buf))
2420                 return -EINVAL;
2421
2422         if (copy_from_user(&buf, ubuf, cnt))
2423                 return -EFAULT;
2424
2425         buf[cnt] = 0;
2426         cmp = strstrip(buf);
2427
2428         if (strncmp(cmp, "no", 2) == 0) {
2429                 neg = 1;
2430                 cmp += 2;
2431         }
2432
2433         for (i = 0; trace_options[i]; i++) {
2434                 if (strcmp(cmp, trace_options[i]) == 0) {
2435                         set_tracer_flags(1 << i, !neg);
2436                         break;
2437                 }
2438         }
2439
2440         /* If no option could be set, test the specific tracer options */
2441         if (!trace_options[i]) {
2442                 mutex_lock(&trace_types_lock);
2443                 ret = set_tracer_option(current_trace, cmp, neg);
2444                 mutex_unlock(&trace_types_lock);
2445                 if (ret)
2446                         return ret;
2447         }
2448
2449         *ppos += cnt;
2450
2451         return cnt;
2452 }
2453
2454 static int tracing_trace_options_open(struct inode *inode, struct file *file)
2455 {
2456         if (tracing_disabled)
2457                 return -ENODEV;
2458         return single_open(file, tracing_trace_options_show, NULL);
2459 }
2460
2461 static const struct file_operations tracing_iter_fops = {
2462         .open           = tracing_trace_options_open,
2463         .read           = seq_read,
2464         .llseek         = seq_lseek,
2465         .release        = single_release,
2466         .write          = tracing_trace_options_write,
2467 };
2468
2469 static const char readme_msg[] =
2470         "tracing mini-HOWTO:\n\n"
2471         "# mount -t debugfs nodev /sys/kernel/debug\n\n"
2472         "# cat /sys/kernel/debug/tracing/available_tracers\n"
2473         "wakeup preemptirqsoff preemptoff irqsoff function sched_switch nop\n\n"
2474         "# cat /sys/kernel/debug/tracing/current_tracer\n"
2475         "nop\n"
2476         "# echo sched_switch > /sys/kernel/debug/tracing/current_tracer\n"
2477         "# cat /sys/kernel/debug/tracing/current_tracer\n"
2478         "sched_switch\n"
2479         "# cat /sys/kernel/debug/tracing/trace_options\n"
2480         "noprint-parent nosym-offset nosym-addr noverbose\n"
2481         "# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
2482         "# echo 1 > /sys/kernel/debug/tracing/tracing_enabled\n"
2483         "# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
2484         "# echo 0 > /sys/kernel/debug/tracing/tracing_enabled\n"
2485 ;
2486
2487 static ssize_t
2488 tracing_readme_read(struct file *filp, char __user *ubuf,
2489                        size_t cnt, loff_t *ppos)
2490 {
2491         return simple_read_from_buffer(ubuf, cnt, ppos,
2492                                         readme_msg, strlen(readme_msg));
2493 }
2494
2495 static const struct file_operations tracing_readme_fops = {
2496         .open           = tracing_open_generic,
2497         .read           = tracing_readme_read,
2498 };
2499
2500 static ssize_t
2501 tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
2502                                 size_t cnt, loff_t *ppos)
2503 {
2504         char *buf_comm;
2505         char *file_buf;
2506         char *buf;
2507         int len = 0;
2508         int pid;
2509         int i;
2510
2511         file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
2512         if (!file_buf)
2513                 return -ENOMEM;
2514
2515         buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
2516         if (!buf_comm) {
2517                 kfree(file_buf);
2518                 return -ENOMEM;
2519         }
2520
2521         buf = file_buf;
2522
2523         for (i = 0; i < SAVED_CMDLINES; i++) {
2524                 int r;
2525
2526                 pid = map_cmdline_to_pid[i];
2527                 if (pid == -1 || pid == NO_CMDLINE_MAP)
2528                         continue;
2529
2530                 trace_find_cmdline(pid, buf_comm);
2531                 r = sprintf(buf, "%d %s\n", pid, buf_comm);
2532                 buf += r;
2533                 len += r;
2534         }
2535
2536         len = simple_read_from_buffer(ubuf, cnt, ppos,
2537                                       file_buf, len);
2538
2539         kfree(file_buf);
2540         kfree(buf_comm);
2541
2542         return len;
2543 }
2544
2545 static const struct file_operations tracing_saved_cmdlines_fops = {
2546     .open       = tracing_open_generic,
2547     .read       = tracing_saved_cmdlines_read,
2548 };
2549
2550 static ssize_t
2551 tracing_ctrl_read(struct file *filp, char __user *ubuf,
2552                   size_t cnt, loff_t *ppos)
2553 {
2554         char buf[64];
2555         int r;
2556
2557         r = sprintf(buf, "%u\n", tracer_enabled);
2558         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2559 }
2560
2561 static ssize_t
2562 tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2563                    size_t cnt, loff_t *ppos)
2564 {
2565         struct trace_array *tr = filp->private_data;
2566         char buf[64];
2567         unsigned long val;
2568         int ret;
2569
2570         if (cnt >= sizeof(buf))
2571                 return -EINVAL;
2572
2573         if (copy_from_user(&buf, ubuf, cnt))
2574                 return -EFAULT;
2575
2576         buf[cnt] = 0;
2577
2578         ret = strict_strtoul(buf, 10, &val);
2579         if (ret < 0)
2580                 return ret;
2581
2582         val = !!val;
2583
2584         mutex_lock(&trace_types_lock);
2585         if (tracer_enabled ^ val) {
2586                 if (val) {
2587                         tracer_enabled = 1;
2588                         if (current_trace->start)
2589                                 current_trace->start(tr);
2590                         tracing_start();
2591                 } else {
2592                         tracer_enabled = 0;
2593                         tracing_stop();
2594                         if (current_trace->stop)
2595                                 current_trace->stop(tr);
2596                 }
2597         }
2598         mutex_unlock(&trace_types_lock);
2599
2600         *ppos += cnt;
2601
2602         return cnt;
2603 }
2604
2605 static ssize_t
2606 tracing_set_trace_read(struct file *filp, char __user *ubuf,
2607                        size_t cnt, loff_t *ppos)
2608 {
2609         char buf[MAX_TRACER_SIZE+2];
2610         int r;
2611
2612         mutex_lock(&trace_types_lock);
2613         if (current_trace)
2614                 r = sprintf(buf, "%s\n", current_trace->name);
2615         else
2616                 r = sprintf(buf, "\n");
2617         mutex_unlock(&trace_types_lock);
2618
2619         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2620 }
2621
2622 int tracer_init(struct tracer *t, struct trace_array *tr)
2623 {
2624         tracing_reset_online_cpus(tr);
2625         return t->init(tr);
2626 }
2627
2628 static int tracing_resize_ring_buffer(unsigned long size)
2629 {
2630         int ret;
2631
2632         /*
2633          * If kernel or user changes the size of the ring buffer
2634          * we use the size that was given, and we can forget about
2635          * expanding it later.
2636          */
2637         ring_buffer_expanded = 1;
2638
2639         ret = ring_buffer_resize(global_trace.buffer, size);
2640         if (ret < 0)
2641                 return ret;
2642
2643         ret = ring_buffer_resize(max_tr.buffer, size);
2644         if (ret < 0) {
2645                 int r;
2646
2647                 r = ring_buffer_resize(global_trace.buffer,
2648                                        global_trace.entries);
2649                 if (r < 0) {
2650                         /*
2651                          * AARGH! We are left with different
2652                          * size max buffer!!!!
2653                          * The max buffer is our "snapshot" buffer.
2654                          * When a tracer needs a snapshot (one of the
2655                          * latency tracers), it swaps the max buffer
2656                          * with the saved snap shot. We succeeded to
2657                          * update the size of the main buffer, but failed to
2658                          * update the size of the max buffer. But when we tried
2659                          * to reset the main buffer to the original size, we
2660                          * failed there too. This is very unlikely to
2661                          * happen, but if it does, warn and kill all
2662                          * tracing.
2663                          */
2664                         WARN_ON(1);
2665                         tracing_disabled = 1;
2666                 }
2667                 return ret;
2668         }
2669
2670         global_trace.entries = size;
2671
2672         return ret;
2673 }
2674
2675 /**
2676  * tracing_update_buffers - used by tracing facility to expand ring buffers
2677  *
2678  * To save on memory when the tracing is never used on a system with it
2679  * configured in. The ring buffers are set to a minimum size. But once
2680  * a user starts to use the tracing facility, then they need to grow
2681  * to their default size.
2682  *
2683  * This function is to be called when a tracer is about to be used.
2684  */
2685 int tracing_update_buffers(void)
2686 {
2687         int ret = 0;
2688
2689         mutex_lock(&trace_types_lock);
2690         if (!ring_buffer_expanded)
2691                 ret = tracing_resize_ring_buffer(trace_buf_size);
2692         mutex_unlock(&trace_types_lock);
2693
2694         return ret;
2695 }
2696
2697 struct trace_option_dentry;
2698
2699 static struct trace_option_dentry *
2700 create_trace_option_files(struct tracer *tracer);
2701
2702 static void
2703 destroy_trace_option_files(struct trace_option_dentry *topts);
2704
2705 static int tracing_set_tracer(const char *buf)
2706 {
2707         static struct trace_option_dentry *topts;
2708         struct trace_array *tr = &global_trace;
2709         struct tracer *t;
2710         int ret = 0;
2711
2712         mutex_lock(&trace_types_lock);
2713
2714         if (!ring_buffer_expanded) {
2715                 ret = tracing_resize_ring_buffer(trace_buf_size);
2716                 if (ret < 0)
2717                         goto out;
2718                 ret = 0;
2719         }
2720
2721         for (t = trace_types; t; t = t->next) {
2722                 if (strcmp(t->name, buf) == 0)
2723                         break;
2724         }
2725         if (!t) {
2726                 ret = -EINVAL;
2727                 goto out;
2728         }
2729         if (t == current_trace)
2730                 goto out;
2731
2732         trace_branch_disable();
2733         if (current_trace && current_trace->reset)
2734                 current_trace->reset(tr);
2735
2736         destroy_trace_option_files(topts);
2737
2738         current_trace = t;
2739
2740         topts = create_trace_option_files(current_trace);
2741
2742         if (t->init) {
2743                 ret = tracer_init(t, tr);
2744                 if (ret)
2745                         goto out;
2746         }
2747
2748         trace_branch_enable(tr);
2749  out:
2750         mutex_unlock(&trace_types_lock);
2751
2752         return ret;
2753 }
2754
2755 static ssize_t
2756 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
2757                         size_t cnt, loff_t *ppos)
2758 {
2759         char buf[MAX_TRACER_SIZE+1];
2760         int i;
2761         size_t ret;
2762         int err;
2763
2764         ret = cnt;
2765
2766         if (cnt > MAX_TRACER_SIZE)
2767                 cnt = MAX_TRACER_SIZE;
2768
2769         if (copy_from_user(&buf, ubuf, cnt))
2770                 return -EFAULT;
2771
2772         buf[cnt] = 0;
2773
2774         /* strip ending whitespace. */
2775         for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
2776                 buf[i] = 0;
2777
2778         err = tracing_set_tracer(buf);
2779         if (err)
2780                 return err;
2781
2782         *ppos += ret;
2783
2784         return ret;
2785 }
2786
2787 static ssize_t
2788 tracing_max_lat_read(struct file *filp, char __user *ubuf,
2789                      size_t cnt, loff_t *ppos)
2790 {
2791         unsigned long *ptr = filp->private_data;
2792         char buf[64];
2793         int r;
2794
2795         r = snprintf(buf, sizeof(buf), "%ld\n",
2796                      *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
2797         if (r > sizeof(buf))
2798                 r = sizeof(buf);
2799         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2800 }
2801
2802 static ssize_t
2803 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
2804                       size_t cnt, loff_t *ppos)
2805 {
2806         unsigned long *ptr = filp->private_data;
2807         char buf[64];
2808         unsigned long val;
2809         int ret;
2810
2811         if (cnt >= sizeof(buf))
2812                 return -EINVAL;
2813
2814         if (copy_from_user(&buf, ubuf, cnt))
2815                 return -EFAULT;
2816
2817         buf[cnt] = 0;
2818
2819         ret = strict_strtoul(buf, 10, &val);
2820         if (ret < 0)
2821                 return ret;
2822
2823         *ptr = val * 1000;
2824
2825         return cnt;
2826 }
2827
2828 static int tracing_open_pipe(struct inode *inode, struct file *filp)
2829 {
2830         long cpu_file = (long) inode->i_private;
2831         struct trace_iterator *iter;
2832         int ret = 0;
2833
2834         if (tracing_disabled)
2835                 return -ENODEV;
2836
2837         mutex_lock(&trace_types_lock);
2838
2839         /* We only allow one reader per cpu */
2840         if (cpu_file == TRACE_PIPE_ALL_CPU) {
2841                 if (!cpumask_empty(tracing_reader_cpumask)) {
2842                         ret = -EBUSY;
2843                         goto out;
2844                 }
2845                 cpumask_setall(tracing_reader_cpumask);
2846         } else {
2847                 if (!cpumask_test_cpu(cpu_file, tracing_reader_cpumask))
2848                         cpumask_set_cpu(cpu_file, tracing_reader_cpumask);
2849                 else {
2850                         ret = -EBUSY;
2851                         goto out;
2852                 }
2853         }
2854
2855         /* create a buffer to store the information to pass to userspace */
2856         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2857         if (!iter) {
2858                 ret = -ENOMEM;
2859                 goto out;
2860         }
2861
2862         /*
2863          * We make a copy of the current tracer to avoid concurrent
2864          * changes on it while we are reading.
2865          */
2866         iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
2867         if (!iter->trace) {
2868                 ret = -ENOMEM;
2869                 goto fail;
2870         }
2871         if (current_trace)
2872                 *iter->trace = *current_trace;
2873
2874         if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
2875                 ret = -ENOMEM;
2876                 goto fail;
2877         }
2878
2879         /* trace pipe does not show start of buffer */
2880         cpumask_setall(iter->started);
2881
2882         if (trace_flags & TRACE_ITER_LATENCY_FMT)
2883                 iter->iter_flags |= TRACE_FILE_LAT_FMT;
2884
2885         iter->cpu_file = cpu_file;
2886         iter->tr = &global_trace;
2887         mutex_init(&iter->mutex);
2888         filp->private_data = iter;
2889
2890         if (iter->trace->pipe_open)
2891                 iter->trace->pipe_open(iter);
2892
2893 out:
2894         mutex_unlock(&trace_types_lock);
2895         return ret;
2896
2897 fail:
2898         kfree(iter->trace);
2899         kfree(iter);
2900         mutex_unlock(&trace_types_lock);
2901         return ret;
2902 }
2903
2904 static int tracing_release_pipe(struct inode *inode, struct file *file)
2905 {
2906         struct trace_iterator *iter = file->private_data;
2907
2908         mutex_lock(&trace_types_lock);
2909
2910         if (iter->cpu_file == TRACE_PIPE_ALL_CPU)
2911                 cpumask_clear(tracing_reader_cpumask);
2912         else
2913                 cpumask_clear_cpu(iter->cpu_file, tracing_reader_cpumask);
2914
2915
2916         if (iter->trace->pipe_close)
2917                 iter->trace->pipe_close(iter);
2918
2919         mutex_unlock(&trace_types_lock);
2920
2921         free_cpumask_var(iter->started);
2922         mutex_destroy(&iter->mutex);
2923         kfree(iter->trace);
2924         kfree(iter);
2925
2926         return 0;
2927 }
2928
2929 static unsigned int
2930 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
2931 {
2932         struct trace_iterator *iter = filp->private_data;
2933
2934         if (trace_flags & TRACE_ITER_BLOCK) {
2935                 /*
2936                  * Always select as readable when in blocking mode
2937                  */
2938                 return POLLIN | POLLRDNORM;
2939         } else {
2940                 if (!trace_empty(iter))
2941                         return POLLIN | POLLRDNORM;
2942                 poll_wait(filp, &trace_wait, poll_table);
2943                 if (!trace_empty(iter))
2944                         return POLLIN | POLLRDNORM;
2945
2946                 return 0;
2947         }
2948 }
2949
2950
2951 void default_wait_pipe(struct trace_iterator *iter)
2952 {
2953         DEFINE_WAIT(wait);
2954
2955         prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
2956
2957         if (trace_empty(iter))
2958                 schedule();
2959
2960         finish_wait(&trace_wait, &wait);
2961 }
2962
2963 /*
2964  * This is a make-shift waitqueue.
2965  * A tracer might use this callback on some rare cases:
2966  *
2967  *  1) the current tracer might hold the runqueue lock when it wakes up
2968  *     a reader, hence a deadlock (sched, function, and function graph tracers)
2969  *  2) the function tracers, trace all functions, we don't want
2970  *     the overhead of calling wake_up and friends
2971  *     (and tracing them too)
2972  *
2973  *     Anyway, this is really very primitive wakeup.
2974  */
2975 void poll_wait_pipe(struct trace_iterator *iter)
2976 {
2977         set_current_state(TASK_INTERRUPTIBLE);
2978         /* sleep for 100 msecs, and try again. */
2979         schedule_timeout(HZ / 10);
2980 }
2981
2982 /* Must be called with trace_types_lock mutex held. */
2983 static int tracing_wait_pipe(struct file *filp)
2984 {
2985         struct trace_iterator *iter = filp->private_data;
2986
2987         while (trace_empty(iter)) {
2988
2989                 if ((filp->f_flags & O_NONBLOCK)) {
2990                         return -EAGAIN;
2991                 }
2992
2993                 mutex_unlock(&iter->mutex);
2994
2995                 iter->trace->wait_pipe(iter);
2996
2997                 mutex_lock(&iter->mutex);
2998
2999                 if (signal_pending(current))
3000                         return -EINTR;
3001
3002                 /*
3003                  * We block until we read something and tracing is disabled.
3004                  * We still block if tracing is disabled, but we have never
3005                  * read anything. This allows a user to cat this file, and
3006                  * then enable tracing. But after we have read something,
3007                  * we give an EOF when tracing is again disabled.
3008                  *
3009                  * iter->pos will be 0 if we haven't read anything.
3010                  */
3011                 if (!tracer_enabled && iter->pos)
3012                         break;
3013         }
3014
3015         return 1;
3016 }
3017
3018 /*
3019  * Consumer reader.
3020  */
3021 static ssize_t
3022 tracing_read_pipe(struct file *filp, char __user *ubuf,
3023                   size_t cnt, loff_t *ppos)
3024 {
3025         struct trace_iterator *iter = filp->private_data;
3026         static struct tracer *old_tracer;
3027         ssize_t sret;
3028
3029         /* return any leftover data */
3030         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3031         if (sret != -EBUSY)
3032                 return sret;
3033
3034         trace_seq_init(&iter->seq);
3035
3036         /* copy the tracer to avoid using a global lock all around */
3037         mutex_lock(&trace_types_lock);
3038         if (unlikely(old_tracer != current_trace && current_trace)) {
3039                 old_tracer = current_trace;
3040                 *iter->trace = *current_trace;
3041         }
3042         mutex_unlock(&trace_types_lock);
3043
3044         /*
3045          * Avoid more than one consumer on a single file descriptor
3046          * This is just a matter of traces coherency, the ring buffer itself
3047          * is protected.
3048          */
3049         mutex_lock(&iter->mutex);
3050         if (iter->trace->read) {
3051                 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3052                 if (sret)
3053                         goto out;
3054         }
3055
3056 waitagain:
3057         sret = tracing_wait_pipe(filp);
3058         if (sret <= 0)
3059                 goto out;
3060
3061         /* stop when tracing is finished */
3062         if (trace_empty(iter)) {
3063                 sret = 0;
3064                 goto out;
3065         }
3066
3067         if (cnt >= PAGE_SIZE)
3068                 cnt = PAGE_SIZE - 1;
3069
3070         /* reset all but tr, trace, and overruns */
3071         memset(&iter->seq, 0,
3072                sizeof(struct trace_iterator) -
3073                offsetof(struct trace_iterator, seq));
3074         iter->pos = -1;
3075
3076         trace_event_read_lock();
3077         while (find_next_entry_inc(iter) != NULL) {
3078                 enum print_line_t ret;
3079                 int len = iter->seq.len;
3080
3081                 ret = print_trace_line(iter);
3082                 if (ret == TRACE_TYPE_PARTIAL_LINE) {
3083                         /* don't print partial lines */
3084                         iter->seq.len = len;
3085                         break;
3086                 }
3087                 if (ret != TRACE_TYPE_NO_CONSUME)
3088                         trace_consume(iter);
3089
3090                 if (iter->seq.len >= cnt)
3091                         break;
3092         }
3093         trace_event_read_unlock();
3094
3095         /* Now copy what we have to the user */
3096         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3097         if (iter->seq.readpos >= iter->seq.len)
3098                 trace_seq_init(&iter->seq);
3099
3100         /*
3101          * If there was nothing to send to user, inspite of consuming trace
3102          * entries, go back to wait for more entries.
3103          */
3104         if (sret == -EBUSY)
3105                 goto waitagain;
3106
3107 out:
3108         mutex_unlock(&iter->mutex);
3109
3110         return sret;
3111 }
3112
3113 static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
3114                                      struct pipe_buffer *buf)
3115 {
3116         __free_page(buf->page);
3117 }
3118
3119 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
3120                                      unsigned int idx)
3121 {
3122         __free_page(spd->pages[idx]);
3123 }
3124
3125 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
3126         .can_merge              = 0,
3127         .map                    = generic_pipe_buf_map,
3128         .unmap                  = generic_pipe_buf_unmap,
3129         .confirm                = generic_pipe_buf_confirm,
3130         .release                = tracing_pipe_buf_release,
3131         .steal                  = generic_pipe_buf_steal,
3132         .get                    = generic_pipe_buf_get,
3133 };
3134
3135 static size_t
3136 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
3137 {
3138         size_t count;
3139         int ret;
3140
3141         /* Seq buffer is page-sized, exactly what we need. */
3142         for (;;) {
3143                 count = iter->seq.len;
3144                 ret = print_trace_line(iter);
3145                 count = iter->seq.len - count;
3146                 if (rem < count) {
3147                         rem = 0;
3148                         iter->seq.len -= count;
3149                         break;
3150                 }
3151                 if (ret == TRACE_TYPE_PARTIAL_LINE) {
3152                         iter->seq.len -= count;
3153                         break;
3154                 }
3155
3156                 if (ret != TRACE_TYPE_NO_CONSUME)
3157                         trace_consume(iter);
3158                 rem -= count;
3159                 if (!find_next_entry_inc(iter)) {
3160                         rem = 0;
3161                         iter->ent = NULL;
3162                         break;
3163                 }
3164         }
3165
3166         return rem;
3167 }
3168
3169 static ssize_t tracing_splice_read_pipe(struct file *filp,
3170                                         loff_t *ppos,
3171                                         struct pipe_inode_info *pipe,
3172                                         size_t len,
3173                                         unsigned int flags)
3174 {
3175         struct page *pages[PIPE_BUFFERS];
3176         struct partial_page partial[PIPE_BUFFERS];
3177         struct trace_iterator *iter = filp->private_data;
3178         struct splice_pipe_desc spd = {
3179                 .pages          = pages,
3180                 .partial        = partial,
3181                 .nr_pages       = 0, /* This gets updated below. */
3182                 .flags          = flags,
3183                 .ops            = &tracing_pipe_buf_ops,
3184                 .spd_release    = tracing_spd_release_pipe,
3185         };
3186         static struct tracer *old_tracer;
3187         ssize_t ret;
3188         size_t rem;
3189         unsigned int i;
3190
3191         /* copy the tracer to avoid using a global lock all around */
3192         mutex_lock(&trace_types_lock);
3193         if (unlikely(old_tracer != current_trace && current_trace)) {
3194                 old_tracer = current_trace;
3195                 *iter->trace = *current_trace;
3196         }
3197         mutex_unlock(&trace_types_lock);
3198
3199         mutex_lock(&iter->mutex);
3200
3201         if (iter->trace->splice_read) {
3202                 ret = iter->trace->splice_read(iter, filp,
3203                                                ppos, pipe, len, flags);
3204                 if (ret)
3205                         goto out_err;
3206         }
3207
3208         ret = tracing_wait_pipe(filp);
3209         if (ret <= 0)
3210                 goto out_err;
3211
3212         if (!iter->ent && !find_next_entry_inc(iter)) {
3213                 ret = -EFAULT;
3214                 goto out_err;
3215         }
3216
3217         trace_event_read_lock();
3218
3219         /* Fill as many pages as possible. */
3220         for (i = 0, rem = len; i < PIPE_BUFFERS && rem; i++) {
3221                 pages[i] = alloc_page(GFP_KERNEL);
3222                 if (!pages[i])
3223                         break;
3224
3225                 rem = tracing_fill_pipe_page(rem, iter);
3226
3227                 /* Copy the data into the page, so we can start over. */
3228                 ret = trace_seq_to_buffer(&iter->seq,
3229                                           page_address(pages[i]),
3230                                           iter->seq.len);
3231                 if (ret < 0) {
3232                         __free_page(pages[i]);
3233                         break;
3234                 }
3235                 partial[i].offset = 0;
3236                 partial[i].len = iter->seq.len;
3237
3238                 trace_seq_init(&iter->seq);
3239         }
3240
3241         trace_event_read_unlock();
3242         mutex_unlock(&iter->mutex);
3243
3244         spd.nr_pages = i;
3245
3246         return splice_to_pipe(pipe, &spd);
3247
3248 out_err:
3249         mutex_unlock(&iter->mutex);
3250
3251         return ret;
3252 }
3253
3254 static ssize_t
3255 tracing_entries_read(struct file *filp, char __user *ubuf,
3256                      size_t cnt, loff_t *ppos)
3257 {
3258         struct trace_array *tr = filp->private_data;
3259         char buf[96];
3260         int r;
3261
3262         mutex_lock(&trace_types_lock);
3263         if (!ring_buffer_expanded)
3264                 r = sprintf(buf, "%lu (expanded: %lu)\n",
3265                             tr->entries >> 10,
3266                             trace_buf_size >> 10);
3267         else
3268                 r = sprintf(buf, "%lu\n", tr->entries >> 10);
3269         mutex_unlock(&trace_types_lock);
3270
3271         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3272 }
3273
3274 static ssize_t
3275 tracing_entries_write(struct file *filp, const char __user *ubuf,
3276                       size_t cnt, loff_t *ppos)
3277 {
3278         unsigned long val;
3279         char buf[64];
3280         int ret, cpu;
3281
3282         if (cnt >= sizeof(buf))
3283                 return -EINVAL;
3284
3285         if (copy_from_user(&buf, ubuf, cnt))
3286                 return -EFAULT;
3287
3288         buf[cnt] = 0;
3289
3290         ret = strict_strtoul(buf, 10, &val);
3291         if (ret < 0)
3292                 return ret;
3293
3294         /* must have at least 1 entry */
3295         if (!val)
3296                 return -EINVAL;
3297
3298         mutex_lock(&trace_types_lock);
3299
3300         tracing_stop();
3301
3302         /* disable all cpu buffers */
3303         for_each_tracing_cpu(cpu) {
3304                 if (global_trace.data[cpu])
3305                         atomic_inc(&global_trace.data[cpu]->disabled);
3306                 if (max_tr.data[cpu])
3307                         atomic_inc(&max_tr.data[cpu]->disabled);
3308         }
3309
3310         /* value is in KB */
3311         val <<= 10;
3312
3313         if (val != global_trace.entries) {
3314                 ret = tracing_resize_ring_buffer(val);
3315                 if (ret < 0) {
3316                         cnt = ret;
3317                         goto out;
3318                 }
3319         }
3320
3321         *ppos += cnt;
3322
3323         /* If check pages failed, return ENOMEM */
3324         if (tracing_disabled)
3325                 cnt = -ENOMEM;
3326  out:
3327         for_each_tracing_cpu(cpu) {
3328                 if (global_trace.data[cpu])
3329                         atomic_dec(&global_trace.data[cpu]->disabled);
3330                 if (max_tr.data[cpu])
3331                         atomic_dec(&max_tr.data[cpu]->disabled);
3332         }
3333
3334         tracing_start();
3335         max_tr.entries = global_trace.entries;
3336         mutex_unlock(&trace_types_lock);
3337
3338         return cnt;
3339 }
3340
3341 static int mark_printk(const char *fmt, ...)
3342 {
3343         int ret;
3344         va_list args;
3345         va_start(args, fmt);
3346         ret = trace_vprintk(0, fmt, args);
3347         va_end(args);
3348         return ret;
3349 }
3350
3351 static ssize_t
3352 tracing_mark_write(struct file *filp, const char __user *ubuf,
3353                                         size_t cnt, loff_t *fpos)
3354 {
3355         char *buf;
3356
3357         if (tracing_disabled)
3358                 return -EINVAL;
3359
3360         if (cnt > TRACE_BUF_SIZE)
3361                 cnt = TRACE_BUF_SIZE;
3362
3363         buf = kmalloc(cnt + 2, GFP_KERNEL);
3364         if (buf == NULL)
3365                 return -ENOMEM;
3366
3367         if (copy_from_user(buf, ubuf, cnt)) {
3368                 kfree(buf);
3369                 return -EFAULT;
3370         }
3371         if (buf[cnt-1] != '\n') {
3372                 buf[cnt] = '\n';
3373                 buf[cnt+1] = '\0';
3374         } else
3375                 buf[cnt] = '\0';
3376
3377         cnt = mark_printk("%s", buf);
3378         kfree(buf);
3379         *fpos += cnt;
3380
3381         return cnt;
3382 }
3383
3384 static int tracing_clock_show(struct seq_file *m, void *v)
3385 {
3386         int i;
3387
3388         for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
3389                 seq_printf(m,
3390                         "%s%s%s%s", i ? " " : "",
3391                         i == trace_clock_id ? "[" : "", trace_clocks[i].name,
3392                         i == trace_clock_id ? "]" : "");
3393         seq_putc(m, '\n');
3394
3395         return 0;
3396 }
3397
3398 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
3399                                    size_t cnt, loff_t *fpos)
3400 {
3401         char buf[64];
3402         const char *clockstr;
3403         int i;
3404
3405         if (cnt >= sizeof(buf))
3406                 return -EINVAL;
3407
3408         if (copy_from_user(&buf, ubuf, cnt))
3409                 return -EFAULT;
3410
3411         buf[cnt] = 0;
3412
3413         clockstr = strstrip(buf);
3414
3415         for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
3416                 if (strcmp(trace_clocks[i].name, clockstr) == 0)
3417                         break;
3418         }
3419         if (i == ARRAY_SIZE(trace_clocks))
3420                 return -EINVAL;
3421
3422         trace_clock_id = i;
3423
3424         mutex_lock(&trace_types_lock);
3425
3426         ring_buffer_set_clock(global_trace.buffer, trace_clocks[i].func);
3427         if (max_tr.buffer)
3428                 ring_buffer_set_clock(max_tr.buffer, trace_clocks[i].func);
3429
3430         mutex_unlock(&trace_types_lock);
3431
3432         *fpos += cnt;
3433
3434         return cnt;
3435 }
3436
3437 static int tracing_clock_open(struct inode *inode, struct file *file)
3438 {
3439         if (tracing_disabled)
3440                 return -ENODEV;