tracing: Support to dump instance traces by ftrace_dump_on_oops
authorHuang Yiwei <quic_hyiwei@quicinc.com>
Fri, 23 Feb 2024 08:31:26 +0000 (16:31 +0800)
committerSteven Rostedt (Google) <rostedt@goodmis.org>
Mon, 18 Mar 2024 14:33:06 +0000 (10:33 -0400)
Currently ftrace only dumps the global trace buffer on an OOPs. For
debugging a production usecase, instance trace will be helpful to
check specific problems since global trace buffer may be used for
other purposes.

This patch extend the ftrace_dump_on_oops parameter to dump a specific
or multiple trace instances:

  - ftrace_dump_on_oops=0: as before -- don't dump
  - ftrace_dump_on_oops[=1]: as before -- dump the global trace buffer
  on all CPUs
  - ftrace_dump_on_oops=2 or =orig_cpu: as before -- dump the global
  trace buffer on CPU that triggered the oops
  - ftrace_dump_on_oops=<instance_name>: new behavior -- dump the
  tracing instance matching <instance_name>
  - ftrace_dump_on_oops[=2/orig_cpu],<instance1_name>[=2/orig_cpu],
  <instrance2_name>[=2/orig_cpu]: new behavior -- dump the global trace
  buffer and multiple instance buffer on all CPUs, or only dump on CPU
  that triggered the oops if =2 or =orig_cpu is given

Also, the sysctl node can handle the input accordingly.

Link: https://lore.kernel.org/linux-trace-kernel/20240223083126.1817731-1-quic_hyiwei@quicinc.com
Cc: Ross Zwisler <zwisler@google.com>
Cc: <mhiramat@kernel.org>
Cc: <mark.rutland@arm.com>
Cc: <mcgrof@kernel.org>
Cc: <keescook@chromium.org>
Cc: <j.granados@samsung.com>
Cc: <mathieu.desnoyers@efficios.com>
Cc: <corbet@lwn.net>
Signed-off-by: Huang Yiwei <quic_hyiwei@quicinc.com>
Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
Documentation/admin-guide/kernel-parameters.txt
Documentation/admin-guide/sysctl/kernel.rst
include/linux/ftrace.h
include/linux/kernel.h
kernel/sysctl.c
kernel/trace/trace.c
kernel/trace/trace_selftest.c

index 479ff1737c2fd6f779c48109cd9026307c401746..fa871d53641c6ffc3258a295d5ba225134ba1178 100644 (file)
                        The above will cause the "foo" tracing instance to trigger
                        a snapshot at the end of boot up.
 
-       ftrace_dump_on_oops[=orig_cpu]
+       ftrace_dump_on_oops[=2(orig_cpu) | =<instance>][,<instance> |
+                         ,<instance>=2(orig_cpu)]
                        [FTRACE] will dump the trace buffers on oops.
-                       If no parameter is passed, ftrace will dump
-                       buffers of all CPUs, but if you pass orig_cpu, it will
-                       dump only the buffer of the CPU that triggered the
-                       oops.
+                       If no parameter is passed, ftrace will dump global
+                       buffers of all CPUs, if you pass 2 or orig_cpu, it
+                       will dump only the buffer of the CPU that triggered
+                       the oops, or the specific instance will be dumped if
+                       its name is passed. Multiple instance dump is also
+                       supported, and instances are separated by commas. Each
+                       instance supports only dump on CPU that triggered the
+                       oops by passing 2 or orig_cpu to it.
+
+                       ftrace_dump_on_oops=foo=orig_cpu
+
+                       The above will dump only the buffer of "foo" instance
+                       on CPU that triggered the oops.
+
+                       ftrace_dump_on_oops,foo,bar=orig_cpu
+
+                       The above will dump global buffer on all CPUs, the
+                       buffer of "foo" instance on all CPUs and the buffer
+                       of "bar" instance on CPU that triggered the oops.
 
        ftrace_filter=[function-list]
                        [FTRACE] Limit the functions traced by the function
index 6584a1f9bfe39de58774996f5a37b9e4b5d5545f..ea8e5f152edcf5d142044ce632fe13747516185c 100644 (file)
@@ -296,12 +296,30 @@ kernel panic). This will output the contents of the ftrace buffers to
 the console.  This is very useful for capturing traces that lead to
 crashes and outputting them to a serial console.
 
-= ===================================================
-0 Disabled (default).
-1 Dump buffers of all CPUs.
-2 Dump the buffer of the CPU that triggered the oops.
-= ===================================================
-
+======================= ===========================================
+0                       Disabled (default).
+1                       Dump buffers of all CPUs.
+2(orig_cpu)             Dump the buffer of the CPU that triggered the
+                        oops.
+<instance>              Dump the specific instance buffer on all CPUs.
+<instance>=2(orig_cpu)  Dump the specific instance buffer on the CPU
+                        that triggered the oops.
+======================= ===========================================
+
+Multiple instance dump is also supported, and instances are separated
+by commas. If global buffer also needs to be dumped, please specify
+the dump mode (1/2/orig_cpu) first for global buffer.
+
+So for example to dump "foo" and "bar" instance buffer on all CPUs,
+user can::
+
+  echo "foo,bar" > /proc/sys/kernel/ftrace_dump_on_oops
+
+To dump global buffer and "foo" instance buffer on all
+CPUs along with the "bar" instance buffer on CPU that triggered the
+oops, user can::
+
+  echo "1,foo,bar=2" > /proc/sys/kernel/ftrace_dump_on_oops
 
 ftrace_enabled, stack_tracer_enabled
 ====================================
index e8921871ef9aaa4ff9a8c74359bf49ebe95d8b3e..54d53f345d14974720f308b9fba1a754b3c6d712 100644 (file)
@@ -1151,7 +1151,9 @@ static inline void unpause_graph_tracing(void) { }
 #ifdef CONFIG_TRACING
 enum ftrace_dump_mode;
 
-extern enum ftrace_dump_mode ftrace_dump_on_oops;
+#define MAX_TRACER_SIZE                100
+extern char ftrace_dump_on_oops[];
+extern int ftrace_dump_on_oops_enabled(void);
 extern int tracepoint_printk;
 
 extern void disable_trace_on_warning(void);
index d718fbec72dd661f920171b4beed68f7c319ae60..be2e8c0a187e4b26a704ee5e628b7afe404d9211 100644 (file)
@@ -215,6 +215,7 @@ enum ftrace_dump_mode {
        DUMP_NONE,
        DUMP_ALL,
        DUMP_ORIG,
+       DUMP_PARAM,
 };
 
 #ifdef CONFIG_TRACING
index 157f7ce2942d26a6401bc3c8f0dd2077ad69ff5d..81cc974913bb901ec9319d836d09fbb24bf60c56 100644 (file)
@@ -1710,9 +1710,9 @@ static struct ctl_table kern_table[] = {
        {
                .procname       = "ftrace_dump_on_oops",
                .data           = &ftrace_dump_on_oops,
-               .maxlen         = sizeof(int),
+               .maxlen         = MAX_TRACER_SIZE,
                .mode           = 0644,
-               .proc_handler   = proc_dointvec,
+               .proc_handler   = proc_dostring,
        },
        {
                .procname       = "traceoff_on_warning",
index 1e1fd377a1cfdd6e532c3155055c28f9aaf86fbd..233d1af39fffe29b459c09e9a5e0a82fa07e992a 100644 (file)
@@ -130,9 +130,12 @@ cpumask_var_t __read_mostly        tracing_buffer_mask;
  * /proc/sys/kernel/ftrace_dump_on_oops
  * Set 1 if you want to dump buffers of all CPUs
  * Set 2 if you want to dump the buffer of the CPU that triggered oops
+ * Set instance name if you want to dump the specific trace instance
+ * Multiple instance dump is also supported, and instances are seperated
+ * by commas.
  */
-
-enum ftrace_dump_mode ftrace_dump_on_oops;
+/* Set to string format zero to disable by default */
+char ftrace_dump_on_oops[MAX_TRACER_SIZE] = "0";
 
 /* When set, tracing will stop when a WARN*() is hit */
 int __disable_trace_on_warning;
@@ -178,7 +181,6 @@ static void ftrace_trace_userstack(struct trace_array *tr,
                                   struct trace_buffer *buffer,
                                   unsigned int trace_ctx);
 
-#define MAX_TRACER_SIZE                100
 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
 static char *default_bootup_tracer;
 
@@ -201,19 +203,33 @@ static int __init set_cmdline_ftrace(char *str)
 }
 __setup("ftrace=", set_cmdline_ftrace);
 
+int ftrace_dump_on_oops_enabled(void)
+{
+       if (!strcmp("0", ftrace_dump_on_oops))
+               return 0;
+       else
+               return 1;
+}
+
 static int __init set_ftrace_dump_on_oops(char *str)
 {
-       if (*str++ != '=' || !*str || !strcmp("1", str)) {
-               ftrace_dump_on_oops = DUMP_ALL;
+       if (!*str) {
+               strscpy(ftrace_dump_on_oops, "1", MAX_TRACER_SIZE);
                return 1;
        }
 
-       if (!strcmp("orig_cpu", str) || !strcmp("2", str)) {
-               ftrace_dump_on_oops = DUMP_ORIG;
-                return 1;
-        }
+       if (*str == ',') {
+               strscpy(ftrace_dump_on_oops, "1", MAX_TRACER_SIZE);
+               strscpy(ftrace_dump_on_oops + 1, str, MAX_TRACER_SIZE - 1);
+               return 1;
+       }
+
+       if (*str++ == '=') {
+               strscpy(ftrace_dump_on_oops, str, MAX_TRACER_SIZE);
+               return 1;
+       }
 
-        return 0;
+       return 0;
 }
 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
 
@@ -9832,14 +9848,14 @@ static struct notifier_block trace_die_notifier = {
 static int trace_die_panic_handler(struct notifier_block *self,
                                unsigned long ev, void *unused)
 {
-       if (!ftrace_dump_on_oops)
+       if (!ftrace_dump_on_oops_enabled())
                return NOTIFY_DONE;
 
        /* The die notifier requires DIE_OOPS to trigger */
        if (self == &trace_die_notifier && ev != DIE_OOPS)
                return NOTIFY_DONE;
 
-       ftrace_dump(ftrace_dump_on_oops);
+       ftrace_dump(DUMP_PARAM);
 
        return NOTIFY_DONE;
 }
@@ -9880,12 +9896,12 @@ trace_printk_seq(struct trace_seq *s)
        trace_seq_init(s);
 }
 
-void trace_init_global_iter(struct trace_iterator *iter)
+static void trace_init_iter(struct trace_iterator *iter, struct trace_array *tr)
 {
-       iter->tr = &global_trace;
+       iter->tr = tr;
        iter->trace = iter->tr->current_trace;
        iter->cpu_file = RING_BUFFER_ALL_CPUS;
-       iter->array_buffer = &global_trace.array_buffer;
+       iter->array_buffer = &tr->array_buffer;
 
        if (iter->trace && iter->trace->open)
                iter->trace->open(iter);
@@ -9905,22 +9921,19 @@ void trace_init_global_iter(struct trace_iterator *iter)
        iter->fmt_size = STATIC_FMT_BUF_SIZE;
 }
 
-void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
+void trace_init_global_iter(struct trace_iterator *iter)
+{
+       trace_init_iter(iter, &global_trace);
+}
+
+static void ftrace_dump_one(struct trace_array *tr, enum ftrace_dump_mode dump_mode)
 {
        /* use static because iter can be a bit big for the stack */
        static struct trace_iterator iter;
-       static atomic_t dump_running;
-       struct trace_array *tr = &global_trace;
        unsigned int old_userobj;
        unsigned long flags;
        int cnt = 0, cpu;
 
-       /* Only allow one dump user at a time. */
-       if (atomic_inc_return(&dump_running) != 1) {
-               atomic_dec(&dump_running);
-               return;
-       }
-
        /*
         * Always turn off tracing when we dump.
         * We don't need to show trace output of what happens
@@ -9929,12 +9942,12 @@ void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
         * If the user does a sysrq-z, then they can re-enable
         * tracing with echo 1 > tracing_on.
         */
-       tracing_off();
+       tracer_tracing_off(tr);
 
        local_irq_save(flags);
 
        /* Simulate the iterator */
-       trace_init_global_iter(&iter);
+       trace_init_iter(&iter, tr);
 
        for_each_tracing_cpu(cpu) {
                atomic_inc(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
@@ -9945,21 +9958,15 @@ void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
        /* don't look at user memory in panic mode */
        tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
 
-       switch (oops_dump_mode) {
-       case DUMP_ALL:
-               iter.cpu_file = RING_BUFFER_ALL_CPUS;
-               break;
-       case DUMP_ORIG:
+       if (dump_mode == DUMP_ORIG)
                iter.cpu_file = raw_smp_processor_id();
-               break;
-       case DUMP_NONE:
-               goto out_enable;
-       default:
-               printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
+       else
                iter.cpu_file = RING_BUFFER_ALL_CPUS;
-       }
 
-       printk(KERN_TRACE "Dumping ftrace buffer:\n");
+       if (tr == &global_trace)
+               printk(KERN_TRACE "Dumping ftrace buffer:\n");
+       else
+               printk(KERN_TRACE "Dumping ftrace instance %s buffer:\n", tr->name);
 
        /* Did function tracer already get disabled? */
        if (ftrace_is_dead()) {
@@ -10001,15 +10008,84 @@ void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
        else
                printk(KERN_TRACE "---------------------------------\n");
 
- out_enable:
        tr->trace_flags |= old_userobj;
 
        for_each_tracing_cpu(cpu) {
                atomic_dec(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);
        }
-       atomic_dec(&dump_running);
        local_irq_restore(flags);
 }
+
+static void ftrace_dump_by_param(void)
+{
+       bool first_param = true;
+       char dump_param[MAX_TRACER_SIZE];
+       char *buf, *token, *inst_name;
+       struct trace_array *tr;
+
+       strscpy(dump_param, ftrace_dump_on_oops, MAX_TRACER_SIZE);
+       buf = dump_param;
+
+       while ((token = strsep(&buf, ",")) != NULL) {
+               if (first_param) {
+                       first_param = false;
+                       if (!strcmp("0", token))
+                               continue;
+                       else if (!strcmp("1", token)) {
+                               ftrace_dump_one(&global_trace, DUMP_ALL);
+                               continue;
+                       }
+                       else if (!strcmp("2", token) ||
+                         !strcmp("orig_cpu", token)) {
+                               ftrace_dump_one(&global_trace, DUMP_ORIG);
+                               continue;
+                       }
+               }
+
+               inst_name = strsep(&token, "=");
+               tr = trace_array_find(inst_name);
+               if (!tr) {
+                       printk(KERN_TRACE "Instance %s not found\n", inst_name);
+                       continue;
+               }
+
+               if (token && (!strcmp("2", token) ||
+                         !strcmp("orig_cpu", token)))
+                       ftrace_dump_one(tr, DUMP_ORIG);
+               else
+                       ftrace_dump_one(tr, DUMP_ALL);
+       }
+}
+
+void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
+{
+       static atomic_t dump_running;
+
+       /* Only allow one dump user at a time. */
+       if (atomic_inc_return(&dump_running) != 1) {
+               atomic_dec(&dump_running);
+               return;
+       }
+
+       switch (oops_dump_mode) {
+       case DUMP_ALL:
+               ftrace_dump_one(&global_trace, DUMP_ALL);
+               break;
+       case DUMP_ORIG:
+               ftrace_dump_one(&global_trace, DUMP_ORIG);
+               break;
+       case DUMP_PARAM:
+               ftrace_dump_by_param();
+               break;
+       case DUMP_NONE:
+               break;
+       default:
+               printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
+               ftrace_dump_one(&global_trace, DUMP_ALL);
+       }
+
+       atomic_dec(&dump_running);
+}
 EXPORT_SYMBOL_GPL(ftrace_dump);
 
 #define WRITE_BUFSIZE  4096
index 529590499b1fa6b5877232aa82bead760813d2ac..e9c5058a8efd8c7e96114df839192f2fd7046e28 100644 (file)
@@ -768,7 +768,7 @@ static int trace_graph_entry_watchdog(struct ftrace_graph_ent *trace)
        if (unlikely(++graph_hang_thresh > GRAPH_MAX_FUNC_TEST)) {
                ftrace_graph_stop();
                printk(KERN_WARNING "BUG: Function graph tracer hang!\n");
-               if (ftrace_dump_on_oops) {
+               if (ftrace_dump_on_oops_enabled()) {
                        ftrace_dump(DUMP_ALL);
                        /* ftrace_dump() disables tracing */
                        tracing_on();