From e11c9c8365cd7655e81c4e6611492f1b15850aa5 Mon Sep 17 00:00:00 2001 From: Crystal Wood Date: Mon, 11 May 2026 17:31:43 -0500 Subject: [PATCH 1/2] tracing/osnoise: Dump stack on timerlat uret threshold event Dump the saved IRQ stack trace regardless of whether the event was THREAD_CONTEXT or THREAD_URET. In the uret case, the latency presumably had not yet crossed the threshold at IRQ time (or else it would have dumped the stack at thread wakeup time, unless we're racing with a change to the threshold), but it may have at least contributed -- and this is possible with THREAD_CONTEXT as well. In any case, it helps with writing reliable rtla tests if we always get a stack trace on a threshold event. Cc: John Kacur Cc: Tomas Glozar Cc: Costa Shulyupin Cc: Wander Lairson Costa Link: https://patch.msgid.link/20260511223143.1477332-1-crwood@redhat.com Signed-off-by: Crystal Wood Signed-off-by: Steven Rostedt --- kernel/trace/trace_osnoise.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c index 75678053b21c..62c2667d97fa 100644 --- a/kernel/trace/trace_osnoise.c +++ b/kernel/trace/trace_osnoise.c @@ -2544,9 +2544,12 @@ timerlat_fd_read(struct file *file, char __user *ubuf, size_t count, notify_new_max_latency(diff); tlat->tracing_thread = false; - if (osnoise_data.stop_tracing_total) - if (time_to_us(diff) >= osnoise_data.stop_tracing_total) + if (osnoise_data.stop_tracing_total) { + if (time_to_us(diff) >= osnoise_data.stop_tracing_total) { + timerlat_dump_stack(time_to_us(diff)); osnoise_stop_tracing(); + } + } } else { tlat->tracing_thread = false; tlat->kthread = current; From 9cb99c598643ba78638dfd668cf020544159cf70 Mon Sep 17 00:00:00 2001 From: Crystal Wood Date: Mon, 11 May 2026 17:30:35 -0500 Subject: [PATCH 2/2] tracing/osnoise: Array printk init and cleanup None of the calls to trace_array_printk_buf() will do anything if we don't initialize the buffer on instance creation (unless some other tracer called it), so do that. Add an osnoise_print() function to facilitate adding debug prints (without tainting). Use trace_array_printk() instead of trace_array_printk_buf(), as we're only writing to the main buffer (of a non-main instance) anyway -- and trace_array_printk_buf() skips the check to make sure we're not printing to the global instance. Link: https://patch.msgid.link/20260511223035.1475676-1-crwood@redhat.com Signed-off-by: Crystal Wood Signed-off-by: Steven Rostedt --- kernel/trace/trace_osnoise.c | 39 ++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c index 62c2667d97fa..5e83c4f6f2b4 100644 --- a/kernel/trace/trace_osnoise.c +++ b/kernel/trace/trace_osnoise.c @@ -83,6 +83,22 @@ struct osnoise_instance { static struct list_head osnoise_instances; +static void osnoise_print(const char *fmt, ...) +{ + struct osnoise_instance *inst; + struct trace_array *tr; + va_list ap; + + rcu_read_lock(); + list_for_each_entry_rcu(inst, &osnoise_instances, list) { + tr = inst->tr; + va_start(ap, fmt); + trace_array_vprintk(tr, _RET_IP_, fmt, ap); + va_end(ap); + } + rcu_read_unlock(); +} + static bool osnoise_has_registered_instances(void) { return !!list_first_or_null_rcu(&osnoise_instances, @@ -123,6 +139,7 @@ static int osnoise_register_instance(struct trace_array *tr) * trace_types_lock. */ lockdep_assert_held(&trace_types_lock); + trace_array_init_printk(tr); inst = kmalloc_obj(*inst); if (!inst) @@ -471,15 +488,7 @@ static void print_osnoise_headers(struct seq_file *s) * osnoise_taint - report an osnoise error. */ #define osnoise_taint(msg) ({ \ - struct osnoise_instance *inst; \ - struct trace_buffer *buffer; \ - \ - rcu_read_lock(); \ - list_for_each_entry_rcu(inst, &osnoise_instances, list) { \ - buffer = inst->tr->array_buffer.buffer; \ - trace_array_printk_buf(buffer, _THIS_IP_, msg); \ - } \ - rcu_read_unlock(); \ + osnoise_print(msg); \ osnoise_data.tainted = true; \ }) @@ -1189,10 +1198,10 @@ static __always_inline void osnoise_stop_exception(char *msg, int cpu) rcu_read_lock(); list_for_each_entry_rcu(inst, &osnoise_instances, list) { tr = inst->tr; - trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_, - "stop tracing hit on cpu %d due to exception: %s\n", - smp_processor_id(), - msg); + trace_array_printk(tr, _THIS_IP_, + "stop tracing hit on cpu %d due to exception: %s\n", + smp_processor_id(), + msg); if (test_bit(OSN_PANIC_ON_STOP, &osnoise_options)) panic("tracer hit on cpu %d due to exception: %s\n", @@ -1362,8 +1371,8 @@ static __always_inline void osnoise_stop_tracing(void) rcu_read_lock(); list_for_each_entry_rcu(inst, &osnoise_instances, list) { tr = inst->tr; - trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_, - "stop tracing hit on cpu %d\n", smp_processor_id()); + trace_array_printk(tr, _THIS_IP_, + "stop tracing hit on cpu %d\n", smp_processor_id()); if (test_bit(OSN_PANIC_ON_STOP, &osnoise_options)) panic("tracer hit stop condition on CPU %d\n", smp_processor_id());