From ade2e3c3aede458c8ffed62ef0320aeb60a53a92 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 7 May 2026 16:00:18 -0700 Subject: [PATCH 01/47] rcu-tasks: TASKS_TRACE_RCU doesn't need IRQ_WORK Now that Tasks Trace RCU is implemented in terms of SRCU, it no longer has any particular need for the IRQ_WORK Kconfig option. This commit therefore removes the "select IRQ_WORK" from the TASKS_TRACE_RCU Kconfig option. Signed-off-by: Paul E. McKenney --- kernel/rcu/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/kernel/rcu/Kconfig b/kernel/rcu/Kconfig index 762299291e09..332df7a7a634 100644 --- a/kernel/rcu/Kconfig +++ b/kernel/rcu/Kconfig @@ -140,7 +140,6 @@ config FORCE_TASKS_TRACE_RCU config TASKS_TRACE_RCU bool default n - select IRQ_WORK config TASKS_TRACE_RCU_NO_MB bool "Override RCU Tasks Trace inclusion of read-side memory barriers" From bc892f8249780cacf65636f9a57cc9736b410490 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 30 Apr 2026 17:05:35 -0700 Subject: [PATCH 02/47] rcutorture: Abstract reader-segment dump into rcu_torture_dump_read_segs() This commit abstracts the open-coded dumping of reader segments in the rcu_torture_cleanup() function into a new rcu_torture_dump_read_segs() function. This abstraction will allow reader segments to be dumped for other purposes. Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 148 +++++++++++++++++++++------------------- 1 file changed, 76 insertions(+), 72 deletions(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 882a158ada7b..ffeca1b7e170 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -2401,6 +2401,80 @@ struct rcu_torture_one_read_state { unsigned long long ts; }; +static void rcu_torture_dump_read_segs(struct rt_read_seg *rrsp, int nsegs) +{ + bool firsttime; + int i; + int j; + + firsttime = 1; + for (i = 0; i < nsegs; i++) { + if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP)) + pr_alert("\t%lluus ", div64_u64(rrsp[i].rt_ts, 1000ULL)); + else + pr_alert("\t"); + pr_cont("%d: %#4x", i, rrsp[i].rt_readstate); + if (rrsp[i].rt_delay_jiffies != 0) { + pr_cont("%s%ldjiffies", firsttime ? "" : "+", + rrsp[i].rt_delay_jiffies); + firsttime = 0; + } + if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_CPU)) { + pr_cont(" CPU %2d", rrsp[i].rt_cpu); + if (rrsp[i].rt_cpu != rrsp[i].rt_end_cpu) + pr_cont("->%-2d", rrsp[i].rt_end_cpu); + else + pr_cont(" ..."); + } + if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP) && + cur_ops->gather_gp_seqs && cur_ops->format_gp_seqs) { + char buf1[20+1]; + char buf2[20+1]; + char sepchar = '-'; + + cur_ops->format_gp_seqs(rrsp[i].rt_gp_seq, buf1, ARRAY_SIZE(buf1)); + cur_ops->format_gp_seqs(rrsp[i].rt_gp_seq_end, buf2, ARRAY_SIZE(buf2)); + if (rrsp[i].rt_gp_seq == rrsp[i].rt_gp_seq_end) { + if (buf2[0]) { + for (j = 0; buf2[j]; j++) + buf2[j] = '.'; + if (j) + buf2[j - 1] = ' '; + } + sepchar = ' '; + } + pr_cont(" %s%c%s", buf1, sepchar, buf2); + } + if (rrsp[i].rt_delay_ms != 0) { + pr_cont(" %s%ldms", firsttime ? "" : "+", rrsp[i].rt_delay_ms); + firsttime = 0; + } + if (rrsp[i].rt_delay_us != 0) { + pr_cont(" %s%ldus", firsttime ? "" : "+", rrsp[i].rt_delay_us); + firsttime = 0; + } + pr_cont("%s", rrsp[i].rt_preempted ? " preempted" : ""); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_BH) + pr_cont(" BH"); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_IRQ) + pr_cont(" IRQ"); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_PREEMPT) + pr_cont(" PREEMPT"); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_RBH) + pr_cont(" RBH"); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_SCHED) + pr_cont(" SCHED"); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_RCU_1) + pr_cont(" RCU_1"); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_RCU_2) + pr_cont(" RCU_2"); + pr_cont("\n"); + + } + if (rt_read_preempted) + pr_alert("\tReader was preempted.\n"); +} + static void init_rcu_torture_one_read_state(struct rcu_torture_one_read_state *rtorsp, struct torture_random_state *trsp) { @@ -4094,11 +4168,9 @@ static void rcu_gpwrap_lag_cleanup(void) static void rcu_torture_cleanup(void) { - int firsttime; int flags = 0; unsigned long gp_seq = 0; int i; - int j; if (torture_cleanup_begin()) { if (cur_ops->cb_barrier != NULL) { @@ -4183,76 +4255,8 @@ rcu_torture_cleanup(void) pr_alert("Failure/close-call rcutorture reader segments:\n"); if (rt_read_nsegs == 0) pr_alert("\t: No segments recorded!!!\n"); - firsttime = 1; - for (i = 0; i < rt_read_nsegs; i++) { - if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP)) - pr_alert("\t%lluus ", div64_u64(err_segs[i].rt_ts, 1000ULL)); - else - pr_alert("\t"); - pr_cont("%d: %#4x", i, err_segs[i].rt_readstate); - if (err_segs[i].rt_delay_jiffies != 0) { - pr_cont("%s%ldjiffies", firsttime ? "" : "+", - err_segs[i].rt_delay_jiffies); - firsttime = 0; - } - if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_CPU)) { - pr_cont(" CPU %2d", err_segs[i].rt_cpu); - if (err_segs[i].rt_cpu != err_segs[i].rt_end_cpu) - pr_cont("->%-2d", err_segs[i].rt_end_cpu); - else - pr_cont(" ..."); - } - if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP) && - cur_ops->gather_gp_seqs && cur_ops->format_gp_seqs) { - char buf1[20+1]; - char buf2[20+1]; - char sepchar = '-'; - - cur_ops->format_gp_seqs(err_segs[i].rt_gp_seq, - buf1, ARRAY_SIZE(buf1)); - cur_ops->format_gp_seqs(err_segs[i].rt_gp_seq_end, - buf2, ARRAY_SIZE(buf2)); - if (err_segs[i].rt_gp_seq == err_segs[i].rt_gp_seq_end) { - if (buf2[0]) { - for (j = 0; buf2[j]; j++) - buf2[j] = '.'; - if (j) - buf2[j - 1] = ' '; - } - sepchar = ' '; - } - pr_cont(" %s%c%s", buf1, sepchar, buf2); - } - if (err_segs[i].rt_delay_ms != 0) { - pr_cont(" %s%ldms", firsttime ? "" : "+", - err_segs[i].rt_delay_ms); - firsttime = 0; - } - if (err_segs[i].rt_delay_us != 0) { - pr_cont(" %s%ldus", firsttime ? "" : "+", - err_segs[i].rt_delay_us); - firsttime = 0; - } - pr_cont("%s", err_segs[i].rt_preempted ? " preempted" : ""); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_BH) - pr_cont(" BH"); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_IRQ) - pr_cont(" IRQ"); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_PREEMPT) - pr_cont(" PREEMPT"); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_RBH) - pr_cont(" RBH"); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_SCHED) - pr_cont(" SCHED"); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_RCU_1) - pr_cont(" RCU_1"); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_RCU_2) - pr_cont(" RCU_2"); - pr_cont("\n"); - - } - if (rt_read_preempted) - pr_alert("\tReader was preempted.\n"); + else + rcu_torture_dump_read_segs(err_segs, rt_read_nsegs); } if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error) rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE"); From a47dea1baa3b445365ad0757e6dee4ef8fde49e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Onur=20=C3=96zkan?= Date: Sat, 13 Jun 2026 09:40:07 +0300 Subject: [PATCH 03/47] srcu: make init_srcu_struct() consistently wrap __init_srcu_struct() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restructure the SRCU initialization functions so it always follows one direction: init_srcu_struct() -> __init_srcu_struct() -> lockdep or generic This uses the same wrapper style as mutex. It avoids the old confusing style where init_srcu_struct() and __init_srcu_struct() called each other in different configs. It also helps Rust side to have simpler helper for SRCU initialization. Signed-off-by: Onur Özkan Reviewed-by: Gary Guo Reviewed-by: Alice Ryhl Reviewed-by: Boqun Feng Signed-off-by: Paul E. McKenney --- include/linux/srcu.h | 29 ++++++++++++++++++++--------- kernel/rcu/srcutiny.c | 12 ++++++------ kernel/rcu/srcutree.c | 11 ++++++----- 3 files changed, 32 insertions(+), 20 deletions(-) diff --git a/include/linux/srcu.h b/include/linux/srcu.h index a54ce9e808b9..c5ab7df6fe5c 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h @@ -25,20 +25,19 @@ context_lock_struct(srcu_struct, __reentrant_ctx_lock); #ifdef CONFIG_DEBUG_LOCK_ALLOC -int __init_srcu_struct(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); +int init_srcu_struct_lockdep(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key); +static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key) +{ + return init_srcu_struct_lockdep(ssp, name, key); +} #ifndef CONFIG_TINY_SRCU int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); #endif // #ifndef CONFIG_TINY_SRCU -#define init_srcu_struct(ssp) \ -({ \ - static struct lock_class_key __srcu_key; \ - \ - __init_srcu_struct((ssp), #ssp, &__srcu_key); \ -}) - #define init_srcu_struct_fast(ssp) \ ({ \ static struct lock_class_key __srcu_key; \ @@ -56,7 +55,12 @@ int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, #define __SRCU_DEP_MAP_INIT(srcu_name) .dep_map = { .name = #srcu_name }, #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ -int init_srcu_struct(struct srcu_struct *ssp); +int init_srcu_struct_generic(struct srcu_struct *ssp); +static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key) +{ + return init_srcu_struct_generic(ssp); +} #ifndef CONFIG_TINY_SRCU int init_srcu_struct_fast(struct srcu_struct *ssp); int init_srcu_struct_fast_updown(struct srcu_struct *ssp); @@ -65,6 +69,13 @@ int init_srcu_struct_fast_updown(struct srcu_struct *ssp); #define __SRCU_DEP_MAP_INIT(srcu_name) #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ +#define init_srcu_struct(ssp) \ +({ \ + static struct lock_class_key __srcu_key; \ + \ + __init_srcu_struct((ssp), #ssp, &__srcu_key); \ +}) + /* Values for SRCU Tree srcu_data ->srcu_reader_flavor, but also used by rcutorture. */ #define SRCU_READ_FLAVOR_NORMAL 0x1 // srcu_read_lock(). #define SRCU_READ_FLAVOR_NMI 0x2 // srcu_read_lock_nmisafe(). diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c index a2e2d516e51b..47d48ed31848 100644 --- a/kernel/rcu/srcutiny.c +++ b/kernel/rcu/srcutiny.c @@ -48,31 +48,31 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp) #ifdef CONFIG_DEBUG_LOCK_ALLOC -int __init_srcu_struct(struct srcu_struct *ssp, const char *name, - struct lock_class_key *key) +int init_srcu_struct_lockdep(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key) { /* Don't re-initialize a lock while it is held. */ debug_check_no_locks_freed((void *)ssp, sizeof(*ssp)); lockdep_init_map(&ssp->dep_map, name, key, 0); return init_srcu_struct_fields(ssp); } -EXPORT_SYMBOL_GPL(__init_srcu_struct); +EXPORT_SYMBOL_GPL(init_srcu_struct_lockdep); #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ /* - * init_srcu_struct - initialize a sleep-RCU structure + * init_srcu_struct_generic - initialize a sleep-RCU structure * @ssp: structure to initialize. * * Must invoke this on a given srcu_struct before passing that srcu_struct * to any other function. Each srcu_struct represents a separate domain * of SRCU protection. */ -int init_srcu_struct(struct srcu_struct *ssp) +int init_srcu_struct_generic(struct srcu_struct *ssp) { return init_srcu_struct_fields(ssp); } -EXPORT_SYMBOL_GPL(init_srcu_struct); +EXPORT_SYMBOL_GPL(init_srcu_struct_generic); #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 7c2f7cc131f7..56c5db5c29a0 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -266,12 +266,13 @@ __init_srcu_struct_common(struct srcu_struct *ssp, const char *name, struct lock return init_srcu_struct_fields(ssp, false); } -int __init_srcu_struct(struct srcu_struct *ssp, const char *name, struct lock_class_key *key) +int init_srcu_struct_lockdep(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key) { ssp->srcu_reader_flavor = 0; return __init_srcu_struct_common(ssp, name, key); } -EXPORT_SYMBOL_GPL(__init_srcu_struct); +EXPORT_SYMBOL_GPL(init_srcu_struct_lockdep); int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key) { @@ -291,7 +292,7 @@ EXPORT_SYMBOL_GPL(__init_srcu_struct_fast_updown); #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ /** - * init_srcu_struct - initialize a sleep-RCU structure + * init_srcu_struct_generic - initialize a sleep-RCU structure * @ssp: structure to initialize. * * Use this in place of DEFINE_SRCU() and DEFINE_STATIC_SRCU() @@ -301,12 +302,12 @@ EXPORT_SYMBOL_GPL(__init_srcu_struct_fast_updown); * to any other function. Each srcu_struct represents a separate domain * of SRCU protection. */ -int init_srcu_struct(struct srcu_struct *ssp) +int init_srcu_struct_generic(struct srcu_struct *ssp) { ssp->srcu_reader_flavor = 0; return init_srcu_struct_fields(ssp, false); } -EXPORT_SYMBOL_GPL(init_srcu_struct); +EXPORT_SYMBOL_GPL(init_srcu_struct_generic); /** * init_srcu_struct_fast - initialize a fast-reader sleep-RCU structure From 8f63d947bedfddefeb1c10f1dfc36f16407ce22c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Onur=20=C3=96zkan?= Date: Sat, 13 Jun 2026 09:40:08 +0300 Subject: [PATCH 04/47] rust: helpers: add SRCU helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add helper wrappers for SRCU functions that are exposed to Rust through generated bindings. Signed-off-by: Onur Özkan Reviewed-by: Gary Guo Reviewed-by: Alice Ryhl Reviewed-by: Boqun Feng Signed-off-by: Paul E. McKenney --- rust/helpers/helpers.c | 1 + rust/helpers/srcu.c | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 rust/helpers/srcu.c diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 998e31052e66..35954f38ff6a 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -91,6 +91,7 @@ #include "slab.c" #include "spinlock.c" #include "string.c" +#include "srcu.c" #include "sync.c" #include "task.c" #include "time.c" diff --git a/rust/helpers/srcu.c b/rust/helpers/srcu.c new file mode 100644 index 000000000000..225b3bf9334a --- /dev/null +++ b/rust/helpers/srcu.c @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper int rust_helper_init_srcu_struct_with_key(struct srcu_struct *ssp, + const char *name, + struct lock_class_key *key) +{ + return __init_srcu_struct(ssp, name, key); +} + +__rust_helper int rust_helper_srcu_read_lock(struct srcu_struct *ssp) +{ + return srcu_read_lock(ssp); +} + +__rust_helper void rust_helper_srcu_read_unlock(struct srcu_struct *ssp, int idx) +{ + srcu_read_unlock(ssp, idx); +} + +__rust_helper void rust_helper_srcu_barrier(struct srcu_struct *ssp) +{ + srcu_barrier(ssp); +} + +__rust_helper void rust_helper_synchronize_srcu_expedited(struct srcu_struct *ssp) +{ + synchronize_srcu_expedited(ssp); +} From 91d5185187e97611e328d03ee5e00e54b5d6d2b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Onur=20=C3=96zkan?= Date: Sat, 13 Jun 2026 09:40:09 +0300 Subject: [PATCH 05/47] srcu: expose srcu_readers_active() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is needed by rust/helpers/srcu.c which now adds rust_helper_srcu_readers_active() as a wrapper around the SRCU helper for Rust callers. To achive this: 1- Move the srcu_readers_active() implementation from "kernel/rcu/srcutree.c" to "include/linux/srcutree.h". 2- Implement a matching srcu_readers_active() in "include/linux/srcutiny.h" and use it on the existing open-coded WARN_ON() check in cleanup_srcu_struct(). Signed-off-by: Onur Özkan Reviewed-by: Gary Guo Reviewed-by: Alice Ryhl Reviewed-by: Boqun Feng Signed-off-by: Paul E. McKenney --- include/linux/srcutiny.h | 13 +++++++++++++ include/linux/srcutree.h | 24 ++++++++++++++++++++++++ kernel/rcu/srcutiny.c | 2 +- kernel/rcu/srcutree.c | 25 ------------------------- rust/helpers/srcu.c | 5 +++++ 5 files changed, 43 insertions(+), 26 deletions(-) diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h index 905b629e8fa3..fbcf13bc12d1 100644 --- a/include/linux/srcutiny.h +++ b/include/linux/srcutiny.h @@ -154,4 +154,17 @@ static inline void srcu_torture_stats_print(struct srcu_struct *ssp, data_race(READ_ONCE(ssp->srcu_idx_max))); } +/** + * srcu_readers_active - returns true if there are readers. and false otherwise. + * @ssp: which srcu_struct to count active readers (holding srcu_read_lock). + * + * Note that this is not an atomic primitive, and can therefore suffer + * severe errors when invoked on an active srcu_struct. That said, it + * can be useful as an error check at cleanup time. + */ +static inline bool srcu_readers_active(struct srcu_struct *ssp) +{ + return READ_ONCE(ssp->srcu_lock_nesting[0]) || READ_ONCE(ssp->srcu_lock_nesting[1]); +} + #endif diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h index fd1a9270cb9a..75e54e4f963f 100644 --- a/include/linux/srcutree.h +++ b/include/linux/srcutree.h @@ -374,4 +374,28 @@ static inline void srcu_check_read_flavor(struct srcu_struct *ssp, int read_flav __srcu_check_read_flavor(ssp, read_flavor); } +/** + * srcu_readers_active - returns true if there are readers. and false otherwise. + * @ssp: which srcu_struct to count active readers (holding srcu_read_lock). + * + * Note that this is not an atomic primitive, and can therefore suffer + * severe errors when invoked on an active srcu_struct. That said, it + * can be useful as an error check at cleanup time. + */ +static inline bool srcu_readers_active(struct srcu_struct *ssp) +{ + int cpu; + unsigned long sum = 0; + + for_each_possible_cpu(cpu) { + struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); + + sum += atomic_long_read(&sdp->srcu_ctrs[0].srcu_locks); + sum += atomic_long_read(&sdp->srcu_ctrs[1].srcu_locks); + sum -= atomic_long_read(&sdp->srcu_ctrs[0].srcu_unlocks); + sum -= atomic_long_read(&sdp->srcu_ctrs[1].srcu_unlocks); + } + return sum; +} + #endif diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c index 47d48ed31848..558ba8d316db 100644 --- a/kernel/rcu/srcutiny.c +++ b/kernel/rcu/srcutiny.c @@ -85,7 +85,7 @@ EXPORT_SYMBOL_GPL(init_srcu_struct_generic); */ void cleanup_srcu_struct(struct srcu_struct *ssp) { - WARN_ON(ssp->srcu_lock_nesting[0] || ssp->srcu_lock_nesting[1]); + WARN_ON(srcu_readers_active(ssp)); irq_work_sync(&ssp->srcu_irq_work); flush_work(&ssp->srcu_work); WARN_ON(ssp->srcu_gp_running); diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 56c5db5c29a0..4a00e90e17fc 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -599,31 +599,6 @@ static bool srcu_readers_active_idx_check(struct srcu_struct *ssp, int idx) return srcu_readers_lock_idx(ssp, idx, did_gp, unlocks); } -/** - * srcu_readers_active - returns true if there are readers. and false - * otherwise - * @ssp: which srcu_struct to count active readers (holding srcu_read_lock). - * - * Note that this is not an atomic primitive, and can therefore suffer - * severe errors when invoked on an active srcu_struct. That said, it - * can be useful as an error check at cleanup time. - */ -static bool srcu_readers_active(struct srcu_struct *ssp) -{ - int cpu; - unsigned long sum = 0; - - for_each_possible_cpu(cpu) { - struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); - - sum += atomic_long_read(&sdp->srcu_ctrs[0].srcu_locks); - sum += atomic_long_read(&sdp->srcu_ctrs[1].srcu_locks); - sum -= atomic_long_read(&sdp->srcu_ctrs[0].srcu_unlocks); - sum -= atomic_long_read(&sdp->srcu_ctrs[1].srcu_unlocks); - } - return sum; -} - /* * We use an adaptive strategy for synchronize_srcu() and especially for * synchronize_srcu_expedited(). We spin for a fixed time period diff --git a/rust/helpers/srcu.c b/rust/helpers/srcu.c index 225b3bf9334a..1a2f563640e0 100644 --- a/rust/helpers/srcu.c +++ b/rust/helpers/srcu.c @@ -9,6 +9,11 @@ __rust_helper int rust_helper_init_srcu_struct_with_key(struct srcu_struct *ssp, return __init_srcu_struct(ssp, name, key); } +__rust_helper bool rust_helper_srcu_readers_active(struct srcu_struct *ssp) +{ + return srcu_readers_active(ssp); +} + __rust_helper int rust_helper_srcu_read_lock(struct srcu_struct *ssp) { return srcu_read_lock(ssp); From 59cf3a5bda61cd3c300f0db952afcb031806e802 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Onur=20=C3=96zkan?= Date: Sat, 13 Jun 2026 09:40:10 +0300 Subject: [PATCH 06/47] rust: sync: add SRCU abstraction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a Rust abstraction for sleepable RCU (SRCU), backed by C srcu_struct. Provide FFI helpers and a safe wrapper with a guard-based API for read-side critical sections. Cleanup is handled via `PinnedDrop`. It first checks for active read-side sections and emits a warning if any guards were leaked. In that case, it waits in `synchronize_srcu()` rather than risking a UAF by freeing the `srcu_struct` that is still reachable from the C side. It then uses `srcu_barrier()` to drain pending callbacks before finally calling `cleanup_srcu_struct()`. Signed-off-by: Onur Özkan Reviewed-by: Gary Guo Reviewed-by: Alice Ryhl Reviewed-by: Boqun Feng Signed-off-by: Paul E. McKenney --- rust/kernel/sync.rs | 2 + rust/kernel/sync/srcu.rs | 171 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 173 insertions(+) create mode 100644 rust/kernel/sync/srcu.rs diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index 993dbf2caa0e..0d6a5f1300c3 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -21,6 +21,7 @@ pub mod rcu; mod refcount; mod set_once; +pub mod srcu; pub use arc::{Arc, ArcBorrow, UniqueArc}; pub use completion::Completion; @@ -31,6 +32,7 @@ pub use locked_by::LockedBy; pub use refcount::Refcount; pub use set_once::SetOnce; +pub use srcu::Srcu; /// Represents a lockdep class. /// diff --git a/rust/kernel/sync/srcu.rs b/rust/kernel/sync/srcu.rs new file mode 100644 index 000000000000..723e5e277fd6 --- /dev/null +++ b/rust/kernel/sync/srcu.rs @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Sleepable read-copy update (SRCU) support. +//! +//! C header: [`include/linux/srcu.h`](srctree/include/linux/srcu.h) + +use crate::{ + bindings, + error::to_result, + prelude::*, + sync::LockClassKey, + types::{ + NotThreadSafe, + Opaque, // + }, +}; + +use pin_init::pin_data; + +/// Creates an [`Srcu`] initialiser with the given name and a newly-created lock class. +#[doc(hidden)] +#[macro_export] +macro_rules! new_srcu { + ($($name:literal)?) => { + $crate::sync::Srcu::new($crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} +pub use new_srcu; + +/// Sleepable read-copy update primitive. +/// +/// SRCU readers may sleep while holding the read-side guard. +/// +/// The destructor waits for active readers and callbacks, so it may sleep. +/// If a read-side guard has been leaked, dropping an [`Srcu`] may never return. +/// +/// # Invariants +/// +/// This represents a valid `struct srcu_struct` initialized by the C SRCU API +/// and it remains pinned and valid until the pinned destructor runs. +#[repr(transparent)] +#[pin_data(PinnedDrop)] +pub struct Srcu { + #[pin] + inner: Opaque, +} + +impl Srcu { + /// Creates a new SRCU instance. + #[inline] + pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit { + try_pin_init!(Self { + // INVARIANT: On success, the C initializer creates a valid `srcu_struct` and + // it remains pinned until `PinnedDrop` runs. + inner <- Opaque::try_ffi_init(|ptr: *mut bindings::srcu_struct| { + // SAFETY: `ptr` points to valid uninitialised memory for a `srcu_struct`. + to_result(unsafe { + bindings::init_srcu_struct_with_key(ptr, name.as_char_ptr(), key.as_ptr()) + }) + }), + }) + } + + /// Enters an SRCU read-side critical section. + /// + /// Leaking the returned [`Guard`] leaves the SRCU read-side critical + /// section active and makes `drop` sleep forever. + #[inline] + pub fn read_lock(&self) -> Guard<'_> { + // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`. + let idx = unsafe { bindings::srcu_read_lock(self.inner.get()) }; + + // INVARIANT: `idx` was returned by `srcu_read_lock()` for this `Srcu`. + Guard { + srcu: self, + idx, + _not_send: NotThreadSafe, + } + } + + /// Waits until all pre-existing SRCU readers have completed. + #[inline] + pub fn synchronize(&self) { + // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`. + unsafe { bindings::synchronize_srcu(self.inner.get()) }; + } + + /// Waits until all pre-existing SRCU readers have completed, expedited. + /// + /// This requests a lower-latency grace period than [`Srcu::synchronize`] typically + /// at the cost of higher system-wide overhead. Prefer [`Srcu::synchronize`] by default + /// and use this variant only when reducing reset or teardown latency is more important + /// than the extra cost. + #[inline] + pub fn synchronize_expedited(&self) { + // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`. + unsafe { bindings::synchronize_srcu_expedited(self.inner.get()) }; + } +} + +#[pinned_drop] +impl PinnedDrop for Srcu { + fn drop(self: Pin<&mut Self>) { + let ptr = self.inner.get(); + + if crate::warn_on!( + // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct` + // and `srcu_readers_active()` only checks the active reader count. + unsafe { bindings::srcu_readers_active(ptr) } + ) { + // `cleanup_srcu_struct()` may return early if there are still active readers. + // This should only happen if a guard was leaked with `mem::forget`, which is + // "WRONG" code and may cause a UAF because Rust will free the `srcu_struct` + // while it is still referenced from the C side (e.g. by `call_srcu()` callbacks). + // + // Another consequence of leaking guards is that `call_srcu()` callbacks will + // never run because the grace period can never complete due to permanently + // active readers (i.e. leaked guards). + // + // If this ever happens, that means the guard was leaked by mistake and the + // caller must fix the bug. Sleeping here is intentional and less harmful + // than risking a UAF. + // + // SAFETY: By the type invariants, `self` contains a valid and pinned + // `struct srcu_struct`. + unsafe { bindings::synchronize_srcu(ptr) }; + } + + // Ensure all SRCU callbacks have been finished before freeing. + // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct`. + unsafe { bindings::srcu_barrier(ptr) }; + + // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct`. + unsafe { bindings::cleanup_srcu_struct(ptr) }; + } +} + +// SAFETY: `srcu_struct` may be shared and used across threads. +unsafe impl Send for Srcu {} +// SAFETY: `srcu_struct` may be shared and used concurrently. +unsafe impl Sync for Srcu {} + +/// Guard for an active SRCU read-side critical section on a particular [`Srcu`]. +/// +/// Leaking this guard with [`core::mem::forget`] leaves the SRCU read-side +/// critical section active and makes dropping the associated [`Srcu`] sleep forever. +/// +/// # Invariants +/// +/// `idx` is the index returned by `srcu_read_lock()` for `srcu`. +#[must_use = "if unused, the lock will be immediately unlocked"] +pub struct Guard<'a> { + srcu: &'a Srcu, + idx: i32, + _not_send: NotThreadSafe, +} + +impl Guard<'_> { + /// Explicitly releases the SRCU read-side critical section. + #[inline] + pub fn unlock(self) {} +} + +impl Drop for Guard<'_> { + #[inline] + fn drop(&mut self) { + // SAFETY: `Guard` is only constructible through `Srcu::read_lock()`, + // which returns a valid index for the SRCU instance. + unsafe { bindings::srcu_read_unlock(self.srcu.inner.get(), self.idx) }; + } +} From 3f90d04303c11a5006059016fc1ebedcbbff682d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Onur=20=C3=96zkan?= Date: Sat, 13 Jun 2026 09:40:11 +0300 Subject: [PATCH 07/47] MAINTAINERS: add Rust SRCU files to SRCU entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Include Rust side implementation files to the SRCU maintainer entry. Signed-off-by: Onur Özkan Reviewed-by: Alice Ryhl Reviewed-by: Boqun Feng Signed-off-by: Paul E. McKenney --- MAINTAINERS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 806bd2d80d15..b6ef971ee408 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -24933,6 +24933,7 @@ SLEEPABLE READ-COPY UPDATE (SRCU) M: Lai Jiangshan M: "Paul E. McKenney" M: Josh Triplett +M: Onur Özkan (RUST) R: Steven Rostedt R: Mathieu Desnoyers L: rcu@vger.kernel.org @@ -24941,6 +24942,8 @@ W: http://www.rdrop.com/users/paulmck/RCU/ T: git git://git.kernel.org/pub/scm/linux/kernel/git/rcu/linux.git rcu/dev F: include/linux/srcu*.h F: kernel/rcu/srcu*.c +F: rust/helpers/srcu.c +F: rust/kernel/sync/srcu.rs SMACK SECURITY MODULE M: Casey Schaufler From 1384ea908e5de451061b50b4eb76c4440c4d4a0d Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Fri, 12 Jun 2026 10:34:12 +0800 Subject: [PATCH 08/47] rcu: Use task_state_to_char() in stall-warning prints RCU stall warnings currently print task states as raw hexadecimal values, requiring developers to manually decode them. Use task_state_to_char() so that stall warnings show the same symbolic task-state representation used elsewhere in the kernel. For example: ->state=0x402 becomes ->state=I ->state=0x0 becomes ->state=R ->state=0x2 becomes ->state=D This improves readability while preserving the underlying diagnostic information. Suggested-by: Zqiang Co-developed-by: Wang Lian Signed-off-by: Wang Lian Signed-off-by: Kunwu Chan Signed-off-by: Paul E. McKenney --- kernel/rcu/tree_stall.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index cf7ae51cba40..45b9856ccd2b 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -573,13 +573,13 @@ static void rcu_check_gp_kthread_starvation(void) if (rcu_is_gp_kthread_starving(&j)) { cpu = gpk ? task_cpu(gpk) : -1; - pr_err("%s kthread starved for %ld jiffies! g%ld f%#x %s(%d) ->state=%#x ->cpu=%d\n", + pr_err("%s kthread starved for %ld jiffies! g%ld f%#x %s(%d) ->state=%c ->cpu=%d\n", rcu_state.name, j, (long)rcu_seq_current(&rcu_state.gp_seq), data_race(READ_ONCE(rcu_state.gp_flags)), gp_state_getname(rcu_state.gp_state), data_race(READ_ONCE(rcu_state.gp_state)), - gpk ? data_race(READ_ONCE(gpk->__state)) : ~0, cpu); + gpk ? task_state_to_char(gpk) : '?', cpu); if (gpk) { struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); @@ -616,12 +616,12 @@ static void rcu_check_gp_kthread_expired_fqs_timer(void) time_after(jiffies, jiffies_fqs + RCU_STALL_MIGHT_MIN) && gpk && !READ_ONCE(gpk->on_rq)) { cpu = task_cpu(gpk); - pr_err("%s kthread timer wakeup didn't happen for %ld jiffies! g%ld f%#x %s(%d) ->state=%#x\n", + pr_err("%s kthread timer wakeup didn't happen for %ld jiffies! g%ld f%#x %s(%d) ->state=%c\n", rcu_state.name, (jiffies - jiffies_fqs), (long)rcu_seq_current(&rcu_state.gp_seq), data_race(READ_ONCE(rcu_state.gp_flags)), // Diagnostic read gp_state_getname(RCU_GP_WAIT_FQS), RCU_GP_WAIT_FQS, - data_race(READ_ONCE(gpk->__state))); + task_state_to_char(gpk)); pr_err("\tPossible timer handling issue on cpu=%d timer-softirq=%u\n", cpu, kstat_softirqs_cpu(TIMER_SOFTIRQ, cpu)); } @@ -948,10 +948,10 @@ void show_rcu_gp_kthreads(void) jr = j - data_race(READ_ONCE(rcu_state.gp_req_activity)); js = j - data_race(READ_ONCE(rcu_state.gp_start)); jw = j - data_race(READ_ONCE(rcu_state.gp_wake_time)); - pr_info("%s: wait state: %s(%d) ->state: %#x ->rt_priority %u delta ->gp_start %lu ->gp_activity %lu ->gp_req_activity %lu ->gp_wake_time %lu ->gp_wake_seq %ld ->gp_seq %ld ->gp_seq_needed %ld ->gp_max %lu ->gp_flags %#x\n", + pr_info("%s: wait state: %s(%d) ->state: %c ->rt_priority %u delta ->gp_start %lu ->gp_activity %lu ->gp_req_activity %lu ->gp_wake_time %lu ->gp_wake_seq %ld ->gp_seq %ld ->gp_seq_needed %ld ->gp_max %lu ->gp_flags %#x\n", rcu_state.name, gp_state_getname(rcu_state.gp_state), data_race(READ_ONCE(rcu_state.gp_state)), - t ? data_race(READ_ONCE(t->__state)) : 0x1ffff, t ? t->rt_priority : 0xffU, + t ? task_state_to_char(t) : '?', t ? t->rt_priority : 0xffU, js, ja, jr, jw, (long)data_race(READ_ONCE(rcu_state.gp_wake_seq)), (long)data_race(READ_ONCE(rcu_state.gp_seq)), (long)data_race(READ_ONCE(rcu_get_root()->gp_seq_needed)), From 573d76f7db5ba050bc7c7c73d30e1364fcee269b Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Sun, 28 Jun 2026 09:00:01 -0700 Subject: [PATCH 09/47] rcu: Mark __rcu_access_pointer() as context_unsafe() A simple comparison of a pointer returned by rcu_access_pointer() results in a context-analysis warning for lockless inspection of the RCU-protected (also known as __rcu-protected) pointer. This can be suppressed by placing context_unsafe() calls around calls rcu_access_pointer(), but this is messy and distracting. This commit therefore wraps the underlying __rcu_access_pointer() macro with a call to context_unsafe(), thereby informing the context-analysis code that rcu_access_pointer() may safely be invoked outside of an RCU read-side critical section. Reported-by: Christoph Hellwig Suggested-by: Marco Elver Signed-off-by: Paul E. McKenney Tested-by: Nilay Shroff Reviewed-by: Marco Elver --- include/linux/rcupdate.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 5e95acc33989..e40dc2e20c5b 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -490,12 +490,12 @@ context_unsafe( \ */ #define unrcu_pointer(p) __unrcu_pointer(p, __UNIQUE_ID(rcu)) -#define __rcu_access_pointer(p, local, space) \ +#define __rcu_access_pointer(p, local, space) context_unsafe( \ ({ \ typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \ rcu_check_sparse(p, space); \ ((typeof(*p) __force __kernel *)(local)); \ -}) +}) ) #define __rcu_dereference_check(p, local, c, space) \ ({ \ /* Dependency order vs. p above. */ \ From dc7e119abd58297408d2ca34d216242715eda7b7 Mon Sep 17 00:00:00 2001 From: Manuel Ebner Date: Sat, 27 Jun 2026 11:06:06 +0200 Subject: [PATCH 10/47] doc: RCU: Adopt new coding style of type-aware kmalloc-family - part 2/2 Update Documentation/RCU/* to suggest using the new type-aware kmalloc_obj() per commit 2932ba8d9c99 ("slab: Introduce kmalloc_obj() and family") p = kmalloc(...); -> p = kmalloc_obj(...); Signed-off-by: Manuel Ebner Signed-off-by: Paul E. McKenney --- Documentation/RCU/rcu_dereference.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/RCU/rcu_dereference.rst b/Documentation/RCU/rcu_dereference.rst index 2524dcdadde2..5bc3785ebfc2 100644 --- a/Documentation/RCU/rcu_dereference.rst +++ b/Documentation/RCU/rcu_dereference.rst @@ -236,7 +236,7 @@ precautions. To see this, consider the following code fragment:: { struct foo *p; - p = kmalloc(...); + p = kmalloc_obj(*p); if (p == NULL) deal_with_it(); p->a = 42; /* Each field in its own cache line. */ @@ -293,7 +293,7 @@ Then one approach is to use locking, for example, as follows:: { struct foo *p; - p = kmalloc(...); + p = kmalloc_obj(*p); if (p == NULL) deal_with_it(); spin_lock(&p->lock); From 2b109eccd80b660e58d4523e8b80ce11bc3e532e Mon Sep 17 00:00:00 2001 From: Manuel Ebner Date: Sat, 27 Jun 2026 11:26:45 +0200 Subject: [PATCH 11/47] doc: RCU: Fix brackets Remove needless brackets and add missing brackets. Signed-off-by: Manuel Ebner Signed-off-by: Paul E. McKenney --- .../Design/Expedited-Grace-Periods/Expedited-Grace-Periods.rst | 2 +- Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp.svg | 2 +- Documentation/RCU/Design/Memory-Ordering/TreeRCU-qs.svg | 2 +- Documentation/RCU/Design/Requirements/Requirements.rst | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/RCU/Design/Expedited-Grace-Periods/Expedited-Grace-Periods.rst b/Documentation/RCU/Design/Expedited-Grace-Periods/Expedited-Grace-Periods.rst index 414f8a2012d6..cf0f9cdca7e8 100644 --- a/Documentation/RCU/Design/Expedited-Grace-Periods/Expedited-Grace-Periods.rst +++ b/Documentation/RCU/Design/Expedited-Grace-Periods/Expedited-Grace-Periods.rst @@ -410,7 +410,7 @@ workqueues (see Documentation/core-api/workqueue.rst). The requesting task still does counter snapshotting and funnel-lock processing, but the task reaching the top of the funnel lock does a -``schedule_work()`` (from ``_synchronize_rcu_expedited()`` so that a +``schedule_work()`` (from ``_synchronize_rcu_expedited()``) so that a workqueue kthread does the actual grace-period processing. Because workqueue kthreads do not accept POSIX signals, grace-period-wait processing need not allow for POSIX signals. In addition, this approach diff --git a/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp.svg b/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp.svg index d05bc7b27edb..95a66de40ca5 100644 --- a/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp.svg +++ b/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp.svg @@ -3933,7 +3933,7 @@ font-style="normal" y="-3914.085" x="3745.7725" - xml:space="preserve">rcu__report_qs_rdp()) + xml:space="preserve">rcu__report_qs_rdp() rcu__report_qs_rdp()) + xml:space="preserve">rcu__report_qs_rdp() Date: Thu, 25 Jun 2026 20:42:54 -0400 Subject: [PATCH 12/47] rcu: introduce rcu_defer_qs_clear() helper Currently rdp->defer_qs_pending transitions from DEFER_QS_PENDING to DEFER_QS_IDLE at two sites: rcu_preempt_deferred_qs_irqrestore() and rcu_preempt_deferred_qs_handler() (depth>0 reset). Both write the IDLE value directly. Introduce a single inline helper rcu_defer_qs_clear() in tree.h and route both sites through it. This becomes the single PENDING->IDLE transition point for upcoming work. Signed-off-by: Joel Fernandes Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.h | 5 +++++ kernel/rcu/tree_plugin.h | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 7dfc57e9adb1..4069132f9d44 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -296,6 +296,11 @@ struct rcu_data { int cpu; }; +static inline void rcu_defer_qs_clear(struct rcu_data *rdp) +{ + WRITE_ONCE(rdp->defer_qs_pending, DEFER_QS_IDLE); +} + /* Values for nocb_defer_wakeup field in struct rcu_data. */ #define RCU_NOCB_WAKE_NOT 0 #define RCU_NOCB_WAKE_BYPASS 1 diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 95ad967adcf3..8637f405cb47 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -488,7 +488,7 @@ rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags) rdp = this_cpu_ptr(&rcu_data); if (rdp->defer_qs_pending == DEFER_QS_PENDING) - rdp->defer_qs_pending = DEFER_QS_IDLE; + rcu_defer_qs_clear(rdp); /* * If RCU core is waiting for this CPU to exit its critical section, @@ -645,7 +645,7 @@ static void rcu_preempt_deferred_qs_handler(struct irq_work *iwp) * 5. Deferred QS reporting does not happen. */ if (rcu_preempt_depth() > 0) - WRITE_ONCE(rdp->defer_qs_pending, DEFER_QS_IDLE); + rcu_defer_qs_clear(rdp); } /* From d972d106965f16348f2c44035440f4926a5880d3 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Thu, 25 Jun 2026 20:43:00 -0400 Subject: [PATCH 13/47] rcu: clear defer_qs_pending in deferred-QS bail when nesting > 0 Paul McKenney noted that a softirq (or irq_work) handler arming for a deferred QS can fire and find rcu_preempt_depth() > 0 -- the task is still inside its outer reader, so rcu_preempt_need_deferred_qs() bails without reporting the QS. At that point the queued mechanism has been consumed but ->defer_qs_pending stays in DEFER_QS_PENDING. In the meantime, the only remaining path back to a quiescent state on this CPU may be a local_irq_disable()/_enable() pair that does not call preempt_check_resched() (it is just `sti`/`cli`). patch 6's unconditional set_need_resched_current() makes need_resched true, but without an irq_work being raised the next outer rcu_read_unlock_special() hits the P-gate at the arming code: if (rdp->defer_qs_pending != DEFER_QS_PENDING) { rdp->defer_qs_pending = DEFER_QS_PENDING; irq_work_queue_on(...); // <-- skipped } so no irq_work is queued for the hardirq-exit preempt_schedule_irq() path either. The deferred QS now waits until the next timer tick (or similar preempt-safe boundary), needlessly extending expedited grace period latency. Clear ->defer_qs_pending in the bail-out path of rcu_preempt_deferred_qs() when rcu_preempt_depth() > 0. The recursion guard semantics introduced by commit b41642c87716 ("rcu: Fix rcu_read_unlock() deadloop due to IRQ work"). The clear is also safe against fresh recursion at this exact program point: rcu_preempt_depth() > 0 guarantees we are still inside an outer reader, so any inner rcu_read_unlock() from tracing infrastructure brings nesting back to outer (>0), never to 0. The slow path of rcu_read_unlock_special() is structurally unreachable under that condition, so no recursive raise_softirq_irqoff()/irq_work_queue_on() can be triggered by the clear. Essentially, the mechanism will work to prevent the following recursion which Xiongfeng had previously reported: irq_exit() -> __irq_exit_rcu() -> tick_irq_exit() -> tick_nohz_irq_exit() -> tick_nohz_stop_sched_tick() -> trace_tick_stop() // BPF prog hooked here -> rcu_read_unlock_special() -> irq_work_queue_on(&rdp->defer_qs_iw, rdp->cpu) // self-IPI re-enters irq_exit Reported-by: Paul E. McKenney Signed-off-by: Joel Fernandes Signed-off-by: Paul E. McKenney --- kernel/rcu/tree_plugin.h | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 8637f405cb47..9ba136a4233a 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -614,9 +614,35 @@ static notrace bool rcu_preempt_need_deferred_qs(struct task_struct *t) notrace void rcu_preempt_deferred_qs(struct task_struct *t) { unsigned long flags; + struct rcu_data *rdp; - if (!rcu_preempt_need_deferred_qs(t)) + if (!rcu_preempt_need_deferred_qs(t)) { + /* + * If we got here from a softirq/irq_work that fired while + * rcu_preempt_depth() > 0, the deferred-QS mechanism has been + * consumed without doing any work: rcu_preempt_need_deferred_qs() + * just returned false because the task is still in a reader, so + * the actual QS report has to wait for the next + * rcu_read_unlock(). + * + * Clear ->defer_qs_pending here so the next outer + * rcu_read_unlock_special() can re-arm a fresh mechanism (in + * particular the irq_work path, which the local_irq_enable() + * recovery boundary cannot itself reschedule from). + * + * Recursion safety: rcu_preempt_depth() > 0 means we are inside + * an outer reader, so any inner rcu_read_unlock() reached via + * tracing (bpf programs attached to trace points) brings + * nesting to outer (> 0), never to 0, so no recursive + * raise_softirq_irqoff()/irq_work_queue_on() can be triggered + * by this clear. + */ + if (rcu_preempt_depth() > 0) { + rdp = this_cpu_ptr(&rcu_data); + rcu_defer_qs_clear(rdp); + } return; + } local_irq_save(flags); rcu_preempt_deferred_qs_irqrestore(t, flags); } From 5b79f1d5d791bb41c66cbb75387a53c19f1ffe1e Mon Sep 17 00:00:00 2001 From: Zqiang Date: Mon, 6 Jul 2026 18:03:09 +0800 Subject: [PATCH 14/47] rcu-tasks: Remove unused struct rcu_tasks's->n_ipis_fails variables The RCU tasks trace has been reimplemented by 'commit c27cea4416a3 ("rcu: Re-implement RCU Tasks Trace in terms of SRCU-fast")', the rcu_tasks structure's->n_ipis_fails is no longer used, this commit therefore remove it. Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index f4da5fad70f5..7f7d1e636217 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -67,7 +67,6 @@ struct rcu_tasks_percpu { * @gp_start: Most recent grace-period start in jiffies. * @tasks_gp_seq: Number of grace periods completed since boot in upper bits. * @n_ipis: Number of IPIs sent to encourage grace periods to end. - * @n_ipis_fails: Number of IPI-send failures. * @kthread_ptr: This flavor's grace-period/callback-invocation kthread. * @lazy_jiffies: Number of jiffies to allow callbacks to be lazy. * @pregp_func: This flavor's pre-grace-period function (optional). @@ -102,7 +101,6 @@ struct rcu_tasks { unsigned long gp_start; unsigned long tasks_gp_seq; unsigned long n_ipis; - unsigned long n_ipis_fails; struct task_struct *kthread_ptr; unsigned long lazy_jiffies; rcu_tasks_gp_func_t gp_func; @@ -735,12 +733,12 @@ static void show_rcu_tasks_generic_gp_kthread(struct rcu_tasks *rtp, char *s) if (havecbs && haveurgent && haveurgentcbs) break; } - pr_info("%s: %s(%d) since %lu g:%lu i:%lu/%lu %c%c%c%c l:%lu %s\n", + pr_info("%s: %s(%d) since %lu g:%lu i:%lu %c%c%c%c l:%lu %s\n", rtp->kname, tasks_gp_state_getname(rtp), data_race(rtp->gp_state), jiffies - data_race(rtp->gp_jiffies), data_race(rcu_seq_current(&rtp->tasks_gp_seq)), - data_race(rtp->n_ipis_fails), data_race(rtp->n_ipis), + data_race(rtp->n_ipis), ".k"[!!data_race(rtp->kthread_ptr)], ".C"[havecbs], ".u"[haveurgent], From ef21090071b33844f55fac7c5470729b862dad0c Mon Sep 17 00:00:00 2001 From: Zqiang Date: Mon, 6 Jul 2026 18:03:10 +0800 Subject: [PATCH 15/47] rcu-tasks: Dump rcu tasks status when the boot-test failed This commit adds a dump RCU task status function to rcu tasks tests, used to obtain more information to help debug when the RCU tasks boot-time tests failed. Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 7f7d1e636217..d9e1e53f4ccf 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1453,6 +1453,7 @@ struct rcu_tasks_test_desc { const char *name; bool notrun; unsigned long runstart; + void (*gp_dbg)(void); }; static struct rcu_tasks_test_desc tests[] = { @@ -1460,6 +1461,8 @@ static struct rcu_tasks_test_desc tests[] = { .name = "call_rcu_tasks()", /* If not defined, the test is skipped. */ .notrun = IS_ENABLED(CONFIG_TASKS_RCU), + /* Dump rcu tasks status, if test failed. */ + .gp_dbg = show_rcu_tasks_classic_gp_kthread }, { .name = "call_rcu_tasks_trace()", @@ -1519,6 +1522,8 @@ static int rcu_tasks_verify_self_tests(void) while (tests[i].notrun) { // still hanging. if (time_after(jiffies, tests[i].runstart + bst)) { pr_err("%s has failed boot-time tests.\n", tests[i].name); + if (tests[i].gp_dbg) + tests[i].gp_dbg(); ret = -1; break; } From 647dd6e09481daa23d93e927c2b7f46a3acf8a03 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Sun, 12 Jul 2026 10:50:08 -0700 Subject: [PATCH 16/47] rcu-tasks: Apply READ_ONCE() and WRITE_ONCE() to fix data race Now that rcutorture tests readers from interrupt handlers, KCSAN spotted an additional data race. This commit therefore fixes it by applying READ_ONCE() and WRITE_ONCE(). Signed-off-by: Paul E. McKenney --- include/linux/rcupdate_trace.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h index cee89e51e45c..fd3ddeb6aa3b 100644 --- a/include/linux/rcupdate_trace.h +++ b/include/linux/rcupdate_trace.h @@ -95,10 +95,13 @@ static inline void rcu_read_unlock_tasks_trace(struct srcu_ctr __percpu *scp) */ static inline void rcu_read_lock_trace(void) { + int n; struct task_struct *t = current; rcu_try_lock_acquire(&rcu_tasks_trace_srcu_struct.dep_map); - if (t->trc_reader_nesting++) { + n = READ_ONCE(t->trc_reader_nesting); + WRITE_ONCE(t->trc_reader_nesting, n + 1); + if (n) { // In case we interrupted a Tasks Trace RCU reader. return; } @@ -119,12 +122,15 @@ static inline void rcu_read_lock_trace(void) */ static inline void rcu_read_unlock_trace(void) { + int n; struct srcu_ctr __percpu *scp; struct task_struct *t = current; scp = t->trc_reader_scp; barrier(); // scp before nesting to protect against interrupt handler. - if (!--t->trc_reader_nesting) { + n = READ_ONCE(t->trc_reader_nesting) - 1; + WRITE_ONCE(t->trc_reader_nesting, n); + if (!n) { if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB)) smp_mb(); // Placeholder for more selective ordering __srcu_read_unlock_fast(&rcu_tasks_trace_srcu_struct, scp); From a781301250bc003dc18688d31db3e39a546b55f8 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Mon, 13 Jul 2026 18:31:12 +0800 Subject: [PATCH 17/47] rcu-tasks: Remove smp_mb() in rcu_spawn_tasks_kthread_generic() For the smp_mb() after kthread_run() in rcu_spawn_tasks_kthread_generic() from these commits: 'commit eacd6f04a133 ("rcu-tasks: Move Tasks RCU to its own file")' 'commit 84a8f446ffd7 ("rcu: Defer rcu_tasks_kthread() creation till first call_rcu_tasks()")' the memory order pairing as follows: rcu_spawn_tasks_kthread() ->t = kthread_run(rcu_tasks_kthread, ...); ->smp_mb(); /* Ensure others see full kthread. */ ->WRITE_ONCE(rcu_tasks_kthread_ptr, t); call_rcu_tasks() ->if (READ_ONCE(rcu_tasks_kthread_ptr)) ->wake_up(&rcu_tasks_cbs_wq) ->try_to_wake_up() lock pi_lock ->smp_mb__after_spinlock() //see full kthread Because the 'commit d119357d0743 ("rcu-tasks: Treat only synchronous grace periods urgently")' moved the kthread_ptr assignment into the rcu_tasks_kthread() function, the following memory order pairings are sufficient: The runq's raw_spinlock/unlock(or smp_mb__after_spinlock()) from wake_up_process() in kthread_run() and __schedule() provides memory order barrier when the kthread is first scheduled, this ensures the kthread's func observes all of the kthread's initialization. The kthread's smp_store_release(&rtp->kthread_ptr, ...) in rcu_tasks_kthread() and smp_load_acquire(&rtp->kthread_ptr) in call_rcu_tasks_generic() compose release/acquire pairing, the cumulativity of smp_store_release() propagates visibility of the kthread's initialization through the scheduler chain. This commit therefore remove smp_mb() in rcu_spawn_tasks_kthread_generic(). Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 1 - 1 file changed, 1 deletion(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index d9e1e53f4ccf..79aa5d51388c 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -681,7 +681,6 @@ static void __init rcu_spawn_tasks_kthread_generic(struct rcu_tasks *rtp) t = kthread_run(rcu_tasks_kthread, rtp, "%s_kthread", rtp->kname); if (WARN_ONCE(IS_ERR(t), "%s: Could not start %s grace-period kthread, OOM is now expected behavior\n", __func__, rtp->name)) return; - smp_mb(); /* Ensure others see full kthread. */ } #ifndef CONFIG_TINY_RCU From d2282f9bc75ca39a8ea5b7fe87191609985aca30 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 16 Jul 2026 12:53:56 -0700 Subject: [PATCH 18/47] rcu: Use this_cpu_{read,write}() for ->cpu_no_qs.b.exp Currently __this_cpu_read() and __this_cpu_write() is used to access the ->cpu_no_qs.b.exp field of the per-CPU rcu_data structure. However, this can fail when the accesses can happen in interrupt handlers, as recently started being exercised by rcutorture. This commit therefore upgrades the uses of __this_cpu_read() and __this_cpu_write() to their interrupt-safe counterparts this_cpu_read() and this_cpu_write(). KCSAN located this issue. Signed-off-by: Paul E. McKenney --- kernel/rcu/tree_exp.h | 4 ++-- kernel/rcu/tree_plugin.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index 82cada459e5d..e03cb3d447a5 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h @@ -731,7 +731,7 @@ static void rcu_exp_need_qs(void) { lockdep_assert_irqs_disabled(); ASSERT_EXCLUSIVE_WRITER_SCOPED(*this_cpu_ptr(&rcu_data.cpu_no_qs.b.exp)); - __this_cpu_write(rcu_data.cpu_no_qs.b.exp, true); + this_cpu_write(rcu_data.cpu_no_qs.b.exp, true); /* Store .exp before .rcu_urgent_qs. */ smp_store_release(this_cpu_ptr(&rcu_data.rcu_urgent_qs), true); set_need_resched_current(); @@ -870,7 +870,7 @@ static void rcu_exp_handler(void *unused) ASSERT_EXCLUSIVE_WRITER_SCOPED(rdp->cpu_no_qs.b.exp); if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) || - __this_cpu_read(rcu_data.cpu_no_qs.b.exp)) + this_cpu_read(rcu_data.cpu_no_qs.b.exp)) return; if (rcu_is_cpu_rrupt_from_idle() || (IS_ENABLED(CONFIG_PREEMPT_COUNT) && preempt_bh_enabled)) { diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 9ba136a4233a..c3db1ad4a280 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -599,7 +599,7 @@ rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags) */ static notrace bool rcu_preempt_need_deferred_qs(struct task_struct *t) { - return (__this_cpu_read(rcu_data.cpu_no_qs.b.exp) || + return (this_cpu_read(rcu_data.cpu_no_qs.b.exp) || READ_ONCE(t->rcu_read_unlock_special.s)) && rcu_preempt_depth() == 0; } @@ -981,7 +981,7 @@ static void rcu_qs(void) trace_rcu_grace_period(TPS("rcu_sched"), __this_cpu_read(rcu_data.gp_seq), TPS("cpuqs")); __this_cpu_write(rcu_data.cpu_no_qs.b.norm, false); - if (__this_cpu_read(rcu_data.cpu_no_qs.b.exp)) + if (this_cpu_read(rcu_data.cpu_no_qs.b.exp)) rcu_report_exp_rdp(this_cpu_ptr(&rcu_data)); } From a050e671d3545cf1890ab764387958be6c542f6c Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 16 Jul 2026 13:05:00 -0700 Subject: [PATCH 19/47] rcu: Use WRITE_ONCE() for ->rcu_need_heavy_qs Currently raw_cpu_write() is used to clear the ->rcu_need_heavy_qs field of the per-CPU rcu_data structure. However, on x86 this is a normal assignment, which does not play well with concurrent accesses. This commit therefore upgrades the uses of raw_cpu_write() to its concurrency-safe counterpart WRITE_ONCE() of an rdp pointer obtained from this_cpu_ptr(&rcu_data). KCSAN located this issue. Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 03a43d3d2616..91d5b4dd08bf 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -357,9 +357,10 @@ bool rcu_watching_zero_in_eqs(int cpu, int *vp) */ notrace void rcu_momentary_eqs(void) { + struct rcu_data *rdp = this_cpu_ptr(&rcu_data); int seq; - raw_cpu_write(rcu_data.rcu_need_heavy_qs, false); + WRITE_ONCE(rdp->rcu_need_heavy_qs, false); seq = ct_state_inc(2 * CT_RCU_WATCHING); /* It is illegal to call this from idle state. */ WARN_ON_ONCE(!(seq & CT_RCU_WATCHING)); From 4137d69124a29506a09756af0aaf0c7acd2a60f6 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Sat, 18 Jul 2026 13:15:32 -0400 Subject: [PATCH 20/47] rcu: Remove unused expedited_need_qs field from rcu_state The ->expedited_need_qs counter was part of the old expedited grace-period machinery that spun waiting for each CPU to check in. The current implementation instead tracks holdout CPUs via the rcu_node structures' ->expmask fields and waits on the ->exp_wq[] wait queues, so nothing reads or writes ->expedited_need_qs any longer. Remove the field to avoid wasting space in rcu_state and to keep readers of the expedited code from searching for nonexistent users. Signed-off-by: Joel Fernandes Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.h | 1 - 1 file changed, 1 deletion(-) diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 4069132f9d44..587bd71396ea 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -391,7 +391,6 @@ struct rcu_state { struct mutex exp_mutex; /* Serialize expedited GP. */ struct mutex exp_wake_mutex; /* Serialize wakeup. */ unsigned long expedited_sequence; /* Take a ticket. */ - atomic_t expedited_need_qs; /* # CPUs left to check in. */ struct swait_queue_head expedited_wq; /* Wait for check-ins. */ int ncpus_snap; /* # CPUs seen last time. */ u8 cbovld; /* Callback overload now? */ From 545b3ac4c1db3c455cbe1bd1ed305e54ad87512e Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Sat, 18 Jul 2026 13:15:34 -0400 Subject: [PATCH 21/47] rcu: Remove unused func parameter from callback-enqueue functions Ever since the kvfree_rcu() tracing moved out of the callback-enqueue path, rcutree_enqueue() no longer looks at the callback function pointer: By the time it is invoked, __call_rcu_common() has already stored the function into rhp->func, and the enqueue path only adds the rcu_head to the segmented callback list and emits tracepoints that do not take the function pointer. Nevertheless, the function pointer is still threaded through call_rcu_core(), call_rcu_nocb(), and rcutree_enqueue(), forcing each level to carry a dead argument. Remove the parameter from all three functions, from the no-CBs stub, and from the corresponding declarations. Anything needing the callback function can still get it from rhp->func. No functional change. Signed-off-by: Joel Fernandes Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 10 +++++----- kernel/rcu/tree.h | 2 +- kernel/rcu/tree_nocb.h | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 91d5b4dd08bf..390dad82675d 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -3024,7 +3024,7 @@ static int __init rcu_spawn_core_kthreads(void) return 0; } -static void rcutree_enqueue(struct rcu_data *rdp, struct rcu_head *head, rcu_callback_t func) +static void rcutree_enqueue(struct rcu_data *rdp, struct rcu_head *head) { rcu_segcblist_enqueue(&rdp->cblist, head); trace_rcu_callback(rcu_state.name, head, @@ -3036,9 +3036,9 @@ static void rcutree_enqueue(struct rcu_data *rdp, struct rcu_head *head, rcu_cal * Handle any core-RCU processing required by a call_rcu() invocation. */ static void call_rcu_core(struct rcu_data *rdp, struct rcu_head *head, - rcu_callback_t func, unsigned long flags) + unsigned long flags) { - rcutree_enqueue(rdp, head, func); + rcutree_enqueue(rdp, head); /* * If called from an extended quiescent state, invoke the RCU * core in order to force a re-evaluation of RCU's idleness. @@ -3179,9 +3179,9 @@ __call_rcu_common(struct rcu_head *head, rcu_callback_t func, bool lazy_in) check_cb_ovld(rdp); if (unlikely(rcu_rdp_is_offloaded(rdp))) - call_rcu_nocb(rdp, head, func, flags, lazy); + call_rcu_nocb(rdp, head, flags, lazy); else - call_rcu_core(rdp, head, func, flags); + call_rcu_core(rdp, head, flags); local_irq_restore(flags); } diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 587bd71396ea..ca88aaa29d61 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -507,7 +507,7 @@ static bool wake_nocb_gp(struct rcu_data *rdp); static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, unsigned long j, bool lazy); static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head, - rcu_callback_t func, unsigned long flags, bool lazy); + unsigned long flags, bool lazy); static void __maybe_unused __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_empty, unsigned long flags); static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp, int level); diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index 373b877cf171..95dbf33462a2 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -603,13 +603,13 @@ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, } static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head, - rcu_callback_t func, unsigned long flags, bool lazy) + unsigned long flags, bool lazy) { bool was_alldone; if (!rcu_nocb_try_bypass(rdp, head, &was_alldone, flags, lazy)) { /* Not enqueued on bypass but locked, do regular enqueue */ - rcutree_enqueue(rdp, head, func); + rcutree_enqueue(rdp, head); __call_rcu_nocb_wake(rdp, was_alldone, flags); /* unlocks */ } } @@ -1666,7 +1666,7 @@ static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, } static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head, - rcu_callback_t func, unsigned long flags, bool lazy) + unsigned long flags, bool lazy) { WARN_ON_ONCE(1); /* Should be dead code! */ } From 2aaee3fc84e10c5949e24a8778414ade2db44a55 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 20 Jul 2026 16:36:55 -0700 Subject: [PATCH 22/47] rcu: Mark accesses to rdp->rcu_cpu_has_work Although the rdp->rcu_cpu_has_work field is accessed only by the corresponding CPU, it can be accessed by both interrupt handlers via invoke_rcu_core_kthread() and at task level via rcu_cpu_kthread(). This means that we need this_cpu_read() rather than __this_cpu_read(), this_cpu_write() rather than __this_cpu_write(), and READ_ONCE() rather than plain C-language loads. The exception is the boot-time rcu_spawn_core_kthreads(), which cannot race with kthreads that have not yet been spawned. This commit therefore makes it so. KCSAN located this issue. Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 390dad82675d..c9780e7c0e2a 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2671,7 +2671,7 @@ static void rcu_do_batch(struct rcu_data *rdp) // reporting, so check time limits for them. if (rdp->rcu_cpu_kthread_status == RCU_KTHREAD_RUNNING && rcu_do_batch_check_time(count, tlimit, jlimit_check, jlimit)) { - rdp->rcu_cpu_has_work = 1; + WRITE_ONCE(rdp->rcu_cpu_has_work, 1); break; } } @@ -2931,7 +2931,7 @@ static void invoke_rcu_core_kthread(void) unsigned long flags; local_irq_save(flags); - __this_cpu_write(rcu_data.rcu_cpu_has_work, 1); + this_cpu_write(rcu_data.rcu_cpu_has_work, 1); t = __this_cpu_read(rcu_data.rcu_cpu_kthread_task); if (t != NULL && t != current) rcu_wake_cond(t, __this_cpu_read(rcu_data.rcu_cpu_kthread_status)); @@ -2958,7 +2958,7 @@ static void rcu_cpu_kthread_park(unsigned int cpu) static int rcu_cpu_kthread_should_run(unsigned int cpu) { - return __this_cpu_read(rcu_data.rcu_cpu_has_work); + return this_cpu_read(rcu_data.rcu_cpu_has_work); } /* @@ -2979,7 +2979,7 @@ static void rcu_cpu_kthread(unsigned int cpu) local_bh_disable(); *statusp = RCU_KTHREAD_RUNNING; local_irq_disable(); - work = *workp; + work = READ_ONCE(*workp); WRITE_ONCE(*workp, 0); local_irq_enable(); if (work) From 5976694882f6ebccd0cf7c81a6fdf5c1e08d9d67 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Mon, 13 Jul 2026 18:31:26 +0800 Subject: [PATCH 23/47] rcu-tasks: Update comments in call_rcu_tasks_generic() This commit updates the comment "We can't create the thread unless interrupts are enabled." from 'commit 4929c913bda5 ("rcu: Make call_rcu_tasks() tolerate first call with irqs disabled")' to be more clear and also to cover deferred wakeup and to take into account the fact that kthread creation has been move to core_initcall() time by 'commit c63eb17ff06d ("rcu: Create call_rcu_tasks() kthread at boot time")'. Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 79aa5d51388c..bc49698a3bcc 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -395,7 +395,11 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags); } rcu_read_unlock(); - /* We can't create the thread unless interrupts are enabled. */ + // We can't create the kthread with interrupts disabled because a + // scheduler spinlock might be held, so kthread creation is deferred + // until core_initcall() time. Similarly, wakeups are deferred using + // irq_work in order to avoid potential scheduler-lock-deadlock + // lockdep splats. if (needwake && READ_ONCE(rtp->kthread_ptr)) irq_work_queue(&rtpcp->rtp_irq_work); } From 7a455d3caef8c1cae44b0b0ea103d1d9fb197aa6 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Thu, 16 Jul 2026 17:57:54 +0800 Subject: [PATCH 24/47] rcu-tasks: Dump rtpcp->lazy_timer status in show_rcu_tasks_generic_gp_kthread() Add 'P' flag to the diagnostic line to indicate whether any per-cpu rtpcp's lazy_timer is pending. this helps diagnose stalls where rcu-task callbacks are queued but the kthread stay sleep because the lazy_timer has not yet fired and no grace period has started. The output is as follows: [ 31.319540][ T77] call_rcu_tasks() has failed boot-time tests. [ 31.320205][ T77] rcu_tasks: RTGS_WAIT_CBS(11) since 7518 g:4 i:0 kCuUP l:150000 Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index bc49698a3bcc..0306a0568f01 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -723,6 +723,7 @@ static void show_rcu_tasks_generic_gp_kthread(struct rcu_tasks *rtp, char *s) bool havecbs = false; bool haveurgent = false; bool haveurgentcbs = false; + bool havependtimer = false; for_each_possible_cpu(cpu) { struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); @@ -733,10 +734,12 @@ static void show_rcu_tasks_generic_gp_kthread(struct rcu_tasks *rtp, char *s) haveurgent = true; if (!data_race(rcu_segcblist_empty(&rtpcp->cblist)) && data_race(rtpcp->urgent_gp)) haveurgentcbs = true; - if (havecbs && haveurgent && haveurgentcbs) + if (data_race(timer_pending(&rtpcp->lazy_timer))) + havependtimer = true; + if (havecbs && haveurgent && haveurgentcbs && havependtimer) break; } - pr_info("%s: %s(%d) since %lu g:%lu i:%lu %c%c%c%c l:%lu %s\n", + pr_info("%s: %s(%d) since %lu g:%lu i:%lu %c%c%c%c%c l:%lu %s\n", rtp->kname, tasks_gp_state_getname(rtp), data_race(rtp->gp_state), jiffies - data_race(rtp->gp_jiffies), @@ -746,6 +749,7 @@ static void show_rcu_tasks_generic_gp_kthread(struct rcu_tasks *rtp, char *s) ".C"[havecbs], ".u"[haveurgent], ".U"[haveurgentcbs], + ".P"[havependtimer], rtp->lazy_jiffies, s); } From f99dc9e288c5a2e191cca8a9af59113a8bc2768a Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 20 Jul 2026 08:27:59 -0700 Subject: [PATCH 25/47] rcu-tasks: Fix IRQ read lock/unlock data race As noted by Marco Elver: rcu_read_lock_trace() .... t->trc_reader_scp = __srcu_read_lock_fast(&rcu_tasks_trace_srcu_struct); rcu_read_unlock_trace() < ... var decls only ... > scp = t->trc_reader_scp; This constitutes a data race between these two accesses to t->trc_reader_scp. If rcu_read_lock_trace() were to tear its store, this value would be corrupted. This commit therefore defers the rcu_read_lock_untrace() function's load from t->trc_reader_scp until after it has verified that this is the outermost rcu_read_unlock_trace(). With this change, the interrupt handler increments and decrements t->trc_reader_nesting and does not access t->trc_reader_scp, thus avoiding the data race. KCSAN located this issue. Signed-off-by: Paul E. McKenney --- include/linux/rcupdate_trace.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h index fd3ddeb6aa3b..70decf877348 100644 --- a/include/linux/rcupdate_trace.h +++ b/include/linux/rcupdate_trace.h @@ -126,11 +126,13 @@ static inline void rcu_read_unlock_trace(void) struct srcu_ctr __percpu *scp; struct task_struct *t = current; - scp = t->trc_reader_scp; - barrier(); // scp before nesting to protect against interrupt handler. n = READ_ONCE(t->trc_reader_nesting) - 1; - WRITE_ONCE(t->trc_reader_nesting, n); - if (!n) { + if (n) { + WRITE_ONCE(t->trc_reader_nesting, n); + } else { + scp = t->trc_reader_scp; // Compiler cannot hoist load due to data raciness. + barrier(); // scp before nesting to protect against interrupt handler. + WRITE_ONCE(t->trc_reader_nesting, n); if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB)) smp_mb(); // Placeholder for more selective ordering __srcu_read_unlock_fast(&rcu_tasks_trace_srcu_struct, scp); From 8fe479568cddbe7f2023069a539d4040e05f3f87 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 21 Jul 2026 21:03:06 -0700 Subject: [PATCH 26/47] rcu: Mark interrupts-enabled accesses to rdp->cpu_no_qs.b.norm The rdp->cpu_no_qs.b.norm field is accessed only by the current CPU, but can be accessed both at task level and from interrupt handlers. All accesses from interrupts-enabled code must therefore be marked. This commit therefore converts from __this_cpu_read() to this_cpu_read(), from __this_cpu_write() to this_cpu_write(), and plain C-language accesses to READ_ONCE() and WRITE_ONCE(), but only in interrupts-enabled code. KCSAN located this issue. Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 2 +- kernel/rcu/tree_plugin.h | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index c9780e7c0e2a..377153e0d934 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2540,7 +2540,7 @@ rcu_check_quiescent_state(struct rcu_data *rdp) * Was there a quiescent state since the beginning of the grace * period? If no, then exit and wait for the next call. */ - if (rdp->cpu_no_qs.b.norm) + if (READ_ONCE(rdp->cpu_no_qs.b.norm)) return; /* diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index c3db1ad4a280..353effac5a37 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -298,11 +298,11 @@ static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp) static void rcu_qs(void) { RCU_LOCKDEP_WARN(preemptible(), "rcu_qs() invoked with preemption enabled!!!\n"); - if (__this_cpu_read(rcu_data.cpu_no_qs.b.norm)) { + if (this_cpu_read(rcu_data.cpu_no_qs.b.norm)) { trace_rcu_grace_period(TPS("rcu_preempt"), __this_cpu_read(rcu_data.gp_seq), TPS("cpuqs")); - __this_cpu_write(rcu_data.cpu_no_qs.b.norm, false); + this_cpu_write(rcu_data.cpu_no_qs.b.norm, false); barrier(); /* Coordinate with rcu_flavor_sched_clock_irq(). */ WRITE_ONCE(current->rcu_read_unlock_special.b.need_qs, false); } @@ -952,7 +952,7 @@ void rcu_read_unlock_strict(void) * __rcu_read_unlock(). */ rdp = this_cpu_ptr(&rcu_data); - rdp->cpu_no_qs.b.norm = false; + WRITE_ONCE(rdp->cpu_no_qs.b.norm, false); rcu_report_qs_rdp(rdp); udelay(rcu_unlock_delay); } @@ -980,7 +980,7 @@ static void rcu_qs(void) return; trace_rcu_grace_period(TPS("rcu_sched"), __this_cpu_read(rcu_data.gp_seq), TPS("cpuqs")); - __this_cpu_write(rcu_data.cpu_no_qs.b.norm, false); + this_cpu_write(rcu_data.cpu_no_qs.b.norm, false); if (this_cpu_read(rcu_data.cpu_no_qs.b.exp)) rcu_report_exp_rdp(this_cpu_ptr(&rcu_data)); } From 7f28a4c8f3f723a3b99cc59425e906b429108c48 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Wed, 22 Jul 2026 18:23:47 +0800 Subject: [PATCH 27/47] rcu: Remove unused rdp parameter from rcu_check_gp_start_stall() The rcu_check_gp_start_stall() works entirely on rnp parameter, and never uses it's rdp parameter. this commit therefore drop it, updating both callers and the declaration. No functional change. Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 2 +- kernel/rcu/tree.h | 3 +-- kernel/rcu/tree_stall.h | 5 ++--- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 377153e0d934..5dbc7506c0c0 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2890,7 +2890,7 @@ static __latent_entropy void rcu_core(void) rcu_accelerate_cbs_unlocked(rnp, rdp); } - rcu_check_gp_start_stall(rnp, rdp, rcu_jiffies_till_stall_check()); + rcu_check_gp_start_stall(rnp, rcu_jiffies_till_stall_check()); /* If there are callbacks ready, invoke them. */ if (!rcu_rdp_is_offloaded(rdp) && rcu_segcblist_ready_cbs(&rdp->cblist) && diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index ca88aaa29d61..c3662c6ba395 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -544,8 +544,7 @@ static bool rcu_nohz_full_cpu(void); static void record_gp_stall_check_time(void); static void rcu_iw_handler(struct irq_work *iwp); static void check_cpu_stall(struct rcu_data *rdp); -static void rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp, - const unsigned long gpssdelay); +static void rcu_check_gp_start_stall(struct rcu_node *rnp, const unsigned long gpssdelay); /* Forward declarations for tree_exp.h. */ static void sync_rcu_do_polled_gp(struct work_struct *wp); diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index 45b9856ccd2b..02684dadb4ee 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -998,8 +998,7 @@ EXPORT_SYMBOL_GPL(show_rcu_gp_kthreads); * This function checks for grace-period requests that fail to motivate * RCU to come out of its idle mode. */ -static void rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp, - const unsigned long gpssdelay) +static void rcu_check_gp_start_stall(struct rcu_node *rnp, const unsigned long gpssdelay) { unsigned long flags; unsigned long j; @@ -1074,7 +1073,7 @@ void rcu_fwd_progress_check(unsigned long j) __func__, jiffies - data_race(READ_ONCE(rcu_state.gp_end))); preempt_disable(); rdp = this_cpu_ptr(&rcu_data); - rcu_check_gp_start_stall(rdp->mynode, rdp, j); + rcu_check_gp_start_stall(rdp->mynode, j); preempt_enable(); } for_each_possible_cpu(cpu) { From 27d73e81195b395270117ff77c47be2ed9b09b12 Mon Sep 17 00:00:00 2001 From: Itai Handler Date: Thu, 23 Jul 2026 10:41:38 +0300 Subject: [PATCH 28/47] rcu: Mark accesses to ->rcu_urgent_qs and ->rcu_need_heavy_qs rcu_all_qs() and rcu_note_context_switch() read/clear the per-CPU ->rcu_urgent_qs and ->rcu_need_heavy_qs flags with plain raw_cpu_read() and this_cpu_write(), while the RCU core clears them with WRITE_ONCE() in rcu_disable_urgency_upon_qs(). KCSAN flags the resulting same-CPU race: BUG: KCSAN: data-race in rcu_all_qs / rcu_disable_urgency_upon_qs It is benign -- the flags are advisory and rcu_all_qs() re-reads ->rcu_urgent_qs with smp_load_acquire() before acting on it -- but these are the last unmarked accesses to the two flags; every other access already uses READ_ONCE()/WRITE_ONCE()/smp_*. Mark them to match. No functional change. Reproduced on a PREEMPT_NONE, CONFIG_KCSAN_INTERRUPT_WATCHER=y kernel with a pthreads program whose threads (two per CPU) loop reading a large file: for (;;) { int fd = open("/proc/kallsyms", O_RDONLY); while (read(fd, buf, sizeof(buf)) > 0) ; close(fd); } The read()s drive cond_resched() -> rcu_all_qs() while the busy CPUs keep the grace period urgent, so the RCU core clears the flags concurrently. Fixes: 2dba13f0b6c2 ("rcu: Switch urgent quiescent-state requests to rcu_data structure") Signed-off-by: Itai Handler Signed-off-by: Paul E. McKenney --- kernel/rcu/tree_plugin.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 353effac5a37..844a9ced67cc 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -996,7 +996,7 @@ void rcu_all_qs(void) { unsigned long flags; - if (!raw_cpu_read(rcu_data.rcu_urgent_qs)) + if (!READ_ONCE(*raw_cpu_ptr(&rcu_data.rcu_urgent_qs))) return; preempt_disable(); // For CONFIG_PREEMPT_COUNT=y kernels /* Load rcu_urgent_qs before other flags. */ @@ -1004,8 +1004,8 @@ void rcu_all_qs(void) preempt_enable(); return; } - this_cpu_write(rcu_data.rcu_urgent_qs, false); - if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs))) { + WRITE_ONCE(*this_cpu_ptr(&rcu_data.rcu_urgent_qs), false); + if (unlikely(READ_ONCE(*this_cpu_ptr(&rcu_data.rcu_need_heavy_qs)))) { local_irq_save(flags); rcu_momentary_eqs(); local_irq_restore(flags); @@ -1025,8 +1025,8 @@ void rcu_note_context_switch(bool preempt) /* Load rcu_urgent_qs before other flags. */ if (!smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) goto out; - this_cpu_write(rcu_data.rcu_urgent_qs, false); - if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs))) + WRITE_ONCE(*this_cpu_ptr(&rcu_data.rcu_urgent_qs), false); + if (unlikely(READ_ONCE(*this_cpu_ptr(&rcu_data.rcu_need_heavy_qs)))) rcu_momentary_eqs(); out: rcu_tasks_qs(current, preempt); From df0ecbc497de851891847260f45805ca6ccb393f Mon Sep 17 00:00:00 2001 From: Zqiang Date: Thu, 23 Jul 2026 18:04:30 +0800 Subject: [PATCH 29/47] rcu: Reduce stack usage in show_rcu_gp_kthreads() When CONFIG_KASAN=y and CONFIG_KASAN_STACK=y builds, the show_rcu_gp_kthreads() exceeds the 1024-byte frame-size limit: make kernel/rcu/tree.o KCFLAGS="-fstack-usage" DESCEND objtool DESCEND bpf/resolve_btfids INSTALL libsubcmd_headers CC kernel/rcu/tree.o In file included from kernel/rcu/tree.c:4998: kernel/rcu/tree_stall.h: In function 'show_rcu_gp_kthreads': kernel/rcu/tree_stall.h:994:1: warning: the frame size of 1656 bytes is larger than 1024 bytes [-Wframe-larger-than=] grep show_rcu kernel/rcu/tree.su tree_nocb.h:1622:13:show_rcu_nocb_state 896 dynamic,bounded tree_stall.h:933:6:show_rcu_gp_kthreads 1784 dynamic,bounded tree_stall.h:1102:13:sysrq_show_rcu 16 static Wrap the pr_info() into two noinline_for_stack helpers function: show_rcu_state() print rcu_state status, and show_rcu_node() print single rcu_node status. After apply this change: grep show_rcu kernel/rcu/tree.su tree_stall.h:955:22:show_rcu_node 696 dynamic,bounded tree_stall.h:930:22:show_rcu_state 872 dynamic,bounded tree_nocb.h:1622:13:show_rcu_nocb_state 896 dynamic,bounded tree_stall.h:972:6:show_rcu_gp_kthreads 544 static tree_stall.h:1113:13:sysrq_show_rcu 16 static Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/tree_stall.h | 47 +++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index 02684dadb4ee..20634edfb44d 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -927,20 +927,13 @@ bool rcu_check_boost_fail(unsigned long gp_state, int *cpup) } EXPORT_SYMBOL_GPL(rcu_check_boost_fail); -/* - * Show the state of the grace-period kthreads. - */ -void show_rcu_gp_kthreads(void) +static noinline_for_stack void show_rcu_state(void) { - unsigned long cbs = 0; - int cpu; unsigned long j; unsigned long ja; unsigned long jr; unsigned long js; unsigned long jw; - struct rcu_data *rdp; - struct rcu_node *rnp; struct task_struct *t = READ_ONCE(rcu_state.gp_kthread); j = jiffies; @@ -957,21 +950,39 @@ void show_rcu_gp_kthreads(void) (long)data_race(READ_ONCE(rcu_get_root()->gp_seq_needed)), data_race(READ_ONCE(rcu_state.gp_max)), data_race(READ_ONCE(rcu_state.gp_flags))); +} + +static noinline_for_stack void show_rcu_node(struct rcu_node *rnp) +{ + pr_info("\trcu_node %d:%d ->gp_seq %ld ->gp_seq_needed %ld ->qsmask %#lx %c%c%c%c ->n_boosts %ld\n", + rnp->grplo, rnp->grphi, + (long)data_race(READ_ONCE(rnp->gp_seq)), + (long)data_race(READ_ONCE(rnp->gp_seq_needed)), + data_race(READ_ONCE(rnp->qsmask)), + ".b"[!!data_race(READ_ONCE(rnp->boost_kthread_task))], + ".B"[!!data_race(READ_ONCE(rnp->boost_tasks))], + ".E"[!!data_race(READ_ONCE(rnp->exp_tasks))], + ".G"[!!data_race(READ_ONCE(rnp->gp_tasks))], + data_race(READ_ONCE(rnp->n_boosts))); +} + +/* + * Show the state of the grace-period kthreads. + */ +void show_rcu_gp_kthreads(void) +{ + unsigned long cbs = 0; + int cpu; + struct rcu_data *rdp; + struct rcu_node *rnp; + + show_rcu_state(); rcu_for_each_node_breadth_first(rnp) { if (ULONG_CMP_GE(READ_ONCE(rcu_state.gp_seq), READ_ONCE(rnp->gp_seq_needed)) && !data_race(READ_ONCE(rnp->qsmask)) && !data_race(READ_ONCE(rnp->boost_tasks)) && !data_race(READ_ONCE(rnp->exp_tasks)) && !data_race(READ_ONCE(rnp->gp_tasks))) continue; - pr_info("\trcu_node %d:%d ->gp_seq %ld ->gp_seq_needed %ld ->qsmask %#lx %c%c%c%c ->n_boosts %ld\n", - rnp->grplo, rnp->grphi, - (long)data_race(READ_ONCE(rnp->gp_seq)), - (long)data_race(READ_ONCE(rnp->gp_seq_needed)), - data_race(READ_ONCE(rnp->qsmask)), - ".b"[!!data_race(READ_ONCE(rnp->boost_kthread_task))], - ".B"[!!data_race(READ_ONCE(rnp->boost_tasks))], - ".E"[!!data_race(READ_ONCE(rnp->exp_tasks))], - ".G"[!!data_race(READ_ONCE(rnp->gp_tasks))], - data_race(READ_ONCE(rnp->n_boosts))); + show_rcu_node(rnp); if (!rcu_is_leaf_node(rnp)) continue; for_each_leaf_node_possible_cpu(rnp, cpu) { From 76892716e187e1bb4be30c8c980204b501c28555 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 24 Jul 2026 14:44:15 -0700 Subject: [PATCH 30/47] rcu: Mark interrupts-enabled accesses to rdp->cpu_no_qs.s The rdp->cpu_no_qs.s field is accessed only by the current CPU, but can be accessed both at task level and from interrupt handlers. All accesses from interrupts-enabled code must therefore be marked. This commit therefore converts from __this_cpu_read() to this_cpu_read(), but only in interrupts-enabled code, that is, the rcu_qs() function. KCSAN located this issue. Signed-off-by: Paul E. McKenney --- kernel/rcu/tree_plugin.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 844a9ced67cc..6904a4f826d1 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -976,7 +976,7 @@ static void __init rcu_bootup_announce(void) static void rcu_qs(void) { RCU_LOCKDEP_WARN(preemptible(), "rcu_qs() invoked with preemption enabled!!!"); - if (!__this_cpu_read(rcu_data.cpu_no_qs.s)) + if (!this_cpu_read(rcu_data.cpu_no_qs.s)) return; trace_rcu_grace_period(TPS("rcu_sched"), __this_cpu_read(rcu_data.gp_seq), TPS("cpuqs")); From ed61912b40dda92ef24014fb0daca7de852f9407 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Mon, 20 Jul 2026 17:49:58 +0800 Subject: [PATCH 31/47] rcu-tasks: Rename tasks_rcu_exit_srcu_stall_timer to tasks_rcu_exit_stall_timer This 'commit 1612160b9127 ("rcu-tasks: Eliminate deadlocks involving do_exit() and RCU tasks")' has reimplemented the rcu_tasks_postscan(), and the synchronize_srcu() has been removed, this commit therefore rename tasks_rcu_exit_srcu_stall_timer to tasks_rcu_exit_stall_timer. Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 0306a0568f01..fce57d90db1a 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -155,8 +155,8 @@ static struct rcu_tasks rt_name = \ #ifdef CONFIG_TASKS_RCU /* Report delay of scan exiting tasklist in rcu_tasks_postscan(). */ -static void tasks_rcu_exit_srcu_stall(struct timer_list *unused); -static DEFINE_TIMER(tasks_rcu_exit_srcu_stall_timer, tasks_rcu_exit_srcu_stall); +static void tasks_rcu_exit_stall(struct timer_list *unused); +static DEFINE_TIMER(tasks_rcu_exit_stall_timer, tasks_rcu_exit_stall); #endif /* Control stall timeouts. Disable with <= 0, otherwise jiffies till stall. */ @@ -1032,8 +1032,8 @@ static void rcu_tasks_postscan(struct list_head *hop) int rtsi = READ_ONCE(rcu_task_stall_info); if (!IS_ENABLED(CONFIG_TINY_RCU)) { - tasks_rcu_exit_srcu_stall_timer.expires = jiffies + rtsi; - add_timer(&tasks_rcu_exit_srcu_stall_timer); + tasks_rcu_exit_stall_timer.expires = jiffies + rtsi; + add_timer(&tasks_rcu_exit_stall_timer); } /* @@ -1086,7 +1086,7 @@ static void rcu_tasks_postscan(struct list_head *hop) } if (!IS_ENABLED(CONFIG_TINY_RCU)) - timer_delete_sync(&tasks_rcu_exit_srcu_stall_timer); + timer_delete_sync(&tasks_rcu_exit_stall_timer); } /* See if tasks are still holding out, complain if so. */ @@ -1158,7 +1158,7 @@ static void rcu_tasks_postgp(struct rcu_tasks *rtp) synchronize_rcu(); } -static void tasks_rcu_exit_srcu_stall(struct timer_list *unused) +static void tasks_rcu_exit_stall(struct timer_list *unused) { #ifndef CONFIG_TINY_RCU int rtsi; @@ -1168,8 +1168,8 @@ static void tasks_rcu_exit_srcu_stall(struct timer_list *unused) __func__, rcu_tasks.kname, rcu_tasks.tasks_gp_seq, tasks_gp_state_getname(&rcu_tasks), jiffies - rcu_tasks.gp_jiffies); pr_info("Please check any exiting tasks stuck between calls to exit_tasks_rcu_start() and exit_tasks_rcu_finish()\n"); - tasks_rcu_exit_srcu_stall_timer.expires = jiffies + rtsi; - add_timer(&tasks_rcu_exit_srcu_stall_timer); + tasks_rcu_exit_stall_timer.expires = jiffies + rtsi; + add_timer(&tasks_rcu_exit_stall_timer); #endif // #ifndef CONFIG_TINY_RCU } From 885d8314e5706ce7d176f2029454246c2d940bb3 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Tue, 21 Jul 2026 17:48:07 +0800 Subject: [PATCH 32/47] rcu-tasks: Fix some comments for call_rcu_tasks() and call_rcu_tasks_rude() The comments for call_rcu_tasks() and call_rcu_tasks_rude() referred to "RCU read-side critical sections", which is ambiguous. specify "rcu-tasks" and "rcu-tasks rude" respectively to match the actual flavor being described. Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index fce57d90db1a..851ac225770c 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1179,7 +1179,7 @@ static void tasks_rcu_exit_stall(struct timer_list *unused) * @func: actual callback function to be invoked after the grace period * * The callback function will be invoked some time after a full grace - * period elapses, in other words after all currently executing RCU + * period elapses, in other words after all currently executing rcu-tasks * read-side critical sections have completed. call_rcu_tasks() assumes * that the read-side critical sections end at a voluntary context * switch (not a preemption!), cond_resched_tasks_rcu_qs(), entry into idle, @@ -1365,8 +1365,8 @@ DEFINE_RCU_TASKS(rcu_tasks_rude, rcu_tasks_rude_wait_gp, call_rcu_tasks_rude, * @func: actual callback function to be invoked after the grace period * * The callback function will be invoked some time after a full grace - * period elapses, in other words after all currently executing RCU - * read-side critical sections have completed. call_rcu_tasks_rude() + * period elapses, in other words after all currently executing rude + * rcu-tasks read-side critical sections have completed. call_rcu_tasks_rude() * assumes that the read-side critical sections end at context switch, * cond_resched_tasks_rcu_qs(), or transition to usermode execution (as * usermode execution is schedulable). As such, there are no read-side @@ -1390,7 +1390,7 @@ static void call_rcu_tasks_rude(struct rcu_head *rhp, rcu_callback_t func) * * Control will return to the caller some time after a rude rcu-tasks * grace period has elapsed, in other words after all currently - * executing rcu-tasks read-side critical sections have elapsed. These + * executing rude rcu-tasks read-side critical sections have elapsed. These * read-side critical sections are delimited by calls to schedule(), * cond_resched_tasks_rcu_qs(), userspace execution (which is a schedulable * context), and (in theory, anyway) cond_resched(). From 0f5fe218d9d4e23d47a49873c9bfacd717b9c145 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 27 Jul 2026 15:11:04 -0700 Subject: [PATCH 33/47] rcu-tasks: Convert cond_resched_tasks_rcu_qs() to static inline In order to make "cc -E" output less annoying, this commit converts cond_resched_tasks_rcu_qs() to static inline. You know, the READ_ONCE() and WRITE_ONCE() macros used to be *so* simple. ;-) Reported-by: Andrew Morton Signed-off-by: Paul E. McKenney --- include/linux/rcupdate.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 5e95acc33989..5e8770fd7b72 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -208,15 +208,15 @@ static inline void exit_tasks_rcu_finish(void) { } /** * cond_resched_tasks_rcu_qs - Report potential quiescent states to RCU * - * This macro resembles cond_resched(), except that it is defined to + * This function resembles cond_resched(), except that it is defined to * report potential quiescent states to RCU-tasks even if the cond_resched() * machinery were to be shut off, as some advocate for PREEMPTION kernels. */ -#define cond_resched_tasks_rcu_qs() \ -do { \ - rcu_tasks_qs(current, false); \ - cond_resched(); \ -} while (0) +static inline void cond_resched_tasks_rcu_qs(void) +{ + rcu_tasks_qs(current, false); + cond_resched(); +} /** * rcu_softirq_qs_periodic - Report RCU and RCU-Tasks quiescent states From 78a38cbf6f20bc8247e93d1149f97c12dba9fbfb Mon Sep 17 00:00:00 2001 From: Zqiang Date: Thu, 9 Jul 2026 18:06:02 +0800 Subject: [PATCH 34/47] srcu: Queue sdp->work when the delay timer is successfully deleted In the cleanup_srcu_struct() function, when iterating over per-cpu's srcu_data, timer_delete_sync(&sdp->delay_work) is called to cancel the delayed work before doing flush_work(&sdp->work). However, suppose that timer_delete_sync() returns 1, which means that it successfully deleted an pending timer before it had a chance to fire. But this also means that the sdp->work will not be queued, so that the subsequent flush_work(&sdp->work) will returns immediately without waiting for anything. Taken together, all of this means that any recently queued SRCU callbacks to not be invoked, which can result in memory leaks, hangs, or worse. Fix this by checking the return value of timer_delete_sync(), if it returns 1, explicitly queue sdp->work so that the callbacks will be invoked and the following flush_work() will correctly wait for all of those callbacks to finish executing. [ Zqiang: Apply feedback from Breno Leitao and kernel test robot. ] Signed-off-by: Zqiang Tested-by: kernel test robot Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- kernel/rcu/srcutree.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 4a00e90e17fc..02a101824f12 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -701,7 +701,12 @@ void cleanup_srcu_struct(struct srcu_struct *ssp) for_each_possible_cpu(cpu) { struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); - timer_delete_sync(&sdp->delay_work); + // Call srcu_barrier() before this cleanup_srcu_struct() + // to avoid triggering this WARN_ON(). + if (WARN_ON(timer_delete_sync(&sdp->delay_work) && + rcu_segcblist_n_cbs(&sdp->srcu_cblist)) && + rcu_cpu_beenfullyonline(sdp->cpu)) + queue_work_on(sdp->cpu, rcu_gp_wq, &sdp->work); flush_work(&sdp->work); if (WARN_ON(rcu_segcblist_n_cbs(&sdp->srcu_cblist))) return; /* Forgot srcu_barrier(), so just leak it! */ From 50df5afa6d410ddfa32f58cdb2976d16fc3785b3 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Sat, 25 Apr 2026 10:50:57 -0700 Subject: [PATCH 35/47] rcutorture: Check for immediate deboosting at reader end This commit adds a check for failure to have fully deboosted a multi-segmented RCU reader at the end of the full read-side critical section. This check only happens for fully task-level readers, because a a handler might have interrupted an already-boosted task-level RCU reader, and a reader in that handler could then cause false positives. The first failed check (due to an RCU reader that was not immediately deboosted) causes a splat, but only when the disabled-by-default deboost_timeliness_check module parameter is enabled. Regardless of the value of this parameter, it produces a list of the segments making up that RCU reader following a "Slow-deboost rcutorture reader segments" heading. Subsequent failures fail silently, all in the name of keeping console output down to a dull roar. Although most uses of RCU priority boosting serve as debugging aids, this might change, and in fact might already have changed. And allowing (for example) RCU priority boosting to persist until the next scheduler tick could cause an aggressively real-time system to miss sub-millisecond deadlines. So we do need to find this sort of problem during testing, and preferably not in the field. The name and type of the newly added rcu_torture_ops function pointer (named "->is_task_rcu_boosted()") may need to change should other end-of-reader checks be needed. But let's start simple. Oh, and Claude figured out that rcu_is_task_rcu_boosted() could be lockless. Perhaps there is hope for AI yet! ;-) [ paulmck: Apply Akira Yokosawa feedback. ] Signed-off-by: Paul E. McKenney --- kernel/rcu/rcu.h | 7 +++++++ kernel/rcu/rcutorture.c | 24 ++++++++++++++++++++++++ kernel/rcu/tree_plugin.h | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+) diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index fa6d30ce73d1..14faa11ef23c 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h @@ -695,4 +695,11 @@ static inline int rcu_stall_notifier_call_chain(unsigned long val, void *v) { re void synchronize_rcu_trivial_preempt(void); #endif // #ifdef CONFIG_TRIVIAL_PREEMPT_RCU +#if defined(CONFIG_RCU_TORTURE_TEST) && defined(CONFIG_RCU_BOOST) +bool rcu_is_task_rcu_boosted(void); +#else // #if defined(CONFIG_RCU_TORTURE_TEST) && defined(CONFIG_RCU_BOOST) +static inline bool rcu_is_task_rcu_boosted(void) { return false; } +#endif // #else // #if defined(CONFIG_RCU_TORTURE_TEST) && defined(CONFIG_RCU_BOOST) + + #endif /* __LINUX_RCU_H */ diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index ffeca1b7e170..1a033ae1f1eb 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -80,6 +80,7 @@ MODULE_AUTHOR("Paul E. McKenney and Josh Triplett is_task_rcu_boosted && cur_ops->is_task_rcu_boosted() && + !in_serving_softirq() && !in_hardirq() && !in_nmi() && + READ_ONCE(firsttime) && xchg(&firsttime, 0)) { + WARN_ON_ONCE(deboost_timeliness_check); + nsegs = rtors.rtrsp - rtors.rtseg; + nsegs = clamp_val(nsegs, 0, RCUTORTURE_RDR_MAX_SEGS); + pr_alert("Slow-deboost rcutorture reader segments:\n"); + rcu_torture_dump_read_segs(rtors.rtseg, nsegs); + } return true; } diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 95ad967adcf3..d950722eb8f9 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -1320,6 +1320,41 @@ static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp) wake_up_process(t); /* get to TASK_INTERRUPTIBLE quickly. */ } +#ifdef CONFIG_RCU_TORTURE_TEST + +/* + * Is the current task RCU priority boosted? This is used by + * rcutorture to check that tasks are always deboosted once then exit + * an RCU read-side critical section, no matter how many overlapping + * segments of rcu_read_lock(), preempt_disable(), local_bh_disable(), + * or local_irq_disable() made up that reader. + * + * The lockless accesses in rt_mutex_owner(&rnp->boost_mtx.rtmutex) + * are safe because tasks release ->boost_mtx when they own it, they + * cannot be boosted unless current->rcu_blocked_node is non-NULL, + * current->rcu_blocked_node is modified only by the current task, + * rt_mutex_owner() uses READ_ONCE() on the ->owner field, and the owner + * switching among other tasks cannot force an equality comparison. + */ +bool rcu_is_task_rcu_boosted(void) +{ + bool ret; + struct rcu_node *rnp; + struct task_struct *t = current; + + preempt_disable(); // Stabilize ->rcu_blocked_node + rnp = t->rcu_blocked_node; + if (!rnp) + ret = false; + else + ret = (rt_mutex_owner(&rnp->boost_mtx.rtmutex) == t); + preempt_enable(); + return ret; +} +EXPORT_SYMBOL_GPL(rcu_is_task_rcu_boosted); + +#endif // #ifdef CONFIG_RCU_TORTURE_TEST + #else /* #ifdef CONFIG_RCU_BOOST */ static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags) From 5ac693414b51acb8c4b92813ca3ab4fc6686ddf9 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 7 May 2026 15:03:21 -0700 Subject: [PATCH 36/47] rcutorture: Test RCU readers from hardware interrupt handlers Although rcutorture has long had the irqreader module parameter, this parameter results only in RCU readers in softirq handlers, specifically, timers. This commit therefore uses smp_call_function_single() to test RCU readers in real hardware interrupt handlers, thus providing the full effect from the irqreader module parameter. However, consistency/debug checks must account for the possibility that the smp_call_function_single() handler function is directly invoked from the idle loop, in which case, for example, in_hardirq() will return false. This commit uses a per-CPU variable to record being in the rcu_torture_irq() smp_call_function_single() handler function. Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 77 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 71 insertions(+), 6 deletions(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 1a033ae1f1eb..57cae3fe23fd 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -213,6 +213,7 @@ static long n_rcu_torture_boost_ktrerror; static long n_rcu_torture_boost_failure; static long n_rcu_torture_boosts; static atomic_long_t n_rcu_torture_timers; +static atomic_long_t n_rcu_torture_irqs; static long n_barrier_attempts; static long n_barrier_successes; /* did rcu_barrier test succeed? */ static unsigned long n_read_exits; @@ -2127,6 +2128,8 @@ static void rcu_torture_reader_do_mbchk(long myid, struct rcu_torture *rtp, smp_store_release(&rtrcp_assigner->rtc_chkrdr, -1); // Assigner can again assign. } +static DEFINE_PER_CPU(bool, torture_in_scf_handler); + // Verify the specified RCUTORTURE_RDR* state. #define ROEC_ARGS "%s %s: Current %#x To add %#x To remove %#x preempt_count() %#x\n", __func__, s, curstate, new, old, preempt_count() static void rcutorture_one_extend_check(char *s, int curstate, int new, int old) @@ -2136,7 +2139,7 @@ static void rcutorture_one_extend_check(char *s, int curstate, int new, int old) if (!IS_ENABLED(CONFIG_RCU_TORTURE_TEST_CHK_RDR_STATE) || in_nmi()) return; - WARN_ONCE(!(curstate & RCUTORTURE_RDR_IRQ) && irqs_disabled() && !in_hardirq(), ROEC_ARGS); + WARN_ONCE(!(curstate & RCUTORTURE_RDR_IRQ) && irqs_disabled() && !in_hardirq() && !this_cpu_read(torture_in_scf_handler), ROEC_ARGS); WARN_ONCE((curstate & RCUTORTURE_RDR_IRQ) && !irqs_disabled(), ROEC_ARGS); // If CONFIG_PREEMPT_COUNT=n, further checks are unreliable. @@ -2153,7 +2156,7 @@ static void rcutorture_one_extend_check(char *s, int curstate, int new, int old) // Interrupt handlers have all sorts of stuff disabled, so ignore // unintended disabling. - if (in_serving_softirq() || in_hardirq()) + if (in_serving_softirq() || in_hardirq() || this_cpu_read(torture_in_scf_handler)) return; WARN_ONCE(cur_ops->extendables && @@ -2345,12 +2348,19 @@ rcutorture_extend_mask(int oldmask, struct torture_random_state *trsp) mask |= RCUTORTURE_RDR_RCU_1; } + /* + * Don't mess with interrupt masking in interrupt handlers. + */ + if (in_hardirq() || this_cpu_read(torture_in_scf_handler)) + mask &= ~(preempts_irq | bhs); + /* * Can't enable bh w/irq disabled. */ if (mask & RCUTORTURE_RDR_IRQ) mask |= oldmask & bhs; + /* * Ideally these sequences would be detected in debug builds * (regardless of RT), but until then don't stop testing @@ -2605,6 +2615,7 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) return false; rtors.rtrsp = rcutorture_loop_extend(&rtors.readstate, trsp, rtors.rtrsp); rcu_torture_one_read_end(&rtors, trsp); + // This splat will happen on systems built with CONFIG_IRQ_WORK=n // and on systems where arch_irq_work_has_interrupt() returns false. // It might also happen on systems using a short-duration clock @@ -2643,7 +2654,7 @@ static void rcu_torture_timer(struct timer_list *unused) atomic_long_inc(&n_rcu_torture_timers); (void)rcu_torture_one_read(this_cpu_ptr(&rcu_torture_timer_rand), -1); - /* Test call_rcu() invocation from interrupt handler. */ + /* Test call_rcu() invocation from softirq handler. */ if (cur_ops->call) { struct rcu_head *rhp = kmalloc_obj(*rhp, GFP_NOWAIT); @@ -2652,6 +2663,41 @@ static void rcu_torture_timer(struct timer_list *unused) } } +static DEFINE_TORTURE_RANDOM_PERCPU(rcu_torture_irq_rand); + +/* + * RCU torture reader from timer handler. Dereferences rcu_torture_current, + * incrementing the corresponding element of the pipeline array. The + * counter in the element should never be greater than 1, otherwise, the + * RCU implementation is broken. + * + * Note that on some systems, "interrupts" from idle are direct calls + * rather than interrupts. The torture_in_scf_handler per-CPU variable + * accounts for this case. + */ +static void rcu_torture_irq(void *unused) +{ + WARN_ON_ONCE(in_nmi()); + lockdep_assert_irqs_disabled(); + atomic_long_inc(&n_rcu_torture_irqs); + this_cpu_write(torture_in_scf_handler, true); + (void)rcu_torture_one_read(this_cpu_ptr(&rcu_torture_irq_rand), -1); + this_cpu_write(torture_in_scf_handler, false); + + // Test call_rcu() invocation from interrupt handler. Interrupts + // will always be disabled here, even in CONFIG_PREEMPT_RT=y kernels. + // The "right" thing to do would be to create a special-purpose + // lockless or raw-spinlock-protected allocator, but in the meantime, + // skip testing call_rcu() from interrupt handlers in kernels built + // with either CONFIG_PREEMPT_RT=y or CONFIG_PROVE_LOCKING=y. + if (cur_ops->call && !IS_ENABLED(CONFIG_PROVE_LOCKING) && !IS_ENABLED(CONFIG_PREEMPT_RT)) { + struct rcu_head *rhp = kmalloc_obj(*rhp, GFP_NOWAIT); + + if (rhp) + cur_ops->call(rhp, rcu_torture_timer_cb); + } +} + /* * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, * incrementing the corresponding element of the pipeline array. The @@ -2661,6 +2707,7 @@ static void rcu_torture_timer(struct timer_list *unused) static int rcu_torture_reader(void *arg) { + unsigned long lastscf = jiffies; unsigned long lastsleep = jiffies; long myid = (long)arg; int mynumonline = myid; @@ -2674,8 +2721,25 @@ rcu_torture_reader(void *arg) tick_dep_set_task(current, TICK_DEP_BIT_RCU); // CPU bound, so need tick. do { if (irqreader && cur_ops->irq_capable) { - if (!timer_pending(&t)) + if (!timer_pending(&t)) { + int cpu; + mod_timer(&t, jiffies + 1); + preempt_disable(); + cpu = torture_random(&rand) % nr_cpu_ids; + if (!cpu_online(cpu)) { + cpu = cpumask_next(cpu, cpu_online_mask); + if (cpu >= nr_cpu_ids) + cpu = cpumask_next(-1, cpu_online_mask); + } + // An smp_call_function_single() to self is not an interrupt! + if (cpu != smp_processor_id() && + time_after(jiffies, lastscf + HZ * nrealreaders / 50)) { + smp_call_function_single(cpu, rcu_torture_irq, NULL, 0); + lastscf = jiffies; + } + preempt_enable(); + } } if (!rcu_torture_one_read(&rand, myid) && !torture_must_stop()) schedule_timeout_interruptible(HZ); @@ -2951,10 +3015,11 @@ rcu_torture_stats_print(void) atomic_read(&n_rcu_torture_mbchk_fail), atomic_read(&n_rcu_torture_mbchk_tries), n_rcu_torture_barrier_error, n_rcu_torture_boost_ktrerror); - pr_cont("rtbf: %ld rtb: %ld nt: %ld ", + pr_cont("rtbf: %ld rtb: %ld nt: %ld ni: %ld ", n_rcu_torture_boost_failure, n_rcu_torture_boosts, - atomic_long_read(&n_rcu_torture_timers)); + atomic_long_read(&n_rcu_torture_timers), + atomic_long_read(&n_rcu_torture_irqs)); if (updownreaders) pr_cont("ndowns: %lu nups: %lu nhrt: %lu nmigrates: %lu ", ndowns, nups, nunexpired, nmigrates); torture_onoff_stats(); From f424566105c946ef42f2643230362598c06a29f1 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 4 Jun 2026 20:42:58 -0700 Subject: [PATCH 37/47] rcutorture: Use cpumask_next_wrap() in rcu_torture_preempt() The rcu_torture_preempt() function uses cpumask_next(), and if that returns an out-of-bounds result, re-invokes cpumask_next() on -1. Which is exactly what cpumask_next_wrap() does. This commit therefore saves a couple of lines by instead using cpumask_next_wrap(). This was reported by metacode when asked to look for opportunities to use cpumask_next_wrap() in kernel/rcu. Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 57cae3fe23fd..afd5045263be 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -4183,9 +4183,7 @@ static int rcu_torture_preempt(void *unused) // Wait for preempt_interval ms with up to 100us fuzz. torture_hrtimeout_ms(preempt_interval, 100, &rand); // Select online CPU. - cpu = cpumask_next(cpu, cpu_online_mask); - if (cpu >= nr_cpu_ids) - cpu = cpumask_next(-1, cpu_online_mask); + cpu = cpumask_next_wrap(cpu, cpu_online_mask); WARN_ON_ONCE(cpu >= nr_cpu_ids); // Move to that CPU, if can't do so, retry later. if (torture_sched_setaffinity(current->pid, cpumask_of(cpu), false)) From e7b6d505c7080d6d40e5dbbeb01b612de9f08a78 Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Fri, 12 Jun 2026 10:34:13 +0800 Subject: [PATCH 38/47] rcutorture: Use task_state_to_char() for task-state reporting Use the kernel's standard symbolic task-state representation instead of printing raw hexadecimal task-state values. Suggested-by: Zqiang Co-developed-by: Wang Lian Signed-off-by: Wang Lian Signed-off-by: Kunwu Chan Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index afd5045263be..c9fb375701f8 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -3076,10 +3076,10 @@ rcu_torture_stats_print(void) if (cur_ops->get_gp_data) cur_ops->get_gp_data(&flags, &gp_seq); wtp = READ_ONCE(writer_task); - pr_alert("??? Writer stall state %s(%d) g%lu f%#x ->state %#x cpu %d\n", + pr_alert("??? Writer stall state %s(%d) g%lu f%#x ->state %c cpu %d\n", rcu_torture_writer_state_getname(), rcu_torture_writer_state, gp_seq, flags, - wtp == NULL ? ~0U : wtp->__state, + wtp == NULL ? '?' : task_state_to_char(wtp), wtp == NULL ? -1 : (int)task_cpu(wtp)); if (!splatted && wtp) { sched_show_task(wtp); From 75a88fe00b67b2a065461940b5392dc5487acfff Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 25 Jun 2026 10:57:06 -0700 Subject: [PATCH 39/47] rcutorture: Add nwriters module parameter Believe it or not, there are people who would like to run rcutorture without actually torturing RCU. For example, some people would like to induce various types of stall warnings without placing any unnecessary additional overhead on their systems running in production. And rcutorture provides the stall_cpu, stall_cpu_holdoff, stall_no_softlockup, stall_cpu_irqsoff, stall_cpu_block, and stall_cpu_repeat module parameters in order to allow the user to force numerous types of stalls. In addition, rcutorture provides a great number of other module parameters to allow the user to reduce other overhead. But unfortunately, there is no way to turn of the rcu_torture_writer() portion of this torture test, which on my x86 laptop consumes somewhere between 40% and 45% of a CPU. Although this is quite lightweight for a torture test, it is not welcome on systems running production workloads. This commit therefore adds an nwriters module parameter that defaults to 1 but can be set to 0 in order to disable the rcu_torture_writer() portion of the torture test, but that cannot be set to any other value (that is what the fakewriters module parameter is for!). This reduces the overhead to well under 1% of a CPU, which is much more likely to be compatible with production workloads. Reported-by: Breno Leitao Reported-by: Puranjay Mohan Reported-by: Usama Arif Signed-off-by: Paul E. McKenney --- Documentation/admin-guide/kernel-parameters.txt | 5 +++++ kernel/rcu/rcutorture.c | 14 ++++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index b5493a7f8f22..b942149720c9 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -6117,6 +6117,11 @@ Kernel parameters the number of CPUs. For example, -2 selects N (the number of CPUs), -3 selects N+1, and so on. + rcutorture.nwriters= [KNL] + Set number of RCU writers, which must be either + zero or one. For additional writers, use instead + the rcutorture.nfakewriters parameter. + rcutorture.object_debug= [KNL] Enable debug-object double-call_rcu() testing. diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index c9fb375701f8..acda5c5fca5f 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -116,6 +116,7 @@ torture_param(int, n_barrier_cbs, 0, "# of callbacks/kthreads for barrier testin torture_param(int, n_up_down, 32, "# of concurrent up/down hrtimer-based RCU readers"); torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads"); torture_param(int, nreaders, -1, "Number of RCU reader threads"); +torture_param(bool, nwriters, 1, "Number of RCU writer threads (0 or 1)"); torture_param(int, object_debug, 0, "Enable debug-object double call_rcu() testing"); torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); torture_param(int, onoff_interval, 0, "Time between CPU hotplugs (jiffies), 0=disable"); @@ -3163,7 +3164,7 @@ static void rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) { pr_alert("%s" TORTURE_FLAG - "--- %s: nreaders=%d nfakewriters=%d " + "--- %s: nreaders=%d nwriters=%d nfakewriters=%d " "stat_interval=%d verbose=%d test_no_idle_hz=%d " "shuffle_interval=%d stutter=%d irqreader=%d " "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d " @@ -3178,7 +3179,7 @@ rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) "nocbs_nthreads=%d nocbs_toggle=%d " "test_nmis=%d " "preempt_duration=%d preempt_interval=%d n_up_down=%d\n", - torture_type, tag, nrealreaders, nrealfakewriters, + torture_type, tag, nrealreaders, nwriters, nrealfakewriters, stat_interval, verbose, test_no_idle_hz, shuffle_interval, stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter, test_boost, cur_ops->can_boost, @@ -4754,10 +4755,11 @@ rcu_torture_init(void) goto unwind; } - firsterr = torture_create_kthread(rcu_torture_writer, NULL, - writer_task); - if (torture_init_error(firsterr)) - goto unwind; + if (nwriters) { + firsterr = torture_create_kthread(rcu_torture_writer, NULL, writer_task); + if (torture_init_error(firsterr)) + goto unwind; + } firsterr = rcu_torture_updown_init(); if (torture_init_error(firsterr)) From d1ff05605db14d8d6d1578f289595210c60b385f Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 26 Jun 2026 12:44:20 -0700 Subject: [PATCH 40/47] rcutorture: Add a stall_only module parameter This commit adds a stall_only module parameter that shuts off all rcutorture kthreads other than the RCU CPU stall-warning test kthreads. The purpose of this is to test production applictions' reactions to CPU stalls, and with minimal additional overhead. Or you can omit the stall-warning tests as well and get a heavy no-op, your choice! Signed-off-by: Paul E. McKenney --- .../admin-guide/kernel-parameters.txt | 9 +++++++++ kernel/rcu/rcutorture.c | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index b942149720c9..40d2831add4a 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -6211,6 +6211,15 @@ Kernel parameters and stall_gp_kthread are specified, the kthread is starved first, then the CPU. + rcutorture.stall_only= [KNL] + Shut off all rcutorture kthreads other than the + RCU CPU stall-warning test kthreads. The purpose + of this is to test production applictions' + reactions to CPU stalls, and with minimal + additional overhead. Or you can omit the + stall-warning tests as well and get a heavy + no-op, your choice! + rcutorture.stat_interval= [KNL] Time (s) between statistics printk()s. diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index acda5c5fca5f..6e9822baa236 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -140,6 +140,7 @@ torture_param(int, stall_cpu_irqsoff, 0, "Disable interrupts while stalling."); torture_param(int, stall_cpu_block, 0, "Sleep while stalling."); torture_param(int, stall_cpu_repeat, 0, "Number of additional stalls after the first one."); torture_param(int, stall_gp_kthread, 0, "Grace-period kthread stall duration (s)."); +torture_param(bool, stall_only, 0, "Suppress all non-CPU-stall kthreads."); torture_param(int, stat_interval, 60, "Number of seconds between stats printk()s"); torture_param(int, stutter, 5, "Number of seconds to run/halt test"); torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes."); @@ -4649,6 +4650,23 @@ rcu_torture_init(void) cur_ops = NULL; goto unwind; } + if (stall_only) { + pr_alert("rcu-torture: stall_only specified, suppressing all else.\n"); + fqs_stutter = 0; + fwd_progress = 0; + n_barrier_cbs = 0; + nfakewriters = 0; + nocbs_nthreads = 0; + nreaders = 0; + n_up_down = 0; + nwriters = 0; + onoff_interval = 0; + preempt_duration = 0; + read_exit_burst = 0; + shuffle_interval = 0; + stutter = 0; + test_boost = 0; + } if (cur_ops->fqs == NULL && fqs_duration != 0) { pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n"); fqs_duration = 0; From 560b35bd7c8aa1b7cd20adf28c401e93da567400 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 8 Jul 2026 13:50:35 -0700 Subject: [PATCH 41/47] rcutorture: Test RCU Tasks Trace GP implying RCU GP An RCU Tasks Trace grace period is supposed to imply an RCU grace period, and this implication is relied on by BPF. But this is not currently tested. This commit therefore makes tasks_tracing_torture_read_lock() sometimes use rcu_read_lock() instead of rcu_read_lock_trace(), thus testing the required implication. Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 6e9822baa236..c8118c38812a 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -766,11 +766,12 @@ srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) const long uspertick = 1000000 / HZ; const long longdelay = 10; - /* We want there to be long-running readers, but not all the time. */ + // We want there to be long-running readers, but not all the time. + // The !rcu_preempt_depth() is for RCU Tasks Trace. delay = torture_random(rrsp) % (nrealreaders * 2 * longdelay * uspertick); - if (!delay && in_task()) { + if (!delay && !in_atomic() && !rcu_preempt_depth() && !irqs_disabled()) { schedule_timeout_interruptible(longdelay); rtrsp->rt_delay_jiffies = longdelay; } else { @@ -1219,15 +1220,24 @@ static struct rcu_torture_ops tasks_rude_ops = { * Definitions for tracing RCU-tasks torture testing. */ +// Note that an RCU Tasks Trace GP must imply an RCU GP. static int tasks_tracing_torture_read_lock(void) { - rcu_read_lock_trace(); - return 0; + int use_rcu = !(jiffies & 0xff); + + if (use_rcu) + rcu_read_lock(); + else + rcu_read_lock_trace(); + return use_rcu; } -static void tasks_tracing_torture_read_unlock(int idx) +static void tasks_tracing_torture_read_unlock(int use_rcu) { - rcu_read_unlock_trace(); + if (use_rcu) + rcu_read_unlock(); + else + rcu_read_unlock_trace(); } static void rcu_tasks_tracing_torture_deferred_free(struct rcu_torture *p) From 86fa5387b473c160c26b781dff55014c4e1b3db0 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 8 Jul 2026 17:34:07 -0700 Subject: [PATCH 42/47] rcutorture: Make RCU Tasks Trace track Reader Batches This commit adds the ->get_sp_seq and ->gp_diff fields to the tasks_tracing_ops structure so that RCU Tasks Trace rcutorture runs will track Reader Batch. Signed-off-by: Paul E. McKenney --- include/linux/rcupdate_trace.h | 3 +++ kernel/rcu/rcutorture.c | 3 ++- kernel/rcu/tasks.h | 6 ++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h index cee89e51e45c..f2c4173bf14f 100644 --- a/include/linux/rcupdate_trace.h +++ b/include/linux/rcupdate_trace.h @@ -198,10 +198,13 @@ static inline void rcu_tasks_trace_expedite_current(void) srcu_expedite_current(&rcu_tasks_trace_srcu_struct); } +unsigned long rcu_tasks_trace_batches_completed(void); + // Placeholders to enable stepwise transition. void __init rcu_tasks_trace_suppress_unused(void); #else +static inline unsigned long rcu_tasks_trace_batches_completed(void) { return 0; } /* * The BPF JIT forms these addresses even when it doesn't call these * functions, so provide definitions that result in runtime errors. diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index c8118c38812a..8514fef3ec10 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -1252,7 +1252,8 @@ static struct rcu_torture_ops tasks_tracing_ops = { .read_delay = srcu_read_delay, /* just reuse srcu's version. */ .readunlock = tasks_tracing_torture_read_unlock, .readlock_held = rcu_read_lock_trace_held, - .get_gp_seq = rcu_no_completed, + .get_gp_seq = rcu_tasks_trace_batches_completed, + .gp_diff = rcu_seq_diff, .deferred_free = rcu_tasks_tracing_torture_deferred_free, .sync = synchronize_rcu_tasks_trace, .exp_sync = synchronize_rcu_tasks_trace, diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index f4da5fad70f5..496acac8796f 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1606,4 +1606,10 @@ static inline void rcu_tasks_bootup_oddness(void) {} DEFINE_SRCU_FAST(rcu_tasks_trace_srcu_struct); EXPORT_SYMBOL_GPL(rcu_tasks_trace_srcu_struct); +unsigned long rcu_tasks_trace_batches_completed(void) +{ + return srcu_batches_completed(&rcu_tasks_trace_srcu_struct); +} +EXPORT_SYMBOL_GPL(rcu_tasks_trace_batches_completed); + #endif /* #else #ifdef CONFIG_TASKS_TRACE_RCU */ From ed55bb915047934acd28e2f0ea1ef9586b254bf0 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Sat, 11 Jul 2026 16:51:41 -0700 Subject: [PATCH 43/47] rcutorture: Use this_cpu_inc() for rcu_torture_count[] and rcu_torture_batch[] Currently __this_cpu_inc() is used to increment elements of both the rcu_torture_count[] and rcu_torture_batch[] arrays. However, this can fail when the increments can happen in interrupt handlers, as recently became possible. This commit therefore upgrades the uses of __this_cpu_inc() to the interrupt-safe this_cpu_inc(). KCSAN located this issue. Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 8514fef3ec10..9180016d87c0 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -2565,13 +2565,13 @@ static void rcu_torture_one_read_end(struct rcu_torture_one_read_state *rtorsp, rtorsp->ts, rtorsp->started, completed); rcu_ftrace_dump(DUMP_ALL); } - __this_cpu_inc(rcu_torture_count[pipe_count]); + this_cpu_inc(rcu_torture_count[pipe_count]); completed = rcutorture_seq_diff(completed, rtorsp->started); if (completed > RCU_TORTURE_PIPE_LEN) { /* Should not happen, but... */ completed = RCU_TORTURE_PIPE_LEN; } - __this_cpu_inc(rcu_torture_batch[completed]); + this_cpu_inc(rcu_torture_batch[completed]); preempt_enable(); if (rtorsp->checkpolling) { if (cur_ops->get_gp_state && cur_ops->poll_gp_state) From aaf43c458c491748ae79f8e6afc39fd09a36e77b Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Sat, 18 Jul 2026 13:15:35 -0400 Subject: [PATCH 44/47] torture: Don't leak shuffle_tmp_mask when shuffler kthread fails to start If torture_shuffle_init() successfully allocates shuffle_tmp_mask but then fails to create the torture_shuffle kthread, the cpumask is never freed. Free the cpumask directly on the kthread-creation error path. Signed-off-by: Joel Fernandes Signed-off-by: Paul E. McKenney --- kernel/torture.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kernel/torture.c b/kernel/torture.c index 77cb3589b19f..8c4e6b2fe8ba 100644 --- a/kernel/torture.c +++ b/kernel/torture.c @@ -577,6 +577,8 @@ static int torture_shuffle(void *arg) */ int torture_shuffle_init(long shuffint) { + int ret; + shuffle_interval = shuffint; shuffle_idle_cpu = -1; @@ -587,7 +589,10 @@ int torture_shuffle_init(long shuffint) } /* Create the shuffler thread */ - return torture_create_kthread(torture_shuffle, NULL, shuffler_task); + ret = torture_create_kthread(torture_shuffle, NULL, shuffler_task); + if (ret) + free_cpumask_var(shuffle_tmp_mask); + return ret; } EXPORT_SYMBOL_GPL(torture_shuffle_init); From 6c22d640117c3da7fa7fffa0b1ddebc667507ad9 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 28 Jul 2026 13:55:15 -0700 Subject: [PATCH 45/47] rcutorture: Announce declining to forward-progress test Currently, rcu_torture_fwd_prog_cr() announces function entry, but silently exits if rcu_fwd_emergency_stop is set or if there is no ->call() function available. This could cause confusion, leading people to believe that rcu_torture_fwd_prog_cr() is on the job when it is not. This commit therefore also announces the early exits from this function. Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 9180016d87c0..46d97055c2fc 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -3623,13 +3623,17 @@ static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp) unsigned long stopat; unsigned long stoppedat; - pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); - if (READ_ONCE(rcu_fwd_emergency_stop)) + if (READ_ONCE(rcu_fwd_emergency_stop)) { + pr_alert("%s: Emergency stop, so no forward-progress test %d\n", __func__, rfp->rcu_fwd_id); return; /* Get out of the way quickly, no GP wait! */ - if (!cur_ops->call) + } + if (!cur_ops->call) { + pr_alert("%s: No ->call(), so no forward-progress test %d\n", __func__, rfp->rcu_fwd_id); return; /* Can't do call_rcu() fwd prog without ->call. */ + } /* Loop continuously posting RCU callbacks. */ + pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); atomic_inc(&rcu_fwd_cb_nodelay); cur_ops->sync(); /* Later readers see above write. */ WRITE_ONCE(rfp->rcu_fwd_startat, jiffies); From 20ed97cfde0536991fcb1785453a7748fb131620 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 28 Jul 2026 14:58:49 -0700 Subject: [PATCH 46/47] rcutorture: Make {,s}rcu_read_delay() better handle forward-progress testing The rcu_read_delay() and srcu_read_delay() functions are suppose to avoid (or at least to minimize) read-side delays during call_rcu()-based forward-progress testing. Although rcu_read_delay() does account for this by testing rcu_fwd_cb_nodelay, it only does so in some cases, and srcu_read_delay() does not bother at all. And rcutorture testing does occasionally turn up the MIN_FWD_CBS_LAUNDERED failure when running forward-progress tests on Tree SRCU flavors. This commit therefore makes both rcu_read_delay() and srcu_read_delay() take an immediate exit if rcu_fwd_cb_nodelay is set. Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 46d97055c2fc..9c7b293d1caf 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -469,12 +469,14 @@ rcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) unsigned long longdelay_ms = 300; unsigned long long ts; - /* We want a short delay sometimes to make a reader delay the grace - * period, and we want a long delay occasionally to trigger - * force_quiescent_state. */ + // If there is a forward-progress test in flight, don't delay. + if (atomic_read(&rcu_fwd_cb_nodelay)) + return; - if (!atomic_read(&rcu_fwd_cb_nodelay) && - !(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) { + // We want a short delay sometimes to make a reader delay the grace + // period, and we want a long delay occasionally to trigger + // force_quiescent_state. + if (!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) { started = cur_ops->get_gp_seq(); ts = rcu_trace_clock_local(); if ((preempt_count() & HARDIRQ_MASK) || softirq_count()) @@ -766,6 +768,10 @@ srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) const long uspertick = 1000000 / HZ; const long longdelay = 10; + // If there is a forward-progress test in flight, don't delay. + if (atomic_read(&rcu_fwd_cb_nodelay)) + return; + // We want there to be long-running readers, but not all the time. // The !rcu_preempt_depth() is for RCU Tasks Trace. From 301d28fefd141d1504b4226e8fa87db6a31f3ca1 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 14 Aug 2026 15:37:15 -0700 Subject: [PATCH 47/47] rcu: Add closing parenthesis in comment in rcu_read_unlock_strict() This comment is missing a closing parenthesis: * The in_atomic_preempt_off() check ensures that we come here holding * the last preempt_count (which will get dropped once we return to * __rcu_read_unlock(). This commit therefore adds it at the end of the sentence. Signed-off-by: Paul E. McKenney --- kernel/rcu/tree_plugin.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 6904a4f826d1..9c17dec821d3 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -949,7 +949,7 @@ void rcu_read_unlock_strict(void) * * The in_atomic_preempt_off() check ensures that we come here holding * the last preempt_count (which will get dropped once we return to - * __rcu_read_unlock(). + * __rcu_read_unlock()). */ rdp = this_cpu_ptr(&rcu_data); WRITE_ONCE(rdp->cpu_no_qs.b.norm, false);