From ade2e3c3aede458c8ffed62ef0320aeb60a53a92 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 7 May 2026 16:00:18 -0700 Subject: [PATCH 01/59] rcu-tasks: TASKS_TRACE_RCU doesn't need IRQ_WORK Now that Tasks Trace RCU is implemented in terms of SRCU, it no longer has any particular need for the IRQ_WORK Kconfig option. This commit therefore removes the "select IRQ_WORK" from the TASKS_TRACE_RCU Kconfig option. Signed-off-by: Paul E. McKenney --- kernel/rcu/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/kernel/rcu/Kconfig b/kernel/rcu/Kconfig index 762299291e09..332df7a7a634 100644 --- a/kernel/rcu/Kconfig +++ b/kernel/rcu/Kconfig @@ -140,7 +140,6 @@ config FORCE_TASKS_TRACE_RCU config TASKS_TRACE_RCU bool default n - select IRQ_WORK config TASKS_TRACE_RCU_NO_MB bool "Override RCU Tasks Trace inclusion of read-side memory barriers" From bc892f8249780cacf65636f9a57cc9736b410490 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 30 Apr 2026 17:05:35 -0700 Subject: [PATCH 02/59] rcutorture: Abstract reader-segment dump into rcu_torture_dump_read_segs() This commit abstracts the open-coded dumping of reader segments in the rcu_torture_cleanup() function into a new rcu_torture_dump_read_segs() function. This abstraction will allow reader segments to be dumped for other purposes. Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 148 +++++++++++++++++++++------------------- 1 file changed, 76 insertions(+), 72 deletions(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 882a158ada7b..ffeca1b7e170 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -2401,6 +2401,80 @@ struct rcu_torture_one_read_state { unsigned long long ts; }; +static void rcu_torture_dump_read_segs(struct rt_read_seg *rrsp, int nsegs) +{ + bool firsttime; + int i; + int j; + + firsttime = 1; + for (i = 0; i < nsegs; i++) { + if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP)) + pr_alert("\t%lluus ", div64_u64(rrsp[i].rt_ts, 1000ULL)); + else + pr_alert("\t"); + pr_cont("%d: %#4x", i, rrsp[i].rt_readstate); + if (rrsp[i].rt_delay_jiffies != 0) { + pr_cont("%s%ldjiffies", firsttime ? "" : "+", + rrsp[i].rt_delay_jiffies); + firsttime = 0; + } + if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_CPU)) { + pr_cont(" CPU %2d", rrsp[i].rt_cpu); + if (rrsp[i].rt_cpu != rrsp[i].rt_end_cpu) + pr_cont("->%-2d", rrsp[i].rt_end_cpu); + else + pr_cont(" ..."); + } + if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP) && + cur_ops->gather_gp_seqs && cur_ops->format_gp_seqs) { + char buf1[20+1]; + char buf2[20+1]; + char sepchar = '-'; + + cur_ops->format_gp_seqs(rrsp[i].rt_gp_seq, buf1, ARRAY_SIZE(buf1)); + cur_ops->format_gp_seqs(rrsp[i].rt_gp_seq_end, buf2, ARRAY_SIZE(buf2)); + if (rrsp[i].rt_gp_seq == rrsp[i].rt_gp_seq_end) { + if (buf2[0]) { + for (j = 0; buf2[j]; j++) + buf2[j] = '.'; + if (j) + buf2[j - 1] = ' '; + } + sepchar = ' '; + } + pr_cont(" %s%c%s", buf1, sepchar, buf2); + } + if (rrsp[i].rt_delay_ms != 0) { + pr_cont(" %s%ldms", firsttime ? "" : "+", rrsp[i].rt_delay_ms); + firsttime = 0; + } + if (rrsp[i].rt_delay_us != 0) { + pr_cont(" %s%ldus", firsttime ? "" : "+", rrsp[i].rt_delay_us); + firsttime = 0; + } + pr_cont("%s", rrsp[i].rt_preempted ? " preempted" : ""); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_BH) + pr_cont(" BH"); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_IRQ) + pr_cont(" IRQ"); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_PREEMPT) + pr_cont(" PREEMPT"); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_RBH) + pr_cont(" RBH"); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_SCHED) + pr_cont(" SCHED"); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_RCU_1) + pr_cont(" RCU_1"); + if (rrsp[i].rt_readstate & RCUTORTURE_RDR_RCU_2) + pr_cont(" RCU_2"); + pr_cont("\n"); + + } + if (rt_read_preempted) + pr_alert("\tReader was preempted.\n"); +} + static void init_rcu_torture_one_read_state(struct rcu_torture_one_read_state *rtorsp, struct torture_random_state *trsp) { @@ -4094,11 +4168,9 @@ static void rcu_gpwrap_lag_cleanup(void) static void rcu_torture_cleanup(void) { - int firsttime; int flags = 0; unsigned long gp_seq = 0; int i; - int j; if (torture_cleanup_begin()) { if (cur_ops->cb_barrier != NULL) { @@ -4183,76 +4255,8 @@ rcu_torture_cleanup(void) pr_alert("Failure/close-call rcutorture reader segments:\n"); if (rt_read_nsegs == 0) pr_alert("\t: No segments recorded!!!\n"); - firsttime = 1; - for (i = 0; i < rt_read_nsegs; i++) { - if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP)) - pr_alert("\t%lluus ", div64_u64(err_segs[i].rt_ts, 1000ULL)); - else - pr_alert("\t"); - pr_cont("%d: %#4x", i, err_segs[i].rt_readstate); - if (err_segs[i].rt_delay_jiffies != 0) { - pr_cont("%s%ldjiffies", firsttime ? "" : "+", - err_segs[i].rt_delay_jiffies); - firsttime = 0; - } - if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_CPU)) { - pr_cont(" CPU %2d", err_segs[i].rt_cpu); - if (err_segs[i].rt_cpu != err_segs[i].rt_end_cpu) - pr_cont("->%-2d", err_segs[i].rt_end_cpu); - else - pr_cont(" ..."); - } - if (IS_ENABLED(CONFIG_RCU_TORTURE_TEST_LOG_GP) && - cur_ops->gather_gp_seqs && cur_ops->format_gp_seqs) { - char buf1[20+1]; - char buf2[20+1]; - char sepchar = '-'; - - cur_ops->format_gp_seqs(err_segs[i].rt_gp_seq, - buf1, ARRAY_SIZE(buf1)); - cur_ops->format_gp_seqs(err_segs[i].rt_gp_seq_end, - buf2, ARRAY_SIZE(buf2)); - if (err_segs[i].rt_gp_seq == err_segs[i].rt_gp_seq_end) { - if (buf2[0]) { - for (j = 0; buf2[j]; j++) - buf2[j] = '.'; - if (j) - buf2[j - 1] = ' '; - } - sepchar = ' '; - } - pr_cont(" %s%c%s", buf1, sepchar, buf2); - } - if (err_segs[i].rt_delay_ms != 0) { - pr_cont(" %s%ldms", firsttime ? "" : "+", - err_segs[i].rt_delay_ms); - firsttime = 0; - } - if (err_segs[i].rt_delay_us != 0) { - pr_cont(" %s%ldus", firsttime ? "" : "+", - err_segs[i].rt_delay_us); - firsttime = 0; - } - pr_cont("%s", err_segs[i].rt_preempted ? " preempted" : ""); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_BH) - pr_cont(" BH"); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_IRQ) - pr_cont(" IRQ"); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_PREEMPT) - pr_cont(" PREEMPT"); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_RBH) - pr_cont(" RBH"); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_SCHED) - pr_cont(" SCHED"); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_RCU_1) - pr_cont(" RCU_1"); - if (err_segs[i].rt_readstate & RCUTORTURE_RDR_RCU_2) - pr_cont(" RCU_2"); - pr_cont("\n"); - - } - if (rt_read_preempted) - pr_alert("\tReader was preempted.\n"); + else + rcu_torture_dump_read_segs(err_segs, rt_read_nsegs); } if (atomic_read(&n_rcu_torture_error) || n_rcu_torture_barrier_error) rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE"); From a47dea1baa3b445365ad0757e6dee4ef8fde49e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Onur=20=C3=96zkan?= Date: Sat, 13 Jun 2026 09:40:07 +0300 Subject: [PATCH 03/59] srcu: make init_srcu_struct() consistently wrap __init_srcu_struct() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restructure the SRCU initialization functions so it always follows one direction: init_srcu_struct() -> __init_srcu_struct() -> lockdep or generic This uses the same wrapper style as mutex. It avoids the old confusing style where init_srcu_struct() and __init_srcu_struct() called each other in different configs. It also helps Rust side to have simpler helper for SRCU initialization. Signed-off-by: Onur Özkan Reviewed-by: Gary Guo Reviewed-by: Alice Ryhl Reviewed-by: Boqun Feng Signed-off-by: Paul E. McKenney --- include/linux/srcu.h | 29 ++++++++++++++++++++--------- kernel/rcu/srcutiny.c | 12 ++++++------ kernel/rcu/srcutree.c | 11 ++++++----- 3 files changed, 32 insertions(+), 20 deletions(-) diff --git a/include/linux/srcu.h b/include/linux/srcu.h index a54ce9e808b9..c5ab7df6fe5c 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h @@ -25,20 +25,19 @@ context_lock_struct(srcu_struct, __reentrant_ctx_lock); #ifdef CONFIG_DEBUG_LOCK_ALLOC -int __init_srcu_struct(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); +int init_srcu_struct_lockdep(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key); +static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key) +{ + return init_srcu_struct_lockdep(ssp, name, key); +} #ifndef CONFIG_TINY_SRCU int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); #endif // #ifndef CONFIG_TINY_SRCU -#define init_srcu_struct(ssp) \ -({ \ - static struct lock_class_key __srcu_key; \ - \ - __init_srcu_struct((ssp), #ssp, &__srcu_key); \ -}) - #define init_srcu_struct_fast(ssp) \ ({ \ static struct lock_class_key __srcu_key; \ @@ -56,7 +55,12 @@ int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, #define __SRCU_DEP_MAP_INIT(srcu_name) .dep_map = { .name = #srcu_name }, #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ -int init_srcu_struct(struct srcu_struct *ssp); +int init_srcu_struct_generic(struct srcu_struct *ssp); +static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key) +{ + return init_srcu_struct_generic(ssp); +} #ifndef CONFIG_TINY_SRCU int init_srcu_struct_fast(struct srcu_struct *ssp); int init_srcu_struct_fast_updown(struct srcu_struct *ssp); @@ -65,6 +69,13 @@ int init_srcu_struct_fast_updown(struct srcu_struct *ssp); #define __SRCU_DEP_MAP_INIT(srcu_name) #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ +#define init_srcu_struct(ssp) \ +({ \ + static struct lock_class_key __srcu_key; \ + \ + __init_srcu_struct((ssp), #ssp, &__srcu_key); \ +}) + /* Values for SRCU Tree srcu_data ->srcu_reader_flavor, but also used by rcutorture. */ #define SRCU_READ_FLAVOR_NORMAL 0x1 // srcu_read_lock(). #define SRCU_READ_FLAVOR_NMI 0x2 // srcu_read_lock_nmisafe(). diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c index a2e2d516e51b..47d48ed31848 100644 --- a/kernel/rcu/srcutiny.c +++ b/kernel/rcu/srcutiny.c @@ -48,31 +48,31 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp) #ifdef CONFIG_DEBUG_LOCK_ALLOC -int __init_srcu_struct(struct srcu_struct *ssp, const char *name, - struct lock_class_key *key) +int init_srcu_struct_lockdep(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key) { /* Don't re-initialize a lock while it is held. */ debug_check_no_locks_freed((void *)ssp, sizeof(*ssp)); lockdep_init_map(&ssp->dep_map, name, key, 0); return init_srcu_struct_fields(ssp); } -EXPORT_SYMBOL_GPL(__init_srcu_struct); +EXPORT_SYMBOL_GPL(init_srcu_struct_lockdep); #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ /* - * init_srcu_struct - initialize a sleep-RCU structure + * init_srcu_struct_generic - initialize a sleep-RCU structure * @ssp: structure to initialize. * * Must invoke this on a given srcu_struct before passing that srcu_struct * to any other function. Each srcu_struct represents a separate domain * of SRCU protection. */ -int init_srcu_struct(struct srcu_struct *ssp) +int init_srcu_struct_generic(struct srcu_struct *ssp) { return init_srcu_struct_fields(ssp); } -EXPORT_SYMBOL_GPL(init_srcu_struct); +EXPORT_SYMBOL_GPL(init_srcu_struct_generic); #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 7c2f7cc131f7..56c5db5c29a0 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -266,12 +266,13 @@ __init_srcu_struct_common(struct srcu_struct *ssp, const char *name, struct lock return init_srcu_struct_fields(ssp, false); } -int __init_srcu_struct(struct srcu_struct *ssp, const char *name, struct lock_class_key *key) +int init_srcu_struct_lockdep(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key) { ssp->srcu_reader_flavor = 0; return __init_srcu_struct_common(ssp, name, key); } -EXPORT_SYMBOL_GPL(__init_srcu_struct); +EXPORT_SYMBOL_GPL(init_srcu_struct_lockdep); int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key) { @@ -291,7 +292,7 @@ EXPORT_SYMBOL_GPL(__init_srcu_struct_fast_updown); #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ /** - * init_srcu_struct - initialize a sleep-RCU structure + * init_srcu_struct_generic - initialize a sleep-RCU structure * @ssp: structure to initialize. * * Use this in place of DEFINE_SRCU() and DEFINE_STATIC_SRCU() @@ -301,12 +302,12 @@ EXPORT_SYMBOL_GPL(__init_srcu_struct_fast_updown); * to any other function. Each srcu_struct represents a separate domain * of SRCU protection. */ -int init_srcu_struct(struct srcu_struct *ssp) +int init_srcu_struct_generic(struct srcu_struct *ssp) { ssp->srcu_reader_flavor = 0; return init_srcu_struct_fields(ssp, false); } -EXPORT_SYMBOL_GPL(init_srcu_struct); +EXPORT_SYMBOL_GPL(init_srcu_struct_generic); /** * init_srcu_struct_fast - initialize a fast-reader sleep-RCU structure From 8f63d947bedfddefeb1c10f1dfc36f16407ce22c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Onur=20=C3=96zkan?= Date: Sat, 13 Jun 2026 09:40:08 +0300 Subject: [PATCH 04/59] rust: helpers: add SRCU helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add helper wrappers for SRCU functions that are exposed to Rust through generated bindings. Signed-off-by: Onur Özkan Reviewed-by: Gary Guo Reviewed-by: Alice Ryhl Reviewed-by: Boqun Feng Signed-off-by: Paul E. McKenney --- rust/helpers/helpers.c | 1 + rust/helpers/srcu.c | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 rust/helpers/srcu.c diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 998e31052e66..35954f38ff6a 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -91,6 +91,7 @@ #include "slab.c" #include "spinlock.c" #include "string.c" +#include "srcu.c" #include "sync.c" #include "task.c" #include "time.c" diff --git a/rust/helpers/srcu.c b/rust/helpers/srcu.c new file mode 100644 index 000000000000..225b3bf9334a --- /dev/null +++ b/rust/helpers/srcu.c @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper int rust_helper_init_srcu_struct_with_key(struct srcu_struct *ssp, + const char *name, + struct lock_class_key *key) +{ + return __init_srcu_struct(ssp, name, key); +} + +__rust_helper int rust_helper_srcu_read_lock(struct srcu_struct *ssp) +{ + return srcu_read_lock(ssp); +} + +__rust_helper void rust_helper_srcu_read_unlock(struct srcu_struct *ssp, int idx) +{ + srcu_read_unlock(ssp, idx); +} + +__rust_helper void rust_helper_srcu_barrier(struct srcu_struct *ssp) +{ + srcu_barrier(ssp); +} + +__rust_helper void rust_helper_synchronize_srcu_expedited(struct srcu_struct *ssp) +{ + synchronize_srcu_expedited(ssp); +} From 91d5185187e97611e328d03ee5e00e54b5d6d2b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Onur=20=C3=96zkan?= Date: Sat, 13 Jun 2026 09:40:09 +0300 Subject: [PATCH 05/59] srcu: expose srcu_readers_active() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is needed by rust/helpers/srcu.c which now adds rust_helper_srcu_readers_active() as a wrapper around the SRCU helper for Rust callers. To achive this: 1- Move the srcu_readers_active() implementation from "kernel/rcu/srcutree.c" to "include/linux/srcutree.h". 2- Implement a matching srcu_readers_active() in "include/linux/srcutiny.h" and use it on the existing open-coded WARN_ON() check in cleanup_srcu_struct(). Signed-off-by: Onur Özkan Reviewed-by: Gary Guo Reviewed-by: Alice Ryhl Reviewed-by: Boqun Feng Signed-off-by: Paul E. McKenney --- include/linux/srcutiny.h | 13 +++++++++++++ include/linux/srcutree.h | 24 ++++++++++++++++++++++++ kernel/rcu/srcutiny.c | 2 +- kernel/rcu/srcutree.c | 25 ------------------------- rust/helpers/srcu.c | 5 +++++ 5 files changed, 43 insertions(+), 26 deletions(-) diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h index 905b629e8fa3..fbcf13bc12d1 100644 --- a/include/linux/srcutiny.h +++ b/include/linux/srcutiny.h @@ -154,4 +154,17 @@ static inline void srcu_torture_stats_print(struct srcu_struct *ssp, data_race(READ_ONCE(ssp->srcu_idx_max))); } +/** + * srcu_readers_active - returns true if there are readers. and false otherwise. + * @ssp: which srcu_struct to count active readers (holding srcu_read_lock). + * + * Note that this is not an atomic primitive, and can therefore suffer + * severe errors when invoked on an active srcu_struct. That said, it + * can be useful as an error check at cleanup time. + */ +static inline bool srcu_readers_active(struct srcu_struct *ssp) +{ + return READ_ONCE(ssp->srcu_lock_nesting[0]) || READ_ONCE(ssp->srcu_lock_nesting[1]); +} + #endif diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h index fd1a9270cb9a..75e54e4f963f 100644 --- a/include/linux/srcutree.h +++ b/include/linux/srcutree.h @@ -374,4 +374,28 @@ static inline void srcu_check_read_flavor(struct srcu_struct *ssp, int read_flav __srcu_check_read_flavor(ssp, read_flavor); } +/** + * srcu_readers_active - returns true if there are readers. and false otherwise. + * @ssp: which srcu_struct to count active readers (holding srcu_read_lock). + * + * Note that this is not an atomic primitive, and can therefore suffer + * severe errors when invoked on an active srcu_struct. That said, it + * can be useful as an error check at cleanup time. + */ +static inline bool srcu_readers_active(struct srcu_struct *ssp) +{ + int cpu; + unsigned long sum = 0; + + for_each_possible_cpu(cpu) { + struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); + + sum += atomic_long_read(&sdp->srcu_ctrs[0].srcu_locks); + sum += atomic_long_read(&sdp->srcu_ctrs[1].srcu_locks); + sum -= atomic_long_read(&sdp->srcu_ctrs[0].srcu_unlocks); + sum -= atomic_long_read(&sdp->srcu_ctrs[1].srcu_unlocks); + } + return sum; +} + #endif diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c index 47d48ed31848..558ba8d316db 100644 --- a/kernel/rcu/srcutiny.c +++ b/kernel/rcu/srcutiny.c @@ -85,7 +85,7 @@ EXPORT_SYMBOL_GPL(init_srcu_struct_generic); */ void cleanup_srcu_struct(struct srcu_struct *ssp) { - WARN_ON(ssp->srcu_lock_nesting[0] || ssp->srcu_lock_nesting[1]); + WARN_ON(srcu_readers_active(ssp)); irq_work_sync(&ssp->srcu_irq_work); flush_work(&ssp->srcu_work); WARN_ON(ssp->srcu_gp_running); diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 56c5db5c29a0..4a00e90e17fc 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -599,31 +599,6 @@ static bool srcu_readers_active_idx_check(struct srcu_struct *ssp, int idx) return srcu_readers_lock_idx(ssp, idx, did_gp, unlocks); } -/** - * srcu_readers_active - returns true if there are readers. and false - * otherwise - * @ssp: which srcu_struct to count active readers (holding srcu_read_lock). - * - * Note that this is not an atomic primitive, and can therefore suffer - * severe errors when invoked on an active srcu_struct. That said, it - * can be useful as an error check at cleanup time. - */ -static bool srcu_readers_active(struct srcu_struct *ssp) -{ - int cpu; - unsigned long sum = 0; - - for_each_possible_cpu(cpu) { - struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); - - sum += atomic_long_read(&sdp->srcu_ctrs[0].srcu_locks); - sum += atomic_long_read(&sdp->srcu_ctrs[1].srcu_locks); - sum -= atomic_long_read(&sdp->srcu_ctrs[0].srcu_unlocks); - sum -= atomic_long_read(&sdp->srcu_ctrs[1].srcu_unlocks); - } - return sum; -} - /* * We use an adaptive strategy for synchronize_srcu() and especially for * synchronize_srcu_expedited(). We spin for a fixed time period diff --git a/rust/helpers/srcu.c b/rust/helpers/srcu.c index 225b3bf9334a..1a2f563640e0 100644 --- a/rust/helpers/srcu.c +++ b/rust/helpers/srcu.c @@ -9,6 +9,11 @@ __rust_helper int rust_helper_init_srcu_struct_with_key(struct srcu_struct *ssp, return __init_srcu_struct(ssp, name, key); } +__rust_helper bool rust_helper_srcu_readers_active(struct srcu_struct *ssp) +{ + return srcu_readers_active(ssp); +} + __rust_helper int rust_helper_srcu_read_lock(struct srcu_struct *ssp) { return srcu_read_lock(ssp); From 59cf3a5bda61cd3c300f0db952afcb031806e802 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Onur=20=C3=96zkan?= Date: Sat, 13 Jun 2026 09:40:10 +0300 Subject: [PATCH 06/59] rust: sync: add SRCU abstraction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a Rust abstraction for sleepable RCU (SRCU), backed by C srcu_struct. Provide FFI helpers and a safe wrapper with a guard-based API for read-side critical sections. Cleanup is handled via `PinnedDrop`. It first checks for active read-side sections and emits a warning if any guards were leaked. In that case, it waits in `synchronize_srcu()` rather than risking a UAF by freeing the `srcu_struct` that is still reachable from the C side. It then uses `srcu_barrier()` to drain pending callbacks before finally calling `cleanup_srcu_struct()`. Signed-off-by: Onur Özkan Reviewed-by: Gary Guo Reviewed-by: Alice Ryhl Reviewed-by: Boqun Feng Signed-off-by: Paul E. McKenney --- rust/kernel/sync.rs | 2 + rust/kernel/sync/srcu.rs | 171 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 173 insertions(+) create mode 100644 rust/kernel/sync/srcu.rs diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index 993dbf2caa0e..0d6a5f1300c3 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -21,6 +21,7 @@ pub mod rcu; mod refcount; mod set_once; +pub mod srcu; pub use arc::{Arc, ArcBorrow, UniqueArc}; pub use completion::Completion; @@ -31,6 +32,7 @@ pub use locked_by::LockedBy; pub use refcount::Refcount; pub use set_once::SetOnce; +pub use srcu::Srcu; /// Represents a lockdep class. /// diff --git a/rust/kernel/sync/srcu.rs b/rust/kernel/sync/srcu.rs new file mode 100644 index 000000000000..723e5e277fd6 --- /dev/null +++ b/rust/kernel/sync/srcu.rs @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Sleepable read-copy update (SRCU) support. +//! +//! C header: [`include/linux/srcu.h`](srctree/include/linux/srcu.h) + +use crate::{ + bindings, + error::to_result, + prelude::*, + sync::LockClassKey, + types::{ + NotThreadSafe, + Opaque, // + }, +}; + +use pin_init::pin_data; + +/// Creates an [`Srcu`] initialiser with the given name and a newly-created lock class. +#[doc(hidden)] +#[macro_export] +macro_rules! new_srcu { + ($($name:literal)?) => { + $crate::sync::Srcu::new($crate::optional_name!($($name)?), $crate::static_lock_class!()) + }; +} +pub use new_srcu; + +/// Sleepable read-copy update primitive. +/// +/// SRCU readers may sleep while holding the read-side guard. +/// +/// The destructor waits for active readers and callbacks, so it may sleep. +/// If a read-side guard has been leaked, dropping an [`Srcu`] may never return. +/// +/// # Invariants +/// +/// This represents a valid `struct srcu_struct` initialized by the C SRCU API +/// and it remains pinned and valid until the pinned destructor runs. +#[repr(transparent)] +#[pin_data(PinnedDrop)] +pub struct Srcu { + #[pin] + inner: Opaque, +} + +impl Srcu { + /// Creates a new SRCU instance. + #[inline] + pub fn new(name: &'static CStr, key: Pin<&'static LockClassKey>) -> impl PinInit { + try_pin_init!(Self { + // INVARIANT: On success, the C initializer creates a valid `srcu_struct` and + // it remains pinned until `PinnedDrop` runs. + inner <- Opaque::try_ffi_init(|ptr: *mut bindings::srcu_struct| { + // SAFETY: `ptr` points to valid uninitialised memory for a `srcu_struct`. + to_result(unsafe { + bindings::init_srcu_struct_with_key(ptr, name.as_char_ptr(), key.as_ptr()) + }) + }), + }) + } + + /// Enters an SRCU read-side critical section. + /// + /// Leaking the returned [`Guard`] leaves the SRCU read-side critical + /// section active and makes `drop` sleep forever. + #[inline] + pub fn read_lock(&self) -> Guard<'_> { + // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`. + let idx = unsafe { bindings::srcu_read_lock(self.inner.get()) }; + + // INVARIANT: `idx` was returned by `srcu_read_lock()` for this `Srcu`. + Guard { + srcu: self, + idx, + _not_send: NotThreadSafe, + } + } + + /// Waits until all pre-existing SRCU readers have completed. + #[inline] + pub fn synchronize(&self) { + // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`. + unsafe { bindings::synchronize_srcu(self.inner.get()) }; + } + + /// Waits until all pre-existing SRCU readers have completed, expedited. + /// + /// This requests a lower-latency grace period than [`Srcu::synchronize`] typically + /// at the cost of higher system-wide overhead. Prefer [`Srcu::synchronize`] by default + /// and use this variant only when reducing reset or teardown latency is more important + /// than the extra cost. + #[inline] + pub fn synchronize_expedited(&self) { + // SAFETY: By the type invariants, `self` contains a valid `struct srcu_struct`. + unsafe { bindings::synchronize_srcu_expedited(self.inner.get()) }; + } +} + +#[pinned_drop] +impl PinnedDrop for Srcu { + fn drop(self: Pin<&mut Self>) { + let ptr = self.inner.get(); + + if crate::warn_on!( + // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct` + // and `srcu_readers_active()` only checks the active reader count. + unsafe { bindings::srcu_readers_active(ptr) } + ) { + // `cleanup_srcu_struct()` may return early if there are still active readers. + // This should only happen if a guard was leaked with `mem::forget`, which is + // "WRONG" code and may cause a UAF because Rust will free the `srcu_struct` + // while it is still referenced from the C side (e.g. by `call_srcu()` callbacks). + // + // Another consequence of leaking guards is that `call_srcu()` callbacks will + // never run because the grace period can never complete due to permanently + // active readers (i.e. leaked guards). + // + // If this ever happens, that means the guard was leaked by mistake and the + // caller must fix the bug. Sleeping here is intentional and less harmful + // than risking a UAF. + // + // SAFETY: By the type invariants, `self` contains a valid and pinned + // `struct srcu_struct`. + unsafe { bindings::synchronize_srcu(ptr) }; + } + + // Ensure all SRCU callbacks have been finished before freeing. + // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct`. + unsafe { bindings::srcu_barrier(ptr) }; + + // SAFETY: By the type invariants, `self` contains a valid and pinned `struct srcu_struct`. + unsafe { bindings::cleanup_srcu_struct(ptr) }; + } +} + +// SAFETY: `srcu_struct` may be shared and used across threads. +unsafe impl Send for Srcu {} +// SAFETY: `srcu_struct` may be shared and used concurrently. +unsafe impl Sync for Srcu {} + +/// Guard for an active SRCU read-side critical section on a particular [`Srcu`]. +/// +/// Leaking this guard with [`core::mem::forget`] leaves the SRCU read-side +/// critical section active and makes dropping the associated [`Srcu`] sleep forever. +/// +/// # Invariants +/// +/// `idx` is the index returned by `srcu_read_lock()` for `srcu`. +#[must_use = "if unused, the lock will be immediately unlocked"] +pub struct Guard<'a> { + srcu: &'a Srcu, + idx: i32, + _not_send: NotThreadSafe, +} + +impl Guard<'_> { + /// Explicitly releases the SRCU read-side critical section. + #[inline] + pub fn unlock(self) {} +} + +impl Drop for Guard<'_> { + #[inline] + fn drop(&mut self) { + // SAFETY: `Guard` is only constructible through `Srcu::read_lock()`, + // which returns a valid index for the SRCU instance. + unsafe { bindings::srcu_read_unlock(self.srcu.inner.get(), self.idx) }; + } +} From 3f90d04303c11a5006059016fc1ebedcbbff682d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Onur=20=C3=96zkan?= Date: Sat, 13 Jun 2026 09:40:11 +0300 Subject: [PATCH 07/59] MAINTAINERS: add Rust SRCU files to SRCU entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Include Rust side implementation files to the SRCU maintainer entry. Signed-off-by: Onur Özkan Reviewed-by: Alice Ryhl Reviewed-by: Boqun Feng Signed-off-by: Paul E. McKenney --- MAINTAINERS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 806bd2d80d15..b6ef971ee408 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -24933,6 +24933,7 @@ SLEEPABLE READ-COPY UPDATE (SRCU) M: Lai Jiangshan M: "Paul E. McKenney" M: Josh Triplett +M: Onur Özkan (RUST) R: Steven Rostedt R: Mathieu Desnoyers L: rcu@vger.kernel.org @@ -24941,6 +24942,8 @@ W: http://www.rdrop.com/users/paulmck/RCU/ T: git git://git.kernel.org/pub/scm/linux/kernel/git/rcu/linux.git rcu/dev F: include/linux/srcu*.h F: kernel/rcu/srcu*.c +F: rust/helpers/srcu.c +F: rust/kernel/sync/srcu.rs SMACK SECURITY MODULE M: Casey Schaufler From 767ce74b33741c730777df5ab77bd3ab7dc2c74c Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Sat, 18 Jul 2026 13:15:33 -0400 Subject: [PATCH 08/59] rcu: Remove unused rdp parameter from trace_rcu_this_gp() The trace_rcu_this_gp() wrapper forwards only the rcu_node structure's fields and the requested grace-period sequence number to the rcu_future_grace_period tracepoint. Its rcu_data pointer parameter has no users, but every one of the ten call sites must nevertheless come up with an rcu_data pointer to pass in. Remove the parameter and update all callers. This also allows rcu_future_gp_cleanup() to drop the local rcu_data pointer that existed solely to feed this trace call. No functional change. Signed-off-by: Joel Fernandes Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 23 ++++++++++------------- kernel/rcu/tree_nocb.h | 4 ++-- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 03a43d3d2616..ef553189ee24 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -986,8 +986,8 @@ static int rcu_watching_snap_recheck(struct rcu_data *rdp) } /* Trace-event wrapper function for trace_rcu_future_grace_period. */ -static void trace_rcu_this_gp(struct rcu_node *rnp, struct rcu_data *rdp, - unsigned long gp_seq_req, const char *s) +static void trace_rcu_this_gp(struct rcu_node *rnp, unsigned long gp_seq_req, + const char *s) { trace_rcu_future_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq), gp_seq_req, rnp->level, @@ -1026,7 +1026,7 @@ static bool rcu_start_this_gp(struct rcu_node *rnp_start, struct rcu_data *rdp, * Note that rnp_start->lock must not be released. */ raw_lockdep_assert_held_rcu_node(rnp_start); - trace_rcu_this_gp(rnp_start, rdp, gp_seq_req, TPS("Startleaf")); + trace_rcu_this_gp(rnp_start, gp_seq_req, TPS("Startleaf")); for (rnp = rnp_start; 1; rnp = rnp->parent) { if (rnp != rnp_start) raw_spin_lock_rcu_node(rnp); @@ -1034,8 +1034,7 @@ static bool rcu_start_this_gp(struct rcu_node *rnp_start, struct rcu_data *rdp, rcu_seq_started(&rnp->gp_seq, gp_seq_req) || (rnp != rnp_start && rcu_seq_state(rcu_seq_current(&rnp->gp_seq)))) { - trace_rcu_this_gp(rnp, rdp, gp_seq_req, - TPS("Prestarted")); + trace_rcu_this_gp(rnp, gp_seq_req, TPS("Prestarted")); goto unlock_out; } WRITE_ONCE(rnp->gp_seq_needed, gp_seq_req); @@ -1046,7 +1045,7 @@ static bool rcu_start_this_gp(struct rcu_node *rnp_start, struct rcu_data *rdp, * rcu_gp_cleanup() will see the marking. Bail to * reduce contention. */ - trace_rcu_this_gp(rnp_start, rdp, gp_seq_req, + trace_rcu_this_gp(rnp_start, gp_seq_req, TPS("Startedleaf")); goto unlock_out; } @@ -1058,14 +1057,14 @@ static bool rcu_start_this_gp(struct rcu_node *rnp_start, struct rcu_data *rdp, /* If GP already in progress, just leave, otherwise start one. */ if (rcu_gp_in_progress()) { - trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedleafroot")); + trace_rcu_this_gp(rnp, gp_seq_req, TPS("Startedleafroot")); goto unlock_out; } - trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedroot")); + trace_rcu_this_gp(rnp, gp_seq_req, TPS("Startedroot")); WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags | RCU_GP_FLAG_INIT); WRITE_ONCE(rcu_state.gp_req_activity, jiffies); if (!READ_ONCE(rcu_state.gp_kthread)) { - trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("NoGPkthread")); + trace_rcu_this_gp(rnp, gp_seq_req, TPS("NoGPkthread")); goto unlock_out; } trace_rcu_grace_period(rcu_state.name, data_race(rcu_state.gp_seq), TPS("newreq")); @@ -1088,12 +1087,11 @@ static bool rcu_start_this_gp(struct rcu_node *rnp_start, struct rcu_data *rdp, static bool rcu_future_gp_cleanup(struct rcu_node *rnp) { bool needmore; - struct rcu_data *rdp = this_cpu_ptr(&rcu_data); needmore = ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed); if (!needmore) rnp->gp_seq_needed = rnp->gp_seq; /* Avoid counter wrap. */ - trace_rcu_this_gp(rnp, rdp, rnp->gp_seq, + trace_rcu_this_gp(rnp, rnp->gp_seq, needmore ? TPS("CleanupMore") : TPS("Cleanup")); return needmore; } @@ -2252,8 +2250,7 @@ static noinline void rcu_gp_cleanup(void) /* Check for GP requests since above loop. */ rdp = this_cpu_ptr(&rcu_data); if (!needgp && ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed)) { - trace_rcu_this_gp(rnp, rdp, rnp->gp_seq_needed, - TPS("CleanupMore")); + trace_rcu_this_gp(rnp, rnp->gp_seq_needed, TPS("CleanupMore")); needgp = true; } /* Advance CBs to reduce false positives below. */ diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index 373b877cf171..c4a5b4662b3a 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -802,12 +802,12 @@ static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp) nocb_gp_sleep(my_rdp, cpu); } else { rnp = my_rdp->mynode; - trace_rcu_this_gp(rnp, my_rdp, wait_gp_seq, TPS("StartWait")); + trace_rcu_this_gp(rnp, wait_gp_seq, TPS("StartWait")); swait_event_interruptible_exclusive( rnp->nocb_gp_wq[rcu_seq_ctr(wait_gp_seq) & 0x1], rcu_seq_done(&rnp->gp_seq, wait_gp_seq) || !READ_ONCE(my_rdp->nocb_gp_sleep)); - trace_rcu_this_gp(rnp, my_rdp, wait_gp_seq, TPS("EndWait")); + trace_rcu_this_gp(rnp, wait_gp_seq, TPS("EndWait")); } if (!rcu_nocb_poll) { From 736084507faa3596caad181a2ba6aa2a96197c52 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Wed, 24 Jun 2026 06:23:43 -0700 Subject: [PATCH 09/59] rcu: Rename struct rcu_gp_oldstate to rcu_gp_seq The polled grace-period state structure rcu_gp_oldstate holds a snapshot of the normal (and, on SMP, expedited) grace-period sequence numbers. Upcoming changes store this structure in the callback segment list, where the "oldstate" name reads poorly: there it represents the grace period a segment is waiting on and is also compared against the current grace-period state. Rename struct rcu_gp_oldstate to the more neutral struct rcu_gp_seq, and shorten its members rgos_norm and rgos_exp to norm and exp. Local variables and parameters of this type are renamed from rgosp/rgos to gsp/gs accordingly. While at it, provide a single definition of the structure in rcupdate.h rather than separate Tiny-RCU and Tree-RCU definitions, and give it the ->exp field unconditionally. Tiny RCU does not track expedited grace periods and leaves ->exp unused, but a single definition that always has ->exp lets the shared callback code in rcu_segcblist.c reference it without CONFIG_SMP guards, including on !SMP builds. No functional change. Signed-off-by: Puranjay Mohan Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- include/linux/rcupdate.h | 13 ++++++-- include/linux/rcupdate_wait.h | 2 +- include/linux/rcutiny.h | 36 +++++++++----------- include/linux/rcutree.h | 29 +++++++--------- kernel/rcu/rcutorture.c | 30 ++++++++--------- kernel/rcu/tiny.c | 4 +-- kernel/rcu/tree.c | 62 +++++++++++++++++------------------ kernel/rcu/tree_exp.h | 18 +++++----- mm/slab_common.c | 6 ++-- 9 files changed, 100 insertions(+), 100 deletions(-) diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 5e95acc33989..ce00f1726e95 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -52,9 +52,18 @@ void call_rcu(struct rcu_head *head, rcu_callback_t func); void rcu_barrier_tasks(void); void synchronize_rcu(void); -struct rcu_gp_oldstate; +/* + * Grace-period sequence snapshot for the polled RCU APIs: ->norm for the + * normal grace period and ->exp for the expedited one. ->exp is unused by + * Tiny RCU, but is present unconditionally so that a single definition + * serves both Tiny RCU and Tree RCU. + */ +struct rcu_gp_seq { + unsigned long norm; + unsigned long exp; +}; unsigned long get_completed_synchronize_rcu(void); -void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); +void get_completed_synchronize_rcu_full(struct rcu_gp_seq *gsp); // Maximum number of unsigned long values corresponding to // not-yet-completed RCU grace periods. diff --git a/include/linux/rcupdate_wait.h b/include/linux/rcupdate_wait.h index 4c92d4291cce..fa884704a3b7 100644 --- a/include/linux/rcupdate_wait.h +++ b/include/linux/rcupdate_wait.h @@ -18,7 +18,7 @@ struct rcu_synchronize { struct completion completion; /* This is for debugging. */ - struct rcu_gp_oldstate oldstate; + struct rcu_gp_seq oldstate; }; void wakeme_after_rcu(struct rcu_head *head); diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h index f519cd680228..e56ded733b1b 100644 --- a/include/linux/rcutiny.h +++ b/include/linux/rcutiny.h @@ -14,11 +14,7 @@ #include /* for HZ */ -struct rcu_gp_oldstate { - unsigned long rgos_norm; -}; - -// Maximum number of rcu_gp_oldstate values corresponding to +// Maximum number of rcu_gp_seq values corresponding to // not-yet-completed RCU grace periods. #define NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE 2 @@ -26,31 +22,31 @@ struct rcu_gp_oldstate { * Are the two oldstate values the same? See the Tree RCU version for * docbook header. */ -static inline bool same_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp1, - struct rcu_gp_oldstate *rgosp2) +static inline bool same_state_synchronize_rcu_full(struct rcu_gp_seq *rgosp1, + struct rcu_gp_seq *rgosp2) { - return rgosp1->rgos_norm == rgosp2->rgos_norm; + return rgosp1->norm == rgosp2->norm; } unsigned long get_state_synchronize_rcu(void); -static inline void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +static inline void get_state_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - rgosp->rgos_norm = get_state_synchronize_rcu(); + gsp->norm = get_state_synchronize_rcu(); } unsigned long start_poll_synchronize_rcu(void); -static inline void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +static inline void start_poll_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - rgosp->rgos_norm = start_poll_synchronize_rcu(); + gsp->norm = start_poll_synchronize_rcu(); } bool poll_state_synchronize_rcu(unsigned long oldstate); -static inline bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +static inline bool poll_state_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - return poll_state_synchronize_rcu(rgosp->rgos_norm); + return poll_state_synchronize_rcu(gsp->norm); } static inline void cond_synchronize_rcu(unsigned long oldstate) @@ -58,9 +54,9 @@ static inline void cond_synchronize_rcu(unsigned long oldstate) might_sleep(); } -static inline void cond_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +static inline void cond_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - cond_synchronize_rcu(rgosp->rgos_norm); + cond_synchronize_rcu(gsp->norm); } static inline unsigned long start_poll_synchronize_rcu_expedited(void) @@ -68,9 +64,9 @@ static inline unsigned long start_poll_synchronize_rcu_expedited(void) return start_poll_synchronize_rcu(); } -static inline void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp) +static inline void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_seq *gsp) { - rgosp->rgos_norm = start_poll_synchronize_rcu_expedited(); + gsp->norm = start_poll_synchronize_rcu_expedited(); } static inline void cond_synchronize_rcu_expedited(unsigned long oldstate) @@ -78,9 +74,9 @@ static inline void cond_synchronize_rcu_expedited(unsigned long oldstate) cond_synchronize_rcu(oldstate); } -static inline void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp) +static inline void cond_synchronize_rcu_expedited_full(struct rcu_gp_seq *gsp) { - cond_synchronize_rcu_expedited(rgosp->rgos_norm); + cond_synchronize_rcu_expedited(gsp->norm); } extern void rcu_barrier(void); diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h index 9d2d7bd251d4..16a04202888b 100644 --- a/include/linux/rcutree.h +++ b/include/linux/rcutree.h @@ -38,12 +38,7 @@ void synchronize_rcu_expedited(void); void rcu_barrier(void); void rcu_momentary_eqs(void); -struct rcu_gp_oldstate { - unsigned long rgos_norm; - unsigned long rgos_exp; -}; - -// Maximum number of rcu_gp_oldstate values corresponding to +// Maximum number of rcu_gp_seq values corresponding to // not-yet-completed RCU grace periods. #define NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE 4 @@ -60,29 +55,29 @@ struct rcu_gp_oldstate { * to a list header, allowing those structures to be slightly smaller. * * Note that equality is judged on a bitwise basis, so that an - * @rcu_gp_oldstate structure with an already-completed state in one field + * @rcu_gp_seq structure with an already-completed state in one field * will compare not-equal to a structure with an already-completed state - * in the other field. After all, the @rcu_gp_oldstate structure is opaque + * in the other field. After all, the @rcu_gp_seq structure is opaque * so how did such a situation come to pass in the first place? */ -static inline bool same_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp1, - struct rcu_gp_oldstate *rgosp2) +static inline bool same_state_synchronize_rcu_full(struct rcu_gp_seq *rgosp1, + struct rcu_gp_seq *rgosp2) { - return rgosp1->rgos_norm == rgosp2->rgos_norm && rgosp1->rgos_exp == rgosp2->rgos_exp; + return rgosp1->norm == rgosp2->norm && rgosp1->exp == rgosp2->exp; } unsigned long start_poll_synchronize_rcu_expedited(void); -void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp); +void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_seq *gsp); void cond_synchronize_rcu_expedited(unsigned long oldstate); -void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp); +void cond_synchronize_rcu_expedited_full(struct rcu_gp_seq *gsp); unsigned long get_state_synchronize_rcu(void); -void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); +void get_state_synchronize_rcu_full(struct rcu_gp_seq *gsp); unsigned long start_poll_synchronize_rcu(void); -void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); +void start_poll_synchronize_rcu_full(struct rcu_gp_seq *gsp); bool poll_state_synchronize_rcu(unsigned long oldstate); -bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); +bool poll_state_synchronize_rcu_full(struct rcu_gp_seq *gsp); void cond_synchronize_rcu(unsigned long oldstate); -void cond_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); +void cond_synchronize_rcu_full(struct rcu_gp_seq *gsp); #ifdef CONFIG_PROVE_RCU void rcu_irq_exit_check_preempt(void); diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 882a158ada7b..d71911ac911a 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -392,23 +392,23 @@ struct rcu_torture_ops { void (*exp_current)(void); unsigned long (*get_gp_state_exp)(void); unsigned long (*start_gp_poll_exp)(void); - void (*start_gp_poll_exp_full)(struct rcu_gp_oldstate *rgosp); + void (*start_gp_poll_exp_full)(struct rcu_gp_seq *gsp); bool (*poll_gp_state_exp)(unsigned long oldstate); void (*cond_sync_exp)(unsigned long oldstate); - void (*cond_sync_exp_full)(struct rcu_gp_oldstate *rgosp); + void (*cond_sync_exp_full)(struct rcu_gp_seq *gsp); unsigned long (*get_comp_state)(void); - void (*get_comp_state_full)(struct rcu_gp_oldstate *rgosp); + void (*get_comp_state_full)(struct rcu_gp_seq *gsp); bool (*same_gp_state)(unsigned long oldstate1, unsigned long oldstate2); - bool (*same_gp_state_full)(struct rcu_gp_oldstate *rgosp1, struct rcu_gp_oldstate *rgosp2); + bool (*same_gp_state_full)(struct rcu_gp_seq *rgosp1, struct rcu_gp_seq *rgosp2); unsigned long (*get_gp_state)(void); - void (*get_gp_state_full)(struct rcu_gp_oldstate *rgosp); + void (*get_gp_state_full)(struct rcu_gp_seq *gsp); unsigned long (*start_gp_poll)(void); - void (*start_gp_poll_full)(struct rcu_gp_oldstate *rgosp); + void (*start_gp_poll_full)(struct rcu_gp_seq *gsp); bool (*poll_gp_state)(unsigned long oldstate); - bool (*poll_gp_state_full)(struct rcu_gp_oldstate *rgosp); + bool (*poll_gp_state_full)(struct rcu_gp_seq *gsp); bool (*poll_need_2gp)(bool poll, bool poll_full); void (*cond_sync)(unsigned long oldstate); - void (*cond_sync_full)(struct rcu_gp_oldstate *rgosp); + void (*cond_sync_full)(struct rcu_gp_seq *gsp); int poll_active; int poll_active_full; call_rcu_func_t call; @@ -1605,7 +1605,7 @@ static void rcu_torture_write_types(void) static void do_rtws_sync(struct torture_random_state *trsp, void (*sync)(void)) { unsigned long cookie; - struct rcu_gp_oldstate cookie_full; + struct rcu_gp_seq cookie_full; bool dopoll; bool dopoll_full; unsigned long r = torture_random(trsp); @@ -1653,18 +1653,18 @@ rcu_torture_writer(void *arg) bool booting_still = false; bool can_expedite = !rcu_gp_is_expedited() && !rcu_gp_is_normal(); unsigned long cookie; - struct rcu_gp_oldstate cookie_full; + struct rcu_gp_seq cookie_full; int expediting = 0; unsigned long gp_snap; unsigned long gp_snap1; - struct rcu_gp_oldstate gp_snap_full; - struct rcu_gp_oldstate gp_snap1_full; + struct rcu_gp_seq gp_snap_full; + struct rcu_gp_seq gp_snap1_full; int i; int idx; unsigned long j; struct work_struct lazy_work; int oldnice = task_nice(current); - struct rcu_gp_oldstate *rgo = NULL; + struct rcu_gp_seq *rgo = NULL; int rgo_size = 0; struct rcu_torture *rp; struct rcu_torture *old_rp; @@ -1963,7 +1963,7 @@ static int rcu_torture_fakewriter(void *arg) { unsigned long gp_snap; - struct rcu_gp_oldstate gp_snap_full; + struct rcu_gp_seq gp_snap_full; DEFINE_TORTURE_RANDOM(rand); VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started"); @@ -2392,7 +2392,7 @@ rcutorture_loop_extend(int *readstate, struct torture_random_state *trsp, struct struct rcu_torture_one_read_state { bool checkpolling; unsigned long cookie; - struct rcu_gp_oldstate cookie_full; + struct rcu_gp_seq cookie_full; unsigned long started; struct rcu_torture *p; int readstate; diff --git a/kernel/rcu/tiny.c b/kernel/rcu/tiny.c index 585cade21010..dccccd6be941 100644 --- a/kernel/rcu/tiny.c +++ b/kernel/rcu/tiny.c @@ -187,9 +187,9 @@ EXPORT_SYMBOL_GPL(call_rcu); * Store a grace-period-counter "cookie". For more information, * see the Tree RCU header comment. */ -void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +void get_completed_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - rgosp->rgos_norm = RCU_GET_STATE_COMPLETED; + gsp->norm = RCU_GET_STATE_COMPLETED; } EXPORT_SYMBOL_GPL(get_completed_synchronize_rcu_full); diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index ef553189ee24..0d2d877eb945 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -3284,7 +3284,7 @@ EXPORT_SYMBOL_GPL(call_rcu); * Later on, this could in theory be the case for kernels built with * CONFIG_SMP=y && CONFIG_PREEMPTION=y running on a single CPU, but this * is not a common case. Furthermore, this optimization would cause - * the rcu_gp_oldstate structure to expand by 50%, so this potential + * the rcu_gp_seq structure to expand by 50%, so this potential * grace-period optimization is ignored once the scheduler is running. */ static int rcu_blocking_is_gp(void) @@ -3413,16 +3413,16 @@ EXPORT_SYMBOL_GPL(synchronize_rcu); /** * get_completed_synchronize_rcu_full - Return a full pre-completed polled state cookie - * @rgosp: Place to put state cookie + * @gsp: Place to put state cookie * - * Stores into @rgosp a value that will always be treated by functions + * Stores into @gsp a value that will always be treated by functions * like poll_state_synchronize_rcu_full() as a cookie whose grace period * has already completed. */ -void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +void get_completed_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - rgosp->rgos_norm = RCU_GET_STATE_COMPLETED; - rgosp->rgos_exp = RCU_GET_STATE_COMPLETED; + gsp->norm = RCU_GET_STATE_COMPLETED; + gsp->exp = RCU_GET_STATE_COMPLETED; } EXPORT_SYMBOL_GPL(get_completed_synchronize_rcu_full); @@ -3446,13 +3446,13 @@ EXPORT_SYMBOL_GPL(get_state_synchronize_rcu); /** * get_state_synchronize_rcu_full - Snapshot RCU state, both normal and expedited - * @rgosp: location to place combined normal/expedited grace-period state + * @gsp: location to place combined normal/expedited grace-period state * - * Places the normal and expedited grace-period states in @rgosp. This + * Places the normal and expedited grace-period states in @gsp. This * state value can be passed to a later call to cond_synchronize_rcu_full() * or poll_state_synchronize_rcu_full() to determine whether or not a * grace period (whether normal or expedited) has elapsed in the meantime. - * The rcu_gp_oldstate structure takes up twice the memory of an unsigned + * The rcu_gp_seq structure takes up twice the memory of an unsigned * long, but is guaranteed to see all grace periods. In contrast, the * combined state occupies less memory, but can sometimes fail to take * grace periods into account. @@ -3460,7 +3460,7 @@ EXPORT_SYMBOL_GPL(get_state_synchronize_rcu); * This does not guarantee that the needed grace period will actually * start. */ -void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +void get_state_synchronize_rcu_full(struct rcu_gp_seq *gsp) { /* * Any prior manipulation of RCU-protected data must happen @@ -3472,8 +3472,8 @@ void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) // in poll_state_synchronize_rcu_full() notwithstanding. Use of // the latter here would result in too-short grace periods due to // interactions with newly onlined CPUs. - rgosp->rgos_norm = rcu_seq_snap(&rcu_state.gp_seq); - rgosp->rgos_exp = rcu_seq_snap(&rcu_state.expedited_sequence); + gsp->norm = rcu_seq_snap(&rcu_state.gp_seq); + gsp->exp = rcu_seq_snap(&rcu_state.expedited_sequence); } EXPORT_SYMBOL_GPL(get_state_synchronize_rcu_full); @@ -3524,18 +3524,18 @@ EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu); /** * start_poll_synchronize_rcu_full - Take a full snapshot and start RCU grace period - * @rgosp: value from get_state_synchronize_rcu_full() or start_poll_synchronize_rcu_full() + * @gsp: value from get_state_synchronize_rcu_full() or start_poll_synchronize_rcu_full() * - * Places the normal and expedited grace-period states in *@rgos. This + * Places the normal and expedited grace-period states in *@gs. This * state value can be passed to a later call to cond_synchronize_rcu_full() * or poll_state_synchronize_rcu_full() to determine whether or not a * grace period (whether normal or expedited) has elapsed in the meantime. * If the needed grace period is not already slated to start, notifies * RCU core of the need for that grace period. */ -void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +void start_poll_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - get_state_synchronize_rcu_full(rgosp); + get_state_synchronize_rcu_full(gsp); start_poll_synchronize_rcu_common(); } @@ -3587,19 +3587,19 @@ EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu); /** * poll_state_synchronize_rcu_full - Has the specified RCU grace period completed? - * @rgosp: value from get_state_synchronize_rcu_full() or start_poll_synchronize_rcu_full() + * @gsp: value from get_state_synchronize_rcu_full() or start_poll_synchronize_rcu_full() * * If a full RCU grace period has elapsed since the earlier call from - * which *rgosp was obtained, return @true, otherwise return @false. + * which *gsp was obtained, return @true, otherwise return @false. * If @false is returned, it is the caller's responsibility to invoke this * function later on until it does return @true. Alternatively, the caller - * can explicitly wait for a grace period, for example, by passing @rgosp + * can explicitly wait for a grace period, for example, by passing @gsp * to cond_synchronize_rcu() or by directly invoking synchronize_rcu(). * * Yes, this function does not take counter wrap into account. * But counter wrap is harmless. If the counter wraps, we have waited * for more than a billion grace periods (and way more on a 64-bit - * system!). Those needing to keep rcu_gp_oldstate values for very + * system!). Those needing to keep rcu_gp_seq values for very * long time periods (many hours even on 32-bit systems) should check * them occasionally and either refresh them or set a flag indicating * that the grace period has completed. Alternatively, they can use @@ -3608,7 +3608,7 @@ EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu); * * This function provides the same memory-ordering guarantees that would * be provided by a synchronize_rcu() that was invoked at the call to - * the function that provided @rgosp, and that returned at the end of this + * the function that provided @gsp, and that returned at the end of this * function. And this guarantee requires that the root rcu_node structure's * ->gp_seq field be checked instead of that of the rcu_state structure. * The problem is that the just-ending grace-period's callbacks can be @@ -3618,15 +3618,15 @@ EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu); * cause a subsequent poll_state_synchronize_rcu_full() to return @true, * then the root rcu_node structure is the one that needs to be polled. */ -bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +bool poll_state_synchronize_rcu_full(struct rcu_gp_seq *gsp) { struct rcu_node *rnp = rcu_get_root(); smp_mb(); // Order against root rcu_node structure grace-period cleanup. - if (rgosp->rgos_norm == RCU_GET_STATE_COMPLETED || - rcu_seq_done_exact(&rnp->gp_seq, rgosp->rgos_norm) || - rgosp->rgos_exp == RCU_GET_STATE_COMPLETED || - rcu_seq_done_exact(&rcu_state.expedited_sequence, rgosp->rgos_exp)) { + if (gsp->norm == RCU_GET_STATE_COMPLETED || + rcu_seq_done_exact(&rnp->gp_seq, gsp->norm) || + gsp->exp == RCU_GET_STATE_COMPLETED || + rcu_seq_done_exact(&rcu_state.expedited_sequence, gsp->exp)) { smp_mb(); /* Ensure GP ends before subsequent accesses. */ return true; } @@ -3661,11 +3661,11 @@ EXPORT_SYMBOL_GPL(cond_synchronize_rcu); /** * cond_synchronize_rcu_full - Conditionally wait for an RCU grace period - * @rgosp: value from get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), or start_poll_synchronize_rcu_expedited_full() + * @gsp: value from get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), or start_poll_synchronize_rcu_expedited_full() * * If a full RCU grace period has elapsed since the call to * get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), - * or start_poll_synchronize_rcu_expedited_full() from which @rgosp was + * or start_poll_synchronize_rcu_expedited_full() from which @gsp was * obtained, just return. Otherwise, invoke synchronize_rcu() to wait * for a full grace period. * @@ -3676,12 +3676,12 @@ EXPORT_SYMBOL_GPL(cond_synchronize_rcu); * * This function provides the same memory-ordering guarantees that * would be provided by a synchronize_rcu() that was invoked at the call - * to the function that provided @rgosp and that returned at the end of + * to the function that provided @gsp and that returned at the end of * this function. */ -void cond_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +void cond_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - if (!poll_state_synchronize_rcu_full(rgosp)) + if (!poll_state_synchronize_rcu_full(gsp)) synchronize_rcu(); } EXPORT_SYMBOL_GPL(cond_synchronize_rcu_full); diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index 82cada459e5d..7024131574df 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h @@ -1047,18 +1047,18 @@ EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu_expedited); /** * start_poll_synchronize_rcu_expedited_full - Take a full snapshot and start expedited grace period - * @rgosp: Place to put snapshot of grace-period state + * @gsp: Place to put snapshot of grace-period state * - * Places the normal and expedited grace-period states in rgosp. This + * Places the normal and expedited grace-period states in gsp. This * state value can be passed to a later call to cond_synchronize_rcu_full() * or poll_state_synchronize_rcu_full() to determine whether or not a * grace period (whether normal or expedited) has elapsed in the meantime. * If the needed expedited grace period is not already slated to start, * initiates that grace period. */ -void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp) +void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_seq *gsp) { - get_state_synchronize_rcu_full(rgosp); + get_state_synchronize_rcu_full(gsp); (void)start_poll_synchronize_rcu_expedited(); } EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu_expedited_full); @@ -1092,11 +1092,11 @@ EXPORT_SYMBOL_GPL(cond_synchronize_rcu_expedited); /** * cond_synchronize_rcu_expedited_full - Conditionally wait for an expedited RCU grace period - * @rgosp: value from get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), or start_poll_synchronize_rcu_expedited_full() + * @gsp: value from get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), or start_poll_synchronize_rcu_expedited_full() * * If a full RCU grace period has elapsed since the call to * get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), - * or start_poll_synchronize_rcu_expedited_full() from which @rgosp was + * or start_poll_synchronize_rcu_expedited_full() from which @gsp was * obtained, just return. Otherwise, invoke synchronize_rcu_expedited() * to wait for a full grace period. * @@ -1107,12 +1107,12 @@ EXPORT_SYMBOL_GPL(cond_synchronize_rcu_expedited); * * This function provides the same memory-ordering guarantees that * would be provided by a synchronize_rcu() that was invoked at the call - * to the function that provided @rgosp and that returned at the end of + * to the function that provided @gsp and that returned at the end of * this function. */ -void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp) +void cond_synchronize_rcu_expedited_full(struct rcu_gp_seq *gsp) { - if (!poll_state_synchronize_rcu_full(rgosp)) + if (!poll_state_synchronize_rcu_full(gsp)) synchronize_rcu_expedited(); } EXPORT_SYMBOL_GPL(cond_synchronize_rcu_expedited_full); diff --git a/mm/slab_common.c b/mm/slab_common.c index b6426d7ceec9..d34743d1f3cb 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -1322,7 +1322,7 @@ static struct workqueue_struct *rcu_reclaim_wq; */ struct kvfree_rcu_bulk_data { struct list_head list; - struct rcu_gp_oldstate gp_snap; + struct rcu_gp_seq gp_snap; unsigned long nr_records; void *records[] __counted_by(nr_records); }; @@ -1347,7 +1347,7 @@ struct kvfree_rcu_bulk_data { struct kfree_rcu_cpu_work { struct rcu_work rcu_work; struct rcu_head *head_free; - struct rcu_gp_oldstate head_free_gp_snap; + struct rcu_gp_seq head_free_gp_snap; struct list_head bulk_head_free[FREE_N_CHANNELS]; struct kfree_rcu_cpu *krcp; }; @@ -1555,7 +1555,7 @@ static void kfree_rcu_work(struct work_struct *work) struct rcu_head *head; struct kfree_rcu_cpu *krcp; struct kfree_rcu_cpu_work *krwp; - struct rcu_gp_oldstate head_gp_snap; + struct rcu_gp_seq head_gp_snap; int i; krwp = container_of(to_rcu_work(work), From ca1f962a82692aecb06e92c2609200d918d57dbd Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Wed, 24 Jun 2026 06:23:44 -0700 Subject: [PATCH 10/59] rcu/segcblist: Add SRCU and Tasks RCU wrapper functions Add srcu_segcblist_advance() and srcu_segcblist_accelerate() wrappers that forward to the core rcu_segcblist_advance() and rcu_segcblist_accelerate() functions, and switch all SRCU (srcutree.c) and Tasks RCU (tasks.h) callers to use these wrappers. This isolates SRCU and Tasks RCU from upcoming changes to the core advance/accelerate functions, which will switch to struct rcu_gp_seq for dual normal/expedited GP tracking. Because SRCU and Tasks RCU use only normal GP sequences, their wrappers will maintain the existing unsigned long interface. No functional change. Reviewed-by: Paul E. McKenney Signed-off-by: Puranjay Mohan Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- kernel/rcu/rcu_segcblist.c | 10 ++++++++++ kernel/rcu/rcu_segcblist.h | 2 ++ kernel/rcu/srcutree.c | 14 +++++++------- kernel/rcu/tasks.h | 8 ++++---- 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/kernel/rcu/rcu_segcblist.c b/kernel/rcu/rcu_segcblist.c index 298a2c573f02..da39d818b01b 100644 --- a/kernel/rcu/rcu_segcblist.c +++ b/kernel/rcu/rcu_segcblist.c @@ -620,3 +620,13 @@ void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp, rcu_segcblist_init(src_rsclp); } + +void srcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) +{ + rcu_segcblist_advance(rsclp, seq); +} + +bool srcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq) +{ + return rcu_segcblist_accelerate(rsclp, seq); +} diff --git a/kernel/rcu/rcu_segcblist.h b/kernel/rcu/rcu_segcblist.h index fadc08ad4b7b..956f2967d9d2 100644 --- a/kernel/rcu/rcu_segcblist.h +++ b/kernel/rcu/rcu_segcblist.h @@ -143,3 +143,5 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq); bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq); void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp, struct rcu_segcblist *src_rsclp); +void srcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq); +bool srcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq); diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 7c2f7cc131f7..519a35719c89 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -1351,7 +1351,7 @@ static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp, * 2) The grace period for RCU_WAIT_TAIL is seen as started but not * completed so rcu_seq_current() returns X + SRCU_STATE_SCAN1. * - * 3) This value is passed to rcu_segcblist_advance() which can't move + * 3) This value is passed to srcu_segcblist_advance() which can't move * any segment forward and fails. * * 4) srcu_gp_start_if_needed() still proceeds with callback acceleration. @@ -1360,15 +1360,15 @@ static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp, * RCU_NEXT_READY_TAIL segment as started (ie: X + 4 + SRCU_STATE_SCAN1) * so it returns a snapshot of the next grace period, which is X + 12. * - * 5) The value of X + 12 is passed to rcu_segcblist_accelerate() but the + * 5) The value of X + 12 is passed to srcu_segcblist_accelerate() but the * freshly enqueued callback in RCU_NEXT_TAIL can't move to * RCU_NEXT_READY_TAIL which already has callbacks for a previous grace * period (gp_num = X + 8). So acceleration fails. */ s = rcu_seq_snap(&ssp->srcu_sup->srcu_gp_seq); if (rhp) { - rcu_segcblist_advance(&sdp->srcu_cblist, - rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq)); + srcu_segcblist_advance(&sdp->srcu_cblist, + rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq)); /* * Acceleration can never fail because the base current gp_seq * used for acceleration is <= the value of gp_seq used for @@ -1376,7 +1376,7 @@ static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp, * always be able to be emptied by the acceleration into the * RCU_NEXT_READY_TAIL or RCU_WAIT_TAIL segments. */ - WARN_ON_ONCE(!rcu_segcblist_accelerate(&sdp->srcu_cblist, s)); + WARN_ON_ONCE(!srcu_segcblist_accelerate(&sdp->srcu_cblist, s)); } if (ULONG_CMP_LT(sdp->srcu_gp_seq_needed, s)) { sdp->srcu_gp_seq_needed = s; @@ -1891,8 +1891,8 @@ static void srcu_invoke_callbacks(struct work_struct *work) rcu_cblist_init(&ready_cbs); raw_spin_lock_irq_rcu_node(sdp); WARN_ON_ONCE(!rcu_segcblist_segempty(&sdp->srcu_cblist, RCU_NEXT_TAIL)); - rcu_segcblist_advance(&sdp->srcu_cblist, - rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq)); + srcu_segcblist_advance(&sdp->srcu_cblist, + rcu_seq_current(&ssp->srcu_sup->srcu_gp_seq)); /* * Although this function is theoretically re-entrant, concurrent * callbacks invocation is disallowed to avoid executing an SRCU barrier diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index f4da5fad70f5..92971499a12c 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -481,8 +481,8 @@ static int rcu_tasks_need_gpcb(struct rcu_tasks *rtp) if (cpu > 0) ncbsnz += n; } - rcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq)); - (void)rcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq)); + srcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq)); + (void)srcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq)); if (rtpcp->urgent_gp > 0 && rcu_segcblist_pend_cbs(&rtpcp->cblist)) { if (rtp->lazy_jiffies) rtpcp->urgent_gp--; @@ -565,7 +565,7 @@ static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu if (rcu_segcblist_empty(&rtpcp->cblist)) return; raw_spin_lock_irqsave_rcu_node(rtpcp, flags); - rcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq)); + srcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq)); rcu_segcblist_extract_done_cbs(&rtpcp->cblist, &rcl); raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); len = rcl.len; @@ -578,7 +578,7 @@ static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu } raw_spin_lock_irqsave_rcu_node(rtpcp, flags); rcu_segcblist_add_len(&rtpcp->cblist, -len); - (void)rcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq)); + (void)srcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq)); raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); } From 97b4c6b933f28a34f549c2e87852c83eede3ccc0 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Wed, 24 Jun 2026 06:23:45 -0700 Subject: [PATCH 11/59] rcu/segcblist: Factor out rcu_segcblist_advance_compact() helper This commit extracts the tail-pointer cleanup and segment compaction logic from rcu_segcblist_advance() into a new static helper function, rcu_segcblist_advance_compact(). This shared logic will be reused by the upcoming srcu_segcblist_advance() standalone implementation, which cannot call the core rcu_segcblist_advance() because that function will use RCU-specific globals. No functional change. Reviewed-by: Paul E. McKenney Signed-off-by: Puranjay Mohan Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- kernel/rcu/rcu_segcblist.c | 50 ++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/kernel/rcu/rcu_segcblist.c b/kernel/rcu/rcu_segcblist.c index da39d818b01b..421f1dadb5e5 100644 --- a/kernel/rcu/rcu_segcblist.c +++ b/kernel/rcu/rcu_segcblist.c @@ -462,13 +462,43 @@ void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp, WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], rclp->tail); } +/* + * Clean up and compact the segmented callback list after callbacks have been + * advanced to the RCU_DONE_TAIL segment. The @i parameter is the index of the + * first segment that was NOT advanced (i.e., the segment after the last one + * moved to RCU_DONE_TAIL). This function fixes up tail pointers and compacts + * any gaps left by the moved segments. + */ +static void rcu_segcblist_advance_compact(struct rcu_segcblist *rsclp, int i) +{ + int j; + + /* Clean up tail pointers that might have been misordered above. */ + for (j = RCU_WAIT_TAIL; j < i; j++) + WRITE_ONCE(rsclp->tails[j], rsclp->tails[RCU_DONE_TAIL]); + + /* + * Callbacks moved, so there might be an empty RCU_WAIT_TAIL + * and a non-empty RCU_NEXT_READY_TAIL. If so, copy the + * RCU_NEXT_READY_TAIL segment to fill the RCU_WAIT_TAIL gap + * created by the now-ready-to-invoke segments. + */ + for (j = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++, j++) { + if (rsclp->tails[j] == rsclp->tails[RCU_NEXT_TAIL]) + break; /* No more callbacks. */ + WRITE_ONCE(rsclp->tails[j], rsclp->tails[i]); + rcu_segcblist_move_seglen(rsclp, i, j); + rsclp->gp_seq[j] = rsclp->gp_seq[i]; + } +} + /* * Advance the callbacks in the specified rcu_segcblist structure based * on the current value passed in for the grace-period counter. */ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) { - int i, j; + int i; WARN_ON_ONCE(!rcu_segcblist_is_enabled(rsclp)); if (rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL)) @@ -489,23 +519,7 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) if (i == RCU_WAIT_TAIL) return; - /* Clean up tail pointers that might have been misordered above. */ - for (j = RCU_WAIT_TAIL; j < i; j++) - WRITE_ONCE(rsclp->tails[j], rsclp->tails[RCU_DONE_TAIL]); - - /* - * Callbacks moved, so there might be an empty RCU_WAIT_TAIL - * and a non-empty RCU_NEXT_READY_TAIL. If so, copy the - * RCU_NEXT_READY_TAIL segment to fill the RCU_WAIT_TAIL gap - * created by the now-ready-to-invoke segments. - */ - for (j = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++, j++) { - if (rsclp->tails[j] == rsclp->tails[RCU_NEXT_TAIL]) - break; /* No more callbacks. */ - WRITE_ONCE(rsclp->tails[j], rsclp->tails[i]); - rcu_segcblist_move_seglen(rsclp, i, j); - rsclp->gp_seq[j] = rsclp->gp_seq[i]; - } + rcu_segcblist_advance_compact(rsclp, i); } /* From fb5c815f12b5c17ed6c8c5879b3acb1407e5bac4 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Wed, 24 Jun 2026 06:23:46 -0700 Subject: [PATCH 12/59] rcu/segcblist: Track segment grace periods with struct rcu_gp_seq Change the type of the per-segment ->gp_seq[] array in struct rcu_segcblist from unsigned long to struct rcu_gp_seq. This prepares the callback tracking infrastructure to record both normal and expedited grace periods per segment. The rcu_segcblist_nextgp(), rcu_segcblist_advance(), and rcu_segcblist_accelerate() helpers now take a struct rcu_gp_seq * instead of an unsigned long, and all callers use the .norm field for comparisons and assignments. The SRCU and Tasks RCU wrappers construct a struct rcu_gp_seq with only .norm set and forward to the core helpers. No functional change: only the .norm field is used. Signed-off-by: Puranjay Mohan Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- include/linux/rcu_segcblist.h | 2 +- include/trace/events/rcu.h | 5 +++-- kernel/rcu/rcu_segcblist.c | 24 ++++++++++++++---------- kernel/rcu/rcu_segcblist.h | 6 +++--- kernel/rcu/tree.c | 25 ++++++++++++++----------- kernel/rcu/tree_nocb.h | 21 +++++++++++---------- 6 files changed, 46 insertions(+), 37 deletions(-) diff --git a/include/linux/rcu_segcblist.h b/include/linux/rcu_segcblist.h index 2fdc2208f1ca..137cc23b024c 100644 --- a/include/linux/rcu_segcblist.h +++ b/include/linux/rcu_segcblist.h @@ -190,7 +190,7 @@ struct rcu_cblist { struct rcu_segcblist { struct rcu_head *head; struct rcu_head **tails[RCU_CBLIST_NSEGS]; - unsigned long gp_seq[RCU_CBLIST_NSEGS]; + struct rcu_gp_seq gp_seq[RCU_CBLIST_NSEGS]; #ifdef CONFIG_RCU_NOCB_CPU atomic_long_t len; #else diff --git a/include/trace/events/rcu.h b/include/trace/events/rcu.h index 5fbdabe3faea..c84309c38834 100644 --- a/include/trace/events/rcu.h +++ b/include/trace/events/rcu.h @@ -547,10 +547,11 @@ TRACE_EVENT_RCU(rcu_segcb_stats, ), TP_fast_assign( + int i; __entry->ctx = ctx; memcpy(__entry->seglen, rs->seglen, RCU_CBLIST_NSEGS * sizeof(long)); - memcpy(__entry->gp_seq, rs->gp_seq, RCU_CBLIST_NSEGS * sizeof(unsigned long)); - + for (i = 0; i < RCU_CBLIST_NSEGS; i++) + __entry->gp_seq[i] = rs->gp_seq[i].norm; ), TP_printk("%s seglen: (DONE=%ld, WAIT=%ld, NEXT_READY=%ld, NEXT=%ld) " diff --git a/kernel/rcu/rcu_segcblist.c b/kernel/rcu/rcu_segcblist.c index 421f1dadb5e5..4e3dfe42bc09 100644 --- a/kernel/rcu/rcu_segcblist.c +++ b/kernel/rcu/rcu_segcblist.c @@ -307,13 +307,13 @@ struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp) /* * Return false if there are no CBs awaiting grace periods, otherwise, - * return true and store the nearest waited-upon grace period into *lp. + * return true and store the nearest waited-upon grace period state into *gsp. */ -bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp) +bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gsp) { if (!rcu_segcblist_pend_cbs(rsclp)) return false; - *lp = rsclp->gp_seq[RCU_WAIT_TAIL]; + *gsp = rsclp->gp_seq[RCU_WAIT_TAIL]; return true; } @@ -496,7 +496,7 @@ static void rcu_segcblist_advance_compact(struct rcu_segcblist *rsclp, int i) * Advance the callbacks in the specified rcu_segcblist structure based * on the current value passed in for the grace-period counter. */ -void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) +void rcu_segcblist_advance(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gsp) { int i; @@ -509,7 +509,7 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) * are ready to invoke, and put them into the RCU_DONE_TAIL segment. */ for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++) { - if (ULONG_CMP_LT(seq, rsclp->gp_seq[i])) + if (ULONG_CMP_LT(gsp->norm, rsclp->gp_seq[i].norm)) break; WRITE_ONCE(rsclp->tails[RCU_DONE_TAIL], rsclp->tails[i]); rcu_segcblist_move_seglen(rsclp, i, RCU_DONE_TAIL); @@ -537,7 +537,7 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) * ready to invoke. Returns true if there are callbacks that won't be * ready to invoke until seq, false otherwise. */ -bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq) +bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gsp) { int i, j; @@ -555,7 +555,7 @@ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq) */ for (i = RCU_NEXT_READY_TAIL; i > RCU_DONE_TAIL; i--) if (!rcu_segcblist_segempty(rsclp, i) && - ULONG_CMP_LT(rsclp->gp_seq[i], seq)) + ULONG_CMP_LT(rsclp->gp_seq[i].norm, gsp->norm)) break; /* @@ -595,7 +595,7 @@ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq) */ for (; i < RCU_NEXT_TAIL; i++) { WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_NEXT_TAIL]); - rsclp->gp_seq[i] = seq; + rsclp->gp_seq[i].norm = gsp->norm; } return true; } @@ -637,10 +637,14 @@ void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp, void srcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) { - rcu_segcblist_advance(rsclp, seq); + struct rcu_gp_seq gs = { .norm = seq }; + + rcu_segcblist_advance(rsclp, &gs); } bool srcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq) { - return rcu_segcblist_accelerate(rsclp, seq); + struct rcu_gp_seq gs = { .norm = seq }; + + return rcu_segcblist_accelerate(rsclp, &gs); } diff --git a/kernel/rcu/rcu_segcblist.h b/kernel/rcu/rcu_segcblist.h index 956f2967d9d2..16b0cb6b3250 100644 --- a/kernel/rcu/rcu_segcblist.h +++ b/kernel/rcu/rcu_segcblist.h @@ -124,7 +124,7 @@ bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp); bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp); struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp); struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp); -bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp); +bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gsp); void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp, struct rcu_head *rhp); bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp, @@ -139,8 +139,8 @@ void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp, struct rcu_cblist *rclp); void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp, struct rcu_cblist *rclp); -void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq); -bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq); +void rcu_segcblist_advance(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gsp); +bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gsp); void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp, struct rcu_segcblist *src_rsclp); void srcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq); diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 0d2d877eb945..9ca811e534b5 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -1137,7 +1137,7 @@ static void rcu_gp_kthread_wake(void) */ static bool rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp) { - unsigned long gp_seq_req; + struct rcu_gp_seq gs; bool ret = false; rcu_lockdep_assert_cblist_protected(rdp); @@ -1159,15 +1159,15 @@ static bool rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp) * accelerating callback invocation to an earlier grace-period * number. */ - gp_seq_req = rcu_seq_snap(&rcu_state.gp_seq); - if (rcu_segcblist_accelerate(&rdp->cblist, gp_seq_req)) - ret = rcu_start_this_gp(rnp, rdp, gp_seq_req); + gs.norm = rcu_seq_snap(&rcu_state.gp_seq); + if (rcu_segcblist_accelerate(&rdp->cblist, &gs)) + ret = rcu_start_this_gp(rnp, rdp, gs.norm); /* Trace depending on how much we were able to accelerate. */ if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL)) - trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccWaitCB")); + trace_rcu_grace_period(rcu_state.name, gs.norm, TPS("AccWaitCB")); else - trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccReadyCB")); + trace_rcu_grace_period(rcu_state.name, gs.norm, TPS("AccReadyCB")); trace_rcu_segcb_stats(&rdp->cblist, TPS("SegCbPostAcc")); @@ -1184,14 +1184,14 @@ static bool rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp) static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp, struct rcu_data *rdp) { - unsigned long c; + struct rcu_gp_seq gs; bool needwake; rcu_lockdep_assert_cblist_protected(rdp); - c = rcu_seq_snap(&rcu_state.gp_seq); - if (!READ_ONCE(rdp->gpwrap) && ULONG_CMP_GE(rdp->gp_seq_needed, c)) { + gs.norm = rcu_seq_snap(&rcu_state.gp_seq); + if (!READ_ONCE(rdp->gpwrap) && ULONG_CMP_GE(rdp->gp_seq_needed, gs.norm)) { /* Old request still live, so mark recent callbacks. */ - (void)rcu_segcblist_accelerate(&rdp->cblist, c); + (void)rcu_segcblist_accelerate(&rdp->cblist, &gs); return; } raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */ @@ -1213,6 +1213,8 @@ static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp, */ static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp) { + struct rcu_gp_seq gs; + rcu_lockdep_assert_cblist_protected(rdp); raw_lockdep_assert_held_rcu_node(rnp); @@ -1224,7 +1226,8 @@ static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp) * Find all callbacks whose ->gp_seq numbers indicate that they * are ready to invoke, and put them into the RCU_DONE_TAIL sublist. */ - rcu_segcblist_advance(&rdp->cblist, rnp->gp_seq); + gs.norm = rnp->gp_seq; + rcu_segcblist_advance(&rdp->cblist, &gs); /* Classify any remaining callbacks. */ return rcu_accelerate_cbs(rnp, rdp); diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index c4a5b4662b3a..eaa1ad5297e2 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -433,7 +433,7 @@ static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, bool lazy) { unsigned long c; - unsigned long cur_gp_seq; + struct rcu_gp_seq cur_gp_seq; unsigned long j = jiffies; long ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass); long lazy_len = READ_ONCE(rdp->lazy_len); @@ -502,7 +502,7 @@ static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, } if (j != rdp->nocb_gp_adv_time && rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && - rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) { + rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq.norm)) { rcu_advance_cbs_nowake(rdp->mynode, rdp); rdp->nocb_gp_adv_time = j; } @@ -659,7 +659,7 @@ static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp) { bool bypass = false; int __maybe_unused cpu = my_rdp->cpu; - unsigned long cur_gp_seq; + struct rcu_gp_seq cur_gp_seq; unsigned long flags; bool gotcbs = false; unsigned long j = jiffies; @@ -731,7 +731,7 @@ static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp) if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL) || (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && - rcu_seq_done(&rnp->gp_seq, cur_gp_seq))) { + rcu_seq_done(&rnp->gp_seq, cur_gp_seq.norm))) { raw_spin_lock_rcu_node(rnp); /* irqs disabled. */ needwake_gp = rcu_advance_cbs(rnp, rdp); wasempty = rcu_segcblist_restempty(&rdp->cblist, @@ -744,8 +744,8 @@ static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp) RCU_NEXT_READY_TAIL)); if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq)) { if (!needwait_gp || - ULONG_CMP_LT(cur_gp_seq, wait_gp_seq)) - wait_gp_seq = cur_gp_seq; + ULONG_CMP_LT(cur_gp_seq.norm, wait_gp_seq)) + wait_gp_seq = cur_gp_seq.norm; needwait_gp = true; trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("NeedWaitGP")); @@ -877,7 +877,7 @@ static inline bool nocb_cb_wait_cond(struct rcu_data *rdp) static void nocb_cb_wait(struct rcu_data *rdp) { struct rcu_segcblist *cblist = &rdp->cblist; - unsigned long cur_gp_seq; + struct rcu_gp_seq cur_gp_seq; unsigned long flags; bool needwake_gp = false; struct rcu_node *rnp = rdp->mynode; @@ -919,7 +919,7 @@ static void nocb_cb_wait(struct rcu_data *rdp) lockdep_assert_irqs_enabled(); rcu_nocb_lock_irqsave(rdp, flags); if (rcu_segcblist_nextgp(cblist, &cur_gp_seq) && - rcu_seq_done(&rnp->gp_seq, cur_gp_seq) && + rcu_seq_done(&rnp->gp_seq, cur_gp_seq.norm) && raw_spin_trylock_rcu_node(rnp)) { /* irqs already disabled. */ needwake_gp = rcu_advance_cbs(rdp->mynode, rdp); raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ @@ -1569,9 +1569,10 @@ static void show_rcu_nocb_state(struct rcu_data *rdp) nocb_entry_rdp); sprintf(bufd, "%ld", rsclp->seglen[RCU_DONE_TAIL]); - sprintf(bufw, "%ld(%ld)", rsclp->seglen[RCU_WAIT_TAIL], rsclp->gp_seq[RCU_WAIT_TAIL]); + sprintf(bufw, "%ld(%ld)", rsclp->seglen[RCU_WAIT_TAIL], + rsclp->gp_seq[RCU_WAIT_TAIL].norm); sprintf(bufr, "%ld(%ld)", rsclp->seglen[RCU_NEXT_READY_TAIL], - rsclp->gp_seq[RCU_NEXT_READY_TAIL]); + rsclp->gp_seq[RCU_NEXT_READY_TAIL].norm); sprintf(bufn, "%ld", rsclp->seglen[RCU_NEXT_TAIL]); sprintf(bufb, "%ld", rcu_cblist_n_cbs(&rdp->nocb_bypass)); pr_info(" CB %d^%d->%d %c%c%c%c%c F%ld L%ld C%d %c%s%c%s%c%s%c%s%c%s q%ld %c CPU %d%s\n", From 60b54875bd0ba87a7d39204e0cd494a29d22e72f Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Wed, 24 Jun 2026 06:23:47 -0700 Subject: [PATCH 13/59] rcu: Add RCU_GET_STATE_NOT_TRACKED for subsystems without expedited GPs SRCU and Tasks RCU do not track expedited grace periods. When their callback state is checked via poll_state_synchronize_rcu_full(), the uninitialized or zeroed exp field could cause false-positive completion detection. This commit adds an RCU_GET_STATE_NOT_TRACKED sentinel value (0x2) that these subsystems can place into exp to indicate that expedited GP tracking is not applicable. The expedited sequence check in poll_state_synchronize_rcu_full() is guarded to skip entries marked with this sentinel. This is needed to allow rcu_segcblist_advance() and rcu_accelerate_cbs() to work with both normal and expedited grace periods via get_state_synchronize_rcu_full() and poll_state_synchronize_rcu_full(). Reviewed-by: Paul E. McKenney Signed-off-by: Puranjay Mohan Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- kernel/rcu/rcu.h | 13 +++++++++++-- kernel/rcu/tree.c | 3 ++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index fa6d30ce73d1..0507bb7dac12 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h @@ -46,16 +46,25 @@ * the number of pending readers that will use * this inactive index is bounded). * - * RCU polled GP special control value: + * RCU polled GP special control values: * * RCU_GET_STATE_COMPLETED : State value indicating an already-completed * polled GP has completed. This value covers * both the state and the counter of the * grace-period sequence number. + * + * RCU_GET_STATE_NOT_TRACKED : State value indicating that a GP component + * is not tracked by this subsystem and should + * not be checked. Used by SRCU and RCU Tasks + * which do not track expedited GPs, to prevent + * false-positive completion when their + * gp_seq entries are checked via + * poll_state_synchronize_rcu_full(). */ -/* Low-order bit definition for polled grace-period APIs. */ +/* Low-order bit definitions for polled grace-period APIs. */ #define RCU_GET_STATE_COMPLETED 0x1 +#define RCU_GET_STATE_NOT_TRACKED 0x2 /* A complete grace period count */ #define RCU_SEQ_GP (RCU_SEQ_STATE_MASK + 1) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 9ca811e534b5..640ebe055a24 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -3629,7 +3629,8 @@ bool poll_state_synchronize_rcu_full(struct rcu_gp_seq *gsp) if (gsp->norm == RCU_GET_STATE_COMPLETED || rcu_seq_done_exact(&rnp->gp_seq, gsp->norm) || gsp->exp == RCU_GET_STATE_COMPLETED || - rcu_seq_done_exact(&rcu_state.expedited_sequence, gsp->exp)) { + (gsp->exp != RCU_GET_STATE_NOT_TRACKED && + rcu_seq_done_exact(&rcu_state.expedited_sequence, gsp->exp))) { smp_mb(); /* Ensure GP ends before subsequent accesses. */ return true; } From 1384ea908e5de451061b50b4eb76c4440c4d4a0d Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Fri, 12 Jun 2026 10:34:12 +0800 Subject: [PATCH 14/59] rcu: Use task_state_to_char() in stall-warning prints RCU stall warnings currently print task states as raw hexadecimal values, requiring developers to manually decode them. Use task_state_to_char() so that stall warnings show the same symbolic task-state representation used elsewhere in the kernel. For example: ->state=0x402 becomes ->state=I ->state=0x0 becomes ->state=R ->state=0x2 becomes ->state=D This improves readability while preserving the underlying diagnostic information. Suggested-by: Zqiang Co-developed-by: Wang Lian Signed-off-by: Wang Lian Signed-off-by: Kunwu Chan Signed-off-by: Paul E. McKenney --- kernel/rcu/tree_stall.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index cf7ae51cba40..45b9856ccd2b 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -573,13 +573,13 @@ static void rcu_check_gp_kthread_starvation(void) if (rcu_is_gp_kthread_starving(&j)) { cpu = gpk ? task_cpu(gpk) : -1; - pr_err("%s kthread starved for %ld jiffies! g%ld f%#x %s(%d) ->state=%#x ->cpu=%d\n", + pr_err("%s kthread starved for %ld jiffies! g%ld f%#x %s(%d) ->state=%c ->cpu=%d\n", rcu_state.name, j, (long)rcu_seq_current(&rcu_state.gp_seq), data_race(READ_ONCE(rcu_state.gp_flags)), gp_state_getname(rcu_state.gp_state), data_race(READ_ONCE(rcu_state.gp_state)), - gpk ? data_race(READ_ONCE(gpk->__state)) : ~0, cpu); + gpk ? task_state_to_char(gpk) : '?', cpu); if (gpk) { struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); @@ -616,12 +616,12 @@ static void rcu_check_gp_kthread_expired_fqs_timer(void) time_after(jiffies, jiffies_fqs + RCU_STALL_MIGHT_MIN) && gpk && !READ_ONCE(gpk->on_rq)) { cpu = task_cpu(gpk); - pr_err("%s kthread timer wakeup didn't happen for %ld jiffies! g%ld f%#x %s(%d) ->state=%#x\n", + pr_err("%s kthread timer wakeup didn't happen for %ld jiffies! g%ld f%#x %s(%d) ->state=%c\n", rcu_state.name, (jiffies - jiffies_fqs), (long)rcu_seq_current(&rcu_state.gp_seq), data_race(READ_ONCE(rcu_state.gp_flags)), // Diagnostic read gp_state_getname(RCU_GP_WAIT_FQS), RCU_GP_WAIT_FQS, - data_race(READ_ONCE(gpk->__state))); + task_state_to_char(gpk)); pr_err("\tPossible timer handling issue on cpu=%d timer-softirq=%u\n", cpu, kstat_softirqs_cpu(TIMER_SOFTIRQ, cpu)); } @@ -948,10 +948,10 @@ void show_rcu_gp_kthreads(void) jr = j - data_race(READ_ONCE(rcu_state.gp_req_activity)); js = j - data_race(READ_ONCE(rcu_state.gp_start)); jw = j - data_race(READ_ONCE(rcu_state.gp_wake_time)); - pr_info("%s: wait state: %s(%d) ->state: %#x ->rt_priority %u delta ->gp_start %lu ->gp_activity %lu ->gp_req_activity %lu ->gp_wake_time %lu ->gp_wake_seq %ld ->gp_seq %ld ->gp_seq_needed %ld ->gp_max %lu ->gp_flags %#x\n", + pr_info("%s: wait state: %s(%d) ->state: %c ->rt_priority %u delta ->gp_start %lu ->gp_activity %lu ->gp_req_activity %lu ->gp_wake_time %lu ->gp_wake_seq %ld ->gp_seq %ld ->gp_seq_needed %ld ->gp_max %lu ->gp_flags %#x\n", rcu_state.name, gp_state_getname(rcu_state.gp_state), data_race(READ_ONCE(rcu_state.gp_state)), - t ? data_race(READ_ONCE(t->__state)) : 0x1ffff, t ? t->rt_priority : 0xffU, + t ? task_state_to_char(t) : '?', t ? t->rt_priority : 0xffU, js, ja, jr, jw, (long)data_race(READ_ONCE(rcu_state.gp_wake_seq)), (long)data_race(READ_ONCE(rcu_state.gp_seq)), (long)data_race(READ_ONCE(rcu_get_root()->gp_seq_needed)), From 573d76f7db5ba050bc7c7c73d30e1364fcee269b Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Sun, 28 Jun 2026 09:00:01 -0700 Subject: [PATCH 15/59] rcu: Mark __rcu_access_pointer() as context_unsafe() A simple comparison of a pointer returned by rcu_access_pointer() results in a context-analysis warning for lockless inspection of the RCU-protected (also known as __rcu-protected) pointer. This can be suppressed by placing context_unsafe() calls around calls rcu_access_pointer(), but this is messy and distracting. This commit therefore wraps the underlying __rcu_access_pointer() macro with a call to context_unsafe(), thereby informing the context-analysis code that rcu_access_pointer() may safely be invoked outside of an RCU read-side critical section. Reported-by: Christoph Hellwig Suggested-by: Marco Elver Signed-off-by: Paul E. McKenney Tested-by: Nilay Shroff Reviewed-by: Marco Elver --- include/linux/rcupdate.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 5e95acc33989..e40dc2e20c5b 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -490,12 +490,12 @@ context_unsafe( \ */ #define unrcu_pointer(p) __unrcu_pointer(p, __UNIQUE_ID(rcu)) -#define __rcu_access_pointer(p, local, space) \ +#define __rcu_access_pointer(p, local, space) context_unsafe( \ ({ \ typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \ rcu_check_sparse(p, space); \ ((typeof(*p) __force __kernel *)(local)); \ -}) +}) ) #define __rcu_dereference_check(p, local, c, space) \ ({ \ /* Dependency order vs. p above. */ \ From dc7e119abd58297408d2ca34d216242715eda7b7 Mon Sep 17 00:00:00 2001 From: Manuel Ebner Date: Sat, 27 Jun 2026 11:06:06 +0200 Subject: [PATCH 16/59] doc: RCU: Adopt new coding style of type-aware kmalloc-family - part 2/2 Update Documentation/RCU/* to suggest using the new type-aware kmalloc_obj() per commit 2932ba8d9c99 ("slab: Introduce kmalloc_obj() and family") p = kmalloc(...); -> p = kmalloc_obj(...); Signed-off-by: Manuel Ebner Signed-off-by: Paul E. McKenney --- Documentation/RCU/rcu_dereference.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/RCU/rcu_dereference.rst b/Documentation/RCU/rcu_dereference.rst index 2524dcdadde2..5bc3785ebfc2 100644 --- a/Documentation/RCU/rcu_dereference.rst +++ b/Documentation/RCU/rcu_dereference.rst @@ -236,7 +236,7 @@ precautions. To see this, consider the following code fragment:: { struct foo *p; - p = kmalloc(...); + p = kmalloc_obj(*p); if (p == NULL) deal_with_it(); p->a = 42; /* Each field in its own cache line. */ @@ -293,7 +293,7 @@ Then one approach is to use locking, for example, as follows:: { struct foo *p; - p = kmalloc(...); + p = kmalloc_obj(*p); if (p == NULL) deal_with_it(); spin_lock(&p->lock); From 2b109eccd80b660e58d4523e8b80ce11bc3e532e Mon Sep 17 00:00:00 2001 From: Manuel Ebner Date: Sat, 27 Jun 2026 11:26:45 +0200 Subject: [PATCH 17/59] doc: RCU: Fix brackets Remove needless brackets and add missing brackets. Signed-off-by: Manuel Ebner Signed-off-by: Paul E. McKenney --- .../Design/Expedited-Grace-Periods/Expedited-Grace-Periods.rst | 2 +- Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp.svg | 2 +- Documentation/RCU/Design/Memory-Ordering/TreeRCU-qs.svg | 2 +- Documentation/RCU/Design/Requirements/Requirements.rst | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/RCU/Design/Expedited-Grace-Periods/Expedited-Grace-Periods.rst b/Documentation/RCU/Design/Expedited-Grace-Periods/Expedited-Grace-Periods.rst index 414f8a2012d6..cf0f9cdca7e8 100644 --- a/Documentation/RCU/Design/Expedited-Grace-Periods/Expedited-Grace-Periods.rst +++ b/Documentation/RCU/Design/Expedited-Grace-Periods/Expedited-Grace-Periods.rst @@ -410,7 +410,7 @@ workqueues (see Documentation/core-api/workqueue.rst). The requesting task still does counter snapshotting and funnel-lock processing, but the task reaching the top of the funnel lock does a -``schedule_work()`` (from ``_synchronize_rcu_expedited()`` so that a +``schedule_work()`` (from ``_synchronize_rcu_expedited()``) so that a workqueue kthread does the actual grace-period processing. Because workqueue kthreads do not accept POSIX signals, grace-period-wait processing need not allow for POSIX signals. In addition, this approach diff --git a/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp.svg b/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp.svg index d05bc7b27edb..95a66de40ca5 100644 --- a/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp.svg +++ b/Documentation/RCU/Design/Memory-Ordering/TreeRCU-gp.svg @@ -3933,7 +3933,7 @@ font-style="normal" y="-3914.085" x="3745.7725" - xml:space="preserve">rcu__report_qs_rdp()) + xml:space="preserve">rcu__report_qs_rdp() rcu__report_qs_rdp()) + xml:space="preserve">rcu__report_qs_rdp() Date: Thu, 25 Jun 2026 20:42:54 -0400 Subject: [PATCH 18/59] rcu: introduce rcu_defer_qs_clear() helper Currently rdp->defer_qs_pending transitions from DEFER_QS_PENDING to DEFER_QS_IDLE at two sites: rcu_preempt_deferred_qs_irqrestore() and rcu_preempt_deferred_qs_handler() (depth>0 reset). Both write the IDLE value directly. Introduce a single inline helper rcu_defer_qs_clear() in tree.h and route both sites through it. This becomes the single PENDING->IDLE transition point for upcoming work. Signed-off-by: Joel Fernandes Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.h | 5 +++++ kernel/rcu/tree_plugin.h | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 7dfc57e9adb1..4069132f9d44 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -296,6 +296,11 @@ struct rcu_data { int cpu; }; +static inline void rcu_defer_qs_clear(struct rcu_data *rdp) +{ + WRITE_ONCE(rdp->defer_qs_pending, DEFER_QS_IDLE); +} + /* Values for nocb_defer_wakeup field in struct rcu_data. */ #define RCU_NOCB_WAKE_NOT 0 #define RCU_NOCB_WAKE_BYPASS 1 diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 95ad967adcf3..8637f405cb47 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -488,7 +488,7 @@ rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags) rdp = this_cpu_ptr(&rcu_data); if (rdp->defer_qs_pending == DEFER_QS_PENDING) - rdp->defer_qs_pending = DEFER_QS_IDLE; + rcu_defer_qs_clear(rdp); /* * If RCU core is waiting for this CPU to exit its critical section, @@ -645,7 +645,7 @@ static void rcu_preempt_deferred_qs_handler(struct irq_work *iwp) * 5. Deferred QS reporting does not happen. */ if (rcu_preempt_depth() > 0) - WRITE_ONCE(rdp->defer_qs_pending, DEFER_QS_IDLE); + rcu_defer_qs_clear(rdp); } /* From d972d106965f16348f2c44035440f4926a5880d3 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Thu, 25 Jun 2026 20:43:00 -0400 Subject: [PATCH 19/59] rcu: clear defer_qs_pending in deferred-QS bail when nesting > 0 Paul McKenney noted that a softirq (or irq_work) handler arming for a deferred QS can fire and find rcu_preempt_depth() > 0 -- the task is still inside its outer reader, so rcu_preempt_need_deferred_qs() bails without reporting the QS. At that point the queued mechanism has been consumed but ->defer_qs_pending stays in DEFER_QS_PENDING. In the meantime, the only remaining path back to a quiescent state on this CPU may be a local_irq_disable()/_enable() pair that does not call preempt_check_resched() (it is just `sti`/`cli`). patch 6's unconditional set_need_resched_current() makes need_resched true, but without an irq_work being raised the next outer rcu_read_unlock_special() hits the P-gate at the arming code: if (rdp->defer_qs_pending != DEFER_QS_PENDING) { rdp->defer_qs_pending = DEFER_QS_PENDING; irq_work_queue_on(...); // <-- skipped } so no irq_work is queued for the hardirq-exit preempt_schedule_irq() path either. The deferred QS now waits until the next timer tick (or similar preempt-safe boundary), needlessly extending expedited grace period latency. Clear ->defer_qs_pending in the bail-out path of rcu_preempt_deferred_qs() when rcu_preempt_depth() > 0. The recursion guard semantics introduced by commit b41642c87716 ("rcu: Fix rcu_read_unlock() deadloop due to IRQ work"). The clear is also safe against fresh recursion at this exact program point: rcu_preempt_depth() > 0 guarantees we are still inside an outer reader, so any inner rcu_read_unlock() from tracing infrastructure brings nesting back to outer (>0), never to 0. The slow path of rcu_read_unlock_special() is structurally unreachable under that condition, so no recursive raise_softirq_irqoff()/irq_work_queue_on() can be triggered by the clear. Essentially, the mechanism will work to prevent the following recursion which Xiongfeng had previously reported: irq_exit() -> __irq_exit_rcu() -> tick_irq_exit() -> tick_nohz_irq_exit() -> tick_nohz_stop_sched_tick() -> trace_tick_stop() // BPF prog hooked here -> rcu_read_unlock_special() -> irq_work_queue_on(&rdp->defer_qs_iw, rdp->cpu) // self-IPI re-enters irq_exit Reported-by: Paul E. McKenney Signed-off-by: Joel Fernandes Signed-off-by: Paul E. McKenney --- kernel/rcu/tree_plugin.h | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 8637f405cb47..9ba136a4233a 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -614,9 +614,35 @@ static notrace bool rcu_preempt_need_deferred_qs(struct task_struct *t) notrace void rcu_preempt_deferred_qs(struct task_struct *t) { unsigned long flags; + struct rcu_data *rdp; - if (!rcu_preempt_need_deferred_qs(t)) + if (!rcu_preempt_need_deferred_qs(t)) { + /* + * If we got here from a softirq/irq_work that fired while + * rcu_preempt_depth() > 0, the deferred-QS mechanism has been + * consumed without doing any work: rcu_preempt_need_deferred_qs() + * just returned false because the task is still in a reader, so + * the actual QS report has to wait for the next + * rcu_read_unlock(). + * + * Clear ->defer_qs_pending here so the next outer + * rcu_read_unlock_special() can re-arm a fresh mechanism (in + * particular the irq_work path, which the local_irq_enable() + * recovery boundary cannot itself reschedule from). + * + * Recursion safety: rcu_preempt_depth() > 0 means we are inside + * an outer reader, so any inner rcu_read_unlock() reached via + * tracing (bpf programs attached to trace points) brings + * nesting to outer (> 0), never to 0, so no recursive + * raise_softirq_irqoff()/irq_work_queue_on() can be triggered + * by this clear. + */ + if (rcu_preempt_depth() > 0) { + rdp = this_cpu_ptr(&rcu_data); + rcu_defer_qs_clear(rdp); + } return; + } local_irq_save(flags); rcu_preempt_deferred_qs_irqrestore(t, flags); } From 5b79f1d5d791bb41c66cbb75387a53c19f1ffe1e Mon Sep 17 00:00:00 2001 From: Zqiang Date: Mon, 6 Jul 2026 18:03:09 +0800 Subject: [PATCH 20/59] rcu-tasks: Remove unused struct rcu_tasks's->n_ipis_fails variables The RCU tasks trace has been reimplemented by 'commit c27cea4416a3 ("rcu: Re-implement RCU Tasks Trace in terms of SRCU-fast")', the rcu_tasks structure's->n_ipis_fails is no longer used, this commit therefore remove it. Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index f4da5fad70f5..7f7d1e636217 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -67,7 +67,6 @@ struct rcu_tasks_percpu { * @gp_start: Most recent grace-period start in jiffies. * @tasks_gp_seq: Number of grace periods completed since boot in upper bits. * @n_ipis: Number of IPIs sent to encourage grace periods to end. - * @n_ipis_fails: Number of IPI-send failures. * @kthread_ptr: This flavor's grace-period/callback-invocation kthread. * @lazy_jiffies: Number of jiffies to allow callbacks to be lazy. * @pregp_func: This flavor's pre-grace-period function (optional). @@ -102,7 +101,6 @@ struct rcu_tasks { unsigned long gp_start; unsigned long tasks_gp_seq; unsigned long n_ipis; - unsigned long n_ipis_fails; struct task_struct *kthread_ptr; unsigned long lazy_jiffies; rcu_tasks_gp_func_t gp_func; @@ -735,12 +733,12 @@ static void show_rcu_tasks_generic_gp_kthread(struct rcu_tasks *rtp, char *s) if (havecbs && haveurgent && haveurgentcbs) break; } - pr_info("%s: %s(%d) since %lu g:%lu i:%lu/%lu %c%c%c%c l:%lu %s\n", + pr_info("%s: %s(%d) since %lu g:%lu i:%lu %c%c%c%c l:%lu %s\n", rtp->kname, tasks_gp_state_getname(rtp), data_race(rtp->gp_state), jiffies - data_race(rtp->gp_jiffies), data_race(rcu_seq_current(&rtp->tasks_gp_seq)), - data_race(rtp->n_ipis_fails), data_race(rtp->n_ipis), + data_race(rtp->n_ipis), ".k"[!!data_race(rtp->kthread_ptr)], ".C"[havecbs], ".u"[haveurgent], From ef21090071b33844f55fac7c5470729b862dad0c Mon Sep 17 00:00:00 2001 From: Zqiang Date: Mon, 6 Jul 2026 18:03:10 +0800 Subject: [PATCH 21/59] rcu-tasks: Dump rcu tasks status when the boot-test failed This commit adds a dump RCU task status function to rcu tasks tests, used to obtain more information to help debug when the RCU tasks boot-time tests failed. Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 7f7d1e636217..d9e1e53f4ccf 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1453,6 +1453,7 @@ struct rcu_tasks_test_desc { const char *name; bool notrun; unsigned long runstart; + void (*gp_dbg)(void); }; static struct rcu_tasks_test_desc tests[] = { @@ -1460,6 +1461,8 @@ static struct rcu_tasks_test_desc tests[] = { .name = "call_rcu_tasks()", /* If not defined, the test is skipped. */ .notrun = IS_ENABLED(CONFIG_TASKS_RCU), + /* Dump rcu tasks status, if test failed. */ + .gp_dbg = show_rcu_tasks_classic_gp_kthread }, { .name = "call_rcu_tasks_trace()", @@ -1519,6 +1522,8 @@ static int rcu_tasks_verify_self_tests(void) while (tests[i].notrun) { // still hanging. if (time_after(jiffies, tests[i].runstart + bst)) { pr_err("%s has failed boot-time tests.\n", tests[i].name); + if (tests[i].gp_dbg) + tests[i].gp_dbg(); ret = -1; break; } From 647dd6e09481daa23d93e927c2b7f46a3acf8a03 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Sun, 12 Jul 2026 10:50:08 -0700 Subject: [PATCH 22/59] rcu-tasks: Apply READ_ONCE() and WRITE_ONCE() to fix data race Now that rcutorture tests readers from interrupt handlers, KCSAN spotted an additional data race. This commit therefore fixes it by applying READ_ONCE() and WRITE_ONCE(). Signed-off-by: Paul E. McKenney --- include/linux/rcupdate_trace.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h index cee89e51e45c..fd3ddeb6aa3b 100644 --- a/include/linux/rcupdate_trace.h +++ b/include/linux/rcupdate_trace.h @@ -95,10 +95,13 @@ static inline void rcu_read_unlock_tasks_trace(struct srcu_ctr __percpu *scp) */ static inline void rcu_read_lock_trace(void) { + int n; struct task_struct *t = current; rcu_try_lock_acquire(&rcu_tasks_trace_srcu_struct.dep_map); - if (t->trc_reader_nesting++) { + n = READ_ONCE(t->trc_reader_nesting); + WRITE_ONCE(t->trc_reader_nesting, n + 1); + if (n) { // In case we interrupted a Tasks Trace RCU reader. return; } @@ -119,12 +122,15 @@ static inline void rcu_read_lock_trace(void) */ static inline void rcu_read_unlock_trace(void) { + int n; struct srcu_ctr __percpu *scp; struct task_struct *t = current; scp = t->trc_reader_scp; barrier(); // scp before nesting to protect against interrupt handler. - if (!--t->trc_reader_nesting) { + n = READ_ONCE(t->trc_reader_nesting) - 1; + WRITE_ONCE(t->trc_reader_nesting, n); + if (!n) { if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB)) smp_mb(); // Placeholder for more selective ordering __srcu_read_unlock_fast(&rcu_tasks_trace_srcu_struct, scp); From a781301250bc003dc18688d31db3e39a546b55f8 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Mon, 13 Jul 2026 18:31:12 +0800 Subject: [PATCH 23/59] rcu-tasks: Remove smp_mb() in rcu_spawn_tasks_kthread_generic() For the smp_mb() after kthread_run() in rcu_spawn_tasks_kthread_generic() from these commits: 'commit eacd6f04a133 ("rcu-tasks: Move Tasks RCU to its own file")' 'commit 84a8f446ffd7 ("rcu: Defer rcu_tasks_kthread() creation till first call_rcu_tasks()")' the memory order pairing as follows: rcu_spawn_tasks_kthread() ->t = kthread_run(rcu_tasks_kthread, ...); ->smp_mb(); /* Ensure others see full kthread. */ ->WRITE_ONCE(rcu_tasks_kthread_ptr, t); call_rcu_tasks() ->if (READ_ONCE(rcu_tasks_kthread_ptr)) ->wake_up(&rcu_tasks_cbs_wq) ->try_to_wake_up() lock pi_lock ->smp_mb__after_spinlock() //see full kthread Because the 'commit d119357d0743 ("rcu-tasks: Treat only synchronous grace periods urgently")' moved the kthread_ptr assignment into the rcu_tasks_kthread() function, the following memory order pairings are sufficient: The runq's raw_spinlock/unlock(or smp_mb__after_spinlock()) from wake_up_process() in kthread_run() and __schedule() provides memory order barrier when the kthread is first scheduled, this ensures the kthread's func observes all of the kthread's initialization. The kthread's smp_store_release(&rtp->kthread_ptr, ...) in rcu_tasks_kthread() and smp_load_acquire(&rtp->kthread_ptr) in call_rcu_tasks_generic() compose release/acquire pairing, the cumulativity of smp_store_release() propagates visibility of the kthread's initialization through the scheduler chain. This commit therefore remove smp_mb() in rcu_spawn_tasks_kthread_generic(). Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 1 - 1 file changed, 1 deletion(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index d9e1e53f4ccf..79aa5d51388c 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -681,7 +681,6 @@ static void __init rcu_spawn_tasks_kthread_generic(struct rcu_tasks *rtp) t = kthread_run(rcu_tasks_kthread, rtp, "%s_kthread", rtp->kname); if (WARN_ONCE(IS_ERR(t), "%s: Could not start %s grace-period kthread, OOM is now expected behavior\n", __func__, rtp->name)) return; - smp_mb(); /* Ensure others see full kthread. */ } #ifndef CONFIG_TINY_RCU From aaca360ebb04ccd15a733bcbd863c2aebba8c1d5 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Fri, 24 Jul 2026 11:31:23 -0700 Subject: [PATCH 24/59] rcu: Enable RCU callbacks to benefit from expedited grace periods Currently, RCU callbacks only track normal grace-period sequence numbers. This means callbacks must wait for normal grace periods to complete even when expedited grace periods have already elapsed. Use the full struct rcu_gp_seq (which tracks both the normal and expedited grace-period sequences) throughout the callback infrastructure. rcu_segcblist_advance() now checks both normal and expedited GP completion via poll_state_synchronize_rcu_full(), and becomes parameterless since it reads the grace-period state internally. rcu_segcblist_accelerate() stores the full state (both sequences) instead of just the normal one. rcu_accelerate_cbs() and rcu_accelerate_cbs_unlocked() use get_state_synchronize_rcu_full() to capture both sequences, and the NOCB advance checks use poll_state_synchronize_rcu_full() instead of comparing only the normal sequence. srcu_segcblist_advance() becomes a standalone implementation because it compares SRCU sequences directly and cannot use poll_state_synchronize_rcu_full(), which reads RCU-specific globals. srcu_segcblist_accelerate() sets the ->exp field to RCU_GET_STATE_NOT_TRACKED so that poll_state_synchronize_rcu_full() compares only ->norm and ignores ->exp. Signed-off-by: Puranjay Mohan Signed-off-by: Paul E. McKenney --- kernel/rcu/rcu_segcblist.c | 30 +++++++++++++++++++++++------- kernel/rcu/rcu_segcblist.h | 2 +- kernel/rcu/tree.c | 9 +++------ kernel/rcu/tree_nocb.h | 22 ++++++++++++---------- 4 files changed, 39 insertions(+), 24 deletions(-) diff --git a/kernel/rcu/rcu_segcblist.c b/kernel/rcu/rcu_segcblist.c index 4e3dfe42bc09..cf8951d33e76 100644 --- a/kernel/rcu/rcu_segcblist.c +++ b/kernel/rcu/rcu_segcblist.c @@ -12,6 +12,7 @@ #include #include +#include "rcu.h" #include "rcu_segcblist.h" /* Initialize simple callback list. */ @@ -494,9 +495,9 @@ static void rcu_segcblist_advance_compact(struct rcu_segcblist *rsclp, int i) /* * Advance the callbacks in the specified rcu_segcblist structure based - * on the current value passed in for the grace-period counter. + * on the current value of the grace-period counter. */ -void rcu_segcblist_advance(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gsp) +void rcu_segcblist_advance(struct rcu_segcblist *rsclp) { int i; @@ -509,7 +510,7 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gsp) * are ready to invoke, and put them into the RCU_DONE_TAIL segment. */ for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++) { - if (ULONG_CMP_LT(gsp->norm, rsclp->gp_seq[i].norm)) + if (!poll_state_synchronize_rcu_full(&rsclp->gp_seq[i])) break; WRITE_ONCE(rsclp->tails[RCU_DONE_TAIL], rsclp->tails[i]); rcu_segcblist_move_seglen(rsclp, i, RCU_DONE_TAIL); @@ -595,7 +596,7 @@ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gs */ for (; i < RCU_NEXT_TAIL; i++) { WRITE_ONCE(rsclp->tails[i], rsclp->tails[RCU_NEXT_TAIL]); - rsclp->gp_seq[i].norm = gsp->norm; + rsclp->gp_seq[i] = *gsp; } return true; } @@ -637,14 +638,29 @@ void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp, void srcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) { - struct rcu_gp_seq gs = { .norm = seq }; + int i; - rcu_segcblist_advance(rsclp, &gs); + WARN_ON_ONCE(!rcu_segcblist_is_enabled(rsclp)); + if (rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL)) + return; + + for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++) { + if (ULONG_CMP_LT(seq, rsclp->gp_seq[i].norm)) + break; + WRITE_ONCE(rsclp->tails[RCU_DONE_TAIL], rsclp->tails[i]); + rcu_segcblist_move_seglen(rsclp, i, RCU_DONE_TAIL); + } + + /* If no callbacks moved, nothing more need be done. */ + if (i == RCU_WAIT_TAIL) + return; + + rcu_segcblist_advance_compact(rsclp, i); } bool srcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq) { - struct rcu_gp_seq gs = { .norm = seq }; + struct rcu_gp_seq gs = { .norm = seq, .exp = RCU_GET_STATE_NOT_TRACKED }; return rcu_segcblist_accelerate(rsclp, &gs); } diff --git a/kernel/rcu/rcu_segcblist.h b/kernel/rcu/rcu_segcblist.h index 16b0cb6b3250..431c4466b889 100644 --- a/kernel/rcu/rcu_segcblist.h +++ b/kernel/rcu/rcu_segcblist.h @@ -139,7 +139,7 @@ void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp, struct rcu_cblist *rclp); void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp, struct rcu_cblist *rclp); -void rcu_segcblist_advance(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gsp); +void rcu_segcblist_advance(struct rcu_segcblist *rsclp); bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gsp); void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp, struct rcu_segcblist *src_rsclp); diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 640ebe055a24..f63ce75c5b4f 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -1159,7 +1159,7 @@ static bool rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp) * accelerating callback invocation to an earlier grace-period * number. */ - gs.norm = rcu_seq_snap(&rcu_state.gp_seq); + get_state_synchronize_rcu_full(&gs); if (rcu_segcblist_accelerate(&rdp->cblist, &gs)) ret = rcu_start_this_gp(rnp, rdp, gs.norm); @@ -1188,7 +1188,7 @@ static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp, bool needwake; rcu_lockdep_assert_cblist_protected(rdp); - gs.norm = rcu_seq_snap(&rcu_state.gp_seq); + get_state_synchronize_rcu_full(&gs); if (!READ_ONCE(rdp->gpwrap) && ULONG_CMP_GE(rdp->gp_seq_needed, gs.norm)) { /* Old request still live, so mark recent callbacks. */ (void)rcu_segcblist_accelerate(&rdp->cblist, &gs); @@ -1213,8 +1213,6 @@ static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp, */ static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp) { - struct rcu_gp_seq gs; - rcu_lockdep_assert_cblist_protected(rdp); raw_lockdep_assert_held_rcu_node(rnp); @@ -1226,8 +1224,7 @@ static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp) * Find all callbacks whose ->gp_seq numbers indicate that they * are ready to invoke, and put them into the RCU_DONE_TAIL sublist. */ - gs.norm = rnp->gp_seq; - rcu_segcblist_advance(&rdp->cblist, &gs); + rcu_segcblist_advance(&rdp->cblist); /* Classify any remaining callbacks. */ return rcu_accelerate_cbs(rnp, rdp); diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index eaa1ad5297e2..876dd897f18b 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -502,7 +502,7 @@ static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, } if (j != rdp->nocb_gp_adv_time && rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && - rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq.norm)) { + poll_state_synchronize_rcu_full(&cur_gp_seq)) { rcu_advance_cbs_nowake(rdp->mynode, rdp); rdp->nocb_gp_adv_time = j; } @@ -731,17 +731,17 @@ static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp) if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL) || (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) && - rcu_seq_done(&rnp->gp_seq, cur_gp_seq.norm))) { + poll_state_synchronize_rcu_full(&cur_gp_seq))) { raw_spin_lock_rcu_node(rnp); /* irqs disabled. */ needwake_gp = rcu_advance_cbs(rnp, rdp); wasempty = rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL); raw_spin_unlock_rcu_node(rnp); /* irqs disabled. */ } - // Need to wait on some grace period? WARN_ON_ONCE(wasempty && !rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL)); + // Need to wait on some grace period? if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq)) { if (!needwait_gp || ULONG_CMP_LT(cur_gp_seq.norm, wait_gp_seq)) @@ -919,7 +919,7 @@ static void nocb_cb_wait(struct rcu_data *rdp) lockdep_assert_irqs_enabled(); rcu_nocb_lock_irqsave(rdp, flags); if (rcu_segcblist_nextgp(cblist, &cur_gp_seq) && - rcu_seq_done(&rnp->gp_seq, cur_gp_seq.norm) && + poll_state_synchronize_rcu_full(&cur_gp_seq) && raw_spin_trylock_rcu_node(rnp)) { /* irqs already disabled. */ needwake_gp = rcu_advance_cbs(rdp->mynode, rdp); raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ @@ -1548,8 +1548,8 @@ static void show_rcu_nocb_gp_state(struct rcu_data *rdp) static void show_rcu_nocb_state(struct rcu_data *rdp) { char bufd[22]; - char bufw[45]; - char bufr[45]; + char bufw[64]; + char bufr[64]; char bufn[22]; char bufb[22]; struct rcu_data *nocb_next_rdp; @@ -1569,10 +1569,12 @@ static void show_rcu_nocb_state(struct rcu_data *rdp) nocb_entry_rdp); sprintf(bufd, "%ld", rsclp->seglen[RCU_DONE_TAIL]); - sprintf(bufw, "%ld(%ld)", rsclp->seglen[RCU_WAIT_TAIL], - rsclp->gp_seq[RCU_WAIT_TAIL].norm); - sprintf(bufr, "%ld(%ld)", rsclp->seglen[RCU_NEXT_READY_TAIL], - rsclp->gp_seq[RCU_NEXT_READY_TAIL].norm); + sprintf(bufw, "%ld(%ld/%ld)", rsclp->seglen[RCU_WAIT_TAIL], + rsclp->gp_seq[RCU_WAIT_TAIL].norm, + rsclp->gp_seq[RCU_WAIT_TAIL].exp); + sprintf(bufr, "%ld(%ld/%ld)", rsclp->seglen[RCU_NEXT_READY_TAIL], + rsclp->gp_seq[RCU_NEXT_READY_TAIL].norm, + rsclp->gp_seq[RCU_NEXT_READY_TAIL].exp); sprintf(bufn, "%ld", rsclp->seglen[RCU_NEXT_TAIL]); sprintf(bufb, "%ld", rcu_cblist_n_cbs(&rdp->nocb_bypass)); pr_info(" CB %d^%d->%d %c%c%c%c%c F%ld L%ld C%d %c%s%c%s%c%s%c%s%c%s q%ld %c CPU %d%s\n", From 942b3e8f3f3f51011161cb728375481ebbc84e22 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Wed, 24 Jun 2026 06:23:49 -0700 Subject: [PATCH 25/59] rcu: Update comments for gp_seq and expedited GP tracking Update documentation comments throughout the RCU callback infrastructure to reflect the transition from a single grace-period sequence number to the full struct rcu_gp_seq that tracks both normal and expedited grace periods. The ->gp_seq[] array documentation in rcu_segcblist.h is updated to describe dual (normal and expedited) GP tracking. The rcu_segcblist_advance(), rcu_segcblist_accelerate(), and rcu_advance_cbs() comments are updated to refer to the struct rcu_gp_seq state (gsp) instead of the old bare grace-period sequence number (seq). Reviewed-by: Paul E. McKenney Signed-off-by: Puranjay Mohan Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- include/linux/rcu_segcblist.h | 14 +++++++----- kernel/rcu/rcu_segcblist.c | 43 +++++++++++++++++++++++------------ kernel/rcu/tree.c | 6 ++--- 3 files changed, 39 insertions(+), 24 deletions(-) diff --git a/include/linux/rcu_segcblist.h b/include/linux/rcu_segcblist.h index 137cc23b024c..08b63ecf719b 100644 --- a/include/linux/rcu_segcblist.h +++ b/include/linux/rcu_segcblist.h @@ -50,12 +50,14 @@ struct rcu_cblist { * Note that RCU_WAIT_TAIL cannot be empty unless RCU_NEXT_READY_TAIL is also * empty. * - * The ->gp_seq[] array contains the grace-period number at which the - * corresponding segment of callbacks will be ready to invoke. A given - * element of this array is meaningful only when the corresponding segment - * is non-empty, and it is never valid for RCU_DONE_TAIL (whose callbacks - * are already ready to invoke) or for RCU_NEXT_TAIL (whose callbacks have - * not yet been assigned a grace-period number). + * The ->gp_seq[] array contains the grace-period state at which the + * corresponding segment of callbacks will be ready to invoke. This tracks + * both normal and expedited grace periods, allowing callbacks to complete + * when either type of GP finishes. A given element of this array is + * meaningful only when the corresponding segment is non-empty, and it is + * never valid for RCU_DONE_TAIL (whose callbacks are already ready to + * invoke) or for RCU_NEXT_TAIL (whose callbacks have not yet been assigned + * a grace-period state). */ #define RCU_DONE_TAIL 0 /* Also RCU_WAIT head. */ #define RCU_WAIT_TAIL 1 /* Also RCU_NEXT_READY head. */ diff --git a/kernel/rcu/rcu_segcblist.c b/kernel/rcu/rcu_segcblist.c index cf8951d33e76..dd770006e7f8 100644 --- a/kernel/rcu/rcu_segcblist.c +++ b/kernel/rcu/rcu_segcblist.c @@ -495,7 +495,8 @@ static void rcu_segcblist_advance_compact(struct rcu_segcblist *rsclp, int i) /* * Advance the callbacks in the specified rcu_segcblist structure based - * on the current value of the grace-period counter. + * on the current grace-period state. Checks both normal and expedited + * grace periods, advancing callbacks when either GP type completes. */ void rcu_segcblist_advance(struct rcu_segcblist *rsclp) { @@ -506,8 +507,10 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp) return; /* - * Find all callbacks whose ->gp_seq numbers indicate that they - * are ready to invoke, and put them into the RCU_DONE_TAIL segment. + * Find all callbacks whose grace periods have completed (either + * normal or expedited) and put them into the RCU_DONE_TAIL segment. + * We check against the current global GP state, which includes + * proper memory barriers and handles special completion values. */ for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++) { if (!poll_state_synchronize_rcu_full(&rsclp->gp_seq[i])) @@ -534,9 +537,9 @@ void rcu_segcblist_advance(struct rcu_segcblist *rsclp) * them to complete at the end of the earlier grace period. * * This function operates on an rcu_segcblist structure, and also the - * grace-period sequence number seq at which new callbacks would become + * grace-period state gsp at which new callbacks would become * ready to invoke. Returns true if there are callbacks that won't be - * ready to invoke until seq, false otherwise. + * ready to invoke until the grace period represented by gsp, false otherwise. */ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gsp) { @@ -548,11 +551,11 @@ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gs /* * Find the segment preceding the oldest segment of callbacks - * whose ->gp_seq[] completion is at or after that passed in via - * "seq", skipping any empty segments. This oldest segment, along + * whose grace period completion is at or after that passed in via + * "gsp", skipping any empty segments. This oldest segment, along * with any later segments, can be merged in with any newly arrived - * callbacks in the RCU_NEXT_TAIL segment, and assigned "seq" - * as their ->gp_seq[] grace-period completion sequence number. + * callbacks in the RCU_NEXT_TAIL segment, and assigned "gsp" + * as their grace-period completion state. */ for (i = RCU_NEXT_READY_TAIL; i > RCU_DONE_TAIL; i--) if (!rcu_segcblist_segempty(rsclp, i) && @@ -561,7 +564,7 @@ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gs /* * If all the segments contain callbacks that correspond to - * earlier grace-period sequence numbers than "seq", leave. + * earlier grace-period sequence numbers than "gsp", leave. * Assuming that the rcu_segcblist structure has enough * segments in its arrays, this can only happen if some of * the non-done segments contain callbacks that really are @@ -569,15 +572,15 @@ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gs * out by the next call to rcu_segcblist_advance(). * * Also advance to the oldest segment of callbacks whose - * ->gp_seq[] completion is at or after that passed in via "seq", + * ->gp_seq[] completion is at or after that passed in via "gsp", * skipping any empty segments. * * Note that segment "i" (and any lower-numbered segments * containing older callbacks) will be unaffected, and their - * grace-period numbers remain unchanged. For example, if i == + * grace-period states remain unchanged. For example, if i == * WAIT_TAIL, then neither WAIT_TAIL nor DONE_TAIL will be touched. * Instead, the CBs in NEXT_TAIL will be merged with those in - * NEXT_READY_TAIL and the grace-period number of NEXT_READY_TAIL + * NEXT_READY_TAIL and the grace-period state of NEXT_READY_TAIL * would be updated. NEXT_TAIL would then be empty. */ if (rcu_segcblist_restempty(rsclp, i) || ++i >= RCU_NEXT_TAIL) @@ -589,8 +592,8 @@ bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, struct rcu_gp_seq *gs /* * Merge all later callbacks, including newly arrived callbacks, - * into the segment located by the for-loop above. Assign "seq" - * as the ->gp_seq[] value in order to correctly handle the case + * into the segment located by the for-loop above. Assign "gsp" + * as the grace-period state in order to correctly handle the case * where there were no pending callbacks in the rcu_segcblist * structure other than in the RCU_NEXT_TAIL segment. */ @@ -644,6 +647,10 @@ void srcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) if (rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL)) return; + /* + * Find all callbacks whose normal GP sequence numbers indicate + * that they are ready to invoke. For SRCU, we only check norm. + */ for (i = RCU_WAIT_TAIL; i < RCU_NEXT_TAIL; i++) { if (ULONG_CMP_LT(seq, rsclp->gp_seq[i].norm)) break; @@ -658,6 +665,12 @@ void srcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq) rcu_segcblist_advance_compact(rsclp, i); } +/* + * SRCU wrapper for rcu_segcblist_accelerate() - converts SRCU's unsigned + * long GP sequence to rcu_gp_seq format with exp set to + * RCU_GET_STATE_NOT_TRACKED (since SRCU does not use expedited GPs) + * and calls the core rcu_segcblist_accelerate(). + */ bool srcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq) { struct rcu_gp_seq gs = { .norm = seq, .exp = RCU_GET_STATE_NOT_TRACKED }; diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index f63ce75c5b4f..aa11e9a04830 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -1204,7 +1204,7 @@ static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp, /* * Move any callbacks whose grace period has completed to the * RCU_DONE_TAIL sublist, then compact the remaining sublists and - * assign ->gp_seq numbers to any callbacks in the RCU_NEXT_TAIL + * assign ->gp_seq[] state to any callbacks in the RCU_NEXT_TAIL * sublist. This function is idempotent, so it does not hurt to * invoke it repeatedly. As long as it is not invoked -too- often... * Returns true if the RCU grace-period kthread needs to be awakened. @@ -1221,8 +1221,8 @@ static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp) return false; /* - * Find all callbacks whose ->gp_seq numbers indicate that they - * are ready to invoke, and put them into the RCU_DONE_TAIL sublist. + * Find all callbacks whose grace periods have completed (either + * normal or expedited) and put them into the RCU_DONE_TAIL sublist. */ rcu_segcblist_advance(&rdp->cblist); From 318606106c610714608da56cb9a2699040a26b53 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Wed, 24 Jun 2026 06:23:50 -0700 Subject: [PATCH 26/59] rcu: Wake NOCB rcuog kthreads on expedited grace period completion When an expedited grace period completes, rcu_exp_wait_wake() wakes waiters on rnp->exp_wq[] but does not notify the NOCB rcuog kthreads. An rcuog kthread that is waiting for a grace period sleeps on the leaf rcu_node's ->nocb_gp_wq[] with a wait condition based on the grace-period state, so without a wakeup, callbacks on offloaded CPUs that could benefit from the expedited GP wait until the rcuog kthread wakes for some other reason (e.g. the next normal GP or a timer). Make the rcuog grace-period wait honour expedited GPs and wake it when one completes: - nocb_gp_wait() now records the grace period to wait for as a struct rcu_gp_seq (both normal and expedited), tracks the earliest pending normal and expedited sequence across the group, and releases the wait via poll_state_synchronize_rcu_full() so it wakes for whichever completes first. ->nocb_gp_seq is widened to struct rcu_gp_seq accordingly. - rcu_exp_wait_wake() calls the new rcu_nocb_exp_cleanup() on leaf nodes, which wakes both ->nocb_gp_wq[0] and ->nocb_gp_wq[1] (the expedited sequence does not share parity with the normal ->gp_seq the waiter indexed with). Both this path and rcu_nocb_gp_cleanup() use the shared rcu_nocb_cleanup_wake() helper, which checks swait_active() first; the smp_mb() in rcu_gp_cleanup()/rcu_exp_wait_wake() orders the grace-period state update before that check. A stub rcu_nocb_exp_cleanup() is provided for CONFIG_RCU_NOCB_CPU=n. Signed-off-by: Puranjay Mohan Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 11 ++++- kernel/rcu/tree.h | 3 +- kernel/rcu/tree_exp.h | 2 + kernel/rcu/tree_nocb.h | 95 +++++++++++++++++++++++++++++++++++------- 4 files changed, 94 insertions(+), 17 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index aa11e9a04830..251114eb974b 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2219,8 +2219,15 @@ static noinline void rcu_gp_cleanup(void) dump_blkd_tasks(rnp, 10); WARN_ON_ONCE(rnp->qsmask); WRITE_ONCE(rnp->gp_seq, new_gp_seq); - if (!rnp->parent) - smp_mb(); // Order against failing poll_state_synchronize_rcu_full(). + if (!rnp->parent) { + /* + * Order against failing poll_state_synchronize_rcu_full(), + * and also against rcu_nocb_gp_cleanup() -> swait_active(), + * which relies on this barrier to observe a waiter that + * enqueued before re-checking the grace-period state. + */ + smp_mb(); + } rdp = this_cpu_ptr(&rcu_data); if (rnp == rdp->mynode) needgp = __note_gp_changes(rnp, rdp) || needgp; diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 7dfc57e9adb1..01a1b2985abd 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -253,7 +253,7 @@ struct rcu_data { u8 nocb_gp_sleep; /* Is the nocb GP thread asleep? */ u8 nocb_gp_bypass; /* Found a bypass on last scan? */ u8 nocb_gp_gp; /* GP to wait for on last scan? */ - unsigned long nocb_gp_seq; /* If so, ->gp_seq to wait for. */ + struct rcu_gp_seq nocb_gp_seq; /* If so, GP state to wait for. */ unsigned long nocb_gp_loops; /* # passes through wait code. */ struct swait_queue_head nocb_gp_wq; /* For nocb kthreads to sleep on. */ bool nocb_cb_sleep; /* Is the nocb CB thread asleep? */ @@ -498,6 +498,7 @@ static bool rcu_preempt_need_deferred_qs(struct task_struct *t); static void zero_cpu_stall_ticks(struct rcu_data *rdp); static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp); static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq); +static void rcu_nocb_exp_cleanup(struct rcu_node *rnp); static void rcu_init_one_nocb(struct rcu_node *rnp); static bool wake_nocb_gp(struct rcu_data *rdp); static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index 7024131574df..1ab83c98af51 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h @@ -708,6 +708,8 @@ static void rcu_exp_wait_wake(unsigned long s) } smp_mb(); /* All above changes before wakeup. */ wake_up_all(&rnp->exp_wq[rcu_seq_ctr(s) & 0x3]); + if (rcu_is_leaf_node(rnp)) + rcu_nocb_exp_cleanup(rnp); } trace_rcu_exp_grace_period(rcu_state.name, s, TPS("endwake")); mutex_unlock(&rcu_state.exp_wake_mutex); diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index 876dd897f18b..afb35f921b33 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -170,13 +170,35 @@ static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp) lockdep_assert_held(&rdp->nocb_lock); } +static void rcu_nocb_cleanup_wake(struct swait_queue_head *sq) +{ + if (swait_active(sq)) + swake_up_all(sq); +} + /* * Wake up any no-CBs CPUs' kthreads that were waiting on the just-ended * grace period. */ static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq) { - swake_up_all(sq); + /* + * swait_active() can be checked first because of the following + * ordering, which pairs the smp_mb() in rcu_gp_cleanup() against + * the implicit barrier in prepare_to_swait()/set_current_state() + * on the nocb_gp_wait() side: + * + * rcu_gp_cleanup() nocb_gp_wait() + * --------------- -------------- + * WRITE_ONCE(root->gp_seq, new_gp_seq); swait_event_interruptible_exclusive(sq) + * smp_mb() prepare_to_swait() + * if swait_active(sq) list_add_tail(...) + * swake_up_all(sq) set_current_state() + * smp_mb() + * if (poll_state_synchronize_rcu_full()) + * ... + */ + rcu_nocb_cleanup_wake(sq); } static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp) @@ -190,6 +212,38 @@ static void rcu_init_one_nocb(struct rcu_node *rnp) init_swait_queue_head(&rnp->nocb_gp_wq[1]); } +/* + * Wake NOCB rcuog kthreads on a leaf node so that they can advance + * callbacks that were waiting for the just-completed expedited GP. + * + * The rcuog kthread waiting for a grace period sleeps on the per-leaf-node + * ->nocb_gp_wq[] (not on its rdp_gp's ->nocb_gp_wq, which only signals that + * new callbacks have shown up), so this is the queue that must be woken. + * Both the even and odd waitqueues are woken because the expedited sequence + * does not share parity with the normal ->gp_seq the waiter indexed with. + */ +static void rcu_nocb_exp_cleanup(struct rcu_node *rnp) +{ + /* + * swait_active() can be checked first because of the following + * ordering, which pairs the smp_mb() in rcu_exp_wait_wake() against + * the implicit barrier in prepare_to_swait()/set_current_state() + * on the nocb_gp_wait() side: + * + * rcu_exp_wait_wake() nocb_gp_wait() + * --------------- -------------- + * rcu_seq_end(&rcu_state.expedited_sequence); swait_event_interruptible_exclusive(sq) + * smp_mb() prepare_to_swait() + * if swait_active(sq) list_add_tail(...) + * swake_up_all(sq) set_current_state() + * smp_mb() + * if (poll_state_synchronize_rcu_full()) + * ... + */ + rcu_nocb_cleanup_wake(&rnp->nocb_gp_wq[0]); + rcu_nocb_cleanup_wake(&rnp->nocb_gp_wq[1]); +} + /* Clear any pending deferred wakeup timer (nocb_gp_lock must be held). */ static void nocb_defer_wakeup_cancel(struct rcu_data *rdp_gp) { @@ -659,7 +713,6 @@ static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp) { bool bypass = false; int __maybe_unused cpu = my_rdp->cpu; - struct rcu_gp_seq cur_gp_seq; unsigned long flags; bool gotcbs = false; unsigned long j = jiffies; @@ -669,7 +722,7 @@ static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp) bool needwake_gp; struct rcu_data *rdp, *rdp_toggling = NULL; struct rcu_node *rnp; - unsigned long wait_gp_seq = 0; // Suppress "use uninitialized" warning. + struct rcu_gp_seq wait_gp_seq = {0}; // Suppress "use uninitialized" warning. bool wasempty = false; /* @@ -693,6 +746,7 @@ static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp) * won't be ignored for long. */ list_for_each_entry(rdp, &my_rdp->nocb_head_rdp, nocb_entry_rdp) { + struct rcu_gp_seq cur_gp_seq; long bypass_ncbs; bool flush_bypass = false; long lazy_ncbs; @@ -743,9 +797,15 @@ static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp) RCU_NEXT_READY_TAIL)); // Need to wait on some grace period? if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq)) { - if (!needwait_gp || - ULONG_CMP_LT(cur_gp_seq.norm, wait_gp_seq)) - wait_gp_seq = cur_gp_seq.norm; + /* + * Track the earliest pending normal and expedited GP + * across the group so the wait below can be released by + * whichever completes first. + */ + if (!needwait_gp || ULONG_CMP_LT(cur_gp_seq.norm, wait_gp_seq.norm)) + wait_gp_seq.norm = cur_gp_seq.norm; + if (!needwait_gp || ULONG_CMP_LT(cur_gp_seq.exp, wait_gp_seq.exp)) + wait_gp_seq.exp = cur_gp_seq.exp; needwait_gp = true; trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("NeedWaitGP")); @@ -767,7 +827,8 @@ static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp) my_rdp->nocb_gp_bypass = bypass; my_rdp->nocb_gp_gp = needwait_gp; - my_rdp->nocb_gp_seq = needwait_gp ? wait_gp_seq : 0; + if (needwait_gp) + my_rdp->nocb_gp_seq = wait_gp_seq; // At least one child with non-empty ->nocb_bypass, so set // timer in order to avoid stranding its callbacks. @@ -802,12 +863,12 @@ static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp) nocb_gp_sleep(my_rdp, cpu); } else { rnp = my_rdp->mynode; - trace_rcu_this_gp(rnp, wait_gp_seq, TPS("StartWait")); + trace_rcu_this_gp(rnp, wait_gp_seq.norm, TPS("StartWait")); swait_event_interruptible_exclusive( - rnp->nocb_gp_wq[rcu_seq_ctr(wait_gp_seq) & 0x1], - rcu_seq_done(&rnp->gp_seq, wait_gp_seq) || + rnp->nocb_gp_wq[rcu_seq_ctr(wait_gp_seq.norm) & 0x1], + poll_state_synchronize_rcu_full(&wait_gp_seq) || !READ_ONCE(my_rdp->nocb_gp_sleep)); - trace_rcu_this_gp(rnp, wait_gp_seq, TPS("EndWait")); + trace_rcu_this_gp(rnp, wait_gp_seq.norm, TPS("EndWait")); } if (!rcu_nocb_poll) { @@ -841,7 +902,8 @@ static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp) swake_up_one(&rdp_toggling->nocb_state_wq); } - my_rdp->nocb_gp_seq = -1; + my_rdp->nocb_gp_seq.norm = -1; + my_rdp->nocb_gp_seq.exp = -1; WARN_ON(signal_pending(current)); } @@ -1525,7 +1587,7 @@ static void show_rcu_nocb_gp_state(struct rcu_data *rdp) { struct rcu_node *rnp = rdp->mynode; - pr_info("nocb GP %d %c%c%c%c%c %c[%c%c] %c%c:%ld rnp %d:%d %lu %c CPU %d%s\n", + pr_info("nocb GP %d %c%c%c%c%c %c[%c%c] %c%c:%ld/%ld rnp %d:%d %lu %c CPU %d%s\n", rdp->cpu, "kK"[!!rdp->nocb_gp_kthread], "lL"[raw_spin_is_locked(&rdp->nocb_gp_lock)], @@ -1537,7 +1599,8 @@ static void show_rcu_nocb_gp_state(struct rcu_data *rdp) ".W"[swait_active(&rnp->nocb_gp_wq[1])], ".B"[!!rdp->nocb_gp_bypass], ".G"[!!rdp->nocb_gp_gp], - (long)rdp->nocb_gp_seq, + (long)rdp->nocb_gp_seq.norm, + (long)rdp->nocb_gp_seq.exp, rnp->grplo, rnp->grphi, READ_ONCE(rdp->nocb_gp_loops), rdp->nocb_gp_kthread ? task_state_to_char(rdp->nocb_gp_kthread) : '.', rdp->nocb_gp_kthread ? (int)task_cpu(rdp->nocb_gp_kthread) : -1, @@ -1657,6 +1720,10 @@ static void rcu_init_one_nocb(struct rcu_node *rnp) { } +static void rcu_nocb_exp_cleanup(struct rcu_node *rnp) +{ +} + static bool wake_nocb_gp(struct rcu_data *rdp) { return false; From fde2b18817e4f32b92c1e9881356fbc314983a7a Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Wed, 24 Jun 2026 06:23:51 -0700 Subject: [PATCH 27/59] rcu: Detect expedited grace period completion in rcu_pending() rcu_pending() decides whether rcu_core() should run on the current CPU's timer tick. It does not account for expedited grace periods: after an expedited GP completes, a non-offloaded CPU's callbacks remain in RCU_WAIT_TAIL (not yet advanced to RCU_DONE_TAIL) and rcu_core() is never invoked to advance them. Detect that case via rcu_segcblist_nextgp() combined with a new memory-ordering-free poll variant, poll_state_synchronize_rcu_full_unordered(). This keeps rcu_pending() cheap: it runs on every tick that has pending callbacks, so it must not pay for the two memory barriers in poll_state_synchronize_rcu_full(). The check is only a hint to run rcu_core(); the ordered re-check and the actual callback advancement happen there. Signed-off-by: Puranjay Mohan Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 251114eb974b..3541feed9557 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -3592,6 +3592,24 @@ bool poll_state_synchronize_rcu(unsigned long oldstate) } EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu); +/* + * Racy, memory-ordering-free test of whether the normal or expedited grace + * period recorded in *gsp has completed. Callers that need the full + * memory-ordering guarantees must use poll_state_synchronize_rcu_full(); + * this variant is only a hint (e.g. for rcu_pending()) and leaves any + * required ordering to a subsequent ordered check. + */ +static bool poll_state_synchronize_rcu_full_unordered(struct rcu_gp_seq *gsp) +{ + struct rcu_node *rnp = rcu_get_root(); + + return gsp->norm == RCU_GET_STATE_COMPLETED || + rcu_seq_done_exact(&rnp->gp_seq, gsp->norm) || + gsp->exp == RCU_GET_STATE_COMPLETED || + (gsp->exp != RCU_GET_STATE_NOT_TRACKED && + rcu_seq_done_exact(&rcu_state.expedited_sequence, gsp->exp)); +} + /** * poll_state_synchronize_rcu_full - Has the specified RCU grace period completed? * @gsp: value from get_state_synchronize_rcu_full() or start_poll_synchronize_rcu_full() @@ -3627,14 +3645,8 @@ EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu); */ bool poll_state_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - struct rcu_node *rnp = rcu_get_root(); - smp_mb(); // Order against root rcu_node structure grace-period cleanup. - if (gsp->norm == RCU_GET_STATE_COMPLETED || - rcu_seq_done_exact(&rnp->gp_seq, gsp->norm) || - gsp->exp == RCU_GET_STATE_COMPLETED || - (gsp->exp != RCU_GET_STATE_NOT_TRACKED && - rcu_seq_done_exact(&rcu_state.expedited_sequence, gsp->exp))) { + if (poll_state_synchronize_rcu_full_unordered(gsp)) { smp_mb(); /* Ensure GP ends before subsequent accesses. */ return true; } @@ -3704,6 +3716,7 @@ EXPORT_SYMBOL_GPL(cond_synchronize_rcu_full); static int rcu_pending(int user) { bool gp_in_progress; + struct rcu_gp_seq gp_state; struct rcu_data *rdp = this_cpu_ptr(&rcu_data); struct rcu_node *rnp = rdp->mynode; @@ -3734,6 +3747,17 @@ static int rcu_pending(int user) rcu_segcblist_ready_cbs(&rdp->cblist)) return 1; + /* + * Has a GP (normal or expedited) completed for pending callbacks? + * This is only a racy hint to decide whether to run rcu_core(); the + * ordered re-check and callback advancement happen there, so the + * unordered test avoids paying for memory barriers on every tick. + */ + if (!rcu_rdp_is_offloaded(rdp) && + rcu_segcblist_nextgp(&rdp->cblist, &gp_state) && + poll_state_synchronize_rcu_full_unordered(&gp_state)) + return 1; + /* Has RCU gone idle with this CPU needing another grace period? */ if (!gp_in_progress && rcu_segcblist_is_enabled(&rdp->cblist) && !rcu_rdp_is_offloaded(rdp) && From ec5ee5417b00278fa050b38d878b76f1365420fb Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Wed, 24 Jun 2026 06:23:52 -0700 Subject: [PATCH 28/59] rcu: Advance callbacks for expedited GP completion in rcu_core() Even when rcu_pending() triggers rcu_core(), the normal callback advancement path through note_gp_changes() -> __note_gp_changes() bails out when rdp->gp_seq == rnp->gp_seq (no normal GP change). Since expedited GPs do not update rnp->gp_seq, rcu_advance_cbs() is never called and callbacks remain stuck in RCU_WAIT_TAIL. Add a direct callback advancement block in rcu_core() that checks for GP completion via rcu_segcblist_nextgp() combined with poll_state_synchronize_rcu_full(). When detected, trylock rnp and call rcu_advance_cbs() to move completed callbacks to RCU_DONE_TAIL. Wake the GP kthread if rcu_advance_cbs() requests a new grace period. Uses trylock to avoid adding contention on rnp->lock. If the lock is contended, callbacks will be advanced on the next tick. Reviewed-by: Paul E. McKenney Signed-off-by: Puranjay Mohan Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 3541feed9557..f6809ccedfc1 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2885,6 +2885,23 @@ static __latent_entropy void rcu_core(void) /* Update RCU state based on any recent quiescent states. */ rcu_check_quiescent_state(rdp); + /* Advance callbacks if an expedited GP has completed. */ + if (!rcu_rdp_is_offloaded(rdp) && rcu_segcblist_is_enabled(&rdp->cblist)) { + struct rcu_gp_seq gp_state; + + if (rcu_segcblist_nextgp(&rdp->cblist, &gp_state) && + poll_state_synchronize_rcu_full(&gp_state)) { + guard(irqsave)(); + if (raw_spin_trylock_rcu_node(rnp)) { + bool needwake = rcu_advance_cbs(rnp, rdp); + + raw_spin_unlock_rcu_node(rnp); + if (needwake) + rcu_gp_kthread_wake(); + } + } + } + /* No grace period and unregistered callbacks? */ if (!rcu_gp_in_progress() && rcu_segcblist_is_enabled(&rdp->cblist) && !rcu_rdp_is_offloaded(rdp)) { From 31b780b23f8f4c14db7892a2db29527afb521c4b Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Wed, 24 Jun 2026 06:23:53 -0700 Subject: [PATCH 29/59] rcuscale: Add concurrent expedited GP threads for callback scaling tests Add nexp and exp_interval parameters to rcuscale that spawn kthreads running synchronize_rcu_expedited() in a loop. This generates concurrent expedited GP load while the normal writers measure GP or callback latency. When combined with gp_async=1 (which uses call_rcu() for writers), this tests how effectively callbacks benefit from expedited grace periods. With RCU callback expedited GP tracking, the async callbacks should complete faster because they piggyback on the expedited GPs rather than waiting for normal GPs. Reviewed-by: Paul E. McKenney Signed-off-by: Puranjay Mohan Signed-off-by: Paul E. McKenney --- kernel/rcu/rcuscale.c | 84 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 82 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/rcuscale.c b/kernel/rcu/rcuscale.c index ac0b1c6b7dae..1097ec15879c 100644 --- a/kernel/rcu/rcuscale.c +++ b/kernel/rcu/rcuscale.c @@ -91,6 +91,8 @@ torture_param(int, shutdown_secs, !IS_MODULE(CONFIG_RCU_SCALE_TEST) * 300, torture_param(int, verbose, 1, "Enable verbose debugging printk()s"); torture_param(int, writer_holdoff, 0, "Holdoff (us) between GPs, zero to disable"); torture_param(int, writer_holdoff_jiffies, 0, "Holdoff (jiffies) between GPs, zero to disable"); +torture_param(int, nexp, 0, "Number of expedited GP threads to run concurrently"); +torture_param(int, exp_interval, 0, "Interval (us) between expedited GPs, zero to disable"); torture_param(int, kfree_rcu_test, 0, "Do we run a kfree_rcu() scale test?"); torture_param(int, kfree_mult, 1, "Multiple of kfree_obj size to allocate."); torture_param(int, kfree_by_call_rcu, 0, "Use call_rcu() to emulate kfree_rcu()?"); @@ -115,8 +117,10 @@ struct writer_freelist { static int nrealreaders; static int nrealwriters; +static int nrealexp; static struct task_struct **writer_tasks; static struct task_struct **reader_tasks; +static struct task_struct **exp_tasks; static u64 **writer_durations; static bool *writer_done; @@ -462,6 +466,34 @@ rcu_scale_reader(void *arg) return 0; } +/* + * RCU expedited GP kthread. Repeatedly invokes expedited grace periods + * to generate concurrent expedited GP load while the normal-GP writers + * are being measured. This allows measuring the benefit of callbacks + * that can piggyback on expedited grace periods. + */ +static int +rcu_scale_exp(void *arg) +{ + long me = (long)arg; + + VERBOSE_SCALEOUT_STRING("rcu_scale_exp task started"); + set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids)); + set_user_nice(current, MIN_NICE); + + if (holdoff) + schedule_timeout_idle(holdoff * HZ); + + do { + if (exp_interval) + udelay(exp_interval); + cur_ops->exp_sync(); + rcu_scale_wait_shutdown(); + } while (!torture_must_stop()); + torture_kthread_stopping("rcu_scale_exp"); + return 0; +} + /* * Allocate a writer_mblock structure for the specified rcu_scale_writer * task. @@ -664,8 +696,10 @@ static void rcu_scale_print_module_parms(struct rcu_scale_ops *cur_ops, const char *tag) { pr_alert("%s" SCALE_FLAG - "--- %s: gp_async=%d gp_async_max=%d gp_exp=%d holdoff=%d minruntime=%d nreaders=%d nwriters=%d writer_holdoff=%d writer_holdoff_jiffies=%d verbose=%d shutdown_secs=%d\n", - scale_type, tag, gp_async, gp_async_max, gp_exp, holdoff, minruntime, nrealreaders, nrealwriters, writer_holdoff, writer_holdoff_jiffies, verbose, shutdown_secs); + "--- %s: gp_async=%d gp_async_max=%d gp_exp=%d holdoff=%d minruntime=%d nreaders=%d nwriters=%d nexp=%d exp_interval=%d writer_holdoff=%d writer_holdoff_jiffies=%d verbose=%d shutdown_secs=%d\n", + scale_type, tag, gp_async, gp_async_max, gp_exp, holdoff, + minruntime, nrealreaders, nrealwriters, nrealexp, exp_interval, + writer_holdoff, writer_holdoff_jiffies, verbose, shutdown_secs); } /* @@ -809,6 +843,13 @@ kfree_scale_cleanup(void) if (torture_cleanup_begin()) return; + if (exp_tasks) { + for (i = 0; i < nrealexp; i++) + torture_stop_kthread(rcu_scale_exp, exp_tasks[i]); + kfree(exp_tasks); + exp_tasks = NULL; + } + if (kfree_reader_tasks) { for (i = 0; i < kfree_nrealthreads; i++) torture_stop_kthread(kfree_scale_thread, @@ -903,6 +944,22 @@ kfree_scale_init(void) goto unwind; } + if (nrealexp > 0 && cur_ops->exp_sync) { + exp_tasks = kzalloc_objs(exp_tasks[0], nrealexp); + if (!exp_tasks) { + SCALEOUT_ERRSTRING("out of memory"); + firsterr = -ENOMEM; + goto unwind; + } + for (i = 0; i < nrealexp; i++) { + firsterr = torture_create_kthread(rcu_scale_exp, + (void *)i, + exp_tasks[i]); + if (torture_init_error(firsterr)) + goto unwind; + } + } + while (atomic_read(&n_kfree_scale_thread_started) < kfree_nrealthreads) schedule_timeout_uninterruptible(1); @@ -959,6 +1016,13 @@ rcu_scale_cleanup(void) return; } + if (exp_tasks) { + for (i = 0; i < nrealexp; i++) + torture_stop_kthread(rcu_scale_exp, exp_tasks[i]); + kfree(exp_tasks); + exp_tasks = NULL; + } + if (reader_tasks) { for (i = 0; i < nrealreaders; i++) torture_stop_kthread(rcu_scale_reader, @@ -1076,6 +1140,7 @@ rcu_scale_init(void) if (kthread_tp) kthread_stime = kthread_tp->stime; } + nrealexp = nexp; if (kfree_rcu_test) return kfree_scale_init(); @@ -1107,6 +1172,21 @@ rcu_scale_init(void) } while (atomic_read(&n_rcu_scale_reader_started) < nrealreaders) schedule_timeout_uninterruptible(1); + if (nrealexp > 0 && cur_ops->exp_sync) { + exp_tasks = kzalloc_objs(exp_tasks[0], nrealexp); + if (!exp_tasks) { + SCALEOUT_ERRSTRING("out of memory"); + firsterr = -ENOMEM; + goto unwind; + } + for (i = 0; i < nrealexp; i++) { + firsterr = torture_create_kthread(rcu_scale_exp, + (void *)i, + exp_tasks[i]); + if (torture_init_error(firsterr)) + goto unwind; + } + } writer_tasks = kzalloc_objs(writer_tasks[0], nrealwriters); writer_durations = kcalloc(nrealwriters, sizeof(*writer_durations), GFP_KERNEL); writer_n_durations = kzalloc_objs(*writer_n_durations, nrealwriters); From d2282f9bc75ca39a8ea5b7fe87191609985aca30 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 16 Jul 2026 12:53:56 -0700 Subject: [PATCH 30/59] rcu: Use this_cpu_{read,write}() for ->cpu_no_qs.b.exp Currently __this_cpu_read() and __this_cpu_write() is used to access the ->cpu_no_qs.b.exp field of the per-CPU rcu_data structure. However, this can fail when the accesses can happen in interrupt handlers, as recently started being exercised by rcutorture. This commit therefore upgrades the uses of __this_cpu_read() and __this_cpu_write() to their interrupt-safe counterparts this_cpu_read() and this_cpu_write(). KCSAN located this issue. Signed-off-by: Paul E. McKenney --- kernel/rcu/tree_exp.h | 4 ++-- kernel/rcu/tree_plugin.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index 82cada459e5d..e03cb3d447a5 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h @@ -731,7 +731,7 @@ static void rcu_exp_need_qs(void) { lockdep_assert_irqs_disabled(); ASSERT_EXCLUSIVE_WRITER_SCOPED(*this_cpu_ptr(&rcu_data.cpu_no_qs.b.exp)); - __this_cpu_write(rcu_data.cpu_no_qs.b.exp, true); + this_cpu_write(rcu_data.cpu_no_qs.b.exp, true); /* Store .exp before .rcu_urgent_qs. */ smp_store_release(this_cpu_ptr(&rcu_data.rcu_urgent_qs), true); set_need_resched_current(); @@ -870,7 +870,7 @@ static void rcu_exp_handler(void *unused) ASSERT_EXCLUSIVE_WRITER_SCOPED(rdp->cpu_no_qs.b.exp); if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) || - __this_cpu_read(rcu_data.cpu_no_qs.b.exp)) + this_cpu_read(rcu_data.cpu_no_qs.b.exp)) return; if (rcu_is_cpu_rrupt_from_idle() || (IS_ENABLED(CONFIG_PREEMPT_COUNT) && preempt_bh_enabled)) { diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 9ba136a4233a..c3db1ad4a280 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -599,7 +599,7 @@ rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags) */ static notrace bool rcu_preempt_need_deferred_qs(struct task_struct *t) { - return (__this_cpu_read(rcu_data.cpu_no_qs.b.exp) || + return (this_cpu_read(rcu_data.cpu_no_qs.b.exp) || READ_ONCE(t->rcu_read_unlock_special.s)) && rcu_preempt_depth() == 0; } @@ -981,7 +981,7 @@ static void rcu_qs(void) trace_rcu_grace_period(TPS("rcu_sched"), __this_cpu_read(rcu_data.gp_seq), TPS("cpuqs")); __this_cpu_write(rcu_data.cpu_no_qs.b.norm, false); - if (__this_cpu_read(rcu_data.cpu_no_qs.b.exp)) + if (this_cpu_read(rcu_data.cpu_no_qs.b.exp)) rcu_report_exp_rdp(this_cpu_ptr(&rcu_data)); } From a050e671d3545cf1890ab764387958be6c542f6c Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 16 Jul 2026 13:05:00 -0700 Subject: [PATCH 31/59] rcu: Use WRITE_ONCE() for ->rcu_need_heavy_qs Currently raw_cpu_write() is used to clear the ->rcu_need_heavy_qs field of the per-CPU rcu_data structure. However, on x86 this is a normal assignment, which does not play well with concurrent accesses. This commit therefore upgrades the uses of raw_cpu_write() to its concurrency-safe counterpart WRITE_ONCE() of an rdp pointer obtained from this_cpu_ptr(&rcu_data). KCSAN located this issue. Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 03a43d3d2616..91d5b4dd08bf 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -357,9 +357,10 @@ bool rcu_watching_zero_in_eqs(int cpu, int *vp) */ notrace void rcu_momentary_eqs(void) { + struct rcu_data *rdp = this_cpu_ptr(&rcu_data); int seq; - raw_cpu_write(rcu_data.rcu_need_heavy_qs, false); + WRITE_ONCE(rdp->rcu_need_heavy_qs, false); seq = ct_state_inc(2 * CT_RCU_WATCHING); /* It is illegal to call this from idle state. */ WARN_ON_ONCE(!(seq & CT_RCU_WATCHING)); From 4137d69124a29506a09756af0aaf0c7acd2a60f6 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Sat, 18 Jul 2026 13:15:32 -0400 Subject: [PATCH 32/59] rcu: Remove unused expedited_need_qs field from rcu_state The ->expedited_need_qs counter was part of the old expedited grace-period machinery that spun waiting for each CPU to check in. The current implementation instead tracks holdout CPUs via the rcu_node structures' ->expmask fields and waits on the ->exp_wq[] wait queues, so nothing reads or writes ->expedited_need_qs any longer. Remove the field to avoid wasting space in rcu_state and to keep readers of the expedited code from searching for nonexistent users. Signed-off-by: Joel Fernandes Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.h | 1 - 1 file changed, 1 deletion(-) diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 4069132f9d44..587bd71396ea 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -391,7 +391,6 @@ struct rcu_state { struct mutex exp_mutex; /* Serialize expedited GP. */ struct mutex exp_wake_mutex; /* Serialize wakeup. */ unsigned long expedited_sequence; /* Take a ticket. */ - atomic_t expedited_need_qs; /* # CPUs left to check in. */ struct swait_queue_head expedited_wq; /* Wait for check-ins. */ int ncpus_snap; /* # CPUs seen last time. */ u8 cbovld; /* Callback overload now? */ From 545b3ac4c1db3c455cbe1bd1ed305e54ad87512e Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Sat, 18 Jul 2026 13:15:34 -0400 Subject: [PATCH 33/59] rcu: Remove unused func parameter from callback-enqueue functions Ever since the kvfree_rcu() tracing moved out of the callback-enqueue path, rcutree_enqueue() no longer looks at the callback function pointer: By the time it is invoked, __call_rcu_common() has already stored the function into rhp->func, and the enqueue path only adds the rcu_head to the segmented callback list and emits tracepoints that do not take the function pointer. Nevertheless, the function pointer is still threaded through call_rcu_core(), call_rcu_nocb(), and rcutree_enqueue(), forcing each level to carry a dead argument. Remove the parameter from all three functions, from the no-CBs stub, and from the corresponding declarations. Anything needing the callback function can still get it from rhp->func. No functional change. Signed-off-by: Joel Fernandes Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 10 +++++----- kernel/rcu/tree.h | 2 +- kernel/rcu/tree_nocb.h | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 91d5b4dd08bf..390dad82675d 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -3024,7 +3024,7 @@ static int __init rcu_spawn_core_kthreads(void) return 0; } -static void rcutree_enqueue(struct rcu_data *rdp, struct rcu_head *head, rcu_callback_t func) +static void rcutree_enqueue(struct rcu_data *rdp, struct rcu_head *head) { rcu_segcblist_enqueue(&rdp->cblist, head); trace_rcu_callback(rcu_state.name, head, @@ -3036,9 +3036,9 @@ static void rcutree_enqueue(struct rcu_data *rdp, struct rcu_head *head, rcu_cal * Handle any core-RCU processing required by a call_rcu() invocation. */ static void call_rcu_core(struct rcu_data *rdp, struct rcu_head *head, - rcu_callback_t func, unsigned long flags) + unsigned long flags) { - rcutree_enqueue(rdp, head, func); + rcutree_enqueue(rdp, head); /* * If called from an extended quiescent state, invoke the RCU * core in order to force a re-evaluation of RCU's idleness. @@ -3179,9 +3179,9 @@ __call_rcu_common(struct rcu_head *head, rcu_callback_t func, bool lazy_in) check_cb_ovld(rdp); if (unlikely(rcu_rdp_is_offloaded(rdp))) - call_rcu_nocb(rdp, head, func, flags, lazy); + call_rcu_nocb(rdp, head, flags, lazy); else - call_rcu_core(rdp, head, func, flags); + call_rcu_core(rdp, head, flags); local_irq_restore(flags); } diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 587bd71396ea..ca88aaa29d61 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -507,7 +507,7 @@ static bool wake_nocb_gp(struct rcu_data *rdp); static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, unsigned long j, bool lazy); static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head, - rcu_callback_t func, unsigned long flags, bool lazy); + unsigned long flags, bool lazy); static void __maybe_unused __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_empty, unsigned long flags); static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp, int level); diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index 373b877cf171..95dbf33462a2 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -603,13 +603,13 @@ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, } static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head, - rcu_callback_t func, unsigned long flags, bool lazy) + unsigned long flags, bool lazy) { bool was_alldone; if (!rcu_nocb_try_bypass(rdp, head, &was_alldone, flags, lazy)) { /* Not enqueued on bypass but locked, do regular enqueue */ - rcutree_enqueue(rdp, head, func); + rcutree_enqueue(rdp, head); __call_rcu_nocb_wake(rdp, was_alldone, flags); /* unlocks */ } } @@ -1666,7 +1666,7 @@ static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, } static void call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *head, - rcu_callback_t func, unsigned long flags, bool lazy) + unsigned long flags, bool lazy) { WARN_ON_ONCE(1); /* Should be dead code! */ } From 2aaee3fc84e10c5949e24a8778414ade2db44a55 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 20 Jul 2026 16:36:55 -0700 Subject: [PATCH 34/59] rcu: Mark accesses to rdp->rcu_cpu_has_work Although the rdp->rcu_cpu_has_work field is accessed only by the corresponding CPU, it can be accessed by both interrupt handlers via invoke_rcu_core_kthread() and at task level via rcu_cpu_kthread(). This means that we need this_cpu_read() rather than __this_cpu_read(), this_cpu_write() rather than __this_cpu_write(), and READ_ONCE() rather than plain C-language loads. The exception is the boot-time rcu_spawn_core_kthreads(), which cannot race with kthreads that have not yet been spawned. This commit therefore makes it so. KCSAN located this issue. Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 390dad82675d..c9780e7c0e2a 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2671,7 +2671,7 @@ static void rcu_do_batch(struct rcu_data *rdp) // reporting, so check time limits for them. if (rdp->rcu_cpu_kthread_status == RCU_KTHREAD_RUNNING && rcu_do_batch_check_time(count, tlimit, jlimit_check, jlimit)) { - rdp->rcu_cpu_has_work = 1; + WRITE_ONCE(rdp->rcu_cpu_has_work, 1); break; } } @@ -2931,7 +2931,7 @@ static void invoke_rcu_core_kthread(void) unsigned long flags; local_irq_save(flags); - __this_cpu_write(rcu_data.rcu_cpu_has_work, 1); + this_cpu_write(rcu_data.rcu_cpu_has_work, 1); t = __this_cpu_read(rcu_data.rcu_cpu_kthread_task); if (t != NULL && t != current) rcu_wake_cond(t, __this_cpu_read(rcu_data.rcu_cpu_kthread_status)); @@ -2958,7 +2958,7 @@ static void rcu_cpu_kthread_park(unsigned int cpu) static int rcu_cpu_kthread_should_run(unsigned int cpu) { - return __this_cpu_read(rcu_data.rcu_cpu_has_work); + return this_cpu_read(rcu_data.rcu_cpu_has_work); } /* @@ -2979,7 +2979,7 @@ static void rcu_cpu_kthread(unsigned int cpu) local_bh_disable(); *statusp = RCU_KTHREAD_RUNNING; local_irq_disable(); - work = *workp; + work = READ_ONCE(*workp); WRITE_ONCE(*workp, 0); local_irq_enable(); if (work) From 5976694882f6ebccd0cf7c81a6fdf5c1e08d9d67 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Mon, 13 Jul 2026 18:31:26 +0800 Subject: [PATCH 35/59] rcu-tasks: Update comments in call_rcu_tasks_generic() This commit updates the comment "We can't create the thread unless interrupts are enabled." from 'commit 4929c913bda5 ("rcu: Make call_rcu_tasks() tolerate first call with irqs disabled")' to be more clear and also to cover deferred wakeup and to take into account the fact that kthread creation has been move to core_initcall() time by 'commit c63eb17ff06d ("rcu: Create call_rcu_tasks() kthread at boot time")'. Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 79aa5d51388c..bc49698a3bcc 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -395,7 +395,11 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags); } rcu_read_unlock(); - /* We can't create the thread unless interrupts are enabled. */ + // We can't create the kthread with interrupts disabled because a + // scheduler spinlock might be held, so kthread creation is deferred + // until core_initcall() time. Similarly, wakeups are deferred using + // irq_work in order to avoid potential scheduler-lock-deadlock + // lockdep splats. if (needwake && READ_ONCE(rtp->kthread_ptr)) irq_work_queue(&rtpcp->rtp_irq_work); } From 7a455d3caef8c1cae44b0b0ea103d1d9fb197aa6 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Thu, 16 Jul 2026 17:57:54 +0800 Subject: [PATCH 36/59] rcu-tasks: Dump rtpcp->lazy_timer status in show_rcu_tasks_generic_gp_kthread() Add 'P' flag to the diagnostic line to indicate whether any per-cpu rtpcp's lazy_timer is pending. this helps diagnose stalls where rcu-task callbacks are queued but the kthread stay sleep because the lazy_timer has not yet fired and no grace period has started. The output is as follows: [ 31.319540][ T77] call_rcu_tasks() has failed boot-time tests. [ 31.320205][ T77] rcu_tasks: RTGS_WAIT_CBS(11) since 7518 g:4 i:0 kCuUP l:150000 Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index bc49698a3bcc..0306a0568f01 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -723,6 +723,7 @@ static void show_rcu_tasks_generic_gp_kthread(struct rcu_tasks *rtp, char *s) bool havecbs = false; bool haveurgent = false; bool haveurgentcbs = false; + bool havependtimer = false; for_each_possible_cpu(cpu) { struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); @@ -733,10 +734,12 @@ static void show_rcu_tasks_generic_gp_kthread(struct rcu_tasks *rtp, char *s) haveurgent = true; if (!data_race(rcu_segcblist_empty(&rtpcp->cblist)) && data_race(rtpcp->urgent_gp)) haveurgentcbs = true; - if (havecbs && haveurgent && haveurgentcbs) + if (data_race(timer_pending(&rtpcp->lazy_timer))) + havependtimer = true; + if (havecbs && haveurgent && haveurgentcbs && havependtimer) break; } - pr_info("%s: %s(%d) since %lu g:%lu i:%lu %c%c%c%c l:%lu %s\n", + pr_info("%s: %s(%d) since %lu g:%lu i:%lu %c%c%c%c%c l:%lu %s\n", rtp->kname, tasks_gp_state_getname(rtp), data_race(rtp->gp_state), jiffies - data_race(rtp->gp_jiffies), @@ -746,6 +749,7 @@ static void show_rcu_tasks_generic_gp_kthread(struct rcu_tasks *rtp, char *s) ".C"[havecbs], ".u"[haveurgent], ".U"[haveurgentcbs], + ".P"[havependtimer], rtp->lazy_jiffies, s); } From f99dc9e288c5a2e191cca8a9af59113a8bc2768a Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 20 Jul 2026 08:27:59 -0700 Subject: [PATCH 37/59] rcu-tasks: Fix IRQ read lock/unlock data race As noted by Marco Elver: rcu_read_lock_trace() .... t->trc_reader_scp = __srcu_read_lock_fast(&rcu_tasks_trace_srcu_struct); rcu_read_unlock_trace() < ... var decls only ... > scp = t->trc_reader_scp; This constitutes a data race between these two accesses to t->trc_reader_scp. If rcu_read_lock_trace() were to tear its store, this value would be corrupted. This commit therefore defers the rcu_read_lock_untrace() function's load from t->trc_reader_scp until after it has verified that this is the outermost rcu_read_unlock_trace(). With this change, the interrupt handler increments and decrements t->trc_reader_nesting and does not access t->trc_reader_scp, thus avoiding the data race. KCSAN located this issue. Signed-off-by: Paul E. McKenney --- include/linux/rcupdate_trace.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h index fd3ddeb6aa3b..70decf877348 100644 --- a/include/linux/rcupdate_trace.h +++ b/include/linux/rcupdate_trace.h @@ -126,11 +126,13 @@ static inline void rcu_read_unlock_trace(void) struct srcu_ctr __percpu *scp; struct task_struct *t = current; - scp = t->trc_reader_scp; - barrier(); // scp before nesting to protect against interrupt handler. n = READ_ONCE(t->trc_reader_nesting) - 1; - WRITE_ONCE(t->trc_reader_nesting, n); - if (!n) { + if (n) { + WRITE_ONCE(t->trc_reader_nesting, n); + } else { + scp = t->trc_reader_scp; // Compiler cannot hoist load due to data raciness. + barrier(); // scp before nesting to protect against interrupt handler. + WRITE_ONCE(t->trc_reader_nesting, n); if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB)) smp_mb(); // Placeholder for more selective ordering __srcu_read_unlock_fast(&rcu_tasks_trace_srcu_struct, scp); From 8fe479568cddbe7f2023069a539d4040e05f3f87 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 21 Jul 2026 21:03:06 -0700 Subject: [PATCH 38/59] rcu: Mark interrupts-enabled accesses to rdp->cpu_no_qs.b.norm The rdp->cpu_no_qs.b.norm field is accessed only by the current CPU, but can be accessed both at task level and from interrupt handlers. All accesses from interrupts-enabled code must therefore be marked. This commit therefore converts from __this_cpu_read() to this_cpu_read(), from __this_cpu_write() to this_cpu_write(), and plain C-language accesses to READ_ONCE() and WRITE_ONCE(), but only in interrupts-enabled code. KCSAN located this issue. Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 2 +- kernel/rcu/tree_plugin.h | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index c9780e7c0e2a..377153e0d934 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2540,7 +2540,7 @@ rcu_check_quiescent_state(struct rcu_data *rdp) * Was there a quiescent state since the beginning of the grace * period? If no, then exit and wait for the next call. */ - if (rdp->cpu_no_qs.b.norm) + if (READ_ONCE(rdp->cpu_no_qs.b.norm)) return; /* diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index c3db1ad4a280..353effac5a37 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -298,11 +298,11 @@ static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp) static void rcu_qs(void) { RCU_LOCKDEP_WARN(preemptible(), "rcu_qs() invoked with preemption enabled!!!\n"); - if (__this_cpu_read(rcu_data.cpu_no_qs.b.norm)) { + if (this_cpu_read(rcu_data.cpu_no_qs.b.norm)) { trace_rcu_grace_period(TPS("rcu_preempt"), __this_cpu_read(rcu_data.gp_seq), TPS("cpuqs")); - __this_cpu_write(rcu_data.cpu_no_qs.b.norm, false); + this_cpu_write(rcu_data.cpu_no_qs.b.norm, false); barrier(); /* Coordinate with rcu_flavor_sched_clock_irq(). */ WRITE_ONCE(current->rcu_read_unlock_special.b.need_qs, false); } @@ -952,7 +952,7 @@ void rcu_read_unlock_strict(void) * __rcu_read_unlock(). */ rdp = this_cpu_ptr(&rcu_data); - rdp->cpu_no_qs.b.norm = false; + WRITE_ONCE(rdp->cpu_no_qs.b.norm, false); rcu_report_qs_rdp(rdp); udelay(rcu_unlock_delay); } @@ -980,7 +980,7 @@ static void rcu_qs(void) return; trace_rcu_grace_period(TPS("rcu_sched"), __this_cpu_read(rcu_data.gp_seq), TPS("cpuqs")); - __this_cpu_write(rcu_data.cpu_no_qs.b.norm, false); + this_cpu_write(rcu_data.cpu_no_qs.b.norm, false); if (this_cpu_read(rcu_data.cpu_no_qs.b.exp)) rcu_report_exp_rdp(this_cpu_ptr(&rcu_data)); } From 7f28a4c8f3f723a3b99cc59425e906b429108c48 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Wed, 22 Jul 2026 18:23:47 +0800 Subject: [PATCH 39/59] rcu: Remove unused rdp parameter from rcu_check_gp_start_stall() The rcu_check_gp_start_stall() works entirely on rnp parameter, and never uses it's rdp parameter. this commit therefore drop it, updating both callers and the declaration. No functional change. Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 2 +- kernel/rcu/tree.h | 3 +-- kernel/rcu/tree_stall.h | 5 ++--- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 377153e0d934..5dbc7506c0c0 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2890,7 +2890,7 @@ static __latent_entropy void rcu_core(void) rcu_accelerate_cbs_unlocked(rnp, rdp); } - rcu_check_gp_start_stall(rnp, rdp, rcu_jiffies_till_stall_check()); + rcu_check_gp_start_stall(rnp, rcu_jiffies_till_stall_check()); /* If there are callbacks ready, invoke them. */ if (!rcu_rdp_is_offloaded(rdp) && rcu_segcblist_ready_cbs(&rdp->cblist) && diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index ca88aaa29d61..c3662c6ba395 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -544,8 +544,7 @@ static bool rcu_nohz_full_cpu(void); static void record_gp_stall_check_time(void); static void rcu_iw_handler(struct irq_work *iwp); static void check_cpu_stall(struct rcu_data *rdp); -static void rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp, - const unsigned long gpssdelay); +static void rcu_check_gp_start_stall(struct rcu_node *rnp, const unsigned long gpssdelay); /* Forward declarations for tree_exp.h. */ static void sync_rcu_do_polled_gp(struct work_struct *wp); diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index 45b9856ccd2b..02684dadb4ee 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -998,8 +998,7 @@ EXPORT_SYMBOL_GPL(show_rcu_gp_kthreads); * This function checks for grace-period requests that fail to motivate * RCU to come out of its idle mode. */ -static void rcu_check_gp_start_stall(struct rcu_node *rnp, struct rcu_data *rdp, - const unsigned long gpssdelay) +static void rcu_check_gp_start_stall(struct rcu_node *rnp, const unsigned long gpssdelay) { unsigned long flags; unsigned long j; @@ -1074,7 +1073,7 @@ void rcu_fwd_progress_check(unsigned long j) __func__, jiffies - data_race(READ_ONCE(rcu_state.gp_end))); preempt_disable(); rdp = this_cpu_ptr(&rcu_data); - rcu_check_gp_start_stall(rdp->mynode, rdp, j); + rcu_check_gp_start_stall(rdp->mynode, j); preempt_enable(); } for_each_possible_cpu(cpu) { From 27d73e81195b395270117ff77c47be2ed9b09b12 Mon Sep 17 00:00:00 2001 From: Itai Handler Date: Thu, 23 Jul 2026 10:41:38 +0300 Subject: [PATCH 40/59] rcu: Mark accesses to ->rcu_urgent_qs and ->rcu_need_heavy_qs rcu_all_qs() and rcu_note_context_switch() read/clear the per-CPU ->rcu_urgent_qs and ->rcu_need_heavy_qs flags with plain raw_cpu_read() and this_cpu_write(), while the RCU core clears them with WRITE_ONCE() in rcu_disable_urgency_upon_qs(). KCSAN flags the resulting same-CPU race: BUG: KCSAN: data-race in rcu_all_qs / rcu_disable_urgency_upon_qs It is benign -- the flags are advisory and rcu_all_qs() re-reads ->rcu_urgent_qs with smp_load_acquire() before acting on it -- but these are the last unmarked accesses to the two flags; every other access already uses READ_ONCE()/WRITE_ONCE()/smp_*. Mark them to match. No functional change. Reproduced on a PREEMPT_NONE, CONFIG_KCSAN_INTERRUPT_WATCHER=y kernel with a pthreads program whose threads (two per CPU) loop reading a large file: for (;;) { int fd = open("/proc/kallsyms", O_RDONLY); while (read(fd, buf, sizeof(buf)) > 0) ; close(fd); } The read()s drive cond_resched() -> rcu_all_qs() while the busy CPUs keep the grace period urgent, so the RCU core clears the flags concurrently. Fixes: 2dba13f0b6c2 ("rcu: Switch urgent quiescent-state requests to rcu_data structure") Signed-off-by: Itai Handler Signed-off-by: Paul E. McKenney --- kernel/rcu/tree_plugin.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 353effac5a37..844a9ced67cc 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -996,7 +996,7 @@ void rcu_all_qs(void) { unsigned long flags; - if (!raw_cpu_read(rcu_data.rcu_urgent_qs)) + if (!READ_ONCE(*raw_cpu_ptr(&rcu_data.rcu_urgent_qs))) return; preempt_disable(); // For CONFIG_PREEMPT_COUNT=y kernels /* Load rcu_urgent_qs before other flags. */ @@ -1004,8 +1004,8 @@ void rcu_all_qs(void) preempt_enable(); return; } - this_cpu_write(rcu_data.rcu_urgent_qs, false); - if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs))) { + WRITE_ONCE(*this_cpu_ptr(&rcu_data.rcu_urgent_qs), false); + if (unlikely(READ_ONCE(*this_cpu_ptr(&rcu_data.rcu_need_heavy_qs)))) { local_irq_save(flags); rcu_momentary_eqs(); local_irq_restore(flags); @@ -1025,8 +1025,8 @@ void rcu_note_context_switch(bool preempt) /* Load rcu_urgent_qs before other flags. */ if (!smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) goto out; - this_cpu_write(rcu_data.rcu_urgent_qs, false); - if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs))) + WRITE_ONCE(*this_cpu_ptr(&rcu_data.rcu_urgent_qs), false); + if (unlikely(READ_ONCE(*this_cpu_ptr(&rcu_data.rcu_need_heavy_qs)))) rcu_momentary_eqs(); out: rcu_tasks_qs(current, preempt); From df0ecbc497de851891847260f45805ca6ccb393f Mon Sep 17 00:00:00 2001 From: Zqiang Date: Thu, 23 Jul 2026 18:04:30 +0800 Subject: [PATCH 41/59] rcu: Reduce stack usage in show_rcu_gp_kthreads() When CONFIG_KASAN=y and CONFIG_KASAN_STACK=y builds, the show_rcu_gp_kthreads() exceeds the 1024-byte frame-size limit: make kernel/rcu/tree.o KCFLAGS="-fstack-usage" DESCEND objtool DESCEND bpf/resolve_btfids INSTALL libsubcmd_headers CC kernel/rcu/tree.o In file included from kernel/rcu/tree.c:4998: kernel/rcu/tree_stall.h: In function 'show_rcu_gp_kthreads': kernel/rcu/tree_stall.h:994:1: warning: the frame size of 1656 bytes is larger than 1024 bytes [-Wframe-larger-than=] grep show_rcu kernel/rcu/tree.su tree_nocb.h:1622:13:show_rcu_nocb_state 896 dynamic,bounded tree_stall.h:933:6:show_rcu_gp_kthreads 1784 dynamic,bounded tree_stall.h:1102:13:sysrq_show_rcu 16 static Wrap the pr_info() into two noinline_for_stack helpers function: show_rcu_state() print rcu_state status, and show_rcu_node() print single rcu_node status. After apply this change: grep show_rcu kernel/rcu/tree.su tree_stall.h:955:22:show_rcu_node 696 dynamic,bounded tree_stall.h:930:22:show_rcu_state 872 dynamic,bounded tree_nocb.h:1622:13:show_rcu_nocb_state 896 dynamic,bounded tree_stall.h:972:6:show_rcu_gp_kthreads 544 static tree_stall.h:1113:13:sysrq_show_rcu 16 static Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/tree_stall.h | 47 +++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index 02684dadb4ee..20634edfb44d 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -927,20 +927,13 @@ bool rcu_check_boost_fail(unsigned long gp_state, int *cpup) } EXPORT_SYMBOL_GPL(rcu_check_boost_fail); -/* - * Show the state of the grace-period kthreads. - */ -void show_rcu_gp_kthreads(void) +static noinline_for_stack void show_rcu_state(void) { - unsigned long cbs = 0; - int cpu; unsigned long j; unsigned long ja; unsigned long jr; unsigned long js; unsigned long jw; - struct rcu_data *rdp; - struct rcu_node *rnp; struct task_struct *t = READ_ONCE(rcu_state.gp_kthread); j = jiffies; @@ -957,21 +950,39 @@ void show_rcu_gp_kthreads(void) (long)data_race(READ_ONCE(rcu_get_root()->gp_seq_needed)), data_race(READ_ONCE(rcu_state.gp_max)), data_race(READ_ONCE(rcu_state.gp_flags))); +} + +static noinline_for_stack void show_rcu_node(struct rcu_node *rnp) +{ + pr_info("\trcu_node %d:%d ->gp_seq %ld ->gp_seq_needed %ld ->qsmask %#lx %c%c%c%c ->n_boosts %ld\n", + rnp->grplo, rnp->grphi, + (long)data_race(READ_ONCE(rnp->gp_seq)), + (long)data_race(READ_ONCE(rnp->gp_seq_needed)), + data_race(READ_ONCE(rnp->qsmask)), + ".b"[!!data_race(READ_ONCE(rnp->boost_kthread_task))], + ".B"[!!data_race(READ_ONCE(rnp->boost_tasks))], + ".E"[!!data_race(READ_ONCE(rnp->exp_tasks))], + ".G"[!!data_race(READ_ONCE(rnp->gp_tasks))], + data_race(READ_ONCE(rnp->n_boosts))); +} + +/* + * Show the state of the grace-period kthreads. + */ +void show_rcu_gp_kthreads(void) +{ + unsigned long cbs = 0; + int cpu; + struct rcu_data *rdp; + struct rcu_node *rnp; + + show_rcu_state(); rcu_for_each_node_breadth_first(rnp) { if (ULONG_CMP_GE(READ_ONCE(rcu_state.gp_seq), READ_ONCE(rnp->gp_seq_needed)) && !data_race(READ_ONCE(rnp->qsmask)) && !data_race(READ_ONCE(rnp->boost_tasks)) && !data_race(READ_ONCE(rnp->exp_tasks)) && !data_race(READ_ONCE(rnp->gp_tasks))) continue; - pr_info("\trcu_node %d:%d ->gp_seq %ld ->gp_seq_needed %ld ->qsmask %#lx %c%c%c%c ->n_boosts %ld\n", - rnp->grplo, rnp->grphi, - (long)data_race(READ_ONCE(rnp->gp_seq)), - (long)data_race(READ_ONCE(rnp->gp_seq_needed)), - data_race(READ_ONCE(rnp->qsmask)), - ".b"[!!data_race(READ_ONCE(rnp->boost_kthread_task))], - ".B"[!!data_race(READ_ONCE(rnp->boost_tasks))], - ".E"[!!data_race(READ_ONCE(rnp->exp_tasks))], - ".G"[!!data_race(READ_ONCE(rnp->gp_tasks))], - data_race(READ_ONCE(rnp->n_boosts))); + show_rcu_node(rnp); if (!rcu_is_leaf_node(rnp)) continue; for_each_leaf_node_possible_cpu(rnp, cpu) { From 76892716e187e1bb4be30c8c980204b501c28555 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 24 Jul 2026 14:44:15 -0700 Subject: [PATCH 42/59] rcu: Mark interrupts-enabled accesses to rdp->cpu_no_qs.s The rdp->cpu_no_qs.s field is accessed only by the current CPU, but can be accessed both at task level and from interrupt handlers. All accesses from interrupts-enabled code must therefore be marked. This commit therefore converts from __this_cpu_read() to this_cpu_read(), but only in interrupts-enabled code, that is, the rcu_qs() function. KCSAN located this issue. Signed-off-by: Paul E. McKenney --- kernel/rcu/tree_plugin.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 844a9ced67cc..6904a4f826d1 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -976,7 +976,7 @@ static void __init rcu_bootup_announce(void) static void rcu_qs(void) { RCU_LOCKDEP_WARN(preemptible(), "rcu_qs() invoked with preemption enabled!!!"); - if (!__this_cpu_read(rcu_data.cpu_no_qs.s)) + if (!this_cpu_read(rcu_data.cpu_no_qs.s)) return; trace_rcu_grace_period(TPS("rcu_sched"), __this_cpu_read(rcu_data.gp_seq), TPS("cpuqs")); From ed61912b40dda92ef24014fb0daca7de852f9407 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Mon, 20 Jul 2026 17:49:58 +0800 Subject: [PATCH 43/59] rcu-tasks: Rename tasks_rcu_exit_srcu_stall_timer to tasks_rcu_exit_stall_timer This 'commit 1612160b9127 ("rcu-tasks: Eliminate deadlocks involving do_exit() and RCU tasks")' has reimplemented the rcu_tasks_postscan(), and the synchronize_srcu() has been removed, this commit therefore rename tasks_rcu_exit_srcu_stall_timer to tasks_rcu_exit_stall_timer. Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 0306a0568f01..fce57d90db1a 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -155,8 +155,8 @@ static struct rcu_tasks rt_name = \ #ifdef CONFIG_TASKS_RCU /* Report delay of scan exiting tasklist in rcu_tasks_postscan(). */ -static void tasks_rcu_exit_srcu_stall(struct timer_list *unused); -static DEFINE_TIMER(tasks_rcu_exit_srcu_stall_timer, tasks_rcu_exit_srcu_stall); +static void tasks_rcu_exit_stall(struct timer_list *unused); +static DEFINE_TIMER(tasks_rcu_exit_stall_timer, tasks_rcu_exit_stall); #endif /* Control stall timeouts. Disable with <= 0, otherwise jiffies till stall. */ @@ -1032,8 +1032,8 @@ static void rcu_tasks_postscan(struct list_head *hop) int rtsi = READ_ONCE(rcu_task_stall_info); if (!IS_ENABLED(CONFIG_TINY_RCU)) { - tasks_rcu_exit_srcu_stall_timer.expires = jiffies + rtsi; - add_timer(&tasks_rcu_exit_srcu_stall_timer); + tasks_rcu_exit_stall_timer.expires = jiffies + rtsi; + add_timer(&tasks_rcu_exit_stall_timer); } /* @@ -1086,7 +1086,7 @@ static void rcu_tasks_postscan(struct list_head *hop) } if (!IS_ENABLED(CONFIG_TINY_RCU)) - timer_delete_sync(&tasks_rcu_exit_srcu_stall_timer); + timer_delete_sync(&tasks_rcu_exit_stall_timer); } /* See if tasks are still holding out, complain if so. */ @@ -1158,7 +1158,7 @@ static void rcu_tasks_postgp(struct rcu_tasks *rtp) synchronize_rcu(); } -static void tasks_rcu_exit_srcu_stall(struct timer_list *unused) +static void tasks_rcu_exit_stall(struct timer_list *unused) { #ifndef CONFIG_TINY_RCU int rtsi; @@ -1168,8 +1168,8 @@ static void tasks_rcu_exit_srcu_stall(struct timer_list *unused) __func__, rcu_tasks.kname, rcu_tasks.tasks_gp_seq, tasks_gp_state_getname(&rcu_tasks), jiffies - rcu_tasks.gp_jiffies); pr_info("Please check any exiting tasks stuck between calls to exit_tasks_rcu_start() and exit_tasks_rcu_finish()\n"); - tasks_rcu_exit_srcu_stall_timer.expires = jiffies + rtsi; - add_timer(&tasks_rcu_exit_srcu_stall_timer); + tasks_rcu_exit_stall_timer.expires = jiffies + rtsi; + add_timer(&tasks_rcu_exit_stall_timer); #endif // #ifndef CONFIG_TINY_RCU } From 885d8314e5706ce7d176f2029454246c2d940bb3 Mon Sep 17 00:00:00 2001 From: Zqiang Date: Tue, 21 Jul 2026 17:48:07 +0800 Subject: [PATCH 44/59] rcu-tasks: Fix some comments for call_rcu_tasks() and call_rcu_tasks_rude() The comments for call_rcu_tasks() and call_rcu_tasks_rude() referred to "RCU read-side critical sections", which is ambiguous. specify "rcu-tasks" and "rcu-tasks rude" respectively to match the actual flavor being described. Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/tasks.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index fce57d90db1a..851ac225770c 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1179,7 +1179,7 @@ static void tasks_rcu_exit_stall(struct timer_list *unused) * @func: actual callback function to be invoked after the grace period * * The callback function will be invoked some time after a full grace - * period elapses, in other words after all currently executing RCU + * period elapses, in other words after all currently executing rcu-tasks * read-side critical sections have completed. call_rcu_tasks() assumes * that the read-side critical sections end at a voluntary context * switch (not a preemption!), cond_resched_tasks_rcu_qs(), entry into idle, @@ -1365,8 +1365,8 @@ DEFINE_RCU_TASKS(rcu_tasks_rude, rcu_tasks_rude_wait_gp, call_rcu_tasks_rude, * @func: actual callback function to be invoked after the grace period * * The callback function will be invoked some time after a full grace - * period elapses, in other words after all currently executing RCU - * read-side critical sections have completed. call_rcu_tasks_rude() + * period elapses, in other words after all currently executing rude + * rcu-tasks read-side critical sections have completed. call_rcu_tasks_rude() * assumes that the read-side critical sections end at context switch, * cond_resched_tasks_rcu_qs(), or transition to usermode execution (as * usermode execution is schedulable). As such, there are no read-side @@ -1390,7 +1390,7 @@ static void call_rcu_tasks_rude(struct rcu_head *rhp, rcu_callback_t func) * * Control will return to the caller some time after a rude rcu-tasks * grace period has elapsed, in other words after all currently - * executing rcu-tasks read-side critical sections have elapsed. These + * executing rude rcu-tasks read-side critical sections have elapsed. These * read-side critical sections are delimited by calls to schedule(), * cond_resched_tasks_rcu_qs(), userspace execution (which is a schedulable * context), and (in theory, anyway) cond_resched(). From 0f5fe218d9d4e23d47a49873c9bfacd717b9c145 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 27 Jul 2026 15:11:04 -0700 Subject: [PATCH 45/59] rcu-tasks: Convert cond_resched_tasks_rcu_qs() to static inline In order to make "cc -E" output less annoying, this commit converts cond_resched_tasks_rcu_qs() to static inline. You know, the READ_ONCE() and WRITE_ONCE() macros used to be *so* simple. ;-) Reported-by: Andrew Morton Signed-off-by: Paul E. McKenney --- include/linux/rcupdate.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 5e95acc33989..5e8770fd7b72 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -208,15 +208,15 @@ static inline void exit_tasks_rcu_finish(void) { } /** * cond_resched_tasks_rcu_qs - Report potential quiescent states to RCU * - * This macro resembles cond_resched(), except that it is defined to + * This function resembles cond_resched(), except that it is defined to * report potential quiescent states to RCU-tasks even if the cond_resched() * machinery were to be shut off, as some advocate for PREEMPTION kernels. */ -#define cond_resched_tasks_rcu_qs() \ -do { \ - rcu_tasks_qs(current, false); \ - cond_resched(); \ -} while (0) +static inline void cond_resched_tasks_rcu_qs(void) +{ + rcu_tasks_qs(current, false); + cond_resched(); +} /** * rcu_softirq_qs_periodic - Report RCU and RCU-Tasks quiescent states From 78a38cbf6f20bc8247e93d1149f97c12dba9fbfb Mon Sep 17 00:00:00 2001 From: Zqiang Date: Thu, 9 Jul 2026 18:06:02 +0800 Subject: [PATCH 46/59] srcu: Queue sdp->work when the delay timer is successfully deleted In the cleanup_srcu_struct() function, when iterating over per-cpu's srcu_data, timer_delete_sync(&sdp->delay_work) is called to cancel the delayed work before doing flush_work(&sdp->work). However, suppose that timer_delete_sync() returns 1, which means that it successfully deleted an pending timer before it had a chance to fire. But this also means that the sdp->work will not be queued, so that the subsequent flush_work(&sdp->work) will returns immediately without waiting for anything. Taken together, all of this means that any recently queued SRCU callbacks to not be invoked, which can result in memory leaks, hangs, or worse. Fix this by checking the return value of timer_delete_sync(), if it returns 1, explicitly queue sdp->work so that the callbacks will be invoked and the following flush_work() will correctly wait for all of those callbacks to finish executing. [ Zqiang: Apply feedback from Breno Leitao and kernel test robot. ] Signed-off-by: Zqiang Tested-by: kernel test robot Reviewed-by: Frederic Weisbecker Signed-off-by: Paul E. McKenney --- kernel/rcu/srcutree.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 4a00e90e17fc..02a101824f12 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -701,7 +701,12 @@ void cleanup_srcu_struct(struct srcu_struct *ssp) for_each_possible_cpu(cpu) { struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); - timer_delete_sync(&sdp->delay_work); + // Call srcu_barrier() before this cleanup_srcu_struct() + // to avoid triggering this WARN_ON(). + if (WARN_ON(timer_delete_sync(&sdp->delay_work) && + rcu_segcblist_n_cbs(&sdp->srcu_cblist)) && + rcu_cpu_beenfullyonline(sdp->cpu)) + queue_work_on(sdp->cpu, rcu_gp_wq, &sdp->work); flush_work(&sdp->work); if (WARN_ON(rcu_segcblist_n_cbs(&sdp->srcu_cblist))) return; /* Forgot srcu_barrier(), so just leak it! */ From 50df5afa6d410ddfa32f58cdb2976d16fc3785b3 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Sat, 25 Apr 2026 10:50:57 -0700 Subject: [PATCH 47/59] rcutorture: Check for immediate deboosting at reader end This commit adds a check for failure to have fully deboosted a multi-segmented RCU reader at the end of the full read-side critical section. This check only happens for fully task-level readers, because a a handler might have interrupted an already-boosted task-level RCU reader, and a reader in that handler could then cause false positives. The first failed check (due to an RCU reader that was not immediately deboosted) causes a splat, but only when the disabled-by-default deboost_timeliness_check module parameter is enabled. Regardless of the value of this parameter, it produces a list of the segments making up that RCU reader following a "Slow-deboost rcutorture reader segments" heading. Subsequent failures fail silently, all in the name of keeping console output down to a dull roar. Although most uses of RCU priority boosting serve as debugging aids, this might change, and in fact might already have changed. And allowing (for example) RCU priority boosting to persist until the next scheduler tick could cause an aggressively real-time system to miss sub-millisecond deadlines. So we do need to find this sort of problem during testing, and preferably not in the field. The name and type of the newly added rcu_torture_ops function pointer (named "->is_task_rcu_boosted()") may need to change should other end-of-reader checks be needed. But let's start simple. Oh, and Claude figured out that rcu_is_task_rcu_boosted() could be lockless. Perhaps there is hope for AI yet! ;-) [ paulmck: Apply Akira Yokosawa feedback. ] Signed-off-by: Paul E. McKenney --- kernel/rcu/rcu.h | 7 +++++++ kernel/rcu/rcutorture.c | 24 ++++++++++++++++++++++++ kernel/rcu/tree_plugin.h | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+) diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index fa6d30ce73d1..14faa11ef23c 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h @@ -695,4 +695,11 @@ static inline int rcu_stall_notifier_call_chain(unsigned long val, void *v) { re void synchronize_rcu_trivial_preempt(void); #endif // #ifdef CONFIG_TRIVIAL_PREEMPT_RCU +#if defined(CONFIG_RCU_TORTURE_TEST) && defined(CONFIG_RCU_BOOST) +bool rcu_is_task_rcu_boosted(void); +#else // #if defined(CONFIG_RCU_TORTURE_TEST) && defined(CONFIG_RCU_BOOST) +static inline bool rcu_is_task_rcu_boosted(void) { return false; } +#endif // #else // #if defined(CONFIG_RCU_TORTURE_TEST) && defined(CONFIG_RCU_BOOST) + + #endif /* __LINUX_RCU_H */ diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index ffeca1b7e170..1a033ae1f1eb 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -80,6 +80,7 @@ MODULE_AUTHOR("Paul E. McKenney and Josh Triplett is_task_rcu_boosted && cur_ops->is_task_rcu_boosted() && + !in_serving_softirq() && !in_hardirq() && !in_nmi() && + READ_ONCE(firsttime) && xchg(&firsttime, 0)) { + WARN_ON_ONCE(deboost_timeliness_check); + nsegs = rtors.rtrsp - rtors.rtseg; + nsegs = clamp_val(nsegs, 0, RCUTORTURE_RDR_MAX_SEGS); + pr_alert("Slow-deboost rcutorture reader segments:\n"); + rcu_torture_dump_read_segs(rtors.rtseg, nsegs); + } return true; } diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 95ad967adcf3..d950722eb8f9 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -1320,6 +1320,41 @@ static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp) wake_up_process(t); /* get to TASK_INTERRUPTIBLE quickly. */ } +#ifdef CONFIG_RCU_TORTURE_TEST + +/* + * Is the current task RCU priority boosted? This is used by + * rcutorture to check that tasks are always deboosted once then exit + * an RCU read-side critical section, no matter how many overlapping + * segments of rcu_read_lock(), preempt_disable(), local_bh_disable(), + * or local_irq_disable() made up that reader. + * + * The lockless accesses in rt_mutex_owner(&rnp->boost_mtx.rtmutex) + * are safe because tasks release ->boost_mtx when they own it, they + * cannot be boosted unless current->rcu_blocked_node is non-NULL, + * current->rcu_blocked_node is modified only by the current task, + * rt_mutex_owner() uses READ_ONCE() on the ->owner field, and the owner + * switching among other tasks cannot force an equality comparison. + */ +bool rcu_is_task_rcu_boosted(void) +{ + bool ret; + struct rcu_node *rnp; + struct task_struct *t = current; + + preempt_disable(); // Stabilize ->rcu_blocked_node + rnp = t->rcu_blocked_node; + if (!rnp) + ret = false; + else + ret = (rt_mutex_owner(&rnp->boost_mtx.rtmutex) == t); + preempt_enable(); + return ret; +} +EXPORT_SYMBOL_GPL(rcu_is_task_rcu_boosted); + +#endif // #ifdef CONFIG_RCU_TORTURE_TEST + #else /* #ifdef CONFIG_RCU_BOOST */ static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags) From 5ac693414b51acb8c4b92813ca3ab4fc6686ddf9 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 7 May 2026 15:03:21 -0700 Subject: [PATCH 48/59] rcutorture: Test RCU readers from hardware interrupt handlers Although rcutorture has long had the irqreader module parameter, this parameter results only in RCU readers in softirq handlers, specifically, timers. This commit therefore uses smp_call_function_single() to test RCU readers in real hardware interrupt handlers, thus providing the full effect from the irqreader module parameter. However, consistency/debug checks must account for the possibility that the smp_call_function_single() handler function is directly invoked from the idle loop, in which case, for example, in_hardirq() will return false. This commit uses a per-CPU variable to record being in the rcu_torture_irq() smp_call_function_single() handler function. Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 77 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 71 insertions(+), 6 deletions(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 1a033ae1f1eb..57cae3fe23fd 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -213,6 +213,7 @@ static long n_rcu_torture_boost_ktrerror; static long n_rcu_torture_boost_failure; static long n_rcu_torture_boosts; static atomic_long_t n_rcu_torture_timers; +static atomic_long_t n_rcu_torture_irqs; static long n_barrier_attempts; static long n_barrier_successes; /* did rcu_barrier test succeed? */ static unsigned long n_read_exits; @@ -2127,6 +2128,8 @@ static void rcu_torture_reader_do_mbchk(long myid, struct rcu_torture *rtp, smp_store_release(&rtrcp_assigner->rtc_chkrdr, -1); // Assigner can again assign. } +static DEFINE_PER_CPU(bool, torture_in_scf_handler); + // Verify the specified RCUTORTURE_RDR* state. #define ROEC_ARGS "%s %s: Current %#x To add %#x To remove %#x preempt_count() %#x\n", __func__, s, curstate, new, old, preempt_count() static void rcutorture_one_extend_check(char *s, int curstate, int new, int old) @@ -2136,7 +2139,7 @@ static void rcutorture_one_extend_check(char *s, int curstate, int new, int old) if (!IS_ENABLED(CONFIG_RCU_TORTURE_TEST_CHK_RDR_STATE) || in_nmi()) return; - WARN_ONCE(!(curstate & RCUTORTURE_RDR_IRQ) && irqs_disabled() && !in_hardirq(), ROEC_ARGS); + WARN_ONCE(!(curstate & RCUTORTURE_RDR_IRQ) && irqs_disabled() && !in_hardirq() && !this_cpu_read(torture_in_scf_handler), ROEC_ARGS); WARN_ONCE((curstate & RCUTORTURE_RDR_IRQ) && !irqs_disabled(), ROEC_ARGS); // If CONFIG_PREEMPT_COUNT=n, further checks are unreliable. @@ -2153,7 +2156,7 @@ static void rcutorture_one_extend_check(char *s, int curstate, int new, int old) // Interrupt handlers have all sorts of stuff disabled, so ignore // unintended disabling. - if (in_serving_softirq() || in_hardirq()) + if (in_serving_softirq() || in_hardirq() || this_cpu_read(torture_in_scf_handler)) return; WARN_ONCE(cur_ops->extendables && @@ -2345,12 +2348,19 @@ rcutorture_extend_mask(int oldmask, struct torture_random_state *trsp) mask |= RCUTORTURE_RDR_RCU_1; } + /* + * Don't mess with interrupt masking in interrupt handlers. + */ + if (in_hardirq() || this_cpu_read(torture_in_scf_handler)) + mask &= ~(preempts_irq | bhs); + /* * Can't enable bh w/irq disabled. */ if (mask & RCUTORTURE_RDR_IRQ) mask |= oldmask & bhs; + /* * Ideally these sequences would be detected in debug builds * (regardless of RT), but until then don't stop testing @@ -2605,6 +2615,7 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) return false; rtors.rtrsp = rcutorture_loop_extend(&rtors.readstate, trsp, rtors.rtrsp); rcu_torture_one_read_end(&rtors, trsp); + // This splat will happen on systems built with CONFIG_IRQ_WORK=n // and on systems where arch_irq_work_has_interrupt() returns false. // It might also happen on systems using a short-duration clock @@ -2643,7 +2654,7 @@ static void rcu_torture_timer(struct timer_list *unused) atomic_long_inc(&n_rcu_torture_timers); (void)rcu_torture_one_read(this_cpu_ptr(&rcu_torture_timer_rand), -1); - /* Test call_rcu() invocation from interrupt handler. */ + /* Test call_rcu() invocation from softirq handler. */ if (cur_ops->call) { struct rcu_head *rhp = kmalloc_obj(*rhp, GFP_NOWAIT); @@ -2652,6 +2663,41 @@ static void rcu_torture_timer(struct timer_list *unused) } } +static DEFINE_TORTURE_RANDOM_PERCPU(rcu_torture_irq_rand); + +/* + * RCU torture reader from timer handler. Dereferences rcu_torture_current, + * incrementing the corresponding element of the pipeline array. The + * counter in the element should never be greater than 1, otherwise, the + * RCU implementation is broken. + * + * Note that on some systems, "interrupts" from idle are direct calls + * rather than interrupts. The torture_in_scf_handler per-CPU variable + * accounts for this case. + */ +static void rcu_torture_irq(void *unused) +{ + WARN_ON_ONCE(in_nmi()); + lockdep_assert_irqs_disabled(); + atomic_long_inc(&n_rcu_torture_irqs); + this_cpu_write(torture_in_scf_handler, true); + (void)rcu_torture_one_read(this_cpu_ptr(&rcu_torture_irq_rand), -1); + this_cpu_write(torture_in_scf_handler, false); + + // Test call_rcu() invocation from interrupt handler. Interrupts + // will always be disabled here, even in CONFIG_PREEMPT_RT=y kernels. + // The "right" thing to do would be to create a special-purpose + // lockless or raw-spinlock-protected allocator, but in the meantime, + // skip testing call_rcu() from interrupt handlers in kernels built + // with either CONFIG_PREEMPT_RT=y or CONFIG_PROVE_LOCKING=y. + if (cur_ops->call && !IS_ENABLED(CONFIG_PROVE_LOCKING) && !IS_ENABLED(CONFIG_PREEMPT_RT)) { + struct rcu_head *rhp = kmalloc_obj(*rhp, GFP_NOWAIT); + + if (rhp) + cur_ops->call(rhp, rcu_torture_timer_cb); + } +} + /* * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, * incrementing the corresponding element of the pipeline array. The @@ -2661,6 +2707,7 @@ static void rcu_torture_timer(struct timer_list *unused) static int rcu_torture_reader(void *arg) { + unsigned long lastscf = jiffies; unsigned long lastsleep = jiffies; long myid = (long)arg; int mynumonline = myid; @@ -2674,8 +2721,25 @@ rcu_torture_reader(void *arg) tick_dep_set_task(current, TICK_DEP_BIT_RCU); // CPU bound, so need tick. do { if (irqreader && cur_ops->irq_capable) { - if (!timer_pending(&t)) + if (!timer_pending(&t)) { + int cpu; + mod_timer(&t, jiffies + 1); + preempt_disable(); + cpu = torture_random(&rand) % nr_cpu_ids; + if (!cpu_online(cpu)) { + cpu = cpumask_next(cpu, cpu_online_mask); + if (cpu >= nr_cpu_ids) + cpu = cpumask_next(-1, cpu_online_mask); + } + // An smp_call_function_single() to self is not an interrupt! + if (cpu != smp_processor_id() && + time_after(jiffies, lastscf + HZ * nrealreaders / 50)) { + smp_call_function_single(cpu, rcu_torture_irq, NULL, 0); + lastscf = jiffies; + } + preempt_enable(); + } } if (!rcu_torture_one_read(&rand, myid) && !torture_must_stop()) schedule_timeout_interruptible(HZ); @@ -2951,10 +3015,11 @@ rcu_torture_stats_print(void) atomic_read(&n_rcu_torture_mbchk_fail), atomic_read(&n_rcu_torture_mbchk_tries), n_rcu_torture_barrier_error, n_rcu_torture_boost_ktrerror); - pr_cont("rtbf: %ld rtb: %ld nt: %ld ", + pr_cont("rtbf: %ld rtb: %ld nt: %ld ni: %ld ", n_rcu_torture_boost_failure, n_rcu_torture_boosts, - atomic_long_read(&n_rcu_torture_timers)); + atomic_long_read(&n_rcu_torture_timers), + atomic_long_read(&n_rcu_torture_irqs)); if (updownreaders) pr_cont("ndowns: %lu nups: %lu nhrt: %lu nmigrates: %lu ", ndowns, nups, nunexpired, nmigrates); torture_onoff_stats(); From f424566105c946ef42f2643230362598c06a29f1 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 4 Jun 2026 20:42:58 -0700 Subject: [PATCH 49/59] rcutorture: Use cpumask_next_wrap() in rcu_torture_preempt() The rcu_torture_preempt() function uses cpumask_next(), and if that returns an out-of-bounds result, re-invokes cpumask_next() on -1. Which is exactly what cpumask_next_wrap() does. This commit therefore saves a couple of lines by instead using cpumask_next_wrap(). This was reported by metacode when asked to look for opportunities to use cpumask_next_wrap() in kernel/rcu. Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 57cae3fe23fd..afd5045263be 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -4183,9 +4183,7 @@ static int rcu_torture_preempt(void *unused) // Wait for preempt_interval ms with up to 100us fuzz. torture_hrtimeout_ms(preempt_interval, 100, &rand); // Select online CPU. - cpu = cpumask_next(cpu, cpu_online_mask); - if (cpu >= nr_cpu_ids) - cpu = cpumask_next(-1, cpu_online_mask); + cpu = cpumask_next_wrap(cpu, cpu_online_mask); WARN_ON_ONCE(cpu >= nr_cpu_ids); // Move to that CPU, if can't do so, retry later. if (torture_sched_setaffinity(current->pid, cpumask_of(cpu), false)) From e7b6d505c7080d6d40e5dbbeb01b612de9f08a78 Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Fri, 12 Jun 2026 10:34:13 +0800 Subject: [PATCH 50/59] rcutorture: Use task_state_to_char() for task-state reporting Use the kernel's standard symbolic task-state representation instead of printing raw hexadecimal task-state values. Suggested-by: Zqiang Co-developed-by: Wang Lian Signed-off-by: Wang Lian Signed-off-by: Kunwu Chan Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index afd5045263be..c9fb375701f8 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -3076,10 +3076,10 @@ rcu_torture_stats_print(void) if (cur_ops->get_gp_data) cur_ops->get_gp_data(&flags, &gp_seq); wtp = READ_ONCE(writer_task); - pr_alert("??? Writer stall state %s(%d) g%lu f%#x ->state %#x cpu %d\n", + pr_alert("??? Writer stall state %s(%d) g%lu f%#x ->state %c cpu %d\n", rcu_torture_writer_state_getname(), rcu_torture_writer_state, gp_seq, flags, - wtp == NULL ? ~0U : wtp->__state, + wtp == NULL ? '?' : task_state_to_char(wtp), wtp == NULL ? -1 : (int)task_cpu(wtp)); if (!splatted && wtp) { sched_show_task(wtp); From 75a88fe00b67b2a065461940b5392dc5487acfff Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 25 Jun 2026 10:57:06 -0700 Subject: [PATCH 51/59] rcutorture: Add nwriters module parameter Believe it or not, there are people who would like to run rcutorture without actually torturing RCU. For example, some people would like to induce various types of stall warnings without placing any unnecessary additional overhead on their systems running in production. And rcutorture provides the stall_cpu, stall_cpu_holdoff, stall_no_softlockup, stall_cpu_irqsoff, stall_cpu_block, and stall_cpu_repeat module parameters in order to allow the user to force numerous types of stalls. In addition, rcutorture provides a great number of other module parameters to allow the user to reduce other overhead. But unfortunately, there is no way to turn of the rcu_torture_writer() portion of this torture test, which on my x86 laptop consumes somewhere between 40% and 45% of a CPU. Although this is quite lightweight for a torture test, it is not welcome on systems running production workloads. This commit therefore adds an nwriters module parameter that defaults to 1 but can be set to 0 in order to disable the rcu_torture_writer() portion of the torture test, but that cannot be set to any other value (that is what the fakewriters module parameter is for!). This reduces the overhead to well under 1% of a CPU, which is much more likely to be compatible with production workloads. Reported-by: Breno Leitao Reported-by: Puranjay Mohan Reported-by: Usama Arif Signed-off-by: Paul E. McKenney --- Documentation/admin-guide/kernel-parameters.txt | 5 +++++ kernel/rcu/rcutorture.c | 14 ++++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index b5493a7f8f22..b942149720c9 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -6117,6 +6117,11 @@ Kernel parameters the number of CPUs. For example, -2 selects N (the number of CPUs), -3 selects N+1, and so on. + rcutorture.nwriters= [KNL] + Set number of RCU writers, which must be either + zero or one. For additional writers, use instead + the rcutorture.nfakewriters parameter. + rcutorture.object_debug= [KNL] Enable debug-object double-call_rcu() testing. diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index c9fb375701f8..acda5c5fca5f 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -116,6 +116,7 @@ torture_param(int, n_barrier_cbs, 0, "# of callbacks/kthreads for barrier testin torture_param(int, n_up_down, 32, "# of concurrent up/down hrtimer-based RCU readers"); torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads"); torture_param(int, nreaders, -1, "Number of RCU reader threads"); +torture_param(bool, nwriters, 1, "Number of RCU writer threads (0 or 1)"); torture_param(int, object_debug, 0, "Enable debug-object double call_rcu() testing"); torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); torture_param(int, onoff_interval, 0, "Time between CPU hotplugs (jiffies), 0=disable"); @@ -3163,7 +3164,7 @@ static void rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) { pr_alert("%s" TORTURE_FLAG - "--- %s: nreaders=%d nfakewriters=%d " + "--- %s: nreaders=%d nwriters=%d nfakewriters=%d " "stat_interval=%d verbose=%d test_no_idle_hz=%d " "shuffle_interval=%d stutter=%d irqreader=%d " "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d " @@ -3178,7 +3179,7 @@ rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) "nocbs_nthreads=%d nocbs_toggle=%d " "test_nmis=%d " "preempt_duration=%d preempt_interval=%d n_up_down=%d\n", - torture_type, tag, nrealreaders, nrealfakewriters, + torture_type, tag, nrealreaders, nwriters, nrealfakewriters, stat_interval, verbose, test_no_idle_hz, shuffle_interval, stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter, test_boost, cur_ops->can_boost, @@ -4754,10 +4755,11 @@ rcu_torture_init(void) goto unwind; } - firsterr = torture_create_kthread(rcu_torture_writer, NULL, - writer_task); - if (torture_init_error(firsterr)) - goto unwind; + if (nwriters) { + firsterr = torture_create_kthread(rcu_torture_writer, NULL, writer_task); + if (torture_init_error(firsterr)) + goto unwind; + } firsterr = rcu_torture_updown_init(); if (torture_init_error(firsterr)) From d1ff05605db14d8d6d1578f289595210c60b385f Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 26 Jun 2026 12:44:20 -0700 Subject: [PATCH 52/59] rcutorture: Add a stall_only module parameter This commit adds a stall_only module parameter that shuts off all rcutorture kthreads other than the RCU CPU stall-warning test kthreads. The purpose of this is to test production applictions' reactions to CPU stalls, and with minimal additional overhead. Or you can omit the stall-warning tests as well and get a heavy no-op, your choice! Signed-off-by: Paul E. McKenney --- .../admin-guide/kernel-parameters.txt | 9 +++++++++ kernel/rcu/rcutorture.c | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index b942149720c9..40d2831add4a 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -6211,6 +6211,15 @@ Kernel parameters and stall_gp_kthread are specified, the kthread is starved first, then the CPU. + rcutorture.stall_only= [KNL] + Shut off all rcutorture kthreads other than the + RCU CPU stall-warning test kthreads. The purpose + of this is to test production applictions' + reactions to CPU stalls, and with minimal + additional overhead. Or you can omit the + stall-warning tests as well and get a heavy + no-op, your choice! + rcutorture.stat_interval= [KNL] Time (s) between statistics printk()s. diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index acda5c5fca5f..6e9822baa236 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -140,6 +140,7 @@ torture_param(int, stall_cpu_irqsoff, 0, "Disable interrupts while stalling."); torture_param(int, stall_cpu_block, 0, "Sleep while stalling."); torture_param(int, stall_cpu_repeat, 0, "Number of additional stalls after the first one."); torture_param(int, stall_gp_kthread, 0, "Grace-period kthread stall duration (s)."); +torture_param(bool, stall_only, 0, "Suppress all non-CPU-stall kthreads."); torture_param(int, stat_interval, 60, "Number of seconds between stats printk()s"); torture_param(int, stutter, 5, "Number of seconds to run/halt test"); torture_param(int, test_boost, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes."); @@ -4649,6 +4650,23 @@ rcu_torture_init(void) cur_ops = NULL; goto unwind; } + if (stall_only) { + pr_alert("rcu-torture: stall_only specified, suppressing all else.\n"); + fqs_stutter = 0; + fwd_progress = 0; + n_barrier_cbs = 0; + nfakewriters = 0; + nocbs_nthreads = 0; + nreaders = 0; + n_up_down = 0; + nwriters = 0; + onoff_interval = 0; + preempt_duration = 0; + read_exit_burst = 0; + shuffle_interval = 0; + stutter = 0; + test_boost = 0; + } if (cur_ops->fqs == NULL && fqs_duration != 0) { pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n"); fqs_duration = 0; From 560b35bd7c8aa1b7cd20adf28c401e93da567400 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 8 Jul 2026 13:50:35 -0700 Subject: [PATCH 53/59] rcutorture: Test RCU Tasks Trace GP implying RCU GP An RCU Tasks Trace grace period is supposed to imply an RCU grace period, and this implication is relied on by BPF. But this is not currently tested. This commit therefore makes tasks_tracing_torture_read_lock() sometimes use rcu_read_lock() instead of rcu_read_lock_trace(), thus testing the required implication. Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 6e9822baa236..c8118c38812a 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -766,11 +766,12 @@ srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) const long uspertick = 1000000 / HZ; const long longdelay = 10; - /* We want there to be long-running readers, but not all the time. */ + // We want there to be long-running readers, but not all the time. + // The !rcu_preempt_depth() is for RCU Tasks Trace. delay = torture_random(rrsp) % (nrealreaders * 2 * longdelay * uspertick); - if (!delay && in_task()) { + if (!delay && !in_atomic() && !rcu_preempt_depth() && !irqs_disabled()) { schedule_timeout_interruptible(longdelay); rtrsp->rt_delay_jiffies = longdelay; } else { @@ -1219,15 +1220,24 @@ static struct rcu_torture_ops tasks_rude_ops = { * Definitions for tracing RCU-tasks torture testing. */ +// Note that an RCU Tasks Trace GP must imply an RCU GP. static int tasks_tracing_torture_read_lock(void) { - rcu_read_lock_trace(); - return 0; + int use_rcu = !(jiffies & 0xff); + + if (use_rcu) + rcu_read_lock(); + else + rcu_read_lock_trace(); + return use_rcu; } -static void tasks_tracing_torture_read_unlock(int idx) +static void tasks_tracing_torture_read_unlock(int use_rcu) { - rcu_read_unlock_trace(); + if (use_rcu) + rcu_read_unlock(); + else + rcu_read_unlock_trace(); } static void rcu_tasks_tracing_torture_deferred_free(struct rcu_torture *p) From 86fa5387b473c160c26b781dff55014c4e1b3db0 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 8 Jul 2026 17:34:07 -0700 Subject: [PATCH 54/59] rcutorture: Make RCU Tasks Trace track Reader Batches This commit adds the ->get_sp_seq and ->gp_diff fields to the tasks_tracing_ops structure so that RCU Tasks Trace rcutorture runs will track Reader Batch. Signed-off-by: Paul E. McKenney --- include/linux/rcupdate_trace.h | 3 +++ kernel/rcu/rcutorture.c | 3 ++- kernel/rcu/tasks.h | 6 ++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h index cee89e51e45c..f2c4173bf14f 100644 --- a/include/linux/rcupdate_trace.h +++ b/include/linux/rcupdate_trace.h @@ -198,10 +198,13 @@ static inline void rcu_tasks_trace_expedite_current(void) srcu_expedite_current(&rcu_tasks_trace_srcu_struct); } +unsigned long rcu_tasks_trace_batches_completed(void); + // Placeholders to enable stepwise transition. void __init rcu_tasks_trace_suppress_unused(void); #else +static inline unsigned long rcu_tasks_trace_batches_completed(void) { return 0; } /* * The BPF JIT forms these addresses even when it doesn't call these * functions, so provide definitions that result in runtime errors. diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index c8118c38812a..8514fef3ec10 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -1252,7 +1252,8 @@ static struct rcu_torture_ops tasks_tracing_ops = { .read_delay = srcu_read_delay, /* just reuse srcu's version. */ .readunlock = tasks_tracing_torture_read_unlock, .readlock_held = rcu_read_lock_trace_held, - .get_gp_seq = rcu_no_completed, + .get_gp_seq = rcu_tasks_trace_batches_completed, + .gp_diff = rcu_seq_diff, .deferred_free = rcu_tasks_tracing_torture_deferred_free, .sync = synchronize_rcu_tasks_trace, .exp_sync = synchronize_rcu_tasks_trace, diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index f4da5fad70f5..496acac8796f 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1606,4 +1606,10 @@ static inline void rcu_tasks_bootup_oddness(void) {} DEFINE_SRCU_FAST(rcu_tasks_trace_srcu_struct); EXPORT_SYMBOL_GPL(rcu_tasks_trace_srcu_struct); +unsigned long rcu_tasks_trace_batches_completed(void) +{ + return srcu_batches_completed(&rcu_tasks_trace_srcu_struct); +} +EXPORT_SYMBOL_GPL(rcu_tasks_trace_batches_completed); + #endif /* #else #ifdef CONFIG_TASKS_TRACE_RCU */ From ed55bb915047934acd28e2f0ea1ef9586b254bf0 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Sat, 11 Jul 2026 16:51:41 -0700 Subject: [PATCH 55/59] rcutorture: Use this_cpu_inc() for rcu_torture_count[] and rcu_torture_batch[] Currently __this_cpu_inc() is used to increment elements of both the rcu_torture_count[] and rcu_torture_batch[] arrays. However, this can fail when the increments can happen in interrupt handlers, as recently became possible. This commit therefore upgrades the uses of __this_cpu_inc() to the interrupt-safe this_cpu_inc(). KCSAN located this issue. Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 8514fef3ec10..9180016d87c0 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -2565,13 +2565,13 @@ static void rcu_torture_one_read_end(struct rcu_torture_one_read_state *rtorsp, rtorsp->ts, rtorsp->started, completed); rcu_ftrace_dump(DUMP_ALL); } - __this_cpu_inc(rcu_torture_count[pipe_count]); + this_cpu_inc(rcu_torture_count[pipe_count]); completed = rcutorture_seq_diff(completed, rtorsp->started); if (completed > RCU_TORTURE_PIPE_LEN) { /* Should not happen, but... */ completed = RCU_TORTURE_PIPE_LEN; } - __this_cpu_inc(rcu_torture_batch[completed]); + this_cpu_inc(rcu_torture_batch[completed]); preempt_enable(); if (rtorsp->checkpolling) { if (cur_ops->get_gp_state && cur_ops->poll_gp_state) From aaf43c458c491748ae79f8e6afc39fd09a36e77b Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Sat, 18 Jul 2026 13:15:35 -0400 Subject: [PATCH 56/59] torture: Don't leak shuffle_tmp_mask when shuffler kthread fails to start If torture_shuffle_init() successfully allocates shuffle_tmp_mask but then fails to create the torture_shuffle kthread, the cpumask is never freed. Free the cpumask directly on the kthread-creation error path. Signed-off-by: Joel Fernandes Signed-off-by: Paul E. McKenney --- kernel/torture.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kernel/torture.c b/kernel/torture.c index 77cb3589b19f..8c4e6b2fe8ba 100644 --- a/kernel/torture.c +++ b/kernel/torture.c @@ -577,6 +577,8 @@ static int torture_shuffle(void *arg) */ int torture_shuffle_init(long shuffint) { + int ret; + shuffle_interval = shuffint; shuffle_idle_cpu = -1; @@ -587,7 +589,10 @@ int torture_shuffle_init(long shuffint) } /* Create the shuffler thread */ - return torture_create_kthread(torture_shuffle, NULL, shuffler_task); + ret = torture_create_kthread(torture_shuffle, NULL, shuffler_task); + if (ret) + free_cpumask_var(shuffle_tmp_mask); + return ret; } EXPORT_SYMBOL_GPL(torture_shuffle_init); From 6c22d640117c3da7fa7fffa0b1ddebc667507ad9 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 28 Jul 2026 13:55:15 -0700 Subject: [PATCH 57/59] rcutorture: Announce declining to forward-progress test Currently, rcu_torture_fwd_prog_cr() announces function entry, but silently exits if rcu_fwd_emergency_stop is set or if there is no ->call() function available. This could cause confusion, leading people to believe that rcu_torture_fwd_prog_cr() is on the job when it is not. This commit therefore also announces the early exits from this function. Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 9180016d87c0..46d97055c2fc 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -3623,13 +3623,17 @@ static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp) unsigned long stopat; unsigned long stoppedat; - pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); - if (READ_ONCE(rcu_fwd_emergency_stop)) + if (READ_ONCE(rcu_fwd_emergency_stop)) { + pr_alert("%s: Emergency stop, so no forward-progress test %d\n", __func__, rfp->rcu_fwd_id); return; /* Get out of the way quickly, no GP wait! */ - if (!cur_ops->call) + } + if (!cur_ops->call) { + pr_alert("%s: No ->call(), so no forward-progress test %d\n", __func__, rfp->rcu_fwd_id); return; /* Can't do call_rcu() fwd prog without ->call. */ + } /* Loop continuously posting RCU callbacks. */ + pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); atomic_inc(&rcu_fwd_cb_nodelay); cur_ops->sync(); /* Later readers see above write. */ WRITE_ONCE(rfp->rcu_fwd_startat, jiffies); From 20ed97cfde0536991fcb1785453a7748fb131620 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 28 Jul 2026 14:58:49 -0700 Subject: [PATCH 58/59] rcutorture: Make {,s}rcu_read_delay() better handle forward-progress testing The rcu_read_delay() and srcu_read_delay() functions are suppose to avoid (or at least to minimize) read-side delays during call_rcu()-based forward-progress testing. Although rcu_read_delay() does account for this by testing rcu_fwd_cb_nodelay, it only does so in some cases, and srcu_read_delay() does not bother at all. And rcutorture testing does occasionally turn up the MIN_FWD_CBS_LAUNDERED failure when running forward-progress tests on Tree SRCU flavors. This commit therefore makes both rcu_read_delay() and srcu_read_delay() take an immediate exit if rcu_fwd_cb_nodelay is set. Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 46d97055c2fc..9c7b293d1caf 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -469,12 +469,14 @@ rcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) unsigned long longdelay_ms = 300; unsigned long long ts; - /* We want a short delay sometimes to make a reader delay the grace - * period, and we want a long delay occasionally to trigger - * force_quiescent_state. */ + // If there is a forward-progress test in flight, don't delay. + if (atomic_read(&rcu_fwd_cb_nodelay)) + return; - if (!atomic_read(&rcu_fwd_cb_nodelay) && - !(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) { + // We want a short delay sometimes to make a reader delay the grace + // period, and we want a long delay occasionally to trigger + // force_quiescent_state. + if (!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) { started = cur_ops->get_gp_seq(); ts = rcu_trace_clock_local(); if ((preempt_count() & HARDIRQ_MASK) || softirq_count()) @@ -766,6 +768,10 @@ srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) const long uspertick = 1000000 / HZ; const long longdelay = 10; + // If there is a forward-progress test in flight, don't delay. + if (atomic_read(&rcu_fwd_cb_nodelay)) + return; + // We want there to be long-running readers, but not all the time. // The !rcu_preempt_depth() is for RCU Tasks Trace. From 301d28fefd141d1504b4226e8fa87db6a31f3ca1 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 14 Aug 2026 15:37:15 -0700 Subject: [PATCH 59/59] rcu: Add closing parenthesis in comment in rcu_read_unlock_strict() This comment is missing a closing parenthesis: * The in_atomic_preempt_off() check ensures that we come here holding * the last preempt_count (which will get dropped once we return to * __rcu_read_unlock(). This commit therefore adds it at the end of the sentence. Signed-off-by: Paul E. McKenney --- kernel/rcu/tree_plugin.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 6904a4f826d1..9c17dec821d3 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -949,7 +949,7 @@ void rcu_read_unlock_strict(void) * * The in_atomic_preempt_off() check ensures that we come here holding * the last preempt_count (which will get dropped once we return to - * __rcu_read_unlock(). + * __rcu_read_unlock()). */ rdp = this_cpu_ptr(&rcu_data); WRITE_ONCE(rdp->cpu_no_qs.b.norm, false);