From b3eb16109ae108c3464b0f5f8850ca4b6085561a Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 5 Aug 2026 21:21:29 -0700 Subject: [PATCH 01/46] doc: Update stallwarn.rst based on RCU Tasks Trace as SRCU Now that RCU Tasks Trace has been re-implemented in terms of SRCU, update stallwarn.rst to remove obsolete verbiage about RCU Tasks Trace stall warnings. Signed-off-by: Paul E. McKenney --- Documentation/RCU/stallwarn.rst | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/Documentation/RCU/stallwarn.rst b/Documentation/RCU/stallwarn.rst index d7c8eff63317..a986358a5c27 100644 --- a/Documentation/RCU/stallwarn.rst +++ b/Documentation/RCU/stallwarn.rst @@ -122,10 +122,11 @@ warnings: of RCU CPU stall warnings, eventually leading to the realization that the CPU had failed. -The RCU, RCU-sched, RCU-tasks, and RCU-tasks-trace implementations have -CPU stall warning. Note that SRCU does *not* have CPU stall warnings. -Please note that RCU only detects CPU stalls when there is a grace period -in progress. No grace period, no CPU stall warnings. +The RCU and RCU-tasks implementations have CPU stall warning. Note that +SRCU does *not* have CPU stall warnings, and now that RCU-tasks-trace +is implemented in terms of SRCU, neither does it. Please note that +RCU only detects CPU stalls when there is a grace period in progress. +No grace period, no CPU stall warnings. To diagnose the cause of the stall, inspect the stack traces. The offending function will usually be near the top of the stack. @@ -213,21 +214,16 @@ RCU_STALL_RAT_DELAY rcupdate.rcu_task_stall_timeout ------------------------------- - This boot/sysfs parameter controls the RCU-tasks and - RCU-tasks-trace stall warning intervals. A value of zero or less - suppresses RCU-tasks stall warnings. A positive value sets the - stall-warning interval in seconds. An RCU-tasks stall warning - starts with the line: + This boot/sysfs parameter controls the RCU-tasks stall warning + interval. A value of zero or less suppresses RCU-tasks stall + warnings. A positive value sets the stall-warning interval + in seconds. An RCU-tasks stall warning starts with the line: INFO: rcu_tasks detected stalls on tasks: And continues with the output of sched_show_task() for each task stalling the current RCU-tasks grace period. - An RCU-tasks-trace stall warning starts (and continues) similarly: - - INFO: rcu_tasks_trace detected stalls on tasks - Interpreting RCU's CPU Stall-Detector "Splats" ============================================== From 4c714f015d9ecc37d7a330b31822ccbcaff5ddc9 Mon Sep 17 00:00:00 2001 From: Longlong Xia Date: Wed, 26 Aug 2026 16:21:50 +0800 Subject: [PATCH 02/46] rcu: fix shrink budget underflow in lazy_rcu_shrink_scan The lazy RCU shrinker decremented sc->nr_to_scan (unsigned long) and then tested the result with <= 0. When a single CPU's lazy callback count exceeds the remaining budget, the subtraction wraps to a large positive value and the <= 0 comparison, which is equivalent to == 0 for an unsigned type, never fires again. The scan loop then iterates through every nocb CPU instead of honouring the reclaim budget. Accumulate into count and stop once count >= nr_to_scan. Fixes: c945b4da7a448 ("rcu: Shrinker for lazy rcu") Assisted-by: Zcode:GLM-5.2 Signed-off-by: Longlong Xia Signed-off-by: Paul E. McKenney --- kernel/rcu/tree_nocb.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index 19bb42672baf..b6cfee4e5039 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -1332,9 +1332,8 @@ lazy_rcu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) rcu_nocb_try_flush_bypass(rdp, jiffies); rcu_nocb_unlock_irqrestore(rdp, flags); wake_nocb_gp(rdp); - sc->nr_to_scan -= _count; count += _count; - if (sc->nr_to_scan <= 0) + if (count >= sc->nr_to_scan) break; } From e71035eb6e2d8af8bfde78938671695571e6d063 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 2 Sep 2026 17:30:02 -0700 Subject: [PATCH 03/46] rcu: Don't panic on already-ended RCU CPU stalls The current RCU CPU stall-warning code can say "INFO: Stall ended before state dump start" but then panic the system anyway. This commit therefore avoids panicking the system if the stall has already ended. However, the sched_ext scheduler will still be ejected, as it might have been the cause of the almost stall. This can easily be changed if needed. Signed-off-by: Paul E. McKenney --- kernel/rcu/tree_exp.h | 2 +- kernel/rcu/tree_stall.h | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h index 81d42b6e2e31..01bdbd7e28cc 100644 --- a/kernel/rcu/tree_exp.h +++ b/kernel/rcu/tree_exp.h @@ -678,7 +678,7 @@ static void synchronize_rcu_expedited_wait(void) nbcon_cpu_emergency_exit(); - panic_on_rcu_stall(&rcu_exp_stall_cpumask); + panic_on_rcu_stall(&rcu_exp_stall_cpumask, NULL, 0); } } diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index 091e7850ab6e..93ba31a619b6 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -159,7 +159,8 @@ static int __init check_cpu_stall_init(void) early_initcall(check_cpu_stall_init); /* If so specified via sysctl, panic, yielding cleaner stall-warning output. */ -static void panic_on_rcu_stall(const struct cpumask *stalled_mask) +static void panic_on_rcu_stall(const struct cpumask *stalled_mask, + unsigned long *gsp, unsigned long gp_seq) { static int cpu_stall; @@ -170,6 +171,11 @@ static void panic_on_rcu_stall(const struct cpumask *stalled_mask) if (scx_rcu_cpu_stall(stalled_mask)) return; + if (gsp && rcu_seq_current(gsp) != gp_seq) { + pr_err("INFO: Stall ended before panic check.\n"); + return; + } + if (++cpu_stall < sysctl_max_rcu_stall_to_panic) return; @@ -703,7 +709,7 @@ static void print_other_cpu_stall(unsigned long gp_seq, unsigned long gps) nbcon_cpu_emergency_exit(); - panic_on_rcu_stall(&rcu_stall_cpumask); + panic_on_rcu_stall(&rcu_stall_cpumask, &rcu_state.gp_seq, gp_seq); rcu_force_quiescent_state(); /* Kick them all. */ } @@ -758,7 +764,7 @@ static void print_cpu_stall(unsigned long gp_seq, unsigned long gps) cpumask_clear(&rcu_stall_cpumask); cpumask_set_cpu(smp_processor_id(), &rcu_stall_cpumask); - panic_on_rcu_stall(&rcu_stall_cpumask); + panic_on_rcu_stall(&rcu_stall_cpumask, &rcu_state.gp_seq, gp_seq); /* * Attempt to revive the RCU machinery by forcing a context switch. From af78eca13b0350ee04665ffa4af88ff4dae6c3e0 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Mon, 10 Aug 2026 05:27:50 -0700 Subject: [PATCH 04/46] rcu: Make call_rcu() safe to call from any context RCU's per-CPU callback list is only touched with interrupts disabled: the enqueue runs under local_irq_save() (and the nocb locks when offloaded), as do callback invocation and grace-period work. A call_rcu() that arrives with interrupts already disabled, whether from an NMI or from instrumentation that re-enters RCU, can interrupt one of those and corrupt the list or deadlock. Defer instead: stage the callback on a per-CPU llist and raise an irq_work that re-issues it once interrupts are on, straight to the enqueue so it cannot defer again. The gate is bare irqs_disabled(), so callers that merely hold interrupts off are deferred too and pay one irq_work hop. Skip it while the scheduler is down (RCU_SCHEDULER_INACTIVE): irq_work is not usable that early, rcu_init() already calls call_rcu(), and the per-CPU deferral state is not initialised until rcu_init_one() runs later in it. rcu_barrier() drains every CPU's ->defer_head before it scans the lists, and rcutree_migrate_callbacks() drains an outgoing CPU's. A drain re-issues onto the draining CPU, so a barrier moves other CPUs' staged callbacks onto its own ->cblist; call_rcu() promises no CPU affinity for invocation. ->defer_lock is held across llist_del_all() and the whole re-issue so the drainers serialize: one that finds the list empty can conclude that everything staged before it is already on a callback list. Interrupts stay off for the batch. Where the arch has an irq_work self-IPI that is what one interrupts-disabled region could stage, normally a single callback; where arch_irq_work_has_interrupt() is false the drain waits for the tick, so several regions can accumulate first. The drain clears ->next before re-issuing. A double call_rcu() on a head that is already debug-object-active self-links the staged node, and rcu_do_enqueue()'s duplicate path returns without clearing it, so the drain would spin. A re-add behind other staged callbacks makes a longer cycle, which that does not bound; a double call_rcu() stays undefined. llist_del_all() yields newest-first, so a batch is re-issued in reverse call order; nothing depends on call_rcu() ordering. The re-issue drops the lazy hint, since staging records only ->func, so a deferred callback loses its batching on CONFIG_RCU_LAZY. kasan_record_aux_stack() moves to __call_rcu_common() so a use-after-free report names the caller rather than the irq_work. The re-issue runs with interrupts disabled, so instrumentation on the enqueue path can re-enter call_rcu(), stage another callback and re-raise the irq_work, livelocking the drain. A per-CPU flag guards it: a deferral that arrives while this CPU is draining, and is not from an NMI, is dropped. The WARN_ONCE() is under CONFIG_PROVE_RCU, so a production kernel drops it silently. That leaks the callback and can strand state the caller tied to it, since a one-shot flag only the callback clears never resets, but the alternative is an unbounded loop. A callback deferred past the CPUHP_AP_SMPCFD_DYING irq_work flush leaves ->defer_work claimed with its self-IPI lost. rcutree_migrate_callbacks() still re-issues the callback, but the first deferral after that CPU comes back raises no IPI and waits for the next irq_work there, or for rcu_barrier(). Unqueueing an irq_work is not something the API offers. The irq_work is IRQ_WORK_INIT_HARD so the re-issue stays prompt on PREEMPT_RT, where a non-HARD irq_work runs in a kthread that can be delayed under load. A hidden CONFIG_RCU_DEFER gates the deferral code and its IRQ_WORK dependency, though the rcu_data members are unconditional; without it call_rcu() enqueues directly as before. Under CONFIG_PROVE_RCU, warn if the direct path is reached from an NMI. Suggested-by: Paul E. McKenney Signed-off-by: Puranjay Mohan Signed-off-by: Paul E. McKenney --- kernel/rcu/Kconfig | 6 +++ kernel/rcu/rcu.h | 11 ++++ kernel/rcu/tree.c | 131 +++++++++++++++++++++++++++++++++++++++++---- kernel/rcu/tree.h | 6 +++ 4 files changed, 143 insertions(+), 11 deletions(-) diff --git a/kernel/rcu/Kconfig b/kernel/rcu/Kconfig index 332df7a7a634..bc9483c0b5d8 100644 --- a/kernel/rcu/Kconfig +++ b/kernel/rcu/Kconfig @@ -175,6 +175,12 @@ config RCU_STALL_COMMON config RCU_NEED_SEGCBLIST def_bool ( TREE_RCU || TREE_SRCU || TASKS_RCU_GENERIC ) +# The deferral (and the IRQ_WORK it uses) is only needed where call_rcu() / +# call_srcu() can be invoked while a callback-list operation is in flight. +config RCU_DEFER + def_bool HAVE_NMI || KPROBES || FUNCTION_TRACER || TRACEPOINTS + select IRQ_WORK + config RCU_FANOUT int "Tree-based hierarchical RCU fanout value" range 2 64 if 64BIT diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index 39a9f6fa9a7b..91e33571a554 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h @@ -572,6 +572,17 @@ static inline void tasks_cblist_init_generic(void) { } #define RCU_SCHEDULER_INIT 1 #define RCU_SCHEDULER_RUNNING 2 +/* + * Defer whenever interrupts are disabled, since a callback-list operation may + * be in flight on this CPU. Not before the scheduler is up: irq_work is not + * usable that early, and rcu_init() itself calls call_rcu(). + */ +static inline bool should_rcu_defer(void) +{ + return IS_ENABLED(CONFIG_RCU_DEFER) && irqs_disabled() && + rcu_scheduler_active != RCU_SCHEDULER_INACTIVE; +} + enum rcutorture_type { RCU_FLAVOR, RCU_TASKS_FLAVOR, diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 96848fc1f02b..ff9a2395c9e8 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -3148,21 +3149,19 @@ static void check_cb_ovld(struct rcu_data *rdp) raw_spin_unlock_rcu_node(rnp); } -static void -__call_rcu_common(struct rcu_head *head, rcu_callback_t func, bool lazy_in) +/* + * Also called by __rcu_defer_drain() to re-issue a deferred callback, so it + * must not re-check the deferral condition. Either caller may have interrupts + * already disabled, and a drain of a remote CPU re-issues onto the draining + * CPU. + */ +static void rcu_do_enqueue(struct rcu_head *head, rcu_callback_t func, bool lazy_in) { static atomic_t doublefrees; unsigned long flags; bool lazy; struct rcu_data *rdp; - /* Misaligned rcu_head! */ - WARN_ON_ONCE((unsigned long)head & (sizeof(void *) - 1)); - - /* Avoid NULL dereference if callback is NULL. */ - if (WARN_ON_ONCE(!func)) - return; - if (debug_rcu_head_queue(head)) { /* * Probable double call_rcu(), so leak the callback. @@ -3178,7 +3177,6 @@ __call_rcu_common(struct rcu_head *head, rcu_callback_t func, bool lazy_in) } head->func = func; head->next = NULL; - kasan_record_aux_stack(head); local_irq_save(flags); rdp = this_cpu_ptr(&rcu_data); @@ -3206,6 +3204,103 @@ __call_rcu_common(struct rcu_head *head, rcu_callback_t func, bool lazy_in) local_irq_restore(flags); } +/* + * Re-issue deferred callbacks straight to the enqueue so they cannot defer + * again. ->defer_lock serializes the drainers: this CPU's irq_work, + * rcu_defer_flush() and rcutree_migrate_callbacks(). + */ +static void __rcu_defer_drain(struct rcu_data *rdp) +{ + struct llist_node *node, *next; + unsigned long flags; + + if (!IS_ENABLED(CONFIG_RCU_DEFER)) + return; + + raw_spin_lock_irqsave(&rdp->defer_lock, flags); + llist_for_each_safe(node, next, llist_del_all(&rdp->defer_head)) { + struct rcu_head *head = (struct rcu_head *)node; + + /* Bounds a node self-linked by a double call_rcu(). */ + head->next = NULL; + rcu_do_enqueue(head, head->func, false); + } + raw_spin_unlock_irqrestore(&rdp->defer_lock, flags); +} + +/* + * Only the irq_work drain can be re-fed by its own re-issue, so only it sets + * ->defer_draining. Anything staged during a direct drain is picked up by the + * staging CPU's own irq_work. Every caller of irq_work_run_list() has + * interrupts disabled, so the flag is never visible with them enabled. + */ +static void rcu_defer_drain(struct irq_work *iw) +{ + struct rcu_data *rdp = container_of(iw, struct rcu_data, defer_work); + + WRITE_ONCE(rdp->defer_draining, true); + __rcu_defer_drain(rdp); + WRITE_ONCE(rdp->defer_draining, false); +} + +/* + * Stage @head for this CPU's irq_work to re-issue once interrupts are on. Only + * the drain side takes a lock, so this stays safe from NMI. + */ +static void call_rcu_defer(struct rcu_head *head, rcu_callback_t func) +{ + struct rcu_data *rdp = this_cpu_ptr(&rcu_data); + + /* + * Instrumentation on the enqueue path can re-enter here from inside the + * drain. Re-queuing would livelock it, so drop the callback; an NMI + * cannot loop, so let it through. + */ + if (READ_ONCE(rdp->defer_draining) && !in_nmi()) { + WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU), + "call_rcu() re-entered during callback drain; leaking callback\n"); + return; + } + head->func = func; + if (llist_add((struct llist_node *)head, &rdp->defer_head)) + irq_work_queue(&rdp->defer_work); +} + +static void rcu_defer_flush(void) +{ + int cpu; + + for_each_possible_cpu(cpu) + __rcu_defer_drain(per_cpu_ptr(&rcu_data, cpu)); +} + +static void +__call_rcu_common(struct rcu_head *head, rcu_callback_t func, bool lazy_in) +{ + /* Misaligned rcu_head! */ + WARN_ON_ONCE((unsigned long)head & (sizeof(void *) - 1)); + + /* Avoid NULL dereference if callback is NULL. */ + if (WARN_ON_ONCE(!func)) + return; + + /* Record the caller: the irq_work's stack says nothing about it. */ + kasan_record_aux_stack(head); + + if (should_rcu_defer()) { + call_rcu_defer(head, func); + return; + } + + /* + * Only reachable from an NMI when deferral is off: before the scheduler + * is up, or with CONFIG_RCU_DEFER=n. The enqueue can then race. + */ + WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi()); + + rcu_do_enqueue(head, func, lazy_in); +} + #ifdef CONFIG_RCU_LAZY static bool enable_rcu_lazy __read_mostly = !IS_ENABLED(CONFIG_RCU_LAZY_DEFAULT_OFF); module_param(enable_rcu_lazy, bool, 0444); @@ -3896,8 +3991,12 @@ void rcu_barrier(void) unsigned long flags; unsigned long gseq; struct rcu_data *rdp; - unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence); + unsigned long s; + /* Register any deferred callbacks before snapshotting the sequence. */ + rcu_defer_flush(); + + s = rcu_seq_snap(&rcu_state.barrier_sequence); rcu_barrier_trace(TPS("Begin"), -1, s); /* Take mutex to serialize concurrent rcu_barrier() requests. */ @@ -4231,6 +4330,9 @@ rcu_boot_init_percpu_data(int cpu) rdp->rcu_onl_gp_state = RCU_GP_CLEANED; rdp->last_sched_clock = jiffies; rdp->cpu = cpu; + init_llist_head(&rdp->defer_head); + raw_spin_lock_init(&rdp->defer_lock); + rdp->defer_work = IRQ_WORK_INIT_HARD(rcu_defer_drain); rcu_boot_init_nocb_percpu_data(rdp); } @@ -4528,6 +4630,13 @@ void rcutree_migrate_callbacks(int cpu) struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); bool needwake; + /* + * Callbacks deferred past the point the outgoing CPU's irq_work can run + * sit on ->defer_head, which the ->cblist migration below does not + * cover. Drain them here, before the early returns. + */ + __rcu_defer_drain(rdp); + if (rcu_rdp_is_offloaded(rdp)) return; diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index eedfa43059e8..b7cac7a13b4f 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -229,6 +229,12 @@ struct rcu_data { struct rcu_head barrier_head; int exp_watching_snap; /* Double-check need for IPI. */ + /* Deferral of an NMI/reentrant call_rcu(); see __call_rcu_common(). */ + struct llist_head defer_head; + struct irq_work defer_work; + raw_spinlock_t defer_lock; + bool defer_draining; + /* 5) Callback offloading. */ #ifdef CONFIG_RCU_NOCB_CPU struct swait_queue_head nocb_cb_wq; /* For nocb kthreads to sleep on. */ From 0ed912d346fdb2ecc3bd03674134b956aa8e7c5d Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Mon, 10 Aug 2026 05:27:51 -0700 Subject: [PATCH 05/46] rcu: Make Tiny call_rcu() safe to call from any context Give Tiny call_rcu() the same treatment as Tree RCU. When interrupts are disabled and the scheduler is up, stage the callback on a lockless list that an irq_work re-issues later. One global list and irq_work suffice since Tiny RCU is uniprocessor, and there is no CPU-offline drain. The re-issue runs with interrupts disabled and can be re-entered by instrumentation, so a draining flag drops a deferring call_rcu() seen mid-drain (unless from an NMI), as in Tree RCU. Gated by CONFIG_RCU_DEFER, though the deferral state is unconditional. Interrupts stay off for the whole batch, but the re-issue is a tail append with no locks. TINY_RCU implies !SMP, where arch_irq_work_has_interrupt() is false, so the drain always waits for the tick and a batch is whatever one tick's worth of interrupts-disabled call_rcu()s staged. As in Tree RCU the drain clears ->next before re-issuing, which bounds a node self-linked by a double call_rcu(): rcu_do_enqueue()'s duplicate path returns without clearing it. A longer cycle is not bounded; a double call_rcu() stays undefined. The idle-task reschedule moves out of the enqueue helper so that a drain does it once for the batch rather than once per callback, which would otherwise take the runqueue lock N times with interrupts disabled. Suggested-by: Paul E. McKenney Signed-off-by: Puranjay Mohan Signed-off-by: Paul E. McKenney --- kernel/rcu/tiny.c | 127 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 104 insertions(+), 23 deletions(-) diff --git a/kernel/rcu/tiny.c b/kernel/rcu/tiny.c index dccccd6be941..656b6a682e31 100644 --- a/kernel/rcu/tiny.c +++ b/kernel/rcu/tiny.c @@ -11,6 +11,8 @@ */ #include #include +#include +#include #include #include #include @@ -42,8 +44,100 @@ static struct rcu_ctrlblk rcu_ctrlblk = { .gp_seq = 0 - 300UL, }; +/* + * The callback list is only accessed with interrupts disabled, so a call_rcu() + * that arrives with interrupts off stages the callback on a lockless list that + * an irq_work re-issues later. One global list and irq_work suffice, as Tiny + * RCU is uniprocessor. + */ +static void rcu_defer_drain(struct irq_work *iw); +static LLIST_HEAD(rcu_defer_list); +static struct irq_work rcu_defer_iw = IRQ_WORK_INIT_HARD(rcu_defer_drain); +static bool rcu_defer_draining; + +/* + * Also called by __rcu_defer_drain() to re-issue a deferred callback, so it + * must not re-check the deferral condition. + */ +static void rcu_do_enqueue(struct rcu_head *head, rcu_callback_t func) +{ + static atomic_t doublefrees; + unsigned long flags; + + if (debug_rcu_head_queue(head)) { + if (atomic_inc_return(&doublefrees) < 4) { + pr_err("%s(): Double-freed CB %p->%pS()!!! ", __func__, head, head->func); + mem_dump_obj(head); + } + return; + } + + head->func = func; + head->next = NULL; + + local_irq_save(flags); + *rcu_ctrlblk.curtail = head; + rcu_ctrlblk.curtail = &head->next; + local_irq_restore(flags); +} + +/* Force scheduling for rcu_qs() when enqueuing from the idle task. */ +static void rcu_resched_if_idle(void) +{ + if (unlikely(is_idle_task(current))) + resched_cpu(0); +} + +static void __rcu_defer_drain(void) +{ + struct llist_node *node, *next; + bool drained = false; + unsigned long flags; + + if (!IS_ENABLED(CONFIG_RCU_DEFER)) + return; + + /* Re-issued newest-first; nothing depends on call_rcu() ordering. */ + local_irq_save(flags); + llist_for_each_safe(node, next, llist_del_all(&rcu_defer_list)) { + struct rcu_head *head = (struct rcu_head *)node; + + /* Bounds a node self-linked by a double call_rcu(). */ + head->next = NULL; + rcu_do_enqueue(head, head->func); + drained = true; + } + local_irq_restore(flags); + + if (drained) + rcu_resched_if_idle(); +} + +/* Only the irq_work drain can be re-fed by its own re-issue; see Tree RCU. */ +static void rcu_defer_drain(struct irq_work *iw) +{ + WRITE_ONCE(rcu_defer_draining, true); + __rcu_defer_drain(); + WRITE_ONCE(rcu_defer_draining, false); +} + +static void call_rcu_defer(struct rcu_head *head, rcu_callback_t func) +{ + /* A re-entrant call_rcu() during the drain would livelock it; drop it. */ + if (READ_ONCE(rcu_defer_draining) && !in_nmi()) { + WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU), + "call_rcu() re-entered during callback drain; leaking callback\n"); + return; + } + head->func = func; + if (llist_add((struct llist_node *)head, &rcu_defer_list)) + irq_work_queue(&rcu_defer_iw); +} + void rcu_barrier(void) { + /* Register any deferred callbacks so the wait below covers them. */ + __rcu_defer_drain(); wait_rcu_gp(call_rcu_hurry); } EXPORT_SYMBOL(rcu_barrier); @@ -157,29 +251,19 @@ EXPORT_SYMBOL_GPL(synchronize_rcu); */ void call_rcu(struct rcu_head *head, rcu_callback_t func) { - static atomic_t doublefrees; - unsigned long flags; - - if (debug_rcu_head_queue(head)) { - if (atomic_inc_return(&doublefrees) < 4) { - pr_err("%s(): Double-freed CB %p->%pS()!!! ", __func__, head, head->func); - mem_dump_obj(head); - } + if (should_rcu_defer()) { + call_rcu_defer(head, func); return; } - head->func = func; - head->next = NULL; + /* + * Only reachable from an NMI when deferral is off: before the scheduler + * is up, or with CONFIG_RCU_DEFER=n. The enqueue can then race. + */ + WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi()); - local_irq_save(flags); - *rcu_ctrlblk.curtail = head; - rcu_ctrlblk.curtail = &head->next; - local_irq_restore(flags); - - if (unlikely(is_idle_task(current))) { - /* force scheduling for rcu_qs() */ - resched_cpu(0); - } + rcu_do_enqueue(head, func); + rcu_resched_if_idle(); } EXPORT_SYMBOL_GPL(call_rcu); @@ -211,10 +295,7 @@ unsigned long start_poll_synchronize_rcu(void) { unsigned long gp_seq = get_state_synchronize_rcu(); - if (unlikely(is_idle_task(current))) { - /* force scheduling for rcu_qs() */ - resched_cpu(0); - } + rcu_resched_if_idle(); return gp_seq; } EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu); From 35a26d87042b7c9814165f244a141104655ed345 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Mon, 10 Aug 2026 05:27:52 -0700 Subject: [PATCH 06/46] srcu: Make call_srcu() safe to call from any context call_srcu() has the same constraint as call_rcu(): srcu_gp_start_if_needed() enqueues under raw_spin_lock_irqsave() and may walk the srcu_node tree, as do callback invocation and grace-period work, so a call_srcu() with interrupts already disabled can race an operation in flight on this CPU. call_rcu_tasks_trace() is call_srcu() under the hood, so a sleepable BPF program freeing an object can reach this. Defer as call_rcu() does: stage the callback on the srcu_data's ->defer_cbs, chain that srcu_data onto a per-CPU list, and raise a per-CPU irq_work that re-issues it straight to the enqueue helper, never back through __call_srcu(). The irqs-enabled path is unchanged; as for call_rcu() the gate is bare irqs_disabled(), so callers that merely hold interrupts off are deferred too and pay one irq_work hop, including call_rcu_tasks_trace() from the BPF memalloc irq_work. The irq_work is per-CPU rather than per-srcu_struct and statically initialized, so deferral never runs check_init_srcu_struct(); it is IRQ_WORK_INIT_HARD as for call_rcu(). srcu_barrier() flushes it first, and rcutree_migrate_callbacks() calls srcu_offline_drain() for an outgoing CPU. cleanup_srcu_struct() drains before its "just leak it" early returns, and srcu_module_going() before freeing any ->sda, since a staged srcu_data left chained on a per-CPU list would dangle. Staging is two steps, the callback onto ->defer_cbs and then the srcu_data onto the per-CPU list, so a flusher can find the per-CPU list empty while a callback whose call_srcu() has not returned sits on ->defer_cbs; the staging CPU's own irq_work takes that one. The per-CPU srcu_defer ->lock, not any srcu_data's, is held with interrupts off across the whole nested drain: the chain of srcu_datas staged on that CPU and, for each, its callbacks, with srcu_do_enqueue() taking that srcu_data's ->lock and possibly starting a grace period for every one. That is what serializes the drainers. The bound is as for call_rcu(): what one interrupts-disabled region could stage, normally a single callback, or whatever accumulates before the tick where arch_irq_work_has_interrupt() is false. Both lists are drained newest-first; nothing depends on call_srcu() ordering. The drain clears ->next before re-issuing, which bounds a node self-linked by a double call_srcu(); a longer cycle is not bounded, and a double call_srcu() stays undefined, as for call_rcu(). A callback deferred past the CPUHP_AP_SMPCFD_DYING irq_work flush leaves that CPU's srcu_defer ->iw claimed with its self-IPI lost, as for call_rcu(). srcu_offline_drain() still re-issues the callback, but the irq_work cannot be un-queued, and here the claim is shared by every srcu_struct on the CPU. As in call_rcu(), the re-issue runs with interrupts disabled and can be re-entered by instrumentation, so a per-CPU flag, set only while that CPU is inside its own irq_work drain, drops a deferring call_srcu() seen mid-drain unless it comes from an NMI. Such a drop can strand state the caller associated with the callback, not just the callback itself. Staging records only the callback, so a deferred expedited call_srcu() completes as a normal grace period. Only srcu_expedite_current() can hit that, and only when invoked with interrupts already disabled. Gated by CONFIG_RCU_DEFER, though the srcu_data members and the per-CPU srcu_defer are unconditional. Under CONFIG_PROVE_RCU, warn if the direct path is reached from an NMI. Suggested-by: Paul E. McKenney Signed-off-by: Puranjay Mohan Signed-off-by: Paul E. McKenney --- include/linux/srcutree.h | 4 + kernel/rcu/rcu.h | 3 + kernel/rcu/srcutree.c | 170 ++++++++++++++++++++++++++++++++++++++- kernel/rcu/tree.c | 2 + 4 files changed, 175 insertions(+), 4 deletions(-) diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h index 75e54e4f963f..1ce759fb7094 100644 --- a/include/linux/srcutree.h +++ b/include/linux/srcutree.h @@ -13,6 +13,8 @@ #include #include +#include +#include struct srcu_node; struct srcu_struct; @@ -41,6 +43,8 @@ struct srcu_data { bool srcu_cblist_invoking; /* Invoking these CBs? */ struct timer_list delay_work; /* Delay for CB invoking */ struct work_struct work; /* Context for CB invoking. */ + struct llist_head defer_cbs; /* Callbacks deferred on re-entry. */ + struct llist_node defer_link; /* Links onto the per-CPU deferral drain list */ struct rcu_head srcu_barrier_head; /* For srcu_barrier() use. */ struct rcu_head srcu_ec_head; /* For srcu_expedite_current() use. */ int srcu_ec_state; /* State for srcu_expedite_current(). */ diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h index 91e33571a554..d60444bf3a02 100644 --- a/kernel/rcu/rcu.h +++ b/kernel/rcu/rcu.h @@ -583,6 +583,9 @@ static inline bool should_rcu_defer(void) rcu_scheduler_active != RCU_SCHEDULER_INACTIVE; } +/* Drain an outgoing CPU's deferred SRCU callbacks; see rcutree_migrate_callbacks(). */ +void srcu_offline_drain(int cpu); + enum rcutorture_type { RCU_FLAVOR, RCU_TASKS_FLAVOR, diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index ad27880dd690..b1b3c81a72f4 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -79,6 +80,38 @@ static void process_srcu(struct work_struct *work); static void srcu_irq_work(struct irq_work *work); static void srcu_delay_timer(struct timer_list *t); +struct srcu_defer; +static void srcu_defer_drain(struct irq_work *iw); +static void __srcu_defer_drain(struct srcu_defer *sndp); + +/* + * Per-CPU call_srcu() deferral state, shared by every srcu_struct. A deferred + * callback is staged on its srcu_data's ->defer_cbs; that srcu_data is chained + * via ->defer_link onto ->list, which the irq_work walks. + */ +struct srcu_defer { + struct llist_head list; + struct irq_work iw; + raw_spinlock_t lock; + bool draining; +}; + +static DEFINE_PER_CPU(struct srcu_defer, srcu_defer) = { + .lock = __RAW_SPIN_LOCK_UNLOCKED(srcu_defer.lock), + .iw = IRQ_WORK_INIT_HARD(srcu_defer_drain), +}; + +/* + * Flush pending deferred callbacks so a following srcu_barrier() waits for them. + */ +static void srcu_defer_flush(void) +{ + int cpu; + + for_each_possible_cpu(cpu) + __srcu_defer_drain(&per_cpu(srcu_defer, cpu)); +} + /* * Initialize SRCU per-CPU data. Note that statically allocated * srcu_struct structures might already have srcu_read_lock() and @@ -107,6 +140,11 @@ static void init_srcu_struct_data(struct srcu_struct *ssp) sdp->cpu = cpu; INIT_WORK(&sdp->work, srcu_invoke_callbacks); timer_setup(&sdp->delay_work, srcu_delay_timer, 0); + /* + * ->defer_cbs and ->defer_link are valid when zeroed and are not + * reinitialized here: that would clobber callbacks a reentrant + * call_srcu() already staged. See __call_srcu(). + */ sdp->ssp = ssp; } } @@ -688,6 +726,14 @@ void cleanup_srcu_struct(struct srcu_struct *ssp) unsigned long delay; struct srcu_usage *sup = ssp->srcu_sup; + /* + * Drain before the early returns below: they leak the srcu_struct, but + * srcu_module_going() frees ->sda regardless, and a staged srcu_data + * left chained on a per-CPU list would then dangle. Draining first also + * has to precede the ->irq_work sync, since re-issuing a callback can + * start a grace period and re-queue ->irq_work, which schedules ->work. + */ + srcu_defer_flush(); raw_spin_lock_irq_rcu_node(ssp->srcu_sup); delay = srcu_get_delay(ssp); raw_spin_unlock_irq_rcu_node(ssp->srcu_sup); @@ -695,7 +741,6 @@ void cleanup_srcu_struct(struct srcu_struct *ssp) return; /* Just leak it! */ if (WARN_ON(srcu_readers_active(ssp))) return; /* Just leak it! */ - /* Wait for irq_work to finish first as it may queue a new work. */ irq_work_sync(&sup->irq_work); flush_delayed_work(&sup->work); for_each_possible_cpu(cpu) { @@ -1411,8 +1456,8 @@ static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp, * srcu_read_lock(), and srcu_read_unlock() that are all passed the same * srcu_struct structure. */ -static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, - rcu_callback_t func, bool do_norm) +static void srcu_do_enqueue(struct srcu_struct *ssp, struct rcu_head *rhp, + rcu_callback_t func, bool do_norm) { if (debug_rcu_head_queue(rhp)) { /* Probable double call_srcu(), so leak the callback. */ @@ -1424,6 +1469,108 @@ static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, (void)srcu_gp_start_if_needed(ssp, rhp, do_norm); } +/* + * The srcu_cblist and srcu_node tree are only accessed with interrupts + * disabled, so defer when interrupts are already off rather than enqueue into + * an operation that may be in flight on this CPU. + */ +static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, + rcu_callback_t func, bool do_norm) +{ + if (should_rcu_defer()) { + struct srcu_defer *sndp = this_cpu_ptr(&srcu_defer); + struct srcu_data *sdp; + + /* + * Instrumentation on the enqueue path can re-enter here from + * inside the drain. Re-queuing would livelock it, so drop the + * callback; an NMI cannot loop, so let it through. + */ + if (READ_ONCE(sndp->draining) && !in_nmi()) { + WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU), + "call_srcu() re-entered during callback drain; leaking callback\n"); + return; + } + sdp = this_cpu_ptr(ssp->sda); + rhp->func = func; + if (llist_add((struct llist_node *)rhp, &sdp->defer_cbs)) { + /* + * Chain this srcu_data for the drain. ->ssp must be + * published here: deferral skips + * check_init_srcu_struct(), so on a never-initialized + * static srcu_struct the srcu_data are still zeroed and + * the drain would read a NULL ->ssp. + */ + sdp->ssp = ssp; + if (llist_add(&sdp->defer_link, &sndp->list)) + irq_work_queue(&sndp->iw); + } + return; + } + + /* + * Only reachable from an NMI when deferral is off: before the scheduler + * is up, or with CONFIG_RCU_DEFER=n. The enqueue can then race. + */ + WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi()); + + srcu_do_enqueue(ssp, rhp, func, do_norm); +} + +/* + * Re-issue deferred callbacks straight to srcu_do_enqueue() so they cannot defer + * again. ->lock serializes the drainers: the irq_work, srcu_defer_flush() and + * srcu_offline_drain(). + */ +static void __srcu_defer_drain(struct srcu_defer *sndp) +{ + struct llist_node *snode, *snext; + unsigned long flags; + + if (!IS_ENABLED(CONFIG_RCU_DEFER)) + return; + + raw_spin_lock_irqsave(&sndp->lock, flags); + llist_for_each_safe(snode, snext, llist_del_all(&sndp->list)) { + struct srcu_data *sdp = container_of(snode, struct srcu_data, defer_link); + struct srcu_struct *ssp = sdp->ssp; + struct llist_node *cnode, *cnext; + + cnode = llist_del_all(&sdp->defer_cbs); + llist_for_each_safe(cnode, cnext, cnode) { + struct rcu_head *rhp = (struct rcu_head *)cnode; + + /* Bounds a node self-linked by a double call_srcu(). */ + rhp->next = NULL; + srcu_do_enqueue(ssp, rhp, rhp->func, true); + } + } + raw_spin_unlock_irqrestore(&sndp->lock, flags); +} + +/* + * Only the irq_work drain can be re-fed by its own re-issue, so only it sets + * ->draining. A direct drain re-issues onto this CPU, and anything staged + * during it is picked up by that CPU's own irq_work. + */ +static void srcu_defer_drain(struct irq_work *iw) +{ + struct srcu_defer *sndp = container_of(iw, struct srcu_defer, iw); + + WRITE_ONCE(sndp->draining, true); + __srcu_defer_drain(sndp); + WRITE_ONCE(sndp->draining, false); +} + +/* + * Drain @cpu's deferred call_srcu() callbacks once @cpu is dead. One pass + * covers every srcu_struct; the re-issue lands on the current CPU. + */ +void srcu_offline_drain(int cpu) +{ + __srcu_defer_drain(&per_cpu(srcu_defer, cpu)); +} + /** * call_srcu() - Queue a callback for invocation after an SRCU grace period * @ssp: srcu_struct in queue the callback @@ -1678,9 +1825,18 @@ void srcu_barrier(struct srcu_struct *ssp) { int cpu; int idx; - unsigned long s = rcu_seq_snap(&ssp->srcu_sup->srcu_barrier_seq); + unsigned long s; check_init_srcu_struct(ssp); + + /* + * Register any deferred callbacks before snapshotting the sequence. The + * staging list is per-CPU, not per-srcu_struct, so this also drains + * other srcu_structs'. + */ + srcu_defer_flush(); + + s = rcu_seq_snap(&ssp->srcu_sup->srcu_barrier_seq); mutex_lock(&ssp->srcu_sup->srcu_barrier_mutex); if (rcu_seq_done(&ssp->srcu_sup->srcu_barrier_seq, s)) { smp_mb(); /* Force ordering following return. */ @@ -2135,6 +2291,12 @@ static void srcu_module_going(struct module *mod) struct srcu_struct *ssp; struct srcu_struct **sspp = mod->srcu_struct_ptrs; + /* + * Deferral skips check_init_srcu_struct(), so cleanup_srcu_struct() + * below can be skipped for an srcu_struct that has staged callbacks. + * Drain them before any ->sda is freed. + */ + srcu_defer_flush(); for (i = 0; i < mod->num_srcu_structs; i++) { ssp = *(sspp++); if (!rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq_needed)) && diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index ff9a2395c9e8..e363e1a6a33c 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -4636,6 +4636,8 @@ void rcutree_migrate_callbacks(int cpu) * cover. Drain them here, before the early returns. */ __rcu_defer_drain(rdp); + /* Likewise for the outgoing CPU's deferred call_srcu() callbacks. */ + srcu_offline_drain(cpu); if (rcu_rdp_is_offloaded(rdp)) return; From a04b2854bedc788d2e1bb421592c2a68633fd036 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Mon, 10 Aug 2026 05:27:53 -0700 Subject: [PATCH 07/46] srcu: Make Tiny call_srcu() safe to call from any context Give Tiny call_srcu() the same treatment as Tree SRCU. When interrupts are disabled and the scheduler is up, stage the callback on the srcu_struct's lockless list for an irq_work to re-issue later. Tiny SRCU is uniprocessor, so there is no CPU-offline drain. A draining flag drops a deferring call_srcu() that re-enters mid-drain (unless from an NMI), as in Tree SRCU; such a drop can strand state the caller tied to the callback, not just the callback itself. Interrupts stay off for the whole batch. TINY_SRCU implies !SMP, where arch_irq_work_has_interrupt() is false, so the drain always waits for the tick and a batch is whatever one tick's worth of interrupts-disabled call_srcu()s staged. Unlike the other three flavors srcu_do_enqueue() here has no debug_rcu_head_queue(), so nothing reports a double call_srcu(); termination of the drain rests on srcu_do_enqueue() clearing ->next, and the callback list self-links at the tail exactly as a double call_srcu() made it before. srcu_barrier() (now out of line) and cleanup_srcu_struct() drain the deferred list first, so a deferred callback is re-issued onto the callback list and invoked by the grace-period work that cleanup_srcu_struct() flushes, rather than stranded on a soon-to-be-freed srcu_struct. cleanup_srcu_struct() also syncs ->defer_iw, since that irq_work is embedded in the srcu_struct the caller is about to free. The draining flag is global rather than per-srcu_struct: a re-entrant call_srcu(B) inside a drain of A raises B's own ->defer_iw, whose drain can stage back onto A, so a per-srcu_struct flag would not break the chain. The cost is that a drain of A also drops a non-NMI call_srcu() to any other srcu_struct for its duration. Gated by CONFIG_RCU_DEFER like Tree SRCU, though the srcu_struct members are unconditional. Suggested-by: Paul E. McKenney Signed-off-by: Puranjay Mohan Signed-off-by: Paul E. McKenney --- include/linux/srcutiny.h | 12 ++++-- kernel/rcu/srcutiny.c | 93 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 97 insertions(+), 8 deletions(-) diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h index fbcf13bc12d1..85b5de438450 100644 --- a/include/linux/srcutiny.h +++ b/include/linux/srcutiny.h @@ -12,6 +12,7 @@ #define _LINUX_SRCU_TINY_H #include +#include #include struct srcu_struct { @@ -26,6 +27,8 @@ struct srcu_struct { struct rcu_head **srcu_cb_tail; /* Pending callbacks: Tail. */ struct work_struct srcu_work; /* For driving grace periods. */ struct irq_work srcu_irq_work; /* Defer schedule_work() to irq work. */ + struct llist_head defer_cbs; /* Callbacks deferred on re-entry. */ + struct irq_work defer_iw; /* Re-issues defer_cbs later. */ #ifdef CONFIG_DEBUG_LOCK_ALLOC struct lockdep_map dep_map; #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ @@ -33,6 +36,7 @@ struct srcu_struct { void srcu_drive_gp(struct work_struct *wp); void srcu_tiny_irq_work(struct irq_work *irq_work); +void srcu_defer_drain(struct irq_work *irq_work); #define __SRCU_STRUCT_INIT(name, __ignored, ___ignored, ____ignored) \ { \ @@ -40,6 +44,9 @@ void srcu_tiny_irq_work(struct irq_work *irq_work); .srcu_cb_tail = &name.srcu_cb_head, \ .srcu_work = __WORK_INITIALIZER(name.srcu_work, srcu_drive_gp), \ .srcu_irq_work = { .func = srcu_tiny_irq_work }, \ + .defer_cbs = LLIST_HEAD_INIT(name.defer_cbs), \ + .defer_iw = { .node = { .u_flags = IRQ_WORK_HARD_IRQ }, \ + .func = srcu_defer_drain }, \ __SRCU_DEP_MAP_INIT(name) \ } @@ -131,10 +138,7 @@ static inline void synchronize_srcu_expedited(struct srcu_struct *ssp) synchronize_srcu(ssp); } -static inline void srcu_barrier(struct srcu_struct *ssp) -{ - synchronize_srcu(ssp); -} +void srcu_barrier(struct srcu_struct *ssp); static inline void srcu_expedite_current(struct srcu_struct *ssp) { } #define srcu_check_read_flavor(ssp, read_flavor) do { } while (0) diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c index 558ba8d316db..5de9a6905838 100644 --- a/kernel/rcu/srcutiny.c +++ b/kernel/rcu/srcutiny.c @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -29,6 +30,8 @@ extern int rcu_scheduler_active; static LIST_HEAD(srcu_boot_list); static bool srcu_init_done; +static void __srcu_defer_drain(struct srcu_struct *ssp); + static int init_srcu_struct_fields(struct srcu_struct *ssp) { ssp->srcu_lock_nesting[0] = 0; @@ -43,6 +46,8 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp) INIT_WORK(&ssp->srcu_work, srcu_drive_gp); INIT_LIST_HEAD(&ssp->srcu_work.entry); init_irq_work(&ssp->srcu_irq_work, srcu_tiny_irq_work); + init_llist_head(&ssp->defer_cbs); + ssp->defer_iw = IRQ_WORK_INIT_HARD(srcu_defer_drain); return 0; } @@ -86,6 +91,16 @@ EXPORT_SYMBOL_GPL(init_srcu_struct_generic); void cleanup_srcu_struct(struct srcu_struct *ssp) { WARN_ON(srcu_readers_active(ssp)); + /* + * Re-issue any deferred callbacks, then wait out ->defer_iw before it is + * freed. Skipped entirely with CONFIG_RCU_DEFER=n: irq_work_sync() ends + * in an unconditional synchronize_rcu() wherever + * arch_irq_work_has_interrupt() is false, which is every !SMP target. + */ + if (IS_ENABLED(CONFIG_RCU_DEFER)) { + __srcu_defer_drain(ssp); + irq_work_sync(&ssp->defer_iw); + } irq_work_sync(&ssp->srcu_irq_work); flush_work(&ssp->srcu_work); WARN_ON(ssp->srcu_gp_running); @@ -213,11 +228,11 @@ static void srcu_gp_start_if_needed(struct srcu_struct *ssp) } /* - * Enqueue an SRCU callback on the specified srcu_struct structure, - * initiating grace-period processing if it is not already running. + * Also called by __srcu_defer_drain() to re-issue a deferred callback, so it + * must not re-check the deferral condition. */ -void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, - rcu_callback_t func) +static void srcu_do_enqueue(struct srcu_struct *ssp, struct rcu_head *rhp, + rcu_callback_t func) { unsigned long flags; @@ -231,6 +246,68 @@ void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, srcu_gp_start_if_needed(ssp); preempt_enable(); } + +/* + * Set only by the irq_work drain, the one drain its own re-issue can re-feed; + * a callback staged during a direct drain is taken by ->defer_iw afterwards. + * Global rather than per-srcu_struct: a re-entrant call_srcu(B) inside a drain + * of A raises B's own ->defer_iw, whose drain can stage back onto A. + */ +static bool srcu_defer_draining; + +static void __srcu_defer_drain(struct srcu_struct *ssp) +{ + struct llist_node *node, *next; + unsigned long flags; + + if (!IS_ENABLED(CONFIG_RCU_DEFER)) + return; + + /* Re-issued newest-first; nothing depends on call_srcu() ordering. */ + local_irq_save(flags); + llist_for_each_safe(node, next, llist_del_all(&ssp->defer_cbs)) { + struct rcu_head *rhp = (struct rcu_head *)node; + + srcu_do_enqueue(ssp, rhp, rhp->func); + } + local_irq_restore(flags); +} + +/* Only the irq_work drain can be re-fed by its own re-issue; see Tree SRCU. */ +void srcu_defer_drain(struct irq_work *iw) +{ + struct srcu_struct *ssp = container_of(iw, struct srcu_struct, defer_iw); + + WRITE_ONCE(srcu_defer_draining, true); + __srcu_defer_drain(ssp); + WRITE_ONCE(srcu_defer_draining, false); +} +EXPORT_SYMBOL_GPL(srcu_defer_drain); + +void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, + rcu_callback_t func) +{ + if (should_rcu_defer()) { + /* A re-entrant call_srcu() during the drain would livelock it. */ + if (READ_ONCE(srcu_defer_draining) && !in_nmi()) { + WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU), + "call_srcu() re-entered during callback drain; leaking callback\n"); + return; + } + rhp->func = func; + if (llist_add((struct llist_node *)rhp, &ssp->defer_cbs)) + irq_work_queue(&ssp->defer_iw); + return; + } + + /* + * Only reachable from an NMI when deferral is off: before the scheduler + * is up, or with CONFIG_RCU_DEFER=n. The enqueue can then race. + */ + WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi()); + + srcu_do_enqueue(ssp, rhp, func); +} EXPORT_SYMBOL_GPL(call_srcu); /* @@ -260,6 +337,14 @@ void synchronize_srcu(struct srcu_struct *ssp) } EXPORT_SYMBOL_GPL(synchronize_srcu); +/* Register any deferred callbacks, then wait for all in-flight ones. */ +void srcu_barrier(struct srcu_struct *ssp) +{ + __srcu_defer_drain(ssp); + synchronize_srcu(ssp); +} +EXPORT_SYMBOL_GPL(srcu_barrier); + /* * get_state_synchronize_srcu - Provide an end-of-grace-period cookie */ From d23e30d76c32d88845ce8cf4ce069addb5fa4910 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 5 Aug 2026 13:18:58 -0700 Subject: [PATCH 08/46] rcutorture: Disable fragile readers during overload testing Vanilla RCU readers, when either non-preemptible or preemptible with priority-boosting enabled, can handle heavy overload conditions. Other RCU implementations are more fragile. This commit adds an rcu_torture_ops structure field named ->rdrs_handle_load that is set for non-fragile configurations of vanilla RCU, but cleared otherwise. It also refrains from running fragile readers concurrently with overload testing, thus avoiding false-positive overload-testing failures. Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 794937e13e7c..7e08be857f01 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -440,6 +440,7 @@ struct rcu_torture_ops { int debug_objects; int start_poll_irqsoff; int have_up_down; + int rdrs_handle_load; const char *name; }; @@ -648,6 +649,7 @@ static struct rcu_torture_ops rcu_ops = { .extendables = RCUTORTURE_MAX_EXTEND, .debug_objects = 1, .start_poll_irqsoff = 1, + .rdrs_handle_load = !IS_ENABLED(CONFIG_PREEMPT_RCU) || IS_ENABLED(CONFIG_RCU_BOOST), .name = "rcu" }; @@ -2534,8 +2536,10 @@ static bool rcu_torture_one_read_start(struct rcu_torture_one_read_state *rtorsp rtorsp->p = rcu_dereference_check(rcu_torture_current, !cur_ops->readlock_held || cur_ops->readlock_held() || (rtorsp->readstate & RCUTORTURE_RDR_UPDOWN)); - if (rtorsp->p == NULL) { - /* Wait for rcu_torture_writer to get underway */ + if ((!cur_ops->rdrs_handle_load && atomic_read(&rcu_fwd_cb_nodelay)) || rtorsp->p == NULL) { + // Wait for rcu_torture_writer to get underway and + // (if readers cannot handle heavy loads) for any + // forward-progress testing to complete. rcutorture_one_extend(&rtorsp->readstate, 0, trsp, rtorsp->rtrsp); return false; } From 813f2ad27003296b9a0dc8ffd764c834626a2549 Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Mon, 10 Aug 2026 05:27:54 -0700 Subject: [PATCH 09/46] rcutorture: Exercise ->call() from NMI context call_rcu() and call_srcu() are now safe to invoke from NMI, but rcutorture never does, leaving the deferral path untested. Add an ->nmi_capable flag to rcu_torture_ops. For flavors that set it, arm a per-CPU hardware perf counter whose overflow handler submits a callback via ->call(). The handler acts only when in_nmi(), so only a genuine NMI exercises the deferral path. One preallocated callback per CPU is kept in flight, guarded by an atomic, to avoid allocating in NMI. The counter uses a fixed sample period rather than a frequency: a frequency-based event sets TICK_DEP_BIT_PERF_EVENTS and would pin the tick for the whole run on NO_HZ_FULL kernels. Report the count issued from NMI ("nmi-calls:") and the count invoked ("nmi-cbs:"). rcu_torture_cleanup() disables the counters and then calls cb_barrier(), which drains every deferred callback, so the two counts must then match; a mismatch fails the test. This relies on rcu_barrier()/srcu_barrier() flushing deferred callbacks, as added earlier in the series. Set ->nmi_capable on the NMI-safe flavors: rcu, srcu, srcud, and tasks-tracing (call_srcu() under the hood). Tasks and Tasks Rude are left alone, as call_rcu_tasks_generic() is not yet NMI-safe. Enabled by default; the nmi_calls parameter disables it, which helps rule NMI handling in or out when triaging a failure. Requires CONFIG_PERF_EVENTS and a hardware PMU: without one nothing is issued from NMI and the end-of-test check compares zero against zero, so a pass does not by itself mean the path ran. That is the case under kvm.sh, which boots qemu with -cpu kvm64 and no vPMU; rcu_torture_nmi_cleanup() says so on the console. Signed-off-by: Puranjay Mohan Signed-off-by: Paul E. McKenney --- .../admin-guide/kernel-parameters.txt | 7 + kernel/rcu/rcutorture.c | 150 +++++++++++++++++- 2 files changed, 155 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 68647ff4bdd2..fd9acc3fd9b7 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -6173,6 +6173,13 @@ Kernel parameters stress RCU, they don't participate in the actual test, hence the "fake". + rcutorture.nmi_calls= [KNL] + Enable issuing RCU callbacks from an NMI, on the + RCU flavors that support it, to exercise the + any-context callback path. Requires + CONFIG_PERF_EVENTS and a hardware PMU; without + both, nothing is issued. Defaults to enabled. + rcutorture.nocbs_nthreads= [KNL] Set number of RCU callback-offload togglers. Zero (the default) disables toggling. diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 7e08be857f01..4d1be2a49f01 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -48,6 +48,7 @@ #include #include #include +#include #include "rcu.h" @@ -115,6 +116,7 @@ torture_param(int, leakpointer, 0, "Leak pointer dereferences from readers"); torture_param(int, n_barrier_cbs, 0, "# of callbacks/kthreads for barrier testing"); torture_param(int, n_up_down, 32, "# of concurrent up/down hrtimer-based RCU readers"); torture_param(int, nfakewriters, 4, "Number of RCU fake writer threads"); +torture_param(bool, nmi_calls, true, "Exercise ->call() from NMI on nmi_capable flavors"); torture_param(int, nreaders, -1, "Number of RCU reader threads"); torture_param(bool, nwriters, 1, "Number of RCU writer threads (0 or 1)"); torture_param(int, object_debug, 0, "Enable debug-object double call_rcu() testing"); @@ -216,6 +218,8 @@ static long n_rcu_torture_boost_failure; static long n_rcu_torture_boosts; static atomic_long_t n_rcu_torture_timers; static atomic_long_t n_rcu_torture_irqs; +static atomic_long_t n_rcu_torture_nmi_call; +static atomic_long_t n_rcu_torture_nmi_cb; static long n_barrier_attempts; static long n_barrier_successes; /* did rcu_barrier test succeed? */ static unsigned long n_read_exits; @@ -433,6 +437,7 @@ struct rcu_torture_ops { bool (*is_task_rcu_boosted)(void); long cbflood_max; int irq_capable; + int nmi_capable; int can_boost; int extendables; int slow_gps; @@ -650,6 +655,7 @@ static struct rcu_torture_ops rcu_ops = { .debug_objects = 1, .start_poll_irqsoff = 1, .rdrs_handle_load = !IS_ENABLED(CONFIG_PREEMPT_RCU) || IS_ENABLED(CONFIG_RCU_BOOST), + .nmi_capable = 1, .name = "rcu" }; @@ -944,6 +950,7 @@ static struct rcu_torture_ops srcu_ops = { .debug_objects = 1, .have_up_down = IS_ENABLED(CONFIG_TINY_SRCU) ? 0 : SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_FAST_UPDOWN, + .nmi_capable = 1, .name = "srcu" }; @@ -1007,6 +1014,7 @@ static struct rcu_torture_ops srcud_ops = { .debug_objects = 1, .have_up_down = IS_ENABLED(CONFIG_TINY_SRCU) ? 0 : SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_FAST_UPDOWN, + .nmi_capable = 1, .name = "srcud" }; @@ -1271,6 +1279,7 @@ static struct rcu_torture_ops tasks_tracing_ops = { .cbflood_max = 50000, .irq_capable = 1, .slow_gps = 1, + .nmi_capable = 1, .name = "tasks-tracing" }; @@ -2663,6 +2672,124 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid) static DEFINE_TORTURE_RANDOM_PERCPU(rcu_torture_timer_rand); +/* + * Exercise ->call() from NMI context for flavors that set ->nmi_capable. A + * per-CPU hardware perf counter overflows into an NMI, and its handler submits + * a preallocated callback via ->call(). One callback per CPU is in flight at a + * time (guarded by an atomic) to avoid allocating in NMI. + */ +#ifdef CONFIG_PERF_EVENTS +static struct perf_event_attr rcu_torture_nmi_attr = { + .type = PERF_TYPE_HARDWARE, + .config = PERF_COUNT_HW_CPU_CYCLES, + .size = sizeof(struct perf_event_attr), + .pinned = 1, + .disabled = 1, + /* + * A fixed period rather than .freq: a frequency-based event bumps + * nr_freq_events, which sets TICK_DEP_BIT_PERF_EVENTS and would pin the + * tick for the whole run on NO_HZ_FULL kernels. + */ + .sample_period = 20 * 1000 * 1000, +}; + +/* One in-flight callback per CPU; ->inuse is released by the callback. */ +struct rcu_torture_nmi_cb { + struct rcu_head rh; + atomic_t inuse; +}; + +static struct perf_event **rcu_torture_nmi_events; +static int rcu_torture_nmi_hp_state; +static DEFINE_PER_CPU(struct rcu_torture_nmi_cb, rcu_torture_nmi_cb); + +static void rcu_torture_nmi_invoked(struct rcu_head *rhp) +{ + struct rcu_torture_nmi_cb *rtncp = container_of(rhp, struct rcu_torture_nmi_cb, rh); + + atomic_long_inc(&n_rcu_torture_nmi_cb); + atomic_set(&rtncp->inuse, 0); +} + +static void rcu_torture_nmi_overflow(struct perf_event *event, + struct perf_sample_data *data, + struct pt_regs *regs) +{ + struct rcu_torture_nmi_cb *rtncp = this_cpu_ptr(&rcu_torture_nmi_cb); + + if (!in_nmi()) + return; + if (cur_ops->call && !atomic_xchg(&rtncp->inuse, 1)) { + atomic_long_inc(&n_rcu_torture_nmi_call); + cur_ops->call(&rtncp->rh, rcu_torture_nmi_invoked); + } +} + +static int rcu_torture_nmi_online(unsigned int cpu) +{ + struct perf_event *event; + + event = perf_event_create_kernel_counter(&rcu_torture_nmi_attr, cpu, NULL, + rcu_torture_nmi_overflow, NULL); + if (IS_ERR(event)) + return 0; + rcu_torture_nmi_events[cpu] = event; + perf_event_enable(event); + return 0; +} + +static int rcu_torture_nmi_offline(unsigned int cpu) +{ + struct perf_event *event = rcu_torture_nmi_events[cpu]; + + if (event) { + rcu_torture_nmi_events[cpu] = NULL; + perf_event_disable(event); + perf_event_release_kernel(event); + } + return 0; +} + +/* Drive the counters from hotplug callbacks so coverage survives onoff. */ +static void rcu_torture_nmi_init(void) +{ + int ret; + + if (!nmi_calls || !cur_ops->nmi_capable || !cur_ops->call) + return; + rcu_torture_nmi_events = kcalloc(nr_cpu_ids, sizeof(*rcu_torture_nmi_events), + GFP_KERNEL); + if (!rcu_torture_nmi_events) + return; + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "rcutorture/nmi:online", + rcu_torture_nmi_online, rcu_torture_nmi_offline); + if (ret < 0) { + kfree(rcu_torture_nmi_events); + rcu_torture_nmi_events = NULL; + return; + } + rcu_torture_nmi_hp_state = ret; +} + +static void rcu_torture_nmi_cleanup(void) +{ + if (!rcu_torture_nmi_events) + return; + if (rcu_torture_nmi_hp_state > 0) { + cpuhp_remove_state(rcu_torture_nmi_hp_state); + rcu_torture_nmi_hp_state = 0; + } + kfree(rcu_torture_nmi_events); + rcu_torture_nmi_events = NULL; + if (!atomic_long_read(&n_rcu_torture_nmi_call)) + pr_alert("%s: nmi_calls set but no ->call() ever issued from NMI, so NMI ->call() went untested (no PMU, or NMIs unavailable here).\n", + __func__); +} +#else /* #ifdef CONFIG_PERF_EVENTS */ +static void rcu_torture_nmi_init(void) { } +static void rcu_torture_nmi_cleanup(void) { } +#endif /* #else #ifdef CONFIG_PERF_EVENTS */ + /* * RCU torture reader from timer handler. Dereferences rcu_torture_current, * incrementing the corresponding element of the pipeline array. The @@ -3051,6 +3178,9 @@ rcu_torture_stats_print(void) data_race(n_barrier_attempts), data_race(n_rcu_torture_barrier_error)); pr_cont("read-exits: %ld ", data_race(n_read_exits)); // Statistic. + pr_cont("nmi-calls: %ld nmi-cbs: %ld ", + atomic_long_read(&n_rcu_torture_nmi_call), + atomic_long_read(&n_rcu_torture_nmi_cb)); pr_cont("nocb-toggles: %ld:%ld ", atomic_long_read(&n_nocb_offload), atomic_long_read(&n_nocb_deoffload)); pr_cont("gpwraps: %ld\n", n_gpwraps); @@ -3199,7 +3329,7 @@ rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) "read_exit_delay=%d read_exit_burst=%d " "reader_flavor=%x " "nocbs_nthreads=%d nocbs_toggle=%d " - "test_nmis=%d " + "test_nmis=%d nmi_calls=%d " "preempt_duration=%d preempt_interval=%d n_up_down=%d\n", torture_type, tag, nrealreaders, nwriters, nrealfakewriters, stat_interval, verbose, test_no_idle_hz, shuffle_interval, @@ -3213,7 +3343,7 @@ rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, const char *tag) read_exit_delay, read_exit_burst, reader_flavor, nocbs_nthreads, nocbs_toggle, - test_nmis, + test_nmis, nmi_calls, preempt_duration, preempt_interval, n_up_down); } @@ -4287,6 +4417,7 @@ rcu_torture_cleanup(void) int i; if (torture_cleanup_begin()) { + rcu_torture_nmi_cleanup(); if (cur_ops->cb_barrier != NULL) { pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier); cur_ops->cb_barrier(); @@ -4329,6 +4460,7 @@ rcu_torture_cleanup(void) kfree(reader_tasks); reader_tasks = NULL; } + rcu_torture_nmi_cleanup(); kfree(rcu_torture_reader_mbchk); rcu_torture_reader_mbchk = NULL; @@ -4358,6 +4490,19 @@ rcu_torture_cleanup(void) pr_info("%s: Invoking %pS().\n", __func__, cur_ops->cb_barrier); cur_ops->cb_barrier(); } + + /* + * cb_barrier() above drained every deferred callback, so the count + * issued from NMI must equal the count invoked. + */ + if (atomic_long_read(&n_rcu_torture_nmi_call) != + atomic_long_read(&n_rcu_torture_nmi_cb)) { + pr_alert("%s: NMI ->call() lost a callback: issued %ld invoked %ld\n", + __func__, atomic_long_read(&n_rcu_torture_nmi_call), + atomic_long_read(&n_rcu_torture_nmi_cb)); + atomic_inc(&n_rcu_torture_error); + } + if (cur_ops->cleanup != NULL) cur_ops->cleanup(); @@ -4790,6 +4935,7 @@ rcu_torture_init(void) firsterr = -ENOMEM; goto unwind; } + rcu_torture_nmi_init(); for (i = 0; i < nrealreaders; i++) { rcu_torture_reader_mbchk[i].rtc_chkrdr = -1; firsterr = torture_create_kthread(rcu_torture_reader, (void *)i, From 0464f587c61af5bba0cc8122e8341769c8f8d73d Mon Sep 17 00:00:00 2001 From: Puranjay Mohan Date: Mon, 10 Aug 2026 05:27:55 -0700 Subject: [PATCH 10/46] selftests/bpf: Add a call_srcu() re-entry reproducer Re-enter call_srcu() from a BPF program to exercise its any-context safety, via call_rcu_tasks_trace(), which is call_srcu() on rcu_tasks_trace_srcu_struct. An fentry program on rcu_segcblist_enqueue() fires mid-enqueue: that function is reached from srcu_gp_start_if_needed() with the srcu_data ->lock held. The program does a task-storage delete, whose only deferred work is call_rcu_tasks_trace(), re-entering the enqueue on the same CPU. The handler matches on TID and fires once; pinning the thread removes the migration window between picking the srcu_data and taking its lock. Without the fix the nested call re-takes the same sdp lock and self-deadlocks; with it the nested __call_srcu() sees interrupts disabled and defers via irq_work, so the delete returns and the test passes. The test skips where it does not apply: Tiny RCU has no rcu_segcblist_enqueue() to attach to, and a UP+PREEMPT kernel pairs Tree RCU with Tiny SRCU, so the attach succeeds but call_srcu() never reaches the enqueue. Tiny SRCU is told apart by srcu_expedite_current(), which it stubs out, so on Tree SRCU a zero hit count fails rather than skips and the reproducer cannot quietly stop reproducing. Signed-off-by: Puranjay Mohan Acked-by: Kumar Kartikeya Dwivedi Signed-off-by: Paul E. McKenney --- .../selftests/bpf/prog_tests/rcu_reentry.c | 93 +++++++++++++++++++ .../testing/selftests/bpf/progs/rcu_reentry.c | 51 ++++++++++ 2 files changed, 144 insertions(+) create mode 100644 tools/testing/selftests/bpf/prog_tests/rcu_reentry.c create mode 100644 tools/testing/selftests/bpf/progs/rcu_reentry.c diff --git a/tools/testing/selftests/bpf/prog_tests/rcu_reentry.c b/tools/testing/selftests/bpf/prog_tests/rcu_reentry.c new file mode 100644 index 000000000000..de23a14b3d40 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/rcu_reentry.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Exercise re-entry into call_srcu() from BPF; see progs/rcu_reentry.c. */ +#define _GNU_SOURCE +#include +#include +#include "task_local_storage_helpers.h" +#include "trace_helpers.h" +#include "rcu_reentry.skel.h" + +/* Tiny RCU has no rcu_segcblist_enqueue() to attach to. */ +static bool have_attach_target(void) +{ + unsigned long long addr; + + return kallsyms_find("rcu_segcblist_enqueue", &addr) == 0; +} + +/* Tiny SRCU stubs out srcu_expedite_current(); Tree SRCU exports it. */ +static bool have_tree_srcu(void) +{ + unsigned long long addr; + + return kallsyms_find("srcu_expedite_current", &addr) == 0; +} + +void test_rcu_reentry(void) +{ + struct rcu_reentry *skel; + int err, pidfd = -1, map_fd; + cpu_set_t set, old_set; + bool affinity_saved; + __u64 val = 1; + int cpu; + + if (!have_attach_target()) { + test__skip(); + return; + } + + skel = rcu_reentry__open_and_load(); + if (!ASSERT_OK_PTR(skel, "skel_open_and_load")) + return; + + err = rcu_reentry__attach(skel); + if (!ASSERT_OK(err, "skel_attach")) + goto out; + + /* Keep the re-entry on a single CPU; a cpuset may exclude CPU 0. */ + affinity_saved = !sched_getaffinity(0, sizeof(old_set), &old_set); + cpu = sched_getcpu(); + if (!ASSERT_GE(cpu, 0, "getcpu")) + goto out; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + if (!ASSERT_OK(sched_setaffinity(0, sizeof(set), &set), "setaffinity")) + goto out; + + pidfd = sys_pidfd_open(getpid(), 0); + if (!ASSERT_GE(pidfd, 0, "pidfd_open")) + goto restore; + map_fd = bpf_map__fd(skel->maps.task_stg); + err = bpf_map_update_elem(map_fd, &pidfd, &val, BPF_NOEXIST); + if (!ASSERT_OK(err, "boot_create")) + goto restore; + + /* Arm the handler for this thread, then trigger call_rcu_tasks_trace(). */ + skel->bss->target_pid = syscall(__NR_gettid); + err = bpf_map_delete_elem(map_fd, &pidfd); + if (!ASSERT_OK(err, "boot_delete")) + goto restore; + + /* + * Only Tree SRCU reaches rcu_segcblist_enqueue() from call_srcu(); a + * UP+PREEMPT kernel pairs Tree RCU with Tiny SRCU, so the attach + * succeeds but nothing fires. On Tree SRCU it must fire. + */ + if (!skel->bss->hits) { + if (have_tree_srcu()) + ASSERT_GT(skel->bss->hits, 0, "prog_fired"); + else + test__skip(); + goto restore; + } + ASSERT_EQ(skel->bss->get_errs, 0, "nested_storage_get"); + ASSERT_EQ(skel->bss->del_errs, 0, "nested_storage_delete"); +restore: + if (affinity_saved) + sched_setaffinity(0, sizeof(old_set), &old_set); +out: + if (pidfd >= 0) + close(pidfd); + rcu_reentry__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/progs/rcu_reentry.c b/tools/testing/selftests/bpf/progs/rcu_reentry.c new file mode 100644 index 000000000000..47a36f704cf3 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/rcu_reentry.c @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Re-enter call_srcu() from a BPF program. fentry on rcu_segcblist_enqueue() + * fires inside call_srcu()'s enqueue (reached from srcu_gp_start_if_needed() + * with the srcu_data ->lock held); the handler then calls call_rcu_tasks_trace() + * -- itself call_srcu() on rcu_tasks_trace_srcu_struct -- re-entering the same + * srcu_data on the same CPU. + */ +#include "vmlinux.h" +#include +#include + +char _license[] SEC("license") = "GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_TASK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, __u64); +} task_stg SEC(".maps"); + +int target_pid; +int hits; +int get_errs; +int del_errs; +int done; + +SEC("fentry/rcu_segcblist_enqueue") +int BPF_PROG(reenter) +{ + struct task_struct *cur; + + if (done || !target_pid) + return 0; + + cur = bpf_get_current_task_btf(); + if (cur->pid != target_pid) + return 0; + + /* Issue the nested call exactly once, so the test is deterministic. */ + done = 1; + __sync_fetch_and_add(&hits, 1); + + /* Re-enter via a task-storage delete, which calls call_rcu_tasks_trace(). */ + if (!bpf_task_storage_get(&task_stg, cur, 0, BPF_LOCAL_STORAGE_GET_F_CREATE)) + __sync_fetch_and_add(&get_errs, 1); + else if (bpf_task_storage_delete(&task_stg, cur)) + __sync_fetch_and_add(&del_errs, 1); + + return 0; +} From 835a10eeb028a932a0061ef141537f5d17fc9aac Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 9 Jul 2026 09:36:34 -0700 Subject: [PATCH 11/46] srcutiny: Make a Tiny SRCU grace period imply an RCU grace period In non-preemptible kernels, a Tiny SRCU grace period implies an RCU grace period because any context switch suffices. But in preemptible kernels, it is possible for a Tiny SRCU grace period to elapse without a corresponding RCU grace period. Which was OK until RCU Tasks Trace was re-implemented in terms of SRCU-fast, which in Tiny SRCU is implemented as SRCU, which is already fast. And RCU Tasks Trace grace periods are required to imply RCU grace periods. This commit therefore adds a synchronize_rcu(), but only in preemptible kernels. Because preemptible Tiny SRCU is not in mainline, this added call to synchronize_rcu() will not slow anything down: The comparison would instead be with TREE SRCU. But if this added call ever becomes a problem, the Tiny SRCU srcu_struct structure could track whether or not this is for SRCU-fast, and to add the synchronize_rcu() only in the SRCU-fast case. However, at the moment, this is seen as unnecessary complexity. Signed-off-by: Paul E. McKenney --- kernel/rcu/srcutiny.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c index 5de9a6905838..32b37d63d58a 100644 --- a/kernel/rcu/srcutiny.c +++ b/kernel/rcu/srcutiny.c @@ -159,6 +159,8 @@ void srcu_drive_gp(struct work_struct *wp) WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1); WRITE_ONCE(ssp->srcu_gp_waiting, true); /* srcu_read_unlock() wakes! */ preempt_enable(); + if (IS_ENABLED(CONFIG_PREEMPTION)) + synchronize_rcu(); // Needed for RCU Tasks Trace to imply RCU grace period do { // Deadlock issues prevent __srcu_read_unlock() from // doing an unconditional wakeup, so polling is required. From 21345a8eff2bf1e12a5defdd12fcf364b29791a9 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 18 Aug 2026 10:51:42 -0700 Subject: [PATCH 12/46] srcutree: Suppress srcu_advance_state() mutex_lock in atomic The new SRCU_READ_FLAVOR_ATOMIC prohibits sleeping in both readers and grace-period waits in order to permit waiting on an SRCU grace period within an OOM notifier. This means that srcu_advance_state() cannot acquire ->srcu_gp_mutex in this case, because mutex_lock() can sleep. This commit therefore adds an is_atomic parameter to srcu_advance_state(). When this parameter is false, current behavior is maintained, in other words, ->srcu_gp_mutex is acquired and released as before. But when this is_atomic parameter is false, the caller is responsible for excluding other callers. Signed-off-by: Paul E. McKenney --- kernel/rcu/srcutree.c | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index b1b3c81a72f4..c305c54a3203 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -1941,13 +1941,16 @@ EXPORT_SYMBOL_GPL(srcu_batches_completed); /* * Core SRCU state machine. Push state bits of ->srcu_gp_seq * to SRCU_STATE_SCAN2, and invoke srcu_gp_end() when scan has - * completed in that state. + * completed in that state. Set is_atomic to indicate that + * the caller is excluding other calls so that ->srcu_gp_mutex + * is not needed, and to indicate that blocking is forbidden. */ -static void srcu_advance_state(struct srcu_struct *ssp) +static void srcu_advance_state(struct srcu_struct *ssp, bool is_atomic) { int idx; - mutex_lock(&ssp->srcu_sup->srcu_gp_mutex); + if (!is_atomic) + mutex_lock(&ssp->srcu_sup->srcu_gp_mutex); /* * Because readers might be delayed for an extended period after @@ -1965,7 +1968,8 @@ static void srcu_advance_state(struct srcu_struct *ssp) if (ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed)) { WARN_ON_ONCE(rcu_seq_state(ssp->srcu_sup->srcu_gp_seq)); raw_spin_unlock_irq_rcu_node(ssp->srcu_sup); - mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); + if (!is_atomic) + mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); return; } idx = rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)); @@ -1973,7 +1977,8 @@ static void srcu_advance_state(struct srcu_struct *ssp) srcu_gp_start(ssp); raw_spin_unlock_irq_rcu_node(ssp->srcu_sup); if (idx != SRCU_STATE_IDLE) { - mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); + if (!is_atomic) + mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); return; /* Someone else started the grace period. */ } } @@ -1981,7 +1986,8 @@ static void srcu_advance_state(struct srcu_struct *ssp) if (rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)) == SRCU_STATE_SCAN1) { idx = !(ssp->srcu_ctrp - &ssp->sda->srcu_ctrs[0]); if (!try_check_zero(ssp, idx, 1)) { - mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); + if (!is_atomic) + mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); return; /* readers present, retry later. */ } srcu_flip(ssp); @@ -1999,7 +2005,8 @@ static void srcu_advance_state(struct srcu_struct *ssp) */ idx = !(ssp->srcu_ctrp - &ssp->sda->srcu_ctrs[0]); if (!try_check_zero(ssp, idx, 2)) { - mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); + if (!is_atomic) + mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); return; /* readers present, retry later. */ } ssp->srcu_sup->srcu_n_exp_nodelay = 0; @@ -2107,7 +2114,7 @@ static void process_srcu(struct work_struct *work) sup = container_of(work, struct srcu_usage, work.work); ssp = sup->srcu_ssp; - srcu_advance_state(ssp); + srcu_advance_state(ssp, false); raw_spin_lock_irq_rcu_node(ssp->srcu_sup); curdelay = srcu_get_delay(ssp); raw_spin_unlock_irq_rcu_node(ssp->srcu_sup); From ee6bd18ef1673f526e795b368603f692576bb148 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 18 Aug 2026 11:41:01 -0700 Subject: [PATCH 13/46] srcutree: Suppress to-big transition for atomic SRCU Initially (and perhaps forever), call_srcu() will not be available for atomic srcu_struct structures. There is therefore no reason to transition such a structure to big, because the main purpose of such a transition is to reduce lock contention for concurrent SRCU callback queueing. This commit therefore adds an is_atomic parameter to both the check_init_srcu_struct() and init_srcu_struct_fields() functions, which suppresses the initialization-time transition to big that is enabled by default on large systems. It will still be possible to force a transition using rcutorture as a destructive test. This might (or might not) be adjusted later. [ paulmck: Apply feedback from Kunwu Chan. ] Signed-off-by: Paul E. McKenney --- kernel/rcu/srcutree.c | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index c305c54a3203..0292d3f9a83a 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -238,8 +238,10 @@ static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags) * Initialize non-compile-time initialized fields, including the * associated srcu_node and srcu_data structures. The is_static parameter * tells us that ->sda has already been wired up to srcu_data. + * The is_atomic parameter tells us that there is no reason to + * ever transition to big. */ -static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) +static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static, bool is_atomic) { if (!is_static) ssp->srcu_sup = kzalloc_obj(*ssp->srcu_sup); @@ -267,7 +269,8 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static) init_srcu_struct_data(ssp); ssp->srcu_sup->srcu_gp_seq_needed_exp = SRCU_GP_SEQ_INITIAL_VAL; ssp->srcu_sup->srcu_last_gp_end = ktime_get_mono_fast_ns(); - if (READ_ONCE(ssp->srcu_sup->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) { + if (!is_atomic && + READ_ONCE(ssp->srcu_sup->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) { if (!preemptible()) WRITE_ONCE(ssp->srcu_sup->srcu_size_state, SRCU_SIZE_ALLOC); else if (init_srcu_struct_nodes(ssp, GFP_KERNEL)) @@ -301,7 +304,7 @@ __init_srcu_struct_common(struct srcu_struct *ssp, const char *name, struct lock /* Don't re-initialize a lock while it is held. */ debug_check_no_locks_freed((void *)ssp, sizeof(*ssp)); lockdep_init_map(&ssp->dep_map, name, key, 0); - return init_srcu_struct_fields(ssp, false); + return init_srcu_struct_fields(ssp, false, false); } int init_srcu_struct_lockdep(struct srcu_struct *ssp, const char *name, @@ -343,7 +346,7 @@ EXPORT_SYMBOL_GPL(__init_srcu_struct_fast_updown); int init_srcu_struct_generic(struct srcu_struct *ssp) { ssp->srcu_reader_flavor = 0; - return init_srcu_struct_fields(ssp, false); + return init_srcu_struct_fields(ssp, false, false); } EXPORT_SYMBOL_GPL(init_srcu_struct_generic); @@ -360,7 +363,7 @@ EXPORT_SYMBOL_GPL(init_srcu_struct_generic); int init_srcu_struct_fast(struct srcu_struct *ssp) { ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_FAST; - return init_srcu_struct_fields(ssp, false); + return init_srcu_struct_fields(ssp, false, false); } EXPORT_SYMBOL_GPL(init_srcu_struct_fast); @@ -378,7 +381,7 @@ EXPORT_SYMBOL_GPL(init_srcu_struct_fast); int init_srcu_struct_fast_updown(struct srcu_struct *ssp) { ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_FAST_UPDOWN; - return init_srcu_struct_fields(ssp, false); + return init_srcu_struct_fields(ssp, false, false); } EXPORT_SYMBOL_GPL(init_srcu_struct_fast_updown); @@ -470,9 +473,11 @@ static void raw_spin_lock_irqsave_ssp_contention(struct srcu_struct *ssp, unsign * done with compile-time initialization, so this check is added * to each update-side SRCU primitive. Use ssp->lock, which -is- * compile-time initialized, to resolve races involving multiple - * CPUs trying to garner first-use privileges. + * CPUs trying to garner first-use privileges. The is_atomic + * parameter tells us that there will never be a reason to + * transition to big. */ -static void check_init_srcu_struct(struct srcu_struct *ssp) +static void check_init_srcu_struct(struct srcu_struct *ssp, bool is_atomic) { unsigned long flags; @@ -484,7 +489,7 @@ static void check_init_srcu_struct(struct srcu_struct *ssp) raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); return; } - init_srcu_struct_fields(ssp, true); + init_srcu_struct_fields(ssp, true, is_atomic); raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); } @@ -1279,7 +1284,7 @@ static bool srcu_should_expedite(struct srcu_struct *ssp) unsigned long t; unsigned long tlast; - check_init_srcu_struct(ssp); + check_init_srcu_struct(ssp, false); /* If _lite() readers, don't do unsolicited expediting. */ if (this_cpu_read(ssp->sda->srcu_reader_flavor) & SRCU_READ_FLAVOR_SLOWGP) return false; @@ -1338,7 +1343,7 @@ static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp, struct srcu_node *sdp_mynode; int ss_state; - check_init_srcu_struct(ssp); + check_init_srcu_struct(ssp, false); /* * While starting a new grace period, make sure we are in an * SRCU read-side critical section so that the grace-period @@ -1617,7 +1622,7 @@ static void __synchronize_srcu(struct srcu_struct *ssp, bool do_norm) if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE) return; might_sleep(); - check_init_srcu_struct(ssp); + check_init_srcu_struct(ssp, false); init_completion(&rcu.completion); init_rcu_head_on_stack(&rcu.head); __call_srcu(ssp, &rcu.head, wakeme_after_rcu, do_norm); @@ -1827,7 +1832,7 @@ void srcu_barrier(struct srcu_struct *ssp) int idx; unsigned long s; - check_init_srcu_struct(ssp); + check_init_srcu_struct(ssp, false); /* * Register any deferred callbacks before snapshotting the sequence. The From 49be2ba9f0fa25cf2137ce111c7d84e869398a8f Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 3 Sep 2026 15:26:24 -0700 Subject: [PATCH 14/46] rcu: Add running and boosted indications to RCU task stall dump Currently, rcu_print_task_stall() will dump out the PID, RCU reader nesting level, the rcu_special structure's flags, and whether or not that reader is on the ->blkd_tasks list. When debugging RCU priority boosting, it is also good to know whether the stalled RCU reader is currently running and whether it is currently being RCU priority boosted. This commit therefore adds this information to the output. [ paulmck: Apply kernel test robot feedback. ] [ paulmck: Apply feedback from Arnd Bergmann and kernel test robot. ] Link: https://lore.kernel.org/all/20260925133922.1356404-1-arnd@kernel.org/ Link: https://lore.kernel.org/all/202609261005.GrnaXBXT-lkp@intel.com/ Signed-off-by: Paul E. McKenney Reviewed-by: Bradley Morgan Reviewed-by: Kunwu Chan Tested-by: Kunwu Chan --- kernel/rcu/tree_plugin.h | 2 -- kernel/rcu/tree_stall.h | 24 ++++++++++++++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 743c16247fc0..ae5d21bae7bc 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -11,8 +11,6 @@ * Paul E. McKenney */ -#include "../locking/rtmutex_common.h" - static bool rcu_rdp_is_offloaded(struct rcu_data *rdp) { /* diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h index 93ba31a619b6..803a56af3259 100644 --- a/kernel/rcu/tree_stall.h +++ b/kernel/rcu/tree_stall.h @@ -11,6 +11,8 @@ #include #include #include +#include +#include "../locking/rtmutex_common.h" ////////////////////////////////////////////////////////////////////////////// // @@ -305,6 +307,8 @@ struct rcu_stall_chk_rdr { int nesting; union rcu_special rs; bool on_blkd_list; + bool rcu_rdr_running; + int rcu_rdr_boosted; }; /* @@ -313,6 +317,7 @@ struct rcu_stall_chk_rdr { */ static int check_slow_task(struct task_struct *t, void *arg) { + struct rcu_node *rnp; struct rcu_stall_chk_rdr *rscrp = arg; if (task_curr(t)) @@ -320,6 +325,19 @@ static int check_slow_task(struct task_struct *t, void *arg) rscrp->nesting = t->rcu_read_lock_nesting; rscrp->rs = t->rcu_read_unlock_special; rscrp->on_blkd_list = !list_empty(&t->rcu_node_entry); + rscrp->rcu_rdr_running = task_curr(t); + rscrp->rcu_rdr_boosted = 0; + if (rscrp->on_blkd_list) { + rnp = READ_ONCE(t->rcu_blocked_node); + raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */ + if (rnp == READ_ONCE(t->rcu_blocked_node)) { + if (rt_mutex_owner(&rnp->boost_mtx.rtmutex) == t) + rscrp->rcu_rdr_boosted = 1; + } else { + rscrp->rcu_rdr_boosted = 2; + } + raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */ + } return 0; } @@ -357,12 +375,14 @@ static int rcu_print_task_stall(struct rcu_node *rnp, unsigned long flags) if (task_call_func(t, check_slow_task, &rscr)) pr_cont(" P%d", t->pid); else - pr_cont(" P%d/%d:%c%c%c%c", + pr_cont(" P%d/%d:%c%c%c%c%c%c", t->pid, rscr.nesting, ".b"[rscr.rs.b.blocked], ".q"[rscr.rs.b.need_qs], ".e"[rscr.rs.b.exp_hint], - ".l"[rscr.on_blkd_list]); + ".l"[rscr.on_blkd_list], + ".R"[rscr.rcu_rdr_running], + ".B?"[rscr.rcu_rdr_boosted]); lockdep_assert_irqs_disabled(); put_task_struct(t); ndetected++; From 2eaee974b5da25b55ae72cb6c649524410316ae8 Mon Sep 17 00:00:00 2001 From: Hemanth Selam Date: Mon, 7 Sep 2026 12:06:22 +0530 Subject: [PATCH 15/46] rcu: Fix typo "upto" in comment Correct "upto" to "up to", reported by scripts/checkpatch.pl using the misspelling list in scripts/spelling.txt. Only touches comments, no code changes. Assisted-by: Cursor:claude-opus-5 Signed-off-by: Hemanth Selam Signed-off-by: Paul E. McKenney --- kernel/rcu/srcutree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index ed204b3f4b84..01f19bb17dd6 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -624,7 +624,7 @@ module_param(srcu_retry_check_delay, ulong, 0444); #define SRCU_UL_CLAMP_LO(val, low) ((val) > (low) ? (val) : (low)) #define SRCU_UL_CLAMP_HI(val, high) ((val) < (high) ? (val) : (high)) #define SRCU_UL_CLAMP(val, low, high) SRCU_UL_CLAMP_HI(SRCU_UL_CLAMP_LO((val), (low)), (high)) -// per-GP-phase no-delay instances adjusted to allow non-sleeping poll upto +// per-GP-phase no-delay instances adjusted to allow non-sleeping poll up to // one jiffies time duration. Mult by 2 is done to factor in the srcu_get_delay() // called from process_srcu(). #define SRCU_DEFAULT_MAX_NODELAY_PHASE_ADJUSTED \ From 88f77859564b81543b7efbe0d866b2a0b20963d9 Mon Sep 17 00:00:00 2001 From: Bradley Morgan Date: Thu, 10 Sep 2026 17:25:37 +0000 Subject: [PATCH 16/46] rcu: Drop the private tick-internal.h include from tree.c Nothing in the tree.c translation unit uses anything from the private timekeeping header ../time/tick-internal.h, included since commit 48d07c04b4cc ("rcu: Enable elimination of Tree-RCU softirq processing"). The tick symbols RCU uses, tick_dep_set(), tick_dep_clear(), their _cpu and _task variants, tick_nohz_full_cpu() and TICK_DEP_BIT_RCU, are all declared in the public linux/tick.h, which tree.c already includes. Drop the include. Signed-off-by: Bradley Morgan Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 1 - 1 file changed, 1 deletion(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 96848fc1f02b..338737b9781c 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -64,7 +64,6 @@ #include #include #include -#include "../time/tick-internal.h" #include "tree.h" #include "rcu.h" From c0fa92ccfdc9af4c6db695cbbae3f25262d56ca4 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Fri, 11 Sep 2026 11:40:59 +0800 Subject: [PATCH 17/46] rcu: Make userspace barrier hook drain kvfree_rcu work The bcachefs ktest allocation-leak check writes rcutree.do_rcu_barrier before reading /proc/allocinfo. While testing bcachefs performance changes, small objects released with kfree_rcu() remained visible after repeated writes to the hook and 20 seconds of waiting, causing otherwise clean tests to fail their leak check. The test assumes a stronger contract than the hook currently documents: rcu_barrier() waits for ordinary callbacks, but does not flush objects still held in kfree_rcu() batching or per-CPU SLUB sheaves. The retained population eventually fell as a sheaf filled; there is no evidence here of unbounded growth or OOM. Changing the hook to drain kvfree_rcu() work let the same unmodified bcachefs workload pass its allocation check. All eight checkpoints in one VM, after 50 through 400 option changes, reported zero retained reconcile_scan objects. This motivated the separate private-cache test used to isolate the incomplete drain from bcachefs. Calling kvfree_rcu_barrier() from rcu_barrier_throttled() was proposed when kvfree_rcu_barrier() was added in 2024, to restore a clean baseline between userspace benchmark runs. The discussion concluded that keeping the existing hook name, adding the second operation and documenting both was the safest compatibility choice, but the follow-up was not added. Add that drain and document the stronger test interface. Always retain the existing start-rate limit and perform the kvfree_rcu() drain: an unrelated ordinary barrier does not establish that this work completed. Retain the entry ordinary-barrier sequence snapshot. After draining, skip the final ordinary barrier only if that snapshot is complete, preserving the memory barrier on the completion path. Otherwise, invoke rcu_barrier() explicitly. This keeps the ordinary-callback guarantee independent of whether kvfree_rcu_barrier() embeds an ordinary barrier. Clarify that the documented completion guarantee covers work queued before the request, without preventing new work from being queued. Earlier validation of the unconditional-drain version used four fresh VM pairs with a private-cache fixture: controls retained the queued object (60 to 60 active objects), and treatments drained it (60 to 59). An ordinary-callback test passed on both kernels. Those runs predated the guarded skip and do not validate that change. No elapsed-time improvement is claimed. Link: https://lore.kernel.org/all/20240820155935.1167988-1-urezki@gmail.com/ Signed-off-by: Matthias Goergens Signed-off-by: Paul E. McKenney --- .../admin-guide/kernel-parameters.txt | 9 ++++-- kernel/rcu/tree.c | 30 ++++++++++++------- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 68647ff4bdd2..914b65ae9413 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -5699,9 +5699,12 @@ Kernel parameters there is an ongoing too-long CSD-lock wait. rcutree.do_rcu_barrier= [KNL] - Request a call to rcu_barrier(). This is - throttled so that userspace tests can safely - hammer on the sysfs variable if they so choose. + Wait for deferred kfree_rcu() frees and ordinary + call_rcu() callbacks queued before this request to + complete. This does not prevent new work from being + queued concurrently. Requests are throttled so that + userspace tests can safely hammer on the sysfs + variable if they so choose. If triggered before the RCU grace-period machinery is fully active, this will error out with EAGAIN. diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 338737b9781c..f60252390d5d 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -3988,12 +3988,12 @@ EXPORT_SYMBOL_GPL(rcu_barrier); static unsigned long rcu_barrier_last_throttle; /** - * rcu_barrier_throttled - Do rcu_barrier(), but limit to one per second + * rcu_barrier_throttled - Drain deferred RCU frees, but rate-limit starts * - * This can be thought of as guard rails around rcu_barrier() that - * permits unrestricted userspace use, at least assuming the hardware's - * try_cmpxchg() is robust. There will be at most one call per second to - * rcu_barrier() system-wide from use of this function, which means that + * This can be thought of as guard rails around the deferred-free barriers + * that permit unrestricted userspace use, at least assuming the hardware's + * try_cmpxchg() is robust. There will be at most one drain operation started + * per sixteenth of a second from use of this function, which means that * callers might needlessly wait a second or three. * * This is intended for use by test suites to avoid OOM by flushing RCU @@ -4015,14 +4015,24 @@ static void rcu_barrier_throttled(void) while (time_in_range(j, old, old + HZ / 16) || !try_cmpxchg(&rcu_barrier_last_throttle, &old, j)) { schedule_timeout_idle(HZ / 16); - if (rcu_seq_done(&rcu_state.barrier_sequence, s)) { - smp_mb(); /* caller's subsequent code after above check. */ - return; - } j = jiffies; old = READ_ONCE(rcu_barrier_last_throttle); } - rcu_barrier(); + /* + * kfree_rcu() can retain objects outside the ordinary callback lists in + * per-CPU SLUB sheaves and kvfree_rcu batches. Always drain those queues: + * an ordinary barrier does not establish that this work was drained. + */ + kvfree_rcu_barrier(); + /* + * A completed barrier can still cover ordinary callbacks queued before + * our entry snapshot. Otherwise, retain an explicit ordinary barrier + * without depending on the implementation of kvfree_rcu_barrier(). + */ + if (rcu_seq_done(&rcu_state.barrier_sequence, s)) + smp_mb(); /* caller's subsequent code after above check. */ + else + rcu_barrier(); } /* From 852fbf7fc857fd375d8caccbce576aefc37fb065 Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Fri, 11 Sep 2026 23:36:43 +0200 Subject: [PATCH 18/46] rcuref: Fix the rcuread_is_dead reference in rcuref_read() kernel-doc The kernel-doc comment of rcuref_read() refers to rcuread_is_dead, which does not exist. The name is rcuref_is_dead. Say rcuref_is_dead. Fixes: 3efa66ce6ee1 ("rcuref: Provide rcuref_is_dead()") Assisted-by: LLM Signed-off-by: Karl Mehltretter Reviewed-by: Sebastian Andrzej Siewior Signed-off-by: Paul E. McKenney --- include/linux/rcuref.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/rcuref.h b/include/linux/rcuref.h index 2fb2af6d9824..01fee161b67c 100644 --- a/include/linux/rcuref.h +++ b/include/linux/rcuref.h @@ -34,7 +34,7 @@ static inline void rcuref_init(rcuref_t *ref, unsigned int cnt) * indicate that it is safe to schedule the object, protected by this reference * counter, for deconstruction. * If you want to know if the reference counter has been marked DEAD (as - * signaled by rcuref_put()) please use rcuread_is_dead(). + * signaled by rcuref_put()) please use rcuref_is_dead(). */ static inline unsigned int rcuref_read(rcuref_t *ref) { From 5de907d24919f42ec2e221ed33a50749016368fe Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 11 Aug 2026 11:44:00 -0700 Subject: [PATCH 19/46] rcu-tasks: Disable callback contend/collapse messages by default New workloads can do large bursts of call_rcu_tasks() invocations in a short time period, followed by a quiet time period long enough to drain all of the callbacks, followed by another burst of call_rcu_tasks() invocations. This can cause RCU Tasks to switch back and forth between queuing callbacks only on CPU 0 (during quiet periods) and on all CPUs (during bursts). Which is fine. Except for the fact that each cycle from CPU-0-only to all-CPUs queuing and back generates three console messages, one announcing the shift to all-CPUs queuing, another announcing the start of the shift back to CPU-0-only queuing, and the third announcing completion of this shift after an RCU grace period. And these console messages can overrun console-log communications channels and obscure other console-message-based debugging information. And the only known use for these console messages is debugging RCU Tasks itself. This commit therefore adds a rcupdate.rcu_task_collapse_debug module parameter that defaults to false (suppressing these console messages). Those debugging or otherwise playing with RCU Tasks callback queuing auto-adjustment can set this parameter to the value true. [ paulmck: Apply Breno Leitao feedback. ] Reported-by: Breno Leitao Reported-by: David Dai Signed-off-by: Paul E. McKenney Reviewed-by: Breno Leitao --- Documentation/admin-guide/kernel-parameters.txt | 7 +++++++ kernel/rcu/tasks.h | 15 ++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 914b65ae9413..6cc6d45b59d6 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -6405,6 +6405,13 @@ Kernel parameters period to instead use normal non-expedited grace-period processing. + rcupdate.rcu_task_collapse_debug= [KNL] + Enable debugging prints that record when RCU Tasks + and RCU Tasks Trace expand to per-CPU callback + queuing and collapse back to CPU-0 queuing. + This is default-disabled due to the fact that + some workloads can make it quite noisy. + rcupdate.rcu_task_collapse_lim= [KNL] Set the maximum number of callbacks present at the beginning of a grace period that allows diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 627295396cd9..fcac7361ec51 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -178,6 +178,8 @@ static int rcu_task_contend_lim __read_mostly = 100; module_param(rcu_task_contend_lim, int, 0444); static int rcu_task_collapse_lim __read_mostly = 10; module_param(rcu_task_collapse_lim, int, 0444); +static bool rcu_task_collapse_debug __read_mostly; +module_param(rcu_task_collapse_debug, bool, 0644); static int rcu_task_lazy_lim __read_mostly = 32; module_param(rcu_task_lazy_lim, int, 0444); @@ -390,7 +392,8 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, WRITE_ONCE(rtp->percpu_enqueue_shift, 0); WRITE_ONCE(rtp->percpu_dequeue_lim, rcu_task_cpu_ids); smp_store_release(&rtp->percpu_enqueue_lim, rcu_task_cpu_ids); - pr_info("Switching %s to per-CPU callback queuing.\n", rtp->name); + if (data_race(rcu_task_collapse_debug)) + pr_info("Switching %s to per-CPU callback queuing.\n", rtp->name); } raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags); } @@ -511,7 +514,9 @@ static int rcu_tasks_need_gpcb(struct rcu_tasks *rtp) smp_store_release(&rtp->percpu_enqueue_lim, 1); rtp->percpu_dequeue_gpseq = get_state_synchronize_rcu(); gpdone = false; - pr_info("Starting switch %s to CPU-0 callback queuing.\n", rtp->name); + if (data_race(rcu_task_collapse_debug)) + pr_info("Starting switch %s to CPU-0 callback queuing.\n", + rtp->name); } raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags); } @@ -519,7 +524,9 @@ static int rcu_tasks_need_gpcb(struct rcu_tasks *rtp) raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags); if (rtp->percpu_enqueue_lim < rtp->percpu_dequeue_lim) { WRITE_ONCE(rtp->percpu_dequeue_lim, 1); - pr_info("Completing switch %s to CPU-0 callback queuing.\n", rtp->name); + if (data_race(rcu_task_collapse_debug)) + pr_info("Completing switch %s to CPU-0 callback queuing.\n", + rtp->name); } if (rtp->percpu_dequeue_lim == 1) { for (cpu = rtp->percpu_dequeue_lim; cpu < rcu_task_cpu_ids; cpu++) { @@ -704,6 +711,8 @@ static void __init rcu_tasks_bootup_oddness(void) pr_info("\tTasks-RCU CPU stall info multiplier clamped to %d (rcu_task_stall_info_mult).\n", rtsimc); rcu_task_stall_info_mult = rtsimc; } + if (rcu_task_collapse_debug) + pr_info("\tTasks-RCU callback contend/collapse debug enabled.\n"); #endif /* #ifdef CONFIG_TASKS_RCU */ #ifdef CONFIG_TASKS_RCU pr_info("\tTrampoline variant of Tasks RCU enabled.\n"); From 0381ade2ea82b48584fa57ef23d63f97b2d87d7b Mon Sep 17 00:00:00 2001 From: Ricardo Ribalda Date: Mon, 14 Sep 2026 13:53:31 +0000 Subject: [PATCH 20/46] rcu: Drop the address space qualifier from the dereference macros The RCU dereference macros end with a cast that is meant to hand back a plain kernel pointer from a __rcu pointer. For that it uses: ((typeof(*p) __force __kernel *)(local)) The problem is that typeof preserves every qualifier, including the address space qualifiers (__rcu). Recent versions of smatch[1] care about this and throw tens of warnings like this one: ./include/trace/events/vb2.h:46:1: warning: incorrect type in assignment (different address spaces) ./include/trace/events/vb2.h:46:1: expected struct tracepoint_func *it_func_ptr ./include/trace/events/vb2.h:46:1: got struct tracepoint_func __rcu * Use a new macro TYPEOF_NO_ADDRESS_SPACE() for the result type. This new macro strips all the qualifiers when running with sparse (so const and volatile are gone). But keeps all the qualifiers when running with the compiler, caring about const/volatile mismatch. [1] https://github.com/error27/smatch/commit/e53027a4e816a772403baafa83c09e4a94c1cb8f Suggested-by: Dan Carpenter Link: https://lore.kernel.org/r/20260825-unqual-v1-1-7024fb81b4f9@chromium.org Signed-off-by: Ricardo Ribalda Reviewed-by: Joel Fernandes Signed-off-by: Paul E. McKenney --- include/linux/compiler.h | 17 +++++++++++++++++ include/linux/rcupdate.h | 10 +++++----- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/include/linux/compiler.h b/include/linux/compiler.h index cb2f6050bdf7..70cb31d60538 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -239,6 +239,23 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, # define TYPEOF_UNQUAL(exp) __typeof__(exp) #endif +/* + * TYPEOF_NO_ADDRESS_SPACE() - typeof() without the address space qualifiers + * + * No operator strips only the address space qualifiers: typeof() keeps every + * qualifier and TYPEOF_UNQUAL() drops every qualifier, const and volatile + * included. + * + * Approximate one by dropping the qualifiers for sparse only, as it is the + * only one that knows about address spaces. The compiler keeps seeing the fully + * qualified type, so a missing const or volatile will still throw a warning. + */ +#ifdef __CHECKER__ +# define TYPEOF_NO_ADDRESS_SPACE(exp) TYPEOF_UNQUAL(exp) +#else +# define TYPEOF_NO_ADDRESS_SPACE(exp) __typeof__(exp) +#endif + #endif /* __KERNEL__ */ #if defined(CONFIG_CFI) && !defined(__DISABLE_EXPORTS) && !defined(BUILD_VDSO) diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 44c07a66edff..3f74ae6d6e1f 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -488,7 +488,7 @@ static __always_inline bool lockdep_assert_rcu_helper(bool c, const struct __ctx context_unsafe( \ typeof(*p) *local = (typeof(*p) *__force)(p); \ rcu_check_sparse(p, __rcu); \ - ((typeof(*p) __force __kernel *)(local)) \ + ((TYPEOF_NO_ADDRESS_SPACE(*p) __force __kernel *)(local)) \ ) /** * unrcu_pointer - mark a pointer as not being RCU protected @@ -503,7 +503,7 @@ context_unsafe( \ ({ \ typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \ rcu_check_sparse(p, space); \ - ((typeof(*p) __force __kernel *)(local)); \ + ((TYPEOF_NO_ADDRESS_SPACE(*p) __force __kernel *)(local)); \ }) ) #define __rcu_dereference_check(p, local, c, space) \ ({ \ @@ -511,19 +511,19 @@ context_unsafe( \ typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \ RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_check() usage"); \ rcu_check_sparse(p, space); \ - ((typeof(*p) __force __kernel *)(local)); \ + ((TYPEOF_NO_ADDRESS_SPACE(*p) __force __kernel *)(local)); \ }) #define __rcu_dereference_protected(p, local, c, space) \ ({ \ RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_protected() usage"); \ rcu_check_sparse(p, space); \ - ((typeof(*p) __force __kernel *)(p)); \ + ((TYPEOF_NO_ADDRESS_SPACE(*p) __force __kernel *)(p)); \ }) #define __rcu_dereference_raw(p, local) \ ({ \ /* Dependency order vs. p above. */ \ typeof(p) local = READ_ONCE(p); \ - ((typeof(*p) __force __kernel *)(local)); \ + ((TYPEOF_NO_ADDRESS_SPACE(*p) __force __kernel *)(local)); \ }) #define rcu_dereference_raw(p) __rcu_dereference_raw(p, __UNIQUE_ID(rcu)) From 60df507b60919bdd44ab4ca7352b5ed24808746c Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 18 Aug 2026 17:10:15 -0700 Subject: [PATCH 21/46] srcutree: Add an atomic Tree SRCU MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some dedicated srcu_struct users have read-side critical sections which are short, never sleep, and never block on anything which may itself depend on memory allocation — because they were, until recently, spinlock or rwlock critical sections. For such a domain the update side can safely wait for readers by spinning, from contexts where sleeping is undesirable or the grace-period machinery's latency (workqueue scheduling, jiffy-paced retries) dominates the actual reader drain time. But that is only safe if every reader keeps the promise. Make the promise explicit and machine-checkable: - srcu_read_lock_atomic() / srcu_read_unlock_atomic() enter the usual (smp_mb-based) read-side critical section with preemption disabled and (except in hardirq, where it is redundant) non_block_start() armed, recording SRCU_READ_FLAVOR_ATOMIC in the per-CPU reader flavor. Disabling preemption enforces the no-sleeping promise on every configuration and bounds the section, so it is always running on some CPU; non_block_start() extends the enforcement to even potentially-sleeping calls on paths which happen not to block. The existing reader-flavor consistency checks complain about any mixing with other flavors. - synchronize_srcu_atomic() waits for all pre-existing readers by repeating the try_synchronize_srcu() both-epoch counter proof with cpu_relax() until it succeeds: no sleeping, no index flip, no grace-period sequence update, and therefore no interaction with concurrent call_srcu(), synchronize_srcu() or srcu_barrier(). It always provides the full grace-period guarantee: if the domain turns out to have had readers of any other flavor — a caller bug, since such a reader may be asleep and spinning on it would be unbounded — it complains and falls back to a real (sleeping) grace period internally, that being the only correct wait for a possibly-sleeping reader. The flavor mask is rechecked on every iteration so a first non-atomic reader appearing mid-spin takes the same path. The immediate motivation is the proposed conversion of KVM's gfn_to_pfn_cache to SRCU¹, whose mmu_notifier invalidation path drains readers before the primary MMU zaps a page. With the readers declared atomic, that drain becomes spin-only: no sleeping at all in the notifier, bounded by the longest reader section, satisfying even the strictest reading of the OOM-reaper non-blocking requirement without needing to touch the non_block_start() annotation². This commit also adds atomic-SRCU-specific initializers: init_srcu_struct_atomic(), DEFINE_SRCU_ATOMIC(), and DEFINE_STATIC_SRCU_ATOMIC(). This commit implements only Tree SRCU. Tiny SRCU will follow. ¹ https://lore.kernel.org/all/20260811132237.102400-1-dwmw2@infradead.org/ ² https://lore.kernel.org/all/20260812134934.GC662699@ziepe.ca/ [ paulmck: Apply Kunwu Chan feedback. ] Co-developed-by: David Woodhouse Signed-off-by: David Woodhouse Assisted-by: Claude:claude-mythos-5 Signed-off-by: Paul E. McKenney --- include/linux/srcu.h | 81 ++++++++++++++++++-- include/linux/srcutree.h | 9 ++- kernel/rcu/srcutree.c | 154 +++++++++++++++++++++++++++++++++++++-- 3 files changed, 227 insertions(+), 17 deletions(-) diff --git a/include/linux/srcu.h b/include/linux/srcu.h index 7d9bc06df98d..3f232f2e0524 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h @@ -36,6 +36,8 @@ static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name, int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); +int __init_srcu_struct_atomic(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key); #endif // #ifndef CONFIG_TINY_SRCU #define init_srcu_struct_fast(ssp) \ @@ -52,6 +54,13 @@ int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, __init_srcu_struct_fast_updown((ssp), #ssp, &__srcu_key); \ }) +#define init_srcu_struct_atomic(ssp) \ +({ \ + static struct lock_class_key __srcu_key; \ + \ + __init_srcu_struct_atomic((ssp), #ssp, &__srcu_key); \ +}) + #define __SRCU_DEP_MAP_INIT(srcu_name) .dep_map = { .name = #srcu_name }, #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ @@ -64,6 +73,7 @@ static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name, #ifndef CONFIG_TINY_SRCU int init_srcu_struct_fast(struct srcu_struct *ssp); int init_srcu_struct_fast_updown(struct srcu_struct *ssp); +int init_srcu_struct_atomic(struct srcu_struct *ssp); #endif // #ifndef CONFIG_TINY_SRCU #define __SRCU_DEP_MAP_INIT(srcu_name) @@ -77,14 +87,18 @@ int init_srcu_struct_fast_updown(struct srcu_struct *ssp); }) /* Values for SRCU Tree srcu_data ->srcu_reader_flavor, but also used by rcutorture. */ -#define SRCU_READ_FLAVOR_NORMAL 0x1 // srcu_read_lock(). -#define SRCU_READ_FLAVOR_NMI 0x2 // srcu_read_lock_nmisafe(). -// 0x4 // SRCU-lite is no longer with us. -#define SRCU_READ_FLAVOR_FAST 0x4 // srcu_read_lock_fast(), also NMI-safe. -#define SRCU_READ_FLAVOR_FAST_UPDOWN 0x8 // srcu_read_lock_fast_updown(). +#define SRCU_READ_FLAVOR_NORMAL 0x01 // srcu_read_lock(). +#define SRCU_READ_FLAVOR_NMI 0x02 // srcu_read_lock_nmisafe(). +// 0x04 // SRCU-lite is no longer with us. +#define SRCU_READ_FLAVOR_FAST 0x04 // srcu_read_lock_fast(), also NMI-safe. +#define SRCU_READ_FLAVOR_FAST_UPDOWN 0x08 // srcu_read_lock_fast_updown(). +#define SRCU_READ_FLAVOR_ATOMIC 0x10 // srcu_read_lock_atomic(). #define SRCU_READ_FLAVOR_ALL (SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_NMI | \ - SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_FAST_UPDOWN) + SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_FAST_UPDOWN | \ + SRCU_READ_FLAVOR_ATOMIC) // All of the above. +#define SRCU_READ_FLAVOR_PREDEF (SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_ATOMIC) + // Flavors special DEFINE_SRCU() flavors. #define SRCU_READ_FLAVOR_SLOWGP (SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_FAST_UPDOWN) // Flavors requiring synchronize_rcu() // instead of smp_mb(). @@ -102,6 +116,7 @@ void call_srcu(struct srcu_struct *ssp, struct rcu_head *head, void (*func)(struct rcu_head *head)); void cleanup_srcu_struct(struct srcu_struct *ssp); void synchronize_srcu(struct srcu_struct *ssp); +void synchronize_srcu_atomic(struct srcu_struct *ssp); #define SRCU_GET_STATE_COMPLETED 0x1 @@ -306,6 +321,43 @@ static inline int srcu_read_lock(struct srcu_struct *ssp) return retval; } +/** + * srcu_read_lock_atomic - register a new reader promising an atomic section + * @ssp: srcu_struct in which to register the new reader. + * + * As srcu_read_lock(), but the caller promises that the read-side + * critical section never sleeps and never blocks on anything which + * may itself depend on memory allocation to make progress. Preemption + * is disabled for the duration, which both enforces that promise (any + * sleepable call in the section will splat on every configuration) + * and bounds the section so that the update side may spin rather + * than sleep when waiting for readers: see synchronize_srcu_atomic(). + * + * The lock and matching srcu_read_unlock_atomic() must be invoked on + * the same CPU, from the same context; passing the return value to + * another task is not permitted for this flavor. + */ +static inline int srcu_read_lock_atomic(struct srcu_struct *ssp) + __acquires_shared(ssp) +{ + int retval; + + preempt_disable(); + /* + * Arm might_sleep() to catch even a *potentially* sleeping call + * in the section, not just an actual schedule: the atomic-domain + * promise must hold on every path, contended or not. In hardirq + * the annotation would land on the interrupted task; it is also + * redundant there, so skip it. + */ + if (!in_hardirq()) + non_block_start(); + srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_ATOMIC); + retval = __srcu_read_lock(ssp); + srcu_lock_acquire(&ssp->dep_map); + return retval; +} + /** * srcu_read_lock_fast - register a new reader for an SRCU-protected structure. * @ssp: srcu_struct in which to register the new reader. @@ -498,6 +550,23 @@ static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx) __srcu_read_unlock(ssp, idx); } +/** + * srcu_read_unlock_atomic - unregister an atomic-section reader + * @ssp: srcu_struct from which to unregister the old reader. + * @idx: return value from corresponding srcu_read_lock_atomic(). + */ +static inline void srcu_read_unlock_atomic(struct srcu_struct *ssp, int idx) + __releases_shared(ssp) +{ + WARN_ON_ONCE(idx & ~0x1); + srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_ATOMIC); + srcu_lock_release(&ssp->dep_map); + __srcu_read_unlock(ssp, idx); + if (!in_hardirq()) + non_block_end(); + preempt_enable(); +} + /** * srcu_read_unlock_fast - unregister a old reader from an SRCU-protected structure. * @ssp: srcu_struct in which to unregister the old reader. diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h index 1ce759fb7094..ad9d9658b0a2 100644 --- a/include/linux/srcutree.h +++ b/include/linux/srcutree.h @@ -80,6 +80,7 @@ struct srcu_usage { struct mutex srcu_cb_mutex; /* Serialize CB preparation. */ raw_spinlock_t __private lock; /* Protect counters and size state. */ struct mutex srcu_gp_mutex; /* Serialize GP work. */ + atomic_t srcu_atomic_gp_flag; /* Serialize atomic GP work. */ unsigned long srcu_gp_seq; /* Grace-period seq #. */ unsigned long srcu_gp_seq_needed; /* Latest gp_seq needed. */ unsigned long srcu_gp_seq_needed_exp; /* Furthest future exp GP. */ @@ -229,14 +230,16 @@ struct srcu_struct { is_static struct srcu_struct name = \ __SRCU_STRUCT_INIT(name, name##_srcu_usage, name##_srcu_data, fast) #endif -#define DEFINE_SRCU(name) __DEFINE_SRCU(name, 0, /* not static */) +#define DEFINE_SRCU(name) __DEFINE_SRCU(name, 0, /* !static */) #define DEFINE_STATIC_SRCU(name) __DEFINE_SRCU(name, 0, static) -#define DEFINE_SRCU_FAST(name) __DEFINE_SRCU(name, SRCU_READ_FLAVOR_FAST, /* not static */) +#define DEFINE_SRCU_FAST(name) __DEFINE_SRCU(name, SRCU_READ_FLAVOR_FAST, /* !static */) #define DEFINE_STATIC_SRCU_FAST(name) __DEFINE_SRCU(name, SRCU_READ_FLAVOR_FAST, static) #define DEFINE_SRCU_FAST_UPDOWN(name) __DEFINE_SRCU(name, SRCU_READ_FLAVOR_FAST_UPDOWN, \ - /* not static */) + /* !static */) #define DEFINE_STATIC_SRCU_FAST_UPDOWN(name) \ __DEFINE_SRCU(name, SRCU_READ_FLAVOR_FAST_UPDOWN, static) +#define DEFINE_SRCU_ATOMIC(name) __DEFINE_SRCU(name, SRCU_READ_FLAVOR_ATOMIC, /* !static */) +#define DEFINE_STATIC_SRCU_ATOMIC(name) __DEFINE_SRCU(name, SRCU_READ_FLAVOR_ATOMIC, static) int __srcu_read_lock(struct srcu_struct *ssp) __acquires_shared(ssp); void synchronize_srcu_expedited(struct srcu_struct *ssp); diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 0292d3f9a83a..7ff45e992c01 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -253,6 +253,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static, bool ssp->srcu_sup->node = NULL; mutex_init(&ssp->srcu_sup->srcu_cb_mutex); mutex_init(&ssp->srcu_sup->srcu_gp_mutex); + atomic_set(&ssp->srcu_sup->srcu_atomic_gp_flag, 0); ssp->srcu_sup->srcu_gp_seq = SRCU_GP_SEQ_INITIAL_VAL; ssp->srcu_sup->srcu_barrier_seq = 0; mutex_init(&ssp->srcu_sup->srcu_barrier_mutex); @@ -330,6 +331,13 @@ int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, } EXPORT_SYMBOL_GPL(__init_srcu_struct_fast_updown); +int __init_srcu_struct_atomic(struct srcu_struct *ssp, const char *name, struct lock_class_key *key) +{ + ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_ATOMIC; + return __init_srcu_struct_common(ssp, name, key); +} +EXPORT_SYMBOL_GPL(__init_srcu_struct_atomic); + #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ /** @@ -385,6 +393,26 @@ int init_srcu_struct_fast_updown(struct srcu_struct *ssp) } EXPORT_SYMBOL_GPL(init_srcu_struct_fast_updown); +/** + * init_srcu_struct_atomic - initialize an atomic sleep-RCU structure + * @ssp: structure to initialize. + * + * Use this in place of DEFINE_SRCU_ATOMIC() and DEFINE_STATIC_SRCU_ATOMIC() + * for non-static srcu_struct structures that are to be passed to + * srcu_read_lock_atomic() and friends. It is necessary to invoke this on a + * given srcu_struct before passing that srcu_struct to any other function. + * Each srcu_struct represents a separate domain of SRCU protection. + * + * And yes, we really are defining a sleepable RCU implementation that + * cannot sleep. Strange universe we live in, isn't it? + */ +int init_srcu_struct_atomic(struct srcu_struct *ssp) +{ + ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_ATOMIC; + return init_srcu_struct_fields(ssp, false, false); +} +EXPORT_SYMBOL_GPL(init_srcu_struct_atomic); + #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ /* @@ -802,7 +830,10 @@ void __srcu_check_read_flavor(struct srcu_struct *ssp, int read_flavor) WARN_ON_ONCE(ssp->srcu_reader_flavor && read_flavor != ssp->srcu_reader_flavor); WARN_ON_ONCE(old_read_flavor && ssp->srcu_reader_flavor && old_read_flavor != ssp->srcu_reader_flavor); - WARN_ON_ONCE(read_flavor == SRCU_READ_FLAVOR_FAST && !ssp->srcu_reader_flavor); + WARN_ON_ONCE(!!(read_flavor & SRCU_READ_FLAVOR_PREDEF) && + read_flavor != ssp->srcu_reader_flavor); + WARN_ON_ONCE(!!(ssp->srcu_reader_flavor & SRCU_READ_FLAVOR_PREDEF) && + read_flavor != ssp->srcu_reader_flavor); if (!old_read_flavor) { old_read_flavor = cmpxchg(&sdp->srcu_reader_flavor, 0, read_flavor); if (!old_read_flavor) @@ -942,7 +973,7 @@ static void srcu_schedule_cbs_snp(struct srcu_struct *ssp, struct srcu_node *snp * are initiating callback invocation. This allows the ->srcu_have_cbs[] * array to have a finite number of elements. */ -static void srcu_gp_end(struct srcu_struct *ssp) +static void srcu_gp_end(struct srcu_struct *ssp, bool is_atomic) { unsigned long cbdelay = 1; bool cbs; @@ -958,7 +989,8 @@ static void srcu_gp_end(struct srcu_struct *ssp) struct srcu_usage *sup = ssp->srcu_sup; /* Prevent more than one additional grace period. */ - mutex_lock(&sup->srcu_cb_mutex); + if (!is_atomic) + mutex_lock(&sup->srcu_cb_mutex); /* End the current grace period. */ raw_spin_lock_irq_rcu_node(sup); @@ -973,7 +1005,8 @@ static void srcu_gp_end(struct srcu_struct *ssp) if (ULONG_CMP_LT(sup->srcu_gp_seq_needed_exp, gpseq)) WRITE_ONCE(sup->srcu_gp_seq_needed_exp, gpseq); raw_spin_unlock_irq_rcu_node(sup); - mutex_unlock(&sup->srcu_gp_mutex); + if (!is_atomic) + mutex_unlock(&sup->srcu_gp_mutex); /* A new grace period can start at this point. But only one. */ /* Initiate callback invocation as needed. */ @@ -1018,13 +1051,15 @@ static void srcu_gp_end(struct srcu_struct *ssp) } /* Callback initiation done, allow grace periods after next. */ - mutex_unlock(&sup->srcu_cb_mutex); + if (!is_atomic) + mutex_unlock(&sup->srcu_cb_mutex); /* Start a new grace period if needed. */ raw_spin_lock_irq_rcu_node(sup); gpseq = rcu_seq_current(&sup->srcu_gp_seq); if (!rcu_seq_state(gpseq) && ULONG_CMP_LT(gpseq, sup->srcu_gp_seq_needed)) { + WARN_ON_ONCE(ssp->srcu_reader_flavor & SRCU_READ_FLAVOR_ATOMIC); srcu_gp_start(ssp); raw_spin_unlock_irq_rcu_node(sup); srcu_reschedule(ssp, 0); @@ -1148,6 +1183,7 @@ static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp, /* If grace period not already in progress, start it. */ if (!WARN_ON_ONCE(rcu_seq_done(&sup->srcu_gp_seq, s)) && rcu_seq_state(sup->srcu_gp_seq) == SRCU_STATE_IDLE) { + WARN_ON_ONCE(ssp->srcu_reader_flavor & SRCU_READ_FLAVOR_ATOMIC); srcu_gp_start(ssp); // And how can that list_add() in the "else" clause @@ -1482,6 +1518,8 @@ static void srcu_do_enqueue(struct srcu_struct *ssp, struct rcu_head *rhp, static void __call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp, rcu_callback_t func, bool do_norm) { + if (WARN_ON_ONCE(ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC)) + return; // Leak the callback rather than corrupt SRCU state. if (should_rcu_defer()) { struct srcu_defer *sndp = this_cpu_ptr(&srcu_defer); struct srcu_data *sdp; @@ -1621,6 +1659,11 @@ static void __synchronize_srcu(struct srcu_struct *ssp, bool do_norm) if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE) return; + if (WARN_ON_ONCE(ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC)) { + // This works, and exposes a possible bug. + synchronize_srcu_atomic(ssp); + return; + } might_sleep(); check_init_srcu_struct(ssp, false); init_completion(&rcu.completion); @@ -1741,10 +1784,21 @@ EXPORT_SYMBOL_GPL(get_state_synchronize_srcu); * period has elapsed in the meantime. Unlike get_state_synchronize_srcu(), * this function also ensures that any needed SRCU grace period will be * started. This convenience does come at a cost in terms of CPU overhead. + * + * This function cannot be used with atomic SRCU, which only has + * atomic grace periods. Give a warning if someone tries, and return + * the same cookie that would have been returned, but refrain from + * messing up state by starting a grace period. If someone somewhere + * somehow invokes synchronize_srcu_atomic(), passing this cookie to + * poll_state_synchronize_srcu() will return true. If no one ever invokes + * synchronize_srcu_atomic(), too bad. */ unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp) { - return srcu_gp_start_if_needed(ssp, NULL, true); + if (WARN_ON_ONCE(ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC)) + return get_state_synchronize_srcu(ssp); + else + return srcu_gp_start_if_needed(ssp, NULL, true); } EXPORT_SYMBOL_GPL(start_poll_synchronize_srcu); @@ -1833,6 +1887,12 @@ void srcu_barrier(struct srcu_struct *ssp) unsigned long s; check_init_srcu_struct(ssp, false); + if (WARN_ON_ONCE(ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC)) { + // There shouldn't be any callbacks for atomic SRCU, + // but just in case. + schedule_timeout_uninterruptible(HZ/10); + return; + } /* * Register any deferred callbacks before snapshotting the sequence. The @@ -1904,6 +1964,9 @@ static void srcu_expedite_current_cb(struct rcu_head *rhp) * no current grace period, one might be created. If the current grace * period is currently sleeping, that sleep will complete before expediting * will take effect. + * + * This function must not be invoked on srcu_struct structures that are + * used with srcu_read_lock_atomic() and synchronize_srcu_atomic(). */ void srcu_expedite_current(struct srcu_struct *ssp) { @@ -1911,6 +1974,9 @@ void srcu_expedite_current(struct srcu_struct *ssp) bool needcb = false; struct srcu_data *sdp; + // Atomic SRCU has no callbacks, so there is nothing to expedite. + if (WARN_ON_ONCE(ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC)) + return; migrate_disable(); sdp = this_cpu_ptr(ssp->sda); raw_spin_lock_irqsave_sdp_contention(sdp, &flags); @@ -1978,8 +2044,10 @@ static void srcu_advance_state(struct srcu_struct *ssp, bool is_atomic) return; } idx = rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)); - if (idx == SRCU_STATE_IDLE) + if (idx == SRCU_STATE_IDLE) { + WARN_ON_ONCE(ssp->srcu_reader_flavor & SRCU_READ_FLAVOR_ATOMIC); srcu_gp_start(ssp); + } raw_spin_unlock_irq_rcu_node(ssp->srcu_sup); if (idx != SRCU_STATE_IDLE) { if (!is_atomic) @@ -2015,10 +2083,79 @@ static void srcu_advance_state(struct srcu_struct *ssp, bool is_atomic) return; /* readers present, retry later. */ } ssp->srcu_sup->srcu_n_exp_nodelay = 0; - srcu_gp_end(ssp); /* Releases ->srcu_gp_mutex. */ + srcu_gp_end(ssp, is_atomic); /* Releases ->srcu_gp_mutex. */ } } +/** + * synchronize_srcu_atomic - spin for prior SRCU read-side critical-section completion + * @ssp: srcu_struct with which to synchronize. + * + * Similar to synchronize_srcu(), but spins rather than blocking. + * Use only with srcu_read_lock_atomic() and srcu_read_unlock_atomic(), + * which are forbidden from voluntarily context switching. If + * synchronize_srcu_atomic() is invoked from a more restrictive context + * (for example, interrupts disabled) for a given srcu_struct structure, + * then for that structure, all calls to both srcu_read_lock_atomic() + * and srcu_read_unlock_atomic() must be invoked from that same context, + * or one that is even more strict. + * + * If synchronize_srcu_atomic() is invoked on a given srcu_struct + * structure, then none of call_srcu(), synchronize_srcu(), + * synchronize_srcu_expedited(), start_poll_synchronize_srcu(), + * srcu_barrier(), or srcu_expedite_current() may be invoked on that + * same structure. + * + * Because synchronize_srcu_atomic() is even more expedited than is + * synchronize_srcu_expedited(), there is no expedited counterpart to + * this function. + */ +void synchronize_srcu_atomic(struct srcu_struct *ssp) +{ + unsigned long srcu_state; + struct srcu_usage *sup = ssp->srcu_sup; + + // Initialize. Either init_srcu_struct() was invoked or + // DEFINE_SRCU() or similar was used. Therefore, no allocation + // will be done here. + check_init_srcu_struct(ssp, true); + srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_ATOMIC); + + // Perhaps others will do our work for us. + srcu_state = get_state_synchronize_srcu(ssp); + while (atomic_read(&sup->srcu_atomic_gp_flag) || + atomic_xchg(&sup->srcu_atomic_gp_flag, 1)) { + if (poll_state_synchronize_srcu(ssp, srcu_state)) + return; + cpu_relax(); + } + + // One last check for others doing our work for us under the lock. + raw_spin_lock_irq_rcu_node(sup); + if (poll_state_synchronize_srcu(ssp, srcu_state)) { + raw_spin_unlock_irq_rcu_node(sup); + atomic_set(&sup->srcu_atomic_gp_flag, 0); + return; + } + + // OK, we really have to do it ourselves. Start the grace period. + non_block_start(); // We must not voluntarily block! + smp_store_release(&sup->srcu_gp_seq_needed, srcu_state); // See srcu_funnel_gp_start(). + ASSERT_EXCLUSIVE_WRITER(ssp->srcu_sup->srcu_gp_seq); + srcu_gp_start(ssp); + raw_spin_unlock_irq_rcu_node(sup); + + // Wait for it to complete, helping it along. + while (!poll_state_synchronize_srcu(ssp, srcu_state)) { + cpu_relax(); + srcu_advance_state(ssp, true); + } + ASSERT_EXCLUSIVE_WRITER(sup->srcu_atomic_gp_flag); + atomic_set_release(&sup->srcu_atomic_gp_flag, 0); + non_block_end(); +} +EXPORT_SYMBOL_GPL(synchronize_srcu_atomic); + /* * Invoke a limited number of SRCU callbacks that have passed through * their grace period. If there are more to do, SRCU will reschedule @@ -2098,6 +2235,7 @@ static void srcu_reschedule(struct srcu_struct *ssp, unsigned long delay) } } else if (!rcu_seq_state(ssp->srcu_sup->srcu_gp_seq)) { /* Outstanding request and no GP. Start one. */ + WARN_ON_ONCE(ssp->srcu_reader_flavor & SRCU_READ_FLAVOR_ATOMIC); srcu_gp_start(ssp); } raw_spin_unlock_irq_rcu_node(ssp->srcu_sup); From a48669f0c49c4074079f5d96478a3d47f66135fd Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 18 Aug 2026 17:10:15 -0700 Subject: [PATCH 22/46] srcutiny: Add an atomic Tiny SRCU This commit adds the Tiny SRCU counterpart to Tree SRCU's synchronize_srcu_atomic(). One might hope that this could be as trivial as Tiny RCU's synchronize_rcu(), and there was a time when it would have been. But lazy preemption really can preempt an SRCU read-side critical section, which means that synchronize_srcu_atomic() really must be prepared to spin waiting for it. This spinning currently consists of cond_resched_tasks_rcu_qs() and cpu_relax(). It would be better to have some way of telling the scheduler that there is nothing useful for us to do. We cannot use the traditional wait_event() approach because synchronize_srcu_atomic() is not permitted to block. [ paulmck: Apply Kunwu Chan and Joel Fernandes feedback. ] Co-developed-by: David Woodhouse Signed-off-by: David Woodhouse Assisted-by: Claude:claude-mythos-5 Signed-off-by: Paul E. McKenney --- include/linux/srcutiny.h | 6 +++ kernel/rcu/srcutiny.c | 80 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h index 85b5de438450..47a368f945e3 100644 --- a/include/linux/srcutiny.h +++ b/include/linux/srcutiny.h @@ -19,6 +19,7 @@ struct srcu_struct { short srcu_lock_nesting[2]; /* srcu_read_lock() nesting depth. */ u8 srcu_gp_running; /* GP workqueue running? */ u8 srcu_gp_waiting; /* GP waiting for readers? */ + u8 srcu_atomic_gp_flag; /* Serialize atomic GP work.*/ unsigned long srcu_idx; /* Current reader array element in bit 0x2. */ unsigned long srcu_idx_max; /* Furthest future srcu_idx request. */ struct swait_queue_head srcu_wq; @@ -64,15 +65,20 @@ void srcu_defer_drain(struct irq_work *irq_work); #define DEFINE_SRCU_FAST_UPDOWN(name) DEFINE_SRCU(name) #define DEFINE_STATIC_SRCU_FAST_UPDOWN(name) \ static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name, name) +#define DEFINE_SRCU_ATOMIC(name) DEFINE_SRCU(name) +#define DEFINE_STATIC_SRCU_ATOMIC(name) \ + static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name, name) // Dummy structure for srcu_notifier_head. struct srcu_usage { }; #define __SRCU_USAGE_INIT(name) { } #define __init_srcu_struct_fast __init_srcu_struct #define __init_srcu_struct_fast_updown __init_srcu_struct +#define __init_srcu_struct_atomic __init_srcu_struct #ifndef CONFIG_DEBUG_LOCK_ALLOC #define init_srcu_struct_fast init_srcu_struct #define init_srcu_struct_fast_updown init_srcu_struct +#define init_srcu_struct_atomic init_srcu_struct #endif // #ifndef CONFIG_DEBUG_LOCK_ALLOC void synchronize_srcu(struct srcu_struct *ssp); diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c index 32b37d63d58a..08e089e3e969 100644 --- a/kernel/rcu/srcutiny.c +++ b/kernel/rcu/srcutiny.c @@ -41,6 +41,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp) ssp->srcu_cb_tail = &ssp->srcu_cb_head; ssp->srcu_gp_running = false; ssp->srcu_gp_waiting = false; + ssp->srcu_atomic_gp_flag = 0; ssp->srcu_idx = 0; ssp->srcu_idx_max = 0; INIT_WORK(&ssp->srcu_work, srcu_drive_gp); @@ -339,6 +340,80 @@ void synchronize_srcu(struct srcu_struct *ssp) } EXPORT_SYMBOL_GPL(synchronize_srcu); +/* + * synchronize_srcu_atomic - spinning grace period for atomic-reader domains + * @ssp: srcu_struct with which to synchronize. + * + * On !SMP this cannot spin: a reader observed mid-section is preempted + * or interrupted-out, and can only finish if we yield the CPU. But it + * is also never needed: an atomic-flavor reader (preemption disabled) + * cannot be observed mid-section from process context on the sole CPU. + * So a reader observed here has broken the atomic-domain promise, and + * the only correct wait for it is a real grace period. + * + * (Actual kernel-doc header is in Tree SRCU.) + */ +void synchronize_srcu_atomic(struct srcu_struct *ssp) +{ + int idx; + bool ret; + unsigned long srcu_state = get_state_synchronize_srcu(ssp); + + srcu_lock_sync(&ssp->dep_map); + + if (IS_ENABLED(CONFIG_PREEMPTION)) + synchronize_rcu(); // Needed for RCU Tasks Trace to imply RCU grace period. + // And in Tiny RCU, it is near zero cost and doesn't block. + + // Usually, there will be no readers. + preempt_disable(); // Guard against lazy preemption and some other grace period. + ret = !READ_ONCE(ssp->srcu_lock_nesting[0]) && !READ_ONCE(ssp->srcu_lock_nesting[1]); + if (ret) { + WRITE_ONCE(ssp->srcu_idx_max, ssp->srcu_idx + 2); + WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 2); + preempt_enable(); + return; + } + + // Wait to drive a grace period or for someone else to do it + // for us while we are lazily preempted. + while (ssp->srcu_atomic_gp_flag) { + if (poll_state_synchronize_srcu(ssp, srcu_state)) { + preempt_enable(); + return; + } + preempt_enable(); + cpu_relax(); + cond_resched_tasks_rcu_qs(); + preempt_disable(); + } + ssp->srcu_atomic_gp_flag = 1; // Acquire SRCU grace-period "lock". + preempt_enable(); + + // We get here if a reader has been lazily preempted. + // First, wait for old readers, which are quite unlikely. + WRITE_ONCE(ssp->srcu_idx_max, get_state_synchronize_srcu(ssp)); + idx = !(((READ_ONCE(ssp->srcu_idx) + 1) & 0x2) >> 1); + while (READ_ONCE(ssp->srcu_lock_nesting[idx])) { + cond_resched_tasks_rcu_qs(); + cpu_relax(); + } + + // Next, flip the index and wait for the other group of readers. + WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1); + idx = !idx; + while (READ_ONCE(ssp->srcu_lock_nesting[idx])) { + cond_resched_tasks_rcu_qs(); + cpu_relax(); + } + + // Finally, flip the index again for poll_state_synchronize_srcu(). + WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1); + WARN_ON_ONCE(!poll_state_synchronize_srcu(ssp, srcu_state)); + ssp->srcu_atomic_gp_flag = 0; // Release SRCU grace-period "lock". +} +EXPORT_SYMBOL_GPL(synchronize_srcu_atomic); + /* Register any deferred callbacks, then wait for all in-flight ones. */ void srcu_barrier(struct srcu_struct *ssp) { @@ -367,6 +442,11 @@ EXPORT_SYMBOL_GPL(get_state_synchronize_srcu); * The difference between this and get_state_synchronize_srcu() is that * this function ensures that the poll_state_synchronize_srcu() will * eventually return the value true. + * + * This function cannot be used with atomic SRCU, which only has + * atomic grace periods. Doing so will silently corrupt internal + * SRCU state. Tree SRCU has appropriate checking with splats, + * so please test with CONFIG_SMP=y as well as CONFIG_SMP=n. */ unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp) { From 7fb2388e0da08cd1b3591b651736ce43144d3e6d Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 21 Aug 2026 16:30:31 -0700 Subject: [PATCH 23/46] rcutorture: Add support for testing synchronize_srcu_atomic() This commit adds support for the value 0x10 for reader_flavor, which specifies SRCU_READ_FLAVOR_ATOMIC, that is, srcu_read_lock_atomic(), srcu_read_unlock_atomic(), and synchronize_srcu_atomic(). [ paulmck: Apply Kunwu Chan feedback. ] Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 46 +++++++++++++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 4d1be2a49f01..182d47975efd 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -710,10 +710,21 @@ static struct rcu_torture_ops rcu_busted_ops = { DEFINE_STATIC_SRCU(srcu_ctl); DEFINE_STATIC_SRCU_FAST(srcu_ctlf); DEFINE_STATIC_SRCU_FAST_UPDOWN(srcu_ctlfud); +DEFINE_STATIC_SRCU_ATOMIC(srcu_ctla); static struct srcu_struct srcu_ctld; static struct srcu_struct *srcu_ctlp = &srcu_ctl; static struct rcu_torture_ops srcud_ops; +// Restrict APIs permitted for atomic SRCU. +static void srcu_torture_init_forbidden_apis(void) +{ + cur_ops->call = NULL; + cur_ops->cb_barrier = NULL; + cur_ops->deferred_free = NULL; + cur_ops->exp_current = NULL; + cur_ops->start_gp_poll = NULL; +} + static void srcu_torture_init(void) { rcu_sync_torture_init(); @@ -729,6 +740,11 @@ static void srcu_torture_init(void) srcu_ctlp = &srcu_ctlfud; VERBOSE_TOROUT_STRING("srcu_torture_init fast-up/down SRCU"); } + if (reader_flavor & SRCU_READ_FLAVOR_ATOMIC) { + srcu_ctlp = &srcu_ctla; + VERBOSE_TOROUT_STRING("srcu_torture_init atomic SRCU"); + srcu_torture_init_forbidden_apis(); + } } static void srcu_get_gp_data(int *flags, unsigned long *gp_seq) @@ -766,6 +782,11 @@ static int srcu_torture_read_lock(void) WARN_ON_ONCE(idx & ~0x1); ret += idx << 3; } + if (reader_flavor & SRCU_READ_FLAVOR_ATOMIC) { + idx = srcu_read_lock_atomic(srcu_ctlp); + WARN_ON_ONCE(idx & ~0x1); + ret += idx << 4; + } return ret; } @@ -785,7 +806,8 @@ srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) delay = torture_random(rrsp) % (nrealreaders * 2 * longdelay * uspertick); - if (!delay && !in_atomic() && !rcu_preempt_depth() && !irqs_disabled()) { + if (!delay && !in_atomic() && !rcu_preempt_depth() && !irqs_disabled() && + !(reader_flavor & SRCU_READ_FLAVOR_ATOMIC)) { schedule_timeout_interruptible(longdelay); rtrsp->rt_delay_jiffies = longdelay; } else { @@ -796,6 +818,8 @@ srcu_read_delay(struct torture_random_state *rrsp, struct rt_read_seg *rtrsp) static void srcu_torture_read_unlock(int idx) { WARN_ON_ONCE((reader_flavor && (idx & ~reader_flavor)) || (!reader_flavor && (idx & ~0x1))); + if (reader_flavor & SRCU_READ_FLAVOR_ATOMIC) + srcu_read_unlock_atomic(srcu_ctlp, (idx & 0x10) >> 4); if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN) srcu_read_unlock_fast_updown(srcu_ctlp, __srcu_ctr_to_ptr(srcu_ctlp, (idx & 0x8) >> 3)); @@ -875,7 +899,10 @@ static void srcu_torture_deferred_free(struct rcu_torture *rp) static void srcu_torture_synchronize(void) { - synchronize_srcu(srcu_ctlp); + if (reader_flavor & SRCU_READ_FLAVOR_ATOMIC) + synchronize_srcu_atomic(srcu_ctlp); + else + synchronize_srcu(srcu_ctlp); } static unsigned long srcu_torture_get_gp_state(void) @@ -911,7 +938,10 @@ static void srcu_torture_stats(void) static void srcu_torture_synchronize_expedited(void) { - synchronize_srcu_expedited(srcu_ctlp); + if (reader_flavor & SRCU_READ_FLAVOR_ATOMIC) + synchronize_srcu_atomic(srcu_ctlp); + else + synchronize_srcu_expedited(srcu_ctlp); } static void srcu_torture_expedite_current(void) @@ -969,6 +999,10 @@ static void srcud_torture_init(void) } else if (reader_flavor & SRCU_READ_FLAVOR_FAST_UPDOWN) { WARN_ON(init_srcu_struct_fast_updown(&srcu_ctld)); VERBOSE_TOROUT_STRING("srcud_torture_init fast-up/down SRCU"); + } else if (reader_flavor & SRCU_READ_FLAVOR_ATOMIC) { + WARN_ON(init_srcu_struct_atomic(&srcu_ctld)); + VERBOSE_TOROUT_STRING("srcud_torture_init atomic SRCU"); + srcu_torture_init_forbidden_apis(); } else { WARN_ON(init_srcu_struct(&srcu_ctld)); } @@ -1749,7 +1783,7 @@ rcu_torture_writer(void *arg) pr_alert("%s" TORTURE_FLAG " Waited %lu jiffies for boot to complete.\n", torture_type, jiffies - j); - if (IS_ENABLED(CONFIG_RCU_LAZY)) + if (IS_ENABLED(CONFIG_RCU_LAZY) && cur_ops->call) INIT_WORK_ONSTACK(&lazy_work, rcu_torture_writer_work); do { @@ -1944,7 +1978,7 @@ rcu_torture_writer(void *arg) !rcu_gp_is_normal(); } rcu_torture_writer_state = RTWS_STUTTER; - if (IS_ENABLED(CONFIG_RCU_LAZY)) + if (IS_ENABLED(CONFIG_RCU_LAZY) && cur_ops->call) queue_work(system_percpu_wq, &lazy_work); stutter_waited = stutter_wait("rcu_torture_writer"); if (stutter_waited && @@ -1977,7 +2011,7 @@ rcu_torture_writer(void *arg) " Dynamic grace-period expediting was disabled.\n", torture_type); - if (IS_ENABLED(CONFIG_RCU_LAZY)) { + if (IS_ENABLED(CONFIG_RCU_LAZY) && cur_ops->call) { cancel_work_sync(&lazy_work); destroy_work_on_stack(&lazy_work); } From 18c55d888b0afa3d8156b812e76c3d29a1bb2a2d Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 26 Aug 2026 09:03:57 -0700 Subject: [PATCH 24/46] srcutree: Disable preemption across synchronize_srcu_atomic() Because synchronize_srcu_atomic() cannot sleep, if a pair of them run concurrently pinned to the same CPU, it is possible that one will spin uselessly waiting for the other while at the same time preventing that other from running. This commit therefore disables preemption to prevent this failure mode. Signed-off-by: Paul E. McKenney --- kernel/rcu/srcutree.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 7ff45e992c01..672fb41345c9 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -2123,11 +2123,14 @@ void synchronize_srcu_atomic(struct srcu_struct *ssp) // Perhaps others will do our work for us. srcu_state = get_state_synchronize_srcu(ssp); + preempt_disable(); while (atomic_read(&sup->srcu_atomic_gp_flag) || atomic_xchg(&sup->srcu_atomic_gp_flag, 1)) { + preempt_enable(); if (poll_state_synchronize_srcu(ssp, srcu_state)) return; cpu_relax(); + preempt_disable(); } // One last check for others doing our work for us under the lock. @@ -2135,6 +2138,7 @@ void synchronize_srcu_atomic(struct srcu_struct *ssp) if (poll_state_synchronize_srcu(ssp, srcu_state)) { raw_spin_unlock_irq_rcu_node(sup); atomic_set(&sup->srcu_atomic_gp_flag, 0); + preempt_enable(); return; } @@ -2152,6 +2156,7 @@ void synchronize_srcu_atomic(struct srcu_struct *ssp) } ASSERT_EXCLUSIVE_WRITER(sup->srcu_atomic_gp_flag); atomic_set_release(&sup->srcu_atomic_gp_flag, 0); + preempt_enable(); non_block_end(); } EXPORT_SYMBOL_GPL(synchronize_srcu_atomic); From f8d95888093348be1235815946359f9cbc2f7012 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Fri, 21 Aug 2026 20:38:09 +0200 Subject: [PATCH 25/46] srcu: Use IRQ_WORK_INIT_HARD for srcu's irq_work MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The irq_work in srcu is used to schedule a delayed work. The work item is not scheduled directly because it is not always possible wake a thread directly. On PREEMPT_RT the default irq_work is initialized with IRQ_WORK_LAZY and is delayed to the irq_work thread. A system booted with the command line "trace_event=…" will freeze during boot because the irq_work thread is not yet deployed (and tracing uses synchronize_srcu() in tp_rcu_cond_sync()). The irq_work performs just a wakeup a thread, there is nothing wrong with doing this from hardirq context on PREEMPT_RT. Use IRQ_WORK_INIT_HARD for srcu's irq_work. Fixes: 7c405fb3279b3 ("rcu: Use an intermediate irq_work to start process_srcu()") Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Paul E. McKenney --- kernel/rcu/srcutree.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 672fb41345c9..64081c8eacc8 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -259,7 +259,12 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static, bool mutex_init(&ssp->srcu_sup->srcu_barrier_mutex); atomic_set(&ssp->srcu_sup->srcu_barrier_cpu_cnt, 0); INIT_DELAYED_WORK(&ssp->srcu_sup->work, process_srcu); - init_irq_work(&ssp->srcu_sup->irq_work, srcu_irq_work); + /* + * trace events started on the command line require SRCU before + * the irq_work kthread starts. Since all it does is a simple + * wakeup, having it as a hard irq, even on PREEMPT_RT is fine. + */ + ssp->srcu_sup->irq_work = IRQ_WORK_INIT_HARD(srcu_irq_work); ssp->srcu_sup->sda_is_static = is_static; if (!is_static) { ssp->sda = alloc_percpu(struct srcu_data); From 494fa5fcb1de0a1d3b7f864db35877ad552f39ad Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 2 Sep 2026 08:30:38 -0700 Subject: [PATCH 26/46] srcutree: Warn if Tiny SRCU readers are preempted The fastpath of synchronize_srcu_atomic() should always be taken because readers always disable preemption. Therefore, the fact that synchronize_srcu_atomic() is running at all should mean that all readers have ended. But there are always bugs. And responding to a usage bug with a too-short SRCU grace period, and thus possibly corrupting memory is at best a sadistic response so such a bug. For this reason, synchronize_srcu_atomic() explicitly waits for readers. Except that it does so silently, possibly failing to flag this bug. This commit therefore adds a splat if synchronize_srcu_atomic() fails to take the early exit. Signed-off-by: Paul E. McKenney --- kernel/rcu/srcutiny.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c index 08e089e3e969..a31398def0b1 100644 --- a/kernel/rcu/srcutiny.c +++ b/kernel/rcu/srcutiny.c @@ -375,6 +375,14 @@ void synchronize_srcu_atomic(struct srcu_struct *ssp) return; } + // Because readers disable preemption, we should never get here. + // However, a splat and some spinning is usually preferable to + // memory corruption due to a too-short grace period. There is + // the possibility that this will hang if the preempted reader is + // not looked upon favorably by the scheduler, but this is still + // preferable to memory corruption. + WARN_ON_ONCE(1); + // Wait to drive a grace period or for someone else to do it // for us while we are lazily preempted. while (ssp->srcu_atomic_gp_flag) { From 06726ed5906d6a403a79e78f80a428ffcbe0bbb4 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 2 Sep 2026 12:58:24 -0700 Subject: [PATCH 27/46] srcutree: Explicitly note DEFINE_SRCU() needs for srcu_barrier() In the core kernel, an srcu_struct structure created by DEFINE_SRCU() or friends lives as long as the kernel does, so there are no particular requirements surrounding the end of that structure's life. In contrast, when DEFINE_SRCU() and friends are used within a module, the corresponding srcu_struct structures' lifetimes end when that module exits. This in turn means that if such a structure was passed to call_srcu(), then srcu_barrier() must be invoked after the last call_srcu() invocation but before the module exits. This commit therefore adds a comment stating this. Reported-by: Alexander Aring Signed-off-by: Paul E. McKenney --- include/linux/srcutree.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h index ad9d9658b0a2..93b76e543894 100644 --- a/include/linux/srcutree.h +++ b/include/linux/srcutree.h @@ -214,6 +214,12 @@ struct srcu_struct { * instead of smp_mb(), and given that the first (for example) * srcu_read_lock_fast() might race with the first synchronize_srcu(), * this different must be specified at initialization time. + * + * If you use any of the DEFINE_SRCU() functions within a module, the + * module-entry code will invoke init_srcu_struct() and the module-exit + * code will invoke cleanup_srcu_struct(). This means that if your module + * passes the resulting srcu_struct structure to call_srcu(), you will + * need to also pass this structure to srcu_barrier() prior to module exit. */ #ifdef MODULE # define __DEFINE_SRCU(name, fast, is_static) \ From ddd326d8f36d6943a510e93ddf16c43037fef1c1 Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Mon, 7 Sep 2026 15:58:20 +0800 Subject: [PATCH 28/46] rcutorture: Add atomic-SRCU support to torture.sh Add the --do-atomic-srcu argument to torture.sh, which runs the SRCU-N, SRCU-P, and SRCU-T scenarios, thus covering both Tree SRCU (SRCU-N and SRCU-P) and Tiny SRCU (SRCU-T), with rcutorture.reader_flavor=0x10 appended to the boot parameters so that it takes precedence over each scenario's own reader-flavor setting. This exercises srcu_read_lock_atomic(), srcu_read_unlock_atomic(), and synchronize_srcu_atomic(). As with other torture.sh tests, the --do-kcsan argument runs a KCSAN+PROVE_LOCKING variant of this test. [ paulmck: Make --do-atomic-srcu be default-on. ] Signed-off-by: Kunwu Chan Signed-off-by: Paul E. McKenney --- .../selftests/rcutorture/bin/torture.sh | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tools/testing/selftests/rcutorture/bin/torture.sh b/tools/testing/selftests/rcutorture/bin/torture.sh index f0083891ee81..e7b578ec152f 100755 --- a/tools/testing/selftests/rcutorture/bin/torture.sh +++ b/tools/testing/selftests/rcutorture/bin/torture.sh @@ -68,6 +68,7 @@ do_clocksourcewd="${ifnotaarch64}" do_rt=yes do_rcutasksflavors="${ifnotaarch64}" # FIXME: Back to "yes" when SMP=n auto-avoided do_srcu_lockdep=yes +do_atomic_srcu=yes do_rcu_rust=no # doyesno - Helper function for yes/no arguments @@ -103,6 +104,7 @@ usage () { echo " --do-rcu-rust / --do-no-rcu-rust / --no-rcu-rust" echo " --do-scftorture / --do-no-scftorture / --no-scftorture" echo " --do-srcu-lockdep / --do-no-srcu-lockdep / --no-srcu-lockdep" + echo " --do-atomic-srcu / --do-no-atomic-srcu / --no-atomic-srcu" echo " --duration [ | h | d ]" echo " --guest-cpu-limit N" echo " --kcsan-kmake-arg kernel-make-arguments" @@ -148,6 +150,7 @@ do do_kcsan=yes do_clocksourcewd="${ifnotaarch64}" do_srcu_lockdep=yes + do_atomic_srcu=yes ;; --do-allmodconfig|--do-no-allmodconfig|--no-allmodconfig) do_allmodconfig=`doyesno "$1" --do-allmodconfig` @@ -183,6 +186,7 @@ do do_kcsan=no do_clocksourcewd=no do_srcu_lockdep=no + do_atomic_srcu=no ;; --do-normal|--do-norm|--do-no-normal|--do-no-norm|--no-normal|--no-norm) do_normal=`doyesno "$1" --do-normal` @@ -212,6 +216,9 @@ do --do-srcu-lockdep|--do-no-srcu-lockdep|--no-srcu-lockdep) do_srcu_lockdep=`doyesno "$1" --do-srcu-lockdep` ;; + --do-atomic-srcu|--do-no-atomic-srcu|--no-atomic-srcu) + do_atomic_srcu=`doyesno "$1" --do-atomic-srcu` + ;; --duration) checkarg --duration "(minutes)" $# "$2" '^[0-9][0-9]*\(m\|h\|d\|\)$' '^error' mult=1 @@ -497,6 +504,23 @@ then torture_set "rcutorture" tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration "$duration_rcutorture" --configs "$configs_rcutorture" --trust-make fi +# Test atomic SRCU across Tree SRCU (SRCU-N and SRCU-P) and Tiny SRCU +# (SRCU-T). The reader flavor selects srcu_read_lock_atomic() and +# synchronize_srcu_atomic(). Tiny SRCU requires SMP=n, which aarch64 +# does not support. +if test "$do_atomic_srcu" = "yes" +then + torture_bootargs="rcutorture.reader_flavor=0x10" + configs_atomic_srcu="SRCU-N SRCU-P" + if test "$ifnotaarch64" = yes + then + configs_atomic_srcu="$configs_atomic_srcu SRCU-T" + fi + torture_set "atomic-srcu" tools/testing/selftests/rcutorture/bin/kvm.sh \ + --allcpus --duration "$duration_rcutorture" \ + --configs "$configs_atomic_srcu" --trust-make +fi + if test "$do_locktorture" = "yes" then torture_bootargs="torture.disable_onoff_at_boot" From 30d89cea5fa0118e4aec52511ab8210da4c363d9 Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Mon, 7 Sep 2026 15:58:19 +0800 Subject: [PATCH 29/46] srcutree: Add reader-free fastpath to synchronize_srcu_atomic() synchronize_srcu_atomic() is restricted to srcu_read_lock_atomic() and srcu_read_unlock_atomic(), whose read-side critical sections disable preemption. In the common case where there are no readers at all, the grace period therefore need not do the index flip. Add a fastpath that sums both ranks of the per-CPU ->srcu_ctrs[] counters and, if the lock counts match the unlock counts on both ranks, ends the grace period immediately, skipping the srcu_advance_state() scans, mirroring the similar Tiny SRCU fastpath. Correctness requires the counter-sum proof to follow the grace-period anchor written by srcu_gp_start(); placing it before the anchor could let this grace period miss a pre-existing reader and return without waiting for it. The smp_mb() between the unlock and lock sums pairs with the smp_mb() in __srcu_read_lock(). The grace period is ended manually under ->lock and ->srcu_atomic_gp_flag. In theory, this is slower than David Woodhouse's earlier patch, but David's measurements showed that the performance was close enough that it makes sense to keep the get_state_synchronize_srcu() and poll_state_synchronize_srcu semantics. Link: https://lore.kernel.org/all/20260907075829.2073224-4-kunwu.chan@linux.dev/ Link: https://lore.kernel.org/all/0f4dea21bac43685d3286df329401177e4452b36.camel@infradead.org/ Link: https://lore.kernel.org/all/0d4af6318ac67486858be1df8d436147b444a2d2.camel@infradead.org/ Signed-off-by: Kunwu Chan Signed-off-by: Paul E. McKenney Cc: David Woodhouse --- kernel/rcu/srcutree.c | 48 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 64081c8eacc8..93b8d1088abe 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -2119,6 +2119,8 @@ void synchronize_srcu_atomic(struct srcu_struct *ssp) { unsigned long srcu_state; struct srcu_usage *sup = ssp->srcu_sup; + unsigned long rdm0, rdm1; + unsigned long unlocks0, unlocks1; // Initialize. Either init_srcu_struct() was invoked or // DEFINE_SRCU() or similar was used. Therefore, no allocation @@ -2154,6 +2156,52 @@ void synchronize_srcu_atomic(struct srcu_struct *ssp) srcu_gp_start(ssp); raw_spin_unlock_irq_rcu_node(sup); + // + // Fastpath: If there are no readers at all, neither grace-period + // scan need wait, so both can be satisfied at once without doing + // the index flip. The counter-sum proof is the same as that of + // srcu_readers_active_idx_check(), but spanning both indices. + // Atomic SRCU guarantees that all readers are of + // SRCU_READ_FLAVOR_ATOMIC, so the SLOWGP check never triggers and + // the ->srcu_reader_flavor masks returned by + // srcu_readers_unlock_idx() are unused. + // + // This proof must follow the grace-period anchor written by the + // srcu_gp_start() above, never precede it. With the anchor first, + // a reader whose lock increment is missed by the sums below cannot + // have incremented its lock counter before the anchor, and therefore + // cannot be a pre-existing reader of this grace period. Placing the + // proof before the anchor would let this grace period miss a + // pre-existing reader and return without waiting for it. + // + // The smp_mb() pairs with the smp_mb() in __srcu_read_lock() + // (store-buffering pattern), which guarantees that a lock is always + // counted if the corresponding unlock is counted, the same + // memory-ordering guarantee as is provided by + // srcu_readers_active_idx_check(). + // + unlocks0 = srcu_readers_unlock_idx(ssp, 0, &rdm0); + unlocks1 = srcu_readers_unlock_idx(ssp, 1, &rdm1); + smp_mb(); /* A */ + if (srcu_readers_lock_idx(ssp, 0, false, unlocks0) && + srcu_readers_lock_idx(ssp, 1, false, unlocks1)) { + // No readers, so end this grace period manually, skipping + // the index flip. Advancing the sequence number via + // rcu_seq_start() in srcu_gp_start() above and rcu_seq_end() + // below keeps get_state_synchronize_srcu() and + // poll_state_synchronize_srcu() working, all under ->lock + // and ->srcu_atomic_gp_flag, which excludes concurrent + // sequence-number updates. + raw_spin_lock_irq_rcu_node(sup); + rcu_seq_end(&sup->srcu_gp_seq); + raw_spin_unlock_irq_rcu_node(sup); + WARN_ON_ONCE(!poll_state_synchronize_srcu(ssp, srcu_state)); + atomic_set_release(&sup->srcu_atomic_gp_flag, 0); + preempt_enable(); + non_block_end(); + return; + } + // Wait for it to complete, helping it along. while (!poll_state_synchronize_srcu(ssp, srcu_state)) { cpu_relax(); From 41f899f18d0e1f00acd83b3f85ee3baa5da3d8fe Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Mon, 7 Sep 2026 15:58:26 +0800 Subject: [PATCH 30/46] srcutree: Skip callback scheduling for atomic SRCU grace periods call_srcu() is forbidden on atomic srcu_struct, so srcu_gp_end() never has callbacks to invoke for them. Yet it schedules callback invocation, which for atomic SRCU's SRCU_SIZE_SMALL state arms the boot CPU's ->delay_work timer every grace period, only for srcu_invoke_callbacks() to find nothing to do. Skip this for atomic SRCU. Signed-off-by: Kunwu Chan Signed-off-by: Paul E. McKenney --- kernel/rcu/srcutree.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 93b8d1088abe..0954486880c7 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -1016,10 +1016,10 @@ static void srcu_gp_end(struct srcu_struct *ssp, bool is_atomic) /* Initiate callback invocation as needed. */ ss_state = smp_load_acquire(&sup->srcu_size_state); - if (ss_state < SRCU_SIZE_WAIT_BARRIER) { + if (!is_atomic && ss_state < SRCU_SIZE_WAIT_BARRIER) { srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, get_boot_cpu_id()), cbdelay); - } else { + } else if (!is_atomic) { idx = rcu_seq_ctr(gpseq) % ARRAY_SIZE(snp->srcu_have_cbs); srcu_for_each_node_breadth_first(ssp, snp) { raw_spin_lock_irq_rcu_node(snp); From 47138812bc786b74a9f96874782668c64860dcc1 Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Mon, 7 Sep 2026 15:58:27 +0800 Subject: [PATCH 31/46] srcutree: Remove srcu_barrier() sleep for atomic SRCU The atomic-SRCU path in srcu_barrier() sleeps for 100 milliseconds just in case there are callbacks to wait for. But call_srcu() refuses atomic SRCU with a WARN_ON_ONCE() before reaching the deferred-enqueue path, so there can be no callbacks, deferred or otherwise. Drop the sleep. Signed-off-by: Kunwu Chan Signed-off-by: Paul E. McKenney --- kernel/rcu/srcutree.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 0954486880c7..82e61421142e 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -1892,12 +1892,9 @@ void srcu_barrier(struct srcu_struct *ssp) unsigned long s; check_init_srcu_struct(ssp, false); - if (WARN_ON_ONCE(ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC)) { - // There shouldn't be any callbacks for atomic SRCU, - // but just in case. - schedule_timeout_uninterruptible(HZ/10); + // Atomic SRCU has no callbacks, so there is nothing to wait on. + if (WARN_ON_ONCE(ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC)) return; - } /* * Register any deferred callbacks before snapshotting the sequence. The From cc7287f9ee696e71c3adc23baf21a2dbe0e94874 Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Mon, 7 Sep 2026 15:58:29 +0800 Subject: [PATCH 32/46] srcu: Restrict atomic-SRCU non_block annotation to task context srcu_read_lock_atomic() and srcu_read_unlock_atomic() arm and disarm might_sleep() checks via non_block_start()/non_block_end(), skipping hardirq so the update does not land on the interrupted task's ->non_block_count. Inline softirqs run on the interrupted task's stack as well, so a timer callback running atomic-SRCU readers races with the interrupted task's own ->non_block_count updates, as KCSAN reports: BUG: KCSAN: data-race in srcu_torture_read_lock / srcu_torture_read_unlock write to 0xffffa00f818ea418 of 4 bytes by interrupt on cpu 0: srcu_torture_read_lock+0x422/0x470 rcutorture_one_extend+0xdc/0x600 rcu_torture_one_read+0xd1/0x330 rcu_torture_timer+0x75/0x140 call_timer_fn+0xe6/0x2f0 ... run_timer_softirq+0xb7/0x130 handle_softirqs+0xfc/0x3f0 __irq_exit_rcu+0x8e/0x100 Use in_task() so the annotation is applied only in task context; it is redundant elsewhere because might_sleep() already warns about sleeping from atomic context. Signed-off-by: Kunwu Chan Signed-off-by: Paul E. McKenney --- include/linux/srcu.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/include/linux/srcu.h b/include/linux/srcu.h index 3f232f2e0524..1a8a465a5650 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h @@ -346,11 +346,13 @@ static inline int srcu_read_lock_atomic(struct srcu_struct *ssp) /* * Arm might_sleep() to catch even a *potentially* sleeping call * in the section, not just an actual schedule: the atomic-domain - * promise must hold on every path, contended or not. In hardirq - * the annotation would land on the interrupted task; it is also + * promise must hold on every path, contended or not. In hardirq, + * softirq, or NMI the annotation would land on the interrupted + * task, and can also result in data races against that task's + * own non_block_start()/non_block_end() invocations; it is also * redundant there, so skip it. */ - if (!in_hardirq()) + if (in_task()) non_block_start(); srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_ATOMIC); retval = __srcu_read_lock(ssp); @@ -562,7 +564,7 @@ static inline void srcu_read_unlock_atomic(struct srcu_struct *ssp, int idx) srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_ATOMIC); srcu_lock_release(&ssp->dep_map); __srcu_read_unlock(ssp, idx); - if (!in_hardirq()) + if (in_task()) non_block_end(); preempt_enable(); } From cbd520f619714e88cca1b814ae3486427133e9ed Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Mon, 7 Sep 2026 15:58:22 +0800 Subject: [PATCH 33/46] srcutree: Make init_srcu_struct_atomic() prevent transition to big The is_atomic parameter of init_srcu_struct_fields() exists so that atomic SRCU never transitions to big, but neither init_srcu_struct_atomic() nor its lockdep counterpart __init_srcu_struct_atomic() sets it. On systems where srcutree.convert_to_big selects SRCU_SIZING_INIT, this needlessly allocates a full srcu_node combining tree for any dynamically initialized atomic srcu_struct, despite atomic SRCU having neither callbacks nor srcu_barrier() operations. Pass true from both atomic entry points, adding an is_atomic parameter to __init_srcu_struct_common(). Signed-off-by: Kunwu Chan Signed-off-by: Paul E. McKenney --- kernel/rcu/srcutree.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 82e61421142e..4e9a0b8ea344 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -305,26 +305,27 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static, bool #ifdef CONFIG_DEBUG_LOCK_ALLOC static int -__init_srcu_struct_common(struct srcu_struct *ssp, const char *name, struct lock_class_key *key) +__init_srcu_struct_common(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key, bool is_atomic) { /* Don't re-initialize a lock while it is held. */ debug_check_no_locks_freed((void *)ssp, sizeof(*ssp)); lockdep_init_map(&ssp->dep_map, name, key, 0); - return init_srcu_struct_fields(ssp, false, false); + return init_srcu_struct_fields(ssp, false, is_atomic); } int init_srcu_struct_lockdep(struct srcu_struct *ssp, const char *name, struct lock_class_key *key) { ssp->srcu_reader_flavor = 0; - return __init_srcu_struct_common(ssp, name, key); + return __init_srcu_struct_common(ssp, name, key, false); } EXPORT_SYMBOL_GPL(init_srcu_struct_lockdep); int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key) { ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_FAST; - return __init_srcu_struct_common(ssp, name, key); + return __init_srcu_struct_common(ssp, name, key, false); } EXPORT_SYMBOL_GPL(__init_srcu_struct_fast); @@ -332,14 +333,14 @@ int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, struct lock_class_key *key) { ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_FAST_UPDOWN; - return __init_srcu_struct_common(ssp, name, key); + return __init_srcu_struct_common(ssp, name, key, false); } EXPORT_SYMBOL_GPL(__init_srcu_struct_fast_updown); int __init_srcu_struct_atomic(struct srcu_struct *ssp, const char *name, struct lock_class_key *key) { ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_ATOMIC; - return __init_srcu_struct_common(ssp, name, key); + return __init_srcu_struct_common(ssp, name, key, true); } EXPORT_SYMBOL_GPL(__init_srcu_struct_atomic); @@ -414,7 +415,7 @@ EXPORT_SYMBOL_GPL(init_srcu_struct_fast_updown); int init_srcu_struct_atomic(struct srcu_struct *ssp) { ssp->srcu_reader_flavor = SRCU_READ_FLAVOR_ATOMIC; - return init_srcu_struct_fields(ssp, false, false); + return init_srcu_struct_fields(ssp, false, true); } EXPORT_SYMBOL_GPL(init_srcu_struct_atomic); From 378ee30c813edf9e015eb1493cf2188f25682a31 Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Fri, 11 Sep 2026 15:21:25 +0800 Subject: [PATCH 34/46] srcutree: Don't transition atomic SRCU to big in srcu_gp_end() Atomic SRCU must remain in the small size state. Warn if this invariant is violated and avoid transitioning to big in that case. [ paulmck: Folded "if" onto one line for 100-character limit. ] Signed-off-by: Kunwu Chan Signed-off-by: Paul E. McKenney Reviewed-by: Bradley Morgan --- kernel/rcu/srcutree.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 4e9a0b8ea344..6a05bd8a6f39 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -1073,8 +1073,10 @@ static void srcu_gp_end(struct srcu_struct *ssp, bool is_atomic) raw_spin_unlock_irq_rcu_node(sup); } - /* Transition to big if needed. */ - if (ss_state != SRCU_SIZE_SMALL && ss_state != SRCU_SIZE_BIG) { + /* Transition to big if needed, but never for atomic SRCU. */ + if (ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC && ss_state != SRCU_SIZE_SMALL) { + WARN_ON_ONCE(1); + } else if (ss_state != SRCU_SIZE_SMALL && ss_state != SRCU_SIZE_BIG) { if (ss_state == SRCU_SIZE_ALLOC) init_srcu_struct_nodes(ssp, GFP_KERNEL); else From 0150a00d829e866931775d9503969402d2274251 Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Fri, 11 Sep 2026 12:02:51 +0800 Subject: [PATCH 35/46] srcutree: Skip torture to-big transition for atomic SRCU Atomic SRCU does not use the srcu_node combining tree. Exclude it from the SRCU_SIZING_IS_TORTURE() transition in srcu_torture_stats_print(), and warn if such a transition is requested. Signed-off-by: Kunwu Chan Signed-off-by: Paul E. McKenney --- kernel/rcu/srcutree.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 6a05bd8a6f39..6e37d53b412e 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -2422,8 +2422,10 @@ void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf) } pr_cont(" T(%ld,%ld)\n", s0, s1); } - if (SRCU_SIZING_IS_TORTURE()) - srcu_transition_to_big(ssp); + if (SRCU_SIZING_IS_TORTURE()) { + if (!WARN_ON_ONCE(ssp->srcu_reader_flavor == SRCU_READ_FLAVOR_ATOMIC)) + srcu_transition_to_big(ssp); + } } EXPORT_SYMBOL_GPL(srcu_torture_stats_print); From 91d2ab160c7c3f001847fe56126eb2f65d681f1e Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Fri, 11 Sep 2026 17:35:15 +0800 Subject: [PATCH 36/46] srcutree: Add lockdep and early-boot checks to synchronize_srcu_atomic() Add the lockdep annotation, same-type SRCU nesting warning, and early-boot check used by __synchronize_srcu(). Suggested-by: Zqiang Signed-off-by: Kunwu Chan Reviewed-by: Joel Fernandes Signed-off-by: Paul E. McKenney --- kernel/rcu/srcutree.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 6e37d53b412e..7890f7c7c2ee 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -2122,6 +2122,14 @@ void synchronize_srcu_atomic(struct srcu_struct *ssp) unsigned long rdm0, rdm1; unsigned long unlocks0, unlocks1; + srcu_lock_sync(&ssp->dep_map); + + RCU_LOCKDEP_WARN(lockdep_is_held(ssp), + "Illegal synchronize_srcu_atomic() in same-type SRCU read-side critical section"); + + if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE) + return; + // Initialize. Either init_srcu_struct() was invoked or // DEFINE_SRCU() or similar was used. Therefore, no allocation // will be done here. From 8d99291b14f1cb441279ba800ad7535e0e760b63 Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Sat, 12 Sep 2026 00:09:13 +0800 Subject: [PATCH 37/46] srcutiny: Add lockdep and early-boot checks to synchronize_srcu_atomic() Add the RCU_LOCKDEP_WARN and early-boot check that the Tree version gained, srcu_lock_sync() was already present. Suggested-by: Zqiang Signed-off-by: Kunwu Chan Reviewed-by: Bradley Morgan Reviewed-by: Joel Fernandes Signed-off-by: Paul E. McKenney --- kernel/rcu/srcutiny.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c index a31398def0b1..63a3f0a8bc53 100644 --- a/kernel/rcu/srcutiny.c +++ b/kernel/rcu/srcutiny.c @@ -361,6 +361,12 @@ void synchronize_srcu_atomic(struct srcu_struct *ssp) srcu_lock_sync(&ssp->dep_map); + RCU_LOCKDEP_WARN(lockdep_is_held(ssp), + "Illegal synchronize_srcu_atomic() in same-type SRCU read-side critical section"); + + if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE) + return; + if (IS_ENABLED(CONFIG_PREEMPTION)) synchronize_rcu(); // Needed for RCU Tasks Trace to imply RCU grace period. // And in Tiny RCU, it is near zero cost and doesn't block. From 13fbbaefb0ac4e4a37e357e0083ca67e84302cd6 Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Mon, 14 Sep 2026 17:34:17 +0800 Subject: [PATCH 38/46] srcutree: Preserve IRQ state in synchronize_srcu_atomic() callchain synchronize_srcu_atomic() may be called with interrupts disabled, but its callchain uses raw_spin_lock_irq_rcu_node() / raw_spin_unlock_irq_rcu_node(), which unconditionally enables interrupts on unlock. This can incorrectly change the caller's IRQ state. Use the irqsave/irqrestore variants for the locks in this callchain so that the caller's IRQ state is preserved. Co-developed-by: Zqiang Signed-off-by: Zqiang Signed-off-by: Kunwu Chan Acked-by: Boqun Feng Signed-off-by: Paul E. McKenney --- kernel/rcu/srcutree.c | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 7890f7c7c2ee..c8209aa6552a 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -985,6 +985,7 @@ static void srcu_gp_end(struct srcu_struct *ssp, bool is_atomic) bool cbs; bool last_lvl; int cpu; + unsigned long flags; unsigned long gpseq; int idx; unsigned long mask; @@ -999,7 +1000,7 @@ static void srcu_gp_end(struct srcu_struct *ssp, bool is_atomic) mutex_lock(&sup->srcu_cb_mutex); /* End the current grace period. */ - raw_spin_lock_irq_rcu_node(sup); + raw_spin_lock_irqsave_rcu_node(sup, flags); idx = rcu_seq_state(sup->srcu_gp_seq); WARN_ON_ONCE(idx != SRCU_STATE_SCAN2); if (srcu_gp_is_expedited(ssp)) @@ -1010,7 +1011,7 @@ static void srcu_gp_end(struct srcu_struct *ssp, bool is_atomic) gpseq = rcu_seq_current(&sup->srcu_gp_seq); if (ULONG_CMP_LT(sup->srcu_gp_seq_needed_exp, gpseq)) WRITE_ONCE(sup->srcu_gp_seq_needed_exp, gpseq); - raw_spin_unlock_irq_rcu_node(sup); + raw_spin_unlock_irqrestore_rcu_node(sup, flags); if (!is_atomic) mutex_unlock(&sup->srcu_gp_mutex); /* A new grace period can start at this point. But only one. */ @@ -1048,12 +1049,12 @@ static void srcu_gp_end(struct srcu_struct *ssp, bool is_atomic) if (!(gpseq & counter_wrap_check)) for_each_possible_cpu(cpu) { sdp = per_cpu_ptr(ssp->sda, cpu); - raw_spin_lock_irq_rcu_node(sdp); + raw_spin_lock_irqsave_rcu_node(sdp, flags); if (ULONG_CMP_GE(gpseq, sdp->srcu_gp_seq_needed + 100)) sdp->srcu_gp_seq_needed = gpseq; if (ULONG_CMP_GE(gpseq, sdp->srcu_gp_seq_needed_exp + 100)) sdp->srcu_gp_seq_needed_exp = gpseq; - raw_spin_unlock_irq_rcu_node(sdp); + raw_spin_unlock_irqrestore_rcu_node(sdp, flags); } /* Callback initiation done, allow grace periods after next. */ @@ -1061,16 +1062,16 @@ static void srcu_gp_end(struct srcu_struct *ssp, bool is_atomic) mutex_unlock(&sup->srcu_cb_mutex); /* Start a new grace period if needed. */ - raw_spin_lock_irq_rcu_node(sup); + raw_spin_lock_irqsave_rcu_node(sup, flags); gpseq = rcu_seq_current(&sup->srcu_gp_seq); if (!rcu_seq_state(gpseq) && ULONG_CMP_LT(gpseq, sup->srcu_gp_seq_needed)) { WARN_ON_ONCE(ssp->srcu_reader_flavor & SRCU_READ_FLAVOR_ATOMIC); srcu_gp_start(ssp); - raw_spin_unlock_irq_rcu_node(sup); + raw_spin_unlock_irqrestore_rcu_node(sup, flags); srcu_reschedule(ssp, 0); } else { - raw_spin_unlock_irq_rcu_node(sup); + raw_spin_unlock_irqrestore_rcu_node(sup, flags); } /* Transition to big if needed, but never for atomic SRCU. */ @@ -1220,10 +1221,11 @@ static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp, static bool try_check_zero(struct srcu_struct *ssp, int idx, int trycount) { unsigned long curdelay; + unsigned long flags; - raw_spin_lock_irq_rcu_node(ssp->srcu_sup); + raw_spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags); curdelay = !srcu_get_delay(ssp); - raw_spin_unlock_irq_rcu_node(ssp->srcu_sup); + raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); for (;;) { if (srcu_readers_active_idx_check(ssp, idx)) @@ -2023,6 +2025,7 @@ EXPORT_SYMBOL_GPL(srcu_batches_completed); */ static void srcu_advance_state(struct srcu_struct *ssp, bool is_atomic) { + unsigned long flags; int idx; if (!is_atomic) @@ -2040,10 +2043,10 @@ static void srcu_advance_state(struct srcu_struct *ssp, bool is_atomic) */ idx = rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq)); /* ^^^ */ if (idx == SRCU_STATE_IDLE) { - raw_spin_lock_irq_rcu_node(ssp->srcu_sup); + raw_spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags); if (ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed)) { WARN_ON_ONCE(rcu_seq_state(ssp->srcu_sup->srcu_gp_seq)); - raw_spin_unlock_irq_rcu_node(ssp->srcu_sup); + raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); if (!is_atomic) mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); return; @@ -2053,7 +2056,7 @@ static void srcu_advance_state(struct srcu_struct *ssp, bool is_atomic) WARN_ON_ONCE(ssp->srcu_reader_flavor & SRCU_READ_FLAVOR_ATOMIC); srcu_gp_start(ssp); } - raw_spin_unlock_irq_rcu_node(ssp->srcu_sup); + raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); if (idx != SRCU_STATE_IDLE) { if (!is_atomic) mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex); @@ -2069,10 +2072,10 @@ static void srcu_advance_state(struct srcu_struct *ssp, bool is_atomic) return; /* readers present, retry later. */ } srcu_flip(ssp); - raw_spin_lock_irq_rcu_node(ssp->srcu_sup); + raw_spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags); rcu_seq_set_state(&ssp->srcu_sup->srcu_gp_seq, SRCU_STATE_SCAN2); ssp->srcu_sup->srcu_n_exp_nodelay = 0; - raw_spin_unlock_irq_rcu_node(ssp->srcu_sup); + raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags); } if (rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)) == SRCU_STATE_SCAN2) { @@ -2119,6 +2122,7 @@ void synchronize_srcu_atomic(struct srcu_struct *ssp) { unsigned long srcu_state; struct srcu_usage *sup = ssp->srcu_sup; + unsigned long flags; unsigned long rdm0, rdm1; unsigned long unlocks0, unlocks1; @@ -2149,9 +2153,9 @@ void synchronize_srcu_atomic(struct srcu_struct *ssp) } // One last check for others doing our work for us under the lock. - raw_spin_lock_irq_rcu_node(sup); + raw_spin_lock_irqsave_rcu_node(sup, flags); if (poll_state_synchronize_srcu(ssp, srcu_state)) { - raw_spin_unlock_irq_rcu_node(sup); + raw_spin_unlock_irqrestore_rcu_node(sup, flags); atomic_set(&sup->srcu_atomic_gp_flag, 0); preempt_enable(); return; @@ -2162,7 +2166,7 @@ void synchronize_srcu_atomic(struct srcu_struct *ssp) smp_store_release(&sup->srcu_gp_seq_needed, srcu_state); // See srcu_funnel_gp_start(). ASSERT_EXCLUSIVE_WRITER(ssp->srcu_sup->srcu_gp_seq); srcu_gp_start(ssp); - raw_spin_unlock_irq_rcu_node(sup); + raw_spin_unlock_irqrestore_rcu_node(sup, flags); // // Fastpath: If there are no readers at all, neither grace-period @@ -2200,9 +2204,9 @@ void synchronize_srcu_atomic(struct srcu_struct *ssp) // poll_state_synchronize_srcu() working, all under ->lock // and ->srcu_atomic_gp_flag, which excludes concurrent // sequence-number updates. - raw_spin_lock_irq_rcu_node(sup); + raw_spin_lock_irqsave_rcu_node(sup, flags); rcu_seq_end(&sup->srcu_gp_seq); - raw_spin_unlock_irq_rcu_node(sup); + raw_spin_unlock_irqrestore_rcu_node(sup, flags); WARN_ON_ONCE(!poll_state_synchronize_srcu(ssp, srcu_state)); atomic_set_release(&sup->srcu_atomic_gp_flag, 0); preempt_enable(); From 5b748d8aa6833f2dec3f545311be69a2064bc6d1 Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Thu, 20 Aug 2026 17:42:28 +0800 Subject: [PATCH 39/46] rcutorture: Fix divide-by-zero with fwd_progress_div=1 When fwd_progress_div=1, the forward-progress test computes: sd4 = (sd + div - 1) / div = sd dur = sd4 + torture_random(&trs) % (sd - sd4) = sd4 + % 0 The modulo operation with a zero divisor triggers an integer division by zero (undefined behavior at the C level, #DE trap on x86), causing a kernel Oops and panic. On x86_64, this manifests as: rcu_torture_fwd_prog_nr: Starting forward-progress test 0 Oops: divide error: 0000 [#1] SMP PTI RIP: 0010:rcu_torture_fwd_prog+0x90b/0x1160 R12: 0000000000000000 The existing guard only handles non-positive values. However, fwd_progress_div=1 also makes the random range empty because sd4 == sd. Change the guard to reject values below 2. The forward-progress test only reaches this calculation when stall_dur() is positive, so sd = stall_dur() + 1 >= 2. For fwd_progress_div >= 2, sd4 < sd, ensuring that sd - sd4 is at least 1. Keep the existing fallback to the default value of 4 for invalid values. Verified with QEMU/KVM: a 138-second run with fwd_progress_div=1 completed 81 forward-progress test cycles without a crash. Fixes: 1b27291b1ea4f ("rcutorture: Add forward-progress tests for RCU grace periods") Signed-off-by: Kunwu Chan Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 182d47975efd..79807475b672 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -4024,7 +4024,7 @@ static int __init rcu_torture_fwd_prog_init(void) } if (fwd_progress_holdoff <= 0) fwd_progress_holdoff = 1; - if (fwd_progress_div <= 0) + if (fwd_progress_div < 2) fwd_progress_div = 4; rfp = kzalloc_objs(*rfp, fwd_progress); fwd_prog_tasks = kzalloc_objs(*fwd_prog_tasks, fwd_progress); From 704e59c721d9156201705860c5d0a5e85415f8cc Mon Sep 17 00:00:00 2001 From: Zqiang Date: Mon, 24 Aug 2026 17:24:27 +0800 Subject: [PATCH 40/46] rcutorture: Synchronously wait for all rcu_torture_irq() callbacks to complete The rcu_torture_reader() drives RCU readers from interrupt context via smp_call_function_single(cpu, rcu_torture_irq, NULL, 0) with wait=0, to runs rcu_torture_irq() on a remote CPU. this is async, nothing waits for the remote handler to run. On shutdown, torture_stop_kthread() only waits for each reader kthread to return, and the reader's timer_delete_sync() only drains its timer. Neither waits for a rcu_torture_irq() which still pending or executing on a remote CPU, so it can run after all readers have exited and rcu_torture_cleanup() has already advanced. 1. rcu_torture_irq() may issue cur_ops->call(rhp, rcu_torture_timer_cb) after cur_ops->cb_barrier() has been waiting for all outstanding callbacks complete. once the module is unloaded, fires into freed module text, a use-after-free happen. 2. rcu_torture_irq() may still be inside rcu_torture_one_read(), holding a read-side critical section, when cur_ops->cleanup() tears the flavor down (e.g. cleanup_srcu_struct()), triggering an active-reader warning or use-after-free of the torn-down structure. This commit therefore issue a kick_all_cpus_sync() after all readers kthread have returned and before cur_ops->cb_barrier(), synchronous IPI round trip to every CPU guarantees that every rcu_torture_irq() which previously issued by any reader has completed. Signed-off-by: Zqiang Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 79807475b672..51ed35f5aab1 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -4491,6 +4491,8 @@ rcu_torture_cleanup(void) for (i = 0; i < nrealreaders; i++) torture_stop_kthread(rcu_torture_reader, reader_tasks[i]); + if (irqreader && cur_ops->irq_capable) + kick_all_cpus_sync(); kfree(reader_tasks); reader_tasks = NULL; } From 88dcdd9e6837b0e24eab4d8287f0401f23009181 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 27 Aug 2026 11:25:51 -0700 Subject: [PATCH 41/46] torture: Allow specifying alternative ssh command to kvm-remote.sh Some environments require use of alternative commands to access the test hosts. This commit therefore adds a KVM_REMOTE_SSH environment variable for this purpose. If this variable is unset, ssh is used. Any alternative ssh command must support the usual ssh arguments, including the command to be executed remotely. In some cases, you may need a wrapper script to make the alternative ssh-like command look enough like ssh to satisfy kvm-remote.sh. Signed-off-by: Paul E. McKenney --- .../selftests/rcutorture/bin/kvm-remote.sh | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/tools/testing/selftests/rcutorture/bin/kvm-remote.sh b/tools/testing/selftests/rcutorture/bin/kvm-remote.sh index 48a8052d5dae..a8397f86e49d 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm-remote.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm-remote.sh @@ -6,6 +6,11 @@ # Usage: kvm-remote.sh "systems" [ ] # kvm-remote.sh "systems" /path/to/old/run [ ] # +# The caller may set the KVM_REMOTE_SSH environment in order to specify +# an alternative ssh command, which is necessary in some environments +# for authentication purposes. This alternative ssn command must support +# ssh's usual arguments. +# # Copyright (C) 2021 Facebook, Inc. # # Authors: Paul E. McKenney @@ -13,6 +18,11 @@ scriptname=$0 args="$*" +if test -z "${KVM_REMOTE_SSH}" +then + KVM_REMOTE_SSH=ssh; export KVM_REMOTE_SSH +fi + if ! test -d tools/testing/selftests/rcutorture/bin then echo $scriptname must be run from top-level directory of kernel source tree. @@ -137,7 +147,7 @@ chmod +x $T/bin/kvm-remote-*.sh # Check first to avoid the need for cleanup for system-name typos for i in $systems do - ssh -o BatchMode=yes $i getconf _NPROCESSORS_ONLN > $T/ssh.stdout 2> $T/ssh.stderr + ${KVM_REMOTE_SSH} -o BatchMode=yes $i getconf _NPROCESSORS_ONLN > $T/ssh.stdout 2> $T/ssh.stderr ret=$? if test "$ret" -ne 0 then @@ -158,14 +168,14 @@ echo Build-products tarball: `du -h $T/binres.tgz` | tee -a "$oldrun/remote-log" for i in $systems do echo Downloading tarball to $i `date` | tee -a "$oldrun/remote-log" - cat $T/binres.tgz | ssh -o BatchMode=yes $i "cd /tmp; tar -xzf -" + cat $T/binres.tgz | ${KVM_REMOTE_SSH} -o BatchMode=yes $i "cd /tmp; tar -xzf -" ret=$? tries=0 while test "$ret" -ne 0 do echo Unable to download $T/binres.tgz to system $i, waiting and then retrying. $tries prior retries. | tee -a "$oldrun/remote-log" sleep 60 - cat $T/binres.tgz | ssh -o BatchMode=yes $i "cd /tmp; tar -xzf -" + cat $T/binres.tgz | ${KVM_REMOTE_SSH} -o BatchMode=yes $i "cd /tmp; tar -xzf -" ret=$? if test "$ret" -ne 0 then @@ -191,7 +201,7 @@ checkremotefile () { while : do - ssh -o BatchMode=yes $1 "test -f \"$2\"" + ${KVM_REMOTE_SSH} -o BatchMode=yes $1 "test -f \"$2\"" ret=$? if test "$ret" -eq 255 then @@ -239,7 +249,7 @@ startbatches () { then continue # System still running last test, skip. fi - ssh -o BatchMode=yes "$i" "cd \"$resdir/$ds\"; touch remote.run; PATH=\"$T/bin:$PATH\" nohup kvm-remote-$curbatch.sh > kvm-remote-$curbatch.sh.out 2>&1 &" 1>&2 + ${KVM_REMOTE_SSH} -o BatchMode=yes "$i" "cd \"$resdir/$ds\"; touch remote.run; PATH=\"$T/bin:$PATH\" nohup kvm-remote-$curbatch.sh > kvm-remote-$curbatch.sh.out 2>&1 &" 1>&2 ret=$? if test "$ret" -ne 0 then @@ -281,7 +291,7 @@ do if test "$ret" -eq 1 then echo " ---" Collecting results from $i `date` | tee -a "$oldrun/remote-log" - ( cd "$oldrun"; ssh -o BatchMode=yes $i "cd $rundir; tar -czf - kvm-remote-*.sh.out */console.log */kvm-test-1-run*.sh.out */qemu[_-]pid */qemu-retval */qemu-affinity; rm -rf $T > /dev/null 2>&1" | tar -xzf - ) + ( cd "$oldrun"; ${KVM_REMOTE_SSH} -o BatchMode=yes $i "cd $rundir; tar -czf - kvm-remote-*.sh.out */console.log */kvm-test-1-run*.sh.out */qemu[_-]pid */qemu-retval */qemu-affinity; rm -rf $T > /dev/null 2>&1" | tar -xzf - ) break; fi if test "$ret" -eq 255 From c23eb6c48cf3b8b7392214d0406b0f997025c9db Mon Sep 17 00:00:00 2001 From: Zhouyi Zhou Date: Sat, 19 Sep 2026 01:06:06 +0000 Subject: [PATCH 42/46] rcutorture: Fix kvm-again.sh re-run failure on ppc64 Discovered in the Open Source Lab of Oregon State University when running torture.sh on a ppc64le VM. Two bugs were exposed: 1. The kvm-transform.sh call hard-coded "bzImage" as the kernel image name. On ppc64, the boot image is vmlinux, not bzImage, so the re-run failed to find the image. Fix this by extracting the QEMU binary from the qemu-cmd file and passing it to identify_boot_image() to obtain the correct architecture-specific image name, the same way kvm.sh already does. 2. The rm -f invocation on re-run listed vmlinux among the files to delete. On ppc64 vmlinux is the boot image, so deleting it broke the re-run on that architecture. Drop vmlinux from the unconditional list, and instead conditionally delete it only when the boot image basename is not vmlinux. In addition, identify qemu_binary before the copy step so that on non-PowerPC systems (where vmlinux is not the boot image) the vmlinux file can be removed immediately after the run directory is copied, saving storage before any tests run. This also eliminates the per-iteration re-computation of qemu_binary and boot_image inside the qemu-cmd transform loop. Tested on a local x86_64 machine and on a PPC VM of the Open Source Lab of Oregon State University. Signed-off-by: Zhouyi Zhou Reviewed-by: Joel Fernandes Signed-off-by: Paul E. McKenney --- tools/testing/selftests/rcutorture/bin/kvm-again.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/rcutorture/bin/kvm-again.sh b/tools/testing/selftests/rcutorture/bin/kvm-again.sh index b5239b52cb5d..9060b3dc8926 100755 --- a/tools/testing/selftests/rcutorture/bin/kvm-again.sh +++ b/tools/testing/selftests/rcutorture/bin/kvm-again.sh @@ -180,6 +180,9 @@ fi echo ---- Re-run results directory: $rundir +qemu_binary="$(find "$oldrun" -maxdepth 2 -name 'qemu-cmd' -type f 2>/dev/null | head -1 | xargs -r grep -v '^#' 2>/dev/null | awk 'NF { print $1; exit }')" +boot_image="`identify_boot_image "$qemu_binary"`" + if test "$oldrun" != "$rundir" then # Copy old run directory tree over and adjust. @@ -189,7 +192,7 @@ then echo "Cannot copy from $oldrun to $rundir." usage fi - rm -f "$rundir"/*/{console.log,console.log.diags,qemu_pid,qemu-pid,qemu-retval,Warnings,kvm-test-1-run.sh.out,kvm-test-1-run-qemu.sh.out,vmlinux} "$rundir"/log + rm -f "$rundir"/*/{console.log,console.log.diags,qemu_pid,qemu-pid,qemu-retval,Warnings,kvm-test-1-run.sh.out,kvm-test-1-run-qemu.sh.out} "$rundir"/log touch "$rundir/log" echo $scriptname $args | tee -a "$rundir/log" echo $oldrun > "$rundir/re-run" @@ -197,6 +200,10 @@ then then $arg_link "$oldrun/../../bin" "$rundir/../.." fi + if test "`basename "$boot_image"`" != vmlinux + then + rm -f "$rundir"/*/vmlinux + fi else # Check for a run having already happened. find "$rundir" -name console.log -print > $T/oldrun-console.log @@ -217,7 +224,7 @@ do qemu_cmd_dir="`dirname "$i"`" kernel_dir="`echo $qemu_cmd_dir | sed -e 's/\.[0-9]\+$//'`" jitter_dir="`dirname "$kernel_dir"`" - kvm-transform.sh "$kernel_dir/bzImage" "$qemu_cmd_dir/console.log" "$jitter_dir" "$dur" "$bootargs" < $T/qemu-cmd > $i + kvm-transform.sh "$kernel_dir/`basename $boot_image`" "$qemu_cmd_dir/console.log" "$jitter_dir" "$dur" "$bootargs" < $T/qemu-cmd > $i if test -n "$arg_remote" then echo "# TORTURE_KCONFIG_GDB_ARG=''" >> $i From a692cb8e40d9d7475a3f27b714acc7fa46f9d1df Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Mon, 14 Sep 2026 17:34:15 +0800 Subject: [PATCH 43/46] rcutorture: Add atomic SRCU lockdep support Add three testtypes covering atomic SRCU lockdep behavior: testtype 4: atomic SRCU same-type deadlock testtype 5: atomic SRCU + raw spinlock dependency cycle testtype 6: synchronize_srcu_atomic() inside rcu_read_lock() Co-developed-by: Zqiang Signed-off-by: Zqiang Signed-off-by: Kunwu Chan Acked-by: Boqun Feng Reviewed-by: Joel Fernandes Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 84 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 83 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 51ed35f5aab1..35dc61d871b9 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -4662,6 +4662,17 @@ static DECLARE_RWSEM(rwsem7); static DECLARE_RWSEM(rwsem8); static DECLARE_RWSEM(rwsem9); +static DEFINE_RAW_SPINLOCK(spin0); +static DEFINE_RAW_SPINLOCK(spin1); +static DEFINE_RAW_SPINLOCK(spin2); +static DEFINE_RAW_SPINLOCK(spin3); +static DEFINE_RAW_SPINLOCK(spin4); +static DEFINE_RAW_SPINLOCK(spin5); +static DEFINE_RAW_SPINLOCK(spin6); +static DEFINE_RAW_SPINLOCK(spin7); +static DEFINE_RAW_SPINLOCK(spin8); +static DEFINE_RAW_SPINLOCK(spin9); + DEFINE_STATIC_SRCU(srcu0); DEFINE_STATIC_SRCU(srcu1); DEFINE_STATIC_SRCU(srcu2); @@ -4673,6 +4684,17 @@ DEFINE_STATIC_SRCU(srcu7); DEFINE_STATIC_SRCU(srcu8); DEFINE_STATIC_SRCU(srcu9); +DEFINE_STATIC_SRCU_ATOMIC(srcu0_atomic); +DEFINE_STATIC_SRCU_ATOMIC(srcu1_atomic); +DEFINE_STATIC_SRCU_ATOMIC(srcu2_atomic); +DEFINE_STATIC_SRCU_ATOMIC(srcu3_atomic); +DEFINE_STATIC_SRCU_ATOMIC(srcu4_atomic); +DEFINE_STATIC_SRCU_ATOMIC(srcu5_atomic); +DEFINE_STATIC_SRCU_ATOMIC(srcu6_atomic); +DEFINE_STATIC_SRCU_ATOMIC(srcu7_atomic); +DEFINE_STATIC_SRCU_ATOMIC(srcu8_atomic); +DEFINE_STATIC_SRCU_ATOMIC(srcu9_atomic); + static int srcu_lockdep_next(const char *f, const char *fl, const char *fs, const char *fu, int i, int cyclelen, int deadlock) { @@ -4702,6 +4724,12 @@ static void rcu_torture_init_srcu_lockdep(void) &rwsem5, &rwsem6, &rwsem7, &rwsem8, &rwsem9 }; struct srcu_struct *srcus[] = { &srcu0, &srcu1, &srcu2, &srcu3, &srcu4, &srcu5, &srcu6, &srcu7, &srcu8, &srcu9 }; + raw_spinlock_t *spins[] = { &spin0, &spin1, &spin2, &spin3, &spin4, + &spin5, &spin6, &spin7, &spin8, &spin9 }; + struct srcu_struct *srcus_atomic[] = { &srcu0_atomic, &srcu1_atomic, &srcu2_atomic, + &srcu3_atomic, &srcu4_atomic, &srcu5_atomic, + &srcu6_atomic, &srcu7_atomic, &srcu8_atomic, + &srcu9_atomic }; int testtype; if (!test_srcu_lockdep) @@ -4812,11 +4840,65 @@ static void rcu_torture_init_srcu_lockdep(void) } #endif // #ifdef CONFIG_TASKS_TRACE_RCU + if (testtype == 4) { + pr_info("%s: test_srcu_lockdep = %05d: SRCU_ATOMIC %d-way %sdeadlock.\n", + __func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-"); + if (deadlock && cyclelen == 1) + pr_info("%s: Expect hang.\n", __func__); + for (i = 0; i < cyclelen; i++) { + j = srcu_lockdep_next(__func__, "srcu_read_lock_atomic", + "synchronize_srcu_atomic", + "srcu_read_unlock_atomic", i, + cyclelen, deadlock); + idx = srcu_read_lock_atomic(srcus_atomic[i]); + if (j >= 0) + synchronize_srcu_atomic(srcus_atomic[j]); + srcu_read_unlock_atomic(srcus_atomic[i], idx); + } + return; + } + + if (testtype == 5) { + pr_info("%s: test_srcu_lockdep = %05d: SRCU_ATOMIC/raw_spinlock %d-way %sdeadlock.\n", + __func__, test_srcu_lockdep, cyclelen, deadlock ? "" : "non-"); + for (i = 0; i < cyclelen; i++) { + pr_info("%s: srcu_read_lock_atomic(%d), raw_spin_lock(%d), raw_spin_unlock(%d), srcu_read_unlock_atomic(%d)\n", + __func__, i, i, i, i); + idx = srcu_read_lock_atomic(srcus_atomic[i]); + raw_spin_lock(spins[i]); + raw_spin_unlock(spins[i]); + srcu_read_unlock_atomic(srcus_atomic[i], idx); + + j = srcu_lockdep_next(__func__, "raw_spin_lock", + "synchronize_srcu_atomic", + "raw_spin_unlock", i, cyclelen, + deadlock); + raw_spin_lock(spins[i]); + if (j >= 0) + synchronize_srcu_atomic(srcus_atomic[j]); + raw_spin_unlock(spins[i]); + } + return; + } + + if (testtype == 6) { + pr_info("%s: test_srcu_lockdep = %05d: synchronize_srcu_atomic() inside rcu_read_lock() %d-way.\n", + __func__, test_srcu_lockdep, cyclelen); + for (i = 0; i < cyclelen; i++) { + rcu_read_lock(); + synchronize_srcu_atomic(srcus_atomic[i]); + rcu_read_unlock(); + } + return; + } + err_out: pr_info("%s: test_srcu_lockdep = %05d does nothing.\n", __func__, test_srcu_lockdep); pr_info("%s: test_srcu_lockdep = DNNL.\n", __func__); pr_info("%s: D: Deadlock if nonzero.\n", __func__); - pr_info("%s: NN: Test number, 0=SRCU, 1=SRCU/mutex, 2=SRCU/rwsem, 3=SRCU/Tasks Trace RCU.\n", __func__); + pr_info("%s: NN: Test number, 0=SRCU, 1=SRCU/mutex, 2=SRCU/rwsem, 3=SRCU/Tasks Trace RCU, 4=SRCU_ATOMIC, ", + __func__); + pr_cont("5=SRCU_ATOMIC/raw_spinlock, 6=synchronize_srcu_atomic inside rcu_read_lock.\n"); pr_info("%s: L: Cycle length.\n", __func__); if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU)) pr_info("%s: NN=3 disallowed because kernel is built with CONFIG_TASKS_TRACE_RCU=n\n", __func__); From 321846596c63482e53d23789cb203bd6f965439e Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Mon, 14 Sep 2026 17:34:16 +0800 Subject: [PATCH 44/46] selftests/rcutorture: Wire atomic SRCU into srcu_lockdep.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add testtypes 4 (atomic SRCU same-type deadlock) and 5 (atomic SRCU + raw_spinlock dependency cycle) to the deadlock-detection loop. Add a separate loop for testtype 6 (rcu_read_lock() → synchronize_srcu_atomic()), which also verifies via console.log that no lockdep warning is triggered. Co-developed-by: Zqiang Signed-off-by: Zqiang Signed-off-by: Kunwu Chan Acked-by: Boqun Feng Signed-off-by: Paul E. McKenney --- .../selftests/rcutorture/bin/srcu_lockdep.sh | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/rcutorture/bin/srcu_lockdep.sh b/tools/testing/selftests/rcutorture/bin/srcu_lockdep.sh index 4e98c697def4..72791499dd96 100755 --- a/tools/testing/selftests/rcutorture/bin/srcu_lockdep.sh +++ b/tools/testing/selftests/rcutorture/bin/srcu_lockdep.sh @@ -44,7 +44,7 @@ nerrs=0 # Test lockdep's handling of deadlocks. for d in 0 1 do - for t in 0 1 2 + for t in 0 1 2 4 5 do for c in 1 2 3 do @@ -79,6 +79,43 @@ do done done +# Verify that synchronize_srcu_atomic() does not trigger lockdep +# warnings when called inside rcu_read_lock(). +for c in 1 2 3 +do + err= + val=$((6*10+c)) + tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 5s \ + --configs "SRCU-P" \ + --kconfig "CONFIG_FORCE_NEED_SRCU_NMI_SAFE=y" \ + --bootargs "rcutorture.test_srcu_lockdep=$val" \ + --trust-make --datestamp "$ds/$val" > "$T/kvm.sh.out" 2>&1 + ret=$? + mv "$T/kvm.sh.out" "$RCUTORTURE/res/$ds/$val" + if ! grep -q '^CONFIG_PROVE_LOCKING=y' .config + then + echo "rcu_torture_init_srcu_lockdep:Error: CONFIG_PROVE_LOCKING" \ + "disabled in rcutorture SRCU-P scenario" + err=1 + fi + if test "$ret" -ne 0 + then + err=1 + echo -n Unexpected failure for > "$RCUTORTURE/res/$ds/$val/kvm.sh.err" + elif grep -qE "WARNING: possible (recursive locking|circular locking dependency)" \ + "$RCUTORTURE/res/$ds/$val/SRCU-P/console.log" + then + err=1 + echo -n Unexpected lockdep warning for > "$RCUTORTURE/res/$ds/$val/kvm.sh.err" + fi + if test -n "$err" + then + grep "rcu_torture_init_srcu_lockdep: test_srcu_lockdep = " "$RCUTORTURE/res/$ds/$val/SRCU-P/console.log" | sed -e 's/^.*rcu_torture_init_srcu_lockdep://' >> "$RCUTORTURE/res/$ds/$val/kvm.sh.err" + cat "$RCUTORTURE/res/$ds/$val/kvm.sh.err" + nerrs=$((nerrs+1)) + fi +done + # Test lockdep-enabled testing of mixed SRCU readers. for val in 0x1 0xf do From 7872141f2e5ec3bbcd7097e8084a5a0eff019ba1 Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Mon, 14 Sep 2026 17:34:18 +0800 Subject: [PATCH 45/46] selftests/rcutorture: Fix double nerrs count in srcu_lockdep.sh When CONFIG_PROVE_LOCKING is missing, the deadlock-detection loop and the mixed-SRCU-readers section each increment nerrs once in the configuration check and again in the common error path, counting the same error twice. Remove the stray increments so that each error advances nerrs by one. Signed-off-by: Kunwu Chan Acked-by: Boqun Feng Signed-off-by: Paul E. McKenney --- tools/testing/selftests/rcutorture/bin/srcu_lockdep.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/testing/selftests/rcutorture/bin/srcu_lockdep.sh b/tools/testing/selftests/rcutorture/bin/srcu_lockdep.sh index 72791499dd96..3acaffe52c78 100755 --- a/tools/testing/selftests/rcutorture/bin/srcu_lockdep.sh +++ b/tools/testing/selftests/rcutorture/bin/srcu_lockdep.sh @@ -56,7 +56,6 @@ do if ! grep -q '^CONFIG_PROVE_LOCKING=y' .config then echo "rcu_torture_init_srcu_lockdep:Error: CONFIG_PROVE_LOCKING disabled in rcutorture SRCU-P scenario" - nerrs=$((nerrs+1)) err=1 fi if test "$d" -ne 0 && test "$ret" -eq 0 @@ -126,7 +125,6 @@ do if ! grep -q '^CONFIG_PROVE_LOCKING=y' .config then echo "rcu_torture_init_srcu_lockdep:Error: CONFIG_PROVE_LOCKING disabled in rcutorture SRCU-P scenario" - nerrs=$((nerrs+1)) err=1 fi if test "$val" = 0xf && test "$ret" -eq 0 From 945f87baf00af62ce47237b3c766caa0269f0522 Mon Sep 17 00:00:00 2001 From: Kunwu Chan Date: Mon, 14 Sep 2026 17:34:19 +0800 Subject: [PATCH 46/46] rcutorture: Add atomic SRCU cross-CPU IRQ context mismatch test Add testtype 7 to verify that lockdep detects an IRQ-context mismatch when an atomic SRCU read-side critical section is held with IRQs enabled on one CPU and synchronize_srcu_atomic() is called from an IPI handler on another CPU. This covers the cross-CPU case that cannot be detected by checking the current task's held locks. Co-developed-by: Zqiang Signed-off-by: Zqiang Signed-off-by: Kunwu Chan Acked-by: Boqun Feng Signed-off-by: Paul E. McKenney --- kernel/rcu/rcutorture.c | 27 +++++++++++++- .../selftests/rcutorture/bin/srcu_lockdep.sh | 36 +++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c index 35dc61d871b9..968343be4e0d 100644 --- a/kernel/rcu/rcutorture.c +++ b/kernel/rcu/rcutorture.c @@ -4695,6 +4695,8 @@ DEFINE_STATIC_SRCU_ATOMIC(srcu7_atomic); DEFINE_STATIC_SRCU_ATOMIC(srcu8_atomic); DEFINE_STATIC_SRCU_ATOMIC(srcu9_atomic); +DEFINE_STATIC_SRCU_ATOMIC(srcu_irq); + static int srcu_lockdep_next(const char *f, const char *fl, const char *fs, const char *fu, int i, int cyclelen, int deadlock) { @@ -4709,6 +4711,11 @@ static int srcu_lockdep_next(const char *f, const char *fl, const char *fs, cons return j; } +static void srcu_sync_irq(void *unused) +{ + synchronize_srcu_atomic(&srcu_irq); +} + // Test lockdep on SRCU-based deadlock scenarios. static void rcu_torture_init_srcu_lockdep(void) { @@ -4892,13 +4899,31 @@ static void rcu_torture_init_srcu_lockdep(void) return; } + if (testtype == 7) { + int cpu; + + for (i = 0; i < cyclelen; i++) { + idx = srcu_read_lock_atomic(&srcu_irq); + cpu = cpumask_any_but(cpu_online_mask, + smp_processor_id()); + if (cpu < nr_cpu_ids) { + pr_info("%s: CPU%d sending IPI to CPU%d\n", + __func__, smp_processor_id(), cpu); + smp_call_function_single(cpu, srcu_sync_irq, + NULL, 1); + } + srcu_read_unlock_atomic(&srcu_irq, idx); + } + return; + } + err_out: pr_info("%s: test_srcu_lockdep = %05d does nothing.\n", __func__, test_srcu_lockdep); pr_info("%s: test_srcu_lockdep = DNNL.\n", __func__); pr_info("%s: D: Deadlock if nonzero.\n", __func__); pr_info("%s: NN: Test number, 0=SRCU, 1=SRCU/mutex, 2=SRCU/rwsem, 3=SRCU/Tasks Trace RCU, 4=SRCU_ATOMIC, ", __func__); - pr_cont("5=SRCU_ATOMIC/raw_spinlock, 6=synchronize_srcu_atomic inside rcu_read_lock.\n"); + pr_cont("5=SRCU_ATOMIC/raw_spinlock, 6=synchronize_srcu_atomic inside rcu_read_lock, 7=atomic SRCU cross-CPU IRQ context mismatch.\n"); pr_info("%s: L: Cycle length.\n", __func__); if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU)) pr_info("%s: NN=3 disallowed because kernel is built with CONFIG_TASKS_TRACE_RCU=n\n", __func__); diff --git a/tools/testing/selftests/rcutorture/bin/srcu_lockdep.sh b/tools/testing/selftests/rcutorture/bin/srcu_lockdep.sh index 3acaffe52c78..c29a39f7d891 100755 --- a/tools/testing/selftests/rcutorture/bin/srcu_lockdep.sh +++ b/tools/testing/selftests/rcutorture/bin/srcu_lockdep.sh @@ -115,6 +115,42 @@ do fi done +# Verify that synchronize_srcu_atomic() detects IRQ context mismatch +# when SRCU reader previously ran with IRQs enabled. +for c in 1 2 3 +do + err= + val=$((1000+7*10+c)) + tools/testing/selftests/rcutorture/bin/kvm.sh --allcpus --duration 5s \ + --configs "SRCU-P" \ + --kconfig "CONFIG_FORCE_NEED_SRCU_NMI_SAFE=y" \ + --bootargs "rcutorture.test_srcu_lockdep=$val" \ + --trust-make --datestamp "$ds/$val" > "$T/kvm.sh.out" 2>&1 + ret=$? + mv "$T/kvm.sh.out" "$RCUTORTURE/res/$ds/$val" + if ! grep -q '^CONFIG_PROVE_LOCKING=y' .config + then + echo "rcu_torture_init_srcu_lockdep:Error: CONFIG_PROVE_LOCKING" \ + "disabled in rcutorture SRCU-P scenario" + err=1 + fi + if test "$ret" -eq 0 + then + err=1 + echo -n Missing lockdep warning for > "$RCUTORTURE/res/$ds/$val/kvm.sh.err" + elif ! grep -q "inconsistent {HARDIRQ-ON-R}" "$RCUTORTURE/res/$ds/$val/SRCU-P/console.log" + then + err=1 + echo -n Missing lockdep warning for > "$RCUTORTURE/res/$ds/$val/kvm.sh.err" + fi + if test -n "$err" + then + grep "rcu_torture_init_srcu_lockdep: test_srcu_lockdep = " "$RCUTORTURE/res/$ds/$val/SRCU-P/console.log" | sed -e 's/^.*rcu_torture_init_srcu_lockdep://' >> "$RCUTORTURE/res/$ds/$val/kvm.sh.err" + cat "$RCUTORTURE/res/$ds/$val/kvm.sh.err" + nerrs=$((nerrs+1)) + fi +done + # Test lockdep-enabled testing of mixed SRCU readers. for val in 0x1 0xf do