github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-14 16:10:02 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

mm/slab: allow kfree_rcu_sheaf() on PREEMPT_RT

Browse Source 
As suggested by Vlastimil Babka [1], kfree_rcu_sheaf() can be used
on PREEMPT_RT if we always assume spinning is not allowed on PREEMPT_RT.
This is because local_trylock and spinlock_t are safe to use with
trylock and unlock as long as the kernel does not spin and the context
is not NMI and not hardirq.

Now that __kfree_rcu_sheaf() knows how to handle SLAB_FREE_NOLOCK,
relax the limitation and try the sheaves path on PREEMPT_RT as well.

Keep the lockdep map on non RT kernels. However, do not use the lockdep
map on PREEMPT_RT to avoid suppressing valid lockdep warnings.

As pointed by Vlastimil Babka [2], on PREEMPT_RT it is unnecessary to
defer call_rcu() under IRQ-disabled section or raw spinlock. However,
let us avoid adding more complexity as the scenario is not supposed
to be common on PREEMPT_RT, with a hope that call_rcu_nolock() will be
soon supported in RCU.

Link: https://lore.kernel.org/linux-mm/6811cc17-8ee4-48c8-8cbf-6bf4d9f98162@kernel.org [1]
Link: https://lore.kernel.org/linux-mm/40591888-3a87-433e-b3d2-cda1cab543be@kernel.org [2]
Suggested-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>
Reviewed-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>
Signed-off-by: Harry Yoo (Oracle) <harry@kernel.org>
Link: https://patch.msgid.link/20260729-kfree_rcu_nolock-v5-5-a28cdcda9673@kernel.org
Signed-off-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>
This commit is contained in:
Harry Yoo (Oracle) 2026-07-29 17:20:13 +09:00 committed by Vlastimil Babka (SUSE)
parent 69abda97a0
commit 2a8bb29ec9
2 changed files with 20 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
mm/slab_common.c
View File

@ -1612,6 +1612,14 @@ static bool kfree_rcu_sheaf(void *obj)
{ {
struct kmem_cache *s; struct kmem_cache *s;
struct slab *slab; struct slab *slab;
unsigned int free_flags = SLAB_FREE_DEFAULT;
/*
* It is not safe to spin on PREEMPT_RT because the kernel might be
* holding a raw spinlock and slab acquires sleeping locks.
*/
if (IS_ENABLED(CONFIG_PREEMPT_RT))
free_flags = SLAB_FREE_NOLOCK;
if (is_vmalloc_addr(obj)) if (is_vmalloc_addr(obj))
return false; return false;
@ -1622,7 +1630,7 @@ static bool kfree_rcu_sheaf(void *obj)
s = slab->slab_cache; s = slab->slab_cache;
if (likely(!IS_ENABLED(CONFIG_NUMA) || slab_nid(slab) == numa_mem_id())) if (likely(!IS_ENABLED(CONFIG_NUMA) || slab_nid(slab) == numa_mem_id()))
return __kfree_rcu_sheaf(s, obj, SLAB_FREE_DEFAULT); return __kfree_rcu_sheaf(s, obj, free_flags);
return false; return false;
} }
@ -1971,7 +1979,7 @@ void kvfree_call_rcu(struct rcu_head *head, void *ptr)
if (!head) if (!head)
might_sleep(); might_sleep();
if (!IS_ENABLED(CONFIG_PREEMPT_RT) && kfree_rcu_sheaf(ptr)) if (kfree_rcu_sheaf(ptr))
return; return;
// Queue the object but don't yet schedule the batch. // Queue the object but don't yet schedule the batch.

17
mm/slub.c
View File

@ -6060,12 +6060,13 @@ static void rcu_free_sheaf(struct rcu_head *head)
* kvfree_call_rcu() can be called while holding a raw_spinlock_t. Since * kvfree_call_rcu() can be called while holding a raw_spinlock_t. Since
* __kfree_rcu_sheaf() may acquire a spinlock_t (sleeping lock on PREEMPT_RT), * __kfree_rcu_sheaf() may acquire a spinlock_t (sleeping lock on PREEMPT_RT),
* this would violate lock nesting rules. Therefore, kvfree_call_rcu() avoids * this would violate lock nesting rules. Therefore, kvfree_call_rcu() avoids
* this problem by bypassing the sheaves layer entirely on PREEMPT_RT. * this problem by passing SLAB_FREE_NOLOCK on PREEMPT_RT.
* *
* However, lockdep still complains that it is invalid to acquire spinlock_t * However, lockdep still complains that it is invalid to acquire spinlock_t
* while holding raw_spinlock_t, even on !PREEMPT_RT where spinlock_t is a * while holding raw_spinlock_t, even on !PREEMPT_RT where spinlock_t is a
* spinning lock. Tell lockdep that acquiring spinlock_t is valid here * spinning lock. Tell lockdep that acquiring spinlock_t is valid here
* by temporarily raising the wait-type to LD_WAIT_CONFIG. * by temporarily raising the wait-type to LD_WAIT_CONFIG. Skip the lockdep map
* on PREEMPT_RT to avoid suppressing valid lockdep warnings.
*/ */
static DEFINE_WAIT_OVERRIDE_MAP(kfree_rcu_sheaf_map, LD_WAIT_CONFIG); static DEFINE_WAIT_OVERRIDE_MAP(kfree_rcu_sheaf_map, LD_WAIT_CONFIG);
@ -6075,10 +6076,10 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags)
struct slab_sheaf *rcu_sheaf; struct slab_sheaf *rcu_sheaf;
bool allow_spin = free_flags_allow_spinning(free_flags); bool allow_spin = free_flags_allow_spinning(free_flags);
if (WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_RT))) VM_WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_RT) && allow_spin);
return false;
lock_map_acquire_try(&kfree_rcu_sheaf_map); if (!IS_ENABLED(CONFIG_PREEMPT_RT))
lock_map_acquire_try(&kfree_rcu_sheaf_map);
if (!local_trylock(&s->cpu_sheaves->lock)) if (!local_trylock(&s->cpu_sheaves->lock))
goto fail; goto fail;
@ -6177,12 +6178,14 @@ bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj, unsigned int free_flags)
local_unlock(&s->cpu_sheaves->lock); local_unlock(&s->cpu_sheaves->lock);
stat(s, FREE_RCU_SHEAF); stat(s, FREE_RCU_SHEAF);
lock_map_release(&kfree_rcu_sheaf_map); if (!IS_ENABLED(CONFIG_PREEMPT_RT))
lock_map_release(&kfree_rcu_sheaf_map);
return true; return true;
fail: fail:
stat(s, FREE_RCU_SHEAF_FAIL); stat(s, FREE_RCU_SHEAF_FAIL);
lock_map_release(&kfree_rcu_sheaf_map); if (!IS_ENABLED(CONFIG_PREEMPT_RT))
lock_map_release(&kfree_rcu_sheaf_map);
return false; return false;
} }