github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-27 09:36:22 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

mm/slab: introduce alloc_flags and SLAB_ALLOC_NOLOCK

Browse Source 
Similarly to the page allocators, introduce slab-allocator specific
alloc flags that internally control allocation behavior in addition to
gfp_flags, without occupying the limited gfp flags space.

Introduce the first flag SLAB_ALLOC_NOLOCK that behaves similarly to
page allocator's ALLOC_TRYLOCK and will be used to reimplement
kmalloc_nolock()'s "!allow_spin" behavior. That currently relies on
gfpflags_allow_spinning() and thus the lack of both __GFP_RECLAIM flags,
importantly __GFP_KSWAPD_RECLAIM. This can give false-positive results
e.g. in early boot with a restricted gfp_allowed_mask.

Also introduce alloc_flags_allow_spinning() to replace the usage of
gfpflags_allow_spinning().

Start using alloc_flags and the new check first in alloc_from_pcs() and
__pcs_replace_empty_main(). This means some slab allocations that were
falsely treated as kmalloc_nolock() due to their gfp flags will now have
higher chances of success, and this will further increase with followup
changes.

Remove a WARN_ON_ONCE() from refill_objects() as it's now legitimate to
reach it from a slab allocation that's not _nolock() and yet lacks
__GFP_KSWAPD_RECLAIM for other reasons.

Link: https://patch.msgid.link/20260610-slab_alloc_flags-v2-5-7190909db118@kernel.org
Reviewed-by: Harry Yoo (Oracle) <harry@kernel.org>
Reviewed-by: Hao Li <hao.li@linux.dev>
Reviewed-by: Suren Baghdasaryan <surenb@google.com>
Signed-off-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>
This commit is contained in:
Vlastimil Babka (SUSE) 2026-06-10 17:40:07 +02:00
parent 203890bbd2
commit 74d224ac92
2 changed files with 17 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
mm/slab.h
View File

@ -16,6 +16,15 @@
* Internal slab definitions
*/
/* slab's alloc_flags definitions */
#define SLAB_ALLOC_DEFAULT 0x00 /* no flags */
#define SLAB_ALLOC_NOLOCK 0x01 /* a kmalloc_nolock() allocation */
static inline bool alloc_flags_allow_spinning(const unsigned int alloc_flags)
{
return !(alloc_flags & SLAB_ALLOC_NOLOCK);
}
#ifdef CONFIG_64BIT
# ifdef system_has_cmpxchg128
# define system_has_freelist_aba() system_has_cmpxchg128()

17
mm/slub.c
View File

@ -4641,7 +4641,8 @@ bool slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
* unlocked.
*/
static struct slub_percpu_sheaves *
__pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, gfp_t gfp)
__pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs,
gfp_t gfp, unsigned int alloc_flags)
{
struct slab_sheaf *empty = NULL;
struct slab_sheaf *full;
@ -4667,7 +4668,7 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs,
return NULL;
}
allow_spin = gfpflags_allow_spinning(gfp);
allow_spin = alloc_flags_allow_spinning(alloc_flags);
full = barn_replace_empty_sheaf(barn, pcs->main, allow_spin);
@ -4753,7 +4754,7 @@ __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs,
}
static __fastpath_inline
void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, int node)
void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, unsigned int alloc_flags, int node)
{
struct slub_percpu_sheaves *pcs;
bool node_requested;
@ -4798,7 +4799,7 @@ void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, int node)
pcs = this_cpu_ptr(s->cpu_sheaves);
if (unlikely(pcs->main->size == 0)) {
pcs = __pcs_replace_empty_main(s, pcs, gfp);
pcs = __pcs_replace_empty_main(s, pcs, gfp, alloc_flags);
if (unlikely(!pcs))
return NULL;
}
@ -4931,7 +4932,7 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list
if (unlikely(object))
goto out;
object = alloc_from_pcs(s, gfpflags, node);
object = alloc_from_pcs(s, gfpflags, SLAB_ALLOC_DEFAULT, node);
if (unlikely(!object)) {
const struct slab_alloc_context ac = {
@ -5362,6 +5363,7 @@ void *_kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_flags, in
{
gfp_t alloc_gfp = __GFP_NOWARN | __GFP_NOMEMALLOC | gfp_flags;
size_t orig_size = size;
unsigned int alloc_flags = SLAB_ALLOC_NOLOCK;
struct kmem_cache *s;
bool can_retry = true;
void *ret;
@ -5404,7 +5406,7 @@ void *_kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_flags, in
*/
return NULL;
ret = alloc_from_pcs(s, alloc_gfp, node);
ret = alloc_from_pcs(s, alloc_gfp, alloc_flags, node);
if (ret)
goto success;
@ -7218,9 +7220,6 @@ refill_objects(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int min,
unsigned int refilled;
struct slab *slab;
if (WARN_ON_ONCE(!gfpflags_allow_spinning(gfp)))
return 0;
refilled = __refill_objects_node(s, p, gfp, min, max,
get_node(s, local_node),
/* allow_spin = */ true);