github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-28 01:55:51 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

mm/slab: pass alloc_flags through slab_post_alloc_hook() chain

Browse Source 
Convert the whole following call stack to pass either slab_alloc_context
(thus including alloc_flags) or just alloc_flags as necessary:

slab_post_alloc_hook()
  alloc_tagging_slab_alloc_hook()
    __alloc_tagging_slab_alloc_hook()
      prepare_slab_obj_exts_hook()
        alloc_slab_obj_exts()
  memcg_slab_post_alloc_hook()
    __memcg_slab_post_alloc_hook()
      alloc_slab_obj_exts()

Converting all these at once avoids unnecessary churn and is mostly
mechanical.

This ultimately allows to decide if spinning is allowed using
alloc_flags in alloc_slab_obj_exts(), as well as slab_post_alloc_hook().
Aside from alloc_from_pcs_bulk() (to be handled next) there is nothing
else in slab itself relying on gfpflags_allow_spinning() which can
be false even if not called from kmalloc_nolock().

A followup change will also use the alloc_flags availability in the call
stack above to remove the __GFP_NO_OBJ_EXT flag.

For alloc_slab_obj_exts(), also replace the suboptimal "bool new_slab"
parameter with a SLAB_ALLOC_NEW_SLAB flag with identical functionality.

To further reduce the number of parameters of slab_post_alloc_hook(),
also make 'struct list_lru *lru' (which is NULL for most callers) a new
field of slab_alloc_context.

Link: https://patch.msgid.link/20260610-slab_alloc_flags-v2-9-7190909db118@kernel.org
Reviewed-by: Harry Yoo (Oracle) <harry@kernel.org>
Reviewed-by: Suren Baghdasaryan <surenb@google.com>
Reviewed-by: Hao Li <hao.li@linux.dev>
Signed-off-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>
This commit is contained in:
Vlastimil Babka (SUSE) 2026-06-10 17:40:11 +02:00
parent f39062e83c
commit ef7f97aa2c
3 changed files with 62 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
mm/memcontrol.c
View File

@ -3424,7 +3424,8 @@ static inline size_t obj_full_size(struct kmem_cache *s)
}
bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
gfp_t flags, size_t size, void **p)
gfp_t flags, unsigned int slab_alloc_flags,
size_t size, void **p)
{
size_t obj_size = obj_full_size(s);
struct obj_cgroup *objcg;
@ -3472,7 +3473,7 @@ bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
slab = virt_to_slab(p[i]);
if (!slab_obj_exts(slab) &&
alloc_slab_obj_exts(slab, s, flags, false)) {
alloc_slab_obj_exts(slab, s, flags, slab_alloc_flags)) {
continue;
}

6
mm/slab.h
View File

@ -19,6 +19,7 @@
/* slab's alloc_flags definitions */
#define SLAB_ALLOC_DEFAULT 0x00 /* no flags */
#define SLAB_ALLOC_NOLOCK 0x01 /* a kmalloc_nolock() allocation */
#define SLAB_ALLOC_NEW_SLAB 0x02 /* a flag for alloc_slab_obj_exts() */
static inline bool alloc_flags_allow_spinning(const unsigned int alloc_flags)
{
@ -612,7 +613,7 @@ static inline struct slabobj_ext *slab_obj_ext(struct slab *slab,
}
int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
gfp_t gfp, bool new_slab);
gfp_t gfp, unsigned int alloc_flags);
#else /* CONFIG_SLAB_OBJ_EXT */
@ -642,7 +643,8 @@ static inline enum node_stat_item cache_vmstat_idx(struct kmem_cache *s)
#ifdef CONFIG_MEMCG
bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
gfp_t flags, size_t size, void **p);
gfp_t flags, unsigned int slab_alloc_flags,
size_t size, void **p);
void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,
void **p, int objects, unsigned long obj_exts);
#endif

94
mm/slub.c
View File

@ -218,6 +218,7 @@ struct slab_alloc_context {
unsigned long caller_addr;
size_t orig_size;
unsigned int alloc_flags;
struct list_lru *lru;
};
/* Structure holding parameters for get_partial_node_bulk() */
@ -2155,9 +2156,9 @@ static inline size_t obj_exts_alloc_size(struct kmem_cache *s,
}
int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
gfp_t gfp, bool new_slab)
gfp_t gfp, unsigned int alloc_flags)
{
bool allow_spin = gfpflags_allow_spinning(gfp);
const bool allow_spin = alloc_flags_allow_spinning(alloc_flags);
unsigned int objects = objs_per_slab(s, slab);
unsigned long new_exts;
unsigned long old_exts;
@ -2206,7 +2207,7 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
old_exts = READ_ONCE(slab->obj_exts);
handle_failed_objexts_alloc(old_exts, vec, objects);
if (new_slab) {
if (alloc_flags & SLAB_ALLOC_NEW_SLAB) {
/*
* If the slab is brand new and nobody can yet access its
* obj_exts, no synchronization is required and obj_exts can
@ -2331,7 +2332,7 @@ static inline void init_slab_obj_exts(struct slab *slab)
}
static int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
gfp_t gfp, bool new_slab)
gfp_t gfp, unsigned int alloc_flags)
{
return 0;
}
@ -2351,10 +2352,10 @@ static inline void alloc_slab_obj_exts_early(struct kmem_cache *s,
static inline unsigned long
prepare_slab_obj_exts_hook(struct kmem_cache *s, struct slab *slab,
gfp_t flags, void *p)
gfp_t flags, unsigned int alloc_flags, void *p)
{
if (!slab_obj_exts(slab) &&
alloc_slab_obj_exts(slab, s, flags, false)) {
alloc_slab_obj_exts(slab, s, flags, alloc_flags)) {
pr_warn_once("%s, %s: Failed to create slab extension vector!\n",
__func__, s->name);
return 0;
@ -2366,7 +2367,8 @@ prepare_slab_obj_exts_hook(struct kmem_cache *s, struct slab *slab,
/* Should be called only if mem_alloc_profiling_enabled() */
static noinline void
__alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags)
__alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags,
unsigned int alloc_flags)
{
unsigned long obj_exts;
struct slabobj_ext *obj_ext;
@ -2382,7 +2384,7 @@ __alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags)
return;
slab = virt_to_slab(object);
obj_exts = prepare_slab_obj_exts_hook(s, slab, flags, object);
obj_exts = prepare_slab_obj_exts_hook(s, slab, flags, alloc_flags, object);
/*
* Currently obj_exts is used only for allocation profiling.
* If other users appear then mem_alloc_profiling_enabled()
@ -2401,10 +2403,11 @@ __alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags)
}
static inline void
alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags)
alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags,
unsigned int alloc_flags)
{
if (mem_alloc_profiling_enabled())
__alloc_tagging_slab_alloc_hook(s, object, flags);
__alloc_tagging_slab_alloc_hook(s, object, flags, alloc_flags);
}
/* Should be called only if mem_alloc_profiling_enabled() */
@ -2443,7 +2446,8 @@ alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
#else /* CONFIG_MEM_ALLOC_PROFILING */
static inline void
alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags)
alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags,
unsigned int alloc_flags)
{
}
@ -2461,8 +2465,9 @@ alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
static void memcg_alloc_abort_single(struct kmem_cache *s, void *object);
static __fastpath_inline
bool memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
gfp_t flags, size_t size, void **p)
bool memcg_slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags,
size_t size, void **p,
const struct slab_alloc_context *ac)
{
if (likely(!memcg_kmem_online()))
return true;
@ -2470,7 +2475,8 @@ bool memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
if (likely(!(flags & __GFP_ACCOUNT) && !(s->flags & SLAB_ACCOUNT)))
return true;
if (likely(__memcg_slab_post_alloc_hook(s, lru, flags, size, p)))
if (likely(__memcg_slab_post_alloc_hook(s, ac->lru, flags,
ac->alloc_flags, size, p)))
return true;
if (likely(size == 1)) {
@ -2558,14 +2564,15 @@ bool memcg_slab_post_charge(void *p, gfp_t flags)
put_slab_obj_exts(obj_exts);
}
return __memcg_slab_post_alloc_hook(s, NULL, flags, 1, &p);
return __memcg_slab_post_alloc_hook(s, NULL, flags, SLAB_ALLOC_DEFAULT,
1, &p);
}
#else /* CONFIG_MEMCG */
static inline bool memcg_slab_post_alloc_hook(struct kmem_cache *s,
struct list_lru *lru,
gfp_t flags, size_t size,
void **p)
gfp_t flags,
size_t size, void **p,
const struct slab_alloc_context *ac)
{
return true;
}
@ -3352,12 +3359,14 @@ static inline void init_freelist_randomization(void) { }
#endif /* CONFIG_SLAB_FREELIST_RANDOM */
static __always_inline void account_slab(struct slab *slab, int order,
struct kmem_cache *s, gfp_t gfp)
struct kmem_cache *s, gfp_t gfp,
unsigned int alloc_flags)
{
if (memcg_kmem_online() &&
(s->flags & SLAB_ACCOUNT) &&
!slab_obj_exts(slab))
alloc_slab_obj_exts(slab, s, gfp, true);
alloc_slab_obj_exts(slab, s, gfp,
alloc_flags | SLAB_ALLOC_NEW_SLAB);
mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),
PAGE_SIZE << order);
@ -3430,7 +3439,7 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags,
* to prevent the array from being overwritten.
*/
alloc_slab_obj_exts_early(s, slab);
account_slab(slab, oo_order(oo), s, flags);
account_slab(slab, oo_order(oo), s, flags, alloc_flags);
return slab;
}
@ -4564,9 +4573,8 @@ struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags)
}
static __fastpath_inline
bool slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
gfp_t flags, size_t size, void **p,
unsigned int orig_size)
bool slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags, size_t size,
void **p, const struct slab_alloc_context *ac)
{
bool init = slab_want_init_on_alloc(flags, s);
unsigned int zero_size = s->object_size;
@ -4585,7 +4593,7 @@ bool slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
* orig_size if we track it.
*/
if (slub_debug_orig_size(s))
zero_size = orig_size;
zero_size = ac->orig_size;
/*
* ARM64 can set memory tags and zero the memory using a single
@ -4615,14 +4623,14 @@ bool slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
if (init && p[i] && !is_kfence_address(p[i]))
memset(p[i], 0, zero_size);
if (gfpflags_allow_spinning(flags))
if (alloc_flags_allow_spinning(ac->alloc_flags))
kmemleak_alloc_recursive(p[i], s->object_size, 1,
s->flags, init_flags);
kmsan_slab_alloc(s, p[i], init_flags);
alloc_tagging_slab_alloc_hook(s, p[i], flags);
alloc_tagging_slab_alloc_hook(s, p[i], flags, ac->alloc_flags);
}
return memcg_slab_post_alloc_hook(s, lru, flags, size, p);
return memcg_slab_post_alloc_hook(s, flags, size, p, ac);
}
/*
@ -4917,6 +4925,12 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list
{
const unsigned int alloc_flags = SLAB_ALLOC_DEFAULT;
void *object;
const struct slab_alloc_context ac = {
.caller_addr = addr,
.orig_size = orig_size,
.alloc_flags = alloc_flags,
.lru = lru,
};
s = slab_pre_alloc_hook(s, gfpflags);
if (unlikely(!s))
@ -4928,14 +4942,8 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list
object = alloc_from_pcs(s, gfpflags, alloc_flags, node);
if (unlikely(!object)) {
const struct slab_alloc_context ac = {
.caller_addr = addr,
.orig_size = orig_size,
.alloc_flags = alloc_flags,
};
if (unlikely(!object))
object = __slab_alloc_node(s, gfpflags, node, &ac);
}
maybe_wipe_obj_freeptr(s, object);
@ -4944,7 +4952,7 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list
* In case this fails due to memcg_slab_post_alloc_hook(),
* object is set to NULL
*/
slab_post_alloc_hook(s, lru, gfpflags, 1, &object, orig_size);
slab_post_alloc_hook(s, gfpflags, 1, &object, &ac);
return object;
}
@ -5239,6 +5247,10 @@ kmem_cache_alloc_from_sheaf_noprof(struct kmem_cache *s, gfp_t gfp,
struct slab_sheaf *sheaf)
{
void *ret = NULL;
const struct slab_alloc_context ac = {
.orig_size = s->object_size,
.alloc_flags = SLAB_ALLOC_DEFAULT,
};
if (sheaf->size == 0)
goto out;
@ -5249,7 +5261,7 @@ kmem_cache_alloc_from_sheaf_noprof(struct kmem_cache *s, gfp_t gfp,
ret = sheaf->objects[--sheaf->size];
/* add __GFP_NOFAIL to force successful memcg charging */
slab_post_alloc_hook(s, NULL, gfp | __GFP_NOFAIL, 1, &ret, s->object_size);
slab_post_alloc_hook(s, gfp | __GFP_NOFAIL, 1, &ret, &ac);
out:
trace_kmem_cache_alloc(_RET_IP_, ret, s, gfp, NUMA_NO_NODE);
@ -5435,7 +5447,7 @@ void *_kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_flags, in
success:
maybe_wipe_obj_freeptr(s, ret);
slab_post_alloc_hook(s, NULL, alloc_gfp, 1, &ret, orig_size);
slab_post_alloc_hook(s, alloc_gfp, 1, &ret, &ac);
ret = kasan_kmalloc(s, ret, orig_size, alloc_gfp);
return ret;
@ -7301,6 +7313,10 @@ bool kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags,
{
unsigned int i = 0;
void *kfence_obj;
const struct slab_alloc_context ac = {
.orig_size = s->object_size,
.alloc_flags = SLAB_ALLOC_DEFAULT,
};
if (!size)
return false;
@ -7351,7 +7367,7 @@ bool kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags,
out:
/* memcg and kmem_cache debug support and memory initialization */
return likely(slab_post_alloc_hook(s, NULL, flags, size, p, s->object_size));
return likely(slab_post_alloc_hook(s, flags, size, p, &ac));
}
EXPORT_SYMBOL(kmem_cache_alloc_bulk_noprof);