github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-13 06:23:02 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

mm/slub: deduplicate NUMA policy calculation in allocation paths

Browse Source 
Currently, alloc_from_pcs() and __slab_alloc_node() both calculate the
NUMA policy independently. Since they are called consecutively in paths
like __kmalloc_nolock_noprof() and slab_alloc_node(), this leads to
redundant code snippets.

Introduce a helper function to resolve the NUMA policy once, eliminating
the duplicated code and reducing execution overhead.

Also remove __slab_alloc_node() function because it is almost empty.
The callers of __slab_alloc_node now call ___slab_alloc() directly.

Additional notes:

  Previously, when slab_strict_numa was enabled, alloc_from_pcs() and
  __slab_alloc_node() could each resolve the task mempolicy, so
  MPOL_INTERLEAVE or MPOL_WEIGHTED_INTERLEAVE could advance the
  interleave state twice for a single object allocation attempt.
  And each retry will also advance the interleave state.

  With this change, the strict NUMA node is resolved once and reused by
  both alloc_from_pcs() and ___slab_alloc() in each retry.

  This is a behavior change, but it better matches the intent of
  selecting one policy node for one allocation attempt.

Signed-off-by: Hao Li <hao.li@linux.dev>
Reviewed-by: Harry Yoo (Oracle) <harry@kernel.org>
Link: https://patch.msgid.link/20260624100320.430115-1-hao.li@linux.dev
Signed-off-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>
This commit is contained in:
Hao Li 2026-06-24 18:00:14 +08:00 committed by Vlastimil Babka (SUSE)
parent af9ea231c0
commit 29b3b6cde5
1 changed files with 9 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
mm/slub.c
View File

@ -4526,11 +4526,8 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
return object;
}
static void *__slab_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node,
const struct slab_alloc_context *ac)
static __always_inline int apply_strict_numa_policy(int node)
{
void *object;
#ifdef CONFIG_NUMA
if (static_branch_unlikely(&strict_numa) &&
node == NUMA_NO_NODE) {
@ -4551,10 +4548,7 @@ static void *__slab_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node,
}
}
#endif
object = ___slab_alloc(s, gfpflags, node, ac);
return object;
return node;
}
static __fastpath_inline
@ -4759,28 +4753,6 @@ void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, unsigned int alloc_flags,
bool node_requested;
void *object;
#ifdef CONFIG_NUMA
if (static_branch_unlikely(&strict_numa) &&
node == NUMA_NO_NODE) {
struct mempolicy *mpol = current->mempolicy;
if (mpol) {
/*
* Special BIND rule support. If the local node
* is in permitted set then do not redirect
* to a particular node.
* Otherwise we apply the memory policy to get
* the node we need to allocate on.
*/
if (mpol->mode != MPOL_BIND ||
!node_isset(numa_mem_id(), mpol->nodes))
node = mempolicy_slab_node();
}
}
#endif
node_requested = IS_ENABLED(CONFIG_NUMA) && node != NUMA_NO_NODE;
/*
@ -4930,10 +4902,12 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s,
if (unlikely(object))
goto out;
node = apply_strict_numa_policy(node);
object = alloc_from_pcs(s, gfpflags, ac->alloc_flags, node);
if (unlikely(!object))
object = __slab_alloc_node(s, gfpflags, node, ac);
object = ___slab_alloc(s, gfpflags, node, ac);
maybe_wipe_obj_freeptr(s, object);
@ -5416,6 +5390,8 @@ static void *__kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_f
if (!IS_ENABLED(CONFIG_SMP) && in_nmi())
return NULL;
node = apply_strict_numa_policy(node);
retry:
if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
return NULL;
@ -5440,10 +5416,10 @@ static void *__kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_f
/*
* Do not call slab_alloc_node(), since trylock mode isn't
* compatible with slab_pre_alloc_hook/should_failslab and
* kfence_alloc. Hence call __slab_alloc_node() (at most twice)
* kfence_alloc. Hence call ___slab_alloc() (at most twice)
* and slab_post_alloc_hook() directly.
*/
ret = __slab_alloc_node(s, gfp_flags, node, ac);
ret = ___slab_alloc(s, gfp_flags, node, ac);
/*
* It's possible we failed due to trylock as we preempted someone with