From 29b3b6cde5f2dee1c51a1c03ba2e0015e205125b Mon Sep 17 00:00:00 2001 From: Hao Li Date: Wed, 24 Jun 2026 18:00:14 +0800 Subject: [PATCH] mm/slub: deduplicate NUMA policy calculation in allocation paths Currently, alloc_from_pcs() and __slab_alloc_node() both calculate the NUMA policy independently. Since they are called consecutively in paths like __kmalloc_nolock_noprof() and slab_alloc_node(), this leads to redundant code snippets. Introduce a helper function to resolve the NUMA policy once, eliminating the duplicated code and reducing execution overhead. Also remove __slab_alloc_node() function because it is almost empty. The callers of __slab_alloc_node now call ___slab_alloc() directly. Additional notes: Previously, when slab_strict_numa was enabled, alloc_from_pcs() and __slab_alloc_node() could each resolve the task mempolicy, so MPOL_INTERLEAVE or MPOL_WEIGHTED_INTERLEAVE could advance the interleave state twice for a single object allocation attempt. And each retry will also advance the interleave state. With this change, the strict NUMA node is resolved once and reused by both alloc_from_pcs() and ___slab_alloc() in each retry. This is a behavior change, but it better matches the intent of selecting one policy node for one allocation attempt. Signed-off-by: Hao Li Reviewed-by: Harry Yoo (Oracle) Link: https://patch.msgid.link/20260624100320.430115-1-hao.li@linux.dev Signed-off-by: Vlastimil Babka (SUSE) --- mm/slub.c | 42 +++++++++--------------------------------- 1 file changed, 9 insertions(+), 33 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index 65febe957886..4d7e3067a5d6 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -4526,11 +4526,8 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, return object; } -static void *__slab_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node, - const struct slab_alloc_context *ac) +static __always_inline int apply_strict_numa_policy(int node) { - void *object; - #ifdef CONFIG_NUMA if (static_branch_unlikely(&strict_numa) && node == NUMA_NO_NODE) { @@ -4551,10 +4548,7 @@ static void *__slab_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node, } } #endif - - object = ___slab_alloc(s, gfpflags, node, ac); - - return object; + return node; } static __fastpath_inline @@ -4759,28 +4753,6 @@ void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, unsigned int alloc_flags, bool node_requested; void *object; -#ifdef CONFIG_NUMA - if (static_branch_unlikely(&strict_numa) && - node == NUMA_NO_NODE) { - - struct mempolicy *mpol = current->mempolicy; - - if (mpol) { - /* - * Special BIND rule support. If the local node - * is in permitted set then do not redirect - * to a particular node. - * Otherwise we apply the memory policy to get - * the node we need to allocate on. - */ - if (mpol->mode != MPOL_BIND || - !node_isset(numa_mem_id(), mpol->nodes)) - - node = mempolicy_slab_node(); - } - } -#endif - node_requested = IS_ENABLED(CONFIG_NUMA) && node != NUMA_NO_NODE; /* @@ -4930,10 +4902,12 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, if (unlikely(object)) goto out; + node = apply_strict_numa_policy(node); + object = alloc_from_pcs(s, gfpflags, ac->alloc_flags, node); if (unlikely(!object)) - object = __slab_alloc_node(s, gfpflags, node, ac); + object = ___slab_alloc(s, gfpflags, node, ac); maybe_wipe_obj_freeptr(s, object); @@ -5416,6 +5390,8 @@ static void *__kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_f if (!IS_ENABLED(CONFIG_SMP) && in_nmi()) return NULL; + node = apply_strict_numa_policy(node); + retry: if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) return NULL; @@ -5440,10 +5416,10 @@ static void *__kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_f /* * Do not call slab_alloc_node(), since trylock mode isn't * compatible with slab_pre_alloc_hook/should_failslab and - * kfence_alloc. Hence call __slab_alloc_node() (at most twice) + * kfence_alloc. Hence call ___slab_alloc() (at most twice) * and slab_post_alloc_hook() directly. */ - ret = __slab_alloc_node(s, gfp_flags, node, ac); + ret = ___slab_alloc(s, gfp_flags, node, ac); /* * It's possible we failed due to trylock as we preempted someone with