diff --git a/mm/slub.c b/mm/slub.c index 8845e15cb152..8a0c5553876e 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -213,6 +213,12 @@ DEFINE_STATIC_KEY_FALSE(slub_debug_enabled); static DEFINE_STATIC_KEY_FALSE(strict_numa); #endif +/* Structure holding extra parameters for slab allocations */ +struct slab_alloc_context { + unsigned long caller_addr; + size_t orig_size; +}; + /* Structure holding parameters for get_from_partial() call chain */ struct partial_context { gfp_t flags; @@ -3687,7 +3693,8 @@ static inline void init_slab_obj_iter(struct kmem_cache *s, struct slab *slab, * and put the slab to the partial (or full) list. */ static void *alloc_single_from_new_slab(struct kmem_cache *s, struct slab *slab, - int orig_size, bool allow_spin) + const struct slab_alloc_context *ac, + bool allow_spin) { struct kmem_cache_node *n; struct slab_obj_iter iter; @@ -3705,7 +3712,7 @@ static void *alloc_single_from_new_slab(struct kmem_cache *s, struct slab *slab, /* alloc_debug_processing() always expects a valid freepointer */ set_freepointer(s, object, slab->freelist); - if (!alloc_debug_processing(s, slab, object, orig_size)) { + if (!alloc_debug_processing(s, slab, object, ac->orig_size)) { /* * It's not really expected that this would fail on a * freshly allocated slab, but a concurrent memory @@ -4443,7 +4450,7 @@ static unsigned int alloc_from_new_slab(struct kmem_cache *s, struct slab *slab, * slab. */ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, - unsigned long addr, unsigned int orig_size) + const struct slab_alloc_context *ac) { bool allow_spin = gfpflags_allow_spinning(gfpflags); void *object; @@ -4476,7 +4483,7 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, pc.flags = GFP_NOWAIT | __GFP_THISNODE; } - pc.orig_size = orig_size; + pc.orig_size = ac->orig_size; object = get_from_partial(s, node, &pc); if (object) goto success; @@ -4496,7 +4503,7 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, stat(s, ALLOC_SLAB); if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) { - object = alloc_single_from_new_slab(s, slab, orig_size, allow_spin); + object = alloc_single_from_new_slab(s, slab, ac, allow_spin); if (likely(object)) goto success; @@ -4514,13 +4521,13 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, success: if (kmem_cache_debug_flags(s, SLAB_STORE_USER)) - set_track(s, object, TRACK_ALLOC, addr, gfpflags); + set_track(s, object, TRACK_ALLOC, ac->caller_addr, gfpflags); return object; } static void *__slab_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node, - unsigned long addr, size_t orig_size) + const struct slab_alloc_context *ac) { void *object; @@ -4545,7 +4552,7 @@ static void *__slab_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node, } #endif - object = ___slab_alloc(s, gfpflags, node, addr, orig_size); + object = ___slab_alloc(s, gfpflags, node, ac); return object; } @@ -4926,8 +4933,13 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list object = alloc_from_pcs(s, gfpflags, node); - if (unlikely(!object)) - object = __slab_alloc_node(s, gfpflags, node, addr, orig_size); + if (unlikely(!object)) { + const struct slab_alloc_context ac = { + .caller_addr = addr, + .orig_size = orig_size, + }; + object = __slab_alloc_node(s, gfpflags, node, &ac); + } maybe_wipe_obj_freeptr(s, object); @@ -5353,6 +5365,10 @@ void *_kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_flags, in struct kmem_cache *s; bool can_retry = true; void *ret; + const struct slab_alloc_context ac = { + .caller_addr = _RET_IP_, + .orig_size = orig_size, + }; VM_WARN_ON_ONCE(gfp_flags & ~(__GFP_ACCOUNT | __GFP_ZERO | __GFP_NO_OBJ_EXT)); @@ -5398,7 +5414,7 @@ void *_kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_flags, in * kfence_alloc. Hence call __slab_alloc_node() (at most twice) * and slab_post_alloc_hook() directly. */ - ret = __slab_alloc_node(s, alloc_gfp, node, _RET_IP_, orig_size); + ret = __slab_alloc_node(s, alloc_gfp, node, &ac); /* * It's possible we failed due to trylock as we preempted someone with @@ -7240,10 +7256,13 @@ static bool __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, int i; if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) { + const struct slab_alloc_context ac = { + .caller_addr = _RET_IP_, + .orig_size = s->object_size, + }; for (i = 0; i < size; i++) { - p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE, _RET_IP_, - s->object_size); + p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE, &ac); if (unlikely(!p[i])) goto error;