From e1fa26489025d6deac76d1dbfe2e0720a3ad84b1 Mon Sep 17 00:00:00 2001 From: "Vlastimil Babka (SUSE)" Date: Tue, 7 Jul 2026 14:16:00 +0200 Subject: [PATCH] mm/slab: extract __free_to_pcs_batch() from free_to_pcs_bulk() It has been noted that free_to_pcs_bulk() is difficult to follow, with a number of goto labels, and this has contributed to two memory leak bugs in there. Extract part of the code to __free_to_pcs_batch(), which focuses only on freeing free-hook-processed local objects to a percpu sheaf, and returning how many were freed. Zero means a trylock failure or no empty sheaf available, and thus the caller should fallback to __kmem_cache_free_bulk(). Make free_to_pcs_bulk() call this in a while loop, removing all goto labels from the function. __free_to_pcs_batch() retains two rather straightforward ones. Reviewed-by: Shengming Hu Link: https://patch.msgid.link/20260707-slab-simplify-bulk-pcs-v1-1-4850dbe0d904@kernel.org Reviewed-by: Hao Li Reviewed-by: Harry Yoo (Oracle) Signed-off-by: Vlastimil Babka (SUSE) --- mm/slub.c | 116 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 60 insertions(+), 56 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index 7a8f496b296c..22045dc919ef 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -6179,51 +6179,21 @@ static __always_inline bool can_free_to_pcs(struct slab *slab) } /* - * Bulk free objects to the percpu sheaves. - * Unlike free_to_pcs() this includes the calls to all necessary hooks - * and the fallback to freeing to slab pages. + * Try to free as many objects (already processed by free hooks) as possible to + * a single per-cpu sheaf. + * + * Returns how many objects were freed. Zero means failure and the caller should + * fall back to __kmem_cache_free_bulk(). */ -static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p) +static unsigned int __free_to_pcs_batch(struct kmem_cache *s, size_t size, void **p) { struct slub_percpu_sheaves *pcs; struct slab_sheaf *main, *empty; - bool init = slab_want_init_on_free(s); - unsigned int batch, i = 0; struct node_barn *barn; - void *remote_objects[PCS_BATCH_MAX]; - unsigned int remote_nr = 0; + unsigned int batch; - while (i < size) { - struct slab *slab = virt_to_slab(p[i]); - - memcg_slab_free_hook(s, slab, p + i, 1); - alloc_tagging_slab_free_hook(s, slab, p + i, 1); - - if (unlikely(!slab_free_hook(s, p[i], init, false))) { - p[i] = p[--size]; - continue; - } - - if (unlikely(!can_free_to_pcs(slab))) { - remote_objects[remote_nr] = p[i]; - p[i] = p[--size]; - if (++remote_nr >= PCS_BATCH_MAX) { - __kmem_cache_free_bulk(s, remote_nr, &remote_objects[0]); - stat_add(s, FREE_SLOWPATH, remote_nr); - remote_nr = 0; - } - continue; - } - - i++; - } - - if (!size) - goto flush_remote; - -next_batch: if (!local_trylock(&s->cpu_sheaves->lock)) - goto fallback; + return 0; pcs = this_cpu_ptr(s->cpu_sheaves); @@ -6269,29 +6239,63 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p) stat_add(s, FREE_FASTPATH, batch); - if (batch < size) { - p += batch; - size -= batch; - goto next_batch; - } - - if (remote_nr) - goto flush_remote; - - return; + return batch; no_empty: local_unlock(&s->cpu_sheaves->lock); - /* - * if we depleted all empty sheaves in the barn or there are too - * many full sheaves, free the rest to slab pages - */ -fallback: - __kmem_cache_free_bulk(s, size, p); - stat_add(s, FREE_SLOWPATH, size); + return 0; +} + +/* + * Bulk free objects to the percpu sheaves. + * Unlike free_to_pcs() this includes the calls to all necessary hooks + * and the fallback to freeing to slab pages. + */ +static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p) +{ + bool init = slab_want_init_on_free(s); + void *remote_objects[PCS_BATCH_MAX]; + unsigned int remote_nr = 0; + + for (unsigned int i = 0; i < size;) { + struct slab *slab = virt_to_slab(p[i]); + + memcg_slab_free_hook(s, slab, p + i, 1); + alloc_tagging_slab_free_hook(s, slab, p + i, 1); + + if (unlikely(!slab_free_hook(s, p[i], init, false))) { + p[i] = p[--size]; + continue; + } + + if (unlikely(!can_free_to_pcs(slab))) { + remote_objects[remote_nr] = p[i]; + p[i] = p[--size]; + if (++remote_nr >= PCS_BATCH_MAX) { + __kmem_cache_free_bulk(s, remote_nr, &remote_objects[0]); + stat_add(s, FREE_SLOWPATH, remote_nr); + remote_nr = 0; + } + continue; + } + + i++; + } + + while (size) { + unsigned int batch_freed = __free_to_pcs_batch(s, size, p); + + if (!batch_freed) { + __kmem_cache_free_bulk(s, size, p); + stat_add(s, FREE_SLOWPATH, size); + break; + } + + p += batch_freed; + size -= batch_freed; + } -flush_remote: if (remote_nr) { __kmem_cache_free_bulk(s, remote_nr, &remote_objects[0]); stat_add(s, FREE_SLOWPATH, remote_nr);