From 874d785e89a6925a08a59c65a93526d7a8c91403 Mon Sep 17 00:00:00 2001 From: Zi Yan Date: Sun, 30 Aug 2026 22:23:23 -0400 Subject: [PATCH 01/10] mm/slab: reject unsupported kmalloc() sizes kmalloc() is used to allocate physically contiguous memory for kernel allocations. For requests larger than KMALLOC_MAX_CACHE_SIZE, kmalloc() uses the page allocator and can only support up to KMALLOC_MAX_SIZE. For request sizes bigger than KMALLOC_MAX_SIZE, the page allocator can emit a WARN because kmalloc() allocates an order greater than MAX_PAGE_ORDER. To make the KMALLOC_MAX_SIZE limit more explicit instead of relying on the page allocator check, add a KMALLOC_MAX_SIZE check inside kmalloc() to emit a WARN and return NULL for requested sizes bigger than KMALLOC_MAX_SIZE. Like the WARN in the page allocator and kvmalloc(), allow __GFP_NOWARN to suppress the WARN. Suggested-by: Vlastimil Babka (SUSE) Signed-off-by: Zi Yan Link: https://patch.msgid.link/20260830-limit_kmalloc_size-v2-1-bf4ca9c63c88@nvidia.com Reviewed-by: Harry Yoo (Meta) Signed-off-by: Vlastimil Babka (SUSE) --- mm/slub.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mm/slub.c b/mm/slub.c index f9b56cb439e7..23e57a8fe55b 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -5342,7 +5342,12 @@ static void *___kmalloc_large_node(size_t size, gfp_t flags, int node) { struct page *page; void *ptr = NULL; - unsigned int order = get_order(size); + unsigned int order; + + if (WARN_ON_ONCE_GFP(size > KMALLOC_MAX_SIZE, flags)) + return NULL; + + order = get_order(size); if (unlikely(flags & GFP_SLAB_BUG_MASK)) flags = kmalloc_fix_flags(flags); From 208854282dab2100a33c5eb8859407b5d36b7723 Mon Sep 17 00:00:00 2001 From: Longlong Xia Date: Wed, 26 Aug 2026 09:55:06 +0800 Subject: [PATCH 02/10] mm/slab_common: reject zero object_size before calculate_alignment calculate_alignment() with SLAB_HWCACHE_ALIGN halves ralign in a while (size <= ralign / 2) loop. When size is 0, ralign eventually reaches 0 and the condition stays true indefinitely, hanging the kernel. kmem_cache_sanity_check() rejected size > KMALLOC_MAX_SIZE but not size == 0. Add !size to the check so a 0-size kmem_cache_create() is caught on CONFIG_DEBUG_VM builds, where it can be diagnosed when it surfaces. Suggested-by: Hao Li Assisted-by: Codex:gpt-5.6-sol Signed-off-by: Longlong Xia Reviewed-by: Hao Li Reviewed-by: Harry Yoo (Meta) Link: https://patch.msgid.link/20260826015506.3178758-1-xialonglong2025@163.com Signed-off-by: Vlastimil Babka (SUSE) --- mm/slab_common.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/slab_common.c b/mm/slab_common.c index b19ba1b31484..2a66725a4d17 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -102,7 +102,7 @@ static bool kmem_cache_is_duplicate_name(const char *name) static int kmem_cache_sanity_check(const char *name, unsigned int size) { - if (!name || in_interrupt() || size > KMALLOC_MAX_SIZE) { + if (!name || in_interrupt() || !size || size > KMALLOC_MAX_SIZE) { pr_err("kmem_cache_create(%s) integrity check failed\n", name); return -EINVAL; } From 0710b29e51efbc7ad2567df429492f86349dd4d8 Mon Sep 17 00:00:00 2001 From: Hao Li Date: Fri, 14 Aug 2026 15:17:02 +0800 Subject: [PATCH 03/10] mm/slub: unify slub stat definitions to simplify metric additions Currently, adding a new slub stat requires modifying three separate locations: stat_item, STAT_ATTR, and slab_attrs. This is inconvenient, especially when temporarily adding metrics for debugging during development. This patch borrows the approach from SCHED_FEAT by introducing an ITEM_EXP X-macro, which is then temporarily defined as different operations at the three locations (stat_item, STAT_ATTR, and slab_attrs) to perform distinct macro expansions. Going forward, adding a new metric only requires adding a single line: ITEM_EXP(XXX, xxx) Signed-off-by: Hao Li Reviewed-by: Harry Yoo (Meta) Link: https://patch.msgid.link/20260814071848.46922-1-hao.li@linux.dev Signed-off-by: Vlastimil Babka (SUSE) --- mm/slub.c | 144 ++++++++++++++++++++++++------------------------------ 1 file changed, 65 insertions(+), 79 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index 23e57a8fe55b..0870388b7177 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -360,33 +360,64 @@ enum add_mode { ADD_TO_TAIL, }; +#define STAT_ITEMS \ + /* Allocation from percpu sheaves */ \ + ITEM_EXP(ALLOC_FASTPATH, alloc_fastpath) \ + /* Allocation from partial or new slab */ \ + ITEM_EXP(ALLOC_SLOWPATH, alloc_slowpath) \ + /* Free to rcu_free sheaf */ \ + ITEM_EXP(FREE_RCU_SHEAF, free_rcu_sheaf) \ + /* Failed to free to a rcu_free sheaf */ \ + ITEM_EXP(FREE_RCU_SHEAF_FAIL, free_rcu_sheaf_fail) \ + /* Free to percpu sheaves */ \ + ITEM_EXP(FREE_FASTPATH, free_fastpath) \ + /* Free to a slab */ \ + ITEM_EXP(FREE_SLOWPATH, free_slowpath) \ + /* Freeing moves slab to partial list */ \ + ITEM_EXP(FREE_ADD_PARTIAL, free_add_partial) \ + /* Freeing removes last object */ \ + ITEM_EXP(FREE_REMOVE_PARTIAL, free_remove_partial) \ + /* New slab acquired from page allocator */ \ + ITEM_EXP(ALLOC_SLAB, alloc_slab) \ + /* Requested node different from cpu sheaf */ \ + ITEM_EXP(ALLOC_NODE_MISMATCH, alloc_node_mismatch) \ + /* Slab freed to the page allocator */ \ + ITEM_EXP(FREE_SLAB, free_slab) \ + /* Number of times fallback was necessary */ \ + ITEM_EXP(ORDER_FALLBACK, order_fallback) \ + /* Failures of slab freelist update */ \ + ITEM_EXP(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail) \ + /* Objects flushed from a sheaf */ \ + ITEM_EXP(SHEAF_FLUSH, sheaf_flush) \ + /* Objects refilled to a sheaf */ \ + ITEM_EXP(SHEAF_REFILL, sheaf_refill) \ + /* Allocation of an empty sheaf including oversized ones */ \ + ITEM_EXP(SHEAF_ALLOC, sheaf_alloc) \ + /* Freeing of an empty sheaf including oversized ones */ \ + ITEM_EXP(SHEAF_FREE, sheaf_free) \ + /* Got full sheaf from barn */ \ + ITEM_EXP(BARN_GET, barn_get) \ + /* Failed to get full sheaf from barn */ \ + ITEM_EXP(BARN_GET_FAIL, barn_get_fail) \ + /* Put full sheaf to barn */ \ + ITEM_EXP(BARN_PUT, barn_put) \ + /* Failed to put full sheaf to barn */ \ + ITEM_EXP(BARN_PUT_FAIL, barn_put_fail) \ + /* Sheaf prefill grabbed the spare sheaf */ \ + ITEM_EXP(SHEAF_PREFILL_FAST, sheaf_prefill_fast) \ + /* Sheaf prefill found no spare sheaf */ \ + ITEM_EXP(SHEAF_PREFILL_SLOW, sheaf_prefill_slow) \ + /* Allocation of oversize sheaf for prefill */ \ + ITEM_EXP(SHEAF_PREFILL_OVERSIZE, sheaf_prefill_oversize) \ + /* Sheaf return reattached spare sheaf */ \ + ITEM_EXP(SHEAF_RETURN_FAST, sheaf_return_fast) \ + /* Sheaf return could not reattach spare */ \ + ITEM_EXP(SHEAF_RETURN_SLOW, sheaf_return_slow) + enum stat_item { - ALLOC_FASTPATH, /* Allocation from percpu sheaves */ - ALLOC_SLOWPATH, /* Allocation from partial or new slab */ - FREE_RCU_SHEAF, /* Free to rcu_free sheaf */ - FREE_RCU_SHEAF_FAIL, /* Failed to free to a rcu_free sheaf */ - FREE_FASTPATH, /* Free to percpu sheaves */ - FREE_SLOWPATH, /* Free to a slab */ - FREE_ADD_PARTIAL, /* Freeing moves slab to partial list */ - FREE_REMOVE_PARTIAL, /* Freeing removes last object */ - ALLOC_SLAB, /* New slab acquired from page allocator */ - ALLOC_NODE_MISMATCH, /* Requested node different from cpu sheaf */ - FREE_SLAB, /* Slab freed to the page allocator */ - ORDER_FALLBACK, /* Number of times fallback was necessary */ - CMPXCHG_DOUBLE_FAIL, /* Failures of slab freelist update */ - SHEAF_FLUSH, /* Objects flushed from a sheaf */ - SHEAF_REFILL, /* Objects refilled to a sheaf */ - SHEAF_ALLOC, /* Allocation of an empty sheaf including oversized ones */ - SHEAF_FREE, /* Freeing of an empty sheaf including oversized ones */ - BARN_GET, /* Got full sheaf from barn */ - BARN_GET_FAIL, /* Failed to get full sheaf from barn */ - BARN_PUT, /* Put full sheaf to barn */ - BARN_PUT_FAIL, /* Failed to put full sheaf to barn */ - SHEAF_PREFILL_FAST, /* Sheaf prefill grabbed the spare sheaf */ - SHEAF_PREFILL_SLOW, /* Sheaf prefill found no spare sheaf */ - SHEAF_PREFILL_OVERSIZE, /* Allocation of oversize sheaf for prefill */ - SHEAF_RETURN_FAST, /* Sheaf return reattached spare sheaf */ - SHEAF_RETURN_SLOW, /* Sheaf return could not reattach spare */ +#define ITEM_EXP(name, unused) name, + STAT_ITEMS +#undef ITEM_EXP NR_SLUB_STAT_ITEMS }; @@ -9512,34 +9543,12 @@ static ssize_t text##_store(struct kmem_cache *s, \ clear_stat(s, si); \ return length; \ } \ -SLAB_ATTR(text); \ +SLAB_ATTR(text); + +#define ITEM_EXP(name, text) STAT_ATTR(name, text) +STAT_ITEMS +#undef ITEM_EXP -STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath); -STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath); -STAT_ATTR(FREE_RCU_SHEAF, free_rcu_sheaf); -STAT_ATTR(FREE_RCU_SHEAF_FAIL, free_rcu_sheaf_fail); -STAT_ATTR(FREE_FASTPATH, free_fastpath); -STAT_ATTR(FREE_SLOWPATH, free_slowpath); -STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial); -STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial); -STAT_ATTR(ALLOC_SLAB, alloc_slab); -STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch); -STAT_ATTR(FREE_SLAB, free_slab); -STAT_ATTR(ORDER_FALLBACK, order_fallback); -STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail); -STAT_ATTR(SHEAF_FLUSH, sheaf_flush); -STAT_ATTR(SHEAF_REFILL, sheaf_refill); -STAT_ATTR(SHEAF_ALLOC, sheaf_alloc); -STAT_ATTR(SHEAF_FREE, sheaf_free); -STAT_ATTR(BARN_GET, barn_get); -STAT_ATTR(BARN_GET_FAIL, barn_get_fail); -STAT_ATTR(BARN_PUT, barn_put); -STAT_ATTR(BARN_PUT_FAIL, barn_put_fail); -STAT_ATTR(SHEAF_PREFILL_FAST, sheaf_prefill_fast); -STAT_ATTR(SHEAF_PREFILL_SLOW, sheaf_prefill_slow); -STAT_ATTR(SHEAF_PREFILL_OVERSIZE, sheaf_prefill_oversize); -STAT_ATTR(SHEAF_RETURN_FAST, sheaf_return_fast); -STAT_ATTR(SHEAF_RETURN_SLOW, sheaf_return_slow); #endif /* CONFIG_SLUB_STATS */ #ifdef CONFIG_KFENCE @@ -9602,32 +9611,9 @@ static const struct attribute *const slab_attrs[] = { &remote_node_defrag_ratio_attr.attr, #endif #ifdef CONFIG_SLUB_STATS - &alloc_fastpath_attr.attr, - &alloc_slowpath_attr.attr, - &free_rcu_sheaf_attr.attr, - &free_rcu_sheaf_fail_attr.attr, - &free_fastpath_attr.attr, - &free_slowpath_attr.attr, - &free_add_partial_attr.attr, - &free_remove_partial_attr.attr, - &alloc_slab_attr.attr, - &alloc_node_mismatch_attr.attr, - &free_slab_attr.attr, - &order_fallback_attr.attr, - &cmpxchg_double_fail_attr.attr, - &sheaf_flush_attr.attr, - &sheaf_refill_attr.attr, - &sheaf_alloc_attr.attr, - &sheaf_free_attr.attr, - &barn_get_attr.attr, - &barn_get_fail_attr.attr, - &barn_put_attr.attr, - &barn_put_fail_attr.attr, - &sheaf_prefill_fast_attr.attr, - &sheaf_prefill_slow_attr.attr, - &sheaf_prefill_oversize_attr.attr, - &sheaf_return_fast_attr.attr, - &sheaf_return_slow_attr.attr, +#define ITEM_EXP(unused, text) &text##_attr.attr, + STAT_ITEMS +#undef ITEM_EXP #endif #ifdef CONFIG_FAILSLAB &failslab_attr.attr, From 6932405b8154dd996b892a8a444a5543bb2bc35c Mon Sep 17 00:00:00 2001 From: Hao Li Date: Mon, 21 Sep 2026 20:43:22 +0800 Subject: [PATCH 04/10] mm/slub: make the case handling in __slab_free() easier to follow There are 8 possible transitions in __slab_free(): a. partial->partial, offlist b. partial->partial, onlist c. partial->empty, offlist d. partial->empty, onlist, exceeding min_partial e. partial->empty, onlist, not exceeding min_partial f. full->empty, exceeding min_partial g. full->empty, not exceeding min_partial h. full->partial (The onlist/offlist distinction does not apply to f, g and h, as a full slab is on no list.) Clarify which case each branch handles, and replace the goto with a return at the end of the skipped block so that every branch explicitly states its coverage. Cases 'a' and 'b' are the only paths that need neither list_lock nor list handling. Give them an early continue: handling them upfront is much clearer than forcing every other case into a nested block. Also, read SL_partial once after the loop right where it is used, rather than re-reading it on every iteration. No functional change. Signed-off-by: Hao Li Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Harry Yoo (Meta) Link: https://patch.msgid.link/20260921125421.142452-1-hao.li@linux.dev Signed-off-by: Harry Yoo (Meta) --- mm/slub.c | 98 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 51 insertions(+), 47 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index 0870388b7177..40d3f7216263 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -5779,76 +5779,80 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab, new.inuse -= cnt; /* - * Might need to be taken off (due to becoming empty) or added - * to (due to not being full anymore) the partial list. - * Unless it's frozen. + * partial->partial: if the slab was on the node partial list, + * it stays there, and if it was off, it stays off, so we need + * no list handling and no list_lock. + * + * Note that "continue;" in a do-while goes on to evaluate the + * condition below, so we do perform the freelist update. */ - if (!new.inuse || was_full) { + if (!was_full && new.inuse) + continue; - n = get_node(s, slab_nid(slab)); - /* - * Speculatively acquire the list_lock. - * If the cmpxchg does not succeed then we may - * drop the list_lock without any processing. - * - * Otherwise the list_lock will synchronize with - * other processors updating the list of slabs. - */ - spin_lock_irqsave(&n->list_lock, flags); - - on_node_partial = slab_test_node_partial(slab); - } + /* + * The slab might need to be taken off (due to becoming empty) + * or added to (due to not being full anymore) the partial + * list. + * + * Speculatively acquire list_lock prior to cmpxchg(), as + * performing cmpxchg() before lock acquisition races with + * concurrent partial list operations. + * + * If the cmpxchg does not succeed then we will retry. + */ + n = get_node(s, slab_nid(slab)); + spin_lock_irqsave(&n->list_lock, flags); } while (!slab_update_freelist(s, slab, &old, &new, "__slab_free")); - if (likely(!n)) { - /* - * We didn't take the list_lock because the slab was already on - * the partial list and will remain there. - */ + /* partial->partial: we didn't take the list_lock. */ + if (likely(!n)) return; - } - /* - * This slab was partially empty but not on the per-node partial list, - * in which case we shouldn't manipulate its list, just return. - */ + on_node_partial = slab_test_node_partial(slab); + if (!was_full && !on_node_partial) { + /* + * partial->empty, offlist: a bulk refill has taken the slab + * off the partial list and will put it back, so its list + * handling is not ours to do. + */ spin_unlock_irqrestore(&n->list_lock, flags); return; } /* - * If slab became empty, should we add/keep it on the partial list or we - * have enough? + * full/partial->empty, exceed: we have enough partial slabs already. */ - if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) - goto slab_empty; + if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) { + /* partial->empty, onlist, exceed */ + if (likely(!was_full)) { + remove_partial(n, slab); + stat(s, FREE_REMOVE_PARTIAL); + } + /* else, full->empty, exceed: it is on no list to remove from */ + + spin_unlock_irqrestore(&n->list_lock, flags); + stat(s, FREE_SLAB); + discard_slab(s, slab); + return; + } /* - * Objects left in the slab. If it was not on the partial list before - * then add it. + * At this point, only three cases remain: + * full->partial + * full->empty, not exceed + * partial->empty, onlist, not exceed */ + + /* full->partial; full->empty, not exceed */ if (unlikely(was_full)) { add_partial(n, slab, ADD_TO_TAIL); stat(s, FREE_ADD_PARTIAL); } - spin_unlock_irqrestore(&n->list_lock, flags); - return; - -slab_empty: - /* - * The slab could have a single object and thus go from full to empty in - * a single free, but more likely it was on the partial list. Remove it. - */ - if (likely(!was_full)) { - remove_partial(n, slab); - stat(s, FREE_REMOVE_PARTIAL); - } + /* else, partial->empty, onlist, not exceed: it stays on partial list */ spin_unlock_irqrestore(&n->list_lock, flags); - stat(s, FREE_SLAB); - discard_slab(s, slab); } /* From 1b87ea06a34b213e45e3e0c4effd5043f15046ed Mon Sep 17 00:00:00 2001 From: Hao Li Date: Mon, 21 Sep 2026 17:41:13 +0800 Subject: [PATCH 05/10] mm/slub: refill prefilled sheaves from the barn Currently, when the prefill API refills a non-full sheaf, it takes the objects from partial slabs and never from the full sheaves in the barn, so once the barn's full list becomes saturated, it stays saturated. For objects freed via kfree_rcu(), every RCU sheaf then has to be flushed to slabs because the barn's full list has no room. To fix this, let the sheaf refill from the barn first, and introduce a partial sheaf in the barn, which holds the leftover objects [1]. Only the prefill path needs the partial sheaf. The generic allocation path (__pcs_replace_empty_main()) exchanges an empty sheaf for a full one from the barn, so nothing is left over. The prefill path refills a sheaf that is not necessarily empty, so taking objects from the barn usually leaves leftovers, and the partial sheaf is where they are kept. refill_sheaf() only takes objects from partial slabs and never involves the partial sheaf. The sheaf is refilled by copying objects from the partial sheaf, and then from a full sheaf taken from the barn if it is still not full. A sheaf that objects were copied from stays in the barn as the partial sheaf if it still holds objects, or goes on the empty list if it is empty. The sheaf being refilled is never replaced. The gain comes from two sides: every full sheaf taken out makes room on the barn's full list for a future RCU sheaf, and refilling from the barn is cheaper than refilling from partial slabs under list_lock. Note that putting the sheaf with the leftover objects on the full list instead would not work: it would occupy room on the full list, so the list would stay saturated and rcu_free_sheaf() would still keep flushing. An earlier version of this patch swapped sheaves instead, to minimize the memcpy overhead. As Harry Yoo pointed out [2], replacing the caller's sheaf loses cache affinity, filling it directly is more straightforward, and testing showed no measurable difference between the two approaches, so the approach he suggested is used. Tested with will-it-scale mmap1 (192 processes, one-minute runs) on the maple_node cache. throughput: 27778727 -> 34500556 (+24.2%) metric baseline patched change ===================================================================== alloc_fastpath 54,124 56,696 +4.75% alloc_slab 7,100,651 159,739 -97.75% barn_get 849 194,460,392 +22904539.81% barn_get_fail 2 217 +10750.00% barn_put 851 182,306,701 +21422544.07% barn_put_fail 260,400,484 145,522,675 -44.12% cmpxchg_double_fail 1,029,204 334,228 -67.53% free_add_partial 326,694,687 181,935,790 -44.31% free_fastpath 13,297 15,611 +17.40% free_rcu_sheaf 8,332,839,104 10,490,535,227 +25.89% free_remove_partial 7,099,788 158,297 -97.77% free_slab 7,099,788 158,297 -97.77% free_slowpath 10,969,156 782,851 -92.86% objects 15,295 14,849 -2.92% objects_partial 15,295 14,849 -2.92% partial 1,604 2,336 +45.64% sheaf_alloc 137,757,895 141,208,076 +2.50% sheaf_flush 8,332,823,922 4,656,725,764 -44.12% sheaf_free 137,757,883 141,208,072 +2.50% sheaf_prefill_fast 3,337,502,163 4,196,506,116 +25.74% sheaf_prefill_slow 685 400 -41.61% sheaf_refill 8,343,794,364 4,657,509,724 -44.18% sheaf_return_fast 3,337,502,411 4,196,506,382 +25.74% sheaf_return_slow 437 134 -69.34% slabs 1,604 2,336 +45.64% total_objects 102,656 149,504 +45.64% Here is what the important metric changes mean. barn_get and barn_put: refills now take full sheaves out of the barn, and RCU sheaves are put into the barn again. Before, the barn's full list was saturated once and hardly ever consumed. barn_put_fail: more than half of the RCU sheaves are now put into the barn instead of being flushed. The rest are still flushed because they arrive while the barn's full list is at its limit. sheaf_refill and free_add_partial: fewer objects are taken from partial slabs to refill sheaves, and fewer slabs are added to the partial list, by the same percentage. alloc_slab and free_slab: the partial list is almost never empty when a refill looks at it, so slabs are almost never allocated and freed again only to serve refills. After the test, the maple_node cache holds 1604 slabs without the patch and 2336 with it. After a manual shrink (echo 1 > /sys/kernel/slab/maple_node/shrink), it holds 485 without the patch and 563 with it. So most of the extra slabs are held only by objects sitting in sheaves and are returned by a shrink, and what remains is a small difference, because objects handed out from the barn come from many more slabs than a refill from partial slabs would use. Link: https://lore.kernel.org/linux-mm/aqPlMzUIw-4g2iOX@fedora/ [1] Link: https://lore.kernel.org/linux-mm/aq0ylDidEHa2kg4X@thinkstation/ [2] Signed-off-by: Hao Li Reviewed-by: Harry Yoo (Meta) Link: https://patch.msgid.link/20260921094521.141665-1-hao.li@linux.dev Reviewed-by: Vlastimil Babka (SUSE) Signed-off-by: Harry Yoo (Meta) --- mm/slub.c | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/mm/slub.c b/mm/slub.c index 40d3f7216263..1559cbace469 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -453,6 +453,7 @@ struct node_barn { spinlock_t lock; struct list_head sheaves_full; struct list_head sheaves_empty; + struct slab_sheaf *sheaf_partial; unsigned int nr_full; unsigned int nr_empty; }; @@ -3329,6 +3330,7 @@ static void barn_init(struct node_barn *barn) spin_lock_init(&barn->lock); INIT_LIST_HEAD(&barn->sheaves_full); INIT_LIST_HEAD(&barn->sheaves_empty); + barn->sheaf_partial = NULL; barn->nr_full = 0; barn->nr_empty = 0; } @@ -3346,6 +3348,10 @@ static void barn_shrink(struct kmem_cache *s, struct node_barn *barn) barn->nr_full = 0; list_splice_init(&barn->sheaves_empty, &empty_list); barn->nr_empty = 0; + if (barn->sheaf_partial) { + list_add(&barn->sheaf_partial->barn_list, &full_list); + barn->sheaf_partial = NULL; + } spin_unlock_irqrestore(&barn->lock, flags); @@ -5104,12 +5110,87 @@ void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t gfpflags, int nod } EXPORT_SYMBOL(kmem_cache_alloc_node_noprof); +/* + * Refill @sheaf from the barn: from its partial sheaf first, then from its full + * sheaves. A sheaf that objects were copied from stays in the barn as the + * partial sheaf if it still holds objects, or goes on the empty list if it is + * empty. + * + * Returns true if the sheaf is now full, at s->sheaf_capacity. + * Returns false if the sheaf is still not full. + */ +static bool refill_sheaf_from_barn(struct kmem_cache *s, + struct slab_sheaf *sheaf) +{ + struct node_barn *barn = get_barn(s); + struct slab_sheaf *src; + unsigned int to_move; + unsigned long flags; + + if (!barn) + return false; + + if (!data_race(barn->nr_full) && !data_race(barn->sheaf_partial)) + return false; + + spin_lock_irqsave(&barn->lock, flags); + + /* + * The partial sheaf can hold fewer objects than @sheaf needs to + * reach capacity, and a sheaf on the full list is not necessarily + * full (see the comment in rcu_free_sheaf()), so keep taking from + * the barn until @sheaf is full or nothing is left. + */ + while (sheaf->size < s->sheaf_capacity) { + src = barn->sheaf_partial; + barn->sheaf_partial = NULL; + if (!src) { + if (!barn->nr_full) + break; + src = list_first_entry(&barn->sheaves_full, + struct slab_sheaf, barn_list); + list_del(&src->barn_list); + barn->nr_full--; + } + + to_move = min(s->sheaf_capacity - sheaf->size, src->size); + src->size -= to_move; + memcpy(&sheaf->objects[sheaf->size], &src->objects[src->size], + to_move * sizeof(void *)); + sheaf->size += to_move; + + if (src->size) { + barn->sheaf_partial = src; + } else { + /* + * No empty-limit check: the sheaf put on the empty list + * was already in the barn, so the barn holds no more + * sheaves than before. barn_replace_empty_sheaf() skips + * the check for the same reason. + */ + list_add(&src->barn_list, &barn->sheaves_empty); + barn->nr_empty++; + } + } + + spin_unlock_irqrestore(&barn->lock, flags); + + if (sheaf->size < s->sheaf_capacity) + return false; + + stat(s, BARN_GET); + return true; +} + static int __prefill_sheaf_pfmemalloc(struct kmem_cache *s, struct slab_sheaf *sheaf, gfp_t gfp) { gfp_t gfp_nomemalloc; int ret; + if (refill_sheaf_from_barn(s, sheaf)) + return 0; + gfp_nomemalloc = gfp | __GFP_NOMEMALLOC; if (gfp_pfmemalloc_allowed(gfp)) gfp_nomemalloc |= __GFP_NOWARN; From d755441b7672c74f29d1241ffdd2b067ecab1ca0 Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Wed, 23 Sep 2026 19:36:51 +0100 Subject: [PATCH 06/10] slab: Remove slab_folio() This is the final step of divorcing struct slab from struct folio. It's still tied to struct page, but there are no more mentions of folios in the slab code. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: Gregory Price (Meta) Reviewed-by: Harry Yoo (Meta) Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Hao Li Link: https://patch.msgid.link/20260923183655.3364885-1-willy@infradead.org Signed-off-by: Harry Yoo (Meta) --- mm/slab.h | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/mm/slab.h b/mm/slab.h index 8fd6835e4235..e94ac63d10dc 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -149,21 +149,6 @@ static_assert(sizeof(struct slab) <= sizeof(struct page)); static_assert(IS_ALIGNED(offsetof(struct slab, freelist), sizeof(struct freelist_counters))); #endif -/** - * slab_folio - The folio allocated for a slab - * @s: The slab. - * - * Slabs are allocated as folios that contain the individual objects and are - * using some fields in the first struct page of the folio - those fields are - * now accessed by struct slab. It is occasionally necessary to convert back to - * a folio in order to communicate with the rest of the mm. Please use this - * helper function instead of casting yourself, as the implementation may change - * in the future. - */ -#define slab_folio(s) (_Generic((s), \ - const struct slab *: (const struct folio *)s, \ - struct slab *: (struct folio *)s)) - /** * page_slab - Converts from struct page to its slab. * @page: A page which may or may not belong to a slab. @@ -184,15 +169,16 @@ static inline struct slab *page_slab(const struct page *page) * slab_page - The first struct page allocated for a slab * @s: The slab. * - * A convenience wrapper for converting slab to the first struct page of the - * underlying folio, to communicate with code not yet converted to folio or - * struct slab. + * A convenience wrapper for returning the first page allocated to this slab. + * This is for the benefit of code not yet converted to struct slab. */ -#define slab_page(s) folio_page(slab_folio(s), 0) +#define slab_page(s) (_Generic((s), \ + const struct slab *: (const struct page *)s, \ + struct slab *: (struct page *)s)) static inline void *slab_address(const struct slab *slab) { - return folio_address(slab_folio(slab)); + return page_address(slab_page(slab)); } static inline int slab_nid(const struct slab *slab) @@ -212,7 +198,7 @@ static inline struct slab *virt_to_slab(const void *addr) static inline int slab_order(const struct slab *slab) { - return folio_order(slab_folio(slab)); + return compound_order(slab_page(slab)); } static inline size_t slab_size(const struct slab *slab) From 655668f0caf32a42fef10aaf31137e52f46a0260 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Tue, 13 Oct 2026 22:23:42 -0700 Subject: [PATCH 07/10] mm/slab: Mark the kmem_buckets_create() context as a Context: section kmem_buckets_create() had the same sentence about calling context as kmem_cache_create(), but lacked the "Context:" prefix, so kernel-doc rendered in the body instead of as a "context" section. Give it the missing prefix. Additionally fix the "a interrupt" typo __kmem_cache_create_args() had. Fixes: b32801d1255be ("mm/slab: Introduce kmem_buckets_create() and family") Reviewed-by: Harry Yoo (Meta) Acked-by: Pedro Falcato Reviewed-by: Hao Li Acked-by: Vlastimil Babka (SUSE) Signed-off-by: Kees Cook Link: https://patch.msgid.link/20261014052344.384723-1-kees@kernel.org Signed-off-by: Vlastimil Babka (SUSE) --- mm/slab_common.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/slab_common.c b/mm/slab_common.c index 2a66725a4d17..980ca4cb6c43 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -311,7 +311,7 @@ __kmem_cache_alias(const char *name, unsigned int size, slab_flags_t flags, * &SLAB_TYPESAFE_BY_RCU - Slab page (not individual objects) freeing delayed * by a grace period - see the full description before using. * - * Context: Cannot be called within a interrupt, but can be interrupted. + * Context: Cannot be called within an interrupt, but can be interrupted. * * Return: a pointer to the cache on success, NULL on failure. */ @@ -422,7 +422,7 @@ static struct kmem_cache *kmem_buckets_cache __ro_after_init; * to/from userspace. * @ctor: A constructor for the objects, run when new allocations are made. * - * Cannot be called within an interrupt, but can be interrupted. + * Context: Cannot be called within an interrupt, but can be interrupted. * * Return: a pointer to the cache on success, NULL on failure. When * CONFIG_SLAB_BUCKETS is not enabled, ZERO_SIZE_PTR is returned, and From c57fbbe42e907fa26c5ffada52d913d4ca5fa66e Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Tue, 13 Oct 2026 22:23:43 -0700 Subject: [PATCH 08/10] ipc, msg: Account msg_msg allocations with GFP_KERNEL_ACCOUNT again Since commit 734bbc1c97ea7 ("ipc, msg: Use dedicated slab buckets for alloc_msg()"), alloc_msg() allocates with GFP_KERNEL, and a msg_msg is accounted only through SLAB_ACCOUNT on its bucket caches. With CONFIG_SLAB_BUCKETS=n, kmem_buckets_create() creates no caches and kmem_buckets_alloc() is a GFP_KERNEL kmalloc() from the general caches, which do not account it; the same happens when kmem_buckets_create() fails. Either way, the allocation is not charged to the sender's memory cgroup. Allocate with GFP_KERNEL_ACCOUNT again, as before that commit, so the allocation is charged whichever cache serves it. Build tested ARCH=x86_64 defconfig with GCC 16.2.0, with CONFIG_SLAB_BUCKETS as y and n. Fixes: 734bbc1c97ea7 ("ipc, msg: Use dedicated slab buckets for alloc_msg()") Cc: stable@vger.kernel.org Signed-off-by: Kees Cook Link: https://patch.msgid.link/20261014052344.384723-2-kees@kernel.org Signed-off-by: Vlastimil Babka (SUSE) Reviewed-by: Harry Yoo (Meta) Reviewed-by: Pedro Falcato Signed-off-by: Harry Yoo (Meta) --- ipc/msgutil.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ipc/msgutil.c b/ipc/msgutil.c index e28f0cecb2ec..12663a40e9ae 100644 --- a/ipc/msgutil.c +++ b/ipc/msgutil.c @@ -58,7 +58,8 @@ static struct msg_msg *alloc_msg(size_t len) size_t alen; alen = min(len, DATALEN_MSG); - msg = kmem_buckets_alloc(msg_buckets, sizeof(*msg) + alen, GFP_KERNEL); + msg = kmem_buckets_alloc(msg_buckets, sizeof(*msg) + alen, + GFP_KERNEL_ACCOUNT); if (msg == NULL) return NULL; From 088bfdc0d7947d623f4899b9011c866e4445074d Mon Sep 17 00:00:00 2001 From: Imre Kaloz Date: Wed, 14 Oct 2026 17:08:43 +0200 Subject: [PATCH 09/10] mm/slab: sample slab_state once in kmem_cache_destroy() kmem_cache_release() tests slab_state >= FULL after kmem_cache_destroy() has dropped slab_mutex and unlinked the cache, and for a SLAB_TYPESAFE_BY_RCU cache after rcu_barrier(). If slab_late_init() runs before that test it sets slab_state to FULL but skips the unlinked cache, which never got a sysfs entry, so sysfs_slab_release() calls kobject_put() on a kobject that was never initialized: WARNING: lib/kobject.c:734 at kobject_put+0x64/0x2c0, CPU#1: kworker/u8:3/55 kobject: '(null)' ((____ptrval____)): is not initialized, yet kobject_put() is being called. refcount_t: underflow; use-after-free. kobject_put+0x64/0x2c0 sysfs_slab_release+0xc/0x20 kmem_cache_destroy+0x104/0x1e0 bioset_exit+0x13c/0x1e0 disk_release+0x54/0x140 put_disk+0x18/0x40 floppy_async_init+0xbec/0xd10 slab_state >= FULL asks whether the cache has a kobject, which cache creation skips while slab_state <= UP. UP is the state just below FULL, so a false sample cannot skip a kobject_put() that was owed. Seen on sparc64 at boot, where the asynchronous floppy init tears down its bio slab while the late initcalls are running. Sample slab_state once, under slab_mutex which slab_late_init() holds when it sets FULL, and use the result for both the unlink and the release. The history of this problem dates back to v3.2: Commit 69cb8e6b7c29 ("slub: free slabs without holding locks") in v3.2 moved sysfs_slab_remove() outside slub_lock and introduced a race condition between kmem_cache_destroy() and slab_sysfs_init(). Commit 945cf2b6199be ("mm/sl[aou]b: Extract a common function for kmem_cache_destroy") in v3.7 accidentally closed the race while extracting a common implementation for kmem_cache_destroy(). Commit 5413dfba88d6 ("slub: drop mutex before deleting sysfs entry") in v3.8 re-opened the race by dropping slab_mutex around sysfs_slab_remove() in __kmem_cache_shutdown(), to avoid a lockdep splat between slab_mutex and the sysfs lock. Commit 657dc2f97220 ("slab: remove synchronous rcu_barrier() call in memcg cache release path") in v4.11 shifted from synchronous rcu_barrier() to asynchronous RCU work for SLAB_TYPESAFE_BY_RCU caches. That commit, again, accidentally closed the race by moving sysfs_slab_release() to shutdown_cache(), but only for !SLAB_TYPESAFE_BY_RCU caches because destruction of SLAB_TYPESAFE_BY_RCU caches is done in an RCU work. Commit 0495e337b703 ("mm/slab_common: Deleting kobject in kmem_cache_destroy() without holding slab_mutex/cpu_hotplug_lock") in v6.0 re-introduced the race for !SLAB_TYPESAFE_BY_RCU caches. Fixes: 69cb8e6b7c29 ("slub: free slabs without holding locks") Fixes: 5413dfba88d6 ("slub: drop mutex before deleting sysfs entry") Fixes: 0495e337b703 ("mm/slab_common: Deleting kobject in kmem_cache_destroy() without holding slab_mutex/cpu_hotplug_lock") Cc: stable@vger.kernel.org Signed-off-by: Imre Kaloz Reviewed-by: Jose A. Perez de Azpillaga Reviewed-by: Harry Yoo (Meta) Link: https://patch.msgid.link/20261014150844.8536-1-kaloz@kernel.org Reviewed-by: Vlastimil Babka (SUSE) Signed-off-by: Harry Yoo (Meta) --- mm/slab_common.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/mm/slab_common.c b/mm/slab_common.c index 980ca4cb6c43..e66c0778ee98 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -515,10 +515,10 @@ EXPORT_SYMBOL(kmem_buckets_create); * and release of the kobject does not need slab_mutex or cpu_hotplug_lock * protection. So they are now done without holding those locks. */ -static void kmem_cache_release(struct kmem_cache *s) +static void kmem_cache_release(struct kmem_cache *s, bool sysfs_ready) { kfence_shutdown_cache(s); - if (__is_defined(SLAB_SUPPORTS_SYSFS) && slab_state >= FULL) + if (__is_defined(SLAB_SUPPORTS_SYSFS) && sysfs_ready) sysfs_slab_release(s); else slab_kmem_cache_release(s); @@ -533,6 +533,7 @@ void slab_kmem_cache_release(struct kmem_cache *s) void kmem_cache_destroy(struct kmem_cache *s) { + bool sysfs_ready; int err; if (unlikely(!s) || !kasan_check_byte(s)) @@ -580,10 +581,18 @@ void kmem_cache_destroy(struct kmem_cache *s) list_del(&s->list); + /* + * slab_late_init() sets slab_state to FULL under slab_mutex and adds + * sysfs entries only for caches still on the list. Sample the state + * here, so that a cache unlinked before that point is not handed to + * sysfs_slab_release() with an uninitialized kobject. + */ + sysfs_ready = slab_state >= FULL; + mutex_unlock(&slab_mutex); cpus_read_unlock(); - if (slab_state >= FULL) + if (sysfs_ready) sysfs_slab_unlink(s); debugfs_slab_release(s); @@ -593,7 +602,7 @@ void kmem_cache_destroy(struct kmem_cache *s) if (s->flags & SLAB_TYPESAFE_BY_RCU) rcu_barrier(); - kmem_cache_release(s); + kmem_cache_release(s, sysfs_ready); } EXPORT_SYMBOL(kmem_cache_destroy); From d15d7703166daf7bd62c844a3cb1b3a6279adb4c Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Sat, 10 Oct 2026 22:39:06 +0200 Subject: [PATCH 10/10] mm/slub: initialize the free pointer of the bootstrap kmem_cache_node With slub_debug=FZPU, validating kmem_cache_node reports BUG kmem_cache_node (Not tainted): Freepointer corrupt Object 0xffff888000140040 @offset=64 fp=0x5a5a5a5a5a5a5a5a and taints the kernel. Poisoning places the bootstrap object's free pointer outside the object, where check_object() checks it even while allocated. early_kmem_cache_node_alloc() previously took this object from a freelist with an initialized pointer. It now takes it from the slab iterator and builds the freelist only for the remaining objects, leaving its pointer filled with POISON_INUSE. Initialize the bootstrap object's free pointer to NULL. Fixes: dc795d4c0282 ("mm/slub: defer freelist construction until after bulk allocation from a new slab") Assisted-by: LLM Signed-off-by: Karl Mehltretter Reviewed-by: Harry Yoo (Meta) Link: https://patch.msgid.link/20261010203906.93156-1-kmehltretter@gmail.com Reviewed-by: Vlastimil Babka (SUSE) Reviewed-by: Shengming Hu Tested-by: David Rientjes Signed-off-by: Harry Yoo (Meta) --- mm/slub.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mm/slub.c b/mm/slub.c index 1559cbace469..8fbc6bf8ebc5 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -7891,6 +7891,7 @@ static void early_kmem_cache_node_alloc(int node) n = next_slab_obj(kmem_cache_node, &iter); BUG_ON(!n); + set_freepointer(kmem_cache_node, n, NULL); slab->inuse = 1; build_slab_freelist(kmem_cache_node, slab, &iter);