From 6bb0009862c5f0e89a6e4afc09b499a02576c7da Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 28 May 2026 11:34:32 +0200 Subject: [PATCH 1/2] mm/slab: improve kmem_cache_alloc_bulk The kmem_cache_alloc_bulk return value is weird. It returns the number of allocated objects, but that must always be 0 or the requested number based on the implementations and the handling in the callers, but that assumption is not actually documented anywhere, which confuses automated review tools. Fix this by returning a bool if the allocation succeeded and adding a kerneldoc comment explaining the API. [rob.clark@oss.qualcomm.com: fixups in msm_iommu_pagetable_prealloc_allocate() ] Signed-off-by: Christoph Hellwig Reviewed-by: Alexander Lobakin # skbuff Link: https://patch.msgid.link/20260528093437.2519248-2-hch@lst.de Signed-off-by: Vlastimil Babka (SUSE) --- drivers/gpu/drm/msm/msm_iommu.c | 13 +++--- drivers/gpu/drm/panthor/panthor_mmu.c | 13 +++--- include/linux/slab.h | 6 ++- io_uring/io_uring.c | 23 ++++------ lib/test_meminit.c | 23 +++++----- mm/kasan/kasan_test_c.c | 5 +-- mm/kfence/kfence_test.c | 9 ++-- mm/slub.c | 64 ++++++++++++++------------- net/bpf/test_run.c | 7 ++- net/core/skbuff.c | 24 +++++----- tools/include/linux/slab.h | 2 +- tools/testing/shared/linux.c | 19 ++++---- 12 files changed, 104 insertions(+), 104 deletions(-) diff --git a/drivers/gpu/drm/msm/msm_iommu.c b/drivers/gpu/drm/msm/msm_iommu.c index 7d449e5202c5..35dd43e3ef43 100644 --- a/drivers/gpu/drm/msm/msm_iommu.c +++ b/drivers/gpu/drm/msm/msm_iommu.c @@ -330,17 +330,20 @@ static int msm_iommu_pagetable_prealloc_allocate(struct msm_mmu *mmu, struct msm_mmu_prealloc *p) { struct kmem_cache *pt_cache = get_pt_cache(mmu); - int ret; + + if (!p->count) { + p->pages = NULL; + return 0; + } p->pages = kvmalloc_objs(*p->pages, p->count); if (!p->pages) return -ENOMEM; - ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, p->count, p->pages); - if (ret != p->count) { - kfree(p->pages); + if (!kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, p->count, p->pages)) { + kvfree(p->pages); p->pages = NULL; - p->count = ret; + p->count = 0; return -ENOMEM; } diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c index 75d98dad7b1d..b12c641af46c 100644 --- a/drivers/gpu/drm/panthor/panthor_mmu.c +++ b/drivers/gpu/drm/panthor/panthor_mmu.c @@ -1274,13 +1274,13 @@ static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx, goto err_cleanup; } - ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count, - op_ctx->rsvd_page_tables.pages); - op_ctx->rsvd_page_tables.count = ret; - if (ret != pt_count) { + if (!kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count, + op_ctx->rsvd_page_tables.pages)) { + op_ctx->rsvd_page_tables.count = 0; ret = -ENOMEM; goto err_cleanup; } + op_ctx->rsvd_page_tables.count = pt_count; /* Insert BO into the extobj list last, when we know nothing can fail. */ dma_resv_lock(panthor_vm_resv(vm), NULL); @@ -1328,9 +1328,8 @@ static int panthor_vm_prepare_unmap_op_ctx(struct panthor_vm_op_ctx *op_ctx, goto err_cleanup; } - ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count, - op_ctx->rsvd_page_tables.pages); - if (ret != pt_count) { + if (!kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count, + op_ctx->rsvd_page_tables.pages)) { ret = -ENOMEM; goto err_cleanup; } diff --git a/include/linux/slab.h b/include/linux/slab.h index 15a60b501b95..24b244e63ba9 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -815,8 +815,10 @@ kmem_buckets *kmem_buckets_create(const char *name, slab_flags_t flags, */ void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p); -int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, void **p); -#define kmem_cache_alloc_bulk(...) alloc_hooks(kmem_cache_alloc_bulk_noprof(__VA_ARGS__)) +bool kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, + size_t size, void **p); +#define kmem_cache_alloc_bulk(...) \ + alloc_hooks(kmem_cache_alloc_bulk_noprof(__VA_ARGS__)) static __always_inline void kfree_bulk(size_t size, void **p) { diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 4ed998d60c09..b46d038400ff 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -966,29 +966,24 @@ __cold bool __io_alloc_req_refill(struct io_ring_ctx *ctx) { gfp_t gfp = GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO; void *reqs[IO_REQ_ALLOC_BATCH]; - int ret; - - ret = kmem_cache_alloc_bulk(req_cachep, gfp, ARRAY_SIZE(reqs), reqs); + int nr_reqs = ARRAY_SIZE(reqs); /* - * Bulk alloc is all-or-nothing. If we fail to get a batch, - * retry single alloc to be on the safe side. + * Bulk alloc is all-or-nothing. If we fail to get a batch, retry a + * single allocation to be on the safe side. */ - if (unlikely(ret <= 0)) { + if (!kmem_cache_alloc_bulk(req_cachep, gfp, nr_reqs, reqs)) { reqs[0] = kmem_cache_alloc(req_cachep, gfp); if (!reqs[0]) return false; - ret = 1; + nr_reqs = 1; } - percpu_ref_get_many(&ctx->refs, ret); - ctx->nr_req_allocated += ret; + percpu_ref_get_many(&ctx->refs, nr_reqs); + ctx->nr_req_allocated += nr_reqs; - while (ret--) { - struct io_kiocb *req = reqs[ret]; - - io_req_add_to_cache(req, ctx); - } + while (nr_reqs--) + io_req_add_to_cache(reqs[nr_reqs], ctx); return true; } diff --git a/lib/test_meminit.c b/lib/test_meminit.c index 6298f66c964b..e106a0c0601a 100644 --- a/lib/test_meminit.c +++ b/lib/test_meminit.c @@ -229,16 +229,14 @@ static int __init do_kmem_cache_size(size_t size, bool want_ctor, for (iter = 0; iter < 10; iter++) { /* Do a test of bulk allocations */ if (!want_rcu && !want_ctor) { - int ret; - - ret = kmem_cache_alloc_bulk(c, alloc_mask, BULK_SIZE, bulk_array); - if (!ret) { + if (!kmem_cache_alloc_bulk(c, alloc_mask, BULK_SIZE, + bulk_array)) { fail = true; } else { int i; - for (i = 0; i < ret; i++) + for (i = 0; i < BULK_SIZE; i++) fail |= check_buf(bulk_array[i], size, want_ctor, want_rcu, want_zero); - kmem_cache_free_bulk(c, ret, bulk_array); + kmem_cache_free_bulk(c, BULK_SIZE, bulk_array); } } @@ -348,23 +346,24 @@ static int __init do_kmem_cache_size_bulk(int size, int *total_failures) { struct kmem_cache *c; int i, iter, maxiter = 1024; - int num, bytes; + int bytes; bool fail = false; void *objects[10]; c = kmem_cache_create("test_cache", size, size, 0, NULL); for (iter = 0; (iter < maxiter) && !fail; iter++) { - num = kmem_cache_alloc_bulk(c, GFP_KERNEL, ARRAY_SIZE(objects), - objects); - for (i = 0; i < num; i++) { + if (!kmem_cache_alloc_bulk(c, GFP_KERNEL, ARRAY_SIZE(objects), + objects)) + continue; + + for (i = 0; i < ARRAY_SIZE(objects); i++) { bytes = count_nonzero_bytes(objects[i], size); if (bytes) fail = true; fill_with_garbage(objects[i], size); } - if (num) - kmem_cache_free_bulk(c, num, objects); + kmem_cache_free_bulk(c, ARRAY_SIZE(objects), objects); } kmem_cache_destroy(c); *total_failures += fail; diff --git a/mm/kasan/kasan_test_c.c b/mm/kasan/kasan_test_c.c index 32d06cbf6a31..e41ba69592ef 100644 --- a/mm/kasan/kasan_test_c.c +++ b/mm/kasan/kasan_test_c.c @@ -1215,14 +1215,13 @@ static void kmem_cache_bulk(struct kunit *test) struct kmem_cache *cache; size_t size = 200; char *p[10]; - bool ret; int i; cache = kmem_cache_create("test_cache", size, 0, 0, NULL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache); - ret = kmem_cache_alloc_bulk(cache, GFP_KERNEL, ARRAY_SIZE(p), (void **)&p); - if (!ret) { + if (!kmem_cache_alloc_bulk(cache, GFP_KERNEL, ARRAY_SIZE(p), + (void **)&p)) { kunit_err(test, "Allocation failed: %s\n", __func__); kmem_cache_destroy(cache); return; diff --git a/mm/kfence/kfence_test.c b/mm/kfence/kfence_test.c index 5725a367246d..bac6f2aff101 100644 --- a/mm/kfence/kfence_test.c +++ b/mm/kfence/kfence_test.c @@ -761,9 +761,10 @@ static void test_memcache_alloc_bulk(struct kunit *test) timeout = jiffies + msecs_to_jiffies(100 * kfence_sample_interval); do { void *objects[100]; - int i, num = kmem_cache_alloc_bulk(test_cache, GFP_ATOMIC, ARRAY_SIZE(objects), - objects); - if (!num) + int i; + + if (!kmem_cache_alloc_bulk(test_cache, GFP_ATOMIC, + ARRAY_SIZE(objects), objects)) continue; for (i = 0; i < ARRAY_SIZE(objects); i++) { if (is_kfence_address(objects[i])) { @@ -771,7 +772,7 @@ static void test_memcache_alloc_bulk(struct kunit *test) break; } } - kmem_cache_free_bulk(test_cache, num, objects); + kmem_cache_free_bulk(test_cache, ARRAY_SIZE(objects), objects); /* * kmem_cache_alloc_bulk() disables interrupts, and calling it * in a tight loop may not give KFENCE a chance to switch the diff --git a/mm/slub.c b/mm/slub.c index 0baa906f39ab..711df528c9a6 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -4980,8 +4980,8 @@ static int __prefill_sheaf_pfmemalloc(struct kmem_cache *s, return ret; } -static int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, - size_t size, void **p); +static bool __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, + size_t size, void **p); /* * returns a sheaf that has at least the requested size @@ -5153,9 +5153,8 @@ int kmem_cache_refill_sheaf(struct kmem_cache *s, gfp_t gfp, return __prefill_sheaf_pfmemalloc(s, sheaf, gfp); if (!__kmem_cache_alloc_bulk(s, gfp, sheaf->capacity - sheaf->size, - &sheaf->objects[sheaf->size])) { + &sheaf->objects[sheaf->size])) return -ENOMEM; - } sheaf->size = sheaf->capacity; return 0; @@ -7272,9 +7271,8 @@ refill_objects(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int min, return refilled; } -static inline -int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, - void **p) +static bool __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, + size_t size, void **p) { int i; @@ -7295,30 +7293,43 @@ int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, stat_add(s, ALLOC_SLOWPATH, i); } - return i; + return true; error: __kmem_cache_free_bulk(s, i, p); - return 0; - + return false; } -/* - * Note that interrupts must be enabled when calling this function and gfp - * flags must allow spinning. +/** + * kmem_cache_alloc_bulk - Allocate multiple objects + * @s: The cache to allocate from + * @flags: GFP_* flags. See kmalloc(). + * @size: Number of objects to allocate + * @p: Array of allocated objects + * + * Allocate @size objects from @s and places them into @p. @size must be larger + * than 0. + * + * Interrupts must be enabled when calling this function and @flags must allow + * spinning. + * + * Unlike alloc_pages_bulk(), this function does not check for already allocated + * objects in @p, and thus the caller does not need to zero it. + * + * Return: %true if the allocation succeeded, or %false if it failed. */ -int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, - void **p) +bool kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, + size_t size, void **p) { unsigned int i = 0; void *kfence_obj; if (!size) - return 0; + return false; s = slab_pre_alloc_hook(s, flags); if (unlikely(!s)) - return 0; + return false; /* * to make things simpler, only assume at most once kfence allocated @@ -7335,18 +7346,18 @@ int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, } i = alloc_from_pcs_bulk(s, flags, size, p); - if (i < size) { /* * If we ran out of memory, don't bother with freeing back to * the percpu sheaves, we have bigger problems. */ - if (unlikely(__kmem_cache_alloc_bulk(s, flags, size - i, p + i) == 0)) { + if (unlikely(!__kmem_cache_alloc_bulk(s, flags, size - i, + p + i))) { if (i > 0) __kmem_cache_free_bulk(s, i, p); if (kfence_obj) __kfence_free(kfence_obj); - return 0; + return false; } } @@ -7361,16 +7372,9 @@ int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, } out: - /* - * memcg and kmem_cache debug support and memory initialization. - * Done outside of the IRQ disabled fastpath loop. - */ - if (unlikely(!slab_post_alloc_hook(s, NULL, flags, size, p, - slab_want_init_on_alloc(flags, s), s->object_size))) { - return 0; - } - - return size; + /* memcg and kmem_cache debug support and memory initialization */ + return likely(slab_post_alloc_hook(s, NULL, flags, size, p, + slab_want_init_on_alloc(flags, s), s->object_size)); } EXPORT_SYMBOL(kmem_cache_alloc_bulk_noprof); diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index 2bc04feadfab..dbf0d8eae8d8 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -243,12 +243,11 @@ static int xdp_recv_frames(struct xdp_frame **frames, int nframes, struct net_device *dev) { gfp_t gfp = __GFP_ZERO | GFP_ATOMIC; - int i, n; + int i; LIST_HEAD(list); - n = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, gfp, nframes, - (void **)skbs); - if (unlikely(n == 0)) { + if (unlikely(!kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, gfp, + nframes, (void **)skbs))) { for (i = 0; i < nframes; i++) xdp_return_frame(frames[i]); return -ENOMEM; diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 7dad68e3b518..74f32c581403 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -288,11 +288,11 @@ static inline struct sk_buff *napi_skb_cache_get(bool alloc) local_lock_nested_bh(&napi_alloc_cache.bh_lock); if (unlikely(!nc->skb_count)) { - if (alloc) - nc->skb_count = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, - GFP_ATOMIC | __GFP_NOWARN, - NAPI_SKB_CACHE_BULK, - nc->skb_cache); + if (alloc && kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, + GFP_ATOMIC | __GFP_NOWARN, + NAPI_SKB_CACHE_BULK, + nc->skb_cache)) + nc->skb_count = NAPI_SKB_CACHE_BULK; if (unlikely(!nc->skb_count)) { local_unlock_nested_bh(&napi_alloc_cache.bh_lock); return NULL; @@ -353,16 +353,18 @@ u32 napi_skb_cache_get_bulk(void **skbs, u32 n) /* No enough cached skbs. Try refilling the cache first */ bulk = min(NAPI_SKB_CACHE_SIZE - nc->skb_count, NAPI_SKB_CACHE_BULK); - nc->skb_count += kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, - GFP_ATOMIC | __GFP_NOWARN, bulk, - &nc->skb_cache[nc->skb_count]); + if (kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, + GFP_ATOMIC | __GFP_NOWARN, bulk, + &nc->skb_cache[nc->skb_count])) + nc->skb_count += bulk; if (likely(nc->skb_count >= n)) goto get; /* Still not enough. Bulk-allocate the missing part directly, zeroed */ - n -= kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, - GFP_ATOMIC | __GFP_ZERO | __GFP_NOWARN, - n - nc->skb_count, &skbs[nc->skb_count]); + if (kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, + GFP_ATOMIC | __GFP_ZERO | __GFP_NOWARN, + n - nc->skb_count, &skbs[nc->skb_count])) + n = nc->skb_count; if (likely(nc->skb_count >= n)) goto get; diff --git a/tools/include/linux/slab.h b/tools/include/linux/slab.h index 6d8e9413d5a4..2e63c2e726aa 100644 --- a/tools/include/linux/slab.h +++ b/tools/include/linux/slab.h @@ -183,7 +183,7 @@ __kmem_cache_create(const char *name, unsigned int size, unsigned int align, default: __kmem_cache_create)(__name, __object_size, __args, __VA_ARGS__) void kmem_cache_free_bulk(struct kmem_cache *cachep, size_t size, void **list); -int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size, +bool kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size, void **list); struct slab_sheaf * kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size); diff --git a/tools/testing/shared/linux.c b/tools/testing/shared/linux.c index 8c7257155958..e0a0693df08f 100644 --- a/tools/testing/shared/linux.c +++ b/tools/testing/shared/linux.c @@ -154,7 +154,7 @@ void kmem_cache_shrink(struct kmem_cache *cachep) { } -int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size, +bool kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size, void **p) { size_t i; @@ -213,7 +213,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size, pthread_mutex_unlock(&cachep->lock); if (cachep->callback) cachep->exec_callback = true; - return 0; + return false; } for (i = 0; i < size; i++) { @@ -224,7 +224,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size, printf("Allocating %p from slab\n", p[i]); } - return size; + return true; } struct kmem_cache * @@ -271,8 +271,8 @@ kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size) sheaf->cache = s; sheaf->capacity = capacity; - sheaf->size = kmem_cache_alloc_bulk(s, gfp, size, sheaf->objects); - if (!sheaf->size) { + sheaf->size = size; + if (!kmem_cache_alloc_bulk(s, gfp, size, sheaf->objects)) { free(sheaf); return NULL; } @@ -284,7 +284,6 @@ int kmem_cache_refill_sheaf(struct kmem_cache *s, gfp_t gfp, struct slab_sheaf **sheafp, unsigned int size) { struct slab_sheaf *sheaf = *sheafp; - int refill; if (sheaf->size >= size) return 0; @@ -299,12 +298,10 @@ int kmem_cache_refill_sheaf(struct kmem_cache *s, gfp_t gfp, return 0; } - refill = kmem_cache_alloc_bulk(s, gfp, size - sheaf->size, - &sheaf->objects[sheaf->size]); - if (!refill) + if (!kmem_cache_alloc_bulk(s, gfp, size - sheaf->size, + &sheaf->objects[sheaf->size])) return -ENOMEM; - - sheaf->size += refill; + sheaf->size = size; return 0; } From be6e0a533d4f42f87e5fe1190d742478e4184f86 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 2 Jun 2026 18:00:38 +0200 Subject: [PATCH 2/2] mm: simplify the mempool_alloc_bulk API The mempool_alloc_bulk was modelled after the alloc_pages_bulk API, including some misunderstanding of it. Remove checking for NULL slots in the array, as alloc_pages_bulk and kmem_cache_alloc_bulk always fill the array from the beginning and thus we know the offset of the first failing allocation. This removes support for working well with alloc_pages_bulk used to refill page arrays that might have an entry removed from in the middle, but that is only used by sunrpc and hopefully on it's way out. Also remove the allocated parameter as it is redundant because the caller can simply specific and offset into the entries array. Signed-off-by: Christoph Hellwig Link: https://patch.msgid.link/20260602160038.3976341-1-hch@lst.de Signed-off-by: Vlastimil Babka (SUSE) --- block/blk-crypto-fallback.c | 9 +++++---- include/linux/mempool.h | 2 +- mm/mempool.c | 27 ++++++++++----------------- 3 files changed, 16 insertions(+), 22 deletions(-) diff --git a/block/blk-crypto-fallback.c b/block/blk-crypto-fallback.c index 61f595410832..ab6924fba280 100644 --- a/block/blk-crypto-fallback.c +++ b/block/blk-crypto-fallback.c @@ -199,8 +199,8 @@ static struct bio *blk_crypto_alloc_enc_bio(struct bio *bio_src, pages += nr_segs * (PAGE_PTRS_PER_BVEC - 1); /* - * Try a bulk allocation first. This could leave random pages in the - * array unallocated, but we'll fix that up later in mempool_alloc_bulk. + * Try a bulk allocation first. This might not fill all allocated + * pages, but we'll fix that up later in mempool_alloc_bulk. * * Note: alloc_pages_bulk needs the array to be zeroed, as it assumes * any non-zero slot already contains a valid allocation. @@ -208,8 +208,9 @@ static struct bio *blk_crypto_alloc_enc_bio(struct bio *bio_src, memset(pages, 0, sizeof(struct page *) * nr_segs); nr_allocated = alloc_pages_bulk(GFP_KERNEL, nr_segs, pages); if (nr_allocated < nr_segs) - mempool_alloc_bulk(blk_crypto_bounce_page_pool, (void **)pages, - nr_segs, nr_allocated); + mempool_alloc_bulk(blk_crypto_bounce_page_pool, + (void **)pages + nr_allocated, + nr_segs - nr_allocated); memalloc_noio_restore(memflags); *pages_ret = pages; return bio; diff --git a/include/linux/mempool.h b/include/linux/mempool.h index e8e440e04a06..a0fa6d43e0dc 100644 --- a/include/linux/mempool.h +++ b/include/linux/mempool.h @@ -66,7 +66,7 @@ void *mempool_alloc_noprof(struct mempool *pool, gfp_t gfp_mask) __malloc; #define mempool_alloc(...) \ alloc_hooks(mempool_alloc_noprof(__VA_ARGS__)) int mempool_alloc_bulk_noprof(struct mempool *pool, void **elem, - unsigned int count, unsigned int allocated); + unsigned int count); #define mempool_alloc_bulk(...) \ alloc_hooks(mempool_alloc_bulk_noprof(__VA_ARGS__)) diff --git a/mm/mempool.c b/mm/mempool.c index db23e0eef652..473a029fa31f 100644 --- a/mm/mempool.c +++ b/mm/mempool.c @@ -419,12 +419,8 @@ static unsigned int mempool_alloc_from_pool(struct mempool *pool, void **elems, spin_lock_irqsave(&pool->lock, flags); if (unlikely(pool->curr_nr < count - allocated)) goto fail; - for (i = 0; i < count; i++) { - if (!elems[i]) { - elems[i] = remove_element(pool); - allocated++; - } - } + while (allocated < count) + elems[allocated++] = remove_element(pool); spin_unlock_irqrestore(&pool->lock, flags); /* Paired with rmb in mempool_free(), read comment there. */ @@ -479,22 +475,21 @@ static inline gfp_t mempool_adjust_gfp(gfp_t *gfp_mask) * @pool: pointer to the memory pool * @elems: partially or fully populated elements array * @count: number of entries in @elem that need to be allocated - * @allocated: number of entries in @elem already allocated * - * Allocate elements for each slot in @elem that is non-%NULL. This is done by - * first calling into the alloc_fn supplied at pool initialization time, and - * dipping into the reserved pool when alloc_fn fails to allocate an element. + * Allocate @count elements into @elems. This is done by first calling into the + * alloc_fn supplied at pool initialization time, and dipping into the reserved + * pool when alloc_fn fails to allocate an element. * * On return all @count elements in @elems will be populated. * * Return: Always 0. If it wasn't for %$#^$ alloc tags, it would return void. */ int mempool_alloc_bulk_noprof(struct mempool *pool, void **elems, - unsigned int count, unsigned int allocated) + unsigned int count) { gfp_t gfp_mask = GFP_KERNEL; gfp_t gfp_temp = mempool_adjust_gfp(&gfp_mask); - unsigned int i = 0; + unsigned int allocated = 0; VM_WARN_ON_ONCE(count > pool->min_nr); might_alloc(gfp_mask); @@ -514,11 +509,9 @@ int mempool_alloc_bulk_noprof(struct mempool *pool, void **elems, * Try to allocate the elements using the allocation callback first as * that might succeed even when the caller's bulk allocation did not. */ - for (i = 0; i < count; i++) { - if (elems[i]) - continue; - elems[i] = pool->alloc(gfp_temp, pool->pool_data); - if (unlikely(!elems[i])) + while (allocated < count) { + elems[allocated] = pool->alloc(gfp_temp, pool->pool_data); + if (unlikely(!elems[allocated])) goto use_pool; allocated++; }