github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-28 10:09:10 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge branch 'slab/for-7.2/alloc_bulk' into slab/for-next

Browse Source 
Merge two separately sent but vaguely related patches from Christoph
Hellwig.  One changes the kmem_cache_alloc_bulk() API to return bool,
because it was already actiong as all-or-nothing, and that aspect was
not documented. Existing callers are updated.

The second patch simplifies the mempool_alloc_bulk() API to stop
skipping over non-NULL entries in the array, and removes a related
parameter that said how many are non-NULL.

A similar simplification of alloc_pages_bulk() is being discussed as
well and should follow in near future.
This commit is contained in:
Vlastimil Babka (SUSE) 2026-06-11 12:26:50 +02:00
commit d3c45a0fee
15 changed files with 120 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
block/blk-crypto-fallback.c
View File

@ -199,8 +199,8 @@ static struct bio *blk_crypto_alloc_enc_bio(struct bio *bio_src,
pages += nr_segs * (PAGE_PTRS_PER_BVEC - 1);
/*
* Try a bulk allocation first. This could leave random pages in the
* array unallocated, but we'll fix that up later in mempool_alloc_bulk.
* Try a bulk allocation first. This might not fill all allocated
* pages, but we'll fix that up later in mempool_alloc_bulk.
*
* Note: alloc_pages_bulk needs the array to be zeroed, as it assumes
* any non-zero slot already contains a valid allocation.
@ -208,8 +208,9 @@ static struct bio *blk_crypto_alloc_enc_bio(struct bio *bio_src,
memset(pages, 0, sizeof(struct page *) * nr_segs);
nr_allocated = alloc_pages_bulk(GFP_KERNEL, nr_segs, pages);
if (nr_allocated < nr_segs)
mempool_alloc_bulk(blk_crypto_bounce_page_pool, (void **)pages,
nr_segs, nr_allocated);
mempool_alloc_bulk(blk_crypto_bounce_page_pool,
(void **)pages + nr_allocated,
nr_segs - nr_allocated);
memalloc_noio_restore(memflags);
*pages_ret = pages;
return bio;

13
drivers/gpu/drm/msm/msm_iommu.c
View File

@ -330,17 +330,20 @@ static int
msm_iommu_pagetable_prealloc_allocate(struct msm_mmu *mmu, struct msm_mmu_prealloc *p)
{
struct kmem_cache *pt_cache = get_pt_cache(mmu);
int ret;
if (!p->count) {
p->pages = NULL;
return 0;
}
p->pages = kvmalloc_objs(*p->pages, p->count);
if (!p->pages)
return -ENOMEM;
ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, p->count, p->pages);
if (ret != p->count) {
kfree(p->pages);
if (!kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, p->count, p->pages)) {
kvfree(p->pages);
p->pages = NULL;
p->count = ret;
p->count = 0;
return -ENOMEM;
}

13
drivers/gpu/drm/panthor/panthor_mmu.c
View File

@ -1274,13 +1274,13 @@ static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx,
goto err_cleanup;
}
ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,
op_ctx->rsvd_page_tables.pages);
op_ctx->rsvd_page_tables.count = ret;
if (ret != pt_count) {
if (!kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,
op_ctx->rsvd_page_tables.pages)) {
op_ctx->rsvd_page_tables.count = 0;
ret = -ENOMEM;
goto err_cleanup;
}
op_ctx->rsvd_page_tables.count = pt_count;
/* Insert BO into the extobj list last, when we know nothing can fail. */
dma_resv_lock(panthor_vm_resv(vm), NULL);
@ -1328,9 +1328,8 @@ static int panthor_vm_prepare_unmap_op_ctx(struct panthor_vm_op_ctx *op_ctx,
goto err_cleanup;
}
ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,
op_ctx->rsvd_page_tables.pages);
if (ret != pt_count) {
if (!kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,
op_ctx->rsvd_page_tables.pages)) {
ret = -ENOMEM;
goto err_cleanup;
}

2
include/linux/mempool.h
View File

@ -66,7 +66,7 @@ void *mempool_alloc_noprof(struct mempool *pool, gfp_t gfp_mask) __malloc;
#define mempool_alloc(...) \
alloc_hooks(mempool_alloc_noprof(__VA_ARGS__))
int mempool_alloc_bulk_noprof(struct mempool *pool, void **elem,
unsigned int count, unsigned int allocated);
unsigned int count);
#define mempool_alloc_bulk(...) \
alloc_hooks(mempool_alloc_bulk_noprof(__VA_ARGS__))

6
include/linux/slab.h
View File

@ -815,8 +815,10 @@ kmem_buckets *kmem_buckets_create(const char *name, slab_flags_t flags,
*/
void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p);
int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, void **p);
#define kmem_cache_alloc_bulk(...) alloc_hooks(kmem_cache_alloc_bulk_noprof(__VA_ARGS__))
bool kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags,
size_t size, void **p);
#define kmem_cache_alloc_bulk(...) \
alloc_hooks(kmem_cache_alloc_bulk_noprof(__VA_ARGS__))
static __always_inline void kfree_bulk(size_t size, void **p)
{

23
io_uring/io_uring.c
View File

@ -966,29 +966,24 @@ __cold bool __io_alloc_req_refill(struct io_ring_ctx *ctx)
{
gfp_t gfp = GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO;
void *reqs[IO_REQ_ALLOC_BATCH];
int ret;
ret = kmem_cache_alloc_bulk(req_cachep, gfp, ARRAY_SIZE(reqs), reqs);
int nr_reqs = ARRAY_SIZE(reqs);
/*
* Bulk alloc is all-or-nothing. If we fail to get a batch,
* retry single alloc to be on the safe side.
* Bulk alloc is all-or-nothing. If we fail to get a batch, retry a
* single allocation to be on the safe side.
*/
if (unlikely(ret <= 0)) {
if (!kmem_cache_alloc_bulk(req_cachep, gfp, nr_reqs, reqs)) {
reqs[0] = kmem_cache_alloc(req_cachep, gfp);
if (!reqs[0])
return false;
ret = 1;
nr_reqs = 1;
}
percpu_ref_get_many(&ctx->refs, ret);
ctx->nr_req_allocated += ret;
percpu_ref_get_many(&ctx->refs, nr_reqs);
ctx->nr_req_allocated += nr_reqs;
while (ret--) {
struct io_kiocb *req = reqs[ret];
io_req_add_to_cache(req, ctx);
}
while (nr_reqs--)
io_req_add_to_cache(reqs[nr_reqs], ctx);
return true;
}

23
lib/test_meminit.c
View File

@ -229,16 +229,14 @@ static int __init do_kmem_cache_size(size_t size, bool want_ctor,
for (iter = 0; iter < 10; iter++) {
/* Do a test of bulk allocations */
if (!want_rcu && !want_ctor) {
int ret;
ret = kmem_cache_alloc_bulk(c, alloc_mask, BULK_SIZE, bulk_array);
if (!ret) {
if (!kmem_cache_alloc_bulk(c, alloc_mask, BULK_SIZE,
bulk_array)) {
fail = true;
} else {
int i;
for (i = 0; i < ret; i++)
for (i = 0; i < BULK_SIZE; i++)
fail |= check_buf(bulk_array[i], size, want_ctor, want_rcu, want_zero);
kmem_cache_free_bulk(c, ret, bulk_array);
kmem_cache_free_bulk(c, BULK_SIZE, bulk_array);
}
}
@ -348,23 +346,24 @@ static int __init do_kmem_cache_size_bulk(int size, int *total_failures)
{
struct kmem_cache *c;
int i, iter, maxiter = 1024;
int num, bytes;
int bytes;
bool fail = false;
void *objects[10];
c = kmem_cache_create("test_cache", size, size, 0, NULL);
for (iter = 0; (iter < maxiter) && !fail; iter++) {
num = kmem_cache_alloc_bulk(c, GFP_KERNEL, ARRAY_SIZE(objects),
objects);
for (i = 0; i < num; i++) {
if (!kmem_cache_alloc_bulk(c, GFP_KERNEL, ARRAY_SIZE(objects),
objects))
continue;
for (i = 0; i < ARRAY_SIZE(objects); i++) {
bytes = count_nonzero_bytes(objects[i], size);
if (bytes)
fail = true;
fill_with_garbage(objects[i], size);
}
if (num)
kmem_cache_free_bulk(c, num, objects);
kmem_cache_free_bulk(c, ARRAY_SIZE(objects), objects);
}
kmem_cache_destroy(c);
*total_failures += fail;

5
mm/kasan/kasan_test_c.c
View File

@ -1215,14 +1215,13 @@ static void kmem_cache_bulk(struct kunit *test)
struct kmem_cache *cache;
size_t size = 200;
char *p[10];
bool ret;
int i;
cache = kmem_cache_create("test_cache", size, 0, 0, NULL);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache);
ret = kmem_cache_alloc_bulk(cache, GFP_KERNEL, ARRAY_SIZE(p), (void **)&p);
if (!ret) {
if (!kmem_cache_alloc_bulk(cache, GFP_KERNEL, ARRAY_SIZE(p),
(void **)&p)) {
kunit_err(test, "Allocation failed: %s\n", __func__);
kmem_cache_destroy(cache);
return;

9
mm/kfence/kfence_test.c
View File

@ -761,9 +761,10 @@ static void test_memcache_alloc_bulk(struct kunit *test)
timeout = jiffies + msecs_to_jiffies(100 * kfence_sample_interval);
do {
void *objects[100];
int i, num = kmem_cache_alloc_bulk(test_cache, GFP_ATOMIC, ARRAY_SIZE(objects),
objects);
if (!num)
int i;
if (!kmem_cache_alloc_bulk(test_cache, GFP_ATOMIC,
ARRAY_SIZE(objects), objects))
continue;
for (i = 0; i < ARRAY_SIZE(objects); i++) {
if (is_kfence_address(objects[i])) {
@ -771,7 +772,7 @@ static void test_memcache_alloc_bulk(struct kunit *test)
break;
}
}
kmem_cache_free_bulk(test_cache, num, objects);
kmem_cache_free_bulk(test_cache, ARRAY_SIZE(objects), objects);
/*
* kmem_cache_alloc_bulk() disables interrupts, and calling it
* in a tight loop may not give KFENCE a chance to switch the

27
mm/mempool.c
View File

@ -419,12 +419,8 @@ static unsigned int mempool_alloc_from_pool(struct mempool *pool, void **elems,
spin_lock_irqsave(&pool->lock, flags);
if (unlikely(pool->curr_nr < count - allocated))
goto fail;
for (i = 0; i < count; i++) {
if (!elems[i]) {
elems[i] = remove_element(pool);
allocated++;
}
}
while (allocated < count)
elems[allocated++] = remove_element(pool);
spin_unlock_irqrestore(&pool->lock, flags);
/* Paired with rmb in mempool_free(), read comment there. */
@ -479,22 +475,21 @@ static inline gfp_t mempool_adjust_gfp(gfp_t *gfp_mask)
* @pool: pointer to the memory pool
* @elems: partially or fully populated elements array
* @count: number of entries in @elem that need to be allocated
* @allocated: number of entries in @elem already allocated
*
* Allocate elements for each slot in @elem that is non-%NULL. This is done by
* first calling into the alloc_fn supplied at pool initialization time, and
* dipping into the reserved pool when alloc_fn fails to allocate an element.
* Allocate @count elements into @elems. This is done by first calling into the
* alloc_fn supplied at pool initialization time, and dipping into the reserved
* pool when alloc_fn fails to allocate an element.
*
* On return all @count elements in @elems will be populated.
*
* Return: Always 0. If it wasn't for %$#^$ alloc tags, it would return void.
*/
int mempool_alloc_bulk_noprof(struct mempool *pool, void **elems,
unsigned int count, unsigned int allocated)
unsigned int count)
{
gfp_t gfp_mask = GFP_KERNEL;
gfp_t gfp_temp = mempool_adjust_gfp(&gfp_mask);
unsigned int i = 0;
unsigned int allocated = 0;
VM_WARN_ON_ONCE(count > pool->min_nr);
might_alloc(gfp_mask);
@ -514,11 +509,9 @@ int mempool_alloc_bulk_noprof(struct mempool *pool, void **elems,
* Try to allocate the elements using the allocation callback first as
* that might succeed even when the caller's bulk allocation did not.
*/
for (i = 0; i < count; i++) {
if (elems[i])
continue;
elems[i] = pool->alloc(gfp_temp, pool->pool_data);
if (unlikely(!elems[i]))
while (allocated < count) {
elems[allocated] = pool->alloc(gfp_temp, pool->pool_data);
if (unlikely(!elems[allocated]))
goto use_pool;
allocated++;
}

64
mm/slub.c
View File

@ -5020,8 +5020,8 @@ static int __prefill_sheaf_pfmemalloc(struct kmem_cache *s,
return ret;
}
static int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags,
size_t size, void **p);
static bool __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags,
size_t size, void **p);
/*
* returns a sheaf that has at least the requested size
@ -5192,9 +5192,8 @@ int kmem_cache_refill_sheaf(struct kmem_cache *s, gfp_t gfp,
return __prefill_sheaf_pfmemalloc(s, sheaf, gfp);
if (!__kmem_cache_alloc_bulk(s, gfp, sheaf->capacity - sheaf->size,
&sheaf->objects[sheaf->size])) {
&sheaf->objects[sheaf->size]))
return -ENOMEM;
}
sheaf->size = sheaf->capacity;
return 0;
@ -7332,9 +7331,8 @@ refill_objects(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int min,
return refilled;
}
static inline
int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
void **p)
static bool __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags,
size_t size, void **p)
{
int i;
@ -7355,30 +7353,43 @@ int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
stat_add(s, ALLOC_SLOWPATH, i);
}
return i;
return true;
error:
__kmem_cache_free_bulk(s, i, p);
return 0;
return false;
}
/*
* Note that interrupts must be enabled when calling this function and gfp
* flags must allow spinning.
/**
* kmem_cache_alloc_bulk - Allocate multiple objects
* @s: The cache to allocate from
* @flags: GFP_* flags. See kmalloc().
* @size: Number of objects to allocate
* @p: Array of allocated objects
*
* Allocate @size objects from @s and places them into @p. @size must be larger
* than 0.
*
* Interrupts must be enabled when calling this function and @flags must allow
* spinning.
*
* Unlike alloc_pages_bulk(), this function does not check for already allocated
* objects in @p, and thus the caller does not need to zero it.
*
* Return: %true if the allocation succeeded, or %false if it failed.
*/
int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size,
void **p)
bool kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags,
size_t size, void **p)
{
unsigned int i = 0;
void *kfence_obj;
if (!size)
return 0;
return false;
s = slab_pre_alloc_hook(s, flags);
if (unlikely(!s))
return 0;
return false;
/*
* to make things simpler, only assume at most once kfence allocated
@ -7395,18 +7406,18 @@ int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size,
}
i = alloc_from_pcs_bulk(s, flags, size, p);
if (i < size) {
/*
* If we ran out of memory, don't bother with freeing back to
* the percpu sheaves, we have bigger problems.
*/
if (unlikely(__kmem_cache_alloc_bulk(s, flags, size - i, p + i) == 0)) {
if (unlikely(!__kmem_cache_alloc_bulk(s, flags, size - i,
p + i))) {
if (i > 0)
__kmem_cache_free_bulk(s, i, p);
if (kfence_obj)
__kfence_free(kfence_obj);
return 0;
return false;
}
}
@ -7421,16 +7432,9 @@ int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size,
}
out:
/*
* memcg and kmem_cache debug support and memory initialization.
* Done outside of the IRQ disabled fastpath loop.
*/
if (unlikely(!slab_post_alloc_hook(s, NULL, flags, size, p,
slab_want_init_on_alloc(flags, s), s->object_size))) {
return 0;
}
return size;
/* memcg and kmem_cache debug support and memory initialization */
return likely(slab_post_alloc_hook(s, NULL, flags, size, p,
slab_want_init_on_alloc(flags, s), s->object_size));
}
EXPORT_SYMBOL(kmem_cache_alloc_bulk_noprof);

7
net/bpf/test_run.c
View File

@ -243,12 +243,11 @@ static int xdp_recv_frames(struct xdp_frame **frames, int nframes,
struct net_device *dev)
{
gfp_t gfp = __GFP_ZERO | GFP_ATOMIC;
int i, n;
int i;
LIST_HEAD(list);
n = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, gfp, nframes,
(void **)skbs);
if (unlikely(n == 0)) {
if (unlikely(!kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, gfp,
nframes, (void **)skbs))) {
for (i = 0; i < nframes; i++)
xdp_return_frame(frames[i]);
return -ENOMEM;

24
net/core/skbuff.c
View File

@ -288,11 +288,11 @@ static inline struct sk_buff *napi_skb_cache_get(bool alloc)
local_lock_nested_bh(&napi_alloc_cache.bh_lock);
if (unlikely(!nc->skb_count)) {
if (alloc)
nc->skb_count = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
GFP_ATOMIC | __GFP_NOWARN,
NAPI_SKB_CACHE_BULK,
nc->skb_cache);
if (alloc && kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
GFP_ATOMIC | __GFP_NOWARN,
NAPI_SKB_CACHE_BULK,
nc->skb_cache))
nc->skb_count = NAPI_SKB_CACHE_BULK;
if (unlikely(!nc->skb_count)) {
local_unlock_nested_bh(&napi_alloc_cache.bh_lock);
return NULL;
@ -353,16 +353,18 @@ u32 napi_skb_cache_get_bulk(void **skbs, u32 n)
/* No enough cached skbs. Try refilling the cache first */
bulk = min(NAPI_SKB_CACHE_SIZE - nc->skb_count, NAPI_SKB_CACHE_BULK);
nc->skb_count += kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
GFP_ATOMIC | __GFP_NOWARN, bulk,
&nc->skb_cache[nc->skb_count]);
if (kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
GFP_ATOMIC | __GFP_NOWARN, bulk,
&nc->skb_cache[nc->skb_count]))
nc->skb_count += bulk;
if (likely(nc->skb_count >= n))
goto get;
/* Still not enough. Bulk-allocate the missing part directly, zeroed */
n -= kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
GFP_ATOMIC | __GFP_ZERO | __GFP_NOWARN,
n - nc->skb_count, &skbs[nc->skb_count]);
if (kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
GFP_ATOMIC | __GFP_ZERO | __GFP_NOWARN,
n - nc->skb_count, &skbs[nc->skb_count]))
n = nc->skb_count;
if (likely(nc->skb_count >= n))
goto get;

2
tools/include/linux/slab.h
View File

@ -183,7 +183,7 @@ __kmem_cache_create(const char *name, unsigned int size, unsigned int align,
default: __kmem_cache_create)(__name, __object_size, __args, __VA_ARGS__)
void kmem_cache_free_bulk(struct kmem_cache *cachep, size_t size, void **list);
int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,
bool kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,
void **list);
struct slab_sheaf *
kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size);

19
tools/testing/shared/linux.c
View File

@ -154,7 +154,7 @@ void kmem_cache_shrink(struct kmem_cache *cachep)
{
}
int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,
bool kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,
void **p)
{
size_t i;
@ -213,7 +213,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,
pthread_mutex_unlock(&cachep->lock);
if (cachep->callback)
cachep->exec_callback = true;
return 0;
return false;
}
for (i = 0; i < size; i++) {
@ -224,7 +224,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,
printf("Allocating %p from slab\n", p[i]);
}
return size;
return true;
}
struct kmem_cache *
@ -271,8 +271,8 @@ kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size)
sheaf->cache = s;
sheaf->capacity = capacity;
sheaf->size = kmem_cache_alloc_bulk(s, gfp, size, sheaf->objects);
if (!sheaf->size) {
sheaf->size = size;
if (!kmem_cache_alloc_bulk(s, gfp, size, sheaf->objects)) {
free(sheaf);
return NULL;
}
@ -284,7 +284,6 @@ int kmem_cache_refill_sheaf(struct kmem_cache *s, gfp_t gfp,
struct slab_sheaf **sheafp, unsigned int size)
{
struct slab_sheaf *sheaf = *sheafp;
int refill;
if (sheaf->size >= size)
return 0;
@ -299,12 +298,10 @@ int kmem_cache_refill_sheaf(struct kmem_cache *s, gfp_t gfp,
return 0;
}
refill = kmem_cache_alloc_bulk(s, gfp, size - sheaf->size,
&sheaf->objects[sheaf->size]);
if (!refill)
if (!kmem_cache_alloc_bulk(s, gfp, size - sheaf->size,
&sheaf->objects[sheaf->size]))
return -ENOMEM;
sheaf->size += refill;
sheaf->size = size;
return 0;
}