github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-27 17:47:41 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

mm/slab: improve kmem_cache_alloc_bulk

Browse Source 
The kmem_cache_alloc_bulk return value is weird.  It returns the number
of allocated objects, but that must always be 0 or the requested number
based on the implementations and the handling in the callers, but that
assumption is not actually documented anywhere, which confuses automated
review tools.

Fix this by returning a bool if the allocation succeeded and adding a
kerneldoc comment explaining the API.

[rob.clark@oss.qualcomm.com: fixups in
 msm_iommu_pagetable_prealloc_allocate() ]

Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Alexander Lobakin <aleksander.lobakin@intel.com> # skbuff
Link: https://patch.msgid.link/20260528093437.2519248-2-hch@lst.de
Signed-off-by: Vlastimil Babka (SUSE) <vbabka@kernel.org>
This commit is contained in:
Christoph Hellwig 2026-05-28 11:34:32 +02:00 committed by Vlastimil Babka (SUSE)
parent 5d6919055d
commit 6bb0009862
12 changed files with 104 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
drivers/gpu/drm/msm/msm_iommu.c
View File

@ -330,17 +330,20 @@ static int
msm_iommu_pagetable_prealloc_allocate(struct msm_mmu *mmu, struct msm_mmu_prealloc *p)
{
struct kmem_cache *pt_cache = get_pt_cache(mmu);
int ret;
if (!p->count) {
p->pages = NULL;
return 0;
}
p->pages = kvmalloc_objs(*p->pages, p->count);
if (!p->pages)
return -ENOMEM;
ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, p->count, p->pages);
if (ret != p->count) {
kfree(p->pages);
if (!kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, p->count, p->pages)) {
kvfree(p->pages);
p->pages = NULL;
p->count = ret;
p->count = 0;
return -ENOMEM;
}

13
drivers/gpu/drm/panthor/panthor_mmu.c
View File

@ -1274,13 +1274,13 @@ static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx,
goto err_cleanup;
}
ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,
op_ctx->rsvd_page_tables.pages);
op_ctx->rsvd_page_tables.count = ret;
if (ret != pt_count) {
if (!kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,
op_ctx->rsvd_page_tables.pages)) {
op_ctx->rsvd_page_tables.count = 0;
ret = -ENOMEM;
goto err_cleanup;
}
op_ctx->rsvd_page_tables.count = pt_count;
/* Insert BO into the extobj list last, when we know nothing can fail. */
dma_resv_lock(panthor_vm_resv(vm), NULL);
@ -1328,9 +1328,8 @@ static int panthor_vm_prepare_unmap_op_ctx(struct panthor_vm_op_ctx *op_ctx,
goto err_cleanup;
}
ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,
op_ctx->rsvd_page_tables.pages);
if (ret != pt_count) {
if (!kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,
op_ctx->rsvd_page_tables.pages)) {
ret = -ENOMEM;
goto err_cleanup;
}

6
include/linux/slab.h
View File

@ -815,8 +815,10 @@ kmem_buckets *kmem_buckets_create(const char *name, slab_flags_t flags,
*/
void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p);
int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, void **p);
#define kmem_cache_alloc_bulk(...) alloc_hooks(kmem_cache_alloc_bulk_noprof(__VA_ARGS__))
bool kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags,
size_t size, void **p);
#define kmem_cache_alloc_bulk(...) \
alloc_hooks(kmem_cache_alloc_bulk_noprof(__VA_ARGS__))
static __always_inline void kfree_bulk(size_t size, void **p)
{

23
io_uring/io_uring.c
View File

@ -966,29 +966,24 @@ __cold bool __io_alloc_req_refill(struct io_ring_ctx *ctx)
{
gfp_t gfp = GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO;
void *reqs[IO_REQ_ALLOC_BATCH];
int ret;
ret = kmem_cache_alloc_bulk(req_cachep, gfp, ARRAY_SIZE(reqs), reqs);
int nr_reqs = ARRAY_SIZE(reqs);
/*
* Bulk alloc is all-or-nothing. If we fail to get a batch,
* retry single alloc to be on the safe side.
* Bulk alloc is all-or-nothing. If we fail to get a batch, retry a
* single allocation to be on the safe side.
*/
if (unlikely(ret <= 0)) {
if (!kmem_cache_alloc_bulk(req_cachep, gfp, nr_reqs, reqs)) {
reqs[0] = kmem_cache_alloc(req_cachep, gfp);
if (!reqs[0])
return false;
ret = 1;
nr_reqs = 1;
}
percpu_ref_get_many(&ctx->refs, ret);
ctx->nr_req_allocated += ret;
percpu_ref_get_many(&ctx->refs, nr_reqs);
ctx->nr_req_allocated += nr_reqs;
while (ret--) {
struct io_kiocb *req = reqs[ret];
io_req_add_to_cache(req, ctx);
}
while (nr_reqs--)
io_req_add_to_cache(reqs[nr_reqs], ctx);
return true;
}

23
lib/test_meminit.c
View File

@ -229,16 +229,14 @@ static int __init do_kmem_cache_size(size_t size, bool want_ctor,
for (iter = 0; iter < 10; iter++) {
/* Do a test of bulk allocations */
if (!want_rcu && !want_ctor) {
int ret;
ret = kmem_cache_alloc_bulk(c, alloc_mask, BULK_SIZE, bulk_array);
if (!ret) {
if (!kmem_cache_alloc_bulk(c, alloc_mask, BULK_SIZE,
bulk_array)) {
fail = true;
} else {
int i;
for (i = 0; i < ret; i++)
for (i = 0; i < BULK_SIZE; i++)
fail |= check_buf(bulk_array[i], size, want_ctor, want_rcu, want_zero);
kmem_cache_free_bulk(c, ret, bulk_array);
kmem_cache_free_bulk(c, BULK_SIZE, bulk_array);
}
}
@ -348,23 +346,24 @@ static int __init do_kmem_cache_size_bulk(int size, int *total_failures)
{
struct kmem_cache *c;
int i, iter, maxiter = 1024;
int num, bytes;
int bytes;
bool fail = false;
void *objects[10];
c = kmem_cache_create("test_cache", size, size, 0, NULL);
for (iter = 0; (iter < maxiter) && !fail; iter++) {
num = kmem_cache_alloc_bulk(c, GFP_KERNEL, ARRAY_SIZE(objects),
objects);
for (i = 0; i < num; i++) {
if (!kmem_cache_alloc_bulk(c, GFP_KERNEL, ARRAY_SIZE(objects),
objects))
continue;
for (i = 0; i < ARRAY_SIZE(objects); i++) {
bytes = count_nonzero_bytes(objects[i], size);
if (bytes)
fail = true;
fill_with_garbage(objects[i], size);
}
if (num)
kmem_cache_free_bulk(c, num, objects);
kmem_cache_free_bulk(c, ARRAY_SIZE(objects), objects);
}
kmem_cache_destroy(c);
*total_failures += fail;

5
mm/kasan/kasan_test_c.c
View File

@ -1215,14 +1215,13 @@ static void kmem_cache_bulk(struct kunit *test)
struct kmem_cache *cache;
size_t size = 200;
char *p[10];
bool ret;
int i;
cache = kmem_cache_create("test_cache", size, 0, 0, NULL);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, cache);
ret = kmem_cache_alloc_bulk(cache, GFP_KERNEL, ARRAY_SIZE(p), (void **)&p);
if (!ret) {
if (!kmem_cache_alloc_bulk(cache, GFP_KERNEL, ARRAY_SIZE(p),
(void **)&p)) {
kunit_err(test, "Allocation failed: %s\n", __func__);
kmem_cache_destroy(cache);
return;

9
mm/kfence/kfence_test.c
View File

@ -761,9 +761,10 @@ static void test_memcache_alloc_bulk(struct kunit *test)
timeout = jiffies + msecs_to_jiffies(100 * kfence_sample_interval);
do {
void *objects[100];
int i, num = kmem_cache_alloc_bulk(test_cache, GFP_ATOMIC, ARRAY_SIZE(objects),
objects);
if (!num)
int i;
if (!kmem_cache_alloc_bulk(test_cache, GFP_ATOMIC,
ARRAY_SIZE(objects), objects))
continue;
for (i = 0; i < ARRAY_SIZE(objects); i++) {
if (is_kfence_address(objects[i])) {
@ -771,7 +772,7 @@ static void test_memcache_alloc_bulk(struct kunit *test)
break;
}
}
kmem_cache_free_bulk(test_cache, num, objects);
kmem_cache_free_bulk(test_cache, ARRAY_SIZE(objects), objects);
/*
* kmem_cache_alloc_bulk() disables interrupts, and calling it
* in a tight loop may not give KFENCE a chance to switch the

64
mm/slub.c
View File

@ -4980,8 +4980,8 @@ static int __prefill_sheaf_pfmemalloc(struct kmem_cache *s,
return ret;
}
static int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags,
size_t size, void **p);
static bool __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags,
size_t size, void **p);
/*
* returns a sheaf that has at least the requested size
@ -5153,9 +5153,8 @@ int kmem_cache_refill_sheaf(struct kmem_cache *s, gfp_t gfp,
return __prefill_sheaf_pfmemalloc(s, sheaf, gfp);
if (!__kmem_cache_alloc_bulk(s, gfp, sheaf->capacity - sheaf->size,
&sheaf->objects[sheaf->size])) {
&sheaf->objects[sheaf->size]))
return -ENOMEM;
}
sheaf->size = sheaf->capacity;
return 0;
@ -7272,9 +7271,8 @@ refill_objects(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int min,
return refilled;
}
static inline
int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
void **p)
static bool __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags,
size_t size, void **p)
{
int i;
@ -7295,30 +7293,43 @@ int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
stat_add(s, ALLOC_SLOWPATH, i);
}
return i;
return true;
error:
__kmem_cache_free_bulk(s, i, p);
return 0;
return false;
}
/*
* Note that interrupts must be enabled when calling this function and gfp
* flags must allow spinning.
/**
* kmem_cache_alloc_bulk - Allocate multiple objects
* @s: The cache to allocate from
* @flags: GFP_* flags. See kmalloc().
* @size: Number of objects to allocate
* @p: Array of allocated objects
*
* Allocate @size objects from @s and places them into @p. @size must be larger
* than 0.
*
* Interrupts must be enabled when calling this function and @flags must allow
* spinning.
*
* Unlike alloc_pages_bulk(), this function does not check for already allocated
* objects in @p, and thus the caller does not need to zero it.
*
* Return: %true if the allocation succeeded, or %false if it failed.
*/
int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size,
void **p)
bool kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags,
size_t size, void **p)
{
unsigned int i = 0;
void *kfence_obj;
if (!size)
return 0;
return false;
s = slab_pre_alloc_hook(s, flags);
if (unlikely(!s))
return 0;
return false;
/*
* to make things simpler, only assume at most once kfence allocated
@ -7335,18 +7346,18 @@ int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size,
}
i = alloc_from_pcs_bulk(s, flags, size, p);
if (i < size) {
/*
* If we ran out of memory, don't bother with freeing back to
* the percpu sheaves, we have bigger problems.
*/
if (unlikely(__kmem_cache_alloc_bulk(s, flags, size - i, p + i) == 0)) {
if (unlikely(!__kmem_cache_alloc_bulk(s, flags, size - i,
p + i))) {
if (i > 0)
__kmem_cache_free_bulk(s, i, p);
if (kfence_obj)
__kfence_free(kfence_obj);
return 0;
return false;
}
}
@ -7361,16 +7372,9 @@ int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size,
}
out:
/*
* memcg and kmem_cache debug support and memory initialization.
* Done outside of the IRQ disabled fastpath loop.
*/
if (unlikely(!slab_post_alloc_hook(s, NULL, flags, size, p,
slab_want_init_on_alloc(flags, s), s->object_size))) {
return 0;
}
return size;
/* memcg and kmem_cache debug support and memory initialization */
return likely(slab_post_alloc_hook(s, NULL, flags, size, p,
slab_want_init_on_alloc(flags, s), s->object_size));
}
EXPORT_SYMBOL(kmem_cache_alloc_bulk_noprof);

7
net/bpf/test_run.c
View File

@ -243,12 +243,11 @@ static int xdp_recv_frames(struct xdp_frame **frames, int nframes,
struct net_device *dev)
{
gfp_t gfp = __GFP_ZERO | GFP_ATOMIC;
int i, n;
int i;
LIST_HEAD(list);
n = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, gfp, nframes,
(void **)skbs);
if (unlikely(n == 0)) {
if (unlikely(!kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, gfp,
nframes, (void **)skbs))) {
for (i = 0; i < nframes; i++)
xdp_return_frame(frames[i]);
return -ENOMEM;

24
net/core/skbuff.c
View File

@ -288,11 +288,11 @@ static inline struct sk_buff *napi_skb_cache_get(bool alloc)
local_lock_nested_bh(&napi_alloc_cache.bh_lock);
if (unlikely(!nc->skb_count)) {
if (alloc)
nc->skb_count = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
GFP_ATOMIC | __GFP_NOWARN,
NAPI_SKB_CACHE_BULK,
nc->skb_cache);
if (alloc && kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
GFP_ATOMIC | __GFP_NOWARN,
NAPI_SKB_CACHE_BULK,
nc->skb_cache))
nc->skb_count = NAPI_SKB_CACHE_BULK;
if (unlikely(!nc->skb_count)) {
local_unlock_nested_bh(&napi_alloc_cache.bh_lock);
return NULL;
@ -353,16 +353,18 @@ u32 napi_skb_cache_get_bulk(void **skbs, u32 n)
/* No enough cached skbs. Try refilling the cache first */
bulk = min(NAPI_SKB_CACHE_SIZE - nc->skb_count, NAPI_SKB_CACHE_BULK);
nc->skb_count += kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
GFP_ATOMIC | __GFP_NOWARN, bulk,
&nc->skb_cache[nc->skb_count]);
if (kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
GFP_ATOMIC | __GFP_NOWARN, bulk,
&nc->skb_cache[nc->skb_count]))
nc->skb_count += bulk;
if (likely(nc->skb_count >= n))
goto get;
/* Still not enough. Bulk-allocate the missing part directly, zeroed */
n -= kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
GFP_ATOMIC | __GFP_ZERO | __GFP_NOWARN,
n - nc->skb_count, &skbs[nc->skb_count]);
if (kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
GFP_ATOMIC | __GFP_ZERO | __GFP_NOWARN,
n - nc->skb_count, &skbs[nc->skb_count]))
n = nc->skb_count;
if (likely(nc->skb_count >= n))
goto get;

2
tools/include/linux/slab.h
View File

@ -183,7 +183,7 @@ __kmem_cache_create(const char *name, unsigned int size, unsigned int align,
default: __kmem_cache_create)(__name, __object_size, __args, __VA_ARGS__)
void kmem_cache_free_bulk(struct kmem_cache *cachep, size_t size, void **list);
int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,
bool kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,
void **list);
struct slab_sheaf *
kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size);

19
tools/testing/shared/linux.c
View File

@ -154,7 +154,7 @@ void kmem_cache_shrink(struct kmem_cache *cachep)
{
}
int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,
bool kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,
void **p)
{
size_t i;
@ -213,7 +213,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,
pthread_mutex_unlock(&cachep->lock);
if (cachep->callback)
cachep->exec_callback = true;
return 0;
return false;
}
for (i = 0; i < size; i++) {
@ -224,7 +224,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,
printf("Allocating %p from slab\n", p[i]);
}
return size;
return true;
}
struct kmem_cache *
@ -271,8 +271,8 @@ kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size)
sheaf->cache = s;
sheaf->capacity = capacity;
sheaf->size = kmem_cache_alloc_bulk(s, gfp, size, sheaf->objects);
if (!sheaf->size) {
sheaf->size = size;
if (!kmem_cache_alloc_bulk(s, gfp, size, sheaf->objects)) {
free(sheaf);
return NULL;
}
@ -284,7 +284,6 @@ int kmem_cache_refill_sheaf(struct kmem_cache *s, gfp_t gfp,
struct slab_sheaf **sheafp, unsigned int size)
{
struct slab_sheaf *sheaf = *sheafp;
int refill;
if (sheaf->size >= size)
return 0;
@ -299,12 +298,10 @@ int kmem_cache_refill_sheaf(struct kmem_cache *s, gfp_t gfp,
return 0;
}
refill = kmem_cache_alloc_bulk(s, gfp, size - sheaf->size,
&sheaf->objects[sheaf->size]);
if (!refill)
if (!kmem_cache_alloc_bulk(s, gfp, size - sheaf->size,
&sheaf->objects[sheaf->size]))
return -ENOMEM;
sheaf->size += refill;
sheaf->size = size;
return 0;
}