github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-22 12:44:03 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

io_uring/zcrx: coalesce same-niov RQEs on refill

Browse Source 
With large rx piages I often see >10 sequential RQEs referring to the
same niov. Instead of putting them one by one, count such RQEs during
parsing and batch refcounting for the niov.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://patch.msgid.link/71f11d5de47553f187737e0e892ab783b03f53b1.1786108672.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov 2026-08-07 14:19:23 +01:00 committed by Jens Axboe
parent 888f7ab164
commit 7b188cf990
1 changed files with 36 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
io_uring/zcrx.c
View File

@ -358,16 +358,16 @@ static inline atomic_t *io_get_user_counter(struct net_iov *niov)
return &area->user_refs[net_iov_idx(niov)];
}
static bool io_zcrx_put_niov_uref(struct net_iov *niov)
static bool io_zcrx_put_niov_uref(struct net_iov *niov, unsigned refs)
{
atomic_t *uref = io_get_user_counter(niov);
int old;
old = atomic_read(uref);
do {
if (unlikely(old == 0))
if (unlikely(old < refs))
return false;
} while (!atomic_try_cmpxchg(uref, &old, old - 1));
} while (!atomic_try_cmpxchg(uref, &old, old - refs));
return true;
}
@ -1162,6 +1162,22 @@ static inline bool io_parse_rqe(struct io_uring_zcrx_rqe *rqe,
return true;
}
static bool zcrx_put_refill_niov(struct net_iov *niov, struct page_pool *pp,
unsigned refs)
{
netmem_ref netmem = net_iov_to_netmem(niov);
if (!io_zcrx_put_niov_uref(niov, refs))
return false;
if (page_pool_unref_netmem(netmem, refs) != 0)
return false;
if (unlikely(niov->desc.pp != pp)) {
io_zcrx_return_niov(niov);
return false;
}
return true;
}
static unsigned io_zcrx_ring_refill(struct page_pool *pp,
struct io_zcrx_ifq *ifq,
netmem_ref *netmems, unsigned to_alloc)
@ -1169,34 +1185,34 @@ static unsigned io_zcrx_ring_refill(struct page_pool *pp,
struct zcrx_rq *rq = &ifq->rq;
struct io_uring_zcrx_rqe *rqe;
struct zcrx_rq_iter it;
struct net_iov *niov = NULL;
unsigned niov_refs = 0;
unsigned allocated = 0;
guard(spinlock_bh)(&rq->lock);
zcrx_rq_iter_init(&it, rq);
while (zcrx_rq_iter_next(&it, rq, &rqe)) {
struct net_iov *niov;
netmem_ref netmem;
while (allocated < to_alloc - 1 && zcrx_rq_iter_next(&it, rq, &rqe)) {
struct net_iov *next_niov;
if (!io_parse_rqe(rqe, ifq, &niov))
if (!io_parse_rqe(rqe, ifq, &next_niov))
continue;
if (!io_zcrx_put_niov_uref(niov))
continue;
netmem = net_iov_to_netmem(niov);
if (!page_pool_unref_and_test(netmem))
continue;
if (unlikely(niov->desc.pp != pp)) {
io_zcrx_return_niov(niov);
if (niov == next_niov) {
niov_refs++;
continue;
}
if (niov && zcrx_put_refill_niov(niov, pp, niov_refs)) {
netmems[allocated] = net_iov_to_netmem(niov);
allocated++;
}
niov = next_niov;
niov_refs = 1;
}
netmems[allocated] = netmem;
if (niov && zcrx_put_refill_niov(niov, pp, niov_refs)) {
netmems[allocated] = net_iov_to_netmem(niov);
allocated++;
if (allocated >= to_alloc)
break;
}
smp_store_release(&rq->ring->head, rq->cached_head);
@ -1408,7 +1424,7 @@ static void zcrx_return_buffers(netmem_ref *netmems, unsigned nr)
netmem_ref netmem = netmems[i];
struct net_iov *niov = netmem_to_net_iov(netmem);
if (!io_zcrx_put_niov_uref(niov))
if (!io_zcrx_put_niov_uref(niov, 1))
continue;
if (!page_pool_unref_and_test(netmem))
continue;