github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-22 12:44:03 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

io_uring/zcrx: cache RQ tail

Browse Source 
The RQ tail is updated by the user space. Cache it to reduce cache line
bouncing. Refilling now tries to exhaust the previous batch of rqes, but
since it could be too low, the iterator is allowed to recalculate the
rqes to process once after synching the tail value.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://patch.msgid.link/b42d2ed8b0e697110b646573b5da2a5f8e85f92e.1786108672.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov 2026-08-07 14:19:22 +01:00 committed by Jens Axboe
parent 2292f4fbc5
commit 888f7ab164
2 changed files with 23 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
io_uring/zcrx.c
View File

@ -1089,14 +1089,20 @@ void io_unregister_zcrx(struct io_ring_ctx *ctx)
struct zcrx_rq_iter {
int rqes_left;
bool flushed;
};
static inline u32 __zcrx_rq_entries(struct zcrx_rq *rq)
{
u32 entries = rq->cached_tail - rq->cached_head;
return min(entries, rq->nr_entries);
}
static inline u32 zcrx_rq_entries(struct zcrx_rq *rq)
{
u32 entries;
entries = smp_load_acquire(&rq->ring->tail) - rq->cached_head;
return min(entries, rq->nr_entries);
rq->cached_tail = smp_load_acquire(&rq->ring->tail);
return __zcrx_rq_entries(rq);
}
static struct io_uring_zcrx_rqe *zcrx_next_rqe(struct zcrx_rq *rq, unsigned mask)
@ -1109,7 +1115,8 @@ static struct io_uring_zcrx_rqe *zcrx_next_rqe(struct zcrx_rq *rq, unsigned mask
static inline void zcrx_rq_iter_init(struct zcrx_rq_iter *it,
struct zcrx_rq *rq)
{
it->rqes_left = min_t(unsigned, zcrx_rq_entries(rq), ZCRX_REFILL_CAP);
it->rqes_left = min_t(unsigned, __zcrx_rq_entries(rq), ZCRX_REFILL_CAP);
it->flushed = false;
}
static inline bool zcrx_rq_iter_next(struct zcrx_rq_iter *it,
@ -1117,8 +1124,16 @@ static inline bool zcrx_rq_iter_next(struct zcrx_rq_iter *it,
struct io_uring_zcrx_rqe **rqe)
{
it->rqes_left--;
if (unlikely(it->rqes_left < 0))
return false;
if (unlikely(it->rqes_left < 0)) {
if (it->flushed)
return false;
rq->cached_tail = smp_load_acquire(&rq->ring->tail);
it->rqes_left = min_t(unsigned, __zcrx_rq_entries(rq),
ZCRX_REFILL_CAP);
it->flushed = true;
if (--it->rqes_left < 0)
return false;
}
*rqe = zcrx_next_rqe(rq, rq->nr_entries - 1);
return true;

1
io_uring/zcrx.h
View File

@ -53,6 +53,7 @@ struct zcrx_rq {
struct zcrx_rq_hdr *ring;
struct io_uring_zcrx_rqe *rqes;
u32 cached_head;
u32 cached_tail;
u32 nr_entries;
};