diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index d1b82daf18f0..ef17c0e16bd4 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -1089,14 +1089,20 @@ void io_unregister_zcrx(struct io_ring_ctx *ctx) struct zcrx_rq_iter { int rqes_left; + bool flushed; }; +static inline u32 __zcrx_rq_entries(struct zcrx_rq *rq) +{ + u32 entries = rq->cached_tail - rq->cached_head; + + return min(entries, rq->nr_entries); +} + static inline u32 zcrx_rq_entries(struct zcrx_rq *rq) { - u32 entries; - - entries = smp_load_acquire(&rq->ring->tail) - rq->cached_head; - return min(entries, rq->nr_entries); + rq->cached_tail = smp_load_acquire(&rq->ring->tail); + return __zcrx_rq_entries(rq); } static struct io_uring_zcrx_rqe *zcrx_next_rqe(struct zcrx_rq *rq, unsigned mask) @@ -1109,7 +1115,8 @@ static struct io_uring_zcrx_rqe *zcrx_next_rqe(struct zcrx_rq *rq, unsigned mask static inline void zcrx_rq_iter_init(struct zcrx_rq_iter *it, struct zcrx_rq *rq) { - it->rqes_left = min_t(unsigned, zcrx_rq_entries(rq), ZCRX_REFILL_CAP); + it->rqes_left = min_t(unsigned, __zcrx_rq_entries(rq), ZCRX_REFILL_CAP); + it->flushed = false; } static inline bool zcrx_rq_iter_next(struct zcrx_rq_iter *it, @@ -1117,8 +1124,16 @@ static inline bool zcrx_rq_iter_next(struct zcrx_rq_iter *it, struct io_uring_zcrx_rqe **rqe) { it->rqes_left--; - if (unlikely(it->rqes_left < 0)) - return false; + if (unlikely(it->rqes_left < 0)) { + if (it->flushed) + return false; + rq->cached_tail = smp_load_acquire(&rq->ring->tail); + it->rqes_left = min_t(unsigned, __zcrx_rq_entries(rq), + ZCRX_REFILL_CAP); + it->flushed = true; + if (--it->rqes_left < 0) + return false; + } *rqe = zcrx_next_rqe(rq, rq->nr_entries - 1); return true; diff --git a/io_uring/zcrx.h b/io_uring/zcrx.h index 3cdfa4415d62..0eb7ea35a9ff 100644 --- a/io_uring/zcrx.h +++ b/io_uring/zcrx.h @@ -53,6 +53,7 @@ struct zcrx_rq { struct zcrx_rq_hdr *ring; struct io_uring_zcrx_rqe *rqes; u32 cached_head; + u32 cached_tail; u32 nr_entries; };