diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 7ad52f499f87..19a432df0ebe 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -28,6 +28,13 @@ #include "zcrx.h" #include "rsrc.h" +#define ZCRX_MAX_FRAGS_PER_PAGE MAX(PAGE_SIZE / 1024, 1) +/* + * We need a reasonable limit to be able to fill in 64 entries on average + * for 1500 byte MTU. Over-estimate it to keep it pow2. + */ +#define ZCRX_REFILL_CAP MIN(64 * ZCRX_MAX_FRAGS_PER_PAGE, 1024) + #define IO_ZCRX_AREA_SUPPORTED_FLAGS (IORING_ZCRX_AREA_DMABUF) #define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING) @@ -1124,17 +1131,15 @@ static unsigned io_zcrx_ring_refill(struct page_pool *pp, { struct zcrx_rq *rq = &ifq->rq; unsigned int mask = rq->nr_entries - 1; - unsigned int entries; + unsigned int rqes_left; unsigned allocated = 0; guard(spinlock_bh)(&rq->lock); - entries = zcrx_rq_entries(rq); - entries = min_t(unsigned, entries, to_alloc); - if (unlikely(!entries)) - return 0; + rqes_left = zcrx_rq_entries(rq); + rqes_left = min_t(unsigned, rqes_left, ZCRX_REFILL_CAP); - do { + for (; rqes_left; rqes_left--) { struct io_uring_zcrx_rqe *rqe = zcrx_next_rqe(rq, mask); struct net_iov *niov; netmem_ref netmem; @@ -1155,7 +1160,9 @@ static unsigned io_zcrx_ring_refill(struct page_pool *pp, netmems[allocated] = netmem; allocated++; - } while (--entries); + if (allocated >= to_alloc) + break; + } smp_store_release(&rq->ring->head, rq->cached_head); return allocated;