github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

io_uring: end the drain when the drain request completes

Browse Source 
Since requests are drained based on allocated requests, the drain stays
active after the drain request has posted its CQE, until it is freed and
a new request finds the defer list empty. A drain request run by io-wq
is freed only after the worker drops its reference, so a request
submitted after the drain completed can still be deferred behind it,
and get issued much later than the application expects.

A drain request dispatched from the defer list runs alone. Track it,
and end the drain when its CQE is flushed and nothing else is deferred.
Have io-wq complete drain requests through task_work, so that this
happens under ->uring_lock before the CQE is visible. Linked drain
requests keep waiting until the link is freed.

Reported-by: Wei Gao <wegao@suse.com>
Link: https://lore.kernel.org/io-uring/20261018070154.16944-1-wegao@suse.com/
Fixes: 8fb7aee055 ("io_uring: drain based on allocates reqs")
Cc: stable@vger.kernel.org
Tested-by: Wei Gao <wegao@suse.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Jens Axboe 2026-10-19 13:30:51 -06:00
parent 02caa59221
commit ecb6817b76
2 changed files with 41 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/linux/io_uring_types.h
View File

@ -524,6 +524,8 @@ struct io_ring_ctx {
atomic_t poll_wq_rcu_pending; atomic_t poll_wq_rcu_pending;
struct list_head defer_list; struct list_head defer_list;
unsigned nr_drained; unsigned nr_drained;
/* drain request dispatched from ->defer_list, until it completes */
struct io_kiocb *drain_req;
/* protected by ->completion_lock */ /* protected by ->completion_lock */
unsigned nr_req_allocated; unsigned nr_req_allocated;

42
io_uring/io_uring.c
View File

@ -480,6 +480,9 @@ static __cold noinline void io_queue_deferred(struct io_ring_ctx *ctx)
lockdep_assert_held(&ctx->uring_lock); lockdep_assert_held(&ctx->uring_lock);
__io_req_caches_free(ctx); __io_req_caches_free(ctx);
/* nothing outside of ->defer_list is alive */
if (ctx->nr_req_allocated == ctx->nr_drained)
ctx->drain_req = NULL;
while (!list_empty(&ctx->defer_list)) { while (!list_empty(&ctx->defer_list)) {
struct io_defer_entry *de = list_first_entry(&ctx->defer_list, struct io_defer_entry *de = list_first_entry(&ctx->defer_list,
@ -491,6 +494,9 @@ static __cold noinline void io_queue_deferred(struct io_ring_ctx *ctx)
list_del_init(&de->list); list_del_init(&de->list);
ctx->nr_drained -= io_linked_nr(de->req); ctx->nr_drained -= io_linked_nr(de->req);
/* it runs alone, track its completion unless it's a link */
if ((de->req->flags & REQ_F_IO_DRAIN) && !de->req->link)
ctx->drain_req = de->req;
io_req_task_queue(de->req); io_req_task_queue(de->req);
kfree(de); kfree(de);
first = false; first = false;
@ -980,9 +986,11 @@ static void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
/* /*
* Handle special CQ sync cases via task_work. DEFER_TASKRUN requires * Handle special CQ sync cases via task_work. DEFER_TASKRUN requires
* the submitter task context, IOPOLL protects with uring_lock. * the submitter task context, IOPOLL protects with uring_lock. A drain
* request ends the drain under uring_lock as its CQE is posted.
*/ */
if ((ctx->int_flags & IO_RING_F_LOCKLESS_CQ) || (req->flags & REQ_F_REISSUE)) { if ((ctx->int_flags & IO_RING_F_LOCKLESS_CQ) ||
(req->flags & (REQ_F_REISSUE | REQ_F_IO_DRAIN))) {
defer_complete: defer_complete:
req->io_task_work.func = io_req_task_complete; req->io_task_work.func = io_req_task_complete;
io_req_task_work_add(req); io_req_task_work_add(req);
@ -1185,11 +1193,33 @@ static void io_free_batch_list(struct io_ring_ctx *ctx,
} while (node); } while (node);
} }
/*
* The dispatched drain request ran alone, and anything submitted after it was
* deferred. Once it completed, new requests may run if nothing is waiting.
*/
static __cold bool io_drain_req_completed(struct io_ring_ctx *ctx)
{
struct io_wq_work_node *node;
__wq_list_for_each(node, &ctx->submit_state.compl_reqs) {
struct io_kiocb *req = container_of(node, struct io_kiocb,
comp_list);
/* a requeued request hasn't completed */
if (req == ctx->drain_req && !(req->flags & REQ_F_REISSUE)) {
ctx->drain_req = NULL;
return true;
}
}
return false;
}
void __io_submit_flush_completions(struct io_ring_ctx *ctx) void __io_submit_flush_completions(struct io_ring_ctx *ctx)
__must_hold(&ctx->uring_lock) __must_hold(&ctx->uring_lock)
{ {
struct io_submit_state *state = &ctx->submit_state; struct io_submit_state *state = &ctx->submit_state;
struct io_wq_work_node *node; struct io_wq_work_node *node;
bool drain_done = false;
/* /*
* Drop the files before the CQEs are posted, so they're released by * Drop the files before the CQEs are posted, so they're released by
@ -1224,13 +1254,19 @@ void __io_submit_flush_completions(struct io_ring_ctx *ctx)
} }
__io_cq_unlock_post(ctx); __io_cq_unlock_post(ctx);
if (unlikely(ctx->int_flags & IO_RING_F_DRAIN_ACTIVE) && ctx->drain_req)
drain_done = io_drain_req_completed(ctx);
if (!wq_list_empty(&state->compl_reqs)) { if (!wq_list_empty(&state->compl_reqs)) {
io_free_batch_list(ctx, state->compl_reqs.first); io_free_batch_list(ctx, state->compl_reqs.first);
INIT_WQ_LIST(&state->compl_reqs); INIT_WQ_LIST(&state->compl_reqs);
} }
if (unlikely(ctx->int_flags & IO_RING_F_DRAIN_ACTIVE)) if (unlikely(ctx->int_flags & IO_RING_F_DRAIN_ACTIVE)) {
io_queue_deferred(ctx); io_queue_deferred(ctx);
if (drain_done && list_empty(&ctx->defer_list))
ctx->int_flags &= ~IO_RING_F_DRAIN_ACTIVE;
}
ctx->submit_state.cq_flush = false; ctx->submit_state.cq_flush = false;
} }