github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

io_uring: keep the drain active when a deferred drain request is started

Browse Source 
ecb6817b76 ends the drain when the dispatched drain request has posted
its CQE and nothing else is deferred. But the deferred list is processed
in between, and if the request it starts is itself a drain request, the
drain is over before that one has even run. A request submitted after
that point then runs ahead of it:

  1. P0 in flight, D1 and D2 (both IOSQE_IO_DRAIN) deferred behind it
  2. P0 completes, D1 is started
  3. D1 completes, drain_done is set, D2 is started, the list is empty
  4. IO_RING_F_DRAIN_ACTIVE is cleared with D2 still in flight

Anything submitted now bypasses the drain. The same happens with D2 as
the head of a link, as those aren't tracked in ->drain_req at all.

Have io_queue_deferred() report whether it started a drain request, and
don't end the drain if it did. Starting non-drain requests and ending it
is still fine, they were only waiting on the drain request that just
completed.

Fixes: ecb6817b76 ("io_uring: end the drain when the drain request completes")
Reported-by: Karl Mehltretter <kmehltretter@gmail.com>
Link: https://lore.kernel.org/io-uring/20261027043323.98916-1-kmehltretter@gmail.com/
Cc: stable@vger.kernel.org
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Jens Axboe 2026-10-27 12:02:04 -06:00
parent 71537ed811
commit 2295613cd3
1 changed files with 13 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
io_uring/io_uring.c
View File

@ -474,9 +474,9 @@ unsigned io_linked_nr(struct io_kiocb *req)
return nr;
}
static __cold noinline void io_queue_deferred(struct io_ring_ctx *ctx)
static __cold noinline bool io_queue_deferred(struct io_ring_ctx *ctx)
{
bool drain_seen = false, first = true;
bool drain_seen = false, first = true, drain_queued = false;
lockdep_assert_held(&ctx->uring_lock);
__io_req_caches_free(ctx);
@ -490,17 +490,21 @@ static __cold noinline void io_queue_deferred(struct io_ring_ctx *ctx)
drain_seen |= de->req->flags & REQ_F_IO_DRAIN;
if ((drain_seen || first) && ctx->nr_req_allocated != ctx->nr_drained)
return;
return drain_queued;
list_del_init(&de->list);
ctx->nr_drained -= io_linked_nr(de->req);
/* it runs alone, track its completion unless it's a link */
if ((de->req->flags & REQ_F_IO_DRAIN) && !de->req->link)
ctx->drain_req = de->req;
if (de->req->flags & REQ_F_IO_DRAIN) {
drain_queued = true;
/* it runs alone, track its completion unless it's a link */
if (!de->req->link)
ctx->drain_req = de->req;
}
io_req_task_queue(de->req);
kfree(de);
first = false;
}
return drain_queued;
}
void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
@ -1263,7 +1267,9 @@ void __io_submit_flush_completions(struct io_ring_ctx *ctx)
}
if (unlikely(ctx->int_flags & IO_RING_F_DRAIN_ACTIVE)) {
io_queue_deferred(ctx);
/* a drain request started from the list keeps the drain going */
if (io_queue_deferred(ctx))
drain_done = false;
if (drain_done && list_empty(&ctx->defer_list))
ctx->int_flags &= ~IO_RING_F_DRAIN_ACTIVE;
}