diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 8ffe0c9a4e29..e9c65cf0cc51 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -474,9 +474,9 @@ unsigned io_linked_nr(struct io_kiocb *req) return nr; } -static __cold noinline void io_queue_deferred(struct io_ring_ctx *ctx) +static __cold noinline bool io_queue_deferred(struct io_ring_ctx *ctx) { - bool drain_seen = false, first = true; + bool drain_seen = false, first = true, drain_queued = false; lockdep_assert_held(&ctx->uring_lock); __io_req_caches_free(ctx); @@ -490,17 +490,21 @@ static __cold noinline void io_queue_deferred(struct io_ring_ctx *ctx) drain_seen |= de->req->flags & REQ_F_IO_DRAIN; if ((drain_seen || first) && ctx->nr_req_allocated != ctx->nr_drained) - return; + return drain_queued; list_del_init(&de->list); ctx->nr_drained -= io_linked_nr(de->req); - /* it runs alone, track its completion unless it's a link */ - if ((de->req->flags & REQ_F_IO_DRAIN) && !de->req->link) - ctx->drain_req = de->req; + if (de->req->flags & REQ_F_IO_DRAIN) { + drain_queued = true; + /* it runs alone, track its completion unless it's a link */ + if (!de->req->link) + ctx->drain_req = de->req; + } io_req_task_queue(de->req); kfree(de); first = false; } + return drain_queued; } void __io_commit_cqring_flush(struct io_ring_ctx *ctx) @@ -1263,7 +1267,9 @@ void __io_submit_flush_completions(struct io_ring_ctx *ctx) } if (unlikely(ctx->int_flags & IO_RING_F_DRAIN_ACTIVE)) { - io_queue_deferred(ctx); + /* a drain request started from the list keeps the drain going */ + if (io_queue_deferred(ctx)) + drain_done = false; if (drain_done && list_empty(&ctx->defer_list)) ctx->int_flags &= ~IO_RING_F_DRAIN_ACTIVE; }