github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

io_uring: post io-wq completions from the last request reference

Browse Source 
io-wq holds a reference on a request for the duration of the issue, as
the completion may run before the issue returns. If it does, the CQE is
posted from that completion while the worker still holds its reference,
and the file is only put once the worker drops it. That means the actual
file put happens potentially long after CQE posting.

Post the completion with the last put for refcounted requests.

io_free_req() no longer marks the request as CQE_SKIP, that is now done
by whoever posted the CQE while another reference was still held.

Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Jens Axboe 2026-09-09 12:58:41 -06:00
parent 0c1cf9f3a6
commit 5530a5102e
1 changed files with 30 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
io_uring/io_uring.c
View File

@ -964,9 +964,11 @@ static void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
goto defer_complete;
/*
* We don't free the request here because we know it's called from
* io-wq only, which holds a reference, so it cannot be the last put.
* Request not freed here because we know it's called from io-wq only,
* which holds a reference. Hence it can't be the last put. The CQE
* has been posted, last put frees it.
*/
req->flags |= REQ_F_CQE_SKIP;
req_ref_put(req);
}
@ -1020,8 +1022,6 @@ __cold void io_free_req(struct io_kiocb *req)
{
/* refs were already put, restore them for io_req_task_complete() */
req->flags &= ~REQ_F_REFCOUNT;
/* we only want to free it, don't post CQEs */
req->flags |= REQ_F_CQE_SKIP;
req->io_task_work.func = io_req_task_complete;
io_req_task_work_add(req);
}
@ -1124,6 +1124,8 @@ static void io_free_batch_list(struct io_ring_ctx *ctx,
}
if (req->flags & REQ_F_REFCOUNT) {
node = req->comp_list.next;
/* CQE posted, the last put only frees */
req->flags |= REQ_F_CQE_SKIP;
if (!req_ref_put_and_test(req))
continue;
}
@ -1287,7 +1289,23 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned int min_events)
void io_req_task_complete(struct io_tw_req tw_req, io_tw_token_t tw)
{
io_req_complete_defer(tw_req.req);
struct io_kiocb *req = tw_req.req;
/*
* io-wq may still hold a reference if the issue completed async.
* Defer completion to the last put, so that file drop and CQE
* visibility are ordered. The last reference stays for the free
* path to put, a linked timeout may still look at the request.
*/
if ((req->flags & REQ_F_REFCOUNT) && !(req->flags & REQ_F_REISSUE) &&
atomic_read(&req->refs) != 1) {
if (!req_ref_put_and_test(req))
return;
/* the other put raced us, ours was the last after all */
atomic_set(&req->refs, 1);
}
io_req_complete_defer(req);
}
/*
@ -1476,7 +1494,13 @@ struct io_wq_work *io_wq_free_work(struct io_wq_work *work)
struct io_kiocb *nxt = NULL;
if (req_ref_put_and_test_atomic(req)) {
if (req->flags & IO_REQ_LINK_FLAGS) {
/*
* Only continue the link from here if the completion has been
* posted. If it's still pending, io_free_req() completes the
* request and the next link follows from there, after the CQE.
*/
if ((req->flags & IO_REQ_LINK_FLAGS) &&
(req->flags & REQ_F_CQE_SKIP)) {
struct io_ring_ctx *ctx = req->ctx;
mutex_lock(&ctx->uring_lock);