diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 2fa720160e8f..e6569c204917 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -407,17 +407,46 @@ static void io_prep_async_link(struct io_kiocb *req) } } +/* Called for every request that is issued: keep it inline */ +static __always_inline bool io_assign_file(struct io_kiocb *req, + const struct io_issue_def *def, + unsigned int issue_flags) +{ + if (req->file || !def->needs_file) + return true; + + if (req->flags & REQ_F_FIXED_FILE) + req->file = io_file_get_fixed(req, req->cqe.fd, issue_flags); + else + req->file = io_file_get_normal(req, req->cqe.fd); + + return !!req->file; +} + void io_queue_iowq(struct io_kiocb *req) { struct io_uring_task *tctx = req->tctx; BUG_ON(!tctx); + lockdep_assert_held(&req->ctx->uring_lock); if ((current->flags & PF_KTHREAD) || !tctx->io_wq) { io_req_task_queue_fail(req, -ECANCELED); return; } + /* + * The file is looked up as late as possible, as it may be the result + * of an earlier request of the link. This one is up now, and what kind + * of work it is depends on the file. If that fails, leave the error + * handling to the worker. + * + * Not for a splice: whatever it writes to, it may wait for its input + * for ever. It has to remain unbound work that isn't hashed. + */ + if (req->opcode != IORING_OP_SPLICE) + io_assign_file(req, &io_issue_defs[req->opcode], 0); + /* init ->work of the whole link before punting */ io_prep_async_link(req); @@ -1397,20 +1426,6 @@ static __cold void io_drain_req(struct io_kiocb *req) ctx->int_flags &= ~IO_RING_F_DRAIN_ACTIVE; } -static bool io_assign_file(struct io_kiocb *req, const struct io_issue_def *def, - unsigned int issue_flags) -{ - if (req->file || !def->needs_file) - return true; - - if (req->flags & REQ_F_FIXED_FILE) - req->file = io_file_get_fixed(req, req->cqe.fd, issue_flags); - else - req->file = io_file_get_normal(req, req->cqe.fd); - - return !!req->file; -} - #define REQ_ISSUE_SLOW_FLAGS (REQ_F_CREDS | REQ_F_ARM_LTIMEOUT) static inline int __io_issue_sqe(struct io_kiocb *req,