diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c index 433c003a83e5..a5b3d2ba48bf 100644 --- a/io_uring/io-wq.c +++ b/io_uring/io-wq.c @@ -646,12 +646,13 @@ static void io_worker_handle_work(struct io_wq_acct *acct, /* handle a whole dependent link */ do { bool do_kill = test_bit(IO_WQ_BIT_EXIT, &wq->state); - struct io_wq_work *next_hashed, *linked; + struct io_wq_work *next_hashed, *linked, *done = work; unsigned int work_flags = atomic_read(&work->flags); unsigned int hash = __io_wq_is_hashed(work_flags) ? __io_get_work_hash(work_flags) : -1U; struct io_kiocb *req; + bool last; next_hashed = wq_next_work(work); @@ -670,7 +671,7 @@ static void io_worker_handle_work(struct io_wq_acct *acct, kcov_remote_stop(); io_assign_current_work(worker, NULL); - linked = io_wq_free_work(work); + last = io_wq_put_work(work, &linked); work = next_hashed; if (!work && linked && !io_wq_is_hashed(linked)) { work = linked; @@ -679,6 +680,13 @@ static void io_worker_handle_work(struct io_wq_acct *acct, io_assign_current_work(worker, work); if (linked) io_wq_enqueue(wq, linked); + /* + * Not before the link is current or pending work again. + * This may be the last time that a task is woken up + * which exits and looks for requests to cancel. + */ + if (last) + io_wq_free_work(done); if (hash != -1U && !next_hashed) { /* serialize hash clear with wake_up() */ @@ -1008,9 +1016,13 @@ void io_wq_set_exit_on_idle(struct io_wq *wq, bool enable) static void io_run_cancel(struct io_wq_work *work, struct io_wq *wq) { do { + struct io_wq_work *linked; + atomic_or(IO_WQ_WORK_CANCEL, &work->flags); io_wq_submit_work(work); - work = io_wq_free_work(work); + if (io_wq_put_work(work, &linked)) + io_wq_free_work(work); + work = linked; } while (work); } diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 0c91d466d825..464d9616234c 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1486,28 +1486,40 @@ int io_poll_issue(struct io_kiocb *req, io_tw_token_t tw) return ret; } -struct io_wq_work *io_wq_free_work(struct io_wq_work *work) +/* + * io-wq is done with @work. Returns true if it held the last reference, and + * hands out the next request of the link then. The caller has to make sure + * that cancellations can find the latter before it calls io_wq_free_work(). + */ +bool io_wq_put_work(struct io_wq_work *work, struct io_wq_work **linked) { struct io_kiocb *req = container_of(work, struct io_kiocb, work); struct io_kiocb *nxt = NULL; - if (req_ref_put_and_test_atomic(req)) { - /* - * Only continue the link from here if the completion has been - * posted. If it's still pending, io_free_req() completes the - * request and the next link follows from there, after the CQE. - */ - if ((req->flags & IO_REQ_LINK_FLAGS) && - (req->flags & REQ_F_CQE_SKIP)) { - struct io_ring_ctx *ctx = req->ctx; + *linked = NULL; + if (!req_ref_put_and_test_atomic(req)) + return false; - mutex_lock(&ctx->uring_lock); - nxt = io_req_find_next(req); - mutex_unlock(&ctx->uring_lock); - } - io_free_req(req); + /* + * Only continue the link from here if the completion has been posted. + * If it's still pending, io_free_req() completes the request and the + * next link follows from there, after the CQE. + */ + if ((req->flags & IO_REQ_LINK_FLAGS) && (req->flags & REQ_F_CQE_SKIP)) { + struct io_ring_ctx *ctx = req->ctx; + + mutex_lock(&ctx->uring_lock); + nxt = io_req_find_next(req); + mutex_unlock(&ctx->uring_lock); } - return nxt ? &nxt->work : NULL; + if (nxt) + *linked = &nxt->work; + return true; +} + +void io_wq_free_work(struct io_wq_work *work) +{ + io_free_req(container_of(work, struct io_kiocb, work)); } void io_wq_submit_work(struct io_wq_work *work) @@ -1518,7 +1530,7 @@ void io_wq_submit_work(struct io_wq_work *work) bool needs_poll = false; int ret = 0, err = -ECANCELED; - /* one will be dropped by io_wq_free_work() after returning to io-wq */ + /* one will be dropped by io_wq_put_work() after returning to io-wq */ if (!(req->flags & REQ_F_REFCOUNT)) __io_req_set_refcount(req, 2); else diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index 109261722ca7..c7b340e5d0a8 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -204,7 +204,8 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin); __cold void io_iopoll_try_reap_events(struct io_ring_ctx *ctx); void __io_submit_flush_completions(struct io_ring_ctx *ctx); -struct io_wq_work *io_wq_free_work(struct io_wq_work *work); +bool io_wq_put_work(struct io_wq_work *work, struct io_wq_work **linked); +void io_wq_free_work(struct io_wq_work *work); void io_wq_submit_work(struct io_wq_work *work); void io_free_req(struct io_kiocb *req);