diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 464d9616234c..a5fd1845a1d5 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2348,6 +2348,7 @@ struct io_tctx_exit { struct callback_head task_work; struct completion completion; struct io_ring_ctx *ctx; + bool in_cancel; }; static __cold void io_tctx_exit_cb(struct callback_head *cb) @@ -2362,7 +2363,8 @@ static __cold void io_tctx_exit_cb(struct callback_head *cb) * tctx can be NULL if the queueing of this task_work raced with * work cancelation off the exec path. */ - if (tctx && !atomic_read(&tctx->in_cancel)) + work->in_cancel = tctx && atomic_read(&tctx->in_cancel); + if (tctx && !work->in_cancel) io_uring_del_tctx_node((unsigned long)work->ctx); complete(&work->completion); } @@ -2530,6 +2532,14 @@ static __cold void io_ring_exit_work(struct work_struct *work) * wait is marked as interruptible. */ wait_for_completion_interruptible(&exit.completion); + /* + * The task removes the node when it is done with its + * cancellation. Give it the time for that: it doesn't get back + * to it as long as it finds new task_work each time it has run + * the old one, and we may be one of many who send it. + */ + if (exit.in_cancel) + schedule_timeout_idle(HZ / 20); mutex_lock(&ctx->uring_lock); mutex_lock(&ctx->tctx_lock); }