diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 53941c6a6875..7c5d99a8f75d 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -519,6 +519,9 @@ struct io_ring_ctx { u32 iowq_limits[2]; struct callback_head poll_wq_task_work; + /* deferred ->poll_wq wake up for local task_work */ + struct rcu_head poll_wq_rcu; + atomic_t poll_wq_rcu_pending; struct list_head defer_list; unsigned nr_drained; diff --git a/io_uring/tw.c b/io_uring/tw.c index 98e3b1bc61b4..0b5066f97224 100644 --- a/io_uring/tw.c +++ b/io_uring/tw.c @@ -148,6 +148,37 @@ static void io_ctx_mark_taskrun(struct io_ring_ctx *ctx) } } +static void io_poll_wq_wake_rcu(struct rcu_head *rcu) +{ + struct io_ring_ctx *ctx = container_of(rcu, struct io_ring_ctx, + poll_wq_rcu); + + atomic_set(&ctx->poll_wq_rcu_pending, 0); + io_poll_wq_wake(ctx); + percpu_ref_put(&ctx->refs); +} + +/* + * io_uring_poll() reports pending local work as EPOLLIN, as it takes a trip to + * the kernel to turn it into CQEs. Let those know who are asleep already. + */ +static void io_local_work_wake_poll(struct io_ring_ctx *ctx, unsigned int flags) +{ + if (!(flags & IOU_F_TWQ_IN_WAKE)) { + io_poll_wq_wake(ctx); + return; + } + /* + * Called from a wake up handler with some wait queue lock held, this + * one could lead back to it. Do it from a clean context. + */ + if (!wq_has_sleeper(&ctx->poll_wq) || + atomic_xchg(&ctx->poll_wq_rcu_pending, 1)) + return; + percpu_ref_get(&ctx->refs); + call_rcu_hurry(&ctx->poll_wq_rcu, io_poll_wq_wake_rcu); +} + void io_req_local_work_add(struct io_kiocb *req, unsigned flags) { struct io_ring_ctx *ctx = req->ctx; @@ -174,6 +205,9 @@ void io_req_local_work_add(struct io_kiocb *req, unsigned flags) io_ctx_mark_taskrun(ctx); if (data_race(ctx->int_flags) & IO_RING_F_HAS_EVFD) io_eventfd_signal(ctx, false, flags & IOU_F_TWQ_IN_WAKE); + /* whoever sets the flag wakes ->poll_wq afterwards */ + if (data_race(ctx->int_flags) & IO_RING_F_POLL_ACTIVATED) + io_local_work_wake_poll(ctx, flags); } /*