diff --git a/io_uring/loop.c b/io_uring/loop.c index 3d472b7827fa..2340459164a1 100644 --- a/io_uring/loop.c +++ b/io_uring/loop.c @@ -10,12 +10,6 @@ static inline int io_loop_nr_cqes(const struct io_ring_ctx *ctx, return lp->cq_wait_idx - READ_ONCE(ctx->rings->cq.tail); } -static inline void io_loop_wait_start(struct io_ring_ctx *ctx, unsigned nr_wait) -{ - atomic_set(&ctx->cq_wait_nr, nr_wait); - set_current_state(TASK_INTERRUPTIBLE); -} - static inline void io_loop_wait_finish(struct io_ring_ctx *ctx) { __set_current_state(TASK_RUNNING); @@ -25,7 +19,7 @@ static inline void io_loop_wait_finish(struct io_ring_ctx *ctx) static void io_loop_wait(struct io_ring_ctx *ctx, struct iou_loop_params *lp, unsigned nr_wait) { - io_loop_wait_start(ctx, nr_wait); + io_cq_wait_arm(ctx, nr_wait); if (unlikely(io_local_work_pending(ctx) || io_loop_nr_cqes(ctx, lp) <= 0) || diff --git a/io_uring/tw.c b/io_uring/tw.c index f9affd68dd26..98e3b1bc61b4 100644 --- a/io_uring/tw.c +++ b/io_uring/tw.c @@ -165,7 +165,7 @@ void io_req_local_work_add(struct io_kiocb *req, unsigned flags) /* * The xchg() in mpscq_push() implies a full barrier, which pairs with - * the barrier in set_current_state() on the io_cqring_wait() side. This + * the barrier in io_cq_wait_arm() on the io_cqring_wait() side. This * ensures that either we see the updated ->cq_wait_nr, or waiters going * to sleep will observe the work added to the list, which is similar to * the wait/wake task state sync. diff --git a/io_uring/wait.c b/io_uring/wait.c index e03b40d9c141..5093365e9eb2 100644 --- a/io_uring/wait.c +++ b/io_uring/wait.c @@ -257,8 +257,7 @@ int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, u32 flags, unsigned long check_cq; if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) { - atomic_set(&ctx->cq_wait_nr, nr_wait); - set_current_state(TASK_INTERRUPTIBLE); + io_cq_wait_arm(ctx, nr_wait); } else { prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq, TASK_INTERRUPTIBLE); diff --git a/io_uring/wait.h b/io_uring/wait.h index b7b9c46b1b01..eb5ae8a6fc94 100644 --- a/io_uring/wait.h +++ b/io_uring/wait.h @@ -14,6 +14,21 @@ */ #define IO_CQ_WAKE_INIT (-1) +/* + * Start a wait cycle of the current task. The add that uses up ->cq_wait_nr + * may be one that queued its entry long ago and only now gets to look at the + * count. As nobody else wakes us after that, it has to find us sleeping, hence + * the task state must be visible before the count is. The barrier pairs with + * the xchg() in mpscq_push(), see io_req_local_work_add(). + */ +static inline void io_cq_wait_arm(struct io_ring_ctx *ctx, int nr_wait) +{ + __set_current_state(TASK_INTERRUPTIBLE); + atomic_set_release(&ctx->cq_wait_nr, nr_wait); + /* order the above against looking at the work list */ + smp_mb(); +} + struct ext_arg { size_t argsz; struct timespec64 ts;