github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

io_uring: waitqueue-less cq waiting

Browse Source 
With DEFER_TASKRUN only ctx->submitter_task might be waiting for CQEs,
we can use this to optimise io_cqring_wait(). Replace ->cq_wait
waitqueue with waking the task directly.

It works but misses an important optimisation covered by the following
patch, so this patch without follow ups might hurt performance.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/103d174d35d919d4cb0922d8a9c93a8f0c35f74a.1673274244.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov 2023-01-09 14:46:11 +00:00 committed by Jens Axboe
parent 3181e22fb7
commit 130bd686d9
1 changed files with 12 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
io_uring/io_uring.c
View File

@ -1263,7 +1263,7 @@ static void io_req_local_work_add(struct io_kiocb *req)
percpu_ref_put(&ctx->refs); percpu_ref_put(&ctx->refs);
return; return;
} }
/* need it for the following io_cqring_wake() */ /* needed for the following wake up */
smp_mb__after_atomic(); smp_mb__after_atomic();
if (unlikely(atomic_read(&req->task->io_uring->in_idle))) { if (unlikely(atomic_read(&req->task->io_uring->in_idle))) {
@ -1274,10 +1274,9 @@ static void io_req_local_work_add(struct io_kiocb *req)
if (ctx->flags & IORING_SETUP_TASKRUN_FLAG) if (ctx->flags & IORING_SETUP_TASKRUN_FLAG)
atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags); atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
if (ctx->has_evfd) if (ctx->has_evfd)
io_eventfd_signal(ctx); io_eventfd_signal(ctx);
__io_cqring_wake(ctx); wake_up_state(ctx->submitter_task, TASK_INTERRUPTIBLE);
percpu_ref_put(&ctx->refs); percpu_ref_put(&ctx->refs);
} }
@ -2576,12 +2575,17 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
do { do {
unsigned long check_cq; unsigned long check_cq;
if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) {
set_current_state(TASK_INTERRUPTIBLE);
} else {
prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq, prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq,
TASK_INTERRUPTIBLE); TASK_INTERRUPTIBLE);
}
ret = io_cqring_wait_schedule(ctx, &iowq); ret = io_cqring_wait_schedule(ctx, &iowq);
__set_current_state(TASK_RUNNING);
if (ret < 0) if (ret < 0)
break; break;
__set_current_state(TASK_RUNNING);
/* /*
* Run task_work after scheduling and before io_should_wake(). * Run task_work after scheduling and before io_should_wake().
* If we got woken because of task_work being processed, run it * If we got woken because of task_work being processed, run it
@ -2609,6 +2613,7 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
cond_resched(); cond_resched();
} while (1); } while (1);
if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN))
finish_wait(&ctx->cq_wait, &iowq.wq); finish_wait(&ctx->cq_wait, &iowq.wq);
restore_saved_sigmask_unless(ret == -EINTR); restore_saved_sigmask_unless(ret == -EINTR);