github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-07 06:17:02 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

nvme-tcp: parallelize I/O queue allocation and startup

Browse Source 
Similar to commit 2a8513091d ("nvme-rdma: parallelize I/O queue
allocation and startup"), refactor nvme tcp I/O queue setup to use async
API, combining allocation and startup into a single parallel operation
per queue. This reduces connection and reconnection setup time when
there are delays in establishing connections, which is especially
important for high-core-count hosts.

Key changes:
- Use async API to facilitate parallel calls for io queue setup.
- Add nvme_tcp_queue_setup_ctx for propagating errors from async
  workers.
- Remove nvme_tcp_start_io_queues() and __nvme_tcp_alloc_io_queues();
  their logic is folded into nvme_tcp_setup_io_queues() and
  nvme_tcp_configure_io_queues().
- Allocate the io tag set before the queues so that the queue range is
  known, and only set up the reconnect grow case if the queue count
  actually increased.
- Serialize the cpu scan and claim in nvme_tcp_set_queue_io_cpu() with a
  spinlock, as concurrent callers would otherwise select the same cpu.
  The per-cpu counters no longer need to be atomics.

Testing on a 64-core host with 64 IO-queues shows nvme-tcp connection
time reduced from 61ms to 11ms.

Signed-off-by: Surabhi Gogte <sgogte@purestorage.com>
Reviewed-by: Hannes Reinecke <hare@kernel.org>
Signed-off-by: Keith Busch <kbusch@kernel.org>
This commit is contained in:
Surabhi Gogte 2026-09-22 14:07:18 -06:00 committed by Keith Busch
parent c58465c1e3
commit 61b0ed3be8
1 changed files with 77 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

119
drivers/nvme/host/tcp.c
View File

@ -7,6 +7,7 @@
#include <linux/module.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/async.h>
#include <linux/err.h>
#include <linux/crc32.h>
#include <linux/nvme-tcp.h>
@ -54,7 +55,8 @@ MODULE_PARM_DESC(tls_handshake_timeout,
"nvme TLS handshake timeout in seconds (default 10)");
#endif
static atomic_t nvme_tcp_cpu_queues[NR_CPUS];
static int nvme_tcp_cpu_queues[NR_CPUS];
static DEFINE_SPINLOCK(nvme_tcp_cpu_queues_lock);
enum nvme_tcp_send_state {
NVME_TCP_SEND_CMD_PDU = 0,
@ -154,6 +156,12 @@ struct nvme_tcp_queue {
static DEFINE_MUTEX(nvme_tcp_ctrl_mutex);
static LIST_HEAD_GUARDED(nvme_tcp_ctrl_list, nvme_tcp_ctrl_mutex);
struct nvme_tcp_queue_setup_ctx {
struct nvme_ctrl *ctrl;
int qid;
int *err;
};
struct nvme_tcp_ctrl {
/* read only in the hot path */
struct nvme_tcp_queue *queues;
@ -1731,9 +1739,10 @@ static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue)
goto out;
/* Search for the least used cpu from the mq_map */
spin_lock(&nvme_tcp_cpu_queues_lock);
io_cpu = WORK_CPU_UNBOUND;
for_each_online_cpu(cpu) {
int num_queues = atomic_read(&nvme_tcp_cpu_queues[cpu]);
int num_queues = nvme_tcp_cpu_queues[cpu];
if (mq_map[cpu] != qid)
continue;
@ -1744,9 +1753,10 @@ static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue)
}
if (io_cpu != WORK_CPU_UNBOUND) {
queue->io_cpu = io_cpu;
atomic_inc(&nvme_tcp_cpu_queues[io_cpu]);
nvme_tcp_cpu_queues[io_cpu]++;
set_bit(NVME_TCP_Q_IO_CPU_SET, &queue->flags);
}
spin_unlock(&nvme_tcp_cpu_queues_lock);
out:
dev_dbg(ctrl->ctrl.device, "queue %d: using cpu %d\n",
qid, queue->io_cpu);
@ -2023,8 +2033,11 @@ static void nvme_tcp_stop_queue_nowait(struct nvme_ctrl *nctrl, int qid)
if (!test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
return;
if (test_and_clear_bit(NVME_TCP_Q_IO_CPU_SET, &queue->flags))
atomic_dec(&nvme_tcp_cpu_queues[queue->io_cpu]);
if (test_and_clear_bit(NVME_TCP_Q_IO_CPU_SET, &queue->flags)) {
spin_lock(&nvme_tcp_cpu_queues_lock);
nvme_tcp_cpu_queues[queue->io_cpu]--;
spin_unlock(&nvme_tcp_cpu_queues_lock);
}
mutex_lock(&queue->queue_lock);
if (test_and_clear_bit(NVME_TCP_Q_LIVE, &queue->flags))
@ -2131,25 +2144,6 @@ static void nvme_tcp_stop_io_queues(struct nvme_ctrl *ctrl)
nvme_tcp_wait_queue(ctrl, i);
}
static int nvme_tcp_start_io_queues(struct nvme_ctrl *ctrl,
int first, int last)
{
int i, ret;
for (i = first; i < last; i++) {
ret = nvme_tcp_start_queue(ctrl, i);
if (ret)
goto out_stop_queues;
}
return 0;
out_stop_queues:
for (i--; i >= first; i--)
nvme_tcp_stop_queue(ctrl, i);
return ret;
}
static int nvme_tcp_alloc_admin_queue(struct nvme_ctrl *ctrl)
{
int ret;
@ -2210,22 +2204,67 @@ static int nvme_tcp_tls_check_psk(struct nvme_ctrl *ctrl)
return 0;
}
static int __nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
static void nvme_tcp_setup_queue_async(void *data, async_cookie_t cookie)
{
int i, ret;
struct nvme_tcp_queue_setup_ctx *ctx = data;
struct nvme_ctrl *ctrl = ctx->ctrl;
int ret;
for (i = 1; i < ctrl->queue_count; i++) {
ret = nvme_tcp_alloc_queue(ctrl, i,
ctrl->tls_pskid);
if (ret)
goto out_free_queues;
ret = nvme_tcp_alloc_queue(ctrl, ctx->qid, ctrl->tls_pskid);
if (ret)
goto out_err;
ret = nvme_tcp_start_queue(ctrl, ctx->qid);
if (ret)
goto out_err;
return;
out_err:
WRITE_ONCE(*ctx->err, ret);
}
static int nvme_tcp_setup_io_queues(struct nvme_ctrl *ctrl, unsigned int first,
unsigned int last)
{
ASYNC_DOMAIN_EXCLUSIVE(queue_domain);
struct nvme_tcp_queue_setup_ctx *ctxs;
int nr_queues = last - first;
int err = 0, i, ret;
if (first >= last)
return 0;
ctxs = kmalloc_objs(*ctxs, nr_queues);
if (!ctxs)
return -ENOMEM;
for (i = 0; i < nr_queues; i++) {
ctxs[i].ctrl = ctrl;
ctxs[i].qid = first + i;
ctxs[i].err = &err;
async_schedule_domain(nvme_tcp_setup_queue_async, &ctxs[i],
&queue_domain);
}
async_synchronize_full_domain(&queue_domain);
kfree(ctxs);
ret = READ_ONCE(err);
if (ret)
goto out_free_queues;
return 0;
out_free_queues:
for (i--; i >= 1; i--)
nvme_tcp_free_queue(ctrl, i);
for (i = first; i < last; i++) {
struct nvme_tcp_queue *queue = &to_tcp_ctrl(ctrl)->queues[i];
if (test_bit(NVME_TCP_Q_LIVE, &queue->flags))
nvme_tcp_stop_queue(ctrl, i);
if (test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
nvme_tcp_free_queue(ctrl, i);
}
return ret;
}
@ -2267,10 +2306,6 @@ static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new)
if (ret)
return ret;
ret = __nvme_tcp_alloc_io_queues(ctrl);
if (ret)
return ret;
if (new) {
ret = nvme_alloc_io_tag_set(ctrl, &to_tcp_ctrl(ctrl)->tag_set,
&nvme_tcp_mq_ops,
@ -2281,12 +2316,12 @@ static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new)
}
/*
* Only start IO queues for which we have allocated the tagset
* Only setup IO queues for which we have allocated the tagset
* and limited it to the available queues. On reconnects, the
* queue number might have changed.
*/
nr_queues = min(ctrl->tagset->nr_hw_queues + 1, ctrl->queue_count);
ret = nvme_tcp_start_io_queues(ctrl, 1, nr_queues);
ret = nvme_tcp_setup_io_queues(ctrl, 1, nr_queues);
if (ret)
goto out_cleanup_connect_q;
@ -2310,9 +2345,9 @@ static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new)
/*
* If the number of queues has increased (reconnect case)
* start all new queues now.
* setup all new queues now.
*/
ret = nvme_tcp_start_io_queues(ctrl, nr_queues,
ret = nvme_tcp_setup_io_queues(ctrl, nr_queues,
ctrl->tagset->nr_hw_queues + 1);
if (ret)
goto out_wait_freeze_timed_out;
@ -3157,7 +3192,7 @@ static int __init nvme_tcp_init_module(void)
return -ENOMEM;
for_each_possible_cpu(cpu)
atomic_set(&nvme_tcp_cpu_queues[cpu], 0);
nvme_tcp_cpu_queues[cpu] = 0;
nvmf_register_transport(&nvme_tcp_transport);
return 0;