From 99fd0cbae84988e0af34784aa18ddd51ae094ade Mon Sep 17 00:00:00 2001 From: Daniel Wagner Date: Wed, 14 Oct 2026 09:46:42 +0200 Subject: [PATCH] nvme-fabrics: limit each queue type to the number of online CPUs nvmf_nr_io_queues limits each queue type to the number of online CPUs when asking for I/O queues, but nvmf_set_io_queues hands out the granted queues without these limits, and only nr_io_queues is clamped when parsing the options. E.g. with 2 CPUs, nr_io_queues=2, nr_write_queues=10 and nr_poll_queues=10, 2 + 2 + 2 = 6 queues are requested, but handed out as 4 default, 2 read and 0 poll queues. blk_mq_map_queues can't create more CPU masks than there are CPUs, so default queues 2 and 3 take over the CPUs of default queues 0 and 1, and connecting the I/O queue of default queue 1 fails with -EXDEV. Apply the same limits in nvmf_set_io_queues. A queue type with more queues than online CPUs always has a hardware context without an online CPU, so no working setup changes. The options are not modified, so the limits follow CPU hotplug on reconnect. Fixes: 6486199378a5 ("nvme-tcp: fix queue mapping when queue count is limited") Fixes: a249d3066de6 ("nvme-fabrics: add queue setup helpers") Reviewed-by: Hannes Reinecke Tested-by: Maurizio Lombardi Signed-off-by: Daniel Wagner Signed-off-by: Keith Busch --- drivers/nvme/host/fabrics.c | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/drivers/nvme/host/fabrics.c b/drivers/nvme/host/fabrics.c index 571de549fbaf..27a8ce51d2b3 100644 --- a/drivers/nvme/host/fabrics.c +++ b/drivers/nvme/host/fabrics.c @@ -1125,18 +1125,27 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts, void nvmf_set_io_queues(struct nvmf_ctrl_options *opts, u32 nr_io_queues, u32 io_queues[HCTX_MAX_TYPES]) { + /* + * Each queue type is mapped to the CPUs separately, so a type with + * more queues than online CPUs has hardware contexts without an online + * CPU. Use the same limits as nvmf_nr_io_queues. + */ + u32 nr_io = min(opts->nr_io_queues, num_online_cpus()); + u32 nr_write = min(opts->nr_write_queues, num_online_cpus()); + u32 nr_poll = min(opts->nr_poll_queues, num_online_cpus()); + memset(io_queues, 0, sizeof(*io_queues) * HCTX_MAX_TYPES); - if (opts->nr_write_queues && opts->nr_io_queues < nr_io_queues) { + if (nr_write && nr_io < nr_io_queues) { /* * separate read/write queues * hand out dedicated default queues only after we have * sufficient read queues. */ - io_queues[HCTX_TYPE_READ] = opts->nr_io_queues; + io_queues[HCTX_TYPE_READ] = nr_io; nr_io_queues -= io_queues[HCTX_TYPE_READ]; io_queues[HCTX_TYPE_DEFAULT] = - min(opts->nr_write_queues, nr_io_queues); + min(nr_write, nr_io_queues); nr_io_queues -= io_queues[HCTX_TYPE_DEFAULT]; } else { /* @@ -1145,14 +1154,14 @@ void nvmf_set_io_queues(struct nvmf_ctrl_options *opts, u32 nr_io_queues, * sufficient queue count to have dedicated default queues. */ io_queues[HCTX_TYPE_DEFAULT] = - min(opts->nr_io_queues, nr_io_queues); + min(nr_io, nr_io_queues); nr_io_queues -= io_queues[HCTX_TYPE_DEFAULT]; } - if (opts->nr_poll_queues && nr_io_queues) { + if (nr_poll && nr_io_queues) { /* map dedicated poll queues only if we have queues left */ io_queues[HCTX_TYPE_POLL] = - min(opts->nr_poll_queues, nr_io_queues); + min(nr_poll, nr_io_queues); } } EXPORT_SYMBOL_GPL(nvmf_set_io_queues);