From 6fe0687245e8406bf26143bd45eb16441bbe5280 Mon Sep 17 00:00:00 2001 From: Nick Chan Date: Sun, 7 Jun 2026 14:10:58 +0800 Subject: [PATCH 01/45] nvme-apple: Prevent shared tags across queues on Apple A11 On Apple A11, tags of pending commands must be unique across the admin and IO queues, else the firmware crashes with "duplicate tag error for tag N", with N being the tag. Apply the existing workaround for M1 of reserving two tags for the admin queue to A11. Cc: stable@vger.kernel.org Fixes: 04d8ecf37b5e ("nvme: apple: Add Apple A11 support") Reviewed-by: Sven Peter Signed-off-by: Nick Chan Signed-off-by: Keith Busch --- drivers/nvme/host/apple.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index de0d5126458f..79e1fe2a23f9 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -225,7 +225,7 @@ static unsigned int apple_nvme_queue_depth(struct apple_nvme_queue *q) { struct apple_nvme *anv = queue_to_apple_nvme(q); - if (q->is_adminq && anv->hw->has_lsq_nvmmu) + if (q->is_adminq) return APPLE_NVME_AQ_DEPTH; return anv->hw->max_queue_depth; @@ -303,7 +303,7 @@ static void apple_nvme_submit_cmd_t8015(struct apple_nvme_queue *q, memcpy((void *)q->sqes + (q->sq_tail << APPLE_NVME_IOSQES), cmd, sizeof(*cmd)); - if (++q->sq_tail == anv->hw->max_queue_depth) + if (++q->sq_tail == apple_nvme_queue_depth(q)) q->sq_tail = 0; writel(q->sq_tail, q->sq_db); @@ -1138,10 +1138,7 @@ static void apple_nvme_reset_work(struct work_struct *work) } /* Setup the admin queue */ - if (anv->hw->has_lsq_nvmmu) - aqa = APPLE_NVME_AQ_DEPTH - 1; - else - aqa = anv->hw->max_queue_depth - 1; + aqa = APPLE_NVME_AQ_DEPTH - 1; aqa |= aqa << 16; writel(aqa, anv->mmio_nvme + NVME_REG_AQA); writeq(anv->adminq.sq_dma_addr, anv->mmio_nvme + NVME_REG_ASQ); @@ -1324,8 +1321,7 @@ static int apple_nvme_alloc_tagsets(struct apple_nvme *anv) * both queues. The admin queue gets the first APPLE_NVME_AQ_DEPTH which * must be marked as reserved in the IO queue. */ - if (anv->hw->has_lsq_nvmmu) - anv->tagset.reserved_tags = APPLE_NVME_AQ_DEPTH; + anv->tagset.reserved_tags = APPLE_NVME_AQ_DEPTH; anv->tagset.queue_depth = anv->hw->max_queue_depth - 1; anv->tagset.timeout = NVME_IO_TIMEOUT; anv->tagset.numa_node = NUMA_NO_NODE; From 92f58587a04c94985fd4a9e3575720b054c432bf Mon Sep 17 00:00:00 2001 From: John Garry Date: Thu, 4 Jun 2026 14:58:40 +0000 Subject: [PATCH 02/45] nvme: quieten sparse warning in valid LBA size check Currently building with C=1 generates the following warning: CC drivers/nvme/host/core.o CHECK drivers/nvme/host/core.c drivers/nvme/host/core.c:2426:13: warning: unsigned value that used to be signed checked against zero? drivers/nvme/host/core.c:2426:13: signed value source This issue was introduced when using check_shl_overflow() to check for invalid LBA size. Sparse is having trouble dealing with __bitwise __le64 conversion when passing to check_shl_overflow(). Resolve the issue by moving the check_shl_overflow() call to a separate function, where types are not converted. The id->lbaf[lbaf].ds < SECTOR_SHIFT check is dropped as check_shl_overflow() is able to detect negative shifts. Reviewed-by: Christoph Hellwig Signed-off-by: John Garry Signed-off-by: Keith Busch --- drivers/nvme/host/core.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index efaddab8296e..d37fc70fe48a 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -2379,6 +2379,11 @@ static int nvme_query_fdp_info(struct nvme_ns *ns, struct nvme_ns_info *info) return ret; } +static bool nvme_invalid_lba_sz(u64 nsze, signed int shift, sector_t *capacity) +{ + return check_shl_overflow(nsze, shift, capacity); +} + static int nvme_update_ns_info_block(struct nvme_ns *ns, struct nvme_ns_info *info) { @@ -2422,10 +2427,8 @@ static int nvme_update_ns_info_block(struct nvme_ns *ns, goto out; } - if (id->lbaf[lbaf].ds < SECTOR_SHIFT || - check_shl_overflow(le64_to_cpu(id->nsze), - id->lbaf[lbaf].ds - SECTOR_SHIFT, - &capacity)) { + if (nvme_invalid_lba_sz(le64_to_cpu(id->nsze), + id->lbaf[lbaf].ds - SECTOR_SHIFT, &capacity)) { dev_warn_once(ns->ctrl->device, "invalid LBA data size %u, skipping namespace\n", id->lbaf[lbaf].ds); From 34b9a83c50660148bde01cde16451dbe78369749 Mon Sep 17 00:00:00 2001 From: Wentao Liang Date: Tue, 9 Jun 2026 09:55:05 +0000 Subject: [PATCH 03/45] nvmet: fix refcount leak in nvmet_sq_create() In nvmet_sq_create(), a reference on the ctrl is taken via kref_get_unless_zero() before calling nvmet_check_sqid(). If nvmet_check_sqid() fails, the function returns the error directly without releasing the reference, leading to a leak. Fix this by jumping to the "ctrl_put" label, which already performs the necessary nvmet_ctrl_put(ctrl). This ensures the reference is properly released on this error path. Cc: stable@vger.kernel.org Fixes: 1eb380caf527 ("nvmet: Introduce nvmet_sq_create() and nvmet_cq_create()") Signed-off-by: Wentao Liang Signed-off-by: Keith Busch --- drivers/nvme/target/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index 62dd59b9aa4f..4477c4d6b1ee 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -944,7 +944,7 @@ u16 nvmet_sq_create(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq, status = nvmet_check_sqid(ctrl, sqid, true); if (status != NVME_SC_SUCCESS) - return status; + goto ctrl_put; ret = nvmet_sq_init(sq, cq); if (ret) { From 26acdaa357cded33a37f575cd5f6bae1033b3a5d Mon Sep 17 00:00:00 2001 From: Maurizio Lombardi Date: Mon, 8 Jun 2026 17:53:57 +0200 Subject: [PATCH 04/45] nvme: fix crash and memory leak during invalid cdev teardown In the NVMe multipath code, if nvme_add_ns_head_cdev() fails during nvme_mpath_set_live(), the error is ignored. However, during teardown, nvme_remove_head() unconditionally calls nvme_cdev_del(). This teardown asymmetry leads to a kernel panic if the character device was never successfully initialized. BUG: kernel NULL pointer dereference, address: 00000000000000d0 device_del+0x39/0x3c0 cdev_device_del+0x15/0x50 nvme_cdev_del+0xe/0x20 [nvme_core] nvme_mpath_shutdown_disk+0x38/0x60 [nvme_core] nvme_ns_remove+0x177/0x1f0 [nvme_core] nvme_remove_namespaces+0xdc/0x130 [nvme_core] nvme_do_delete_ctrl+0x71/0xd0 [nvme_core] Additionally, a memory leak exists in the nvme_cdev_add() failure path. Previously, dev_set_name() was called before ida_alloc(). If ida_alloc() subsequently failed, device_initialize() was never called, meaning put_device() could not be used to clean up the kobject, leaking the memory allocated by dev_set_name(). * Introduces the NVME_NSHEAD_CDEV_LIVE and NVME_NS_CDEV_LIVE bits to track the successful creation of the character devices. Teardown routines now check these bits before attempting deletion. * Refactor nvme_cdev_add() to accept the formatted device name as a parameter, moving dev_set_name() after the IDA allocation and immediately before device_initialize(). This ensures any internally allocated strings are safely cleaned up by put_device() upon failure. Signed-off-by: Maurizio Lombardi Signed-off-by: Keith Busch --- drivers/nvme/host/core.c | 33 ++++++++++++++++++++++++--------- drivers/nvme/host/multipath.c | 19 +++++++++++++------ drivers/nvme/host/nvme.h | 5 ++++- 3 files changed, 41 insertions(+), 16 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index d37fc70fe48a..c6930e43cfea 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -3894,7 +3894,8 @@ void nvme_cdev_del(struct cdev *cdev, struct device *cdev_device) put_device(cdev_device); } -int nvme_cdev_add(struct cdev *cdev, struct device *cdev_device, +int nvme_cdev_add(const char *name, struct cdev *cdev, + struct device *cdev_device, const struct file_operations *fops, struct module *owner) { int minor, ret; @@ -3902,6 +3903,12 @@ int nvme_cdev_add(struct cdev *cdev, struct device *cdev_device, minor = ida_alloc(&nvme_ns_chr_minor_ida, GFP_KERNEL); if (minor < 0) return minor; + + ret = dev_set_name(cdev_device, name); + if (ret) { + ida_free(&nvme_ns_chr_minor_ida, minor); + return ret; + } cdev_device->devt = MKDEV(MAJOR(nvme_ns_chr_devt), minor); cdev_device->class = &nvme_ns_chr_class; cdev_device->release = nvme_cdev_rel; @@ -3939,15 +3946,21 @@ static const struct file_operations nvme_ns_chr_fops = { static int nvme_add_ns_cdev(struct nvme_ns *ns) { int ret; + char name[32]; ns->cdev_device.parent = ns->ctrl->device; - ret = dev_set_name(&ns->cdev_device, "ng%dn%d", - ns->ctrl->instance, ns->head->instance); - if (ret) - return ret; + snprintf(name, sizeof(name), "ng%dn%d", ns->ctrl->instance, + ns->head->instance); - return nvme_cdev_add(&ns->cdev, &ns->cdev_device, &nvme_ns_chr_fops, - ns->ctrl->ops->module); + ret = nvme_cdev_add(name, &ns->cdev, &ns->cdev_device, + &nvme_ns_chr_fops, ns->ctrl->ops->module); + if (ret) { + dev_err(ns->ctrl->device, "Unable to create the %s device\n", + name); + } else { + set_bit(NVME_NS_CDEV_LIVE, &ns->flags); + } + return ret; } static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl, @@ -4323,8 +4336,10 @@ static void nvme_ns_remove(struct nvme_ns *ns) /* guarantee not available in head->list */ synchronize_srcu(&ns->head->srcu); - if (!nvme_ns_head_multipath(ns->head)) - nvme_cdev_del(&ns->cdev, &ns->cdev_device); + if (!nvme_ns_head_multipath(ns->head)) { + if (test_and_clear_bit(NVME_NS_CDEV_LIVE, &ns->flags)) + nvme_cdev_del(&ns->cdev, &ns->cdev_device); + } nvme_mpath_remove_sysfs_link(ns); diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index e033ede953cc..9cd49e2f760d 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -642,14 +642,20 @@ static const struct file_operations nvme_ns_head_chr_fops = { static int nvme_add_ns_head_cdev(struct nvme_ns_head *head) { int ret; + char name[32]; head->cdev_device.parent = &head->subsys->dev; - ret = dev_set_name(&head->cdev_device, "ng%dn%d", - head->subsys->instance, head->instance); - if (ret) - return ret; - ret = nvme_cdev_add(&head->cdev, &head->cdev_device, + snprintf(name, sizeof(name), "ng%dn%d", head->subsys->instance, + head->instance); + + ret = nvme_cdev_add(name, &head->cdev, &head->cdev_device, &nvme_ns_head_chr_fops, THIS_MODULE); + if (ret) { + dev_err(disk_to_dev(head->disk), + "Unable to create the %s device\n", name); + } else { + set_bit(NVME_NSHEAD_CDEV_LIVE, &head->flags); + } return ret; } @@ -694,7 +700,8 @@ static void nvme_remove_head(struct nvme_ns_head *head) */ kblockd_schedule_work(&head->requeue_work); - nvme_cdev_del(&head->cdev, &head->cdev_device); + if (test_and_clear_bit(NVME_NSHEAD_CDEV_LIVE, &head->flags)) + nvme_cdev_del(&head->cdev, &head->cdev_device); synchronize_srcu(&head->srcu); del_gendisk(head->disk); } diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index b367c67dcb37..824651cc898d 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -573,6 +573,7 @@ struct nvme_ns_head { atomic_long_t io_fail_no_available_path_count; #define NVME_NSHEAD_DISK_LIVE 0 #define NVME_NSHEAD_QUEUE_IF_NO_PATH 1 +#define NVME_NSHEAD_CDEV_LIVE 2 struct nvme_ns __rcu *current_path[]; #endif }; @@ -611,6 +612,7 @@ struct nvme_ns { #define NVME_NS_FORCE_RO 3 #define NVME_NS_READY 4 #define NVME_NS_SYSFS_ATTR_LINK 5 +#define NVME_NS_CDEV_LIVE 6 struct cdev cdev; struct device cdev_device; @@ -995,7 +997,8 @@ int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi, void *log, size_t size, u64 offset); bool nvme_tryget_ns_head(struct nvme_ns_head *head); void nvme_put_ns_head(struct nvme_ns_head *head); -int nvme_cdev_add(struct cdev *cdev, struct device *cdev_device, +int nvme_cdev_add(const char *name, struct cdev *cdev, + struct device *cdev_device, const struct file_operations *fops, struct module *owner); void nvme_cdev_del(struct cdev *cdev, struct device *cdev_device); int nvme_ioctl(struct block_device *bdev, blk_mode_t mode, From 4fd1f5f6a659886a4ef3a380b2a07207c94a7a24 Mon Sep 17 00:00:00 2001 From: Rosen Penev Date: Sun, 7 Jun 2026 22:12:23 -0700 Subject: [PATCH 05/45] nvme: target: allocate ana_state with port Use a flexible array member to remove one allocation. Simplifies code slightly. Signed-off-by: Rosen Penev Signed-off-by: Keith Busch --- drivers/nvme/target/configfs.c | 9 +-------- drivers/nvme/target/nvmet.h | 2 +- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index b88f897f06e2..2b69ffcfc8df 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -2007,7 +2007,6 @@ static void nvmet_port_release(struct config_item *item) list_del(&port->global_entry); key_put(port->keyring); - kfree(port->ana_state); kfree(port); } @@ -2047,16 +2046,10 @@ static struct config_group *nvmet_ports_make(struct config_group *group, if (kstrtou16(name, 0, &portid)) return ERR_PTR(-EINVAL); - port = kzalloc_obj(*port); + port = kzalloc_flex(*port, ana_state, NVMET_MAX_ANAGRPS + 1); if (!port) return ERR_PTR(-ENOMEM); - port->ana_state = kzalloc_objs(*port->ana_state, NVMET_MAX_ANAGRPS + 1); - if (!port->ana_state) { - kfree(port); - return ERR_PTR(-ENOMEM); - } - if (IS_ENABLED(CONFIG_NVME_TARGET_TCP_TLS) && nvme_keyring_id()) { port->keyring = key_lookup(nvme_keyring_id()); if (IS_ERR(port->keyring)) { diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index 3305a88684ec..aaba745e3c21 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -208,7 +208,6 @@ struct nvmet_port { struct list_head global_entry; struct config_group ana_groups_group; struct nvmet_ana_group ana_default_group; - enum nvme_ana_state *ana_state; struct key *keyring; void *priv; bool enabled; @@ -217,6 +216,7 @@ struct nvmet_port { int mdts; const struct nvmet_fabrics_ops *tr_ops; bool pi_enable; + enum nvme_ana_state ana_state[]; }; static inline struct nvmet_port *to_nvmet_port(struct config_item *item) From 48c0162f647bb47e6084ffbc71b8f213f5e2f4f8 Mon Sep 17 00:00:00 2001 From: Bryam Vargas Date: Thu, 4 Jun 2026 19:36:54 +0000 Subject: [PATCH 06/45] nvmet-rdma: handle inline data with a nonzero offset nvmet_rdma_use_inline_sg() maps the host-controlled inline data offset into the per-command inline scatterlist. The bounds check admits any offset with off + len <= inline_data_size, but the mapping still assumes the data begins in the first inline page: sg->offset = off; sg->length = min_t(int, len, PAGE_SIZE - off); When a port is configured with inline_data_size > PAGE_SIZE (settable up to max(SZ_16K, PAGE_SIZE)), an offset in (PAGE_SIZE, inline_data_size] makes "PAGE_SIZE - off" underflow, so sg->length is set to ~4 GiB and the block backend reads far past the first inline page. num_pages(len) also ignores the offset, so an in-bounds offset whose [off, off+len) span crosses a page boundary under-counts the scatterlist. Map the offset properly: split it into a page index and an in-page offset, start the scatterlist at that page, and size the page count from page_off + len. Because the request scatterlist may now start at inline_sg[page_idx] rather than inline_sg[0], generalize the inline-SGL identity test in nvmet_rdma_release_rsp() to a range test; otherwise the persistent inline scatterlist is mistaken for an allocated one and nvmet_req_free_sgls() frees an inline page (and warns in free_large_kmalloc()). Fixes: 0d5ee2b2ab4f ("nvmet-rdma: support max(16KB, PAGE_SIZE) inline data") Cc: stable@vger.kernel.org Suggested-by: Keith Busch Reported-by: Bryam Vargas Signed-off-by: Bryam Vargas Signed-off-by: Keith Busch --- drivers/nvme/target/rdma.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c index ac26f4f774c4..ea1185b8267e 100644 --- a/drivers/nvme/target/rdma.c +++ b/drivers/nvme/target/rdma.c @@ -666,7 +666,8 @@ static void nvmet_rdma_release_rsp(struct nvmet_rdma_rsp *rsp) if (rsp->n_rdma) nvmet_rdma_rw_ctx_destroy(rsp); - if (rsp->req.sg != rsp->cmd->inline_sg) + if (rsp->req.sg < rsp->cmd->inline_sg || + rsp->req.sg >= rsp->cmd->inline_sg + queue->dev->inline_page_count) nvmet_req_free_sgls(&rsp->req); if (unlikely(!list_empty_careful(&queue->rsp_wr_wait_list))) @@ -821,24 +822,25 @@ static void nvmet_rdma_write_data_done(struct ib_cq *cq, struct ib_wc *wc) static void nvmet_rdma_use_inline_sg(struct nvmet_rdma_rsp *rsp, u32 len, u64 off) { - int sg_count = num_pages(len); + u64 page_off = off % PAGE_SIZE; + u64 page_idx = off / PAGE_SIZE; + int sg_count = num_pages(page_off + len); struct scatterlist *sg; int i; - sg = rsp->cmd->inline_sg; + sg = &rsp->cmd->inline_sg[page_idx]; for (i = 0; i < sg_count; i++, sg++) { if (i < sg_count - 1) sg_unmark_end(sg); else sg_mark_end(sg); - sg->offset = off; - sg->length = min_t(int, len, PAGE_SIZE - off); + sg->offset = page_off; + sg->length = min_t(u64, len, PAGE_SIZE - page_off); len -= sg->length; - if (!i) - off = 0; + page_off = 0; } - rsp->req.sg = rsp->cmd->inline_sg; + rsp->req.sg = &rsp->cmd->inline_sg[page_idx]; rsp->req.sg_cnt = sg_count; } From ac48c49116d3de84fabc224c6e43f08740b1460d Mon Sep 17 00:00:00 2001 From: John Garry Date: Wed, 10 Jun 2026 08:53:32 +0000 Subject: [PATCH 07/45] nvme: make some sysfs diagnostic structures static Building with C=1 generates the following warnings: drivers/nvme/host/sysfs.c:397:25: warning: symbol 'dev_attr_io_errors' was not declared. Should it be static? drivers/nvme/host/sysfs.c:444:30: warning: symbol 'nvme_ns_diag_attr_group' was not declared. Should it be static? drivers/nvme/host/sysfs.c:1150:25: warning: symbol 'dev_attr_adm_errors' was not declared. Should it be static? Make those structures static. Closes: https://lore.kernel.org/oe-kbuild-all/202606101329.T3zXNqdy-lkp@intel.com/ Reviewed-by: Nilay Shroff Reviewed-by: Christoph Hellwig Signed-off-by: John Garry Signed-off-by: Keith Busch --- drivers/nvme/host/sysfs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/nvme/host/sysfs.c b/drivers/nvme/host/sysfs.c index 933a5adfb7af..75b2d69b5957 100644 --- a/drivers/nvme/host/sysfs.c +++ b/drivers/nvme/host/sysfs.c @@ -394,7 +394,7 @@ static ssize_t nvme_io_errors_store(struct device *dev, return count; } -struct device_attribute dev_attr_io_errors = +static struct device_attribute dev_attr_io_errors = __ATTR(command_error_count, 0644, nvme_io_errors_show, nvme_io_errors_store); @@ -441,7 +441,7 @@ static umode_t nvme_ns_diag_attrs_are_visible(struct kobject *kobj, return a->mode; } -const struct attribute_group nvme_ns_diag_attr_group = { +static const struct attribute_group nvme_ns_diag_attr_group = { .name = "diag", .attrs = nvme_ns_diag_attrs, .is_visible = nvme_ns_diag_attrs_are_visible, @@ -1147,7 +1147,7 @@ static ssize_t nvme_adm_errors_store(struct device *dev, return count; } -struct device_attribute dev_attr_adm_errors = +static struct device_attribute dev_attr_adm_errors = __ATTR(command_error_count, 0644, nvme_adm_errors_show, nvme_adm_errors_store); From 869567bcbe2dcc790860e05fc0e0c5e415bb22c2 Mon Sep 17 00:00:00 2001 From: John Garry Date: Wed, 10 Jun 2026 11:16:08 +0000 Subject: [PATCH 08/45] nvme: make nvme_add_ns{_head}_cdev return void The return code from nvme_add_ns_head_cdev() and nvme_add_ns_cdev() is never checked, so make those functions return void. A cdev add failure is tolerated during initialization, and flags NVME_NS_CDEV_LIVE and NVME_NSHEAD_CDEV_LIVE are for determining whether a cdev needs to be deleted during un-initialization. Reviewed-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Signed-off-by: John Garry Signed-off-by: Keith Busch --- drivers/nvme/host/core.c | 13 +++++-------- drivers/nvme/host/multipath.c | 13 +++++-------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index c6930e43cfea..eb268148acec 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -3943,24 +3943,21 @@ static const struct file_operations nvme_ns_chr_fops = { .uring_cmd_iopoll = nvme_ns_chr_uring_cmd_iopoll, }; -static int nvme_add_ns_cdev(struct nvme_ns *ns) +static void nvme_add_ns_cdev(struct nvme_ns *ns) { - int ret; char name[32]; ns->cdev_device.parent = ns->ctrl->device; snprintf(name, sizeof(name), "ng%dn%d", ns->ctrl->instance, ns->head->instance); - ret = nvme_cdev_add(name, &ns->cdev, &ns->cdev_device, - &nvme_ns_chr_fops, ns->ctrl->ops->module); - if (ret) { + if (nvme_cdev_add(name, &ns->cdev, &ns->cdev_device, + &nvme_ns_chr_fops, ns->ctrl->ops->module)) { dev_err(ns->ctrl->device, "Unable to create the %s device\n", name); - } else { - set_bit(NVME_NS_CDEV_LIVE, &ns->flags); + return; } - return ret; + set_bit(NVME_NS_CDEV_LIVE, &ns->flags); } static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl, diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index 9cd49e2f760d..9b9a657fa330 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -639,24 +639,21 @@ static const struct file_operations nvme_ns_head_chr_fops = { .uring_cmd_iopoll = nvme_ns_chr_uring_cmd_iopoll, }; -static int nvme_add_ns_head_cdev(struct nvme_ns_head *head) +static void nvme_add_ns_head_cdev(struct nvme_ns_head *head) { - int ret; char name[32]; head->cdev_device.parent = &head->subsys->dev; snprintf(name, sizeof(name), "ng%dn%d", head->subsys->instance, head->instance); - ret = nvme_cdev_add(name, &head->cdev, &head->cdev_device, - &nvme_ns_head_chr_fops, THIS_MODULE); - if (ret) { + if (nvme_cdev_add(name, &head->cdev, &head->cdev_device, + &nvme_ns_head_chr_fops, THIS_MODULE)) { dev_err(disk_to_dev(head->disk), "Unable to create the %s device\n", name); - } else { - set_bit(NVME_NSHEAD_CDEV_LIVE, &head->flags); + return; } - return ret; + set_bit(NVME_NSHEAD_CDEV_LIVE, &head->flags); } static void nvme_partition_scan_work(struct work_struct *work) From ee38469f88492df99e1d97f03aa40ecfd218934f Mon Sep 17 00:00:00 2001 From: Mohamed Khalfella Date: Thu, 28 May 2026 11:27:34 +0200 Subject: [PATCH 09/45] nvme-fc: Do not cancel requests in io target before it is initialized A new nvme-fc controller in CONNECTING state sees admin request timeout schedules ctrl->ioerr_work to abort inflight requests. This ends up calling __nvme_fc_abort_outstanding_ios() which aborts requests in both admin and io tagsets. In case fc_ctrl->tag_set was not initialized we see the warning below. This is because ctrl.queue_count is initialized early in nvme_fc_alloc_ctrl(). nvme nvme0: NVME-FC{0}: starting error recovery Connectivity Loss INFO: trying to register non-static key. The code is fine but needs lockdep annotation, or maybe lpfc 0000:ab:00.0: queue 0 connect admin queue failed (-6). you didn't initialize this object before use? turning off the locking correctness validator. Workqueue: nvme-reset-wq nvme_fc_ctrl_ioerr_work [nvme_fc] Call Trace: dump_stack_lvl+0x57/0x80 register_lock_class+0x567/0x580 __lock_acquire+0x330/0xb90 lock_acquire.part.0+0xad/0x210 blk_mq_tagset_busy_iter+0xf9/0xc00 __nvme_fc_abort_outstanding_ios+0x23f/0x320 [nvme_fc] nvme_fc_ctrl_ioerr_work+0x172/0x210 [nvme_fc] process_one_work+0x82c/0x1450 worker_thread+0x5ee/0xfd0 kthread+0x3a0/0x750 ret_from_fork+0x439/0x670 ret_from_fork_asm+0x1a/0x30 Update the check in __nvme_fc_abort_outstanding_ios() confirm that io tagset was created before iterating over busy requests. Also make sure to cancel ctrl->ioerr_work before removing io tagset. Reviewed-by: Randy Jennings Reviewed-by: Hannes Reinecke Reviewed-by: Daniel Wagner Reviewed-by: Christoph Hellwig Signed-off-by: Mohamed Khalfella Signed-off-by: James Smart Signed-off-by: Maurizio Lombardi Signed-off-by: Keith Busch --- drivers/nvme/host/fc.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index 2c9a6d3c9797..04363b9c4489 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -2461,7 +2461,7 @@ __nvme_fc_abort_outstanding_ios(struct nvme_fc_ctrl *ctrl, bool start_queues) * io requests back to the block layer as part of normal completions * (but with error status). */ - if (ctrl->ctrl.queue_count > 1) { + if (ctrl->ctrl.queue_count > 1 && ctrl->ctrl.tagset) { nvme_quiesce_io_queues(&ctrl->ctrl); nvme_sync_io_queues(&ctrl->ctrl); blk_mq_tagset_busy_iter(&ctrl->tag_set, @@ -2900,6 +2900,11 @@ nvme_fc_create_io_queues(struct nvme_fc_ctrl *ctrl) out_delete_hw_queues: nvme_fc_delete_hw_io_queues(ctrl); out_cleanup_tagset: + /* + * In CONNECTING state ctrl->ioerr_work will abort both admin + * and io tagsets. Cancel it first before removing io tagset. + */ + cancel_work_sync(&ctrl->ioerr_work); nvme_remove_io_tag_set(&ctrl->ctrl); nvme_fc_free_io_queues(ctrl); From 779575bc35c687697ba69e904f2cd22e60112534 Mon Sep 17 00:00:00 2001 From: Michael Bommarito Date: Tue, 9 Jun 2026 14:24:31 -0400 Subject: [PATCH 10/45] nvmet-auth: reject short AUTH_RECEIVE buffers nvmet_execute_auth_receive() trusts the AUTH_RECEIVE allocation length after checking only that it is nonzero and matches the transfer length. In the SUCCESS1 and FAILURE1/default states, that lets a remote NVMe-oF initiator reach the fixed-size DH-HMAC-CHAP response builders with a kmalloc() buffer shorter than the response, so nvmet_auth_success1() and nvmet_auth_failure1() write past the allocation; both only WARN_ON the short length and then format the message anyway. Impact: A remote NVMe-oF initiator with access to an auth-enabled target can trigger a 16-byte heap out-of-bounds write via a one-byte AUTH_RECEIVE allocation length. Compute the minimum response length for the current DH-HMAC-CHAP step in nvmet_auth_receive_data_len() and report a zero data length when the host-supplied allocation length is shorter, so the existing zero-length check in nvmet_execute_auth_receive() rejects the command before any builder runs. The SUCCESS1 minimum is sizeof(struct nvmf_auth_dhchap_success1_data) plus the HMAC hash length, because the response hash is written into the rval[] flexible-array tail, so the minimum is state dependent rather than a flat sizeof. CHALLENGE keeps its existing variable-length guard in nvmet_auth_challenge(). This is reachable only when in-band DH-HMAC-CHAP authentication is configured on the target. Fixes: db1312dd9548 ("nvmet: implement basic In-Band Authentication") Cc: stable@vger.kernel.org Assisted-by: Codex:gpt-5-5-xhigh Assisted-by: Claude:claude-opus-4-8 Reviewed-by: Hannes Reinecke Signed-off-by: Michael Bommarito Signed-off-by: Keith Busch --- drivers/nvme/target/fabrics-cmd-auth.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/target/fabrics-cmd-auth.c b/drivers/nvme/target/fabrics-cmd-auth.c index 0a85acf1e5c7..45820a12750d 100644 --- a/drivers/nvme/target/fabrics-cmd-auth.c +++ b/drivers/nvme/target/fabrics-cmd-auth.c @@ -493,7 +493,31 @@ static void nvmet_auth_failure1(struct nvmet_req *req, void *d, int al) u32 nvmet_auth_receive_data_len(struct nvmet_req *req) { - return le32_to_cpu(req->cmd->auth_receive.al); + struct nvmet_ctrl *ctrl = req->sq->ctrl; + u32 al = le32_to_cpu(req->cmd->auth_receive.al); + u32 min_len; + + /* + * Reject too-short al before kmalloc(al), since the SUCCESS1 and + * FAILURE1/default builders write fixed response headers into it. + */ + switch (req->sq->dhchap_step) { + case NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE: + return al; + case NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1: + min_len = sizeof(struct nvmf_auth_dhchap_success1_data); + if (req->sq->dhchap_c2) + min_len += nvme_auth_hmac_hash_len(ctrl->shash_id); + break; + default: + min_len = sizeof(struct nvmf_auth_dhchap_failure_data); + break; + } + + if (al < min_len) + return 0; + + return al; } void nvmet_execute_auth_receive(struct nvmet_req *req) From 7d953c75f0a3f905aadf3675c9394a5b9d9897bf Mon Sep 17 00:00:00 2001 From: Maurizio Lombardi Date: Mon, 16 Mar 2026 17:44:41 +0100 Subject: [PATCH 11/45] nvmet-tcp: handle TCP_CLOSING state in nvmet_tcp_state_change When an NVMe/TCP connection shuts down, the underlying TCP socket can enter the TCP_CLOSING state (state 11). Currently, the nvmet_tcp_state_change() callback does not explicitly handle this state, which results in harmless but noisy kernel warnings: nvmet_tcp: queue 2 unhandled state 11 Add TCP_CLOSING to the switch statement alongside TCP_FIN_WAIT2 and TCP_LAST_ACK to silently ignore the state transition. Signed-off-by: Maurizio Lombardi Signed-off-by: Keith Busch --- drivers/nvme/target/tcp.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index 3568fa9a0905..08140f2476d2 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -1678,6 +1678,7 @@ static void nvmet_tcp_state_change(struct sock *sk) switch (sk->sk_state) { case TCP_FIN_WAIT2: case TCP_LAST_ACK: + case TCP_CLOSING: break; case TCP_FIN_WAIT1: case TCP_CLOSE_WAIT: From 4f919141be38ea2b1314e3a531b7b998eb64e8bc Mon Sep 17 00:00:00 2001 From: Yitang Yang Date: Tue, 16 Jun 2026 23:51:29 +0800 Subject: [PATCH 12/45] block: fix IORING_URING_CMD_REISSUE flags check in blkdev_uring_cmd blkdev_uring_cmd() checks IORING_URING_CMD_REISSUE to determine whether this is the first issue. However, this flag lives in cmd->flags instead of issue_flags. Coincidentally, IO_URING_F_NONBLOCK shares bit 31 with IORING_URING_CMD_REISSUE. As a result, the SQE read was never performed, bic->len remained zero, and every BLOCK_URING_CMD_DISCARD failed with -EINVAL. Fix it by checking cmd->flags as intended. Cc: stable@vger.kernel.org Fixes: 212ec34e4e72 ("block: only read from sqe on initial invocation of blkdev_uring_cmd") Signed-off-by: Yitang Yang Link: https://patch.msgid.link/20260616155129.406057-1-yi1tang.yang@gmail.com Signed-off-by: Jens Axboe --- block/ioctl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block/ioctl.c b/block/ioctl.c index ab2c9ed79946..3d4ea1537457 100644 --- a/block/ioctl.c +++ b/block/ioctl.c @@ -951,7 +951,7 @@ int blkdev_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) u32 cmd_op = cmd->cmd_op; /* Read what we need from the SQE on the first issue */ - if (!(issue_flags & IORING_URING_CMD_REISSUE)) { + if (!(cmd->flags & IORING_URING_CMD_REISSUE)) { const struct io_uring_sqe *sqe = cmd->sqe; if (unlikely(sqe->ioprio || sqe->__pad1 || sqe->len || From 9cbbac29d752fb5d95e375fa3685a359b89caa0a Mon Sep 17 00:00:00 2001 From: Wen Xiong Date: Tue, 16 Jun 2026 10:31:21 -0400 Subject: [PATCH 13/45] block: Remove redundant plug in __submit_bio() The patch removes the automatic plug/unplug operations from __submit_bio() that were added to cache nsecs time when no explicit plug is used. The plug mechanism is most effective when batching multiple I/O operations together. Creating a plug for every bio submission provides minimal benefit while adding function call overhead and stack usage for every I/O operation. Below is performance comparison with the latest upstream kernel. Iotype qd nj rmix mpstat busy mpstat busy without plug Randrw 1 20 100 53% 24% Randrw 1 40 100 70% 24% Randrw 1 20 70 40% 24% Randrw 1 40 70 60% 26% Randrw 1 20 0 14% 6% Randrw 1 40 0 20% 7% Fixes: 060406c61c7c ("block: add plug while submitting IO") Signed-off-by: Wen Xiong Reviewed-by: Ming Lei Link: https://patch.msgid.link/20260616143121.878021-1-wenxiong@linux.ibm.com Signed-off-by: Jens Axboe --- block/blk-core.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/block/blk-core.c b/block/blk-core.c index 73a41df98c9a..365641266c9e 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -669,11 +669,6 @@ static inline blk_status_t blk_check_zone_append(struct request_queue *q, static void __submit_bio(struct bio *bio) { - /* If plug is not used, add new plug here to cache nsecs time. */ - struct blk_plug plug; - - blk_start_plug(&plug); - if (!bdev_test_flag(bio->bi_bdev, BD_HAS_SUBMIT_BIO)) { blk_mq_submit_bio(bio); } else if (likely(bio_queue_enter(bio) == 0)) { @@ -686,8 +681,6 @@ static void __submit_bio(struct bio *bio) disk->fops->submit_bio(bio); blk_queue_exit(disk->queue); } - - blk_finish_plug(&plug); } /* From fd38b75c4b43295b10d69772a46d1c74dbd6fc81 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Tue, 16 Jun 2026 07:15:17 -0700 Subject: [PATCH 14/45] kernel/fork: clear PF_BLOCK_TS in copy_process() PF_BLOCK_TS is only set in blk_time_get_ns() when current->plug is non-NULL, and blk_finish_plug() clears it via __blk_flush_plug() before NULLing the plug pointer. copy_process() breaks the invariant by inheriting PF_BLOCK_TS from the parent while resetting the child's plug to NULL. Clear PF_BLOCK_TS alongside that assignment so callers can rely on "PF_BLOCK_TS set implies current->plug != NULL" and dereference current->plug unguarded. Fixes: 06b23f92af87 ("block: update cached timestamp post schedule/preemption") Cc: stable@vger.kernel.org Signed-off-by: Usama Arif Link: https://patch.msgid.link/20260616141604.328820-2-usama.arif@linux.dev Signed-off-by: Jens Axboe --- kernel/fork.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/fork.c b/kernel/fork.c index addc555a1077..1fafcb9bb047 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -2337,6 +2337,7 @@ __latent_entropy struct task_struct *copy_process( #ifdef CONFIG_BLOCK p->plug = NULL; + p->flags &= ~PF_BLOCK_TS; #endif futex_init_task(p); From fad156c2af227f42ca796cbb20ddc354a6dd9932 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Tue, 16 Jun 2026 07:15:18 -0700 Subject: [PATCH 15/45] block: invalidate cached plug timestamp after task switch blk_time_get_ns() caches ktime_get_ns() in current->plug->cur_ktime and marks the task with PF_BLOCK_TS. That cache is only valid while the task keeps running; if the task is switched out, wall-clock time advances and the cached value must not be reused when the task runs again. The existing invalidation covers explicit plug flushes through __blk_flush_plug(), and the schedule() / rtmutex paths through sched_update_worker(). It does not cover in-kernel preemption paths such as preempt_schedule(), preempt_schedule_notrace(), and preempt_schedule_irq(), which enter __schedule(SM_PREEMPT) directly and return without calling sched_update_worker(). As a result, a task preempted while holding a plug with PF_BLOCK_TS set can reuse a stale plug->cur_ktime after it is scheduled back in. blk-iocost then consumes that stale timestamp through ioc_now(), producing stale vnow values for throttle decisions, and through ioc_rqos_done(), inflating on-queue time and feeding false missed-QoS samples into vrate adjustment. Move the schedule-side invalidation to finish_task_switch(), which runs for the scheduled-in task after every actual context switch regardless of which schedule entry point was used. Keep __blk_flush_plug() as the explicit flush/finish-plug invalidation path, and remove only the PF_BLOCK_TS handling from sched_update_worker(). Fixes: 06b23f92af87 ("block: update cached timestamp post schedule/preemption") Cc: stable@vger.kernel.org Signed-off-by: Usama Arif Link: https://patch.msgid.link/20260616141604.328820-3-usama.arif@linux.dev Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 16 ++++++---------- kernel/sched/core.c | 12 ++++++++---- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 5070851cf924..9213a5716f95 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1222,16 +1222,12 @@ static inline void blk_flush_plug(struct blk_plug *plug, bool async) __blk_flush_plug(plug, async); } -/* - * tsk == current here - */ -static inline void blk_plug_invalidate_ts(struct task_struct *tsk) +static __always_inline void blk_plug_invalidate_ts(void) { - struct blk_plug *plug = tsk->plug; - - if (plug) - plug->cur_ktime = 0; - current->flags &= ~PF_BLOCK_TS; + if (unlikely(current->flags & PF_BLOCK_TS)) { + current->plug->cur_ktime = 0; + current->flags &= ~PF_BLOCK_TS; + } } int blkdev_issue_flush(struct block_device *bdev); @@ -1257,7 +1253,7 @@ static inline void blk_flush_plug(struct blk_plug *plug, bool async) { } -static inline void blk_plug_invalidate_ts(struct task_struct *tsk) +static inline void blk_plug_invalidate_ts(void) { } diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 8b791e9e9f67..e97e98c33be5 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5368,6 +5368,12 @@ static struct rq *finish_task_switch(struct task_struct *prev) */ kmap_local_sched_in(); + /* + * Any cached block-layer timestamp (plug->cur_ktime) is stale now, + * invalidate it. + */ + blk_plug_invalidate_ts(); + fire_sched_in_preempt_notifiers(current); /* * When switching through a kernel thread, the loop in @@ -7290,12 +7296,10 @@ static inline void sched_submit_work(struct task_struct *tsk) static void sched_update_worker(struct task_struct *tsk) { - if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER | PF_BLOCK_TS)) { - if (tsk->flags & PF_BLOCK_TS) - blk_plug_invalidate_ts(tsk); + if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER)) { if (tsk->flags & PF_WQ_WORKER) wq_worker_running(tsk); - else if (tsk->flags & PF_IO_WORKER) + else io_wq_worker_running(tsk); } } From b68d4979c88e31488970373f67ac79b4f6267008 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 16 Jun 2026 17:42:35 +0930 Subject: [PATCH 16/45] block: revert the iov_iter after a short copy in bio_iov_iter_bounce_write() For the incoming IOMAP_DIO_BOUNCE flag usage inside btrfs, it's pretty easy to hit short copy inside bio_iov_iter_bounce_write(). This is because btrfs has disabled page fault to avoid certain deadlock during direct writes, and instead btrfs manually fault in the pages then retry. And inside bio_iov_iter_bounce_write(), if we hit a short write, we didn't revert the iov_iter, which can cause problems like unexpected garbage for the next retry. Revert the iov_iter after a short copy. One thing to note is that, the folio is allocated then immediately queued into the bio, so the proper revert size should be (bi_size - this_len + copied). Fixes: 8dd5e7c75d7b ("block: add helpers to bounce buffer an iov_iter into bios") Signed-off-by: Qu Wenruo Reviewed-by: Christoph Hellwig Link: https://patch.msgid.link/c400989f227343b134110773d5acaaacf7024574.1781597506.git.wqu@suse.com Signed-off-by: Jens Axboe --- block/bio.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/block/bio.c b/block/bio.c index 811a96796202..96f40d39b62b 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1321,6 +1321,7 @@ static int bio_iov_iter_bounce_write(struct bio *bio, struct iov_iter *iter, do { size_t this_len = min(total_len, SZ_1M); + size_t copied; struct folio *folio; if (this_len > minsize * 2) @@ -1334,12 +1335,22 @@ static int bio_iov_iter_bounce_write(struct bio *bio, struct iov_iter *iter, break; bio_add_folio_nofail(bio, folio, this_len, 0); - if (copy_from_iter(folio_address(folio), this_len, iter) != - this_len) { + copied = copy_from_iter(folio_address(folio), this_len, iter); + if (copied < this_len) { + /* + * Need to revert the iov iter for all bytes we have + * copied. + * + * However the bio size differs from the real copied + * bytes as @this_len is queued but only advanced + * less than that. + * Need to compensate that for the revert. + */ + iov_iter_revert(iter, bio->bi_iter.bi_size - this_len + + copied); bio_free_folios(bio); return -EFAULT; } - total_len -= this_len; } while (total_len && bio->bi_vcnt < bio->bi_max_vecs); From d5b58fbb2fd7ac25fcd7e1c14730f998a90b0322 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 16 Jun 2026 17:42:36 +0930 Subject: [PATCH 17/45] block: respect iov_iter::nofault flag in bio_iov_iter_bounce_write() For the incoming usage of IOMAP_DIO_BOUNCE in btrfs, btrfs has set iov_iter::nofault to prevent deadlock when a page fault is needed to read out the buffer. However bio_iov_iter_bounce_write() doesn't respect iov_iter::nofault flag, and just call a plain copy_from_iter() so it can still trigger page fault and cause deadlock in btrfs. Fix it by utilizing copy_folio_from_iter_atomic() if nofault flag is set, otherwise use copy_folio_from_iter(). Signed-off-by: Qu Wenruo Reviewed-by: Christoph Hellwig Link: https://patch.msgid.link/9c165a314022b61566eb247852eb773ca6c70889.1781597506.git.wqu@suse.com Signed-off-by: Jens Axboe --- block/bio.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/block/bio.c b/block/bio.c index 96f40d39b62b..f2a5f4d0a967 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1335,7 +1335,11 @@ static int bio_iov_iter_bounce_write(struct bio *bio, struct iov_iter *iter, break; bio_add_folio_nofail(bio, folio, this_len, 0); - copied = copy_from_iter(folio_address(folio), this_len, iter); + if (iter->nofault) + copied = copy_folio_from_iter_atomic(folio, 0, this_len, + iter); + else + copied = copy_folio_from_iter(folio, 0, this_len, iter); if (copied < this_len) { /* * Need to revert the iov iter for all bytes we have From 8e065a1602511282fc0da2dc89445e0eb71a681c Mon Sep 17 00:00:00 2001 From: Abd-Alrhman Masalkhi Date: Sat, 13 Jun 2026 18:28:07 +0000 Subject: [PATCH 18/45] md/raid1: fix writes_pending and barrier reference leaks on write failures raid1_make_request() acquires a writes_pending reference with md_write_start() before calling raid1_write_request(). Several failure paths in raid1_write_request() complete the bio and return without reaching the normal write completion path, causing the corresponding md_write_end() to be skipped. Make raid1_write_request() return a status indicating whether the write request was successfully queued. This allows raid1_make_request() to call md_write_end() when raid1_write_request() fails. Additionally, if wait_blocked_rdev() fails after wait_barrier() succeeds, the associated barrier reference is not released. Call allow_barrier() before returning from that path to keep the barrier accounting balanced. Fixes: b1a7ad8b5c4f ("md/raid1: Handle bio_split() errors") Fixes: f2a38abf5f1c ("md/raid1: Atomic write support") Fixes: 5aa705039c4f ("md: raid1 add nowait support") Reported-by: sashiko-bot Closes: https://sashiko.dev/#/patchset/20260611083514.754922-1-abd.masalkhi@gmail.com?part=1 Closes: https://sashiko.dev/#/patchset/20260611132500.763528-1-abd.masalkhi@gmail.com?part=1 Signed-off-by: Abd-Alrhman Masalkhi Link: https://patch.msgid.link/20260613182810.1317258-2-abd.masalkhi@gmail.com Signed-off-by: Yu Kuai --- drivers/md/raid1.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index 5b9368bd9e70..632d72607e11 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -1501,7 +1501,7 @@ static void raid1_start_write_behind(struct mddev *mddev, struct r1bio *r1_bio, } -static void raid1_write_request(struct mddev *mddev, struct bio *bio, +static bool raid1_write_request(struct mddev *mddev, struct bio *bio, int max_write_sectors) { struct r1conf *conf = mddev->private; @@ -1512,6 +1512,7 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, int max_sectors; bool write_behind = false; bool is_discard = (bio_op(bio) == REQ_OP_DISCARD); + sector_t sector = bio->bi_iter.bi_sector; if (mddev_is_clustered(mddev) && mddev->cluster_ops->area_resyncing(mddev, WRITE, @@ -1519,7 +1520,7 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, if (bio->bi_opf & REQ_NOWAIT) { bio_wouldblock_error(bio); - return; + return false; } wait_event_idle(conf->wait_barrier, !mddev->cluster_ops->area_resyncing(mddev, WRITE, @@ -1535,12 +1536,13 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, if (!wait_barrier(conf, bio->bi_iter.bi_sector, bio->bi_opf & REQ_NOWAIT)) { bio_wouldblock_error(bio); - return; + return false; } if (!wait_blocked_rdev(mddev, bio)) { bio_wouldblock_error(bio); - return; + allow_barrier(conf, sector); + return false; } r1_bio = alloc_r1bio(mddev, bio); @@ -1699,7 +1701,8 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, /* In case raid1d snuck in to freeze_array */ wake_up_barrier(conf); - return; + return true; + err_handle: for (k = 0; k < i; k++) { if (r1_bio->bios[k]) { @@ -1709,6 +1712,7 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio, } raid_end_bio_io(r1_bio); + return false; } static bool raid1_make_request(struct mddev *mddev, struct bio *bio) @@ -1732,8 +1736,9 @@ static bool raid1_make_request(struct mddev *mddev, struct bio *bio) if (bio_data_dir(bio) == READ) raid1_read_request(mddev, bio, sectors, NULL); else { - md_write_start(mddev,bio); - raid1_write_request(mddev, bio, sectors); + md_write_start(mddev, bio); + if (!raid1_write_request(mddev, bio, sectors)) + md_write_end(mddev); } return true; } From e045d6ed33f8faa3e3dd6dc33c62ec01e3ad275d Mon Sep 17 00:00:00 2001 From: Abd-Alrhman Masalkhi Date: Sat, 13 Jun 2026 18:28:08 +0000 Subject: [PATCH 19/45] md/raid10: fix writes_pending leak on write request failures raid10_make_request() acquires a writes_pending reference with md_write_start() before dispatching write requests. Several failure paths in raid10_write_request() complete the bio and return without reaching the normal write completion path, causing the corresponding md_write_end() to be skipped. Make raid10_write_request() return a status indicating whether the write request was successfully queued. This allows raid10_make_request() to release the writes_pending reference with md_write_end() when a write request fails. Fixes: 4cf58d952909 ("md/raid10: Handle bio_split() errors") Fixes: c9aa889b035f ("md: raid10 add nowait support") Signed-off-by: Abd-Alrhman Masalkhi Link: https://patch.msgid.link/20260613182810.1317258-3-abd.masalkhi@gmail.com Signed-off-by: Yu Kuai --- drivers/md/raid10.c | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index cee5a253a281..c123a8c76ddc 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -1349,7 +1349,7 @@ static void wait_blocked_dev(struct mddev *mddev, struct r10bio *r10_bio) } } -static void raid10_write_request(struct mddev *mddev, struct bio *bio, +static bool raid10_write_request(struct mddev *mddev, struct bio *bio, struct r10bio *r10_bio) { struct r10conf *conf = mddev->private; @@ -1365,7 +1365,7 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio, /* Bail out if REQ_NOWAIT is set for the bio */ if (bio->bi_opf & REQ_NOWAIT) { bio_wouldblock_error(bio); - return; + return false; } for (;;) { prepare_to_wait(&conf->wait_barrier, @@ -1381,7 +1381,7 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio, sectors = r10_bio->sectors; if (!regular_request_wait(mddev, conf, bio, sectors)) { free_r10bio(r10_bio); - return; + return false; } if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && @@ -1398,7 +1398,7 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio, if (bio->bi_opf & REQ_NOWAIT) { allow_barrier(conf); bio_wouldblock_error(bio); - return; + return false; } mddev_add_trace_msg(conf->mddev, "raid10 wait reshape metadata"); @@ -1514,7 +1514,8 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio, raid10_write_one_disk(mddev, r10_bio, bio, true, i); } one_write_done(r10_bio); - return; + return true; + err_handle: for (k = 0; k < i; k++) { int d = r10_bio->devs[k].devnum; @@ -1532,10 +1533,12 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio, } raid_end_bio_io(r10_bio); + return false; } -static void __make_request(struct mddev *mddev, struct bio *bio, int sectors) +static bool __make_request(struct mddev *mddev, struct bio *bio, int sectors) { + bool ret; struct r10conf *conf = mddev->private; struct r10bio *r10_bio; @@ -1551,10 +1554,13 @@ static void __make_request(struct mddev *mddev, struct bio *bio, int sectors) memset(r10_bio->devs, 0, sizeof(r10_bio->devs[0]) * conf->geo.raid_disks); + ret = true; if (bio_data_dir(bio) == READ) raid10_read_request(mddev, bio, r10_bio); else - raid10_write_request(mddev, bio, r10_bio); + ret = raid10_write_request(mddev, bio, r10_bio); + + return ret; } static void raid_end_discard_bio(struct r10bio *r10bio) @@ -1900,7 +1906,8 @@ static bool raid10_make_request(struct mddev *mddev, struct bio *bio) sectors = chunk_sects - (bio->bi_iter.bi_sector & (chunk_sects - 1)); - __make_request(mddev, bio, sectors); + if (!__make_request(mddev, bio, sectors)) + md_write_end(mddev); /* In case raid10d snuck in to freeze_array */ wake_up_barrier(conf); From 393d687131d8aa8c7e4de2cb494438e145d20fc2 Mon Sep 17 00:00:00 2001 From: Abd-Alrhman Masalkhi Date: Sat, 13 Jun 2026 18:28:09 +0000 Subject: [PATCH 20/45] md/raid10: fix writes_pending and barrier reference leaks on discard failures raid10_make_request() acquires a writes_pending reference with md_write_start() before calling raid10_handle_discard(). Several failure paths in raid10_handle_discard() complete the bio and return without releasing the corresponding reference, causing md_write_end() to be skipped. Call md_write_end() before returning from these failure paths to keep writes_pending accounting balanced. Additionally, discard split allocation failures can occur after wait_barrier() succeeds. Those paths return without calling allow_barrier(), leaking the associated barrier reference. Release the barrier before returning from those paths. Fixes: c9aa889b035f ("md: raid10 add nowait support") Fixes: 4cf58d952909 ("md/raid10: Handle bio_split() errors") Signed-off-by: Abd-Alrhman Masalkhi Link: https://patch.msgid.link/20260613182810.1317258-4-abd.masalkhi@gmail.com Signed-off-by: Yu Kuai --- drivers/md/raid10.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index c123a8c76ddc..0a3cfdd3f5df 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -1639,6 +1639,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) if (!wait_barrier(conf, bio->bi_opf & REQ_NOWAIT)) { bio_wouldblock_error(bio); + md_write_end(mddev); return 0; } @@ -1681,6 +1682,8 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) if (IS_ERR(split)) { bio->bi_status = errno_to_blk_status(PTR_ERR(split)); bio_endio(bio); + md_write_end(mddev); + allow_barrier(conf); return 0; } @@ -1698,6 +1701,8 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) if (IS_ERR(split)) { bio->bi_status = errno_to_blk_status(PTR_ERR(split)); bio_endio(bio); + md_write_end(mddev); + allow_barrier(conf); return 0; } From a4c55c902670f2784d3001652183aafa17712cfe Mon Sep 17 00:00:00 2001 From: Abd-Alrhman Masalkhi Date: Sat, 13 Jun 2026 18:28:10 +0000 Subject: [PATCH 21/45] md/raid1: simplify raid1_write_request() error handling raid1_write_request() increments rdev->nr_pending before checking the badblocks and then immediately decrements it again when a device is skipped. Move the increment until after the checks succeed so the reference accounting is easier to follow. Consolidate the failure paths so that each error label releases exactly the resources acquired up to that point. err_dec_pending drops pending references and frees the r1bio, while err_allow_barrier handles the barrier release before returning. When a REQ_ATOMIC write cannot be satisfied due to a badblock range, complete the bio with BLK_STS_NOTSUPP rather than reporting an I/O error, since the operation is unsupported rather than having failed during I/O. Rename max_write_sectors to max_sectors and remove the redundant local copy. Signed-off-by: Abd-Alrhman Masalkhi Link: https://patch.msgid.link/20260613182810.1317258-5-abd.masalkhi@gmail.com Signed-off-by: Yu Kuai --- drivers/md/raid1.c | 59 +++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index 632d72607e11..86d4f224ffb1 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -1502,29 +1502,29 @@ static void raid1_start_write_behind(struct mddev *mddev, struct r1bio *r1_bio, } static bool raid1_write_request(struct mddev *mddev, struct bio *bio, - int max_write_sectors) + int max_sectors) { struct r1conf *conf = mddev->private; struct r1bio *r1_bio; int i, disks, k; unsigned long flags; int first_clone; - int max_sectors; bool write_behind = false; - bool is_discard = (bio_op(bio) == REQ_OP_DISCARD); + bool nowait = bio->bi_opf & REQ_NOWAIT; + bool is_discard = op_is_discard(bio->bi_opf); sector_t sector = bio->bi_iter.bi_sector; if (mddev_is_clustered(mddev) && - mddev->cluster_ops->area_resyncing(mddev, WRITE, - bio->bi_iter.bi_sector, bio_end_sector(bio))) { + mddev->cluster_ops->area_resyncing(mddev, WRITE, sector, + bio_end_sector(bio))) { - if (bio->bi_opf & REQ_NOWAIT) { + if (nowait) { bio_wouldblock_error(bio); return false; } wait_event_idle(conf->wait_barrier, !mddev->cluster_ops->area_resyncing(mddev, WRITE, - bio->bi_iter.bi_sector, + sector, bio_end_sector(bio))); } @@ -1533,20 +1533,18 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, * thread has put up a bar for new requests. * Continue immediately if no resync is active currently. */ - if (!wait_barrier(conf, bio->bi_iter.bi_sector, - bio->bi_opf & REQ_NOWAIT)) { + if (!wait_barrier(conf, sector, nowait)) { bio_wouldblock_error(bio); return false; } if (!wait_blocked_rdev(mddev, bio)) { bio_wouldblock_error(bio); - allow_barrier(conf, sector); - return false; + goto err_allow_barrier; } r1_bio = alloc_r1bio(mddev, bio); - r1_bio->sectors = max_write_sectors; + r1_bio->sectors = max_sectors; /* first select target devices under rcu_lock and * inc refcount on their rdev. Record them by setting @@ -1560,7 +1558,6 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, */ disks = conf->raid_disks * 2; - max_sectors = r1_bio->sectors; for (i = 0; i < disks; i++) { struct md_rdev *rdev = conf->mirrors[i].rdev; @@ -1576,23 +1573,21 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, if (!rdev || test_bit(Faulty, &rdev->flags)) continue; - atomic_inc(&rdev->nr_pending); if (test_bit(WriteErrorSeen, &rdev->flags)) { sector_t first_bad; sector_t bad_sectors; int is_bad; - is_bad = is_badblock(rdev, r1_bio->sector, max_sectors, + is_bad = is_badblock(rdev, sector, max_sectors, &first_bad, &bad_sectors); - if (is_bad && first_bad <= r1_bio->sector) { + if (is_bad && first_bad <= sector) { /* Cannot write here at all */ - bad_sectors -= (r1_bio->sector - first_bad); + bad_sectors -= (sector - first_bad); if (bad_sectors < max_sectors) /* mustn't write more than bad_sectors * to other devices yet */ max_sectors = bad_sectors; - rdev_dec_pending(rdev, mddev); continue; } if (is_bad) { @@ -1606,15 +1601,18 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, * the benefit. */ if (bio->bi_opf & REQ_ATOMIC) { - rdev_dec_pending(rdev, mddev); - goto err_handle; + bio->bi_status = BLK_STS_NOTSUPP; + bio_endio(bio); + goto err_dec_pending; } - good_sectors = first_bad - r1_bio->sector; + good_sectors = first_bad - sector; if (good_sectors < max_sectors) max_sectors = good_sectors; } } + + atomic_inc(&rdev->nr_pending); r1_bio->bios[i] = bio; } @@ -1630,10 +1628,8 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, if (max_sectors < bio_sectors(bio)) { bio = bio_submit_split_bioset(bio, max_sectors, &conf->bio_split); - if (!bio) { - set_bit(R1BIO_Returned, &r1_bio->state); - goto err_handle; - } + if (!bio) + goto err_dec_pending; r1_bio->master_bio = bio; r1_bio->sectors = max_sectors; @@ -1677,7 +1673,7 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, mbio->bi_opf &= ~REQ_NOWAIT; r1_bio->bios[i] = mbio; - mbio->bi_iter.bi_sector = (r1_bio->sector + rdev->data_offset); + mbio->bi_iter.bi_sector = sector + rdev->data_offset; mbio->bi_end_io = raid1_end_write_request; if (test_bit(FailFast, &rdev->flags) && !test_bit(WriteMostly, &rdev->flags) && @@ -1686,7 +1682,7 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, mbio->bi_private = r1_bio; atomic_inc(&r1_bio->remaining); - mddev_trace_remap(mddev, mbio, r1_bio->sector); + mddev_trace_remap(mddev, mbio, sector); /* flush_pending_writes() needs access to the rdev so...*/ mbio->bi_bdev = (void *)rdev; if (!raid1_add_bio_to_plug(mddev, mbio, raid1_unplug, disks)) { @@ -1701,9 +1697,10 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, /* In case raid1d snuck in to freeze_array */ wake_up_barrier(conf); + return true; -err_handle: +err_dec_pending: for (k = 0; k < i; k++) { if (r1_bio->bios[k]) { rdev_dec_pending(conf->mirrors[k].rdev, mddev); @@ -1711,7 +1708,11 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, } } - raid_end_bio_io(r1_bio); + free_r1bio(r1_bio); + +err_allow_barrier: + allow_barrier(conf, sector); + return false; } From 74ddbf98e2db646ec58f7e7731c936b7a4a470fe Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Fri, 5 Jun 2026 15:26:37 +0800 Subject: [PATCH 22/45] md/raid5: account discard IO Raid5 handles discard bios internally through make_discard_request() and never passes them through md_account_bio(). As a result, discard IO is missing the md-device iostat accounting that normal raid5 IO and discard IO in other raid levels get from md_account_bio(). Before accounting the bio, trim the request to the full data stripes that raid5 will actually discard. The first full stripe is the ceiling of the bio start divided by data-stripe sectors, and the last full stripe is the floor of the bio end divided by data-stripe sectors. Account that exact MD logical full-stripe range, then restore the original iterator so bio completion and iostat still cover the original request. Link: https://patch.msgid.link/20260605072639.2434847-2-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/raid5.c | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 65ae7d8930fc..debf35342ae0 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -5690,7 +5690,10 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi) { struct r5conf *conf = mddev->private; sector_t logical_sector, last_sector; + sector_t first_stripe, last_stripe; struct stripe_head *sh; + struct bvec_iter bi_iter; + struct bio *orig_bi = bi; int stripe_sectors; /* We need to handle this when io_uring supports discard/trim */ @@ -5701,19 +5704,29 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi) /* Skip discard while reshape is happening */ return; - logical_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1); - last_sector = bio_end_sector(bi); - - bi->bi_next = NULL; - stripe_sectors = conf->chunk_sectors * (conf->raid_disks - conf->max_degraded); - logical_sector = DIV_ROUND_UP_SECTOR_T(logical_sector, - stripe_sectors); - sector_div(last_sector, stripe_sectors); + first_stripe = DIV_ROUND_UP_SECTOR_T(bi->bi_iter.bi_sector, + stripe_sectors); + last_stripe = bio_end_sector(bi); + sector_div(last_stripe, stripe_sectors); - logical_sector *= conf->chunk_sectors; - last_sector *= conf->chunk_sectors; + if (first_stripe >= last_stripe) { + bio_endio(bi); + return; + } + + bi_iter = bi->bi_iter; + bi->bi_iter.bi_sector = first_stripe * stripe_sectors; + bi->bi_iter.bi_size = ((last_stripe - first_stripe) * + stripe_sectors) << 9; + md_account_bio(mddev, &bi); + orig_bi->bi_iter = bi_iter; + bi->bi_iter = bi_iter; + bi->bi_next = NULL; + + logical_sector = first_stripe * conf->chunk_sectors; + last_sector = last_stripe * conf->chunk_sectors; for (; logical_sector < last_sector; logical_sector += RAID5_STRIPE_SECTORS(conf)) { From 90573092673cdbb2f28f0932fd40de65c3f762cf Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Fri, 5 Jun 2026 15:26:38 +0800 Subject: [PATCH 23/45] md/raid5: validate discard support at request time Raid5 used to disable discard limits when devices_handle_discard_safely was not set or when stacked member limits could not support a full-stripe discard. That hides discard from userspace before raid5 can decide whether a request can be handled safely. Follow other virtual drivers and advertise a UINT_MAX discard limit for the md device. Cache lower discard support in r5conf when setting queue limits, and reject unsupported discard bios before queuing stripe work. Link: https://patch.msgid.link/20260605072639.2434847-3-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/raid5.c | 34 +++++++++++++++++++--------------- drivers/md/raid5.h | 1 + 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index debf35342ae0..76e736ee48d3 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -1134,6 +1134,18 @@ static void defer_issue_bios(struct r5conf *conf, sector_t sector, dispatch_bio_list(&tmp); } +static bool raid5_discard_limits(struct mddev *mddev, struct bio *bi) +{ + struct r5conf *conf = mddev->private; + + if (!conf->raid5_discard_unsupported) + return true; + + bi->bi_status = BLK_STS_NOTSUPP; + bio_endio(bi); + return false; +} + static void raid5_end_read_request(struct bio *bi); static void @@ -5704,6 +5716,9 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi) /* Skip discard while reshape is happening */ return; + if (!raid5_discard_limits(mddev, bi)) + return; + stripe_sectors = conf->chunk_sectors * (conf->raid_disks - conf->max_degraded); first_stripe = DIV_ROUND_UP_SECTOR_T(bi->bi_iter.bi_sector, @@ -7817,24 +7832,12 @@ static int raid5_set_limits(struct mddev *mddev) queue_limits_stack_bdev(&lim, rdev->bdev, rdev->new_data_offset, mddev->gendisk->disk_name); - /* - * Zeroing is required for discard, otherwise data could be lost. - * - * Consider a scenario: discard a stripe (the stripe could be - * inconsistent if discard_zeroes_data is 0); write one disk of the - * stripe (the stripe could be inconsistent again depending on which - * disks are used to calculate parity); the disk is broken; The stripe - * data of this disk is lost. - * - * We only allow DISCARD if the sysadmin has confirmed that only safe - * devices are in use by setting a module parameter. A better idea - * might be to turn DISCARD into WRITE_ZEROES requests, as that is - * required to be safe. - */ if (!devices_handle_discard_safely || lim.max_discard_sectors < (stripe >> 9) || lim.discard_granularity < stripe) - lim.max_hw_discard_sectors = 0; + conf->raid5_discard_unsupported = true; + else + conf->raid5_discard_unsupported = false; /* * Requests require having a bitmap for each stripe. @@ -7843,6 +7846,7 @@ static int raid5_set_limits(struct mddev *mddev) lim.max_hw_sectors = RAID5_MAX_REQ_STRIPES << RAID5_STRIPE_SHIFT(conf); if ((lim.max_hw_sectors << 9) < lim.io_opt) lim.max_hw_sectors = lim.io_opt >> 9; + lim.max_hw_discard_sectors = UINT_MAX; /* No restrictions on the number of segments in the request */ lim.max_segments = USHRT_MAX; diff --git a/drivers/md/raid5.h b/drivers/md/raid5.h index 1dfa60a41d91..cb5feae04db2 100644 --- a/drivers/md/raid5.h +++ b/drivers/md/raid5.h @@ -689,6 +689,7 @@ struct r5conf { struct list_head pending_list; int pending_data_cnt; struct r5pending_data *next_pending_data; + bool raid5_discard_unsupported; mempool_t *ctx_pool; int ctx_size; From 53528031e7a6e16f0f05347f3826ffb4f957b7e4 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Fri, 5 Jun 2026 15:26:39 +0800 Subject: [PATCH 24/45] md/raid5: always convert llbitmap bits for discard llbitmap discard is useful even when no underlying member device supports it. The discard still converts the llbitmap range to unwritten, so later reads and recovery do not rely on stale parity for that range. Let llbitmap discard bypass the raid5 lower discard support check. If lower discard is not safe or not supported, complete the accounted clone after md_account_bio() so the llbitmap conversion callbacks run without member discard bios. Link: https://patch.msgid.link/20260605072639.2434847-4-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/raid5.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 76e736ee48d3..180ff0660b6a 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -1138,6 +1138,9 @@ static bool raid5_discard_limits(struct mddev *mddev, struct bio *bi) { struct r5conf *conf = mddev->private; + if (mddev->bitmap_id == ID_LLBITMAP) + return true; + if (!conf->raid5_discard_unsupported) return true; @@ -5740,6 +5743,12 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi) bi->bi_iter = bi_iter; bi->bi_next = NULL; + if (mddev->bitmap_id == ID_LLBITMAP && + conf->raid5_discard_unsupported) { + bio_endio(bi); + return; + } + logical_sector = first_stripe * conf->chunk_sectors; last_sector = last_stripe * conf->chunk_sectors; From a286cb88ddb26c5f4377859d8e77233d9181eb82 Mon Sep 17 00:00:00 2001 From: Abd-Alrhman Masalkhi Date: Thu, 11 Jun 2026 08:35:14 +0000 Subject: [PATCH 25/45] md/raid1: honor REQ_NOWAIT when waiting for behind writes raid1 supports REQ_NOWAIT reads by avoiding waits in the barrier path through wait_read_barrier(). However, a read can still block on a WriteMostly device when the array uses a bitmap and there are outstanding behind writes. In that case raid1 unconditionally calls wait_behind_writes(), which may sleep until all behind writes complete. As a result, a REQ_NOWAIT read can block despite the caller explicitly requesting non-blocking behavior. This ensures that raid1 consistently honors REQ_NOWAIT reads across all paths that may otherwise wait for behind writes. Fixes: 5aa705039c4f ("md: raid1 add nowait support") Signed-off-by: Abd-Alrhman Masalkhi Link: https://patch.msgid.link/20260611083514.754922-1-abd.masalkhi@gmail.com Signed-off-by: Yu Kuai --- drivers/md/md-bitmap.c | 9 +++++++-- drivers/md/md-bitmap.h | 2 +- drivers/md/md-llbitmap.c | 13 ++++++++----- drivers/md/md.c | 2 +- drivers/md/raid1.c | 10 +++++++--- 5 files changed, 24 insertions(+), 12 deletions(-) diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c index 7d778fe1c47c..0f02e2956398 100644 --- a/drivers/md/md-bitmap.c +++ b/drivers/md/md-bitmap.c @@ -2064,18 +2064,23 @@ static void bitmap_end_behind_write(struct mddev *mddev) bitmap->mddev->bitmap_info.max_write_behind); } -static void bitmap_wait_behind_writes(struct mddev *mddev) +static bool bitmap_wait_behind_writes(struct mddev *mddev, bool nowait) { struct bitmap *bitmap = mddev->bitmap; /* wait for behind writes to complete */ if (bitmap && atomic_read(&bitmap->behind_writes) > 0) { + if (nowait) + return false; + pr_debug("md:%s: behind writes in progress - waiting to stop.\n", mdname(mddev)); /* need to kick something here to make sure I/O goes? */ wait_event(bitmap->behind_wait, atomic_read(&bitmap->behind_writes) == 0); } + + return true; } static void bitmap_destroy(struct mddev *mddev) @@ -2085,7 +2090,7 @@ static void bitmap_destroy(struct mddev *mddev) if (!bitmap) /* there was no bitmap */ return; - bitmap_wait_behind_writes(mddev); + bitmap_wait_behind_writes(mddev, false); if (!test_bit(MD_SERIALIZE_POLICY, &mddev->flags)) mddev_destroy_serial_pool(mddev, NULL); diff --git a/drivers/md/md-bitmap.h b/drivers/md/md-bitmap.h index 214f623c7e79..f46674bdfeb9 100644 --- a/drivers/md/md-bitmap.h +++ b/drivers/md/md-bitmap.h @@ -98,7 +98,7 @@ struct bitmap_operations { void (*start_behind_write)(struct mddev *mddev); void (*end_behind_write)(struct mddev *mddev); - void (*wait_behind_writes)(struct mddev *mddev); + bool (*wait_behind_writes)(struct mddev *mddev, bool nowait); md_bitmap_fn *start_write; md_bitmap_fn *end_write; diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index 1adc5b117821..5a4e2abaa757 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -1574,16 +1574,19 @@ static void llbitmap_end_behind_write(struct mddev *mddev) wake_up(&llbitmap->behind_wait); } -static void llbitmap_wait_behind_writes(struct mddev *mddev) +static bool llbitmap_wait_behind_writes(struct mddev *mddev, bool nowait) { struct llbitmap *llbitmap = mddev->bitmap; - if (!llbitmap) - return; + if (llbitmap && atomic_read(&llbitmap->behind_writes) > 0) { + if (nowait) + return false; - wait_event(llbitmap->behind_wait, - atomic_read(&llbitmap->behind_writes) == 0); + wait_event(llbitmap->behind_wait, + atomic_read(&llbitmap->behind_writes) == 0); + } + return true; } static ssize_t bits_show(struct mddev *mddev, char *page) diff --git a/drivers/md/md.c b/drivers/md/md.c index 096bb64e87bd..d1465bcd86c8 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -7050,7 +7050,7 @@ EXPORT_SYMBOL_GPL(md_stop_writes); static void mddev_detach(struct mddev *mddev) { if (md_bitmap_enabled(mddev, false)) - mddev->bitmap_ops->wait_behind_writes(mddev); + mddev->bitmap_ops->wait_behind_writes(mddev, false); if (mddev->pers && mddev->pers->quiesce && !is_md_suspended(mddev)) { mddev->pers->quiesce(mddev, 1); mddev->pers->quiesce(mddev, 0); diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index 86d4f224ffb1..e2a50816e5a0 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -1341,6 +1341,7 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, int max_sectors; int rdisk; bool r1bio_existed = !!r1_bio; + bool nowait = bio->bi_opf & REQ_NOWAIT; /* * An md cloned bio indicates we are in the error path. @@ -1360,8 +1361,7 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, * Still need barrier for READ in case that whole * array is frozen. */ - if (!wait_read_barrier(conf, bio->bi_iter.bi_sector, - bio->bi_opf & REQ_NOWAIT)) { + if (!wait_read_barrier(conf, bio->bi_iter.bi_sector, nowait)) { bio_wouldblock_error(bio); return; } @@ -1402,7 +1402,11 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, * over-take any writes that are 'behind' */ mddev_add_trace_msg(mddev, "raid1 wait behind writes"); - mddev->bitmap_ops->wait_behind_writes(mddev); + if (!mddev->bitmap_ops->wait_behind_writes(mddev, nowait)) { + bio_wouldblock_error(bio); + set_bit(R1BIO_Returned, &r1_bio->state); + goto err_handle; + } } if (max_sectors < bio_sectors(bio)) { From 69ad6ce47f9bf2b9fe0ed69b042db993d33bbf12 Mon Sep 17 00:00:00 2001 From: Abd-Alrhman Masalkhi Date: Thu, 11 Jun 2026 10:13:50 +0000 Subject: [PATCH 26/45] md/raid1: free r1_bio when REQ_NOWAIT is set and read would block on retry When a read is retried, raid1_read_request() may be called with a pre-allocated r1_bio. If wait_read_barrier() fails for a REQ_NOWAIT read, the bio is completed and the function returns immediately. In this case the existing r1_bio is leaked. This fixes a leak of pre-allocated r1_bio structures for retried reads. Fixes: 5aa705039c4f ("md: raid1 add nowait support") Reported-by: sashiko-bot Closes: https://sashiko.dev/#/patchset/20260611083514.754922-1-abd.masalkhi@gmail.com?part=1 Signed-off-by: Abd-Alrhman Masalkhi Link: https://patch.msgid.link/20260611101350.759154-1-abd.masalkhi@gmail.com Signed-off-by: Yu Kuai --- drivers/md/raid1.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index e2a50816e5a0..41d9094fa50a 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -1363,6 +1363,12 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, */ if (!wait_read_barrier(conf, bio->bi_iter.bi_sector, nowait)) { bio_wouldblock_error(bio); + + if (r1bio_existed) { + set_bit(R1BIO_Returned, &r1_bio->state); + raid_end_bio_io(r1_bio); + } + return; } From 601d3c21b2e26b676cc67ae8804e991bbbcd4507 Mon Sep 17 00:00:00 2001 From: Chen Cheng Date: Fri, 19 Jun 2026 12:41:14 +0800 Subject: [PATCH 27/45] md/raid1: protect head_position for read balance KCSAN reports a data race between raid1_end_read_request() and raid1_read_request(). The completion path updates conf->mirrors[disk].head_position in update_head_pos() without a lock, while the read-balance heuristic reads the same field locklessly in is_sequential() and choose_best_rdev(). KCSAN report: ========================= BUG: KCSAN: data-race in raid1_end_read_request / raid1_read_request write to 0xffff8f0306ba7868 of 8 bytes by interrupt on cpu 9: raid1_end_read_request+0xb5/0x440 bio_endio+0x3c9/0x3e0 blk_update_request+0x257/0x770 scsi_end_request+0x4d/0x520 scsi_io_completion+0x6f/0x990 scsi_finish_command+0x188/0x280 scsi_complete+0xac/0x160 blk_complete_reqs+0x8e/0xb0 blk_done_softirq+0x1d/0x30 [...] read to 0xffff8f0306ba7868 of 8 bytes by task 667002 on cpu 11: raid1_read_request+0x497/0x1a10 raid1_make_request+0xdf/0x1950 md_handle_request+0x2c5/0x700 md_submit_bio+0x126/0x320 __submit_bio+0x2ec/0x3a0 submit_bio_noacct_nocheck+0x572/0x890 [...] value changed: 0x0000000000000078 -> 0x00000000005fe448 Signed-off-by: Chen Cheng Link: https://patch.msgid.link/20260619044114.1208456-1-chencheng@fnnas.com Signed-off-by: Yu Kuai --- drivers/md/raid1.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index 41d9094fa50a..afe2ca96ad8c 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -359,8 +359,8 @@ static inline void update_head_pos(int disk, struct r1bio *r1_bio) { struct r1conf *conf = r1_bio->mddev->private; - conf->mirrors[disk].head_position = - r1_bio->sector + (r1_bio->sectors); + WRITE_ONCE(conf->mirrors[disk].head_position, + r1_bio->sector + r1_bio->sectors); } /* @@ -737,7 +737,7 @@ static bool is_sequential(struct r1conf *conf, int disk, struct r1bio *r1_bio) { /* TODO: address issues with this check and concurrency. */ return conf->mirrors[disk].next_seq_sect == r1_bio->sector || - conf->mirrors[disk].head_position == r1_bio->sector; + READ_ONCE(conf->mirrors[disk].head_position) == r1_bio->sector; } /* @@ -814,7 +814,8 @@ static int choose_best_rdev(struct r1conf *conf, struct r1bio *r1_bio) set_bit(R1BIO_FailFast, &r1_bio->state); pending = atomic_read(&rdev->nr_pending); - dist = abs(r1_bio->sector - conf->mirrors[disk].head_position); + dist = abs(r1_bio->sector - + READ_ONCE(conf->mirrors[disk].head_position)); /* Don't change to another disk for sequential reads */ if (is_sequential(conf, disk, r1_bio)) { From 00e93faf4cea9e8802ac5dfee0952d84fc95c40f Mon Sep 17 00:00:00 2001 From: Chen Cheng Date: Thu, 18 Jun 2026 10:57:35 +0800 Subject: [PATCH 28/45] md/raid5: let stripe batch bm_seq comparison wrap-safe Once the 32-bit seq wraps, a newer bm_seq can look smaller than old, so .. covert to wrap-safe calculate way. Signed-off-by: Chen Cheng Link: https://patch.msgid.link/20260618025735.915113-1-chencheng@fnnas.com Signed-off-by: Yu Kuai --- drivers/md/raid5.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 180ff0660b6a..f35c2a7b2be1 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -996,7 +996,7 @@ static void stripe_add_to_batch_list(struct r5conf *conf, if (test_and_clear_bit(STRIPE_BIT_DELAY, &sh->state)) { int seq = sh->bm_seq; if (test_bit(STRIPE_BIT_DELAY, &sh->batch_head->state) && - sh->batch_head->bm_seq > seq) + sh->batch_head->bm_seq - seq > 0) seq = sh->batch_head->bm_seq; set_bit(STRIPE_BIT_DELAY, &sh->batch_head->state); sh->batch_head->bm_seq = seq; From 214cdae69dba9bb1fc0b517b7fb97bab385a2e3a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 22 Jun 2026 18:07:52 +0200 Subject: [PATCH 29/45] block: fix incorrect error injection static key decrement Only decrement the static key when we had items and thus it was incremented before. Fixes: e8dcf2d142bd ("block: add configurable error injection") Reported-by: Damien Le Moal Signed-off-by: Christoph Hellwig Link: https://patch.msgid.link/20260622160752.1552516-1-hch@lst.de Signed-off-by: Jens Axboe --- block/error-injection.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/block/error-injection.c b/block/error-injection.c index d24c90e9a25f..cfb83138960c 100644 --- a/block/error-injection.c +++ b/block/error-injection.c @@ -120,13 +120,13 @@ static void error_inject_removeall(struct gendisk *disk) struct blk_error_inject *inj; mutex_lock(&disk->error_injection_lock); - clear_bit(GD_ERROR_INJECT, &disk->state); + if (test_and_clear_bit(GD_ERROR_INJECT, &disk->state)) + static_branch_dec(&blk_error_injection_enabled); while ((inj = list_first_entry_or_null(&disk->error_injection_list, struct blk_error_inject, entry))) { list_del_rcu(&inj->entry); kfree_rcu_mightsleep(inj); } - static_branch_dec(&blk_error_injection_enabled); mutex_unlock(&disk->error_injection_lock); } From 9280e6edf65662b6aafc8b704ad065b54c08b519 Mon Sep 17 00:00:00 2001 From: Deepanshu Kartikey Date: Mon, 22 Jun 2026 05:22:55 +0530 Subject: [PATCH 30/45] nbd: don't warn when reclassifying a busy socket lock nbd_reclassify_socket() warns via WARN_ON_ONCE() if the socket lock is held at the point of reclassification. That assertion was copied from nvme-tcp, where the socket is created internally by the kernel (sock_create_kern()) and is never visible to user space, so the lock is guaranteed to be free. NBD is different: the socket is looked up from a user-supplied fd in nbd_get_socket(), and user space retains that fd. A concurrent syscall on the same socket (or softirq processing taking bh_lock_sock() on a connected TCP socket) can legitimately hold the lock at the instant NBD reclassifies it. sock_allow_reclassification() then returns false and the WARN_ON_ONCE() fires, which turns into a crash under panic_on_warn. This is reachable by simply racing NBD_CMD_CONNECT against socket activity on the same fd, as reported by syzbot. Hitting a held lock here is expected for an externally owned socket and is not a kernel bug, so skip reclassification silently instead of warning. Reclassification is a lockdep-only annotation, so skipping it in the rare racing case is harmless. Reported-by: syzbot+6b85d1e39a5b8ed9a954@syzkaller.appspotmail.com Closes: https://syzkaller.appspot.com/bug?extid=6b85d1e39a5b8ed9a954 Fixes: d532cddb6c60 ("nbd: Reclassify sockets to avoid lockdep circular dependency") Signed-off-by: Deepanshu Kartikey Acked-by: Eric Dumazet Link: https://patch.msgid.link/20260621235255.66015-1-kartikey406@gmail.com Signed-off-by: Jens Axboe --- drivers/block/nbd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 3a585a0c882a..8f10762e90ef 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -1246,7 +1246,7 @@ static void nbd_reclassify_socket(struct socket *sock) { struct sock *sk = sock->sk; - if (WARN_ON_ONCE(!sock_allow_reclassification(sk))) + if (!sock_allow_reclassification(sk)) return; switch (sk->sk_family) { From 17b2d950a3c0328ed749476e6118ca869b3ca8b5 Mon Sep 17 00:00:00 2001 From: Cen Zhang Date: Sun, 21 Jun 2026 21:59:30 +0800 Subject: [PATCH 31/45] block, bfq: protect async queue reset with blkcg locks Writing 0 to BFQ's low_latency attribute ends weight raising for active, idle and async queues. The async cgroup path walks q->blkg_list, converts each blkg to BFQ policy data and then reads bfqg->async_bfqq and bfqg->async_idle_bfqq. That walk was protected only by bfqd->lock. blkcg release work is serialized by q->blkcg_mutex and q->queue_lock instead, and blkg_free_workfn() can call BFQ's pd_free_fn before it removes blkg->q_node from q->blkg_list. A low_latency reset can therefore still find the blkg on the queue list after the BFQ policy data has been freed. The buggy scenario involves two paths, with each column showing the order within that path: BFQ low_latency reset: blkcg blkg release work: 1. bfq_low_latency_store() 1. blkg_free_workfn() takes calls bfq_end_wr(). q->blkcg_mutex. 2. bfq_end_wr_async() walks 2. BFQ pd_free_fn drops the q->blkg_list. final bfq_group reference. 3. blkg_to_bfqg() returns 3. blkg->q_node remains on the stale policy data. q->blkg_list until list_del_init(). 4. bfq_end_wr_async_queues() reads async queue fields. Fix this by taking q->blkcg_mutex and q->queue_lock around the q->blkg_list walk, then taking bfqd->lock before touching BFQ async queues. The mutex serializes against policy-data free and queue_lock stabilizes the list. Move the async reset out of bfq_end_wr()'s existing bfqd->lock critical section so the lock order matches blkcg policy callbacks. Validation reproduced this kernel report: BUG: KASAN: slab-use-after-free in bfq_end_wr_async_queues+0x246/0x340 Call Trace: dump_stack_lvl+0x66/0xa0 print_report+0xce/0x630 ? bfq_end_wr_async_queues+0x246/0x340 ? srso_alias_return_thunk+0x5/0xfbef5 ? __virt_addr_valid+0x20d/0x410 ? bfq_end_wr_async_queues+0x246/0x340 kasan_report+0xe0/0x110 ? bfq_end_wr_async_queues+0x246/0x340 bfq_end_wr_async_queues+0x246/0x340 bfq_end_wr_async+0xba/0x180 bfq_low_latency_store+0x4e5/0x690 ? 0xffffffffc02150da ? __pfx_bfq_low_latency_store+0x10/0x10 ? __pfx_bfq_low_latency_store+0x10/0x10 elv_attr_store+0xc4/0x110 kernfs_fop_write_iter+0x2f5/0x4a0 vfs_write+0x604/0x11f0 ? __pfx_locks_remove_posix+0x10/0x10 ? __pfx_vfs_write+0x10/0x10 ksys_write+0xf9/0x1d0 ? __pfx_ksys_write+0x10/0x10 do_syscall_64+0x115/0x6a0 entry_SYSCALL_64_after_hwframe+0x77/0x7f Allocated by task 544: kasan_save_stack+0x33/0x60 kasan_save_track+0x14/0x30 __kasan_kmalloc+0xaa/0xb0 bfq_pd_alloc+0xc0/0x1b0 blkg_alloc+0x346/0x960 blkg_create+0x8c2/0x10d0 bio_associate_blkg_from_css+0x9f3/0xfa0 bio_associate_blkg+0xd9/0x200 bio_init+0x303/0x640 __blkdev_direct_IO_simple+0x56b/0x8a0 blkdev_direct_IO+0x8e7/0x2580 blkdev_read_iter+0x205/0x400 vfs_read+0x7b0/0xda0 ksys_read+0xf9/0x1d0 do_syscall_64+0x115/0x6a0 entry_SYSCALL_64_after_hwframe+0x77/0x7f Freed by task 465: kasan_save_stack+0x33/0x60 kasan_save_track+0x14/0x30 kasan_save_free_info+0x3b/0x60 __kasan_slab_free+0x5f/0x80 kfree+0x307/0x580 blkg_free_workfn+0xef/0x460 process_one_work+0x8d0/0x1870 worker_thread+0x575/0xf80 kthread+0x2e7/0x3c0 ret_from_fork+0x576/0x810 ret_from_fork_asm+0x1a/0x30 Fixes: 44e44a1b329e ("block, bfq: improve responsiveness") Assisted-by: Codex:gpt-5.5 Signed-off-by: Cen Zhang Reviewed-by: Tao Cui Link: https://patch.msgid.link/20260621135930.2657810-1-zzzccc427@gmail.com Signed-off-by: Jens Axboe --- block/bfq-cgroup.c | 13 ++++++++++++- block/bfq-iosched.c | 3 ++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c index 0bd0332b3d78..d8fdace464b4 100644 --- a/block/bfq-cgroup.c +++ b/block/bfq-cgroup.c @@ -936,14 +936,23 @@ static void bfq_pd_offline(struct blkg_policy_data *pd) void bfq_end_wr_async(struct bfq_data *bfqd) { + struct request_queue *q = bfqd->queue; struct blkcg_gq *blkg; - list_for_each_entry(blkg, &bfqd->queue->blkg_list, q_node) { + mutex_lock(&q->blkcg_mutex); + spin_lock_irq(&q->queue_lock); + spin_lock(&bfqd->lock); + + list_for_each_entry(blkg, &q->blkg_list, q_node) { struct bfq_group *bfqg = blkg_to_bfqg(blkg); bfq_end_wr_async_queues(bfqd, bfqg); } bfq_end_wr_async_queues(bfqd, bfqd->root_group); + + spin_unlock(&bfqd->lock); + spin_unlock_irq(&q->queue_lock); + mutex_unlock(&q->blkcg_mutex); } static int bfq_io_show_weight_legacy(struct seq_file *sf, void *v) @@ -1416,7 +1425,9 @@ void bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio) {} void bfq_end_wr_async(struct bfq_data *bfqd) { + spin_lock_irq(&bfqd->lock); bfq_end_wr_async_queues(bfqd, bfqd->root_group); + spin_unlock_irq(&bfqd->lock); } struct bfq_group *bfq_bio_bfqg(struct bfq_data *bfqd, struct bio *bio) diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c index 141c602d5e85..eec9be62061b 100644 --- a/block/bfq-iosched.c +++ b/block/bfq-iosched.c @@ -2653,9 +2653,10 @@ static void bfq_end_wr(struct bfq_data *bfqd) } list_for_each_entry(bfqq, &bfqd->idle_list, bfqq_list) bfq_bfqq_end_wr(bfqq); - bfq_end_wr_async(bfqd); spin_unlock_irq(&bfqd->lock); + + bfq_end_wr_async(bfqd); } static sector_t bfq_io_struct_pos(void *io_struct, bool request) From 0ab5ee5a1badb58cbb2242617cb01a4972b1f2a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Koutn=C3=BD?= Date: Thu, 5 Feb 2026 23:54:23 +0800 Subject: [PATCH 32/45] blk-cgroup: fix UAF in __blkcg_rstat_flush() When multiple blkgs in the same blkcg are released concurrently, a use-after-free can occur. The race happens when one blkg's __blkcg_rstat_flush() removes another blkg's iostat entries via llist_del_all(). The second blkg sees an empty list and proceeds to free itself while the first is still iterating over its entries. Move the flush from __blkg_release() (RCU callback) to blkg_release() (before call_rcu). This ensures the RCU grace period waits for any concurrent flush's rcu_read_lock() section to complete before freeing. Cc: stable@vger.kernel.org Cc: Jay Shin Cc: Tejun Heo Cc: Waiman Long Fixes: 20cb1c2fb756 ("blk-cgroup: Flush stats before releasing blkcg_gq") Reported-by: coregee2000@gmail.com Closes: https://lore.kernel.org/linux-block/CAHPqNmwT9oRpem3J3erS_W0uSQND47LGGSBsNxP8E6uSUish1w@mail.gmail.com/ Signed-off-by: Ming Lei Tested-by: Jose Fernandez (Anthropic) Link: https://patch.msgid.link/20260205155425.342084-1-ming.lei@redhat.com Signed-off-by: Jens Axboe --- block/blk-cgroup.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index 3093c1c03902..342816cbbd1b 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -164,20 +164,10 @@ static void blkg_free(struct blkcg_gq *blkg) static void __blkg_release(struct rcu_head *rcu) { struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head); - struct blkcg *blkcg = blkg->blkcg; - int cpu; #ifdef CONFIG_BLK_CGROUP_PUNT_BIO WARN_ON(!bio_list_empty(&blkg->async_bios)); #endif - /* - * Flush all the non-empty percpu lockless lists before releasing - * us, given these stat belongs to us. - * - * blkg_stat_lock is for serializing blkg stat update - */ - for_each_possible_cpu(cpu) - __blkcg_rstat_flush(blkcg, cpu); /* release the blkcg and parent blkg refs this blkg has been holding */ css_put(&blkg->blkcg->css); @@ -195,6 +185,17 @@ static void __blkg_release(struct rcu_head *rcu) static void blkg_release(struct percpu_ref *ref) { struct blkcg_gq *blkg = container_of(ref, struct blkcg_gq, refcnt); + struct blkcg *blkcg = blkg->blkcg; + int cpu; + + /* + * Flush all the non-empty percpu lockless lists before releasing + * us, given these stat belongs to us. + * + * blkg_stat_lock is for serializing blkg stat update + */ + for_each_possible_cpu(cpu) + __blkcg_rstat_flush(blkcg, cpu); call_rcu(&blkg->rcu_head, __blkg_release); } From 3ed9b4779a4aa3f44cd9f78627498d7adac40daa Mon Sep 17 00:00:00 2001 From: Zizhi Wo Date: Tue, 16 Jun 2026 09:17:46 +0800 Subject: [PATCH 33/45] blk-cgroup: defer blkcg css_put until blkg is unlinked from queue [BUG] Our fuzz testing triggered a blkcg use-after-free issue: BUG: KASAN: slab-use-after-free in _raw_spin_lock+0x75/0xe0 Call Trace: ... blkcg_deactivate_policy+0x244/0x4d0 ioc_rqos_exit+0x44/0xe0 rq_qos_exit+0xba/0x120 __del_gendisk+0x50b/0x800 del_gendisk+0xff/0x190 ... [CAUSE] process1 process2 cgroup_rmdir ... css_killed_work_fn offline_css ... blkcg_destroy_blkgs ... __blkg_release css_put(&blkg->blkcg->css) blkg_free INIT_WORK(xxx, blkg_free_workfn) schedule_work css_put ... blkcg_css_free kfree(blkcg)--------blkcg has been freed!!! ====================================schedule_work blkg_free_workfn __del_gendisk rq_qos_exit ioc_rqos_exit blkcg_deactivate_policy mutex_lock(&q->blkcg_mutex) spin_lock_irq(&q->queue_lock) list_for_each_entry(blkg, xxx) blkcg = blkg->blkcg spin_lock(&blkcg->lock)-------UAF!!! mutex_lock(&q->blkcg_mutex) spin_lock_irq(&q->queue_lock) /* Only then is the blkg removed from the list */ list_del_init(&blkg->q_node) As a result, a blkg can still be reachable through q->blkg_list while its ->blkcg has already been freed. [Fix] Fix this by deferring the blkcg css_put() until after the blkg has been unlinked from q->blkg_list in blkg_free_workfn(). This ensures that the blkcg outlives every blkg still reachable through q->blkg_list, so any iterator holding q->queue_lock is guaranteed to observe a valid blkg->blkcg. While at it, move css_tryget_online() from blkg_create() into blkg_alloc() so that the css reference is owned by the alloc/free pair rather than straddling layers: blkg_alloc() <-> blkg_free() blkg_create() <-> blkg_destroy() Fixes: f1c006f1c685 ("blk-cgroup: synchronize pd_free_fn() from blkg_free_workfn() and blkcg_deactivate_policy()") Suggested-by: Hou Tao Signed-off-by: Zizhi Wo Reviewed-by: Yu Kuai Reviewed-by: Tang Yizhou Link: https://patch.msgid.link/20260616011746.2451461-1-wozizhi@huaweicloud.com Signed-off-by: Jens Axboe --- block/blk-cgroup.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index 342816cbbd1b..ee076ab795d3 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -136,6 +136,11 @@ static void blkg_free_workfn(struct work_struct *work) spin_unlock_irq(&q->queue_lock); mutex_unlock(&q->blkcg_mutex); + /* + * Release blkcg css ref only after blkg is removed from q->blkg_list, + * so concurrent iterators won't see a blkg with a freed blkcg. + */ + css_put(&blkg->blkcg->css); blk_put_queue(q); free_percpu(blkg->iostat_cpu); percpu_ref_exit(&blkg->refcnt); @@ -169,8 +174,6 @@ static void __blkg_release(struct rcu_head *rcu) WARN_ON(!bio_list_empty(&blkg->async_bios)); #endif - /* release the blkcg and parent blkg refs this blkg has been holding */ - css_put(&blkg->blkcg->css); blkg_free(blkg); } @@ -314,6 +317,9 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk, goto out_exit_refcnt; if (!blk_get_queue(disk->queue)) goto out_free_iostat; + /* blkg holds a reference to blkcg */ + if (!css_tryget_online(&blkcg->css)) + goto out_put_queue; blkg->q = disk->queue; INIT_LIST_HEAD(&blkg->q_node); @@ -354,6 +360,8 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk, while (--i >= 0) if (blkg->pd[i]) blkcg_policy[i]->pd_free_fn(blkg->pd[i]); + css_put(&blkcg->css); +out_put_queue: blk_put_queue(disk->queue); out_free_iostat: free_percpu(blkg->iostat_cpu); @@ -382,18 +390,12 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct gendisk *disk, goto err_free_blkg; } - /* blkg holds a reference to blkcg */ - if (!css_tryget_online(&blkcg->css)) { - ret = -ENODEV; - goto err_free_blkg; - } - /* allocate */ if (!new_blkg) { new_blkg = blkg_alloc(blkcg, disk, GFP_NOWAIT); if (unlikely(!new_blkg)) { ret = -ENOMEM; - goto err_put_css; + goto err_free_blkg; } } blkg = new_blkg; @@ -403,7 +405,7 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct gendisk *disk, blkg->parent = blkg_lookup(blkcg_parent(blkcg), disk->queue); if (WARN_ON_ONCE(!blkg->parent)) { ret = -ENODEV; - goto err_put_css; + goto err_free_blkg; } blkg_get(blkg->parent); } @@ -443,8 +445,6 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct gendisk *disk, blkg_put(blkg); return ERR_PTR(ret); -err_put_css: - css_put(&blkcg->css); err_free_blkg: if (new_blkg) blkg_free(new_blkg); From 9b249f5ffbeda24c57e6c56ed896c5ca4fc70549 Mon Sep 17 00:00:00 2001 From: Chen Cheng Date: Thu, 18 Jun 2026 21:47:48 +0800 Subject: [PATCH 34/45] md/raid5: use stripe state snapshot in break_stripe_batch_list() The patch just suppress KCSAN noise. No functional change. RAID-5 can group multi full-stripe-write aka stripe_head into a batch aka batch_list, with one head_sh leading them. Call break_stripe_batch_list() when the batch is finished, or, a stripe has to be dropped out of the batch. break_stripe_batch_list() reads stripe state several times while request paths can update thost state words concurrently with lockless bitops, which reported by KCSAN. Use a snapshot to guarantees that the value used for warning, copying, and handle checks is internally consistent at current read moment. KCSAN report: ============================================== BUG: KCSAN: data-race in __add_stripe_bio / break_stripe_batch_list write (marked) to 0xffff8e89d4f0b988 of 8 bytes by task 4323 on cpu 3: __add_stripe_bio+0x35e/0x400 raid5_make_request+0x6ac/0x2930 md_handle_request+0x4a2/0xa40 md_submit_bio+0x109/0x1a0 __submit_bio+0x2ec/0x390 submit_bio_noacct_nocheck+0x457/0x710 submit_bio_noacct+0x2a7/0xc20 submit_bio+0x56/0x250 blkdev_direct_IO+0x54c/0xda0 blkdev_write_iter+0x38f/0x570 aio_write+0x22b/0x490 io_submit_one+0xa51/0xf70 read to 0xffff8e89d4f0b988 of 8 bytes by task 4290 on cpu 4: break_stripe_batch_list+0x3ce/0x480 handle_stripe_clean_event+0x720/0x9b0 handle_stripe+0x32fb/0x4500 handle_active_stripes.isra.0+0x6e0/0xa50 raid5d+0x7e0/0xba0 Signed-off-by: Chen Cheng Link: https://patch.msgid.link/20260618134748.1168360-1-chencheng@fnnas.com Signed-off-by: Yu Kuai --- drivers/md/raid5.c | 45 ++++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index f35c2a7b2be1..6d982c54f2d1 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -4852,31 +4852,35 @@ static void break_stripe_batch_list(struct stripe_head *head_sh, { struct stripe_head *sh, *next; int i; + unsigned long state; list_for_each_entry_safe(sh, next, &head_sh->batch_list, batch_list) { list_del_init(&sh->batch_list); - WARN_ONCE(sh->state & ((1 << STRIPE_ACTIVE) | - (1 << STRIPE_SYNCING) | - (1 << STRIPE_REPLACED) | - (1 << STRIPE_DELAYED) | - (1 << STRIPE_BIT_DELAY) | - (1 << STRIPE_FULL_WRITE) | - (1 << STRIPE_BIOFILL_RUN) | - (1 << STRIPE_COMPUTE_RUN) | - (1 << STRIPE_DISCARD) | - (1 << STRIPE_BATCH_READY) | - (1 << STRIPE_BATCH_ERR)), - "stripe state: %lx\n", sh->state); - WARN_ONCE(head_sh->state & ((1 << STRIPE_DISCARD) | - (1 << STRIPE_REPLACED)), - "head stripe state: %lx\n", head_sh->state); + state = READ_ONCE(sh->state); + WARN_ONCE(state & ((1 << STRIPE_ACTIVE) | + (1 << STRIPE_SYNCING) | + (1 << STRIPE_REPLACED) | + (1 << STRIPE_DELAYED) | + (1 << STRIPE_BIT_DELAY) | + (1 << STRIPE_FULL_WRITE) | + (1 << STRIPE_BIOFILL_RUN) | + (1 << STRIPE_COMPUTE_RUN) | + (1 << STRIPE_DISCARD) | + (1 << STRIPE_BATCH_READY) | + (1 << STRIPE_BATCH_ERR)), + "stripe state: %lx\n", state); + + state = READ_ONCE(head_sh->state); + WARN_ONCE(state & ((1 << STRIPE_DISCARD) | + (1 << STRIPE_REPLACED)), + "head stripe state: %lx\n", state); set_mask_bits(&sh->state, ~(STRIPE_EXPAND_SYNC_FLAGS | (1 << STRIPE_PREREAD_ACTIVE) | (1 << STRIPE_ON_UNPLUG_LIST)), - head_sh->state & (1 << STRIPE_INSYNC)); + state & (1 << STRIPE_INSYNC)); sh->check_state = head_sh->check_state; sh->reconstruct_state = head_sh->reconstruct_state; @@ -4889,8 +4893,9 @@ static void break_stripe_batch_list(struct stripe_head *head_sh, sh->dev[i].flags = head_sh->dev[i].flags & (~((1 << R5_WriteError) | (1 << R5_Overlap))); } - if (handle_flags == 0 || - sh->state & handle_flags) + + state = READ_ONCE(sh->state); + if (handle_flags == 0 || (state & handle_flags)) set_bit(STRIPE_HANDLE, &sh->state); raid5_release_stripe(sh); } @@ -4900,7 +4905,9 @@ static void break_stripe_batch_list(struct stripe_head *head_sh, for (i = 0; i < head_sh->disks; i++) if (test_and_clear_bit(R5_Overlap, &head_sh->dev[i].flags)) wake_up_bit(&head_sh->dev[i].flags, R5_Overlap); - if (head_sh->state & handle_flags) + + state = READ_ONCE(head_sh->state); + if (state & handle_flags) set_bit(STRIPE_HANDLE, &head_sh->state); } From 55b77337bdd088c77461588e5ec094421b89911b Mon Sep 17 00:00:00 2001 From: Chen Cheng Date: Fri, 19 Jun 2026 12:10:13 +0800 Subject: [PATCH 35/45] md/raid5: avoid R5_Overlap races while breaking stripe batches KCSAN report a race in break_stripe_batch_list() vs. raid5_make_request() on sh->dev[i].flags (plain word write vs. atomic bit op).. and .. one possible scenario is: CPU1 CPU2 break_stripe_batch_list(sh1) -> handle sh2 -> lock(sh2) -> sh2->batch_head = NULL -> unlock(sh2) -> test_and_clear_bit(R5_Overlap, sh2->dev[i].flags) -> wake_up_bit(sh2->dev[i].flags) raid5_make_request() -> add_all_stripe_bios(sh2) -> lock(sh2) -> stripe_bio_overlaps(sh2) returns true batch_head is NULL, so new bio overlap exist bio on sh2 -> true -> set_bit(R5_Overlap, sh2->dev[i].flags) -> unlock(sh2) -> wait_on_bit(sh2->dev[i].flags) -> sh2->dev[i].flags = sh1->dev[i].flags & ~R5_Overlap No wait_up_bit(), CPU2 could be wait_on_bit() forever... Fix by : - Expand the protect zone. - Use batch_head's device flag's snaphot when no held head_sh->stripe_lock. - Move sh/head_sh->batch_head = NULL to the end of protected zone , and , any concurrent add_all_stripe_bios() grabs sh->stripe_lock now either: - see batch_head != null, and , is rejected by stripe_bio_overlaps() under the lock (no R5_Overlap wait ) , or , - sees batch_head == NULL, only after dev[i].flags has already been set and the prior R5_Overlap waiters worken. KCSAN report: ================================================ BUG: KCSAN: data-race in break_stripe_batch_list / raid5_make_request write (marked) to 0xffff8e89c8117548 of 8 bytes by task 4042 on cpu 0: raid5_make_request+0xea0/0x2930 md_handle_request+0x4a2/0xa40 md_submit_bio+0x109/0x1a0 __submit_bio+0x2ec/0x390 submit_bio_noacct_nocheck+0x457/0x710 submit_bio_noacct+0x2a7/0xc20 submit_bio+0x56/0x250 blkdev_direct_IO+0x54c/0xda0 blkdev_write_iter+0x38f/0x570 aio_write+0x22b/0x490 io_submit_one+0xa51/0xf70 __x64_sys_io_submit+0xf7/0x220 x64_sys_call+0x1907/0x1c60 do_syscall_64+0x130/0x570 entry_SYSCALL_64_after_hwframe+0x76/0x7e read to 0xffff8e89c8117548 of 8 bytes by task 4010 on cpu 5: break_stripe_batch_list+0x249/0x480 handle_stripe_clean_event+0x720/0x9b0 handle_stripe+0x32fb/0x4500 handle_active_stripes.isra.0+0x6e0/0xa50 raid5d+0x7e0/0xba0 md_thread+0x15a/0x2d0 kthread+0x1e3/0x220 ret_from_fork+0x37a/0x410 ret_from_fork_asm+0x1a/0x30 value changed: 0x0000000000000019 -> 0x0000000000000099 --> R5_Overlap Fixes: fb642b92c267 ("md/raid5: duplicate some more handle_stripe_clean_event code in break_stripe_batch_list") Signed-off-by: Chen Cheng Link: https://patch.msgid.link/20260619041013.1207148-1-chencheng@fnnas.com Signed-off-by: Yu Kuai --- drivers/md/raid5.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 6d982c54f2d1..0c5c9fb0606e 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -4885,14 +4885,14 @@ static void break_stripe_batch_list(struct stripe_head *head_sh, sh->check_state = head_sh->check_state; sh->reconstruct_state = head_sh->reconstruct_state; spin_lock_irq(&sh->stripe_lock); - sh->batch_head = NULL; - spin_unlock_irq(&sh->stripe_lock); for (i = 0; i < sh->disks; i++) { if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) wake_up_bit(&sh->dev[i].flags, R5_Overlap); - sh->dev[i].flags = head_sh->dev[i].flags & + sh->dev[i].flags = READ_ONCE(head_sh->dev[i].flags) & (~((1 << R5_WriteError) | (1 << R5_Overlap))); } + sh->batch_head = NULL; + spin_unlock_irq(&sh->stripe_lock); state = READ_ONCE(sh->state); if (handle_flags == 0 || (state & handle_flags)) @@ -4900,11 +4900,11 @@ static void break_stripe_batch_list(struct stripe_head *head_sh, raid5_release_stripe(sh); } spin_lock_irq(&head_sh->stripe_lock); - head_sh->batch_head = NULL; - spin_unlock_irq(&head_sh->stripe_lock); for (i = 0; i < head_sh->disks; i++) if (test_and_clear_bit(R5_Overlap, &head_sh->dev[i].flags)) wake_up_bit(&head_sh->dev[i].flags, R5_Overlap); + head_sh->batch_head = NULL; + spin_unlock_irq(&head_sh->stripe_lock); state = READ_ONCE(head_sh->state); if (state & handle_flags) From 25656304dabd26198ec69460c594a19d086ef099 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 8 Jun 2026 11:42:42 +0800 Subject: [PATCH 36/45] blk-cgroup: protect iterating blkgs with blkcg->lock in blkcg_print_stat() blkcg_print_one_stat() will be called for each blkg: - access blkg->iostat, which is freed from rcu callback blkg_free_workfn(); - access policy data from pd_stat_fn(), which is freed from pd_free_fn(), while pd_free_fn() can be called by removing blkcg or deactivating policy; Take blkcg->lock while iterating so the blkgs stay online and both blkg->iostat and policy data for activated policies stay valid. Use irq-safe locking because blkcg->lock can be nested under q->queue_lock, which is used from IRQ completion paths. Prepare to convert protecting blkgs from request_queue with mutex. Signed-off-by: Yu Kuai Link: https://patch.msgid.link/05799877e720dcd300e2ddd4625e8e162959d7cc.1780621988.git.yukuai@fygo.io Signed-off-by: Jens Axboe --- block/blk-cgroup.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index ee076ab795d3..c991c263cb5a 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -1191,13 +1191,10 @@ static int blkcg_print_stat(struct seq_file *sf, void *v) else css_rstat_flush(&blkcg->css); - rcu_read_lock(); - hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { - spin_lock_irq(&blkg->q->queue_lock); + guard(spinlock_irq)(&blkcg->lock); + hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) blkcg_print_one_stat(blkg, sf); - spin_unlock_irq(&blkg->q->queue_lock); - } - rcu_read_unlock(); + return 0; } From 0af3fedb8c8ed3c07b4f76927bd7fc88f6f82efb Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 8 Jun 2026 11:42:43 +0800 Subject: [PATCH 37/45] blk-cgroup: delay freeing policy data after rcu grace period Currently blkcg_print_blkgs() must hold RCU to iterate blkgs from a blkcg, and prfill() must hold queue_lock to prevent policy data from being freed by policy deactivation. As a consequence, queue_lock has to be nested under RCU from blkcg_print_blkgs(). Delay freeing policy data until after an RCU grace period so prfill() can be protected by RCU alone. Signed-off-by: Yu Kuai Link: https://patch.msgid.link/e20e5d984b41a026d61851966bed35eb094c4bff.1780621988.git.yukuai@fygo.io Signed-off-by: Jens Axboe --- block/bfq-cgroup.c | 9 ++++++++- block/blk-cgroup.h | 2 ++ block/blk-iocost.c | 14 ++++++++++++-- block/blk-iolatency.c | 10 +++++++++- block/blk-throttle.c | 15 ++++++++++++--- 5 files changed, 43 insertions(+), 7 deletions(-) diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c index d8fdace464b4..0d3e32d246a2 100644 --- a/block/bfq-cgroup.c +++ b/block/bfq-cgroup.c @@ -550,13 +550,20 @@ static void bfq_pd_init(struct blkg_policy_data *pd) bfqg->rq_pos_tree = RB_ROOT; } -static void bfq_pd_free(struct blkg_policy_data *pd) +static void bfqg_release(struct rcu_head *rcu) { + struct blkg_policy_data *pd = + container_of(rcu, struct blkg_policy_data, rcu_head); struct bfq_group *bfqg = pd_to_bfqg(pd); bfqg_put(bfqg); } +static void bfq_pd_free(struct blkg_policy_data *pd) +{ + call_rcu(&pd->rcu_head, bfqg_release); +} + static void bfq_pd_reset_stats(struct blkg_policy_data *pd) { struct bfq_group *bfqg = pd_to_bfqg(pd); diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h index f25fecb87c43..cc603ded6ded 100644 --- a/block/blk-cgroup.h +++ b/block/blk-cgroup.h @@ -140,6 +140,8 @@ struct blkg_policy_data { struct blkcg_gq *blkg; int plid; bool online; + + struct rcu_head rcu_head; }; /* diff --git a/block/blk-iocost.c b/block/blk-iocost.c index 563cc7dcf348..27d2dcaa65f0 100644 --- a/block/blk-iocost.c +++ b/block/blk-iocost.c @@ -3050,6 +3050,16 @@ static void ioc_pd_init(struct blkg_policy_data *pd) spin_unlock_irqrestore(&ioc->lock, flags); } +static void iocg_release(struct rcu_head *rcu) +{ + struct blkg_policy_data *pd = + container_of(rcu, struct blkg_policy_data, rcu_head); + struct ioc_gq *iocg = pd_to_iocg(pd); + + free_percpu(iocg->pcpu_stat); + kfree(iocg); +} + static void ioc_pd_free(struct blkg_policy_data *pd) { struct ioc_gq *iocg = pd_to_iocg(pd); @@ -3074,8 +3084,8 @@ static void ioc_pd_free(struct blkg_policy_data *pd) hrtimer_cancel(&iocg->waitq_timer); } - free_percpu(iocg->pcpu_stat); - kfree(iocg); + + call_rcu(&pd->rcu_head, iocg_release); } static void ioc_pd_stat(struct blkg_policy_data *pd, struct seq_file *s) diff --git a/block/blk-iolatency.c b/block/blk-iolatency.c index 1aaee6fb0f59..cef02b6c5fa9 100644 --- a/block/blk-iolatency.c +++ b/block/blk-iolatency.c @@ -1031,13 +1031,21 @@ static void iolatency_pd_offline(struct blkg_policy_data *pd) iolatency_clear_scaling(blkg); } -static void iolatency_pd_free(struct blkg_policy_data *pd) +static void iolat_release(struct rcu_head *rcu) { + struct blkg_policy_data *pd = + container_of(rcu, struct blkg_policy_data, rcu_head); struct iolatency_grp *iolat = pd_to_lat(pd); + free_percpu(iolat->stats); kfree(iolat); } +static void iolatency_pd_free(struct blkg_policy_data *pd) +{ + call_rcu(&pd->rcu_head, iolat_release); +} + static struct cftype iolatency_files[] = { { .name = "latency", diff --git a/block/blk-throttle.c b/block/blk-throttle.c index 47052ba21d1b..ffc3b70065d4 100644 --- a/block/blk-throttle.c +++ b/block/blk-throttle.c @@ -353,14 +353,23 @@ static void throtl_pd_online(struct blkg_policy_data *pd) tg_update_has_rules(tg); } +static void tg_release(struct rcu_head *rcu) +{ + struct blkg_policy_data *pd = + container_of(rcu, struct blkg_policy_data, rcu_head); + struct throtl_grp *tg = pd_to_tg(pd); + + blkg_rwstat_exit(&tg->stat_bytes); + blkg_rwstat_exit(&tg->stat_ios); + kfree(tg); +} + static void throtl_pd_free(struct blkg_policy_data *pd) { struct throtl_grp *tg = pd_to_tg(pd); timer_delete_sync(&tg->service_queue.pending_timer); - blkg_rwstat_exit(&tg->stat_bytes); - blkg_rwstat_exit(&tg->stat_ios); - kfree(tg); + call_rcu(&pd->rcu_head, tg_release); } static struct throtl_grp * From 56cc24f59c145ce6938959f792df04b8a4f5a4d8 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 8 Jun 2026 11:42:44 +0800 Subject: [PATCH 38/45] blk-cgroup: don't nest queue_lock under rcu in blkcg_print_blkgs() With previous modification to delay freeing policy data after an RCU grace period, prfill() can run under RCU instead of taking queue_lock. However, policy teardown can still clear blkg->pd[plid] after blkcg_print_blkgs() observes the policy enabled bit. Load policy data once with READ_ONCE() and skip the blkg if teardown already cleared it. Do the same in recursive stat walks for descendant blkgs. Remove the stale BFQ debug queue_lock assertion because blkcg_print_blkgs() no longer calls prfill() with queue_lock held. This also lets ioc_qos_prfill() and ioc_cost_model_prfill() use IRQ-safe ioc->lock locking without re-enabling IRQs while queue_lock is still held. Signed-off-by: Yu Kuai Link: https://patch.msgid.link/db7633d5e263dd1c2bf9b901762545a84b7d714e.1780621988.git.yukuai@fygo.io Signed-off-by: Jens Axboe --- block/bfq-cgroup.c | 8 +++++--- block/blk-cgroup-rwstat.c | 15 +++++++++------ block/blk-cgroup.c | 22 +++++++++++++--------- block/blk-cgroup.h | 6 +++--- block/blk-iocost.c | 8 ++++---- 5 files changed, 34 insertions(+), 25 deletions(-) diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c index 0d3e32d246a2..e82ff03bda02 100644 --- a/block/bfq-cgroup.c +++ b/block/bfq-cgroup.c @@ -1163,16 +1163,18 @@ static u64 bfqg_prfill_stat_recursive(struct seq_file *sf, struct cgroup_subsys_state *pos_css; u64 sum = 0; - lockdep_assert_held(&blkg->q->queue_lock); - rcu_read_lock(); blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) { + struct blkg_policy_data *pd; struct bfq_stat *stat; if (!pos_blkg->online) continue; - stat = (void *)blkg_to_pd(pos_blkg, &blkcg_policy_bfq) + off; + pd = blkg_to_pd(pos_blkg, &blkcg_policy_bfq); + if (!pd) + continue; + stat = (void *)pd + off; sum += bfq_stat_read(stat) + atomic64_read(&stat->aux_cnt); } rcu_read_unlock(); diff --git a/block/blk-cgroup-rwstat.c b/block/blk-cgroup-rwstat.c index a55fb0c53558..aae910713814 100644 --- a/block/blk-cgroup-rwstat.c +++ b/block/blk-cgroup-rwstat.c @@ -101,24 +101,27 @@ void blkg_rwstat_recursive_sum(struct blkcg_gq *blkg, struct blkcg_policy *pol, struct cgroup_subsys_state *pos_css; unsigned int i; - lockdep_assert_held(&blkg->q->queue_lock); + WARN_ON_ONCE(!rcu_read_lock_held()); memset(sum, 0, sizeof(*sum)); - rcu_read_lock(); blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) { struct blkg_rwstat *rwstat; if (!pos_blkg->online) continue; - if (pol) - rwstat = (void *)blkg_to_pd(pos_blkg, pol) + off; - else + if (pol) { + struct blkg_policy_data *pd = blkg_to_pd(pos_blkg, pol); + + if (!pd) + continue; + rwstat = (void *)pd + off; + } else { rwstat = (void *)pos_blkg + off; + } for (i = 0; i < BLKG_RWSTAT_NR; i++) sum->cnt[i] += blkg_rwstat_read_counter(rwstat, i); } - rcu_read_unlock(); } EXPORT_SYMBOL_GPL(blkg_rwstat_recursive_sum); diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index c991c263cb5a..d6355338f290 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -699,9 +699,9 @@ const char *blkg_dev_name(struct blkcg_gq *blkg) * * This function invokes @prfill on each blkg of @blkcg if pd for the * policy specified by @pol exists. @prfill is invoked with @sf, the - * policy data and @data and the matching queue lock held. If @show_total - * is %true, the sum of the return values from @prfill is printed with - * "Total" label at the end. + * policy data and @data under RCU read lock. If @show_total is %true, the + * sum of the return values from @prfill is printed with "Total" label at the + * end. * * This is to be used to construct print functions for * cftype->read_seq_string method. @@ -717,10 +717,14 @@ void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg, rcu_read_lock(); hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { - spin_lock_irq(&blkg->q->queue_lock); - if (blkcg_policy_enabled(blkg->q, pol)) - total += prfill(sf, blkg->pd[pol->plid], data); - spin_unlock_irq(&blkg->q->queue_lock); + struct blkg_policy_data *pd; + + if (!blkcg_policy_enabled(blkg->q, pol)) + continue; + + pd = blkg_to_pd(blkg, pol); + if (pd) + total += prfill(sf, pd, data); } rcu_read_unlock(); @@ -1604,7 +1608,7 @@ int blkcg_activate_policy(struct gendisk *disk, const struct blkcg_policy *pol) pd->blkg = blkg; pd->plid = pol->plid; - blkg->pd[pol->plid] = pd; + WRITE_ONCE(blkg->pd[pol->plid], pd); if (pol->pd_init_fn) pol->pd_init_fn(pd); @@ -1643,7 +1647,7 @@ int blkcg_activate_policy(struct gendisk *disk, const struct blkcg_policy *pol) pol->pd_offline_fn(pd); pd->online = false; pol->pd_free_fn(pd); - blkg->pd[pol->plid] = NULL; + WRITE_ONCE(blkg->pd[pol->plid], NULL); } spin_unlock(&blkcg->lock); } diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h index cc603ded6ded..615390f751aa 100644 --- a/block/blk-cgroup.h +++ b/block/blk-cgroup.h @@ -284,9 +284,9 @@ static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, * Return pointer to private data associated with the @blkg-@pol pair. */ static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg, - struct blkcg_policy *pol) + const struct blkcg_policy *pol) { - return blkg ? blkg->pd[pol->plid] : NULL; + return blkg ? READ_ONCE(blkg->pd[pol->plid]) : NULL; } static inline struct blkcg_policy_data *blkcg_to_cpd(struct blkcg *blkcg, @@ -493,7 +493,7 @@ static inline void blkcg_deactivate_policy(struct gendisk *disk, const struct blkcg_policy *pol) { } static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg, - struct blkcg_policy *pol) { return NULL; } + const struct blkcg_policy *pol) { return NULL; } static inline struct blkcg_gq *pd_to_blkg(struct blkg_policy_data *pd) { return NULL; } static inline void blkg_get(struct blkcg_gq *blkg) { } static inline void blkg_put(struct blkcg_gq *blkg) { } diff --git a/block/blk-iocost.c b/block/blk-iocost.c index 27d2dcaa65f0..8b2aeba2e1e3 100644 --- a/block/blk-iocost.c +++ b/block/blk-iocost.c @@ -3221,7 +3221,7 @@ static u64 ioc_qos_prfill(struct seq_file *sf, struct blkg_policy_data *pd, if (!dname) return 0; - spin_lock(&ioc->lock); + spin_lock_irq(&ioc->lock); seq_printf(sf, "%s enable=%d ctrl=%s rpct=%u.%02u rlat=%u wpct=%u.%02u wlat=%u min=%u.%02u max=%u.%02u\n", dname, ioc->enabled, ioc->user_qos_params ? "user" : "auto", ioc->params.qos[QOS_RPPM] / 10000, @@ -3234,7 +3234,7 @@ static u64 ioc_qos_prfill(struct seq_file *sf, struct blkg_policy_data *pd, ioc->params.qos[QOS_MIN] % 10000 / 100, ioc->params.qos[QOS_MAX] / 10000, ioc->params.qos[QOS_MAX] % 10000 / 100); - spin_unlock(&ioc->lock); + spin_unlock_irq(&ioc->lock); return 0; } @@ -3430,14 +3430,14 @@ static u64 ioc_cost_model_prfill(struct seq_file *sf, if (!dname) return 0; - spin_lock(&ioc->lock); + spin_lock_irq(&ioc->lock); seq_printf(sf, "%s ctrl=%s model=linear " "rbps=%llu rseqiops=%llu rrandiops=%llu " "wbps=%llu wseqiops=%llu wrandiops=%llu\n", dname, ioc->user_cost_model ? "user" : "auto", u[I_LCOEF_RBPS], u[I_LCOEF_RSEQIOPS], u[I_LCOEF_RRANDIOPS], u[I_LCOEF_WBPS], u[I_LCOEF_WSEQIOPS], u[I_LCOEF_WRANDIOPS]); - spin_unlock(&ioc->lock); + spin_unlock_irq(&ioc->lock); return 0; } From 9327a865e395a53f67dffac4710beb1d4730495e Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 8 Jun 2026 11:42:45 +0800 Subject: [PATCH 39/45] blk-cgroup: don't nest queue_lock under rcu in blkg_lookup_create() Change this in two steps: 1) hold rcu lock and do blkg_lookup() from fast path; 2) hold queue_lock directly from slow path, and don't nest it under rcu lock; Prepare to convert protecting blkcg with blkcg_mutex instead of queue_lock. Signed-off-by: Yu Kuai Link: https://patch.msgid.link/93f33cc9e5a39dddb78dcd934d0c1d04b564fb00.1780621988.git.yukuai@fygo.io Signed-off-by: Jens Axboe --- block/blk-cgroup.c | 57 +++++++++++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index d6355338f290..fee8c9d5dc2c 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -469,22 +469,17 @@ static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, { struct request_queue *q = disk->queue; struct blkcg_gq *blkg; - unsigned long flags; - WARN_ON_ONCE(!rcu_read_lock_held()); - - blkg = blkg_lookup(blkcg, q); - if (blkg) - return blkg; - - spin_lock_irqsave(&q->queue_lock, flags); + rcu_read_lock(); blkg = blkg_lookup(blkcg, q); if (blkg) { if (blkcg != &blkcg_root && blkg != rcu_dereference(blkcg->blkg_hint)) rcu_assign_pointer(blkcg->blkg_hint, blkg); - goto found; + rcu_read_unlock(); + return blkg; } + rcu_read_unlock(); /* * Create blkgs walking down from blkcg_root to @blkcg, so that all @@ -516,8 +511,6 @@ static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, break; } -found: - spin_unlock_irqrestore(&q->queue_lock, flags); return blkg; } @@ -2046,6 +2039,18 @@ void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta) atomic64_add(delta, &blkg->delay_nsec); } +static inline struct blkcg_gq *blkg_lookup_tryget(struct blkcg_gq *blkg) +{ +retry: + if (blkg_tryget(blkg)) + return blkg; + + blkg = blkg->parent; + if (blkg) + goto retry; + + return NULL; +} /** * blkg_tryget_closest - try and get a blkg ref on the closet blkg * @bio: target bio @@ -2058,20 +2063,30 @@ void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta) static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio, struct cgroup_subsys_state *css) { - struct blkcg_gq *blkg, *ret_blkg = NULL; + struct request_queue *q = bio->bi_bdev->bd_queue; + struct blkcg *blkcg = css_to_blkcg(css); + struct blkcg_gq *blkg; rcu_read_lock(); - blkg = blkg_lookup_create(css_to_blkcg(css), bio->bi_bdev->bd_disk); - while (blkg) { - if (blkg_tryget(blkg)) { - ret_blkg = blkg; - break; - } - blkg = blkg->parent; - } + blkg = blkg_lookup(blkcg, q); + if (likely(blkg)) + blkg = blkg_lookup_tryget(blkg); rcu_read_unlock(); - return ret_blkg; + if (blkg) + return blkg; + + /* + * Fast path failed, we're probably issuing IO in this cgroup the first + * time, hold lock to create new blkg. + */ + spin_lock_irq(&q->queue_lock); + blkg = blkg_lookup_create(blkcg, bio->bi_bdev->bd_disk); + if (blkg) + blkg = blkg_lookup_tryget(blkg); + spin_unlock_irq(&q->queue_lock); + + return blkg; } /** From 457d3c4f0fdd6cf8a4bd8115bf470809984a9f02 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 8 Jun 2026 11:42:46 +0800 Subject: [PATCH 40/45] blk-cgroup: don't nest queue_lock under rcu in bio_associate_blkg() If a bio is already associated with a blkg, the blkcg is already pinned until the bio is done, so there is no need for RCU protection. Otherwise, protect blkcg_css() with RCU independently. Prepare to protect blkcg with blkcg_mutex instead of queue_lock. Signed-off-by: Yu Kuai Link: https://patch.msgid.link/8496fa234b21d4b31b7f068766906d0bffcac8e6.1780621988.git.yukuai@fygo.io Signed-off-by: Jens Axboe --- block/blk-cgroup.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index fee8c9d5dc2c..e1bde48852ae 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -2134,16 +2134,20 @@ void bio_associate_blkg(struct bio *bio) if (blk_op_is_passthrough(bio->bi_opf)) return; - rcu_read_lock(); - - if (bio->bi_blkg) + if (bio->bi_blkg) { css = bio_blkcg_css(bio); - else + bio_associate_blkg_from_css(bio, css); + } else { + rcu_read_lock(); css = blkcg_css(); + if (!css_tryget_online(css)) + css = NULL; + rcu_read_unlock(); - bio_associate_blkg_from_css(bio, css); - - rcu_read_unlock(); + bio_associate_blkg_from_css(bio, css); + if (css) + css_put(css); + } } EXPORT_SYMBOL_GPL(bio_associate_blkg); From 4cfd7c1cff8f4c863b99d420cdbe0563802a9e80 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 8 Jun 2026 11:42:47 +0800 Subject: [PATCH 41/45] blk-cgroup: don't nest queue_lock under blkcg->lock in blkcg_destroy_blkgs() The correct lock order is q->queue_lock before blkcg->lock, and in order to prevent deadlock from blkcg_destroy_blkgs(), trylock is used for q->queue_lock while blkcg->lock is already held, this is hacky. Refactor blkcg_destroy_blkgs() to hold blkcg->lock only long enough to get the first blkg and then release it. Then take q->queue_lock and blkcg->lock in the correct order to destroy the blkg. This is a very cold path, so the extra lock/unlock cycles are acceptable. Also prepare to convert protecting blkcg with blkcg_mutex instead of queue_lock. Signed-off-by: Yu Kuai Link: https://patch.msgid.link/00b03cf74a9937cb4d6dd67a189ddc00a3de0451.1780621988.git.yukuai@fygo.io Signed-off-by: Jens Axboe --- block/blk-cgroup.c | 45 ++++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index e1bde48852ae..d2a1f5903f24 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -1239,6 +1239,21 @@ struct list_head *blkcg_get_cgwb_list(struct cgroup_subsys_state *css) * This finally frees the blkcg. */ +static struct blkcg_gq *blkcg_get_first_blkg(struct blkcg *blkcg) +{ + struct blkcg_gq *blkg = NULL; + + spin_lock_irq(&blkcg->lock); + if (!hlist_empty(&blkcg->blkg_list)) { + blkg = hlist_entry(blkcg->blkg_list.first, struct blkcg_gq, + blkcg_node); + blkg_get(blkg); + } + spin_unlock_irq(&blkcg->lock); + + return blkg; +} + /** * blkcg_destroy_blkgs - responsible for shooting down blkgs * @blkcg: blkcg of interest @@ -1252,32 +1267,24 @@ struct list_head *blkcg_get_cgwb_list(struct cgroup_subsys_state *css) */ static void blkcg_destroy_blkgs(struct blkcg *blkcg) { + struct blkcg_gq *blkg; + might_sleep(); - spin_lock_irq(&blkcg->lock); - - while (!hlist_empty(&blkcg->blkg_list)) { - struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first, - struct blkcg_gq, blkcg_node); + while ((blkg = blkcg_get_first_blkg(blkcg))) { struct request_queue *q = blkg->q; - if (need_resched() || !spin_trylock(&q->queue_lock)) { - /* - * Given that the system can accumulate a huge number - * of blkgs in pathological cases, check to see if we - * need to rescheduling to avoid softlockup. - */ - spin_unlock_irq(&blkcg->lock); - cond_resched(); - spin_lock_irq(&blkcg->lock); - continue; - } + spin_lock_irq(&q->queue_lock); + spin_lock(&blkcg->lock); blkg_destroy(blkg); - spin_unlock(&q->queue_lock); - } - spin_unlock_irq(&blkcg->lock); + spin_unlock(&blkcg->lock); + spin_unlock_irq(&q->queue_lock); + + blkg_put(blkg); + cond_resched(); + } } /** From f928145cbcb52544203808f159461d0a25543df7 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 8 Jun 2026 11:42:48 +0800 Subject: [PATCH 42/45] mm/page_io: don't nest queue_lock under rcu in bio_associate_blkg_from_page() Take a css reference under RCU, drop RCU, and then associate the bio with the blkg. This avoids nesting queue_lock under RCU and prepares to protect blkcg with blkcg_mutex instead of queue_lock. Use css_tryget() instead of css_tryget_online() so swap writeback for pages charged to a dying memcg still passes the dying css to bio_associate_blkg_from_css(). That preserves the existing closest-live ancestor fallback instead of charging those bios to the root blkg. Signed-off-by: Yu Kuai Link: https://patch.msgid.link/c910d2c39d3ec97f67de68af636a52394342d55f.1780621988.git.yukuai@fygo.io Signed-off-by: Jens Axboe --- mm/page_io.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mm/page_io.c b/mm/page_io.c index a59b73f8bdd9..c96d3e4cf872 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -317,8 +317,13 @@ static void bio_associate_blkg_from_page(struct bio *bio, struct folio *folio) rcu_read_lock(); memcg = folio_memcg(folio); css = cgroup_e_css(memcg->css.cgroup, &io_cgrp_subsys); - bio_associate_blkg_from_css(bio, css); + if (!css || !css_tryget(css)) + css = NULL; rcu_read_unlock(); + + bio_associate_blkg_from_css(bio, css); + if (css) + css_put(css); } #else #define bio_associate_blkg_from_page(bio, folio) do { } while (0) From 3ca4f4e3ae811d414076a491cbf0dfcdae0dc01e Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 8 Jun 2026 11:42:49 +0800 Subject: [PATCH 43/45] block, bfq: don't grab queue_lock to initialize bfq The request_queue is frozen and quiesced while the elevator init_sched() method runs, so queue_lock is not needed for BFQ cgroup initialization. Signed-off-by: Yu Kuai Link: https://patch.msgid.link/1965073ea20f33114a8d903816b986e483b9bb34.1780621988.git.yukuai@fygo.io Signed-off-by: Jens Axboe --- block/bfq-iosched.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c index eec9be62061b..0f75301b3115 100644 --- a/block/bfq-iosched.c +++ b/block/bfq-iosched.c @@ -7204,10 +7204,7 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_queue *eq) return -ENOMEM; eq->elevator_data = bfqd; - - spin_lock_irq(&q->queue_lock); q->elevator = eq; - spin_unlock_irq(&q->queue_lock); /* * Our fallback bfqq if bfq_find_alloc_queue() runs into OOM issues. @@ -7240,7 +7237,6 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_queue *eq) * If the disk supports multiple actuators, copy independent * access ranges from the request queue structure. */ - spin_lock_irq(&q->queue_lock); if (ia_ranges) { /* * Check if the disk ia_ranges size exceeds the current bfq @@ -7266,7 +7262,6 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_queue *eq) bfqd->sector[0] = 0; bfqd->nr_sectors[0] = get_capacity(q->disk); } - spin_unlock_irq(&q->queue_lock); INIT_LIST_HEAD(&bfqd->dispatch); From e7c1627afda2484baf65449be15873c2550f917a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 24 Jun 2026 10:00:02 +0200 Subject: [PATCH 44/45] block: fix GFP_ flags confusion in bio_integrity_alloc_buf bio_integrity_alloc_buf usage of GFP_ flags is messed up. For one it mixes GFP_NOFS and GFP_NOIO for neighbouring allocations, but it also makes the allocations fail more often than needed. That code was copied from bio_alloc_bioset which needs to do that so that it can punt to the rescuer workqueue, but none of that is needed for the integrity allocations that either sits in the file system or at the very bottom of the I/O stack. Failing early means we'll do a fully waiting allocation from the mempool ->alloc callback which is usually much larger than required. Fix this by passing a gfp_t so that the file system path can pass GFP_NOFS and the auto-integrity code can pass GFP_NOIO, and don't modify the allocation type except for disabling warnings. Fixes: ec7f31b2a2d3 ("block: make bio auto-integrity deadlock safe") Signed-off-by: Christoph Hellwig Reviewed-by: Martin K. Petersen Link: https://patch.msgid.link/20260624080014.1998650-2-hch@lst.de Signed-off-by: Jens Axboe --- block/bio-integrity-auto.c | 2 +- block/bio-integrity-fs.c | 4 ++-- block/bio-integrity.c | 8 +++----- include/linux/bio-integrity.h | 2 +- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/block/bio-integrity-auto.c b/block/bio-integrity-auto.c index 353eed632fcc..b1c733ecfd2e 100644 --- a/block/bio-integrity-auto.c +++ b/block/bio-integrity-auto.c @@ -94,7 +94,7 @@ void bio_integrity_prep(struct bio *bio, unsigned int action) bio_integrity_init(bio, &bid->bip, &bid->bvec, 1); bid->bio = bio; bid->bip.bip_flags |= BIP_BLOCK_INTEGRITY; - bio_integrity_alloc_buf(bio, action & BI_ACT_ZERO); + bio_integrity_alloc_buf(bio, GFP_NOIO, action & BI_ACT_ZERO); if (action & BI_ACT_CHECK) bio_integrity_setup_default(bio); diff --git a/block/bio-integrity-fs.c b/block/bio-integrity-fs.c index 0daa42d9ead7..9c5fe5fa8f0d 100644 --- a/block/bio-integrity-fs.c +++ b/block/bio-integrity-fs.c @@ -23,10 +23,10 @@ unsigned int fs_bio_integrity_alloc(struct bio *bio) if (!action) return 0; - iib = mempool_alloc(&fs_bio_integrity_pool, GFP_NOIO); + iib = mempool_alloc(&fs_bio_integrity_pool, GFP_NOFS); bio_integrity_init(bio, &iib->bip, &iib->bvec, 1); - bio_integrity_alloc_buf(bio, action & BI_ACT_ZERO); + bio_integrity_alloc_buf(bio, GFP_NOFS, action & BI_ACT_ZERO); if (action & BI_ACT_CHECK) bio_integrity_setup_default(bio); return action; diff --git a/block/bio-integrity.c b/block/bio-integrity.c index e796de1a749e..a53b38cf8a1a 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -64,20 +64,18 @@ unsigned int __bio_integrity_action(struct bio *bio) } EXPORT_SYMBOL_GPL(__bio_integrity_action); -void bio_integrity_alloc_buf(struct bio *bio, bool zero_buffer) +void bio_integrity_alloc_buf(struct bio *bio, gfp_t gfp, bool zero_buffer) { struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk); struct bio_integrity_payload *bip = bio_integrity(bio); unsigned int len = bio_integrity_bytes(bi, bio_sectors(bio)); - gfp_t gfp = GFP_NOIO | (zero_buffer ? __GFP_ZERO : 0); void *buf; - buf = kmalloc(len, (gfp & ~__GFP_DIRECT_RECLAIM) | - __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN); + buf = kmalloc(len, gfp | __GFP_NOWARN | (zero_buffer ? __GFP_ZERO : 0)); if (unlikely(!buf)) { struct page *page; - page = mempool_alloc(&integrity_buf_pool, GFP_NOFS); + page = mempool_alloc(&integrity_buf_pool, gfp); if (zero_buffer) memset(page_address(page), 0, len); bvec_set_page(&bip->bip_vec[0], page, len, 0); diff --git a/include/linux/bio-integrity.h b/include/linux/bio-integrity.h index af5178434ec6..c3dda32fd803 100644 --- a/include/linux/bio-integrity.h +++ b/include/linux/bio-integrity.h @@ -141,7 +141,7 @@ static inline int bio_integrity_add_page(struct bio *bio, struct page *page, } #endif /* CONFIG_BLK_DEV_INTEGRITY */ -void bio_integrity_alloc_buf(struct bio *bio, bool zero_buffer); +void bio_integrity_alloc_buf(struct bio *bio, gfp_t gfp, bool zero_buffer); void bio_integrity_free_buf(struct bio_integrity_payload *bip); void bio_integrity_setup_default(struct bio *bio); From a1c8bdbbd72564cebb0d02948c1ed57b80b2e773 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 24 Jun 2026 10:00:03 +0200 Subject: [PATCH 45/45] block: handle REQ_OP_ZONE_APPEND in __bio_integrity_action Otherwise zone append commands will miss their integrity data. While this works "fine" for auto-PI, it break file system PI and non-PI metadata. With this XFS on ZNS namespace with non-PI metadata and 512 byte sectors with PI work, while PI 4k sector formats with PI work only when Caleb's "block: fix integrity offset/length conversions" is applied as well. Note that unlike regular writes, zone append does need remapping as partitions are not supported on zoned block devices. Fixes: df3c485e0e60 ("block: switch on bio operation in bio_integrity_prep") Signed-off-by: Christoph Hellwig Reviewed-by: Martin K. Petersen Link: https://patch.msgid.link/20260624080014.1998650-3-hch@lst.de Signed-off-by: Jens Axboe --- block/bio-integrity.c | 1 + 1 file changed, 1 insertion(+) diff --git a/block/bio-integrity.c b/block/bio-integrity.c index a53b38cf8a1a..b23e2434d80c 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -38,6 +38,7 @@ unsigned int __bio_integrity_action(struct bio *bio) } return BI_ACT_BUFFER | BI_ACT_CHECK; case REQ_OP_WRITE: + case REQ_OP_ZONE_APPEND: /* * Flush masquerading as write? */